From 222ef91fe9d91110f2933e066b33bf954360e58d Mon Sep 17 00:00:00 2001
From: Jamie Magee <jamie.magee@gmail.com>
Date: Fri, 25 Jun 2021 05:12:23 -0700
Subject: [PATCH] refactor(terraform-module): convert to class-based (#10486)
---
lib/datasource/api.ts | 4 +-
lib/datasource/terraform-module/base.ts | 29 +++
lib/datasource/terraform-module/index.spec.ts | 3 +-
lib/datasource/terraform-module/index.ts | 182 ++++++++----------
lib/datasource/terraform-provider/index.ts | 7 +-
lib/manager/terraform/modules.ts | 4 +-
lib/manager/terragrunt/modules.ts | 4 +-
7 files changed, 122 insertions(+), 111 deletions(-)
create mode 100644 lib/datasource/terraform-module/base.ts
diff --git a/lib/datasource/api.ts b/lib/datasource/api.ts
index 14eecf92ee..8cf8702425 100644
--- a/lib/datasource/api.ts
+++ b/lib/datasource/api.ts
@@ -28,7 +28,7 @@ import * as rubyVersion from './ruby-version';
import * as rubygems from './rubygems';
import * as sbtPackage from './sbt-package';
import * as sbtPlugin from './sbt-plugin';
-import * as terraformModule from './terraform-module';
+import { TerraformModuleDatasource } from './terraform-module';
import { TerraformProviderDatasource } from './terraform-provider';
import type { DatasourceApi } from './types';
@@ -65,5 +65,5 @@ api.set('ruby-version', rubyVersion);
api.set('rubygems', rubygems);
api.set('sbt-package', sbtPackage);
api.set('sbt-plugin', sbtPlugin);
-api.set('terraform-module', terraformModule);
+api.set('terraform-module', new TerraformModuleDatasource());
api.set('terraform-provider', new TerraformProviderDatasource());
diff --git a/lib/datasource/terraform-module/base.ts b/lib/datasource/terraform-module/base.ts
new file mode 100644
index 0000000000..688a9a4291
--- /dev/null
+++ b/lib/datasource/terraform-module/base.ts
@@ -0,0 +1,29 @@
+import { cache } from '../../util/cache/package/decorator';
+import { ensureTrailingSlash } from '../../util/url';
+import { Datasource } from '../datasource';
+import type { ServiceDiscoveryResult } from './types';
+
+// TODO: extract to a separate directory structure (#10532)
+export abstract class TerraformDatasource extends Datasource {
+ static id = 'terraform';
+
+ @cache({
+ namespace: `datasource-${TerraformDatasource.id}`,
+ key: (registryUrl: string) =>
+ TerraformDatasource.getDiscoveryUrl(registryUrl),
+ ttlMinutes: 1440,
+ })
+ async getTerraformServiceDiscoveryResult(
+ registryUrl: string
+ ): Promise<ServiceDiscoveryResult> {
+ const discoveryURL = TerraformDatasource.getDiscoveryUrl(registryUrl);
+ const serviceDiscovery = (
+ await this.http.getJson<ServiceDiscoveryResult>(discoveryURL)
+ ).body;
+ return serviceDiscovery;
+ }
+
+ private static getDiscoveryUrl(registryUrl: string): string {
+ return `${ensureTrailingSlash(registryUrl)}.well-known/terraform.json`;
+ }
+}
diff --git a/lib/datasource/terraform-module/index.spec.ts b/lib/datasource/terraform-module/index.spec.ts
index d93b9a67da..c9576339b3 100644
--- a/lib/datasource/terraform-module/index.spec.ts
+++ b/lib/datasource/terraform-module/index.spec.ts
@@ -1,7 +1,7 @@
import { getPkgReleases } from '..';
import * as httpMock from '../../../test/http-mock';
import { getName, loadFixture } from '../../../test/util';
-import { id as datasource } from '.';
+import { TerraformModuleDatasource } from '.';
const consulData: any = loadFixture('registry-consul.json');
const serviceDiscoveryResult: any = loadFixture('service-discovery.json');
@@ -9,6 +9,7 @@ const serviceDiscoveryCustomResult: any = loadFixture(
'service-custom-discovery.json'
);
+const datasource = TerraformModuleDatasource.id;
const baseUrl = 'https://registry.terraform.io';
const localTerraformEnterprisebaseUrl = 'https://terraform.foo.bar';
diff --git a/lib/datasource/terraform-module/index.ts b/lib/datasource/terraform-module/index.ts
index 42c165da21..6c3f329657 100644
--- a/lib/datasource/terraform-module/index.ts
+++ b/lib/datasource/terraform-module/index.ts
@@ -1,112 +1,62 @@
import { logger } from '../../logger';
import { ExternalHostError } from '../../types/errors/external-host-error';
-import * as packageCache from '../../util/cache/package';
-import { Http } from '../../util/http';
+import { cache } from '../../util/cache/package/decorator';
+import type { HttpError } from '../../util/http/types';
import * as hashicorpVersioning from '../../versioning/hashicorp';
import type { GetReleasesConfig, ReleaseResult } from '../types';
-import type {
- RegistryRepository,
- ServiceDiscoveryResult,
- TerraformRelease,
-} from './types';
+import { TerraformDatasource } from './base';
+import type { RegistryRepository, TerraformRelease } from './types';
-export const id = 'terraform-module';
-export const customRegistrySupport = true;
-export const defaultRegistryUrls = ['https://registry.terraform.io'];
-export const defaultVersioning = hashicorpVersioning.id;
-export const registryStrategy = 'first';
+export class TerraformModuleDatasource extends TerraformDatasource {
+ static readonly id = 'terraform-module';
-const http = new Http(id);
-
-function getRegistryRepository(
- lookupName: string,
- registryUrl: string
-): RegistryRepository {
- let registry: string;
- const split = lookupName.split('/');
- if (split.length > 3 && split[0].includes('.')) {
- [registry] = split;
- split.shift();
- } else {
- registry = registryUrl;
- }
- if (!/^https?:\/\//.test(registry)) {
- registry = `https://${registry}`;
+ constructor() {
+ super(TerraformModuleDatasource.id);
}
- const repository = split.join('/');
- return {
- registry,
- repository,
- };
-}
-export async function getTerraformServiceDiscoveryResult(
- registryUrl: string
-): Promise<ServiceDiscoveryResult> {
- const discoveryURL = `${registryUrl}/.well-known/terraform.json`;
- const cacheNamespace = 'terraform-service-discovery';
- const cachedResult = await packageCache.get<ServiceDiscoveryResult>(
- cacheNamespace,
- registryUrl
- );
- // istanbul ignore if
- if (cachedResult) {
- return cachedResult;
- }
- const serviceDiscovery = (
- await http.getJson<ServiceDiscoveryResult>(discoveryURL)
- ).body;
+ readonly defaultRegistryUrls = ['https://registry.terraform.io'];
- const cacheMinutes = 1440; // 24h
- await packageCache.set(
- cacheNamespace,
- registryUrl,
- serviceDiscovery,
- cacheMinutes
- );
+ readonly defaultVersioning = hashicorpVersioning.id;
- return serviceDiscovery;
-}
-/**
- * terraform.getReleases
- *
- * This function will fetch a package from the specified Terraform registry and return all semver versions.
- * - `sourceUrl` is supported of "source" field is set
- * - `homepage` is set to the Terraform registry's page if it's on the official main registry
- */
-export async function getReleases({
- lookupName,
- registryUrl,
-}: GetReleasesConfig): Promise<ReleaseResult | null> {
- const { registry, repository } = getRegistryRepository(
+ /**
+ * This function will fetch a package from the specified Terraform registry and return all semver versions.
+ * - `sourceUrl` is supported of "source" field is set
+ * - `homepage` is set to the Terraform registry's page if it's on the official main registry
+ */
+ @cache({
+ namespace: `datasource-${TerraformModuleDatasource.id}`,
+ key: (getReleasesConfig: GetReleasesConfig) =>
+ TerraformModuleDatasource.getCacheKey(getReleasesConfig),
+ })
+ async getReleases({
lookupName,
- registryUrl
- );
- logger.debug(
- { registry, terraformRepository: repository },
- 'terraform.getDependencies()'
- );
- const cacheNamespace = 'terraform-module';
- const cacheURL = `${registry}/${repository}`;
- const cachedResult = await packageCache.get<ReleaseResult>(
- cacheNamespace,
- cacheURL
- );
- // istanbul ignore if
- if (cachedResult) {
- return cachedResult;
- }
- try {
- const serviceDiscovery = await getTerraformServiceDiscoveryResult(
- registryUrl
+ registryUrl,
+ }: GetReleasesConfig): Promise<ReleaseResult | null> {
+ const { registry, repository } =
+ TerraformModuleDatasource.getRegistryRepository(lookupName, registryUrl);
+ logger.trace(
+ { registry, terraformRepository: repository },
+ 'terraform-module.getReleases()'
);
- const pkgUrl = `${registry}${serviceDiscovery['modules.v1']}${repository}`;
- const res = (await http.getJson<TerraformRelease>(pkgUrl)).body;
- const returnedName = res.namespace + '/' + res.name + '/' + res.provider;
- if (returnedName !== repository) {
- logger.warn({ pkgUrl }, 'Terraform registry result mismatch');
- return null;
+
+ let res: TerraformRelease;
+ let pkgUrl: string;
+
+ try {
+ const serviceDiscovery = await this.getTerraformServiceDiscoveryResult(
+ registryUrl
+ );
+ pkgUrl = `${registry}${serviceDiscovery['modules.v1']}${repository}`;
+ res = (await this.http.getJson<TerraformRelease>(pkgUrl)).body;
+ const returnedName = res.namespace + '/' + res.name + '/' + res.provider;
+ if (returnedName !== repository) {
+ logger.warn({ pkgUrl }, 'Terraform registry result mismatch');
+ return null;
+ }
+ } catch (err) {
+ this.handleGenericErrors(err);
}
+
// Simplify response before caching and returning
const dep: ReleaseResult = {
releases: null,
@@ -127,16 +77,48 @@ export async function getReleases({
if (latestVersion) {
latestVersion.releaseTimestamp = res.published_at;
}
+
logger.trace({ dep }, 'dep');
- const cacheMinutes = 30;
- await packageCache.set(cacheNamespace, pkgUrl, dep, cacheMinutes);
return dep;
- } catch (err) {
+ }
+
+ // eslint-disable-next-line class-methods-use-this
+ override handleSpecificErrors(err: HttpError): void {
const failureCodes = ['EAI_AGAIN'];
// istanbul ignore if
if (failureCodes.includes(err.code)) {
throw new ExternalHostError(err);
}
- throw err;
+ }
+
+ private static getRegistryRepository(
+ lookupName: string,
+ registryUrl: string
+ ): RegistryRepository {
+ let registry: string;
+ const split = lookupName.split('/');
+ if (split.length > 3 && split[0].includes('.')) {
+ [registry] = split;
+ split.shift();
+ } else {
+ registry = registryUrl;
+ }
+ if (!/^https?:\/\//.test(registry)) {
+ registry = `https://${registry}`;
+ }
+ const repository = split.join('/');
+ return {
+ registry,
+ repository,
+ };
+ }
+
+ private static getCacheKey({
+ lookupName,
+ registryUrl,
+ }: GetReleasesConfig): string {
+ const { registry, repository } =
+ TerraformModuleDatasource.getRegistryRepository(lookupName, registryUrl);
+ return `${registry}/${repository}`;
}
}
diff --git a/lib/datasource/terraform-provider/index.ts b/lib/datasource/terraform-provider/index.ts
index 07e401b8f7..e03b488bec 100644
--- a/lib/datasource/terraform-provider/index.ts
+++ b/lib/datasource/terraform-provider/index.ts
@@ -2,15 +2,14 @@ import { logger } from '../../logger';
import { cache } from '../../util/cache/package/decorator';
import { parseUrl } from '../../util/url';
import * as hashicorpVersioning from '../../versioning/hashicorp';
-import { Datasource } from '../datasource';
-import { getTerraformServiceDiscoveryResult } from '../terraform-module';
+import { TerraformDatasource } from '../terraform-module/base';
import type { GetReleasesConfig, ReleaseResult } from '../types';
import type {
TerraformProvider,
TerraformProviderReleaseBackend,
} from './types';
-export class TerraformProviderDatasource extends Datasource {
+export class TerraformProviderDatasource extends TerraformDatasource {
static readonly id = 'terraform-provider';
static readonly defaultRegistryUrls = [
@@ -63,7 +62,7 @@ export class TerraformProviderDatasource extends Datasource {
registryURL: string,
repository: string
): Promise<ReleaseResult> {
- const serviceDiscovery = await getTerraformServiceDiscoveryResult(
+ const serviceDiscovery = await this.getTerraformServiceDiscoveryResult(
registryURL
);
const backendURL = `${registryURL}${serviceDiscovery['providers.v1']}${repository}`;
diff --git a/lib/manager/terraform/modules.ts b/lib/manager/terraform/modules.ts
index ef9eda0a21..a7903a0233 100644
--- a/lib/manager/terraform/modules.ts
+++ b/lib/manager/terraform/modules.ts
@@ -1,6 +1,6 @@
import * as datasourceGitTags from '../../datasource/git-tags';
import * as datasourceGithubTags from '../../datasource/github-tags';
-import * as datasourceTerraformModule from '../../datasource/terraform-module';
+import { TerraformModuleDatasource } from '../../datasource/terraform-module';
import { logger } from '../../logger';
import { SkipReason } from '../../types';
import type { PackageDependency } from '../types';
@@ -61,7 +61,7 @@ export function analyseTerraformModule(dep: PackageDependency): void {
}
dep.depType = 'module';
dep.depName = moduleParts.join('/');
- dep.datasource = datasourceTerraformModule.id;
+ dep.datasource = TerraformModuleDatasource.id;
}
} else {
logger.debug({ dep }, 'terraform dep has no source');
diff --git a/lib/manager/terragrunt/modules.ts b/lib/manager/terragrunt/modules.ts
index 7b597007a5..3b3f0e4ba5 100644
--- a/lib/manager/terragrunt/modules.ts
+++ b/lib/manager/terragrunt/modules.ts
@@ -1,6 +1,6 @@
import * as datasourceGitTags from '../../datasource/git-tags';
import * as datasourceGithubTags from '../../datasource/github-tags';
-import * as datasourceTerragruntModule from '../../datasource/terraform-module';
+import { TerraformModuleDatasource } from '../../datasource/terraform-module';
import { logger } from '../../logger';
import { SkipReason } from '../../types';
import type { PackageDependency } from '../types';
@@ -62,7 +62,7 @@ export function analyseTerragruntModule(dep: PackageDependency): void {
}
dep.depType = 'terragrunt';
dep.depName = moduleParts.join('/');
- dep.datasource = datasourceTerragruntModule.id;
+ dep.datasource = TerraformModuleDatasource.id;
}
} else {
logger.debug({ dep }, 'terragrunt dep has no source');
--
GitLab