From 1a066c61d8ab9aad1e3b3fe9cc53d7accf045ea0 Mon Sep 17 00:00:00 2001
From: Sebastian Poxhofer <secustor@users.noreply.github.com>
Date: Mon, 22 Jun 2020 10:55:01 +0200
Subject: [PATCH] feat(terraform-provider): Add secondary release repo (#6513)
Co-authored-by: Jamie Magee <jamie.magee@gmail.com>
---
.../__fixtures__/releaseBackendIndex.json | 103 +++++++++++++
.../__snapshots__/index.spec.ts.snap | 75 ++++++++++
.../terraform-provider/index.spec.ts | 58 +++++++-
lib/datasource/terraform-provider/index.ts | 135 ++++++++++++------
4 files changed, 318 insertions(+), 53 deletions(-)
create mode 100644 lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json
diff --git a/lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json b/lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json
new file mode 100644
index 0000000000..a5cd75cb84
--- /dev/null
+++ b/lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json
@@ -0,0 +1,103 @@
+{
+ "terraform-provider-google-beta": {
+ "name": "terraform-provider-google-beta",
+ "versions": {
+ "1.19.0": {
+ "name": "terraform-provider-google-beta",
+ "version": "1.19.0",
+ "shasums": "terraform-provider-google-beta_1.19.0_SHA256SUMS",
+ "shasums_signature": "terraform-provider-google-beta_1.19.0_SHA256SUMS.sig",
+ "builds": [
+ {
+ "name": "terraform-provider-google-beta",
+ "version": "1.19.0",
+ "os": "darwin",
+ "arch": "amd64",
+ "filename": "terraform-provider-google-beta_1.19.0_darwin_amd64.zip",
+ "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.19.0/terraform-provider-google-beta_1.19.0_darwin_amd64.zip"
+ },
+ {
+ "name": "terraform-provider-google-beta",
+ "version": "1.19.0",
+ "os": "freebsd",
+ "arch": "386",
+ "filename": "terraform-provider-google-beta_1.19.0_freebsd_386.zip",
+ "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.19.0/terraform-provider-google-beta_1.19.0_freebsd_386.zip"
+ },
+ {
+ "name": "terraform-provider-google-beta",
+ "version": "1.19.0",
+ "os": "freebsd",
+ "arch": "amd64",
+ "filename": "terraform-provider-google-beta_1.19.0_freebsd_amd64.zip",
+ "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.19.0/terraform-provider-google-beta_1.19.0_freebsd_amd64.zip"
+ }
+ ]
+ },
+ "1.20.0": {
+ "name": "terraform-provider-google-beta",
+ "version": "1.20.0",
+ "shasums": "terraform-provider-google-beta_1.20.0_SHA256SUMS",
+ "shasums_signature": "terraform-provider-google-beta_1.20.0_SHA256SUMS.sig",
+ "builds": [
+ {
+ "name": "terraform-provider-google-beta",
+ "version": "1.20.0",
+ "os": "openbsd",
+ "arch": "386",
+ "filename": "terraform-provider-google-beta_1.20.0_openbsd_386.zip",
+ "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.20.0/terraform-provider-google-beta_1.20.0_openbsd_386.zip"
+ },
+ {
+ "name": "terraform-provider-google-beta",
+ "version": "1.20.0",
+ "os": "openbsd",
+ "arch": "amd64",
+ "filename": "terraform-provider-google-beta_1.20.0_openbsd_amd64.zip",
+ "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.20.0/terraform-provider-google-beta_1.20.0_openbsd_amd64.zip"
+ },
+ {
+ "name": "terraform-provider-google-beta",
+ "version": "1.20.0",
+ "os": "solaris",
+ "arch": "amd64",
+ "filename": "terraform-provider-google-beta_1.20.0_solaris_amd64.zip",
+ "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.20.0/terraform-provider-google-beta_1.20.0_solaris_amd64.zip"
+ }
+ ]
+ },
+ "2.0.0": {
+ "name": "terraform-provider-google-beta",
+ "version": "2.0.0",
+ "shasums": "terraform-provider-google-beta_2.0.0_SHA256SUMS",
+ "shasums_signature": "terraform-provider-google-beta_2.0.0_SHA256SUMS.sig",
+ "builds": [
+ {
+ "name": "terraform-provider-google-beta",
+ "version": "2.0.0",
+ "os": "darwin",
+ "arch": "amd64",
+ "filename": "terraform-provider-google-beta_2.0.0_darwin_amd64.zip",
+ "url": "https://releases.hashicorp.com/terraform-provider-google-beta/2.0.0/terraform-provider-google-beta_2.0.0_darwin_amd64.zip"
+ },
+ {
+ "name": "terraform-provider-google-beta",
+ "version": "2.0.0",
+ "os": "freebsd",
+ "arch": "386",
+ "filename": "terraform-provider-google-beta_2.0.0_freebsd_386.zip",
+ "url": "https://releases.hashicorp.com/terraform-provider-google-beta/2.0.0/terraform-provider-google-beta_2.0.0_freebsd_386.zip"
+ },
+ {
+ "name": "terraform-provider-google-beta",
+ "version": "2.0.0",
+ "os": "freebsd",
+ "arch": "amd64",
+ "filename": "terraform-provider-google-beta_2.0.0_freebsd_amd64.zip",
+ "url": "https://releases.hashicorp.com/terraform-provider-google-beta/2.0.0/terraform-provider-google-beta_2.0.0_freebsd_amd64.zip"
+ }
+ ]
+ }
+ }
+ }
+}
diff --git a/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap b/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap
index d52991debb..577c59d31c 100644
--- a/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap
+++ b/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap
@@ -1,5 +1,48 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
+exports[`datasource/terraform getReleases processes data with alternative backend 1`] = `
+Object {
+ "name": "hashicorp/google-beta",
+ "releases": Array [
+ Object {
+ "version": "1.19.0",
+ },
+ Object {
+ "version": "1.20.0",
+ },
+ Object {
+ "version": "2.0.0",
+ },
+ ],
+ "versions": Object {},
+}
+`;
+
+exports[`datasource/terraform getReleases processes data with alternative backend 2`] = `
+Array [
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate",
+ "host": "registry.terraform.io",
+ "user-agent": "https://github.com/renovatebot/renovate",
+ },
+ "method": "GET",
+ "url": "https://registry.terraform.io/v1/providers/hashicorp/google-beta",
+ },
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate",
+ "host": "releases.hashicorp.com",
+ "user-agent": "https://github.com/renovatebot/renovate",
+ },
+ "method": "GET",
+ "url": "https://releases.hashicorp.com/index.json",
+ },
+]
+`;
+
exports[`datasource/terraform getReleases processes real data 1`] = `
Object {
"homepage": "https://registry.terraform.io/providers/hashicorp/azurerm",
@@ -233,6 +276,16 @@ Array [
"method": "GET",
"url": "https://registry.terraform.io/v1/providers/hashicorp/azurerm",
},
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate",
+ "host": "releases.hashicorp.com",
+ "user-agent": "https://github.com/renovatebot/renovate",
+ },
+ "method": "GET",
+ "url": "https://releases.hashicorp.com/index.json",
+ },
]
`;
@@ -248,6 +301,16 @@ Array [
"method": "GET",
"url": "https://registry.terraform.io/v1/providers/hashicorp/azurerm",
},
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate",
+ "host": "releases.hashicorp.com",
+ "user-agent": "https://github.com/renovatebot/renovate",
+ },
+ "method": "GET",
+ "url": "https://releases.hashicorp.com/index.json",
+ },
]
`;
@@ -263,5 +326,17 @@ Array [
"method": "GET",
"url": "https://registry.terraform.io/v1/providers/hashicorp/azurerm",
},
+ Object {
+ "headers": Object {
+ "accept": "application/json",
+ "accept-encoding": "gzip, deflate",
+ "host": "releases.hashicorp.com",
+ "user-agent": "https://github.com/renovatebot/renovate",
+ },
+ "method": "GET",
+ "url": "https://releases.hashicorp.com/index.json",
+ },
]
`;
+
+exports[`datasource/terraform getReleases simulate failing secondary release source 1`] = `null`;
diff --git a/lib/datasource/terraform-provider/index.spec.ts b/lib/datasource/terraform-provider/index.spec.ts
index 4456859da2..e4e37cc9b1 100644
--- a/lib/datasource/terraform-provider/index.spec.ts
+++ b/lib/datasource/terraform-provider/index.spec.ts
@@ -1,13 +1,17 @@
import fs from 'fs';
import { getPkgReleases } from '..';
import * as httpMock from '../../../test/httpMock';
-import { id as datasource } from '.';
+import { id as datasource, defaultRegistryUrls } from '.';
const consulData: any = fs.readFileSync(
'lib/datasource/terraform-provider/__fixtures__/azurerm-provider.json'
);
+const hashicorpReleases: any = fs.readFileSync(
+ 'lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json'
+);
-const baseUrl = 'https://registry.terraform.io/';
+const primaryUrl = defaultRegistryUrls[0];
+const secondaryUrl = defaultRegistryUrls[1];
describe('datasource/terraform', () => {
describe('getReleases', () => {
@@ -22,9 +26,10 @@ describe('datasource/terraform', () => {
it('returns null for empty result', async () => {
httpMock
- .scope(baseUrl)
+ .scope(primaryUrl)
.get('/v1/providers/hashicorp/azurerm')
.reply(200, {});
+ httpMock.scope(secondaryUrl).get('/index.json').reply(200, {});
expect(
await getPkgReleases({
datasource,
@@ -34,7 +39,11 @@ describe('datasource/terraform', () => {
expect(httpMock.getTrace()).toMatchSnapshot();
});
it('returns null for 404', async () => {
- httpMock.scope(baseUrl).get('/v1/providers/hashicorp/azurerm').reply(404);
+ httpMock
+ .scope(primaryUrl)
+ .get('/v1/providers/hashicorp/azurerm')
+ .reply(404);
+ httpMock.scope(secondaryUrl).get('/index.json').reply(404);
expect(
await getPkgReleases({
datasource,
@@ -45,9 +54,10 @@ describe('datasource/terraform', () => {
});
it('returns null for unknown error', async () => {
httpMock
- .scope(baseUrl)
+ .scope(primaryUrl)
.get('/v1/providers/hashicorp/azurerm')
.replyWithError('');
+ httpMock.scope(secondaryUrl).get('/index.json').replyWithError('');
expect(
await getPkgReleases({
datasource,
@@ -58,7 +68,7 @@ describe('datasource/terraform', () => {
});
it('processes real data', async () => {
httpMock
- .scope(baseUrl)
+ .scope(primaryUrl)
.get('/v1/providers/hashicorp/azurerm')
.reply(200, JSON.parse(consulData));
const res = await getPkgReleases({
@@ -69,5 +79,41 @@ describe('datasource/terraform', () => {
expect(res).not.toBeNull();
expect(httpMock.getTrace()).toMatchSnapshot();
});
+ it('processes data with alternative backend', async () => {
+ httpMock
+ .scope(primaryUrl)
+ .get('/v1/providers/hashicorp/google-beta')
+ .reply(404, {
+ errors: ['Not Found'],
+ });
+ httpMock
+ .scope(secondaryUrl)
+ .get('/index.json')
+ .reply(200, JSON.parse(hashicorpReleases));
+
+ const res = await getPkgReleases({
+ datasource,
+ depName: 'google-beta',
+ });
+ expect(res).toMatchSnapshot();
+ expect(res).not.toBeNull();
+ expect(httpMock.getTrace()).toMatchSnapshot();
+ });
+ it('simulate failing secondary release source', async () => {
+ httpMock
+ .scope(primaryUrl)
+ .get('/v1/providers/hashicorp/google-beta')
+ .reply(404, {
+ errors: ['Not Found'],
+ });
+ httpMock.scope(secondaryUrl).get('/index.json').reply(404);
+
+ const res = await getPkgReleases({
+ datasource,
+ depName: 'datadog',
+ });
+ expect(res).toMatchSnapshot();
+ expect(res).toBeNull();
+ });
});
});
diff --git a/lib/datasource/terraform-provider/index.ts b/lib/datasource/terraform-provider/index.ts
index f818251dad..5ba096a98e 100644
--- a/lib/datasource/terraform-provider/index.ts
+++ b/lib/datasource/terraform-provider/index.ts
@@ -1,9 +1,14 @@
+import URL from 'url';
import { logger } from '../../logger';
-import * as globalCache from '../../util/cache/global';
import { Http } from '../../util/http';
-import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common';
+import { GetReleasesConfig, ReleaseResult } from '../common';
export const id = 'terraform-provider';
+export const defaultRegistryUrls = [
+ 'https://registry.terraform.io',
+ 'https://releases.hashicorp.com',
+];
+export const registryStrategy = 'hunt';
const http = new Http(id);
@@ -15,30 +20,25 @@ interface TerraformProvider {
versions: string[];
}
-/**
- * terraform-provider.getReleases
- *
- * This function will fetch a provider from the public Terraform registry and return all semver versions.
- */
-export async function getReleases({
- lookupName,
-}: GetReleasesConfig): Promise<ReleaseResult | null> {
- const repository = `hashicorp/${lookupName}`;
+interface TerraformProviderReleaseBackend {
+ [key: string]: {
+ name: string;
+ versions: VersionsReleaseBackend;
+ };
+}
- logger.debug({ lookupName }, 'terraform-provider.getDependencies()');
- const cacheNamespace = 'terraform-providers';
- const pkgUrl = `https://registry.terraform.io/v1/providers/${repository}`;
- const cachedResult = await globalCache.get<ReleaseResult>(
- cacheNamespace,
- pkgUrl
- );
- // istanbul ignore if
- if (cachedResult) {
- return cachedResult;
- }
+interface VersionsReleaseBackend {
+ [key: string]: Record<string, any>;
+}
+
+async function queryRegistry(
+ lookupName: string,
+ registryURL: string,
+ repository: string
+): Promise<ReleaseResult> {
try {
- const res = (await http.getJson<TerraformProvider>(pkgUrl)).body;
- // Simplify response before caching and returning
+ const backendURL = `${registryURL}/v1/providers/${repository}`;
+ const res = (await http.getJson<TerraformProvider>(backendURL)).body;
const dep: ReleaseResult = {
name: repository,
versions: {},
@@ -50,33 +50,74 @@ export async function getReleases({
dep.releases = res.versions.map((version) => ({
version,
}));
- if (pkgUrl.startsWith('https://registry.terraform.io/')) {
- dep.homepage = `https://registry.terraform.io/providers/${repository}`;
- }
+ dep.homepage = `${registryURL}/providers/${repository}`;
logger.trace({ dep }, 'dep');
- const cacheMinutes = 30;
- await globalCache.set(cacheNamespace, pkgUrl, dep, cacheMinutes);
return dep;
} catch (err) {
- if (err.statusCode === 404 || err.code === 'ENOTFOUND') {
- logger.debug(
- { lookupName },
- `Terraform registry lookup failure: not found`
- );
- logger.debug({
- err,
- });
- return null;
- }
- const failureCodes = ['EAI_AGAIN'];
- // istanbul ignore if
- if (failureCodes.includes(err.code)) {
- throw new DatasourceError(err);
- }
- logger.warn(
- { err, lookupName },
- 'Terraform registry failure: Unknown error'
+ logger.debug(
+ { lookupName },
+ `Terraform registry ("${registryURL}") lookup failure: not found`
+ );
+ logger.debug({
+ err,
+ });
+ return null;
+ }
+}
+
+async function queryReleaseBackend(
+ lookupName: string,
+ registryURL: string,
+ repository: string
+): Promise<ReleaseResult> {
+ const backendLookUpName = `terraform-provider-${lookupName}`;
+ const backendURL = registryURL + `/index.json`;
+ try {
+ const res = (
+ await http.getJson<TerraformProviderReleaseBackend>(backendURL)
+ ).body;
+ const dep: ReleaseResult = {
+ name: repository,
+ versions: {},
+ releases: null,
+ };
+ dep.releases = Object.keys(res[backendLookUpName].versions).map(
+ (version) => ({
+ version,
+ })
+ );
+ logger.trace({ dep }, 'dep');
+ return dep;
+ } catch (err) {
+ logger.debug(
+ { lookupName },
+ `Terraform registry ("${registryURL}") lookup failure: not found`
);
+ logger.debug({
+ err,
+ });
return null;
}
}
+
+/**
+ * terraform-provider.getReleases
+ *
+ * This function will fetch a provider from the public Terraform registry and return all semver versions.
+ */
+export async function getReleases({
+ lookupName,
+ registryUrl,
+}: GetReleasesConfig): Promise<ReleaseResult | null> {
+ const repository = `hashicorp/${lookupName}`;
+
+ logger.debug({ lookupName }, 'terraform-provider.getDependencies()');
+ let dep: ReleaseResult = null;
+ const registryHost = URL.parse(registryUrl).host;
+ if (registryHost === 'registry.terraform.io') {
+ dep = await queryRegistry(lookupName, registryUrl, repository);
+ } else if (registryHost === 'releases.hashicorp.com') {
+ dep = await queryReleaseBackend(lookupName, registryUrl, repository);
+ }
+ return dep;
+}
--
GitLab