feat(manager): Support poetry custom repositories. (#4524)

lib/manager/poetry/extract.ts

@@ -28,7 +28,8 @@ export function extractPackageFile(
   if (!deps.length) {
     return null;
   }
-  return { deps };
+
+  return { deps, registryUrls: extractRegistries(pyprojectfile) };
 }
 
 function extractFromSection(
@@ -84,3 +85,24 @@ function extractFromSection(
   });
   return deps;
 }
+
+function extractRegistries(pyprojectfile: PoetryFile): string[] {
+  const sources =
+    pyprojectfile.tool &&
+    pyprojectfile.tool.poetry &&
+    pyprojectfile.tool.poetry.source;
+
+  if (!Array.isArray(sources) || sources.length === 0) {
+    return null;
+  }
+
+  const registryUrls = new Set<string>();
+  for (const source of sources) {
+    if (source.url) {
+      registryUrls.add(source.url);
+    }
+  }
+  registryUrls.add('https://pypi.org/pypi/');
+
+  return Array.from(registryUrls);
+}

lib/manager/poetry/types.ts

@@ -2,6 +2,7 @@ export interface PoetrySection {
   dependencies: Record<string, PoetryDependency | string>;
   'dev-dependencies': Record<string, PoetryDependency | string>;
   extras: Record<string, PoetryDependency | string>;
+  source?: PoetrySource[];
 }
 
 export interface PoetryFile {
@@ -15,3 +16,8 @@ export interface PoetryDependency {
   git?: string;
   version?: string;
 }
+
+export interface PoetrySource {
+  name?: string;
+  url?: string;
+}

test/manager/poetry/__snapshots__/extract.spec.ts.snap

 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`lib/manager/poetry/extract extractPackageFile() dedupes registries 1`] = `
+Array [
+  "https://pypi.org/pypi/",
+  "https://bar.baz/+simple/",
+]
+`;
+
 exports[`lib/manager/poetry/extract extractPackageFile() extracts multiple dependencies (with dep = {version = "1.2.3"} case) 1`] = `
 Array [
   Object {
@@ -157,6 +164,14 @@ Array [
 ]
 `;
 
+exports[`lib/manager/poetry/extract extractPackageFile() extracts registries 1`] = `
+Array [
+  "https://foo.bar/simple/",
+  "https://bar.baz/+simple/",
+  "https://pypi.org/pypi/",
+]
+`;
+
 exports[`lib/manager/poetry/extract extractPackageFile() handles multiple constraint dependencies 1`] = `
 Array [
   Object {

test/manager/poetry/_fixtures/pyproject.6.toml

+[tool.poetry]
+name = "example 6"
+version = "0.1.0"
+description = ""
+authors = ["John Doe <john.doe@gmail.com>"]
+
+[tool.poetry.dependencies]
+dep0 = "0.0.0"
+
+[[tool.poetry.source]]
+name = "foo"
+url = "https://foo.bar/simple/"
+
+[[tool.poetry.source]]
+name = "bar"
+url = "https://bar.baz/+simple/"
\ No newline at end of file

test/manager/poetry/_fixtures/pyproject.7.toml

+[tool.poetry]
+name = "example 7"
+version = "0.1.0"
+description = ""
+authors = ["John Doe <john.doe@gmail.com>"]
+source = []
+
+[tool.poetry.dependencies]
+dep0 = "0.0.0"
\ No newline at end of file

test/manager/poetry/_fixtures/pyproject.8.toml

+[tool.poetry]
+name = "example 8"
+version = "0.1.0"
+description = ""
+authors = ["John Doe <john.doe@gmail.com>"]
+
+[tool.poetry.dependencies]
+dep0 = "0.0.0"
+
+[[tool.poetry.source]]
+name = "foo"
+url = "https://pypi.org/pypi/"
+
+[[tool.poetry.source]]
+name = "bar"
+url = "https://bar.baz/+simple/"
+
+[[tool.poetry.source]]
+name = "baz"
+url = "https://bar.baz/+simple/"

test/manager/poetry/extract.spec.ts

@@ -26,41 +26,73 @@ const pyproject5toml = readFileSync(
   'utf8'
 );
 
+const pyproject6toml = readFileSync(
+  'test/manager/poetry/_fixtures/pyproject.6.toml',
+  'utf8'
+);
+
+const pyproject7toml = readFileSync(
+  'test/manager/poetry/_fixtures/pyproject.7.toml',
+  'utf8'
+);
+
+const pyproject8toml = readFileSync(
+  'test/manager/poetry/_fixtures/pyproject.8.toml',
+  'utf8'
+);
+
 describe('lib/manager/poetry/extract', () => {
   describe('extractPackageFile()', () => {
-    let config;
+    let filename: string;
     beforeEach(() => {
-      config = {};
+      filename = '';
     });
     it('returns null for empty', () => {
-      expect(extractPackageFile('nothing here', config)).toBeNull();
+      expect(extractPackageFile('nothing here', filename)).toBeNull();
     });
     it('returns null for parsed file without poetry section', () => {
-      expect(extractPackageFile(pyproject5toml, config)).toBeNull();
+      expect(extractPackageFile(pyproject5toml, filename)).toBeNull();
     });
     it('extracts multiple dependencies', () => {
-      const res = extractPackageFile(pyproject1toml, config);
+      const res = extractPackageFile(pyproject1toml, filename);
       expect(res.deps).toMatchSnapshot();
       expect(res.deps).toHaveLength(9);
     });
     it('extracts multiple dependencies (with dep = {version = "1.2.3"} case)', () => {
-      const res = extractPackageFile(pyproject2toml, config);
+      const res = extractPackageFile(pyproject2toml, filename);
       expect(res.deps).toMatchSnapshot();
       expect(res.deps).toHaveLength(7);
     });
     it('handles case with no dependencies', () => {
-      const res = extractPackageFile(pyproject3toml, config);
+      const res = extractPackageFile(pyproject3toml, filename);
       expect(res).toBeNull();
     });
     it('handles multiple constraint dependencies', () => {
-      const res = extractPackageFile(pyproject4toml, config);
+      const res = extractPackageFile(pyproject4toml, filename);
       expect(res.deps).toMatchSnapshot();
       expect(res.deps).toHaveLength(1);
     });
+    it('extracts registries', () => {
+      const res = extractPackageFile(pyproject6toml, filename);
+      expect(res.registryUrls).toMatchSnapshot();
+      expect(res.registryUrls).toHaveLength(3);
+    });
+    it('can parse empty registries', () => {
+      const res = extractPackageFile(pyproject7toml, filename);
+      expect(res.registryUrls).toBeNull();
+    });
+    it('can parse missing registries', () => {
+      const res = extractPackageFile(pyproject1toml, filename);
+      expect(res.registryUrls).toBeNull();
+    });
+    it('dedupes registries', () => {
+      const res = extractPackageFile(pyproject8toml, filename);
+      expect(res.registryUrls).toMatchSnapshot();
+    });
     it('skips git dependencies', () => {
       const content =
         '[tool.poetry.dependencies]\r\nflask = {git = "https://github.com/pallets/flask.git"}\r\nwerkzeug = ">=0.14"';
-      const res = extractPackageFile(content, config).deps;
+      const res = extractPackageFile(content, filename).deps;
       expect(res[0].depName).toBe('flask');
       expect(res[0].currentValue).toBe('');
       expect(res[0].skipReason).toBe('git-dependency');
@@ -69,7 +101,7 @@ describe('lib/manager/poetry/extract', () => {
     it('skips git dependencies', () => {
       const content =
         '[tool.poetry.dependencies]\r\nflask = {git = "https://github.com/pallets/flask.git", version="1.2.3"}\r\nwerkzeug = ">=0.14"';
-      const res = extractPackageFile(content, config).deps;
+      const res = extractPackageFile(content, filename).deps;
       expect(res[0].depName).toBe('flask');
       expect(res[0].currentValue).toBe('1.2.3');
       expect(res[0].skipReason).toBe('git-dependency');
@@ -78,7 +110,7 @@ describe('lib/manager/poetry/extract', () => {
     it('skips path dependencies', () => {
       const content =
         '[tool.poetry.dependencies]\r\nflask = {path = "/some/path/"}\r\nwerkzeug = ">=0.14"';
-      const res = extractPackageFile(content, config).deps;
+      const res = extractPackageFile(content, filename).deps;
       expect(res[0].depName).toBe('flask');
       expect(res[0].currentValue).toBe('');
       expect(res[0].skipReason).toBe('path-dependency');
@@ -87,7 +119,7 @@ describe('lib/manager/poetry/extract', () => {
     it('skips path dependencies', () => {
       const content =
         '[tool.poetry.dependencies]\r\nflask = {path = "/some/path/", version = "1.2.3"}\r\nwerkzeug = ">=0.14"';
-      const res = extractPackageFile(content, config).deps;
+      const res = extractPackageFile(content, filename).deps;
       expect(res[0].depName).toBe('flask');
       expect(res[0].currentValue).toBe('1.2.3');
       expect(res[0].skipReason).toBe('path-dependency');