diff --git a/.gitignore b/.gitignore index 21e9a12ebb88452482f9f7976308069a525d00d3..28c6004c6f8e75be6a09bcee2d31c37fb4b851f7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ /.vscode /.idea package-lock.json + +*.pyc diff --git a/.travis.yml b/.travis.yml index 47570dfa3115a6a99edf32f7cd54794d549f847e..8568185bf4855e98df0824bf84fa4f8a147cf835 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,9 +18,11 @@ if: tag IS blank before_install: - curl -o- -L https://yarnpkg.com/install.sh | bash - export PATH="$HOME/.yarn/bin:$PATH" + - python --version install: - yarn install --frozen-lockfile + - pip install --user -r requirements.txt cache: yarn: true diff --git a/lib/config/definitions.js b/lib/config/definitions.js index 16e58610c9eebeb2e214b6533d0bf6026b7fc7ed..df073c5276c7ef0a310c30ec795a98959999ea33 100644 --- a/lib/config/definitions.js +++ b/lib/config/definitions.js @@ -1187,6 +1187,18 @@ const options = [ mergeable: true, cli: false, }, + { + name: 'pip_setup', + description: 'Configuration object for setup.py files', + stage: 'package', + type: 'json', + default: { + enabled: false, + fileMatch: ['(^|\\/)setup.py$'], + }, + mergeable: true, + cli: false, + }, { name: 'python', description: 'Configuration object for python', diff --git a/lib/manager/index.js b/lib/manager/index.js index 433e1ccf3cbb5e88d75a5db44c0e02eb31f21f0f..cebad074bec84b1286b4d98cf88e33a0f53215b4 100644 --- a/lib/manager/index.js +++ b/lib/manager/index.js @@ -13,6 +13,7 @@ const managerList = [ 'npm', 'nvm', 'pip_requirements', + 'pip_setup', 'terraform', 'travis', 'nuget', diff --git a/lib/manager/pip_requirements/extract.js b/lib/manager/pip_requirements/extract.js index ec6db816d49273e38e90765710d57963823eeda7..26237ac74fede66b927adbce25a0de7ef48419c5 100644 --- a/lib/manager/pip_requirements/extract.js +++ b/lib/manager/pip_requirements/extract.js @@ -3,6 +3,8 @@ const packagePattern = '[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9._-]*[a-zA-Z0-9]'; const extrasPattern = '(?:\\s*\\[[^\\]]+\\])?'; const rangePattern = require('@renovate/pep440/lib/specifier').RANGE_PATTERN; +const { isSkipComment } = require('../../util/ignore'); + const specifierPartPattern = `\\s*${rangePattern.replace(/\?<\w+>/g, '?:')}`; const specifierPattern = `${specifierPartPattern}(?:\\s*,${specifierPartPattern})*`; const dependencyPattern = `(${packagePattern})(${extrasPattern})(${specifierPattern})`; @@ -30,16 +32,8 @@ function extractPackageFile(content) { .map((rawline, lineNumber) => { let dep = {}; const [line, comment] = rawline.split('#').map(part => part.trim()); - if (comment && comment.match(/^(renovate|pyup):/)) { - const command = comment - .split('#')[0] - .split(':')[1] - .trim(); - if (command === 'ignore') { - dep.skipReason = 'ignored'; - } else { - logger.info('Unknown pip_requirements command: ' + command); - } + if (isSkipComment(comment)) { + dep.skipReason = 'ignored'; } regex.lastIndex = 0; const matches = regex.exec(line); diff --git a/lib/manager/pip_setup/extract.js b/lib/manager/pip_setup/extract.js new file mode 100644 index 0000000000000000000000000000000000000000..20c0e676216a149d92dfa1c89c87ef8e3ccef952 --- /dev/null +++ b/lib/manager/pip_setup/extract.js @@ -0,0 +1,112 @@ +const { exec } = require('child-process-promise'); +const fs = require('fs-extra'); +const { join } = require('upath'); +const { isSkipComment } = require('../../util/ignore'); +const { dependencyPattern } = require('../pip_requirements/extract'); + +module.exports = { + extractPackageFile, + extractSetupFile, +}; + +async function extractSetupFile(content, packageFile, config) { + const cwd = config.localDir; + // extract.py needs setup.py to be written to disk + if (!config.gitFs) { + const localFileName = join(config.localDir, packageFile); + await fs.outputFile(localFileName, content); + } + let cmd; + const args = [join(__dirname, 'extract.py'), packageFile]; + // istanbul ignore if + if (config.binarySource === 'docker') { + logger.info('Running python via docker'); + cmd = 'docker'; + args.unshift( + 'run', + '-i', + '--rm', + // volume + '-v', + `${cwd}:${cwd}`, + '-v', + `${__dirname}:${__dirname}`, + // cwd + '-w', + cwd, + // image + 'renovate/pip', + 'python' + ); + } else { + logger.info('Running python via global command'); + cmd = 'python'; + } + logger.debug({ cmd, args }, 'python command'); + + const { stdout, stderr } = await exec(`${cmd} ${args.join(' ')}`, { + cwd, + shell: true, + timeout: 3000, + }); + // istanbul ignore if + if (stderr) { + logger.warn({ stderr }, 'Error in read setup file'); + } + return JSON.parse(stdout); +} + +async function extractPackageFile(content, packageFile, config) { + logger.debug('pip_setup.extractPackageFile()'); + let setup; + try { + setup = await extractSetupFile(content, packageFile, config); + } catch (err) { + logger.warn({ err }, 'Failed to read setup file'); + return null; + } + const requires = []; + if (setup.install_requires) { + requires.push(...setup.install_requires); + } + if (setup.extras_require) { + for (const req of Object.values(setup.extras_require)) { + requires.push(...req); + } + } + const regex = new RegExp(`^${dependencyPattern}`); + const lines = content.split('\n'); + const deps = requires + .map(req => { + const lineNumber = lines.findIndex(l => l.includes(req)); + if (lineNumber === -1) { + return null; + } + const rawline = lines[lineNumber]; + let dep = {}; + const [, comment] = rawline.split('#').map(part => part.trim()); + if (isSkipComment(comment)) { + dep.skipReason = 'ignored'; + } + regex.lastIndex = 0; + const matches = regex.exec(req); + if (!matches) { + return null; + } + const [, depName, , currentValue] = matches; + dep = { + ...dep, + depName, + currentValue, + lineNumber, + purl: 'pkg:pypi/' + depName, + versionScheme: 'pep440', + }; + return dep; + }) + .filter(Boolean); + if (!deps.length) { + return null; + } + return { deps }; +} diff --git a/lib/manager/pip_setup/extract.py b/lib/manager/pip_setup/extract.py new file mode 100644 index 0000000000000000000000000000000000000000..e49aa52fd8cdceef0ad871e5d1e1c276261f4929 --- /dev/null +++ b/lib/manager/pip_setup/extract.py @@ -0,0 +1,30 @@ +from __future__ import print_function +import sys +import imp +import json +import distutils.core + +try: + import setuptools +except ImportError: + class setuptools: + def setup(): + pass + +try: + from unittest import mock +except ImportError: + # for python3.3+ + import mock + +@mock.patch.object(setuptools, 'setup') +@mock.patch.object(distutils.core, 'setup') +def invoke(mock1, mock2): + # This is setup.py which calls setuptools.setup + imp.load_source('_target_setup_', sys.argv[-1]) + # called arguments are in `mock_setup.call_args` + call_args = mock1.call_args or mock2.call_args + args, kwargs = call_args + print(json.dumps(kwargs, indent=2)) + +invoke() diff --git a/lib/manager/pip_setup/index.js b/lib/manager/pip_setup/index.js new file mode 100644 index 0000000000000000000000000000000000000000..a6af78689a4f6a4c33747f28a3d4225461635416 --- /dev/null +++ b/lib/manager/pip_setup/index.js @@ -0,0 +1,10 @@ +const { extractPackageFile } = require('./extract'); +const { updateDependency } = require('../pip_requirements/update'); + +const language = 'python'; + +module.exports = { + extractPackageFile, + language, + updateDependency, +}; diff --git a/lib/util/ignore.js b/lib/util/ignore.js new file mode 100644 index 0000000000000000000000000000000000000000..dfb2b60a8abadff412cab932166d989569416258 --- /dev/null +++ b/lib/util/ignore.js @@ -0,0 +1,17 @@ +module.exports = { + isSkipComment, +}; + +function isSkipComment(comment) { + if (comment && comment.match(/^(renovate|pyup):/)) { + const command = comment + .split('#')[0] + .split(':')[1] + .trim(); + if (command === 'ignore') { + return true; + } + logger.info('Unknown comment command: ' + command); + } + return false; +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..14fe0d3ef2fd7101edf555e2c4f3906868bfa60f --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +mock==2.0.0 diff --git a/test/_fixtures/pip_setup/setup.py b/test/_fixtures/pip_setup/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..a1aab2a6915cff1309c3c8874d766841f9d5362b --- /dev/null +++ b/test/_fixtures/pip_setup/setup.py @@ -0,0 +1,89 @@ + +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +setup( + author='Simon Davy', + author_email='simon.davy@canonical.com', + classifiers=[ + 'License :: OSI Approved :: Apache Software License', + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Natural Language :: English', + 'Topic :: Internet :: WWW/HTTP :: WSGI', + 'Topic :: Internet :: WWW/HTTP :: WSGI :: Middleware', + 'Topic :: System :: Logging', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: Implementation :: CPython', + ], + description='A common WSGI stack', + entry_points=dict( + console_scripts=[ + 'talisker=talisker:run_gunicorn', + 'talisker.run=talisker:run', + 'talisker.gunicorn=talisker:run_gunicorn', + 'talisker.gunicorn.eventlet=talisker:run_gunicorn_eventlet', + 'talisker.gunicorn.gevent=talisker:run_gunicorn_gevent', + 'talisker.celery=talisker:run_celery', + ], + ), + extras_require=dict( + celery=[ + 'celery>=3.1.13.0,<5.0', + ], + dev=[ + 'logging_tree>=1.7', + 'pygments>=2.2', + 'psutil>=5.0', + 'objgraph>=3.0', + ], + django=[ + 'django>=1.10,<2.0', + ], + flask=[ + 'flask>=0.11,<2.0', + 'blinker>=1.4,<2.0', + ], + pg=[ + 'sqlparse', + 'psycopg2', + ], + prometheus=[ + 'prometheus-client>=0.2.0,<0.5.0' + ',!=0.4.0,!=0.4.1', + ], + ), + include_package_data=True, + install_requires=[ + 'gunicorn>=19.7.0,<20.0', + 'Werkzeug>=0.11.5,<0.15', + 'statsd>=3.2.1,<4.0', + 'requests>=2.10.0,<3.0', # renovate: ignore + 'raven>=5.27.1,<7.0', # pyup: nothing + 'future>=0.15.2,<0.17', + 'ipaddress>=1.0.16,<2.0;python_version<"3.3"', + ], + keywords=[ + 'talisker', + ], + name='talisker', + package_data=dict( + talisker=[ + 'logstash/*', + ], + ), + package_dir=dict( + talisker='talisker', + ), + packages=[ + 'talisker', + ], + test_suite='tests', + url='https://github.com/canonical-ols/talisker', + version='0.9.16', + zip_safe=False, +) diff --git a/test/manager/pip_setup/__snapshots__/extract.spec.js.snap b/test/manager/pip_setup/__snapshots__/extract.spec.js.snap new file mode 100644 index 0000000000000000000000000000000000000000..c6839b1e1c69192c4c0b51bc71a20aef4178dfca --- /dev/null +++ b/test/manager/pip_setup/__snapshots__/extract.spec.js.snap @@ -0,0 +1,200 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`lib/manager/pip_setup/extract extractPackageFile() returns found deps 1`] = ` +Object { + "deps": Array [ + Object { + "currentValue": ">=19.7.0,<20.0", + "depName": "gunicorn", + "lineNumber": 61, + "purl": "pkg:pypi/gunicorn", + "versionScheme": "pep440", + }, + Object { + "currentValue": ">=0.11.5,<0.15", + "depName": "Werkzeug", + "lineNumber": 62, + "purl": "pkg:pypi/Werkzeug", + "versionScheme": "pep440", + }, + Object { + "currentValue": ">=3.2.1,<4.0", + "depName": "statsd", + "lineNumber": 63, + "purl": "pkg:pypi/statsd", + "versionScheme": "pep440", + }, + Object { + "currentValue": ">=2.10.0,<3.0", + "depName": "requests", + "lineNumber": 64, + "purl": "pkg:pypi/requests", + "skipReason": "ignored", + "versionScheme": "pep440", + }, + Object { + "currentValue": ">=5.27.1,<7.0", + "depName": "raven", + "lineNumber": 65, + "purl": "pkg:pypi/raven", + "versionScheme": "pep440", + }, + Object { + "currentValue": ">=0.15.2,<0.17", + "depName": "future", + "lineNumber": 66, + "purl": "pkg:pypi/future", + "versionScheme": "pep440", + }, + Object { + "currentValue": ">=1.0.16,<2.0", + "depName": "ipaddress", + "lineNumber": 67, + "purl": "pkg:pypi/ipaddress", + "versionScheme": "pep440", + }, + Object { + "currentValue": ">=3.1.13.0,<5.0", + "depName": "celery", + "lineNumber": 36, + "purl": "pkg:pypi/celery", + "versionScheme": "pep440", + }, + Object { + "currentValue": ">=0.11,<2.0", + "depName": "flask", + "lineNumber": 48, + "purl": "pkg:pypi/flask", + "versionScheme": "pep440", + }, + Object { + "currentValue": ">=1.4,<2.0", + "depName": "blinker", + "lineNumber": 49, + "purl": "pkg:pypi/blinker", + "versionScheme": "pep440", + }, + Object { + "currentValue": ">=1.7", + "depName": "logging_tree", + "lineNumber": 39, + "purl": "pkg:pypi/logging_tree", + "versionScheme": "pep440", + }, + Object { + "currentValue": ">=2.2", + "depName": "pygments", + "lineNumber": 40, + "purl": "pkg:pypi/pygments", + "versionScheme": "pep440", + }, + Object { + "currentValue": ">=5.0", + "depName": "psutil", + "lineNumber": 41, + "purl": "pkg:pypi/psutil", + "versionScheme": "pep440", + }, + Object { + "currentValue": ">=3.0", + "depName": "objgraph", + "lineNumber": 42, + "purl": "pkg:pypi/objgraph", + "versionScheme": "pep440", + }, + Object { + "currentValue": ">=1.10,<2.0", + "depName": "django", + "lineNumber": 45, + "purl": "pkg:pypi/django", + "versionScheme": "pep440", + }, + ], +} +`; + +exports[`lib/manager/pip_setup/extract extractSetupFile() should return parsed setup() call 1`] = ` +Object { + "author": "Simon Davy", + "author_email": "simon.davy@canonical.com", + "classifiers": Array [ + "License :: OSI Approved :: Apache Software License", + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Natural Language :: English", + "Topic :: Internet :: WWW/HTTP :: WSGI", + "Topic :: Internet :: WWW/HTTP :: WSGI :: Middleware", + "Topic :: System :: Logging", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: Implementation :: CPython", + ], + "description": "A common WSGI stack", + "entry_points": Object { + "console_scripts": Array [ + "talisker=talisker:run_gunicorn", + "talisker.run=talisker:run", + "talisker.gunicorn=talisker:run_gunicorn", + "talisker.gunicorn.eventlet=talisker:run_gunicorn_eventlet", + "talisker.gunicorn.gevent=talisker:run_gunicorn_gevent", + "talisker.celery=talisker:run_celery", + ], + }, + "extras_require": Object { + "celery": Array [ + "celery>=3.1.13.0,<5.0", + ], + "dev": Array [ + "logging_tree>=1.7", + "pygments>=2.2", + "psutil>=5.0", + "objgraph>=3.0", + ], + "django": Array [ + "django>=1.10,<2.0", + ], + "flask": Array [ + "flask>=0.11,<2.0", + "blinker>=1.4,<2.0", + ], + "pg": Array [ + "sqlparse", + "psycopg2", + ], + "prometheus": Array [ + "prometheus-client>=0.2.0,<0.5.0,!=0.4.0,!=0.4.1", + ], + }, + "include_package_data": true, + "install_requires": Array [ + "gunicorn>=19.7.0,<20.0", + "Werkzeug>=0.11.5,<0.15", + "statsd>=3.2.1,<4.0", + "requests>=2.10.0,<3.0", + "raven>=5.27.1,<7.0", + "future>=0.15.2,<0.17", + "ipaddress>=1.0.16,<2.0;python_version<\\"3.3\\"", + ], + "keywords": Array [ + "talisker", + ], + "name": "talisker", + "package_data": Object { + "talisker": Array [ + "logstash/*", + ], + }, + "package_dir": Object { + "talisker": "talisker", + }, + "packages": Array [ + "talisker", + ], + "test_suite": "tests", + "url": "https://github.com/canonical-ols/talisker", + "version": "0.9.16", + "zip_safe": false, +} +`; diff --git a/test/manager/pip_setup/extract.spec.js b/test/manager/pip_setup/extract.spec.js new file mode 100644 index 0000000000000000000000000000000000000000..c5c091b11db4840bd2bdfe2942da8a075590d6e4 --- /dev/null +++ b/test/manager/pip_setup/extract.spec.js @@ -0,0 +1,67 @@ +const fs = require('fs'); +const tmp = require('tmp-promise'); +const { relative } = require('path'); +const { + extractPackageFile, + extractSetupFile, +} = require('../../../lib/manager/pip_setup/extract'); + +const packageFile = 'test/_fixtures/pip_setup/setup.py'; +const content = fs.readFileSync(packageFile, 'utf8'); +const config = { + localDir: '.', +}; + +async function tmpFile() { + const file = await tmp.file({ postfix: '.py' }); + return relative('.', file.path); +} + +describe('lib/manager/pip_setup/extract', () => { + describe('extractPackageFile()', () => { + it('returns found deps', async () => { + expect( + await extractPackageFile(content, packageFile, config) + ).toMatchSnapshot(); + }); + it('should return null for invalid file', async () => { + expect( + await extractPackageFile('raise Exception()', await tmpFile(), config) + ).toBe(null); + }); + it('should return null for no deps file', async () => { + expect( + await extractPackageFile( + 'from setuptools import setup\nsetup()', + await tmpFile(), + config + ) + ).toBe(null); + }); + }); + describe('extractSetupFile()', () => { + it('should return parsed setup() call', async () => { + expect( + await extractSetupFile(content, packageFile, config) + ).toMatchSnapshot(); + }); + it('should support setuptools', async () => { + expect( + await extractSetupFile( + 'from setuptools import setup\nsetup(name="talisker")\n', + await tmpFile(), + config + ) + ).toEqual({ name: 'talisker' }); + }); + it('should support distutils.core', async () => { + expect( + await extractSetupFile( + 'from distutils.core import setup\nsetup(name="talisker")\n', + await tmpFile(), + config + ) + ).toEqual({ name: 'talisker' }); + }); + }); +}); diff --git a/test/workers/repository/extract/__snapshots__/index.spec.js.snap b/test/workers/repository/extract/__snapshots__/index.spec.js.snap index 81f621ea2c5475cbfc1ebad80bb0eddd06db1bbc..f958872656688a92e03d4e3da827cd3408b72eb4 100644 --- a/test/workers/repository/extract/__snapshots__/index.spec.js.snap +++ b/test/workers/repository/extract/__snapshots__/index.spec.js.snap @@ -50,6 +50,9 @@ Object { "pip_requirements": Array [ Object {}, ], + "pip_setup": Array [ + Object {}, + ], "terraform": Array [ Object {}, ], diff --git a/website/docs/configuration-options.md b/website/docs/configuration-options.md index 80ba2cfc5e6681d1b72d0908041512efd021826b..ed370018ddfb815057c2bc38da2069c08d483947 100644 --- a/website/docs/configuration-options.md +++ b/website/docs/configuration-options.md @@ -572,6 +572,10 @@ By default, Renovate will add sha256 digests to Docker source images so that the Add configuration here to specifically override settings for `pip` requirements files. Supports `requirements.txt` and `requirements.pip` files. The default file pattern is fairly flexible in an attempt to catch similarly named ones too but may be extended/changed. +## pip_setup + +Add configuration here to specifically override settings for `setup.py` files. + ## prBodyColumns Use this array to provide a list of column names you wish to include in the PR tables.