Skip to content
Snippets Groups Projects
Unverified Commit b696abb3 authored by Yun Lai's avatar Yun Lai Committed by GitHub
Browse files

feat: add Hermit package manager (#16258)


* feat: add Hermit package manager

* fix: pass bin directory into getRepoStatus as string rather than an array

* fix: fix up hermit manager implementations

* add docker support in exec
* move fs related operations back into  util/fs
* remove ENVVar passed on by process.env
* set concurrency in pMap
* use for instead of pMap for concurrency = 1
* use regex to pick up package reference parts

* fix: fix manager updateArtifacts test after change

* Update lib/modules/manager/hermit/extract.ts

Co-authored-by: default avatarPhilip <42116482+PhilipAbed@users.noreply.github.com>

* fix: fix up test and docker reference for hermit manager

* test refer to internal fs
* docker image change to sidecar
* only symlink are read for the changed file content after hermit
  install
* no more global mock in artifacts test

* fix: use warn instead of error so error better flows up in hermit manager

* fix: partial for test type, use throw instead of reject

* fix: update snapshot

* fix: combine install packages, also make extractPackageFile async

* fix: remove weird generated readLocalSynmlink in test

* fix: removes old test

* fix: use ensureLocalPath and fix test coverage

* fix: more test coverage

* fix: use ensureLocalPath in readLocalSymlink

* Apply suggestions from code review

Co-authored-by: default avatarMichael Kriese <michael.kriese@visualon.de>

* fix: remove unused functions and types

* Apply suggestions from code review

Co-authored-by: default avatarMichael Kriese <michael.kriese@visualon.de>

* Apply suggestions from code review

Co-authored-by: default avatarSergei Zharinov <zharinov@users.noreply.github.com>

* fix: use execSnapshots and for of loop when returning the result

* Update lib/modules/manager/hermit/artifacts.spec.ts

Co-authored-by: default avatarMichael Kriese <michael.kriese@visualon.de>

* fix: move exports below imports

Co-authored-by: default avatarPhilip <42116482+PhilipAbed@users.noreply.github.com>
Co-authored-by: default avatarMichael Kriese <michael.kriese@visualon.de>
Co-authored-by: default avatarSergei Zharinov <zharinov@users.noreply.github.com>
parent bba3c782
No related branches found
No related tags found
No related merge requests found
......@@ -36,6 +36,7 @@ import * as helmValues from './helm-values';
import * as helmfile from './helmfile';
import * as helmsman from './helmsman';
import * as helmv3 from './helmv3';
import * as hermit from './hermit';
import * as homebrew from './homebrew';
import * as html from './html';
import * as jenkins from './jenkins';
......@@ -114,6 +115,7 @@ api.set('helm-values', helmValues);
api.set('helmfile', helmfile);
api.set('helmsman', helmsman);
api.set('helmv3', helmv3);
api.set('hermit', hermit);
api.set('homebrew', homebrew);
api.set('html', html);
api.set('jenkins', jenkins);
......
import { mockExecAll } from '../../../../test/exec-util';
import { mockedFunction, partial } from '../../../../test/util';
import { GlobalConfig } from '../../../config/global';
import { ExecError } from '../../../util/exec/exec-error';
import { localPathIsSymbolicLink, readLocalSymlink } from '../../../util/fs';
import { getRepoStatus } from '../../../util/git';
import type { StatusResult } from '../../../util/git/types';
import type { UpdateArtifact } from '../types';
import { updateArtifacts } from '.';
jest.mock('../../../util/git');
jest.mock('../../../util/fs');
const getRepoStatusMock = mockedFunction(getRepoStatus);
const lstatsMock = mockedFunction(localPathIsSymbolicLink);
const readlinkMock = mockedFunction(readLocalSymlink);
describe('modules/manager/hermit/artifacts', () => {
describe('updateArtifacts', () => {
it('should run hermit install for packages and return updated files', async () => {
lstatsMock.mockResolvedValue(true);
readlinkMock.mockResolvedValue('hermit');
GlobalConfig.set({ localDir: '' });
const execSnapshots = mockExecAll();
getRepoStatusMock.mockResolvedValue(
partial<StatusResult>({
not_added: ['bin/go-1.17.1'],
deleted: ['bin/go-1.17'],
modified: ['bin/go', 'bin/jq'],
created: ['bin/jq-extra'],
renamed: [
{
from: 'bin/jq-1.5',
to: 'bin/jq-1.6',
},
],
})
);
const res = await updateArtifacts(
partial<UpdateArtifact>({
updatedDeps: [
{
depName: 'go',
currentVersion: '1.17',
newValue: '1.17.1',
},
{
depName: 'jq',
currentVersion: '1.5',
newValue: '1.6',
},
],
packageFileName: 'go/bin/hermit',
})
);
expect(execSnapshots).toMatchObject([
{ cmd: './hermit install go-1.17.1 jq-1.6' },
]);
expect(res).toStrictEqual([
{
file: {
path: 'bin/jq-1.5',
type: 'deletion',
},
},
{
file: {
contents: 'hermit',
isSymlink: true,
isExecutable: undefined,
path: 'bin/jq-1.6',
type: 'addition',
},
},
{
file: {
path: 'bin/go',
type: 'deletion',
},
},
{
file: {
contents: 'hermit',
isSymlink: true,
isExecutable: undefined,
path: 'bin/go',
type: 'addition',
},
},
{
file: {
path: 'bin/jq',
type: 'deletion',
},
},
{
file: {
contents: 'hermit',
isSymlink: true,
isExecutable: undefined,
path: 'bin/jq',
type: 'addition',
},
},
{
file: {
contents: 'hermit',
isSymlink: true,
isExecutable: undefined,
path: 'bin/jq-extra',
type: 'addition',
},
},
{
file: {
contents: 'hermit',
isSymlink: true,
isExecutable: undefined,
path: 'bin/go-1.17.1',
type: 'addition',
},
},
{
file: {
path: 'bin/go-1.17',
type: 'deletion',
},
},
]);
});
it('should fail on error getting link content', async () => {
lstatsMock.mockResolvedValue(true);
readlinkMock.mockResolvedValue(null);
GlobalConfig.set({ localDir: '' });
mockExecAll();
getRepoStatusMock.mockResolvedValue(
partial<StatusResult>({
not_added: [],
deleted: [],
modified: [],
created: [],
renamed: [
{
from: 'bin/jq-1.5',
to: 'bin/jq-1.6',
},
],
})
);
const res = await updateArtifacts(
partial<UpdateArtifact>({
updatedDeps: [
{
depName: 'go',
currentVersion: '1.17',
newValue: '1.17.1',
},
{
depName: 'jq',
currentVersion: '1.5',
newValue: '1.6',
},
],
packageFileName: 'go/bin/hermit',
})
);
expect(res).toEqual([
{
artifactError: {
stderr: 'error getting content for bin/jq-1.6',
},
},
]);
});
it('should return error on installation error', async () => {
mockExecAll(
new ExecError('', {
stdout: '',
stderr: 'error executing hermit install',
cmd: '',
options: {
encoding: 'utf-8',
},
})
);
const res = await updateArtifacts(
partial<UpdateArtifact>({
updatedDeps: [
{
depName: 'go',
currentVersion: '1.17',
newValue: '1.17.1',
},
{
depName: 'jq',
currentVersion: '1.5',
newValue: '1.6',
},
],
packageFileName: 'go/bin/hermit',
})
);
expect(res).toStrictEqual([
{
artifactError: {
lockFile: 'from: go-1.17 jq-1.5, to: go-1.17.1 jq-1.6',
stderr: 'error executing hermit install',
},
},
]);
});
it('should return error on invalid update information', async () => {
let res = await updateArtifacts(
partial<UpdateArtifact>({
updatedDeps: [
{
currentVersion: '1.17',
newValue: '1.17.1',
},
],
packageFileName: 'go/bin/hermit',
})
);
expect(res).toStrictEqual([
{
artifactError: {
lockFile: 'from: -1.17, to: -1.17.1',
stderr: `invalid package to update`,
},
},
]);
res = await updateArtifacts(
partial<UpdateArtifact>({
updatedDeps: [
{
depName: 'go',
newValue: '1.17.1',
},
],
packageFileName: 'go/bin/hermit',
})
);
expect(res).toStrictEqual([
{
artifactError: {
lockFile: 'from: go-, to: go-1.17.1',
stderr: `invalid package to update`,
},
},
]);
res = await updateArtifacts(
partial<UpdateArtifact>({
updatedDeps: [
{
depName: 'go',
currentVersion: '1.17',
},
],
packageFileName: 'go/bin/hermit',
})
);
expect(res).toStrictEqual([
{
artifactError: {
lockFile: 'from: go-1.17, to: go-',
stderr: `invalid package to update`,
},
},
]);
});
});
});
import pMap from 'p-map';
import upath from 'upath';
import { logger } from '../../../logger';
import { exec } from '../../../util/exec';
import type { ExecOptions } from '../../../util/exec/types';
import { localPathIsSymbolicLink, readLocalSymlink } from '../../../util/fs';
import { getRepoStatus } from '../../../util/git';
import type { UpdateArtifact, UpdateArtifactsResult } from '../types';
import type { ReadContentResult } from './types';
/**
* updateArtifacts runs hermit install for each updated dependencies
*/
export async function updateArtifacts(
update: UpdateArtifact
): Promise<UpdateArtifactsResult[] | null> {
const { packageFileName } = update;
logger.debug({ packageFileName }, `hermit.updateArtifacts()`);
try {
await updateHermitPackage(update);
} catch (err) {
const execErr: UpdateHermitError = err;
logger.debug({ err }, `error updating hermit packages.`);
return [
{
artifactError: {
lockFile: `from: ${execErr.from}, to: ${execErr.to}`,
stderr: execErr.stderr,
},
},
];
}
logger.debug(`scanning the changes after update`);
let updateResult: UpdateArtifactsResult[] | null = null;
try {
updateResult = await getUpdateResult(packageFileName);
logger.debug({ updateResult }, `update result for hermit`);
} catch (err) {
logger.debug({ err }, 'Error getting hermet update results');
return [
{
artifactError: {
stderr: err.message,
},
},
];
}
return updateResult;
}
/**
* getContent returns the content of either link or a normal file
*/
async function getContent(file: string): Promise<ReadContentResult> {
let contents: string | null = '';
const isSymlink = await localPathIsSymbolicLink(file);
if (isSymlink) {
contents = await readLocalSymlink(file);
}
if (contents === null) {
throw new Error(`error getting content for ${file}`);
}
return {
isSymlink,
contents,
};
}
/**
* getAddResult returns the UpdateArtifactsResult for the added files
*/
function getAddResult(
path: string,
contentRes: ReadContentResult
): UpdateArtifactsResult {
return {
file: {
type: 'addition',
path,
contents: contentRes.contents,
isSymlink: contentRes.isSymlink,
isExecutable: contentRes.isExecutable,
},
};
}
/**
* getDeleteResult returns the UpdateArtifactsResult for deleted files
*/
function getDeleteResult(path: string): UpdateArtifactsResult {
return {
file: {
type: 'deletion',
path,
},
};
}
/**
* getUpdateResult will return the update result after `hermit install`
* has been performed for all packages
*/
async function getUpdateResult(
packageFileName: string
): Promise<UpdateArtifactsResult[]> {
const hermitFolder = `${upath.dirname(packageFileName)}/`;
const hermitChanges = await getRepoStatus(hermitFolder);
logger.debug(
{ hermitChanges, hermitFolder },
`hermit changes after package update`
);
// handle added files
const added = await pMap(
[...hermitChanges.created, ...hermitChanges.not_added],
async (path: string): Promise<UpdateArtifactsResult> => {
const contents = await getContent(path);
return getAddResult(path, contents);
},
{ concurrency: 5 }
);
const deleted = hermitChanges.deleted.map(getDeleteResult);
const modified = await pMap(
hermitChanges.modified,
async (path: string): Promise<UpdateArtifactsResult[]> => {
const contents = await getContent(path);
return [
getDeleteResult(path), // delete existing link
getAddResult(path, contents), // add a new link
];
},
{ concurrency: 5 }
);
const renamed = await pMap(
hermitChanges.renamed,
async (renamed): Promise<UpdateArtifactsResult[]> => {
const from = renamed.from;
const to = renamed.to;
const toContents = await getContent(to);
return [getDeleteResult(from), getAddResult(to, toContents)];
},
{ concurrency: 5 }
);
return [
// rename will need to go first, because
// it needs to create the new link for the new version
// for the modified links to use
...renamed.flat(),
...modified.flat(),
...added,
...deleted,
];
}
/**
* getHermitPackage returns the hermit package for running the hermit install
*/
function getHermitPackage(name: string, version: string): string {
return `${name}-${version}`;
}
/**
* updateHermitPackage runs hermit install for the given package
*/
async function updateHermitPackage(update: UpdateArtifact): Promise<void> {
logger.trace({ update }, `hermit.updateHermitPackage()`);
const toInstall = [];
const from = [];
for (const pkg of update.updatedDeps) {
if (!pkg.depName || !pkg.currentVersion || !pkg.newValue) {
logger.debug(
{
depName: pkg.depName,
currentVersion: pkg.currentVersion,
newValue: pkg.newValue,
},
'missing package update information'
);
throw new UpdateHermitError(
getHermitPackage(pkg.depName ?? '', pkg.currentVersion ?? ''),
getHermitPackage(pkg.depName ?? '', pkg.newValue ?? ''),
'invalid package to update'
);
}
const depName = pkg.depName;
const currentVersion = pkg.currentVersion;
const newValue = pkg.newValue;
const fromPackage = getHermitPackage(depName, currentVersion);
const toPackage = getHermitPackage(depName, newValue);
toInstall.push(toPackage);
from.push(fromPackage);
}
const execOptions: ExecOptions = {
docker: {
image: 'sidecar',
},
cwdFile: update.packageFileName,
};
const packagesToInstall = toInstall.join(' ');
const fromPackages = from.join(' ');
const execCommands = `./hermit install ${packagesToInstall}`;
logger.debug(
{
packageFile: update.packageFileName,
packagesToInstall: packagesToInstall,
},
`performing updates`
);
try {
const result = await exec(execCommands, execOptions);
logger.trace({ stdout: result.stdout }, `hermit command stdout`);
} catch (e) {
logger.warn({ err: e }, `error updating hermit package`);
throw new UpdateHermitError(
fromPackages,
packagesToInstall,
e.stderr,
e.stdout
);
}
}
export class UpdateHermitError extends Error {
stdout: string;
stderr: string;
from: string;
to: string;
constructor(from: string, to: string, stderr: string, stdout = '') {
super();
this.stdout = stdout;
this.stderr = stderr;
this.from = from;
this.to = to;
}
}
import minimatch from 'minimatch';
import { regEx } from '../../../util/regex';
import { defaultConfig } from './default-config';
describe('modules/manager/hermit/default-config', () => {
describe('excludeCommitPaths', () => {
function miniMatches(target: string, patterns: string[]): boolean {
return patterns.some((patt: string) => {
return minimatch(target, patt, { dot: true });
});
}
test.each`
path | expected
${'bin/hermit'} | ${true}
${'gradle/bin/hermit'} | ${true}
${'nested/module/bin/hermit'} | ${true}
${'nested/testbin/hermit'} | ${false}
${'other'} | ${false}
${'nested/other'} | ${false}
${'nested/module/other'} | ${false}
`('minimatches("$path") === $expected', ({ path, expected }) => {
expect(miniMatches(path, defaultConfig.excludeCommitPaths)).toBe(
expected
);
});
});
describe('fileMatch', () => {
function regexMatches(target: string, patterns: string[]): boolean {
return patterns.some((patt: string) => {
const re = regEx(patt);
return re.test(target);
});
}
test.each`
path | expected
${'bin/hermit'} | ${true}
${'gradle/bin/hermit'} | ${true}
${'nested/module/bin/hermit'} | ${true}
${'nested/testbin/hermit'} | ${false}
${'other'} | ${false}
${'nested/other'} | ${false}
${'nested/module/other'} | ${false}
`('regexMatches("$path") === $expected', ({ path, expected }) => {
expect(regexMatches(path, defaultConfig.fileMatch)).toBe(expected);
});
});
});
export const defaultConfig = {
fileMatch: ['(^|/)bin/hermit$'],
// bin/hermit will be changed to trigger artifact update
// but it doesn't need to be committed
excludeCommitPaths: ['**/bin/hermit'],
};
import { mockedFunction } from '../../../../test/util';
import { readLocalDirectory } from '../../../util/fs';
import { HermitDatasource } from '../../datasource/hermit';
import { extractPackageFile } from './extract';
jest.mock('../../../util/fs');
const readdirMock = mockedFunction(readLocalDirectory);
describe('modules/manager/hermit/extract', () => {
describe('extractPackageFile', () => {
it('should list packages on command success', async () => {
const ret = [
'.go-1.17.9.pkg',
'go',
'.golangci-lint-1.40.0.pkg',
'golangci-lint',
'.jq@stable.pkg',
'jq',
'.somepackage-invalid-version.pkg',
];
readdirMock.mockResolvedValue(ret);
const rootPackages = await extractPackageFile('', 'bin/hermit');
expect(rootPackages).toStrictEqual({
deps: [
{
datasource: HermitDatasource.id,
depName: 'go',
currentValue: `1.17.9`,
},
{
datasource: HermitDatasource.id,
depName: 'golangci-lint',
currentValue: `1.40.0`,
},
{
datasource: HermitDatasource.id,
depName: 'jq',
currentValue: `@stable`,
},
],
});
const nestedRet = [
'.gradle-7.4.2.pkg',
'go',
'.openjdk-11.0.11_9-zulu11.48.21.pkg',
'java',
'.maven@3.8.pkg',
'maven',
];
readdirMock.mockResolvedValue(nestedRet);
const nestedPackages = await extractPackageFile('', 'nested/bin/hermit');
expect(nestedPackages).toStrictEqual({
deps: [
{
datasource: HermitDatasource.id,
depName: 'gradle',
currentValue: '7.4.2',
},
{
datasource: HermitDatasource.id,
depName: 'openjdk',
currentValue: `11.0.11_9-zulu11.48.21`,
},
{
datasource: HermitDatasource.id,
depName: 'maven',
currentValue: '@3.8',
},
],
});
});
it('should throw error on execution failure', async () => {
const msg = 'error reading directory';
readdirMock.mockRejectedValue(new Error(msg));
expect(await extractPackageFile('', 'bin/hermit')).toBeNull();
});
});
});
import minimatch from 'minimatch';
import upath from 'upath';
import { logger } from '../../../logger';
import { readLocalDirectory } from '../../../util/fs';
import { regEx } from '../../../util/regex';
import { HermitDatasource } from '../../datasource/hermit';
import type { PackageDependency, PackageFile } from '../types';
import type { HermitListItem } from './types';
const pkgReferenceRegex = regEx(`(?<packageName>.*?)-(?<version>[0-9]{1}.*)`);
/**
* extractPackageFile scans the folder of the package files
* and looking for .{packageName}-{version}.pkg
*/
export async function extractPackageFile(
content: string,
filename: string
): Promise<PackageFile | null> {
logger.trace('hermit.extractPackageFile()');
const dependencies = [] as PackageDependency[];
const packages = await listHermitPackages(filename);
if (!packages?.length) {
return null;
}
for (const p of packages) {
// version of a hermit package is either a Version or a Channel
// Channel will prepend with @ to distinguish from normal version
const version = p.Version === '' ? `@${p.Channel}` : p.Version;
const dep: PackageDependency = {
datasource: HermitDatasource.id,
depName: p.Name,
currentValue: version,
};
dependencies.push(dep);
}
return { deps: dependencies };
}
/**
* listHermitPackages will fetch all installed packages from the bin folder
*/
async function listHermitPackages(
hermitFile: string
): Promise<HermitListItem[] | null> {
logger.trace('hermit.listHermitPackages()');
const hermitFolder = upath.dirname(hermitFile);
let files: string[] = [];
try {
files = await readLocalDirectory(hermitFolder);
} catch (e) {
logger.debug(
{ hermitFolder, err: e },
'error listing hermit package references'
);
return null;
}
logger.trace({ files, hermitFolder }, 'files for hermit package list');
const out = [] as HermitListItem[];
for (const f of files) {
if (!minimatch(f, '.*.pkg')) {
continue;
}
const fileName = f
.replace(`${hermitFolder}/`, '')
.substring(1)
.replace(/\.pkg$/, '');
const channelParts = fileName.split('@');
if (channelParts.length > 1) {
out.push({
Name: channelParts[0],
Channel: channelParts[1],
Version: '',
});
}
const groups = pkgReferenceRegex.exec(fileName)?.groups;
if (!groups) {
logger.debug(
{ fileName },
'invalid hermit package reference file name found'
);
continue;
}
out.push({
Name: groups.packageName,
Version: groups.version,
Channel: '',
});
}
return out;
}
import { HermitDatasource } from '../../datasource/hermit';
import { id as versionId } from '../../versioning/hermit';
import { defaultConfig as partialDefaultConfig } from './default-config';
export { updateArtifacts } from './artifacts';
export { extractPackageFile } from './extract';
export { updateDependency } from './update';
export const defaultConfig = {
fileMatch: partialDefaultConfig.fileMatch,
excludeCommitPaths: partialDefaultConfig.excludeCommitPaths,
versioning: versionId,
};
export const supportedDatasources = [HermitDatasource.id];
**_Hermit package installation token_**
When upgrading private packages through, Hermit manager will uses one of the following two tokens to download private packages.
```
HERMIT_GITHUB_TOKEN
GITHUB_TOKEN
```
These environment variable could be passed on via setting it in `customEnvironmentVariables`.
**_Nested Hermit setup_**
Nested Hermit setup in a single repository is also supported. e.g.
```
├bin
├─hermit
├─(other files)
├nested
├─bin
├──hermit
├──(other files)
```
export interface HermitListItem {
Name: string;
Version: string;
Channel: string;
}
export interface UpdateHermitResult {
from: string;
to: string;
newContent: string;
}
export interface ReadContentResult {
isSymlink?: boolean;
contents: string;
isExecutable?: boolean;
}
import { updateDependency } from '.';
describe('modules/manager/hermit/update', () => {
describe('updateDependency', () => {
it('should append a new marking line at the end to trigger the artifact update', () => {
const fileContent = `#!/bin/bash
#some hermit content
`;
const ret = updateDependency({ fileContent, upgrade: {} });
expect(ret).toBe(`${fileContent}\n#hermit updated`);
});
it('should not update again if the new line has been appended', () => {
const fileContent = `#!/bin/bash
#some hermit content
#hermit updated`;
const ret = updateDependency({ fileContent, upgrade: {} });
expect(ret).toBe(`${fileContent}`);
});
});
});
import { logger } from '../../../logger';
import type { UpdateDependencyConfig } from '../types';
const updateLine = '#hermit updated';
/**
* updateDependency appends a comment line once.
* This is only for the purpose of triggering the artifact update
* Hermit doesn't have a package file to update like other package managers.
*/
export function updateDependency({
fileContent,
upgrade,
}: UpdateDependencyConfig): string | null {
logger.trace({ upgrade }, `hermit.updateDependency()`);
if (!fileContent.endsWith(updateLine)) {
logger.debug(`append update line to the fileContent if it hasn't been`);
return `${fileContent}\n${updateLine}`;
}
return fileContent;
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment