From 974fa7b2a72f3ffa04e0811b7fd20dcbb232555f Mon Sep 17 00:00:00 2001 From: Gabriel-Ladzaretti <97394622+Gabriel-Ladzaretti@users.noreply.github.com> Date: Sat, 9 Apr 2022 14:38:06 +0300 Subject: [PATCH] refactor(versioning/distro-info): for future use in debian and ubuntu versioning (#14880) * refactor(tools/distro-info): for later use in ubuntu and debian versioning distro-info refactor * refactor(tools/distro-info): for later use in ubuntu and debian versioning styling * refactor(tools/distro-info): for later use in ubuntu and debian versioning distro-info refactor * refactor(tools/distro-info): for later use in ubuntu and debian versioning styling * refactor(tools/distro-info): for later use in ubuntu and debian versioning use https://debian.pages.debian.net/distro-info-data/ubuntu.csv as distro info data * refactor(tools/distro-info): for later use in ubuntu and debian versioning cr changes and migrated from mjs to ts * refactor(tools/distro-info): for later use in ubuntu and debian versioning added a crude way to prevent writing corrupted data * refactor(tools/distro-info): for later use in ubuntu and debian versioning lint fixes * refactor(tools/distro-info): for later use in ubuntu and debian versioning cr changes and migrated from mjs to ts * refactor(tools/distro-info): for later use in ubuntu and debian versioning cr changes * refactor(tools/distro-info): for later use in ubuntu and debian versioning dynamically check if stable/released. 22.04 release for example * refactor(tools/distro-info): for later use in ubuntu and debian versioning fixed JSON order * refactor(tools/distro-info): for later use in ubuntu and debian versioning removed ts.ignore * refactor(tools/distro-info): for later use in ubuntu and debian versioning cr changes * refactor(tools/distro-info): for later use in ubuntu and debian versioning cr changes Co-authored-by: Michael Kriese <michael.kriese@visualon.de> --- data/ubuntu-distro-info.json | 303 +++++++++++++++--- lib/modules/versioning/distro.ts | 64 ++++ lib/modules/versioning/ubuntu/distribution.ts | 40 --- lib/modules/versioning/ubuntu/index.ts | 40 +-- tools/distro-json-generate.mjs | 102 +++--- tools/generate-imports.mjs | 2 +- 6 files changed, 406 insertions(+), 145 deletions(-) create mode 100644 lib/modules/versioning/distro.ts delete mode 100644 lib/modules/versioning/ubuntu/distribution.ts diff --git a/data/ubuntu-distro-info.json b/data/ubuntu-distro-info.json index 75d602ec51..d5db78682c 100644 --- a/data/ubuntu-distro-info.json +++ b/data/ubuntu-distro-info.json @@ -1,38 +1,269 @@ { - "4.10": "warty", - "5.04": "hoary", - "5.10": "breezy", - "6.06": "dapper", - "6.10": "edgy", - "7.04": "feisty", - "7.10": "gutsy", - "8.04": "hardy", - "8.10": "intrepid", - "9.04": "jaunty", - "9.10": "karmic", - "10.04": "lucid", - "10.10": "maverick", - "11.04": "natty", - "11.10": "oneiric", - "12.04": "precise", - "12.10": "quantal", - "13.04": "raring", - "13.10": "saucy", - "14.04": "trusty", - "14.10": "utopic", - "15.04": "vivid", - "15.10": "wily", - "16.04": "xenial", - "16.10": "yakkety", - "17.04": "zesty", - "17.10": "artful", - "18.04": "bionic", - "18.10": "cosmic", - "19.04": "disco", - "19.10": "eoan", - "20.04": "focal", - "20.10": "groovy", - "21.04": "hirsute", - "21.10": "impish", - "22.04": "jammy" + "v4.10": { + "codename": "Warty Warthog", + "series": "warty", + "created": "2004-03-05", + "release": "2004-10-20", + "eol": "2006-04-30" + }, + "v5.04": { + "codename": "Hoary Hedgehog", + "series": "hoary", + "created": "2004-10-20", + "release": "2005-04-08", + "eol": "2006-10-31" + }, + "v5.10": { + "codename": "Breezy Badger", + "series": "breezy", + "created": "2005-04-08", + "release": "2005-10-12", + "eol": "2007-04-13" + }, + "v6.06": { + "codename": "Dapper Drake", + "series": "dapper", + "created": "2005-10-12", + "release": "2006-06-01", + "eol": "2009-07-14", + "eol_server": "2011-06-01" + }, + "v6.10": { + "codename": "Edgy Eft", + "series": "edgy", + "created": "2006-06-01", + "release": "2006-10-26", + "eol": "2008-04-26" + }, + "v7.04": { + "codename": "Feisty Fawn", + "series": "feisty", + "created": "2006-10-26", + "release": "2007-04-19", + "eol": "2008-10-19" + }, + "v7.10": { + "codename": "Gutsy Gibbon", + "series": "gutsy", + "created": "2007-04-19", + "release": "2007-10-18", + "eol": "2009-04-18" + }, + "v8.04": { + "codename": "Hardy Heron", + "series": "hardy", + "created": "2007-10-18", + "release": "2008-04-24", + "eol": "2011-05-12", + "eol_server": "2013-05-09" + }, + "v8.10": { + "codename": "Intrepid Ibex", + "series": "intrepid", + "created": "2008-04-24", + "release": "2008-10-30", + "eol": "2010-04-30" + }, + "v9.04": { + "codename": "Jaunty Jackalope", + "series": "jaunty", + "created": "2008-10-30", + "release": "2009-04-23", + "eol": "2010-10-23" + }, + "v9.10": { + "codename": "Karmic Koala", + "series": "karmic", + "created": "2009-04-23", + "release": "2009-10-29", + "eol": "2011-04-30" + }, + "v10.04": { + "codename": "Lucid Lynx", + "series": "lucid", + "created": "2009-10-29", + "release": "2010-04-29", + "eol": "2013-05-09", + "eol_server": "2015-04-30" + }, + "v10.10": { + "codename": "Maverick Meerkat", + "series": "maverick", + "created": "2010-04-29", + "release": "2010-10-10", + "eol": "2012-04-10" + }, + "v11.04": { + "codename": "Natty Narwhal", + "series": "natty", + "created": "2010-10-10", + "release": "2011-04-28", + "eol": "2012-10-28" + }, + "v11.10": { + "codename": "Oneiric Ocelot", + "series": "oneiric", + "created": "2011-04-28", + "release": "2011-10-13", + "eol": "2013-05-09" + }, + "v12.04": { + "codename": "Precise Pangolin", + "series": "precise", + "created": "2011-10-13", + "release": "2012-04-26", + "eol": "2017-04-28", + "eol_server": "2017-04-28", + "eol_esm": "2019-04-28" + }, + "v12.10": { + "codename": "Quantal Quetzal", + "series": "quantal", + "created": "2012-04-26", + "release": "2012-10-18", + "eol": "2014-05-16" + }, + "v13.04": { + "codename": "Raring Ringtail", + "series": "raring", + "created": "2012-10-18", + "release": "2013-04-25", + "eol": "2014-01-27" + }, + "v13.10": { + "codename": "Saucy Salamander", + "series": "saucy", + "created": "2013-04-25", + "release": "2013-10-17", + "eol": "2014-07-17" + }, + "v14.04": { + "codename": "Trusty Tahr", + "series": "trusty", + "created": "2013-10-17", + "release": "2014-04-17", + "eol": "2019-04-25", + "eol_server": "2019-04-25", + "eol_esm": "2024-04-25" + }, + "v14.10": { + "codename": "Utopic Unicorn", + "series": "utopic", + "created": "2014-04-17", + "release": "2014-10-23", + "eol": "2015-07-23" + }, + "v15.04": { + "codename": "Vivid Vervet", + "series": "vivid", + "created": "2014-10-23", + "release": "2015-04-23", + "eol": "2016-02-04" + }, + "v15.10": { + "codename": "Wily Werewolf", + "series": "wily", + "created": "2015-04-23", + "release": "2015-10-22", + "eol": "2016-07-28" + }, + "v16.04": { + "codename": "Xenial Xerus", + "series": "xenial", + "created": "2015-10-22", + "release": "2016-04-21", + "eol": "2021-04-21", + "eol_server": "2021-04-21", + "eol_esm": "2026-04-23" + }, + "v16.10": { + "codename": "Yakkety Yak", + "series": "yakkety", + "created": "2016-04-21", + "release": "2016-10-13", + "eol": "2017-07-20" + }, + "v17.04": { + "codename": "Zesty Zapus", + "series": "zesty", + "created": "2016-10-13", + "release": "2017-04-13", + "eol": "2018-01-13" + }, + "v17.10": { + "codename": "Artful Aardvark", + "series": "artful", + "created": "2017-04-13", + "release": "2017-10-19", + "eol": "2018-07-19" + }, + "v18.04": { + "codename": "Bionic Beaver", + "series": "bionic", + "created": "2017-10-19", + "release": "2018-04-26", + "eol": "2023-04-26", + "eol_server": "2023-04-26", + "eol_esm": "2028-04-26" + }, + "v18.10": { + "codename": "Cosmic Cuttlefish", + "series": "cosmic", + "created": "2018-04-26", + "release": "2018-10-18", + "eol": "2019-07-18" + }, + "v19.04": { + "codename": "Disco Dingo", + "series": "disco", + "created": "2018-10-18", + "release": "2019-04-18", + "eol": "2020-01-23" + }, + "v19.10": { + "codename": "Eoan Ermine", + "series": "eoan", + "created": "2019-04-18", + "release": "2019-10-17", + "eol": "2020-07-17" + }, + "v20.04": { + "codename": "Focal Fossa", + "series": "focal", + "created": "2019-10-17", + "release": "2020-04-23", + "eol": "2025-04-23", + "eol_server": "2025-04-23", + "eol_esm": "2030-04-23" + }, + "v20.10": { + "codename": "Groovy Gorilla", + "series": "groovy", + "created": "2020-04-23", + "release": "2020-10-22", + "eol": "2021-07-22" + }, + "v21.04": { + "codename": "Hirsute Hippo", + "series": "hirsute", + "created": "2020-10-22", + "release": "2021-04-22", + "eol": "2022-01-20" + }, + "v21.10": { + "codename": "Impish Indri", + "series": "impish", + "created": "2021-04-22", + "release": "2021-10-14", + "eol": "2022-07-14" + }, + "v22.04": { + "codename": "Jammy Jellyfish", + "series": "jammy", + "created": "2021-10-14", + "release": "2022-04-21", + "eol": "2027-04-21", + "eol_server": "2027-04-21", + "eol_esm": "2032-04-21" + } } diff --git a/lib/modules/versioning/distro.ts b/lib/modules/versioning/distro.ts new file mode 100644 index 0000000000..075c2adea6 --- /dev/null +++ b/lib/modules/versioning/distro.ts @@ -0,0 +1,64 @@ +import dataFiles, { DataFile } from '../../data-files.generated'; + +interface DistroSchedule { + codename: string; + series: string; + created: string; + release: string; + eol: string; + eol_server?: string; + eol_esm?: string; + eol_lts?: string; + eol_elts?: string; +} + +type DistroDataFile = 'data/ubuntu-distro-info.json'; + +export type DistroInfoRecord = Record<string, DistroSchedule>; + +export type DistroInfoRecordWithVersion = { version: string } & DistroSchedule; + +export class DistroInfo { + private readonly _codenameToVersion = new Map< + string, + DistroInfoRecordWithVersion + >(); + private readonly _distroInfo: DistroInfoRecord; + + constructor(distroJsonKey: DistroDataFile) { + this._distroInfo = JSON.parse( + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + dataFiles.get(distroJsonKey as DataFile)!.replace(/v([\d.]+)\b/gm, '$1') + ); + + for (const version of Object.keys(this._distroInfo)) { + const schedule = this._distroInfo[version]; + this._codenameToVersion.set(schedule.series, { version, ...schedule }); + } + } + + public isCodename(input: string): boolean { + return this._codenameToVersion.has(input); + } + + public getVersionByCodename(input: string): string { + const schedule = this._codenameToVersion.get(input); + if (schedule) { + return schedule.version; + } + return input; + } + + public getCodenameByVersion(input: string): string { + const di = this._distroInfo[input]; + if (di) { + return di.series; + } + // istanbul ignore next + return input; + } + + public getSchedule(input: string): DistroSchedule { + return this._distroInfo[input]; + } +} diff --git a/lib/modules/versioning/ubuntu/distribution.ts b/lib/modules/versioning/ubuntu/distribution.ts deleted file mode 100644 index 999e3c382c..0000000000 --- a/lib/modules/versioning/ubuntu/distribution.ts +++ /dev/null @@ -1,40 +0,0 @@ -import dataFiles from '../../../data-files.generated'; - -export type UbuntuDistroInfo = Record<string, string>; - -// Data file generated with: -// distro-json-generate.mjs -const ubuntuJsonKey = 'data/ubuntu-distro-info.json'; - -const ubuntuDistroInfo: UbuntuDistroInfo = JSON.parse( - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - dataFiles.get(ubuntuJsonKey)! -); - -const codenameToVersion = new Map<string, string>(); - -for (const version of Object.keys(ubuntuDistroInfo)) { - const codename = ubuntuDistroInfo[version]; - codenameToVersion.set(codename, version); -} - -export function isCodename(input: string): boolean { - return codenameToVersion.has(input); -} - -export function getVersionByCodename(input: string): string { - const ver = codenameToVersion.get(input); - if (ver) { - return ver; - } - return input; -} - -export function getCodenameByVersion(input: string): string { - const codename = ubuntuDistroInfo[input]; - if (codename) { - return codename; - } - // istanbul ignore next - return input; -} diff --git a/lib/modules/versioning/ubuntu/index.ts b/lib/modules/versioning/ubuntu/index.ts index 7b57ba9e1c..40c4f2e4bb 100644 --- a/lib/modules/versioning/ubuntu/index.ts +++ b/lib/modules/versioning/ubuntu/index.ts @@ -1,18 +1,17 @@ +import { DateTime } from 'luxon'; import { regEx } from '../../../util/regex'; +import { DistroInfo } from '../distro'; import type { NewValueConfig, VersioningApi } from '../types'; -import { - getCodenameByVersion, - getVersionByCodename, - isCodename, -} from './distribution'; export const id = 'ubuntu'; export const displayName = 'Ubuntu'; -export const urls = ['https://changelogs.ubuntu.com/meta-release']; +export const urls = [ + 'https://changelogs.ubuntu.com/meta-release', + 'https://debian.pages.debian.net/distro-info-data/ubuntu.csv', +]; export const supportsRanges = false; -// #12509 -const temporarilyUnstable = ['22.04']; +const di = new DistroInfo('data/ubuntu-distro-info.json'); // validation @@ -22,7 +21,7 @@ function isValid(input: string): boolean { regEx(/^(0[4-5]|[6-9]|[1-9][0-9])\.[0-9][0-9](\.[0-9]{1,2})?$/).test( input )) || - isCodename(input) + di.isCodename(input) ); } @@ -39,20 +38,23 @@ function isSingleVersion(version: string): boolean { } function isStable(version: string): boolean { - const ver = getVersionByCodename(version); + const ver = di.getVersionByCodename(version); if (!isValid(ver)) { return false; } - if (temporarilyUnstable.includes(ver)) { + + const schedule = di.getSchedule(ver); + if (schedule && DateTime.fromISO(schedule.release) > DateTime.now()) { return false; } + return regEx(/^\d?[02468]\.04/).test(ver); } // digestion of version function getMajor(version: string): null | number { - const ver = getVersionByCodename(version); + const ver = di.getVersionByCodename(version); if (isValid(ver)) { const [major] = ver.split('.'); return parseInt(major, 10); @@ -61,7 +63,7 @@ function getMajor(version: string): null | number { } function getMinor(version: string): null | number { - const ver = getVersionByCodename(version); + const ver = di.getVersionByCodename(version); if (isValid(ver)) { const [, minor] = ver.split('.'); return parseInt(minor, 10); @@ -70,7 +72,7 @@ function getMinor(version: string): null | number { } function getPatch(version: string): null | number { - const ver = getVersionByCodename(version); + const ver = di.getVersionByCodename(version); if (isValid(ver)) { const [, , patch] = ver.split('.'); return patch ? parseInt(patch, 10) : null; @@ -81,8 +83,8 @@ function getPatch(version: string): null | number { // comparison function equals(version: string, other: string): boolean { - const ver = getVersionByCodename(version); - const otherVer = getVersionByCodename(other); + const ver = di.getVersionByCodename(version); + const otherVer = di.getVersionByCodename(other); return isVersion(ver) && isVersion(otherVer) && ver === otherVer; } @@ -130,10 +132,10 @@ function getNewValue({ currentVersion, newVersion, }: NewValueConfig): string { - if (isCodename(currentValue)) { - return getCodenameByVersion(newVersion); + if (di.isCodename(currentValue)) { + return di.getCodenameByVersion(newVersion); } - return getVersionByCodename(newVersion); + return di.getVersionByCodename(newVersion); } function sortVersions(version: string, other: string): number { diff --git a/tools/distro-json-generate.mjs b/tools/distro-json-generate.mjs index 5ee96ccbce..4587351f39 100644 --- a/tools/distro-json-generate.mjs +++ b/tools/distro-json-generate.mjs @@ -1,73 +1,77 @@ import fs from 'fs-extra'; +import got from 'got'; import shell from 'shelljs'; -shell.echo(`Verifying required packages...`); +const url = 'https://debian.pages.debian.net/distro-info-data/ubuntu.csv'; -if (!shell.which(`distro-info`)) { - shell.echo('This script requires distro-info, exiting...'); - shell.exit(2); -} +/** + * Converts valid CSV string into JSON. + * @param {string} raw CSV string + * @returns {string} JSON representation of input CSV + */ +function csvToJson(raw) { + const lines = raw.split(/\r?\n/); -if (!shell.which(`sed`)) { - shell.echo('This script requires sed, exiting...'); - shell.exit(2); -} + /** @type {Record<string, any>} */ + const res = {}; + const headers = lines[0].split(','); -shell.echo(`OK`); + // drop headers + lines.shift(); -const ubuntuDistroInfo = shell.exec( - `ubuntu-distro-info --all -f | sed -r 's/Ubuntu|"|LTS |Debian //g; s/([0-9]+.[0-9]+) /\\1 /; s/.*/\\L&/; s/( [a-z]*) [a-z]*/\\1/g; s/^[ \\t]*//'`, - { silent: true } -); + // drop "version" header + headers.shift(); -/** - * @param {string} str - * @returns {{}} - */ -function objectify(str) { - let obj = {}; + for (const l of lines) { + if (!l) { + continue; + } - for (const line of str.split(/\r?\n/)) { - let [ver, codename] = line.split(' '); - // eslint-disable-next-line - // @ts-ignore - obj[ver] = codename; - } + /** @type {Record<string, any>} */ + const obj = {}; + const line = l.split(','); + let ver = line?.shift()?.replace(/LTS|\s/g, ''); + + for (const [i, h] of headers.entries()) { + obj[h.replace('-', '_')] = line[i]; + } - return obj; + if (ver) { + // Debian related, example codename "hamm" version is 2.0 + // change 2.0 -> 2 + ver = ver.endsWith('.0') ? ver.replace('.0', '') : ver; + res[`v${ver}`] = obj; + } + } + return JSON.stringify(res, undefined, 2); } /** - * @param {string} file - * @param {string} newData + * Update given file with new provided data. + * @param {string} file Path to a data file + * @param {string} newData New data to be written */ async function updateJsonFile(file, newData) { - let oldData; - - try { - oldData = fs.existsSync(file) ? await fs.readFile(file, 'utf8') : null; - // Eliminate formatting - oldData = oldData?.replace(/\s/g, '') ?? null; - } catch (e) { - shell.echo(e.toString()); - shell.exit(1); - } - - const parsedData = JSON.stringify(objectify(newData), undefined, 2); - - if (oldData === parsedData) { - shell.echo(`${file} is up to date.`); - return; - } - try { shell.echo(`Updating ${file}`); - await fs.writeFile(file, parsedData); + await fs.writeFile(file, newData); } catch (e) { shell.echo(e.toString()); shell.exit(1); } } +/** + * Fetch CSV and update data file. + * @param {string} url Url to CSV + * @param {string} file File path to update + */ +async function update(url, file) { + const res = await got(url); + const csv = res.body; + const json = csvToJson(csv); + await updateJsonFile(file, json); +} + // eslint-disable-next-line @typescript-eslint/no-floating-promises -updateJsonFile(`../data/ubuntu-distro-info.json`, ubuntuDistroInfo.toString()); +update(url, `./data/ubuntu-distro-info.json`); diff --git a/tools/generate-imports.mjs b/tools/generate-imports.mjs index bdce7aa37c..69cccfc27f 100644 --- a/tools/generate-imports.mjs +++ b/tools/generate-imports.mjs @@ -131,7 +131,7 @@ async function generateData() { await updateFile( `lib/data-files.generated.ts`, [ - `type DataFile =\n${importDataFileType};`, + `export type DataFile =\n${importDataFileType};`, contentMapDecl, contentMapAssignments.join('\n'), `export default data;\n`, -- GitLab