From 091ccfdbcd28a4829ba80fe3a2d760f743f10e6f Mon Sep 17 00:00:00 2001
From: chris48s <chris48s@users.noreply.github.com>
Date: Thu, 23 Feb 2023 17:18:39 +0000
Subject: [PATCH] migrate token pooling to postgres (#8922)
* add ability to store token pool in Postgres DB
* update transitive ansi-regex dependencies
---
.github/actions/integration-tests/action.yml | 8 +
.github/workflows/test-integration-17.yml | 13 +
.github/workflows/test-integration.yml | 13 +
.gitignore | 3 +
config/custom-environment-variables.yml | 1 +
core/server/server.js | 1 +
.../redis-token-persistence.integration.js | 1 -
.../sql-token-persistence.integration.js | 103 ++++
core/token-pooling/sql-token-persistence.js | 55 ++
migrations/1676731511125_initialize.cjs | 14 +
package-lock.json | 523 +++++++++++++++---
package.json | 5 +-
scripts/migrate-token-pool.js | 42 ++
scripts/write-migrations-config.js | 11 +
server.js | 6 +
services/github/github-constellation.js | 15 +-
16 files changed, 743 insertions(+), 71 deletions(-)
create mode 100644 core/token-pooling/sql-token-persistence.integration.js
create mode 100644 core/token-pooling/sql-token-persistence.js
create mode 100644 migrations/1676731511125_initialize.cjs
create mode 100644 scripts/migrate-token-pool.js
create mode 100644 scripts/write-migrations-config.js
diff --git a/.github/actions/integration-tests/action.yml b/.github/actions/integration-tests/action.yml
index d5068279f5..9214aa6650 100644
--- a/.github/actions/integration-tests/action.yml
+++ b/.github/actions/integration-tests/action.yml
@@ -7,11 +7,19 @@ inputs:
runs:
using: 'composite'
steps:
+ - name: Migrate DB
+ if: always()
+ run: npm run migrate up
+ env:
+ POSTGRES_URL: postgresql://postgres:postgres@localhost:5432/ci_test
+ shell: bash
+
- name: Integration Tests
if: always()
run: npm run test:integration -- --reporter json --reporter-option 'output=reports/integration-tests.json'
env:
GH_TOKEN: '${{ inputs.github-token }}'
+ POSTGRES_URL: postgresql://postgres:postgres@localhost:5432/ci_test
shell: bash
- name: Write Markdown Summary
diff --git a/.github/workflows/test-integration-17.yml b/.github/workflows/test-integration-17.yml
index 2ae38c7de8..d9e6c21fdd 100644
--- a/.github/workflows/test-integration-17.yml
+++ b/.github/workflows/test-integration-17.yml
@@ -23,6 +23,19 @@ jobs:
--health-retries 5
ports:
- 6379:6379
+ postgres:
+ image: postgres
+ env:
+ POSTGRES_USER: postgres
+ POSTGRES_PASSWORD: postgres
+ POSTGRES_DB: ci_test
+ options: >-
+ --health-cmd pg_isready
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 5
+ ports:
+ - 5432:5432
steps:
- name: Checkout
diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml
index 1a5d98cfbd..ffcec2613c 100644
--- a/.github/workflows/test-integration.yml
+++ b/.github/workflows/test-integration.yml
@@ -23,6 +23,19 @@ jobs:
--health-retries 5
ports:
- 6379:6379
+ postgres:
+ image: postgres
+ env:
+ POSTGRES_USER: postgres
+ POSTGRES_PASSWORD: postgres
+ POSTGRES_DB: ci_test
+ options: >-
+ --health-cmd pg_isready
+ --health-interval 10s
+ --health-timeout 5s
+ --health-retries 5
+ ports:
+ - 5432:5432
steps:
- name: Checkout
diff --git a/.gitignore b/.gitignore
index 771e46a75d..a645da247b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -117,3 +117,6 @@ service-definitions.yml
# Flamebearer
flamegraph.html
+
+# config file for node-pg-migrate
+migrations-config.json
diff --git a/config/custom-environment-variables.yml b/config/custom-environment-variables.yml
index d4c64c7bb4..da8e9ec860 100644
--- a/config/custom-environment-variables.yml
+++ b/config/custom-environment-variables.yml
@@ -94,6 +94,7 @@ private:
obs_user: 'OBS_USER'
obs_pass: 'OBS_PASS'
redis_url: 'REDIS_URL'
+ postgres_url: 'POSTGRES_URL'
sentry_dsn: 'SENTRY_DSN'
sl_insight_userUuid: 'SL_INSIGHT_USER_UUID'
sl_insight_apiToken: 'SL_INSIGHT_API_TOKEN'
diff --git a/core/server/server.js b/core/server/server.js
index fcc6de4f2a..a1c0daa315 100644
--- a/core/server/server.js
+++ b/core/server/server.js
@@ -183,6 +183,7 @@ const privateConfigSchema = Joi.object({
obs_user: Joi.string(),
obs_pass: Joi.string(),
redis_url: Joi.string().uri({ scheme: ['redis', 'rediss'] }),
+ postgres_url: Joi.string().uri({ scheme: 'postgresql' }),
sentry_dsn: Joi.string(),
sl_insight_userUuid: Joi.string(),
sl_insight_apiToken: Joi.string(),
diff --git a/core/token-pooling/redis-token-persistence.integration.js b/core/token-pooling/redis-token-persistence.integration.js
index ca1d65cd9b..b11d9304bb 100644
--- a/core/token-pooling/redis-token-persistence.integration.js
+++ b/core/token-pooling/redis-token-persistence.integration.js
@@ -84,7 +84,6 @@ describe('Redis token persistence', function () {
const toRemove = expected.pop()
await persistence.initialize()
-
await persistence.noteTokenRemoved(toRemove)
const savedTokens = await redis.smembers(key)
diff --git a/core/token-pooling/sql-token-persistence.integration.js b/core/token-pooling/sql-token-persistence.integration.js
new file mode 100644
index 0000000000..862a898b5e
--- /dev/null
+++ b/core/token-pooling/sql-token-persistence.integration.js
@@ -0,0 +1,103 @@
+import pg from 'pg'
+import { expect } from 'chai'
+import configModule from 'config'
+import SqlTokenPersistence from './sql-token-persistence.js'
+
+const config = configModule.util.toObject()
+const postgresUrl = config?.private?.postgres_url
+const tableName = 'token_persistence_integration_test'
+
+describe('SQL token persistence', function () {
+ let pool
+ let persistence
+
+ before('Mock db connection and load app', async function () {
+ // Create a new pool with a connection limit of 1
+ pool = new pg.Pool({
+ connectionString: postgresUrl,
+
+ // Reuse the connection to make sure we always hit the same pg_temp schema
+ max: 1,
+
+ // Disable auto-disconnection of idle clients to make sure we always hit the same pg_temp schema
+ idleTimeoutMillis: 0,
+ })
+ persistence = new SqlTokenPersistence({
+ url: postgresUrl,
+ table: tableName,
+ })
+ })
+ after(async function () {
+ if (persistence) {
+ await persistence.stop()
+ persistence = undefined
+ }
+ })
+
+ beforeEach('Create temporary table', async function () {
+ await pool.query(
+ `CREATE TEMPORARY TABLE ${tableName} (LIKE github_user_tokens INCLUDING ALL);`
+ )
+ })
+ afterEach('Drop temporary table', async function () {
+ await pool.query(`DROP TABLE IF EXISTS pg_temp.${tableName};`)
+ })
+
+ context('when the key does not exist', function () {
+ it('does nothing', async function () {
+ const tokens = await persistence.initialize(pool)
+ expect(tokens).to.deep.equal([])
+ })
+ })
+
+ context('when the key exists', function () {
+ const initialTokens = ['a', 'b', 'c'].map(char => char.repeat(40))
+
+ beforeEach(async function () {
+ initialTokens.forEach(async token => {
+ await pool.query(
+ `INSERT INTO pg_temp.${tableName} (token) VALUES ($1::text);`,
+ [token]
+ )
+ })
+ })
+
+ it('loads the contents', async function () {
+ const tokens = await persistence.initialize(pool)
+ expect(tokens.sort()).to.deep.equal(initialTokens)
+ })
+
+ context('when tokens are added', function () {
+ it('saves the change', async function () {
+ const newToken = 'e'.repeat(40)
+ const expected = initialTokens.slice()
+ expected.push(newToken)
+
+ await persistence.initialize(pool)
+ await persistence.noteTokenAdded(newToken)
+
+ const result = await pool.query(
+ `SELECT token FROM pg_temp.${tableName};`
+ )
+ const savedTokens = result.rows.map(row => row.token)
+ expect(savedTokens.sort()).to.deep.equal(expected)
+ })
+ })
+
+ context('when tokens are removed', function () {
+ it('saves the change', async function () {
+ const expected = Array.from(initialTokens)
+ const toRemove = expected.pop()
+
+ await persistence.initialize(pool)
+ await persistence.noteTokenRemoved(toRemove)
+
+ const result = await pool.query(
+ `SELECT token FROM pg_temp.${tableName};`
+ )
+ const savedTokens = result.rows.map(row => row.token)
+ expect(savedTokens.sort()).to.deep.equal(expected)
+ })
+ })
+ })
+})
diff --git a/core/token-pooling/sql-token-persistence.js b/core/token-pooling/sql-token-persistence.js
new file mode 100644
index 0000000000..6f7eab6b73
--- /dev/null
+++ b/core/token-pooling/sql-token-persistence.js
@@ -0,0 +1,55 @@
+import pg from 'pg'
+import log from '../server/log.js'
+
+export default class SqlTokenPersistence {
+ constructor({ url, table }) {
+ this.url = url
+ this.table = table
+ this.noteTokenAdded = this.noteTokenAdded.bind(this)
+ this.noteTokenRemoved = this.noteTokenRemoved.bind(this)
+ }
+
+ async initialize(pool) {
+ if (pool) {
+ this.pool = pool
+ } else {
+ this.pool = new pg.Pool({ connectionString: this.url })
+ }
+ const result = await this.pool.query(`SELECT token FROM ${this.table};`)
+ return result.rows.map(row => row.token)
+ }
+
+ async stop() {
+ await this.pool.end()
+ }
+
+ async onTokenAdded(token) {
+ return await this.pool.query(
+ `INSERT INTO ${this.table} (token) VALUES ($1::text) ON CONFLICT (token) DO NOTHING;`,
+ [token]
+ )
+ }
+
+ async onTokenRemoved(token) {
+ return await this.pool.query(
+ `DELETE FROM ${this.table} WHERE token=$1::text;`,
+ [token]
+ )
+ }
+
+ async noteTokenAdded(token) {
+ try {
+ await this.onTokenAdded(token)
+ } catch (e) {
+ log.error(e)
+ }
+ }
+
+ async noteTokenRemoved(token) {
+ try {
+ await this.onTokenRemoved(token)
+ } catch (e) {
+ log.error(e)
+ }
+ }
+}
diff --git a/migrations/1676731511125_initialize.cjs b/migrations/1676731511125_initialize.cjs
new file mode 100644
index 0000000000..00834fe50f
--- /dev/null
+++ b/migrations/1676731511125_initialize.cjs
@@ -0,0 +1,14 @@
+/* eslint-disable camelcase */
+
+exports.shorthands = undefined
+
+exports.up = pgm => {
+ pgm.createTable('github_user_tokens', {
+ id: 'id',
+ token: { type: 'varchar(1000)', notNull: true, unique: true },
+ })
+}
+
+exports.down = pgm => {
+ pgm.dropTable('github_user_tokens')
+}
diff --git a/package-lock.json b/package-lock.json
index 5de26e652a..c9bf61731a 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -42,8 +42,10 @@
"lodash.groupby": "^4.6.0",
"lodash.times": "^4.3.2",
"node-env-flag": "^0.1.0",
+ "node-pg-migrate": "^6.2.2",
"parse-link-header": "^2.0.0",
"path-to-regexp": "^6.2.1",
+ "pg": "^8.9.0",
"pretty-bytes": "^6.1.0",
"priorityqueuejs": "^2.0.0",
"prom-client": "^14.1.1",
@@ -2852,9 +2854,9 @@
}
},
"node_modules/@gatsbyjs/webpack-hot-middleware/node_modules/ansi-regex": {
- "version": "5.0.0",
- "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
- "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+ "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
"dev": true,
"engines": {
"node": ">=8"
@@ -5683,8 +5685,7 @@
"node_modules/@types/node": {
"version": "16.7.10",
"resolved": "https://registry.npmjs.org/@types/node/-/node-16.7.10.tgz",
- "integrity": "sha512-S63Dlv4zIPb8x6MMTgDq5WWRJQe56iBEY0O3SOFA9JrRienkOVDXSXBjjJw6HTNQYSE2JI6GMCR6LVbIMHJVvA==",
- "dev": true
+ "integrity": "sha512-S63Dlv4zIPb8x6MMTgDq5WWRJQe56iBEY0O3SOFA9JrRienkOVDXSXBjjJw6HTNQYSE2JI6GMCR6LVbIMHJVvA=="
},
"node_modules/@types/node-fetch": {
"version": "2.6.2",
@@ -5722,6 +5723,16 @@
"integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==",
"dev": true
},
+ "node_modules/@types/pg": {
+ "version": "8.6.6",
+ "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.6.6.tgz",
+ "integrity": "sha512-O2xNmXebtwVekJDD+02udOncjVcMZQuTEQEMpKJ0ZRf5E7/9JJX3izhKUcUifBkyKpljyUM6BTgy2trmviKlpw==",
+ "dependencies": {
+ "@types/node": "*",
+ "pg-protocol": "*",
+ "pg-types": "^2.2.0"
+ }
+ },
"node_modules/@types/prop-types": {
"version": "15.7.1",
"resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.1.tgz",
@@ -6775,7 +6786,7 @@
"node_modules/ansi-regex": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
- "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=",
+ "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==",
"dev": true,
"engines": {
"node": ">=0.10.0"
@@ -7925,6 +7936,14 @@
"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==",
"dev": true
},
+ "node_modules/buffer-writer": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
+ "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==",
+ "engines": {
+ "node": ">=4"
+ }
+ },
"node_modules/builtin-modules": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz",
@@ -11684,7 +11703,6 @@
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
"integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
- "dev": true,
"engines": {
"node": ">=6"
}
@@ -16449,7 +16467,6 @@
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
- "dev": true,
"engines": {
"node": "6.* || 8.* || >= 10.*"
}
@@ -21517,6 +21534,110 @@
"node": ">=0.10.0"
}
},
+ "node_modules/node-pg-migrate": {
+ "version": "6.2.2",
+ "resolved": "https://registry.npmjs.org/node-pg-migrate/-/node-pg-migrate-6.2.2.tgz",
+ "integrity": "sha512-0WYLTXpWu2doeZhiwJUW/1u21OqAFU2CMQ8YZ8VBcJ0xrdqYAjtd8GGFe5A5DM4NJdIZsqJcLPDFqY0FQsmivw==",
+ "dependencies": {
+ "@types/pg": "^8.0.0",
+ "decamelize": "^5.0.0",
+ "mkdirp": "~1.0.0",
+ "yargs": "~17.3.0"
+ },
+ "bin": {
+ "node-pg-migrate": "bin/node-pg-migrate"
+ },
+ "engines": {
+ "node": ">=12.20.0"
+ },
+ "peerDependencies": {
+ "pg": ">=4.3.0 <9.0.0"
+ }
+ },
+ "node_modules/node-pg-migrate/node_modules/ansi-regex": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+ "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
+ "engines": {
+ "node": ">=8"
+ }
+ },
+ "node_modules/node-pg-migrate/node_modules/cliui": {
+ "version": "7.0.4",
+ "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
+ "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
+ "dependencies": {
+ "string-width": "^4.2.0",
+ "strip-ansi": "^6.0.0",
+ "wrap-ansi": "^7.0.0"
+ }
+ },
+ "node_modules/node-pg-migrate/node_modules/decamelize": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-5.0.1.tgz",
+ "integrity": "sha512-VfxadyCECXgQlkoEAjeghAr5gY3Hf+IKjKb+X8tGVDtveCjN+USwprd2q3QXBR9T1+x2DG0XZF5/w+7HAtSaXA==",
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/node-pg-migrate/node_modules/mkdirp": {
+ "version": "1.0.4",
+ "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
+ "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==",
+ "bin": {
+ "mkdirp": "bin/cmd.js"
+ },
+ "engines": {
+ "node": ">=10"
+ }
+ },
+ "node_modules/node-pg-migrate/node_modules/strip-ansi": {
+ "version": "6.0.1",
+ "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+ "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+ "dependencies": {
+ "ansi-regex": "^5.0.1"
+ },
+ "engines": {
+ "node": ">=8"
+ }
+ },
+ "node_modules/node-pg-migrate/node_modules/y18n": {
+ "version": "5.0.8",
+ "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
+ "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
+ "engines": {
+ "node": ">=10"
+ }
+ },
+ "node_modules/node-pg-migrate/node_modules/yargs": {
+ "version": "17.3.1",
+ "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.3.1.tgz",
+ "integrity": "sha512-WUANQeVgjLbNsEmGk20f+nlHgOqzRFpiGWVaBrYGYIGANIIu3lWjoyi0fNlFmJkvfhCZ6BXINe7/W2O2bV4iaA==",
+ "dependencies": {
+ "cliui": "^7.0.2",
+ "escalade": "^3.1.1",
+ "get-caller-file": "^2.0.5",
+ "require-directory": "^2.1.1",
+ "string-width": "^4.2.3",
+ "y18n": "^5.0.5",
+ "yargs-parser": "^21.0.0"
+ },
+ "engines": {
+ "node": ">=12"
+ }
+ },
+ "node_modules/node-pg-migrate/node_modules/yargs-parser": {
+ "version": "21.1.1",
+ "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
+ "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==",
+ "engines": {
+ "node": ">=12"
+ }
+ },
"node_modules/node-releases": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.6.tgz",
@@ -22432,6 +22553,11 @@
"semver": "bin/semver.js"
}
},
+ "node_modules/packet-reader": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz",
+ "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ=="
+ },
"node_modules/param-case": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz",
@@ -22762,6 +22888,80 @@
"integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=",
"dev": true
},
+ "node_modules/pg": {
+ "version": "8.9.0",
+ "resolved": "https://registry.npmjs.org/pg/-/pg-8.9.0.tgz",
+ "integrity": "sha512-ZJM+qkEbtOHRuXjmvBtOgNOXOtLSbxiMiUVMgE4rV6Zwocy03RicCVvDXgx8l4Biwo8/qORUnEqn2fdQzV7KCg==",
+ "dependencies": {
+ "buffer-writer": "2.0.0",
+ "packet-reader": "1.0.0",
+ "pg-connection-string": "^2.5.0",
+ "pg-pool": "^3.5.2",
+ "pg-protocol": "^1.6.0",
+ "pg-types": "^2.1.0",
+ "pgpass": "1.x"
+ },
+ "engines": {
+ "node": ">= 8.0.0"
+ },
+ "peerDependencies": {
+ "pg-native": ">=3.0.1"
+ },
+ "peerDependenciesMeta": {
+ "pg-native": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/pg-connection-string": {
+ "version": "2.5.0",
+ "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz",
+ "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ=="
+ },
+ "node_modules/pg-int8": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
+ "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
+ "engines": {
+ "node": ">=4.0.0"
+ }
+ },
+ "node_modules/pg-pool": {
+ "version": "3.5.2",
+ "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.5.2.tgz",
+ "integrity": "sha512-His3Fh17Z4eg7oANLob6ZvH8xIVen3phEZh2QuyrIl4dQSDVEabNducv6ysROKpDNPSD+12tONZVWfSgMvDD9w==",
+ "peerDependencies": {
+ "pg": ">=8.0"
+ }
+ },
+ "node_modules/pg-protocol": {
+ "version": "1.6.0",
+ "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz",
+ "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q=="
+ },
+ "node_modules/pg-types": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
+ "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
+ "dependencies": {
+ "pg-int8": "1.0.1",
+ "postgres-array": "~2.0.0",
+ "postgres-bytea": "~1.0.0",
+ "postgres-date": "~1.0.4",
+ "postgres-interval": "^1.1.0"
+ },
+ "engines": {
+ "node": ">=4"
+ }
+ },
+ "node_modules/pgpass": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
+ "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
+ "dependencies": {
+ "split2": "^4.1.0"
+ }
+ },
"node_modules/physical-cpu-count": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/physical-cpu-count/-/physical-cpu-count-2.0.0.tgz",
@@ -23627,6 +23827,41 @@
"integrity": "sha512-97DXOFbQJhk71ne5/Mt6cOu6yxsSfM0QGQyl0L25Gca4yGWEGJaig7l7gbCX623VqTBNGLRLaVUCnNkcedlRSQ==",
"dev": true
},
+ "node_modules/postgres-array": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
+ "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
+ "engines": {
+ "node": ">=4"
+ }
+ },
+ "node_modules/postgres-bytea": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz",
+ "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/postgres-date": {
+ "version": "1.0.7",
+ "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
+ "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/postgres-interval": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
+ "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
+ "dependencies": {
+ "xtend": "^4.0.0"
+ },
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
"node_modules/prebuild-install": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz",
@@ -25193,7 +25428,6 @@
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
"integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=",
- "dev": true,
"engines": {
"node": ">=0.10.0"
}
@@ -26234,9 +26468,9 @@
}
},
"node_modules/snap-shot-compare/node_modules/ansi-regex": {
- "version": "4.1.0",
- "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz",
- "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==",
+ "version": "4.1.1",
+ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz",
+ "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==",
"dev": true,
"engines": {
"node": ">=6"
@@ -26836,6 +27070,14 @@
"node": ">=6"
}
},
+ "node_modules/split2": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz",
+ "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ==",
+ "engines": {
+ "node": ">= 10.x"
+ }
+ },
"node_modules/sponge-case": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/sponge-case/-/sponge-case-1.0.1.tgz",
@@ -27067,7 +27309,6 @@
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
"integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
- "dev": true,
"dependencies": {
"emoji-regex": "^8.0.0",
"is-fullwidth-code-point": "^3.0.0",
@@ -27081,7 +27322,6 @@
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
"integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
- "dev": true,
"engines": {
"node": ">=8"
}
@@ -27089,14 +27329,12 @@
"node_modules/string-width/node_modules/emoji-regex": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
- "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
- "dev": true
+ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
},
"node_modules/string-width/node_modules/is-fullwidth-code-point": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
- "dev": true,
"engines": {
"node": ">=8"
}
@@ -27105,7 +27343,6 @@
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
"integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
- "dev": true,
"dependencies": {
"ansi-regex": "^5.0.1"
},
@@ -29700,7 +29937,6 @@
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
- "dev": true,
"dependencies": {
"ansi-styles": "^4.0.0",
"string-width": "^4.1.0",
@@ -29714,10 +29950,9 @@
}
},
"node_modules/wrap-ansi/node_modules/ansi-regex": {
- "version": "5.0.0",
- "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
- "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
- "dev": true,
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+ "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
"engines": {
"node": ">=8"
}
@@ -29726,7 +29961,6 @@
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
- "dev": true,
"dependencies": {
"color-convert": "^2.0.1"
},
@@ -29741,7 +29975,6 @@
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
- "dev": true,
"dependencies": {
"color-name": "~1.1.4"
},
@@ -29752,14 +29985,12 @@
"node_modules/wrap-ansi/node_modules/color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
- "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
- "dev": true
+ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
},
"node_modules/wrap-ansi/node_modules/strip-ansi": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
"integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
- "dev": true,
"dependencies": {
"ansi-regex": "^5.0.0"
},
@@ -30020,9 +30251,9 @@
}
},
"node_modules/yargs/node_modules/ansi-regex": {
- "version": "5.0.0",
- "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
- "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+ "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
"dev": true,
"engines": {
"node": ">=8"
@@ -32150,9 +32381,9 @@
},
"dependencies": {
"ansi-regex": {
- "version": "5.0.0",
- "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
- "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+ "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
"dev": true
},
"strip-ansi": {
@@ -34314,8 +34545,7 @@
"@types/node": {
"version": "16.7.10",
"resolved": "https://registry.npmjs.org/@types/node/-/node-16.7.10.tgz",
- "integrity": "sha512-S63Dlv4zIPb8x6MMTgDq5WWRJQe56iBEY0O3SOFA9JrRienkOVDXSXBjjJw6HTNQYSE2JI6GMCR6LVbIMHJVvA==",
- "dev": true
+ "integrity": "sha512-S63Dlv4zIPb8x6MMTgDq5WWRJQe56iBEY0O3SOFA9JrRienkOVDXSXBjjJw6HTNQYSE2JI6GMCR6LVbIMHJVvA=="
},
"@types/node-fetch": {
"version": "2.6.2",
@@ -34352,6 +34582,16 @@
"integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==",
"dev": true
},
+ "@types/pg": {
+ "version": "8.6.6",
+ "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.6.6.tgz",
+ "integrity": "sha512-O2xNmXebtwVekJDD+02udOncjVcMZQuTEQEMpKJ0ZRf5E7/9JJX3izhKUcUifBkyKpljyUM6BTgy2trmviKlpw==",
+ "requires": {
+ "@types/node": "*",
+ "pg-protocol": "*",
+ "pg-types": "^2.2.0"
+ }
+ },
"@types/prop-types": {
"version": "15.7.1",
"resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.1.tgz",
@@ -35154,7 +35394,7 @@
"ansi-regex": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
- "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=",
+ "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==",
"dev": true
},
"ansi-styles": {
@@ -36057,6 +36297,11 @@
"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==",
"dev": true
},
+ "buffer-writer": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
+ "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw=="
+ },
"builtin-modules": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz",
@@ -38973,8 +39218,7 @@
"escalade": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz",
- "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==",
- "dev": true
+ "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw=="
},
"escape-goat": {
"version": "2.1.1",
@@ -42517,8 +42761,7 @@
"get-caller-file": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
- "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
- "dev": true
+ "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
},
"get-func-name": {
"version": "2.0.0",
@@ -46475,6 +46718,76 @@
"integrity": "sha512-jY5dPJzw6NHd/KPSfPKJ+IHoFS81/tJ43r34ZeNMXGzCOM8jwQDCD12HYayKIB6MuznrnqIYy2e891NA2g0ibA==",
"dev": true
},
+ "node-pg-migrate": {
+ "version": "6.2.2",
+ "resolved": "https://registry.npmjs.org/node-pg-migrate/-/node-pg-migrate-6.2.2.tgz",
+ "integrity": "sha512-0WYLTXpWu2doeZhiwJUW/1u21OqAFU2CMQ8YZ8VBcJ0xrdqYAjtd8GGFe5A5DM4NJdIZsqJcLPDFqY0FQsmivw==",
+ "requires": {
+ "@types/pg": "^8.0.0",
+ "decamelize": "^5.0.0",
+ "mkdirp": "~1.0.0",
+ "yargs": "~17.3.0"
+ },
+ "dependencies": {
+ "ansi-regex": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+ "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="
+ },
+ "cliui": {
+ "version": "7.0.4",
+ "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz",
+ "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==",
+ "requires": {
+ "string-width": "^4.2.0",
+ "strip-ansi": "^6.0.0",
+ "wrap-ansi": "^7.0.0"
+ }
+ },
+ "decamelize": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-5.0.1.tgz",
+ "integrity": "sha512-VfxadyCECXgQlkoEAjeghAr5gY3Hf+IKjKb+X8tGVDtveCjN+USwprd2q3QXBR9T1+x2DG0XZF5/w+7HAtSaXA=="
+ },
+ "mkdirp": {
+ "version": "1.0.4",
+ "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
+ "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw=="
+ },
+ "strip-ansi": {
+ "version": "6.0.1",
+ "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+ "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+ "requires": {
+ "ansi-regex": "^5.0.1"
+ }
+ },
+ "y18n": {
+ "version": "5.0.8",
+ "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
+ "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="
+ },
+ "yargs": {
+ "version": "17.3.1",
+ "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.3.1.tgz",
+ "integrity": "sha512-WUANQeVgjLbNsEmGk20f+nlHgOqzRFpiGWVaBrYGYIGANIIu3lWjoyi0fNlFmJkvfhCZ6BXINe7/W2O2bV4iaA==",
+ "requires": {
+ "cliui": "^7.0.2",
+ "escalade": "^3.1.1",
+ "get-caller-file": "^2.0.5",
+ "require-directory": "^2.1.1",
+ "string-width": "^4.2.3",
+ "y18n": "^5.0.5",
+ "yargs-parser": "^21.0.0"
+ }
+ },
+ "yargs-parser": {
+ "version": "21.1.1",
+ "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz",
+ "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw=="
+ }
+ }
+ },
"node-releases": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.6.tgz",
@@ -47147,6 +47460,11 @@
}
}
},
+ "packet-reader": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz",
+ "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ=="
+ },
"param-case": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz",
@@ -47426,6 +47744,61 @@
"integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=",
"dev": true
},
+ "pg": {
+ "version": "8.9.0",
+ "resolved": "https://registry.npmjs.org/pg/-/pg-8.9.0.tgz",
+ "integrity": "sha512-ZJM+qkEbtOHRuXjmvBtOgNOXOtLSbxiMiUVMgE4rV6Zwocy03RicCVvDXgx8l4Biwo8/qORUnEqn2fdQzV7KCg==",
+ "requires": {
+ "buffer-writer": "2.0.0",
+ "packet-reader": "1.0.0",
+ "pg-connection-string": "^2.5.0",
+ "pg-pool": "^3.5.2",
+ "pg-protocol": "^1.6.0",
+ "pg-types": "^2.1.0",
+ "pgpass": "1.x"
+ }
+ },
+ "pg-connection-string": {
+ "version": "2.5.0",
+ "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz",
+ "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ=="
+ },
+ "pg-int8": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
+ "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw=="
+ },
+ "pg-pool": {
+ "version": "3.5.2",
+ "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.5.2.tgz",
+ "integrity": "sha512-His3Fh17Z4eg7oANLob6ZvH8xIVen3phEZh2QuyrIl4dQSDVEabNducv6ysROKpDNPSD+12tONZVWfSgMvDD9w==",
+ "requires": {}
+ },
+ "pg-protocol": {
+ "version": "1.6.0",
+ "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz",
+ "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q=="
+ },
+ "pg-types": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
+ "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
+ "requires": {
+ "pg-int8": "1.0.1",
+ "postgres-array": "~2.0.0",
+ "postgres-bytea": "~1.0.0",
+ "postgres-date": "~1.0.4",
+ "postgres-interval": "^1.1.0"
+ }
+ },
+ "pgpass": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
+ "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
+ "requires": {
+ "split2": "^4.1.0"
+ }
+ },
"physical-cpu-count": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/physical-cpu-count/-/physical-cpu-count-2.0.0.tgz",
@@ -48030,6 +48403,29 @@
"integrity": "sha512-97DXOFbQJhk71ne5/Mt6cOu6yxsSfM0QGQyl0L25Gca4yGWEGJaig7l7gbCX623VqTBNGLRLaVUCnNkcedlRSQ==",
"dev": true
},
+ "postgres-array": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
+ "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="
+ },
+ "postgres-bytea": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz",
+ "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w=="
+ },
+ "postgres-date": {
+ "version": "1.0.7",
+ "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
+ "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q=="
+ },
+ "postgres-interval": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
+ "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
+ "requires": {
+ "xtend": "^4.0.0"
+ }
+ },
"prebuild-install": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz",
@@ -49229,8 +49625,7 @@
"require-directory": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
- "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=",
- "dev": true
+ "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I="
},
"require-from-string": {
"version": "2.0.2",
@@ -50056,9 +50451,9 @@
},
"dependencies": {
"ansi-regex": {
- "version": "4.1.0",
- "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz",
- "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==",
+ "version": "4.1.1",
+ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz",
+ "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==",
"dev": true
},
"debug": {
@@ -50563,6 +50958,11 @@
"integrity": "sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw==",
"dev": true
},
+ "split2": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz",
+ "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ=="
+ },
"sponge-case": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/sponge-case/-/sponge-case-1.0.1.tgz",
@@ -50760,7 +51160,6 @@
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
"integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
- "dev": true,
"requires": {
"emoji-regex": "^8.0.0",
"is-fullwidth-code-point": "^3.0.0",
@@ -50770,26 +51169,22 @@
"ansi-regex": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
- "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
- "dev": true
+ "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="
},
"emoji-regex": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
- "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
- "dev": true
+ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
},
"is-fullwidth-code-point": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
- "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==",
- "dev": true
+ "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="
},
"strip-ansi": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
"integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
- "dev": true,
"requires": {
"ansi-regex": "^5.0.1"
}
@@ -52746,7 +53141,6 @@
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
- "dev": true,
"requires": {
"ansi-styles": "^4.0.0",
"string-width": "^4.1.0",
@@ -52754,16 +53148,14 @@
},
"dependencies": {
"ansi-regex": {
- "version": "5.0.0",
- "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
- "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
- "dev": true
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+ "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="
},
"ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
- "dev": true,
"requires": {
"color-convert": "^2.0.1"
}
@@ -52772,7 +53164,6 @@
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
- "dev": true,
"requires": {
"color-name": "~1.1.4"
}
@@ -52780,14 +53171,12 @@
"color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
- "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
- "dev": true
+ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
},
"strip-ansi": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
"integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
- "dev": true,
"requires": {
"ansi-regex": "^5.0.0"
}
@@ -52937,9 +53326,9 @@
},
"dependencies": {
"ansi-regex": {
- "version": "5.0.0",
- "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
- "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==",
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
+ "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==",
"dev": true
},
"cliui": {
diff --git a/package.json b/package.json
index 33d54aa5a1..878548cfd0 100644
--- a/package.json
+++ b/package.json
@@ -54,8 +54,10 @@
"lodash.groupby": "^4.6.0",
"lodash.times": "^4.3.2",
"node-env-flag": "^0.1.0",
+ "node-pg-migrate": "^6.2.2",
"parse-link-header": "^2.0.0",
"path-to-regexp": "^6.2.1",
+ "pg": "^8.9.0",
"pretty-bytes": "^6.1.0",
"priorityqueuejs": "^2.0.0",
"prom-client": "^14.1.1",
@@ -117,7 +119,8 @@
"e2e": "start-server-and-test start http://localhost:3000 test:e2e",
"e2e-on-build": "cross-env CYPRESS_baseUrl=http://localhost:8080 start-server-and-test start:server:e2e-on-build http://localhost:8080 test:e2e",
"badge": "cross-env NODE_CONFIG_ENV=test TRACE_SERVICES=true node scripts/badge-cli.js",
- "build-docs": "rimraf api-docs/ && jsdoc --pedantic -c ./jsdoc.json . && echo 'contributing.shields.io' > api-docs/CNAME"
+ "build-docs": "rimraf api-docs/ && jsdoc --pedantic -c ./jsdoc.json . && echo 'contributing.shields.io' > api-docs/CNAME",
+ "migrate": "node scripts/write-migrations-config.js > migrations-config.json && node-pg-migrate --config-file=migrations-config.json"
},
"lint-staged": {
"**/*.@(js|ts|tsx)": [
diff --git a/scripts/migrate-token-pool.js b/scripts/migrate-token-pool.js
new file mode 100644
index 0000000000..baaa4cbce1
--- /dev/null
+++ b/scripts/migrate-token-pool.js
@@ -0,0 +1,42 @@
+import pg from 'pg'
+import RedisTokenPersistence from '../core/token-pooling/redis-token-persistence.js'
+
+let redisUrl, postgresUrl
+
+try {
+ redisUrl = process.argv[2]
+ postgresUrl = process.argv[3]
+ if (
+ !redisUrl.startsWith('rediss://') ||
+ !postgresUrl.startsWith('postgresql://')
+ ) {
+ throw new Error()
+ }
+} catch (e) {
+ process.stdout.write(
+ 'Usage: migrate-token-pool.js [redis-url] [postgres-url]\n'
+ )
+ process.exit(1)
+}
+
+;(async () => {
+ const redis = new RedisTokenPersistence({
+ url: redisUrl,
+ key: 'githubUserTokens',
+ })
+
+ const tokens = await redis.initialize()
+ await redis.stop()
+ console.log(`${tokens.length} tokens in redis source`)
+
+ const pool = new pg.Pool({ connectionString: postgresUrl })
+
+ let bulkInsert = 'INSERT INTO github_user_tokens (token) VALUES '
+ bulkInsert += tokens.map(token => `('${token}')`).join(',')
+ bulkInsert += ' ON CONFLICT (token) DO NOTHING;'
+ await pool.query(bulkInsert)
+
+ const result = await pool.query('SELECT COUNT(*) FROM github_user_tokens;')
+ console.log(`${result.rows[0].count} tokens in postgres dest`)
+ await pool.end()
+})()
diff --git a/scripts/write-migrations-config.js b/scripts/write-migrations-config.js
new file mode 100644
index 0000000000..f98e265587
--- /dev/null
+++ b/scripts/write-migrations-config.js
@@ -0,0 +1,11 @@
+import configModule from 'config'
+const config = configModule.util.toObject()
+
+const postgresUrl = config?.private?.postgres_url
+
+if (!postgresUrl) {
+ process.exit(1)
+}
+
+process.stdout.write(JSON.stringify({ url: postgresUrl }))
+process.exit(0)
diff --git a/server.js b/server.js
index 8f74c7d482..e34f305dd6 100644
--- a/server.js
+++ b/server.js
@@ -42,6 +42,12 @@ if (fs.existsSync('.env')) {
process.exit(1)
}
+if (config.private.redis_url != null) {
+ console.warning(
+ 'RedisTokenPersistence is deprecated for token pooling and will be removed in a future release. Migrate to SqlTokenPersistence'
+ )
+}
+
const legacySecretsPath = path.join(
path.dirname(fileURLToPath(import.meta.url)),
'private',
diff --git a/services/github/github-constellation.js b/services/github/github-constellation.js
index dd4016b496..f229185aa8 100644
--- a/services/github/github-constellation.js
+++ b/services/github/github-constellation.js
@@ -1,5 +1,6 @@
import { AuthHelper } from '../../core/base-service/auth-helper.js'
import RedisTokenPersistence from '../../core/token-pooling/redis-token-persistence.js'
+import SqlTokenPersistence from '../../core/token-pooling/sql-token-persistence.js'
import log from '../../core/server/log.js'
import GithubApiProvider from './github-api-provider.js'
import { setRoutes as setAcceptorRoutes } from './auth/acceptor.js'
@@ -23,8 +24,18 @@ class GithubConstellation {
this._debugEnabled = config.service.debug.enabled
this._debugIntervalSeconds = config.service.debug.intervalSeconds
- const { redis_url: redisUrl, gh_token: globalToken } = config.private
- if (redisUrl) {
+ const {
+ postgres_url: pgUrl,
+ redis_url: redisUrl,
+ gh_token: globalToken,
+ } = config.private
+ if (pgUrl) {
+ log.log('Token persistence configured with dbUrl')
+ this.persistence = new SqlTokenPersistence({
+ url: pgUrl,
+ table: 'github_user_tokens',
+ })
+ } else if (redisUrl) {
log.log('Token persistence configured with redisUrl')
this.persistence = new RedisTokenPersistence({
url: redisUrl,
--
GitLab