diff --git a/config/custom-environment-variables.yml b/config/custom-environment-variables.yml
index effec83c27f8e41d441cc5f6947eea2f18c85a6b..f509c63566248fc43d9b1166fe2a770c78cbec4d 100644
--- a/config/custom-environment-variables.yml
+++ b/config/custom-environment-variables.yml
@@ -6,6 +6,15 @@ public:
metrics:
prometheus:
enabled: 'METRICS_PROMETHEUS_ENABLED'
+ endpointEnabled: 'METRICS_PROMETHEUS_ENDPOINT_ENABLED'
+ influx:
+ enabled: 'METRICS_INFLUX_ENABLED'
+ url: 'METRICS_INFLUX_URL'
+ timeoutMilliseconds: 'METRICS_INFLUX_TIMEOUT_MILLISECONDS'
+ intervalSeconds: 'METRICS_INFLUX_INTERVAL_SECONDS'
+ instanceIdFrom: 'METRICS_INFLUX_INSTANCE_ID_FROM'
+ instanceIdEnvVarName: 'METRICS_INFLUX_INSTANCE_ID_ENV_VAR_NAME'
+ envLabel: 'METRICS_INFLUX_ENV_LABEL'
ssl:
isSecure: 'HTTPS'
@@ -85,3 +94,5 @@ private:
twitch_client_id: 'TWITCH_CLIENT_ID'
twitch_client_secret: 'TWITCH_CLIENT_SECRET'
wheelmap_token: 'WHEELMAP_TOKEN'
+ influx_username: 'INFLUX_USERNAME'
+ influx_password: 'INFLUX_PASSWORD'
diff --git a/config/default.yml b/config/default.yml
index 405670561c37a7463f59a295d5a90887763ab01c..e782ca08572cb816eaac144e025c591572edba40 100644
--- a/config/default.yml
+++ b/config/default.yml
@@ -5,7 +5,11 @@ public:
metrics:
prometheus:
enabled: false
-
+ endpointEnabled: false
+ influx:
+ enabled: false
+ timeoutMilliseconds: 1000
+ intervalSeconds: 15
ssl:
isSecure: false
diff --git a/config/shields-io-production.yml b/config/shields-io-production.yml
index 1ca9d5805a5afdfcd3d139994c4f6c9a29bbbb99..e039532cab39f14399cae1dce373e7e20b4a9084 100644
--- a/config/shields-io-production.yml
+++ b/config/shields-io-production.yml
@@ -2,6 +2,7 @@ public:
metrics:
prometheus:
enabled: true
+ endpointEnabled: true
ssl:
isSecure: true
diff --git a/core/server/in-process-server-test-helpers.js b/core/server/in-process-server-test-helpers.js
index fbf530a48d0aa7652abffde47174557aa8260cf3..85280a1a316292c5ec132b890c53c0eb47ae7e13 100644
--- a/core/server/in-process-server-test-helpers.js
+++ b/core/server/in-process-server-test-helpers.js
@@ -1,21 +1,16 @@
'use strict'
+const merge = require('deepmerge')
const config = require('config').util.toObject()
+const portfinder = require('portfinder')
const Server = require('./server')
-function createTestServer({ port }) {
- const serverConfig = {
- ...config,
- public: {
- ...config.public,
- bind: {
- ...config.public.bind,
- port,
- },
- },
+async function createTestServer(customConfig = {}) {
+ const mergedConfig = merge(config, customConfig)
+ if (!mergedConfig.public.bind.port) {
+ mergedConfig.public.bind.port = await portfinder.getPortPromise()
}
-
- return new Server(serverConfig)
+ return new Server(mergedConfig)
}
module.exports = {
diff --git a/core/server/influx-metrics.js b/core/server/influx-metrics.js
new file mode 100644
index 0000000000000000000000000000000000000000..46e10a4f378965f5702455c018a116d158788089
--- /dev/null
+++ b/core/server/influx-metrics.js
@@ -0,0 +1,86 @@
+'use strict'
+const os = require('os')
+const { promisify } = require('util')
+const { post } = require('request')
+const postAsync = promisify(post)
+const generateInstanceId = require('./instance-id-generator')
+const { promClientJsonToInfluxV2 } = require('./metrics/format-converters')
+const log = require('./log')
+
+module.exports = class InfluxMetrics {
+ constructor(metricInstance, config) {
+ this._metricInstance = metricInstance
+ this._config = config
+ this._instanceId = this.getInstanceId()
+ }
+
+ async sendMetrics() {
+ const auth = {
+ user: this._config.username,
+ pass: this._config.password,
+ }
+ const request = {
+ uri: this._config.url,
+ headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+ body: this.metrics(),
+ timeout: this._config.timeoutMillseconds,
+ auth,
+ }
+
+ let response
+ try {
+ response = await postAsync(request)
+ } catch (error) {
+ log.error(
+ new Error(`Cannot push metrics. Cause: ${error.name}: ${error.message}`)
+ )
+ }
+ if (response && response.statusCode >= 300) {
+ log.error(
+ new Error(
+ `Cannot push metrics. ${response.request.href} responded with status code ${response.statusCode}`
+ )
+ )
+ }
+ }
+
+ startPushingMetrics() {
+ this._intervalId = setInterval(
+ () => this.sendMetrics(),
+ this._config.intervalSeconds * 1000
+ )
+ }
+
+ metrics() {
+ return promClientJsonToInfluxV2(this._metricInstance.metrics(), {
+ env: this._config.envLabel,
+ application: 'shields',
+ instance: this._instanceId,
+ })
+ }
+
+ getInstanceId() {
+ const {
+ hostnameAliases = {},
+ instanceIdFrom,
+ instanceIdEnvVarName,
+ } = this._config
+ let instance
+ if (instanceIdFrom === 'env-var') {
+ instance = process.env[instanceIdEnvVarName]
+ } else if (instanceIdFrom === 'hostname') {
+ const hostname = os.hostname()
+ instance = hostnameAliases[hostname] || hostname
+ } else if (instanceIdFrom === 'random') {
+ instance = generateInstanceId()
+ }
+ return instance
+ }
+
+ stopPushingMetrics() {
+ if (this._intervalId) {
+ clearInterval(this._intervalId)
+ this._intervalId = undefined
+ }
+ }
+}
diff --git a/core/server/influx-metrics.spec.js b/core/server/influx-metrics.spec.js
new file mode 100644
index 0000000000000000000000000000000000000000..9885754fbbe7a157aad921355a49cc017f688779
--- /dev/null
+++ b/core/server/influx-metrics.spec.js
@@ -0,0 +1,177 @@
+'use strict'
+const os = require('os')
+const nock = require('nock')
+const sinon = require('sinon')
+const { expect } = require('chai')
+const log = require('./log')
+const InfluxMetrics = require('./influx-metrics')
+require('../register-chai-plugins.spec')
+describe('Influx metrics', function() {
+ const metricInstance = {
+ metrics() {
+ return [
+ {
+ help: 'counter 1 help',
+ name: 'counter1',
+ type: 'counter',
+ values: [{ value: 11, labels: {} }],
+ aggregator: 'sum',
+ },
+ ]
+ },
+ }
+ describe('"metrics" function', function() {
+ let osHostnameStub
+ afterEach(function() {
+ nock.enableNetConnect()
+ delete process.env.INSTANCE_ID
+ if (osHostnameStub) {
+ osHostnameStub.restore()
+ }
+ })
+ it('should use an environment variable value as an instance label', async function() {
+ process.env.INSTANCE_ID = 'instance3'
+ const influxMetrics = new InfluxMetrics(metricInstance, {
+ instanceIdFrom: 'env-var',
+ instanceIdEnvVarName: 'INSTANCE_ID',
+ })
+
+ expect(influxMetrics.metrics()).to.contain('instance=instance3')
+ })
+
+ it('should use a hostname as an instance label', async function() {
+ osHostnameStub = sinon.stub(os, 'hostname').returns('test-hostname')
+ const customConfig = {
+ instanceIdFrom: 'hostname',
+ }
+ const influxMetrics = new InfluxMetrics(metricInstance, customConfig)
+
+ expect(influxMetrics.metrics()).to.be.contain('instance=test-hostname')
+ })
+
+ it('should use a random string as an instance label', async function() {
+ const customConfig = {
+ instanceIdFrom: 'random',
+ }
+ const influxMetrics = new InfluxMetrics(metricInstance, customConfig)
+
+ expect(influxMetrics.metrics()).to.be.match(/instance=\w+ /)
+ })
+
+ it('should use a hostname alias as an instance label', async function() {
+ osHostnameStub = sinon.stub(os, 'hostname').returns('test-hostname')
+ const customConfig = {
+ instanceIdFrom: 'hostname',
+ hostnameAliases: { 'test-hostname': 'test-hostname-alias' },
+ }
+ const influxMetrics = new InfluxMetrics(metricInstance, customConfig)
+
+ expect(influxMetrics.metrics()).to.be.contain(
+ 'instance=test-hostname-alias'
+ )
+ })
+ })
+
+ describe('startPushingMetrics', function() {
+ let influxMetrics, clock
+ beforeEach(function() {
+ clock = sinon.useFakeTimers()
+ })
+ afterEach(function() {
+ influxMetrics.stopPushingMetrics()
+ nock.cleanAll()
+ nock.enableNetConnect()
+ delete process.env.INSTANCE_ID
+ clock.restore()
+ })
+
+ it('should send metrics', async function() {
+ const scope = nock('http://shields-metrics.io/', {
+ reqheaders: {
+ 'Content-Type': 'application/x-www-form-urlencoded',
+ },
+ })
+ .persist()
+ .post(
+ '/metrics',
+ 'prometheus,application=shields,env=test-env,instance=instance2 counter1=11'
+ )
+ .basicAuth({ user: 'metrics-username', pass: 'metrics-password' })
+ .reply(200)
+ process.env.INSTANCE_ID = 'instance2'
+ influxMetrics = new InfluxMetrics(metricInstance, {
+ url: 'http://shields-metrics.io/metrics',
+ timeoutMillseconds: 100,
+ intervalSeconds: 0.001,
+ username: 'metrics-username',
+ password: 'metrics-password',
+ instanceIdFrom: 'env-var',
+ instanceIdEnvVarName: 'INSTANCE_ID',
+ envLabel: 'test-env',
+ })
+
+ influxMetrics.startPushingMetrics()
+
+ await clock.tickAsync(10)
+ expect(scope.isDone()).to.be.equal(
+ true,
+ `pending mocks: ${scope.pendingMocks()}`
+ )
+ })
+ })
+
+ describe('sendMetrics', function() {
+ beforeEach(function() {
+ sinon.spy(log, 'error')
+ })
+ afterEach(function() {
+ log.error.restore()
+ nock.cleanAll()
+ nock.enableNetConnect()
+ })
+
+ const influxMetrics = new InfluxMetrics(metricInstance, {
+ url: 'http://shields-metrics.io/metrics',
+ timeoutMillseconds: 50,
+ intervalSeconds: 0,
+ username: 'metrics-username',
+ password: 'metrics-password',
+ })
+ it('should log errors', async function() {
+ nock.disableNetConnect()
+
+ await influxMetrics.sendMetrics()
+
+ expect(log.error).to.be.calledWith(
+ sinon.match
+ .instanceOf(Error)
+ .and(
+ sinon.match.has(
+ 'message',
+ 'Cannot push metrics. Cause: NetConnectNotAllowedError: Nock: Disallowed net connect for "shields-metrics.io:80/metrics"'
+ )
+ )
+ )
+ })
+
+ it('should log error responses', async function() {
+ nock('http://shields-metrics.io/')
+ .persist()
+ .post('/metrics')
+ .reply(400)
+
+ await influxMetrics.sendMetrics()
+
+ expect(log.error).to.be.calledWith(
+ sinon.match
+ .instanceOf(Error)
+ .and(
+ sinon.match.has(
+ 'message',
+ 'Cannot push metrics. http://shields-metrics.io/metrics responded with status code 400'
+ )
+ )
+ )
+ })
+ })
+})
diff --git a/core/server/instance-id-generator.js b/core/server/instance-id-generator.js
new file mode 100644
index 0000000000000000000000000000000000000000..0036df2648842f0ab739a91a60687377746bcb11
--- /dev/null
+++ b/core/server/instance-id-generator.js
@@ -0,0 +1,10 @@
+'use strict'
+
+function generateInstanceId() {
+ // from https://gist.github.com/gordonbrander/2230317
+ return Math.random()
+ .toString(36)
+ .substr(2, 9)
+}
+
+module.exports = generateInstanceId
diff --git a/core/server/metrics/format-converters.js b/core/server/metrics/format-converters.js
new file mode 100644
index 0000000000000000000000000000000000000000..e6ac8e2332c01cbe28806756ed76b2ab60c03923
--- /dev/null
+++ b/core/server/metrics/format-converters.js
@@ -0,0 +1,27 @@
+'use strict'
+const groupBy = require('lodash.groupby')
+
+function promClientJsonToInfluxV2(metrics, extraLabels = {}) {
+ // TODO Replace with Array.prototype.flatMap() after migrating to Node.js >= 11
+ const flatMap = (f, arr) => arr.reduce((acc, x) => acc.concat(f(x)), [])
+ return flatMap(metric => {
+ const valuesByLabels = groupBy(metric.values, value =>
+ JSON.stringify(Object.entries(value.labels).sort())
+ )
+ return Object.values(valuesByLabels).map(metricsWithSameLabel => {
+ const labels = Object.entries(metricsWithSameLabel[0].labels)
+ .concat(Object.entries(extraLabels))
+ .sort((a, b) => a[0].localeCompare(b[0]))
+ .map(labelEntry => `${labelEntry[0]}=${labelEntry[1]}`)
+ .join(',')
+ const labelsFormatted = labels ? `,${labels}` : ''
+ const values = metricsWithSameLabel
+ .sort((a, b) => a.metricName.localeCompare(b.metricName))
+ .map(value => `${value.metricName || metric.name}=${value.value}`)
+ .join(',')
+ return `prometheus${labelsFormatted} ${values}`
+ })
+ }, metrics).join('\n')
+}
+
+module.exports = { promClientJsonToInfluxV2 }
diff --git a/core/server/metrics/format-converters.spec.js b/core/server/metrics/format-converters.spec.js
new file mode 100644
index 0000000000000000000000000000000000000000..373c3d9cdab2ba7f99382a4896efb22a9efc9384
--- /dev/null
+++ b/core/server/metrics/format-converters.spec.js
@@ -0,0 +1,217 @@
+'use strict'
+
+const { expect } = require('chai')
+const prometheus = require('prom-client')
+const { promClientJsonToInfluxV2 } = require('./format-converters')
+
+describe('Metric format converters', function() {
+ describe('prom-client JSON to InfluxDB line protocol (version 2)', function() {
+ it('converts a counter', function() {
+ const json = [
+ {
+ help: 'counter 1 help',
+ name: 'counter1',
+ type: 'counter',
+ values: [{ value: 11, labels: {} }],
+ aggregator: 'sum',
+ },
+ ]
+
+ const influx = promClientJsonToInfluxV2(json)
+
+ expect(influx).to.be.equal('prometheus counter1=11')
+ })
+
+ it('converts a counter (from prometheus registry)', function() {
+ const register = new prometheus.Registry()
+ const counter = new prometheus.Counter({
+ name: 'counter1',
+ help: 'counter 1 help',
+ registers: [register],
+ })
+ counter.inc(11)
+
+ const influx = promClientJsonToInfluxV2(register.getMetricsAsJSON())
+
+ expect(influx).to.be.equal('prometheus counter1=11')
+ })
+
+ it('converts a gauge', function() {
+ const json = [
+ {
+ help: 'gause 1 help',
+ name: 'gauge1',
+ type: 'gauge',
+ values: [{ value: 20, labels: {} }],
+ aggregator: 'sum',
+ },
+ ]
+
+ const influx = promClientJsonToInfluxV2(json)
+
+ expect(influx).to.be.equal('prometheus gauge1=20')
+ })
+
+ it('converts a gauge (from prometheus registry)', function() {
+ const register = new prometheus.Registry()
+ const gauge = new prometheus.Gauge({
+ name: 'gauge1',
+ help: 'gauge 1 help',
+ registers: [register],
+ })
+ gauge.inc(20)
+
+ const influx = promClientJsonToInfluxV2(register.getMetricsAsJSON())
+
+ expect(influx).to.be.equal('prometheus gauge1=20')
+ })
+
+ const sortLines = text =>
+ text
+ .split('\n')
+ .sort()
+ .join('\n')
+
+ it('converts a histogram', function() {
+ const json = [
+ {
+ name: 'histogram1',
+ help: 'histogram 1 help',
+ type: 'histogram',
+ values: [
+ { labels: { le: 5 }, value: 1, metricName: 'histogram1_bucket' },
+ { labels: { le: 15 }, value: 2, metricName: 'histogram1_bucket' },
+ { labels: { le: 50 }, value: 2, metricName: 'histogram1_bucket' },
+ {
+ labels: { le: '+Inf' },
+ value: 3,
+ metricName: 'histogram1_bucket',
+ },
+ { labels: {}, value: 111, metricName: 'histogram1_sum' },
+ { labels: {}, value: 3, metricName: 'histogram1_count' },
+ ],
+ aggregator: 'sum',
+ },
+ ]
+
+ const influx = promClientJsonToInfluxV2(json)
+
+ expect(sortLines(influx)).to.be.equal(
+ sortLines(`prometheus,le=+Inf histogram1_bucket=3
+prometheus,le=50 histogram1_bucket=2
+prometheus,le=15 histogram1_bucket=2
+prometheus,le=5 histogram1_bucket=1
+prometheus histogram1_count=3,histogram1_sum=111`)
+ )
+ })
+
+ it('converts a histogram (from prometheus registry)', function() {
+ const register = new prometheus.Registry()
+ const histogram = new prometheus.Histogram({
+ name: 'histogram1',
+ help: 'histogram 1 help',
+ buckets: [5, 15, 50],
+ registers: [register],
+ })
+ histogram.observe(100)
+ histogram.observe(10)
+ histogram.observe(1)
+
+ const influx = promClientJsonToInfluxV2(register.getMetricsAsJSON())
+
+ expect(sortLines(influx)).to.be.equal(
+ sortLines(`prometheus,le=+Inf histogram1_bucket=3
+prometheus,le=50 histogram1_bucket=2
+prometheus,le=15 histogram1_bucket=2
+prometheus,le=5 histogram1_bucket=1
+prometheus histogram1_count=3,histogram1_sum=111`)
+ )
+ })
+
+ it('converts a summary', function() {
+ const json = [
+ {
+ name: 'summary1',
+ help: 'summary 1 help',
+ type: 'summary',
+ values: [
+ { labels: { quantile: 0.1 }, value: 1 },
+ { labels: { quantile: 0.9 }, value: 100 },
+ { labels: { quantile: 0.99 }, value: 100 },
+ { metricName: 'summary1_sum', labels: {}, value: 111 },
+ { metricName: 'summary1_count', labels: {}, value: 3 },
+ ],
+ aggregator: 'sum',
+ },
+ ]
+
+ const influx = promClientJsonToInfluxV2(json)
+
+ expect(sortLines(influx)).to.be.equal(
+ sortLines(`prometheus,quantile=0.99 summary1=100
+prometheus,quantile=0.9 summary1=100
+prometheus,quantile=0.1 summary1=1
+prometheus summary1_count=3,summary1_sum=111`)
+ )
+ })
+
+ it('converts a summary (from prometheus registry)', function() {
+ const register = new prometheus.Registry()
+ const summary = new prometheus.Summary({
+ name: 'summary1',
+ help: 'summary 1 help',
+ percentiles: [0.1, 0.9, 0.99],
+ registers: [register],
+ })
+ summary.observe(100)
+ summary.observe(10)
+ summary.observe(1)
+
+ const influx = promClientJsonToInfluxV2(register.getMetricsAsJSON())
+
+ expect(sortLines(influx)).to.be.equal(
+ sortLines(`prometheus,quantile=0.99 summary1=100
+prometheus,quantile=0.9 summary1=100
+prometheus,quantile=0.1 summary1=1
+prometheus summary1_count=3,summary1_sum=111`)
+ )
+ })
+
+ it('converts a counter and skip a timestamp', function() {
+ const json = [
+ {
+ help: 'counter 4 help',
+ name: 'counter4',
+ type: 'counter',
+ values: [{ value: 11, labels: {}, timestamp: 1581850552292 }],
+ aggregator: 'sum',
+ },
+ ]
+
+ const influx = promClientJsonToInfluxV2(json)
+
+ expect(influx).to.be.equal('prometheus counter4=11')
+ })
+
+ it('converts a counter and adds extra labels', function() {
+ const json = [
+ {
+ help: 'counter 1 help',
+ name: 'counter1',
+ type: 'counter',
+ values: [{ value: 11, labels: {} }],
+ aggregator: 'sum',
+ },
+ ]
+
+ const influx = promClientJsonToInfluxV2(json, {
+ instance: 'instance1',
+ env: 'production',
+ })
+
+ expect(influx).to.be.equal(
+ 'prometheus,env=production,instance=instance1 counter1=11'
+ )
+ })
+ })
+})
diff --git a/core/server/prometheus-metrics.js b/core/server/prometheus-metrics.js
index bda312a40109ec6a352e828058cb24ddc14fc629..69748f52cb19f5bd23381febf1b04520e2ba3c59 100644
--- a/core/server/prometheus-metrics.js
+++ b/core/server/prometheus-metrics.js
@@ -68,11 +68,13 @@ module.exports = class PrometheusMetrics {
registers: [this.register],
}),
}
+ this.interval = prometheus.collectDefaultMetrics({
+ register: this.register,
+ })
}
- async initialize(server) {
+ async registerMetricsEndpoint(server) {
const { register } = this
- this.interval = prometheus.collectDefaultMetrics({ register })
server.route(/^\/metrics$/, (data, match, end, ask) => {
ask.res.setHeader('Content-Type', register.contentType)
@@ -88,6 +90,10 @@ module.exports = class PrometheusMetrics {
}
}
+ metrics() {
+ return this.register.getMetricsAsJSON()
+ }
+
/**
* @returns {object} `{ inc() {} }`.
*/
diff --git a/core/server/prometheus-metrics.spec.js b/core/server/prometheus-metrics.spec.js
index 5fb503cadb2b29357d41ff0edc13397915cca2b4..2ea95be77623480ba07f57e68e73d5aaf1fd698e 100644
--- a/core/server/prometheus-metrics.spec.js
+++ b/core/server/prometheus-metrics.spec.js
@@ -7,26 +7,26 @@ const got = require('../got-test-client')
const Metrics = require('./prometheus-metrics')
describe('Prometheus metrics route', function() {
- let port, baseUrl
+ let port, baseUrl, camp, metrics
beforeEach(async function() {
port = await portfinder.getPortPromise()
baseUrl = `http://127.0.0.1:${port}`
- })
-
- let camp
- beforeEach(async function() {
camp = Camp.start({ port, hostname: '::' })
await new Promise(resolve => camp.on('listening', () => resolve()))
})
afterEach(async function() {
+ if (metrics) {
+ metrics.stop()
+ }
if (camp) {
await new Promise(resolve => camp.close(resolve))
camp = undefined
}
})
- it('returns metrics', async function() {
- new Metrics({ enabled: true }).initialize(camp)
+ it('returns default metrics', async function() {
+ metrics = new Metrics()
+ metrics.registerMetricsEndpoint(camp)
const { statusCode, body } = await got(`${baseUrl}/metrics`)
diff --git a/core/server/server.js b/core/server/server.js
index 3f4f5a4238ad0db17e0f1830c70305874a731146..d2f2ef8d7c3bfe73efeea4854cca4d595bbe0f12 100644
--- a/core/server/server.js
+++ b/core/server/server.js
@@ -23,6 +23,7 @@ const { rasterRedirectUrl } = require('../badge-urls/make-badge-url')
const log = require('./log')
const sysMonitor = require('./monitor')
const PrometheusMetrics = require('./prometheus-metrics')
+const InfluxMetrics = require('./influx-metrics')
const Joi = originalJoi
.extend(base => ({
@@ -81,6 +82,33 @@ const publicConfigSchema = Joi.object({
metrics: {
prometheus: {
enabled: Joi.boolean().required(),
+ endpointEnabled: Joi.boolean().required(),
+ },
+ influx: {
+ enabled: Joi.boolean().required(),
+ url: Joi.string()
+ .uri()
+ .when('enabled', { is: true, then: Joi.required() }),
+ timeoutMilliseconds: Joi.number()
+ .integer()
+ .min(1)
+ .when('enabled', { is: true, then: Joi.required() }),
+ intervalSeconds: Joi.number()
+ .integer()
+ .min(1)
+ .when('enabled', { is: true, then: Joi.required() }),
+ instanceIdFrom: Joi.string()
+ .equal('hostname', 'env-var', 'random')
+ .when('enabled', { is: true, then: Joi.required() }),
+ instanceIdEnvVarName: Joi.string().when('instanceIdFrom', {
+ is: 'env-var',
+ then: Joi.required(),
+ }),
+ envLabel: Joi.string().when('enabled', {
+ is: true,
+ then: Joi.required(),
+ }),
+ hostnameAliases: Joi.object(),
},
},
ssl: {
@@ -160,8 +188,13 @@ const privateConfigSchema = Joi.object({
twitch_client_id: Joi.string(),
twitch_client_secret: Joi.string(),
wheelmap_token: Joi.string(),
+ influx_username: Joi.string(),
+ influx_password: Joi.string(),
}).required()
-
+const privateMetricsInfluxConfigSchema = privateConfigSchema.append({
+ influx_username: Joi.string().required(),
+ influx_password: Joi.string().required(),
+})
/**
* The Server is based on the web framework Scoutcamp. It creates
* an http server, sets up helpers for token persistence and monitoring.
@@ -173,22 +206,25 @@ class Server {
* Badge Server Constructor
*
* @param {object} config Configuration object read from config yaml files
- * by https://www.npmjs.com/package/config and validated against
- * publicConfigSchema and privateConfigSchema
+ * by https://www.npmjs.com/package/config and validated against
+ * publicConfigSchema and privateConfigSchema
* @see https://github.com/badges/shields/blob/master/doc/production-hosting.md#configuration
* @see https://github.com/badges/shields/blob/master/doc/server-secrets.md
*/
constructor(config) {
const publicConfig = Joi.attempt(config.public, publicConfigSchema)
- let privateConfig
- try {
- privateConfig = Joi.attempt(config.private, privateConfigSchema)
- } catch (e) {
- const badPaths = e.details.map(({ path }) => path)
- throw Error(
- `Private configuration is invalid. Check these paths: ${badPaths.join(
- ','
- )}`
+ const privateConfig = this.validatePrivateConfig(
+ config.private,
+ privateConfigSchema
+ )
+ // We want to require an username and a password for the influx metrics
+ // only if the influx metrics are enabled. The private config schema
+ // and the public config schema are two separate schemas so we have to run
+ // validation manually.
+ if (publicConfig.metrics.influx && publicConfig.metrics.influx.enabled) {
+ this.validatePrivateConfig(
+ config.private,
+ privateMetricsInfluxConfigSchema
)
}
this.config = {
@@ -201,8 +237,31 @@ class Server {
service: publicConfig.services.github,
private: privateConfig,
})
+
if (publicConfig.metrics.prometheus.enabled) {
this.metricInstance = new PrometheusMetrics()
+ if (publicConfig.metrics.influx.enabled) {
+ this.influxMetrics = new InfluxMetrics(
+ this.metricInstance,
+ Object.assign({}, publicConfig.metrics.influx, {
+ username: privateConfig.influx_username,
+ password: privateConfig.influx_password,
+ })
+ )
+ }
+ }
+ }
+
+ validatePrivateConfig(privateConfig, privateConfigSchema) {
+ try {
+ return Joi.attempt(privateConfig, privateConfigSchema)
+ } catch (e) {
+ const badPaths = e.details.map(({ path }) => path)
+ throw Error(
+ `Private configuration is invalid. Check these paths: ${badPaths.join(
+ ','
+ )}`
+ )
}
}
@@ -381,7 +440,12 @@ class Server {
const { githubConstellation } = this
githubConstellation.initialize(camp)
if (metricInstance) {
- metricInstance.initialize(camp)
+ if (this.config.public.metrics.prometheus.endpointEnabled) {
+ metricInstance.registerMetricsEndpoint(camp)
+ }
+ if (this.influxMetrics) {
+ this.influxMetrics.startPushingMetrics()
+ }
}
const { apiProvider: githubApiProvider } = this.githubConstellation
@@ -425,6 +489,9 @@ class Server {
}
if (this.metricInstance) {
+ if (this.influxMetrics) {
+ this.influxMetrics.stopPushingMetrics()
+ }
this.metricInstance.stop()
}
}
diff --git a/core/server/server.spec.js b/core/server/server.spec.js
index 177b12f5c7ee229c815bb65ed720584aad83f415..c2482b5d4177964c70c0c4ace705932f2c1f9aa1 100644
--- a/core/server/server.spec.js
+++ b/core/server/server.spec.js
@@ -2,163 +2,333 @@
const { expect } = require('chai')
const isSvg = require('is-svg')
-const portfinder = require('portfinder')
+const config = require('config')
const got = require('../got-test-client')
+const Server = require('./server')
const { createTestServer } = require('./in-process-server-test-helpers')
describe('The server', function() {
- let server, baseUrl
- before('Start the server', async function() {
- // Fixes https://github.com/badges/shields/issues/2611
- this.timeout(10000)
- const port = await portfinder.getPortPromise()
- server = createTestServer({ port })
- baseUrl = server.baseUrl
- await server.start()
- })
- after('Shut down the server', async function() {
- if (server) {
- await server.stop()
- }
- server = undefined
- })
+ describe('running', function() {
+ let server, baseUrl
+ before('Start the server', async function() {
+ // Fixes https://github.com/badges/shields/issues/2611
+ this.timeout(10000)
+ server = await createTestServer()
+ baseUrl = server.baseUrl
+ await server.start()
+ })
+ after('Shut down the server', async function() {
+ if (server) {
+ await server.stop()
+ }
+ server = undefined
+ })
- it('should allow strings for port', async function() {
- // fixes #4391 - This allows the app to be run using iisnode, which uses a named pipe for the port.
- const pipeServer = createTestServer({
- port: '\\\\.\\pipe\\9c137306-7c4d-461e-b7cf-5213a3939ad6',
+ it('should allow strings for port', async function() {
+ // fixes #4391 - This allows the app to be run using iisnode, which uses a named pipe for the port.
+ const pipeServer = await createTestServer({
+ public: {
+ bind: {
+ port: '\\\\.\\pipe\\9c137306-7c4d-461e-b7cf-5213a3939ad6',
+ },
+ },
+ })
+ expect(pipeServer).to.not.be.undefined
})
- expect(pipeServer).to.not.be.undefined
- })
- it('should produce colorscheme badges', async function() {
- const { statusCode, body } = await got(`${baseUrl}:fruit-apple-green.svg`)
- expect(statusCode).to.equal(200)
- expect(body)
- .to.satisfy(isSvg)
- .and.to.include('fruit')
- .and.to.include('apple')
- })
+ it('should produce colorscheme badges', async function() {
+ const { statusCode, body } = await got(`${baseUrl}:fruit-apple-green.svg`)
+ expect(statusCode).to.equal(200)
+ expect(body)
+ .to.satisfy(isSvg)
+ .and.to.include('fruit')
+ .and.to.include('apple')
+ })
- it('should redirect colorscheme PNG badges as configured', async function() {
- const { statusCode, headers } = await got(
- `${baseUrl}:fruit-apple-green.png`,
- {
+ it('should redirect colorscheme PNG badges as configured', async function() {
+ const { statusCode, headers } = await got(
+ `${baseUrl}:fruit-apple-green.png`,
+ {
+ followRedirect: false,
+ }
+ )
+ expect(statusCode).to.equal(301)
+ expect(headers.location).to.equal(
+ 'http://raster.example.test/:fruit-apple-green.png'
+ )
+ })
+
+ it('should redirect modern PNG badges as configured', async function() {
+ const { statusCode, headers } = await got(`${baseUrl}npm/v/express.png`, {
followRedirect: false,
- }
- )
- expect(statusCode).to.equal(301)
- expect(headers.location).to.equal(
- 'http://raster.example.test/:fruit-apple-green.png'
- )
- })
+ })
+ expect(statusCode).to.equal(301)
+ expect(headers.location).to.equal(
+ 'http://raster.example.test/npm/v/express.png'
+ )
+ })
- it('should redirect modern PNG badges as configured', async function() {
- const { statusCode, headers } = await got(`${baseUrl}npm/v/express.png`, {
- followRedirect: false,
+ it('should produce json badges', async function() {
+ const { statusCode, body, headers } = await got(
+ `${baseUrl}twitter/follow/_Pyves.json`
+ )
+ expect(statusCode).to.equal(200)
+ expect(headers['content-type']).to.equal('application/json')
+ expect(() => JSON.parse(body)).not.to.throw()
})
- expect(statusCode).to.equal(301)
- expect(headers.location).to.equal(
- 'http://raster.example.test/npm/v/express.png'
- )
- })
- it('should produce json badges', async function() {
- const { statusCode, body, headers } = await got(
- `${baseUrl}twitter/follow/_Pyves.json`
- )
- expect(statusCode).to.equal(200)
- expect(headers['content-type']).to.equal('application/json')
- expect(() => JSON.parse(body)).not.to.throw()
- })
+ it('should preserve label case', async function() {
+ const { statusCode, body } = await got(`${baseUrl}:fRuiT-apple-green.svg`)
+ expect(statusCode).to.equal(200)
+ expect(body)
+ .to.satisfy(isSvg)
+ .and.to.include('fRuiT')
+ })
- it('should preserve label case', async function() {
- const { statusCode, body } = await got(`${baseUrl}:fRuiT-apple-green.svg`)
- expect(statusCode).to.equal(200)
- expect(body)
- .to.satisfy(isSvg)
- .and.to.include('fRuiT')
- })
+ // https://github.com/badges/shields/pull/1319
+ it('should not crash with a numeric logo', async function() {
+ const { statusCode, body } = await got(
+ `${baseUrl}:fruit-apple-green.svg?logo=1`
+ )
+ expect(statusCode).to.equal(200)
+ expect(body)
+ .to.satisfy(isSvg)
+ .and.to.include('fruit')
+ .and.to.include('apple')
+ })
- // https://github.com/badges/shields/pull/1319
- it('should not crash with a numeric logo', async function() {
- const { statusCode, body } = await got(
- `${baseUrl}:fruit-apple-green.svg?logo=1`
- )
- expect(statusCode).to.equal(200)
- expect(body)
- .to.satisfy(isSvg)
- .and.to.include('fruit')
- .and.to.include('apple')
- })
+ it('should not crash with a numeric link', async function() {
+ const { statusCode, body } = await got(
+ `${baseUrl}:fruit-apple-green.svg?link=1`
+ )
+ expect(statusCode).to.equal(200)
+ expect(body)
+ .to.satisfy(isSvg)
+ .and.to.include('fruit')
+ .and.to.include('apple')
+ })
- it('should not crash with a numeric link', async function() {
- const { statusCode, body } = await got(
- `${baseUrl}:fruit-apple-green.svg?link=1`
- )
- expect(statusCode).to.equal(200)
- expect(body)
- .to.satisfy(isSvg)
- .and.to.include('fruit')
- .and.to.include('apple')
- })
+ it('should not crash with a boolean link', async function() {
+ const { statusCode, body } = await got(
+ `${baseUrl}:fruit-apple-green.svg?link=true`
+ )
+ expect(statusCode).to.equal(200)
+ expect(body)
+ .to.satisfy(isSvg)
+ .and.to.include('fruit')
+ .and.to.include('apple')
+ })
- it('should not crash with a boolean link', async function() {
- const { statusCode, body } = await got(
- `${baseUrl}:fruit-apple-green.svg?link=true`
- )
- expect(statusCode).to.equal(200)
- expect(body)
- .to.satisfy(isSvg)
- .and.to.include('fruit')
- .and.to.include('apple')
- })
+ it('should return the 404 badge for unknown badges', async function() {
+ const { statusCode, body } = await got(
+ `${baseUrl}this/is/not/a/badge.svg`,
+ {
+ throwHttpErrors: false,
+ }
+ )
+ expect(statusCode).to.equal(404)
+ expect(body)
+ .to.satisfy(isSvg)
+ .and.to.include('404')
+ .and.to.include('badge not found')
+ })
- it('should return the 404 badge for unknown badges', async function() {
- const { statusCode, body } = await got(
- `${baseUrl}this/is/not/a/badge.svg`,
- { throwHttpErrors: false }
- )
- expect(statusCode).to.equal(404)
- expect(body)
- .to.satisfy(isSvg)
- .and.to.include('404')
- .and.to.include('badge not found')
- })
+ it('should return the 404 badge page for rando links', async function() {
+ const { statusCode, body } = await got(
+ `${baseUrl}this/is/most/definitely/not/a/badge.js`,
+ {
+ throwHttpErrors: false,
+ }
+ )
+ expect(statusCode).to.equal(404)
+ expect(body)
+ .to.satisfy(isSvg)
+ .and.to.include('404')
+ .and.to.include('badge not found')
+ })
- it('should return the 404 badge page for rando links', async function() {
- const { statusCode, body } = await got(
- `${baseUrl}this/is/most/definitely/not/a/badge.js`,
- {
+ it('should redirect the root as configured', async function() {
+ const { statusCode, headers } = await got(baseUrl, {
+ followRedirect: false,
+ })
+
+ expect(statusCode).to.equal(302)
+ // This value is set in `config/test.yml`
+ expect(headers.location).to.equal('http://frontend.example.test')
+ })
+
+ it('should return the 410 badge for obsolete formats', async function() {
+ const { statusCode, body } = await got(`${baseUrl}npm/v/express.jpg`, {
throwHttpErrors: false,
- }
- )
- expect(statusCode).to.equal(404)
- expect(body)
- .to.satisfy(isSvg)
- .and.to.include('404')
- .and.to.include('badge not found')
+ })
+ // TODO It would be nice if this were 404 or 410.
+ expect(statusCode).to.equal(200)
+ expect(body)
+ .to.satisfy(isSvg)
+ .and.to.include('410')
+ .and.to.include('jpg no longer available')
+ })
})
- it('should redirect the root as configured', async function() {
- const { statusCode, headers } = await got(baseUrl, {
- followRedirect: false,
+ describe('configuration', function() {
+ let server
+ afterEach(async function() {
+ if (server) {
+ server.stop()
+ }
+ })
+
+ it('should allow to enable prometheus metrics', async function() {
+ // Fixes https://github.com/badges/shields/issues/2611
+ this.timeout(10000)
+ server = await createTestServer({
+ public: {
+ metrics: { prometheus: { enabled: true, endpointEnabled: true } },
+ },
+ })
+ await server.start()
+
+ const { statusCode } = await got(`${server.baseUrl}metrics`)
+
+ expect(statusCode).to.be.equal(200)
})
- expect(statusCode).to.equal(302)
- // This value is set in `config/test.yml`
- expect(headers.location).to.equal('http://frontend.example.test')
+ it('should allow to disable prometheus metrics', async function() {
+ // Fixes https://github.com/badges/shields/issues/2611
+ this.timeout(10000)
+ server = await createTestServer({
+ public: {
+ metrics: { prometheus: { enabled: true, endpointEnabled: false } },
+ },
+ })
+ await server.start()
+
+ const { statusCode } = await got(`${server.baseUrl}metrics`, {
+ throwHttpErrors: false,
+ })
+
+ expect(statusCode).to.be.equal(404)
+ })
})
- it('should return the 410 badge for obsolete formats', async function() {
- const { statusCode, body } = await got(`${baseUrl}npm/v/express.jpg`, {
- throwHttpErrors: false,
+ describe('configuration validation', function() {
+ describe('influx', function() {
+ let customConfig
+ beforeEach(function() {
+ customConfig = config.util.toObject()
+ customConfig.public.metrics.influx = {
+ enabled: true,
+ url: 'http://localhost:8081/telegraf',
+ timeoutMilliseconds: 1000,
+ intervalSeconds: 2,
+ instanceIdFrom: 'random',
+ instanceIdEnvVarName: 'INSTANCE_ID',
+ hostnameAliases: { 'metrics-hostname': 'metrics-hostname-alias' },
+ envLabel: 'test-env',
+ }
+ customConfig.private = {
+ influx_username: 'telegraf',
+ influx_password: 'telegrafpass',
+ }
+ })
+
+ it('should not require influx configuration', function() {
+ delete customConfig.public.metrics.influx
+ expect(() => new Server(config.util.toObject())).to.not.throw()
+ })
+
+ it('should require url when influx configuration is enabled', function() {
+ delete customConfig.public.metrics.influx.url
+ expect(() => new Server(customConfig)).to.throw(
+ '"metrics.influx.url" is required'
+ )
+ })
+
+ it('should not require url when influx configuration is disabled', function() {
+ customConfig.public.metrics.influx.enabled = false
+ delete customConfig.public.metrics.influx.url
+ expect(() => new Server(customConfig)).to.not.throw()
+ })
+
+ it('should require timeoutMilliseconds when influx configuration is enabled', function() {
+ delete customConfig.public.metrics.influx.timeoutMilliseconds
+ expect(() => new Server(customConfig)).to.throw(
+ '"metrics.influx.timeoutMilliseconds" is required'
+ )
+ })
+
+ it('should require intervalSeconds when influx configuration is enabled', function() {
+ delete customConfig.public.metrics.influx.intervalSeconds
+ expect(() => new Server(customConfig)).to.throw(
+ '"metrics.influx.intervalSeconds" is required'
+ )
+ })
+
+ it('should require instanceIdFrom when influx configuration is enabled', function() {
+ delete customConfig.public.metrics.influx.instanceIdFrom
+ expect(() => new Server(customConfig)).to.throw(
+ '"metrics.influx.instanceIdFrom" is required'
+ )
+ })
+
+ it('should require instanceIdEnvVarName when instanceIdFrom is env-var', function() {
+ customConfig.public.metrics.influx.instanceIdFrom = 'env-var'
+ delete customConfig.public.metrics.influx.instanceIdEnvVarName
+ expect(() => new Server(customConfig)).to.throw(
+ '"metrics.influx.instanceIdEnvVarName" is required'
+ )
+ })
+
+ it('should allow instanceIdFrom = hostname', function() {
+ customConfig.public.metrics.influx.instanceIdFrom = 'hostname'
+ expect(() => new Server(customConfig)).to.not.throw()
+ })
+
+ it('should allow instanceIdFrom = env-var', function() {
+ customConfig.public.metrics.influx.instanceIdFrom = 'env-var'
+ expect(() => new Server(customConfig)).to.not.throw()
+ })
+
+ it('should allow instanceIdFrom = random', function() {
+ customConfig.public.metrics.influx.instanceIdFrom = 'random'
+ expect(() => new Server(customConfig)).to.not.throw()
+ })
+
+ it('should require envLabel when influx configuration is enabled', function() {
+ delete customConfig.public.metrics.influx.envLabel
+ expect(() => new Server(customConfig)).to.throw(
+ '"metrics.influx.envLabel" is required'
+ )
+ })
+
+ it('should not require hostnameAliases', function() {
+ delete customConfig.public.metrics.influx.hostnameAliases
+ expect(() => new Server(customConfig)).to.not.throw()
+ })
+
+ it('should allow empty hostnameAliases', function() {
+ customConfig.public.metrics.influx.hostnameAliases = {}
+ expect(() => new Server(customConfig)).to.not.throw()
+ })
+
+ it('should require username when influx configuration is enabled', function() {
+ delete customConfig.private.influx_username
+ expect(() => new Server(customConfig)).to.throw(
+ 'Private configuration is invalid. Check these paths: influx_username'
+ )
+ })
+
+ it('should require password when influx configuration is enabled', function() {
+ delete customConfig.private.influx_password
+ expect(() => new Server(customConfig)).to.throw(
+ 'Private configuration is invalid. Check these paths: influx_password'
+ )
+ })
+
+ it('should allow other private keys', function() {
+ customConfig.private.gh_token = 'my-token'
+ expect(() => new Server(customConfig)).to.not.throw()
+ })
})
- // TODO It would be nice if this were 404 or 410.
- expect(statusCode).to.equal(200)
- expect(body)
- .to.satisfy(isSvg)
- .and.to.include('410')
- .and.to.include('jpg no longer available')
})
})
diff --git a/core/service-test-runner/cli.js b/core/service-test-runner/cli.js
index 1b7979e15fa9080b1ae6416c0658c582353c54e0..03e789731f0f93bb3563f80e3f2975074b7bffda 100644
--- a/core/service-test-runner/cli.js
+++ b/core/service-test-runner/cli.js
@@ -73,8 +73,14 @@ if (process.env.TESTED_SERVER_URL) {
} else {
const port = 1111
baseUrl = 'http://localhost:1111'
- before('Start running the server', function() {
- server = createTestServer({ port })
+ before('Start running the server', async function() {
+ server = await createTestServer({
+ public: {
+ bind: {
+ port,
+ },
+ },
+ })
server.start()
})
after('Shut down the server', async function() {
diff --git a/package-lock.json b/package-lock.json
index 07dc250e2d22e62b8a3001876083c75cfcc60645..35757ce458b8ed055e265b6d2a2e3972ef72dc9a 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -9905,6 +9905,12 @@
"resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz",
"integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ="
},
+ "deepmerge": {
+ "version": "4.2.2",
+ "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz",
+ "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==",
+ "dev": true
+ },
"default-gateway": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/default-gateway/-/default-gateway-4.2.0.tgz",
@@ -20171,8 +20177,7 @@
"lodash.groupby": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz",
- "integrity": "sha1-Cwih3PaDl8OXhVwyOXg4Mt90A9E=",
- "dev": true
+ "integrity": "sha1-Cwih3PaDl8OXhVwyOXg4Mt90A9E="
},
"lodash.includes": {
"version": "4.3.0",
diff --git a/package.json b/package.json
index e258411c1445274ebf267ad0c7b06ee864d8498f..1774a857a3b5acf496b74b2cbcb664dffebebaa9 100644
--- a/package.json
+++ b/package.json
@@ -48,6 +48,7 @@
"js-yaml": "^3.13.1",
"jsonpath": "~1.0.2",
"lodash.countby": "^4.6.0",
+ "lodash.groupby": "^4.6.0",
"lodash.times": "^4.3.2",
"moment": "^2.24.0",
"node-env-flag": "^0.1.0",
@@ -173,6 +174,7 @@
"cypress": "^4.4.0",
"danger": "^10.1.1",
"danger-plugin-no-test-shortcuts": "^2.0.0",
+ "deepmerge": "^4.2.2",
"enzyme": "^3.11.0",
"enzyme-adapter-react-16": "^1.15.2",
"eslint": "^6.8.0",
@@ -212,7 +214,6 @@
"lint-staged": "^9.5.0",
"lodash.debounce": "^4.0.8",
"lodash.difference": "^4.5.0",
- "lodash.groupby": "^4.6.0",
"minimist": "^1.2.5",
"mocha": "^6.2.3",
"mocha-env-reporter": "^4.0.0",