An option to retry a failed service; test on [dynamicxml] (#4166)

* An option to retry a failed service test

* Convert retry options to integers

* Info about the unit

* JSDoc for retry configuraion

core/service-test-runner/cli.js

@@ -15,6 +15,11 @@
 // Run tests on a given instance:
 //   SKIP_INTERCEPTED=TRUE TESTED_SERVER_URL=https://test.shields.io npm run test:services --
 //
+// Run tests with given number of retries and backoff (in milliseconds):
+//   RETRY_COUNT=3 RETRY_BACKOFF=100 npm run test:services --
+// Retry option documentation:
+// https://github.com/IcedFrisby/IcedFrisby/blob/master/API.md#retrycount-backoff
+//
 // Service tests are run in CI in two cases: scheduled builds and pull
 // requests. The scheduled builds run _all_ the service tests, whereas the
 // pull requests run service tests designated in the PR title. In this way,
@@ -59,6 +64,9 @@ const Runner = require('./runner')
 
 require('../unhandled-rejection.spec')
 
+const retry = {}
+retry.count = parseInt(process.env.RETRY_COUNT) || 0
+retry.backoff = parseInt(process.env.RETRY_BACKOFF) || 0
 let baseUrl, server
 if (process.env.TESTED_SERVER_URL) {
   baseUrl = process.env.TESTED_SERVER_URL
@@ -77,7 +85,7 @@ if (process.env.TESTED_SERVER_URL) {
 }
 
 const skipIntercepted = envFlag(process.env.SKIP_INTERCEPTED, false)
-const runner = new Runner({ baseUrl, skipIntercepted })
+const runner = new Runner({ baseUrl, skipIntercepted, retry })
 runner.prepare()
 
 // The server's request cache causes side effects between tests.

core/service-test-runner/runner.js

@@ -9,9 +9,10 @@ const { loadTesters } = require('../base-service/loader')
  * Load a collection of ServiceTester objects and register them with Mocha.
  */
 class Runner {
-  constructor({ baseUrl, skipIntercepted }) {
+  constructor({ baseUrl, skipIntercepted, retry }) {
     this.baseUrl = baseUrl
     this.skipIntercepted = skipIntercepted
+    this.retry = retry
   }
 
   /**
@@ -67,8 +68,8 @@ class Runner {
    * Register the tests with Mocha.
    */
   toss() {
-    const { testers, baseUrl, skipIntercepted } = this
-    testers.forEach(tester => tester.toss({ baseUrl, skipIntercepted }))
+    const { testers, baseUrl, skipIntercepted, retry } = this
+    testers.forEach(tester => tester.toss({ baseUrl, skipIntercepted, retry }))
   }
 }
 module.exports = Runner

core/service-test-runner/service-tester.js

@@ -115,8 +115,11 @@ class ServiceTester {
    * @param {object} attrs Refer to individual attrs
    * @param {string} attrs.baseUrl base URL for test server
    * @param {boolean} attrs.skipIntercepted skip tests which intercept requests
+   * @param {object} attrs.retry retry configuration
+   * @param {number} attrs.retry.count number of times to retry test
+   * @param {number} attrs.retry.backoff number of milliseconds to add to the wait between each retry
    */
-  toss({ baseUrl, skipIntercepted }) {
+  toss({ baseUrl, skipIntercepted, retry }) {
     const { specs, pathPrefix } = this
     const testerBaseUrl = `${baseUrl}${pathPrefix}`
 
@@ -129,6 +132,7 @@ class ServiceTester {
         }`
         if (!skipIntercepted || !spec.intercepted) {
           spec.baseUri(testerBaseUrl)
+          spec.retry(retry.count, retry.backoff)
           spec.toss()
         }
       })