Rewrite the test runner to work with Jest 27

.eslintignore

@@ -5,4 +5,12 @@ node_modules/*
 production_node_modules/*
 test/fixtures/*
 tmp/*
-jest.config.js
+
+/babel.config.js
+/jest.config.js
+/jest.runner.config.js
+
+/.eslintrc
+/.eslintrc.*
+/test/.eslintrc
+/test/.eslintrc.*

.eslintrc.js

@@ -0,0 +1,8 @@
+module.exports = {
+  root: true,
+  parserOptions: {
+    tsconfigRootDir: __dirname,
+    project: ['./tsconfig.json'],
+  },
+  extends: '@exercism/eslint-config-tooling',
+}

.github/workflows/ci.js.yml

@@ -13,10 +13,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - name: Use Node.js LTS (14.x)
+      - name: Use Node.js LTS (16.x)
         uses: actions/setup-node@v1
         with:
-          node-version: 14.x
+          node-version: 16.x
 
       - name: Install project dependencies
         run: yarn install --frozen-lockfile --ignore-scripts
@@ -29,7 +29,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12.x, 14.x, 15.x]
+        node-version: [16.x]
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/format-code.yml

@@ -62,8 +62,6 @@ jobs:
 
       - name: 'Format code'
         run: ./bin/format.sh
-        env:
-          EXERCISM_PRETTIER_VERSION: '2.2.1'
 
       - name: 'Commit formatted code'
         run: |

.github/workflows/pr.ci.js.yml

@@ -13,10 +13,10 @@ jobs:
       - name: Checkout PR
         uses: actions/checkout@v2
 
-      - name: Use Node.js LTS (14.x)
+      - name: Use Node.js LTS (16.x)
         uses: actions/setup-node@v1
         with:
-          node-version: 14.x
+          node-version: 16.x
 
       - name: Install project dependencies
         run: yarn install --frozen-lockfile --ignore-scripts
@@ -29,7 +29,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12.x, 14.x, 15.x]
+        node-version: [16.x]
 
     steps:
       - name: Checkout PR

.github/workflows/verify-code-formatting.yml

@@ -14,5 +14,3 @@ jobs:
 
       - name: 'Verify formatting of all files'
         run: ./bin/check-formatting.sh
-        env:
-          EXERCISM_PRETTIER_VERSION: '2.2.1'

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 3.0.0
+
+- Rewrite for Jest 27
+- Sanitize jest reported syntax errors
+
 ## 2.5.1
 
 - Fix skipped tests showing up as failed despite passed complete status.

Dockerfile

@@ -1,6 +1,6 @@
-FROM node:erbium-buster-slim as runner
-# Node.js v12 LTS (Erbium)
-# Debian Buster (v10.4)
+FROM node:16-bullseye-slim as runner
+# Node.js 16 (curently LTS)
+# Debian bullseye
 
 # fetch latest security updates
 RUN set -ex; \

README.md

@@ -4,21 +4,21 @@
 
 The Docker image for automatically run tests on JavaScript solutions submitted to [exercism][web-exercism].
 
-> At this moment, the input path _must_ be relative to the `package.json` of this respository.
+> At this moment, the input path _must_ be relative to the `package.json` of this repository.
 > `jest` doesn't like running outside of its tree.
 > This might change in the future.
 
 ## Installation
 
 Clone this repository and then run:
 
-```bash
+```shell
 yarn install
 ```
 
 You'll need at least Node LTS for this to work.
 
-```
+```shell
 yarn build
 ```
 
@@ -48,11 +48,15 @@ For example:
 ```shell
 $ ./bin/run.sh two-fer ./test/fixtures/two-fer/pass
 
-PASS  test/fixtures/two-fer/pass/two-fer.spec.js
-Test Suites: 1 passed, 1 total
-Tests:       3 passed, 3 total
-Snapshots:   0 total
-Time:        2.817s
+Using reporter : **/dist/reporter.js
+Using test-root: **/test/fixtures/two-fer/pass/
+Using base-root: **/
+Using setup-env: **/dist/jest/setup.js
+
+test/fixtures/two-fer/pass/ matches test/fixtures/two-fer/pass/. Not copying anything.
+Using test/fixtures/two-fer/pass/.meta/config.json as base configuration
+Enabling tests in test/fixtures/two-fer/pass/two-fer.spec.js
+Determining test suites to run...
 Find the output at:
 test/fixtures/two-fer/pass/results.json
 ```
@@ -76,11 +80,10 @@ Exercism remote UUID: a7d1b71693fb4298a3a99bd352dd4d74
 
 Downloaded to
 C:\Users\Derk-Jan\Exercism\javascript\clock
-PASS  tmp/clock/a7d1b71693fb4298a3a99bd352dd4d74/clock/clock.spec.js
-Test Suites: 1 passed, 1 total
-Tests:       52 passed, 52 total
-Snapshots:   0 total
-Time:        2.987s
+
+...
+
+Determining test suites to run...
 Find the output at:
 ./tmp/clock/a7d1b71693fb4298a3a99bd352dd4d74/clock/results.json
 ```

babel.config.js

@@ -1,13 +1,6 @@
+// This file is only used by jest.runner.config.js, when running the
+// test-runner. The tool itself uses typescript's compilation instead.
 module.exports = {
-  presets: [
-    [
-      '@babel/env',
-      {
-        targets: {
-          node: 'current',
-        },
-        useBuiltIns: false,
-      },
-    ],
-  ],
+  presets: ['@exercism/babel-preset-javascript'],
+  plugins: [],
 }

bin/run.sh

@@ -80,6 +80,7 @@ set -euo pipefail
 ROOT="$(realpath $(dirname "$0")/..)"
 REPORTER="$ROOT/dist/reporter.js"
 SETUP="$ROOT/dist/jest/setup.js"
+CONFIG="$ROOT/jest.runner.config.js"
 
 if test -f "$REPORTER"; then
   echo "Using reporter : $REPORTER"
@@ -132,17 +133,21 @@ set +e
 
 # Run tests
 "$ROOT/node_modules/.bin/jest" "${OUTPUT}*" \
+                               --bail 1 \
+                               --ci \
+                               --colors \
+                               --config ${CONFIG} \
+                               --noStackTrace \
                                --outputFile="${result_file}" \
+                               --passWithNoTests \
                                --reporters "${REPORTER}" \
-                               --noStackTrace \
-                               --verbose=false \
                                --roots "${OUTPUT}" \
-                               --passWithNoTests \
-                               --ci \
-                               --runInBand \
-                               --bail 1 \
-                               --setupFilesAfterEnv ${SETUP}
+                               --setupFilesAfterEnv ${SETUP} \
+                               --verbose false \
+                               --testLocationInResults
+
 
+# --runInBand \
 # Convert exit(1) (jest worked, but there are failing tests) to exit(0)
 test_exit=$?
 

jest.config.js

@@ -1,7 +1,5 @@
+/** @type {import('ts-jest/dist/types').InitialOptionsTsJest} */
 module.exports = {
-  verbose: true,
-  modulePathIgnorePatterns: ['package.json'],
-  transform: {
-    '^.+\\.[t|j]sx?$': 'babel-jest',
-  },
+  preset: 'ts-jest',
+  testEnvironment: 'node',
 }

jest.runner.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+  verbose: true,
+  modulePathIgnorePatterns: ['package.json'],
+  transform: {
+    '^.+\\.[t|j]sx?$': 'babel-jest',
+  },
+  reporters: [],
+}

package.json

@@ -2,7 +2,7 @@
   "name": "@exercism/javascript-test-runner",
   "description": "Automated Test runner for exercism solutions in Javascript.",
   "author": "Derk-Jan Karrenbeld <[email protected]>",
-  "version": "2.5.1",
+  "version": "3.0.0",
   "license": "AGPL-3.0-or-later",
   "repository": {
     "type": "git",
@@ -24,38 +24,37 @@
     "watch": "yarn build -w",
     "prepare": "yarn build",
     "prepublishOnly": "yarn test:bare && yarn lint",
-    "lint": "yarn eslint . --ext ts,js,tsx,jsx,mjs -c .eslintrc",
+    "lint": "yarn eslint src --ext ts,js,tsx,jsx,mjs -c .eslintrc.js && yarn eslint test --ext ts,js,tsx,jsx,mjs -c test/.eslintrc.js",
     "test": "yarn build && yarn test:bare",
     "test:bare": "jest --roots test --testPathIgnorePatterns=\"fixtures/\""
   },
   "dependencies": {
-    "@babel/cli": "^7.15.4",
-    "@babel/core": "^7.15.5",
-    "@babel/node": "^7.15.4",
-    "@babel/preset-env": "^7.15.4",
-    "@babel/preset-typescript": "^7.15.0",
-    "@exercism/static-analysis": "^0.10.0",
-    "@typescript-eslint/typescript-estree": "^4.30.0",
-    "@typescript-eslint/visitor-keys": "^4.30.0",
-    "babel-jest": "^26.6.3",
-    "chalk": "^4.1.2",
-    "jest": "^26.6.3",
-    "jest-util": "^26.6.2",
-    "slash": "^3.0.0",
-    "string-length": "^4.0.2"
+    "@babel/core": "^7.17.0",
+    "@babel/node": "^7.16.8",
+    "@babel/preset-env": "^7.16.11",
+    "@babel/preset-typescript": "^7.16.7",
+    "@exercism/babel-preset-javascript": "^0.1.2",
+    "@exercism/static-analysis": "^0.11.0",
+    "@typescript-eslint/typescript-estree": "^5.10.2",
+    "@typescript-eslint/visitor-keys": "^5.10.2",
+    "babel-jest": "^27.5.0",
+    "chalk": "^5.0.0",
+    "jest": "^27.5.0"
   },
   "devDependencies": {
-    "@types/jest": "^26.0.24",
-    "@types/node": "^14.17.14",
-    "@typescript-eslint/eslint-plugin": "^4.30.0",
-    "@typescript-eslint/parser": "^4.30.0",
-    "babel-eslint": "^10.1.0",
-    "eslint": "^7.32.0",
+    "@exercism/eslint-config-tooling": "^0.4.0",
+    "@tsconfig/node16": "^1.0.2",
+    "@types/jest": "^27.4.0",
+    "@types/node": "^16.11.22",
+    "@typescript-eslint/eslint-plugin": "^5.10.2",
+    "@typescript-eslint/parser": "^5.10.2",
+    "eslint": "^8.8.0",
     "eslint-config-prettier": "^8.3.0",
-    "eslint-plugin-import": "^2.24.2",
-    "eslint-plugin-jest": "^24.4.0",
-    "prettier": "^2.3.2",
+    "eslint-plugin-import": "^2.25.4",
+    "eslint-plugin-jest": "^26.0.0",
+    "prettier": "^2.5.1",
     "rimraf": "^3.0.2",
-    "typescript": "^4.4.2"
+    "ts-jest": "^27.1.3",
+    "typescript": "^4.5.5"
   }
 }

src/jest/setup.ts

@@ -1,7 +1,7 @@
 // import spyConsole from './console'
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-const originalDescribe = (jasmine as any).getEnv().describe
+/*const originalDescribe = (jasmine as any).getEnv().describe
 
 // eslint-disable-next-line @typescript-eslint/explicit-function-return-type, @typescript-eslint/no-explicit-any
 ;(jasmine as any).getEnv().describe = <T extends unknown[] = any[]>(
@@ -27,3 +27,4 @@ const originalDescribe = (jasmine as any).getEnv().describe
 
   return originalDescribe(description, spiedSpecDefinition, ...describeArgs)
 }
+*/

src/output.ts

@@ -1,18 +1,14 @@
-import { ConsoleBuffer } from '@jest/console'
-import {
+import { AstParser, extractTests } from '@exercism/static-analysis'
+import type { ExtractedTestCase, ParsedSource } from '@exercism/static-analysis'
+import type { ConsoleBuffer } from '@jest/console'
+import type {
   AggregatedResult,
   AssertionResult,
   TestResult,
 } from '@jest/test-result'
-import { Config } from '@jest/types'
+import type { Config } from '@jest/types'
 import fs from 'fs'
 import path from 'path'
-import {
-  AstParser,
-  ExtractedTestCase,
-  extractTests,
-  ParsedSource,
-} from '@exercism/static-analysis'
 
 interface OutputInterface {
   status: 'fail' | 'pass' | 'error'
@@ -25,6 +21,7 @@ interface OutputTestInterface {
   status: 'fail' | 'pass' | 'error'
   message: string
   output: string | null
+  // eslint-disable-next-line @typescript-eslint/naming-convention
   test_code: string
 }
 
@@ -33,12 +30,17 @@ export class Output {
   private results: Partial<OutputInterface> & Pick<OutputInterface, 'tests'>
   private readonly globalConfig: Config.GlobalConfig
   private readonly outputFile: string
+  private readonly sources: Record<string, ParsedSource>
+  private readonly tests: Record<string, ReturnType<typeof extractTests>>
 
   constructor(globalConfig: Config.GlobalConfig) {
     this.globalConfig = globalConfig
     this.results = { tests: [] }
     this.outputFile =
-      this.globalConfig.outputFile || path.join(process.cwd(), 'results.json')
+      this.globalConfig.outputFile ?? path.join(process.cwd(), 'results.json')
+
+    this.sources = {}
+    this.tests = {}
   }
 
   public error(message: string): void {
@@ -98,13 +100,16 @@ export class Output {
           parsedSources[file] = {
             program,
             source,
-            tests: tests.reduce((results, item) => {
-              results[item.name(' > ')] = item
-              results[item.name(' ' as ' > ')] = item
-              return results
-            }, {} as Record<string, ExtractedTestCase>),
+            tests: tests.reduce<Record<string, ExtractedTestCase>>(
+              (results, item) => {
+                results[item.name(' > ')] = item
+                results[item.name(' ' as ' > ')] = item
+                return results
+              },
+              {}
+            ),
           }
-        } catch (err) {
+        } catch (err: unknown) {
           console.error(
             `When trying to parse ${file}, the following error occurred`,
             err
@@ -116,14 +121,17 @@ export class Output {
     const tests = this.results.tests.map((test) => {
       const parsedSource = parsedSources[test.test_code]
       if (!parsedSource) {
+        // eslint-disable-next-line @typescript-eslint/naming-convention
         return { ...test, test_code: null }
       }
 
       const testCase = parsedSource.tests[test.name]
       if (!testCase) {
+        // eslint-disable-next-line @typescript-eslint/naming-convention
         return { ...test, test_code: null }
       }
 
+      // eslint-disable-next-line @typescript-eslint/naming-convention
       return { ...test, test_code: testCase.testCode(parsedSource.source) }
     })
 
@@ -148,7 +156,7 @@ export class Output {
       this.error(
         sanitizeErrorMessage(
           specFilePath,
-          testResult.failureMessage ||
+          testResult.failureMessage ??
             'Something went wrong when running the tests.'
         )
       )
@@ -157,41 +165,40 @@ export class Output {
 
     // Suites ran fine. Output normally.
     results.testResults.forEach((testResult) => {
-      return this.testInnerFinished(
-        specFilePath,
-        testResult,
-        testResult.testResults
-      )
+      this.testInnerFinished(specFilePath, testResult, testResult.testResults)
     })
   }
 
+  private getSource(specFilePath: string): {
+    source: ParsedSource
+    tests: ExtractedTestCase[]
+  } {
+    if (!this.sources[specFilePath]) {
+      const [{ program, source }] = AstParser.ANALYZER.parseSync(
+        fs.readFileSync(specFilePath).toString()
+      )
+      this.tests[specFilePath] = extractTests(program)
+      this.sources[specFilePath] = { program, source }
+    }
+
+    return {
+      source: this.sources[specFilePath],
+      tests: this.tests[specFilePath],
+    }
+  }
+
   public testInnerFinished(
     specFilePath: string,
     testResult: TestResult,
     innerResults: AssertionResult[]
   ): void {
-    if (testResult.console) {
-      /*
-      // The code below works, but is not accepted by the current runner spec on exercism
-      const name = [
-        typeof testResult.displayName === 'string' && testResult.displayName,
-        typeof testResult.displayName === 'object' && testResult.displayName && testResult.displayName.name,
-        innerResults[0] && innerResults[0].ancestorTitles.join(' > '),
-        trimAndFormatPath(0, this.globalConfig, path, 80)
-      ].filter(Boolean)[0] as string
-
-      this.results.tests.push({
-        name: name,
-        output: buildOutput(testResult.console),
-        status: testResult.numPendingTests === 0 && testResult.numFailingTests === 0 ? 'pass' : 'fail',
-        message: ''
-      })
-      */
-    }
-
-    const consoleOutputs = testResult.console
-      ? buildOutput(specFilePath, testResult.console)
-      : ({} as Record<string, string>)
+    const consoleOutputs: Record<string, string> = testResult.console
+      ? buildOutput(
+          specFilePath,
+          testResult.console,
+          this.getSource(specFilePath)
+        )
+      : {}
 
     const outputs = buildTestOutput(specFilePath, testResult, innerResults)
     const firstFailureIndex = outputs.findIndex(
@@ -203,15 +210,16 @@ export class Output {
         const isFirstFailure =
           firstFailureIndex === i ||
           (firstFailureIndex === -1 && self.length === i + 1)
-        const outputMessage =
-          consoleOutputs[withoutOutput.name.replace(/ > /g, ' ')] || null
+
+        const outputMessage = consoleOutputs[withoutOutput.name] || null
 
         return {
           ...withoutOutput,
           output: isFirstFailure
             ? [consoleOutputs[''], outputMessage].filter(Boolean).join('\n') ||
               null
             : outputMessage,
+          // eslint-disable-next-line @typescript-eslint/naming-convention
           test_code: specFilePath,
         }
       })
@@ -225,31 +233,83 @@ export class Output {
 
 function buildOutput(
   specFilePath: string,
-  buffer: ConsoleBuffer
+  buffer: ConsoleBuffer,
+  { tests }: { source: ParsedSource; tests: ExtractedTestCase[] }
 ): Record<string, string> {
-  const [, outputs] = buffer.reduce(
-    ([lastTest, messages], entry) => {
-      // Change current test messages
-      if (entry.message.startsWith('@exercism/javascript:')) {
-        return [
-          entry.message.slice('@exercism/javascript:'.length).trim(),
-          messages,
-        ] as const
+  const outputs = buffer.reduce<Record<string, string[]>>((outputs, entry) => {
+    // The origin stack trace will look something like this:
+    //
+    //   at twoFer (<specFile>:4:11)
+    //   at Object.<anonymous> (<specFilePath>:5:12)
+    //
+    // We want to extract the :5:12) part, convert it to a proper line, column
+    // and then look in the test file which test is "around" it. This only
+    // works if the test file looks like this:
+    //
+    // test('my-test', () => {
+    //   ...
+    //   codeThatEventuallyResultsInLog();
+    //   ...
+    // })
+
+    let foundTest: ExtractedTestCase | undefined
+    // Using find because it will break when something is found!
+
+    entry.origin.split('\n').find((originLine) => {
+      if (!originLine.includes(specFilePath)) {
+        return undefined
       }
 
-      const sanitized = `[${entry.type}] ${sanitizeErrorMessage(
-        specFilePath,
-        entry.message
-      )}`
-      messages[lastTest] = messages[lastTest] || []
-      messages[lastTest].push(sanitized)
+      const [control, line, column] = originLine
+        .slice(originLine.indexOf(specFilePath) + specFilePath.length)
+        .split(':')
 
-      return [lastTest, messages] as const
-    },
-    ['', {}] as readonly [string, Record<string, string[]>]
-  )
+      // Stacktrace didn't look like what we expected
+      if (control !== '') {
+        return undefined
+      }
+
+      const messageLoc = {
+        line: Number(line),
+        column: Number(column.slice(0, column.length - 1)),
+      }
+      return (foundTest = tests.find(({ testNode }) => {
+        if (messageLoc.line < testNode.loc.start.line) {
+          return false
+        }
+        if (messageLoc.line > testNode.loc.end.line) {
+          return false
+        }
+        if (
+          messageLoc.line === testNode.loc.start.line &&
+          messageLoc.column < testNode.loc.start.column
+        ) {
+          return false
+        }
+        if (
+          messageLoc.line === testNode.loc.end.line &&
+          messageLoc.column > testNode.loc.end.column
+        ) {
+          return false
+        }
 
-  return Object.keys(outputs).reduce((results, key) => {
+        return true
+      }))
+    })
+
+    const testName = foundTest?.name(' > ') ?? ''
+    const sanitized = `[${entry.type}] ${sanitizeErrorMessage(
+      specFilePath,
+      entry.message
+    )}`
+
+    outputs[testName] = outputs[testName] || []
+    outputs[testName].push(sanitized)
+
+    return outputs
+  }, {})
+
+  return Object.keys(outputs).reduce<Record<string, string>>((results, key) => {
     const message = (outputs[key] || []).join('\n') || ''
     if (message.length <= 500) {
       results[key] = message
@@ -260,7 +320,7 @@ function buildOutput(
     }
 
     return results
-  }, {} as Record<string, string>)
+  }, {})
 }
 
 function buildTestOutput(
@@ -318,11 +378,15 @@ function sanitizeErrorMessage(specFilePath: string, message: string): string {
     path.dirname(process.cwd()),
   ]
 
-  dirs.forEach((sensativePath) => {
-    while (message.indexOf(sensativePath) !== -1) {
-      message = message.replace(sensativePath, '<solution>')
+  dirs.forEach((sensitivePath) => {
+    while (message.includes(sensitivePath)) {
+      message = message.replace(sensitivePath, '<solution>')
     }
   })
 
+  if (message.includes('SyntaxError: <solution>')) {
+    return message.slice(message.indexOf('SyntaxError: <solution>'))
+  }
+
   return message
 }

src/reporter.ts

@@ -1,44 +1,26 @@
-import { Status } from './status'
-import { Output } from './output'
-import { isInteractive } from 'jest-util'
-
-import { Config } from '@jest/types'
-import {
+import { ParsedSource } from '@exercism/static-analysis'
+import type {
   Context,
-  Test,
   Reporter,
   ReporterOnStartOptions,
+  Test,
 } from '@jest/reporters'
-import { AggregatedResult, TestResult } from '@jest/test-result'
-
-import { getResultHeader } from './getResultHeader'
-import { getSnapshotStatus } from './getSnapshotStatus'
-import { getTestResults } from './getTestResults'
-import { getSummary } from './getSummary'
-import { Stdlib } from './stdlib'
+import type { AggregatedResult, TestResult } from '@jest/test-result'
+import type { Config } from '@jest/types'
+import { Output } from './output'
 
-class StandardReporter implements Reporter {
-  private readonly globalConfig: Config.GlobalConfig
-  private readonly stdlib: Stdlib
-  private readonly status: Status
+// eslint-disable-next-line import/no-default-export
+export default class StandardReporter implements Reporter {
   private readonly output: Output
 
   private error: null | Error
 
   constructor(globalConfig: Config.GlobalConfig) {
-    this.globalConfig = globalConfig
-    this.stdlib = new Stdlib(globalConfig)
-
     this.error = null
-    this.status = new Status()
     this.output = new Output(globalConfig)
-
-    this.status.onChange(() => {
-      /*noop*/
-    })
   }
 
-  public getLastError(): Error | void {
+  public getLastError(): Error | undefined {
     if (this.error) {
       return this.error
     }
@@ -57,17 +39,13 @@ class StandardReporter implements Reporter {
     results: AggregatedResult,
     options: ReporterOnStartOptions
   ): void {
-    this.status.runStarted(results, options)
-    if (isInteractive) {
-      this.stdlib.clear()
-    }
+    // no-op
   }
 
   /**
    * @param test
    */
   public onTestStart(test: Test): void {
-    this.status.testStarted(test.path, test.context.config)
     this.output.testStarted(test.path)
   }
 
@@ -81,31 +59,7 @@ class StandardReporter implements Reporter {
     testResult: TestResult,
     results: AggregatedResult
   ): void {
-    this.status.testFinished(test.context.config, testResult, results)
     this.output.testFinished(test.path, testResult, results)
-
-    if (!testResult.skipped) {
-      this.stdlib.log(
-        getResultHeader(testResult, this.globalConfig, test.context.config)
-      )
-
-      if (
-        this.globalConfig.verbose &&
-        !testResult.testExecError &&
-        !testResult.skipped
-      ) {
-        this.stdlib.log(getTestResults(testResult.testResults))
-      }
-
-      if (testResult.failureMessage) {
-        this.stdlib.error(testResult.failureMessage)
-      }
-
-      const didUpdate = this.globalConfig.updateSnapshot === 'all'
-      const snapshotStatuses = getSnapshotStatus(testResult.snapshot, didUpdate)
-      snapshotStatuses.forEach(this.stdlib.log.bind(this.stdlib))
-    }
-    this.stdlib.forceFlushBufferedOutput()
   }
 
   /**
@@ -116,13 +70,6 @@ class StandardReporter implements Reporter {
     _contexts: Set<Context>,
     aggregatedResults: AggregatedResult
   ): Promise<void> | void {
-    this.stdlib.log(getSummary(aggregatedResults, undefined))
-    this.status.runFinished()
-    this.stdlib.close()
-    this.stdlib.clear()
-
     this.output.finish(aggregatedResults)
   }
 }
-
-export = StandardReporter

test/.eslintignore

@@ -1 +1,4 @@
 fixtures/*
+
+.eslintrc
+.eslintrc.*

test/.eslintrc.js

@@ -0,0 +1,10 @@
+module.exports = {
+  parserOptions: {
+    tsconfigRootDir: __dirname,
+    project: ['./tsconfig.json'],
+  },
+  env: {
+    jest: true,
+  },
+  extends: ['../.eslintrc.js'],
+}

test/fixtures/clock/pass/clock.spec.js

@@ -6,213 +6,213 @@ describe('Clock', () => {
       expect(new Clock(8).toString()).toEqual('08:00');
     });
 
-    test('past the hour', () => {
+    xtest('past the hour', () => {
       expect(new Clock(11, 9).toString()).toEqual('11:09');
     });
 
-    test('midnight is zero hours', () => {
+    xtest('midnight is zero hours', () => {
       expect(new Clock(24, 0).toString()).toEqual('00:00');
     });
 
-    test('hour rolls over', () => {
+    xtest('hour rolls over', () => {
       expect(new Clock(25, 0).toString()).toEqual('01:00');
     });
 
-    test('hour rolls over continuously', () => {
+    xtest('hour rolls over continuously', () => {
       expect(new Clock(100, 0).toString()).toEqual('04:00');
     });
 
-    test('sixty minutes is next hour', () => {
+    xtest('sixty minutes is next hour', () => {
       expect(new Clock(1, 60).toString()).toEqual('02:00');
     });
 
-    test('minutes roll over', () => {
+    xtest('minutes roll over', () => {
       expect(new Clock(0, 160).toString()).toEqual('02:40');
     });
 
-    test('minutes roll over continuously', () => {
+    xtest('minutes roll over continuously', () => {
       expect(new Clock(0, 1723).toString()).toEqual('04:43');
     });
 
-    test('hour and minutes roll over', () => {
+    xtest('hour and minutes roll over', () => {
       expect(new Clock(25, 160).toString()).toEqual('03:40');
     });
 
-    test('hour and minutes roll over continuously', () => {
+    xtest('hour and minutes roll over continuously', () => {
       expect(new Clock(201, 3001).toString()).toEqual('11:01');
     });
 
-    test('hour and minutes roll over to exactly midnight', () => {
+    xtest('hour and minutes roll over to exactly midnight', () => {
       expect(new Clock(72, 8640).toString()).toEqual('00:00');
     });
 
-    test('negative hour', () => {
+    xtest('negative hour', () => {
       expect(new Clock(-1, 15).toString()).toEqual('23:15');
     });
 
-    test('negative hour rolls over', () => {
+    xtest('negative hour rolls over', () => {
       expect(new Clock(-25, 0).toString()).toEqual('23:00');
     });
 
-    test('negative hour rolls over continuously', () => {
+    xtest('negative hour rolls over continuously', () => {
       expect(new Clock(-91, 0).toString()).toEqual('05:00');
     });
 
-    test('negative minutes', () => {
+    xtest('negative minutes', () => {
       expect(new Clock(1, -40).toString()).toEqual('00:20');
     });
 
-    test('negative minutes rolls over', () => {
+    xtest('negative minutes rolls over', () => {
       expect(new Clock(1, -160).toString()).toEqual('22:20');
     });
 
-    test('negative minutes rolls over continuously', () => {
+    xtest('negative minutes rolls over continuously', () => {
       expect(new Clock(1, -4820).toString()).toEqual('16:40');
     });
 
-    test('negative sixty minutes is previous hour', () => {
+    xtest('negative sixty minutes is previous hour', () => {
       expect(new Clock(2, -60).toString()).toEqual('01:00');
     });
 
-    test('negative hour and minutes both roll over', () => {
+    xtest('negative hour and minutes both roll over', () => {
       expect(new Clock(-25, -160).toString()).toEqual('20:20');
     });
 
-    test('negative hour and minutes both roll over continuously', () => {
+    xtest('negative hour and minutes both roll over continuously', () => {
       expect(new Clock(-121, -5810).toString()).toEqual('22:10');
     });
   });
 
   describe('Adding minutes', () => {
-    test('add minutes', () => {
+    xtest('add minutes', () => {
       expect(new Clock(10, 0).plus(3).toString()).toEqual('10:03');
     });
 
-    test('add no minutes', () => {
+    xtest('add no minutes', () => {
       expect(new Clock(6, 41).plus(0).toString()).toEqual('06:41');
     });
 
-    test('add to next hour', () => {
+    xtest('add to next hour', () => {
       expect(new Clock(0, 45).plus(40).toString()).toEqual('01:25');
     });
 
-    test('add more than one hour', () => {
+    xtest('add more than one hour', () => {
       expect(new Clock(10, 0).plus(61).toString()).toEqual('11:01');
     });
 
-    test('add more than two hours with carry', () => {
+    xtest('add more than two hours with carry', () => {
       expect(new Clock(0, 45).plus(160).toString()).toEqual('03:25');
     });
 
-    test('add across midnight', () => {
+    xtest('add across midnight', () => {
       expect(new Clock(23, 59).plus(2).toString()).toEqual('00:01');
     });
 
-    test('add more than one day (1500 min = 25 hrs)', () => {
+    xtest('add more than one day (1500 min = 25 hrs)', () => {
       expect(new Clock(5, 32).plus(1500).toString()).toEqual('06:32');
     });
 
-    test('add more than two days', () => {
+    xtest('add more than two days', () => {
       expect(new Clock(1, 1).plus(3500).toString()).toEqual('11:21');
     });
   });
 
   describe('Subtract minutes', () => {
-    test('subtract minutes', () => {
+    xtest('subtract minutes', () => {
       expect(new Clock(10, 3).minus(3).toString()).toEqual('10:00');
     });
 
-    test('subtract to previous hour', () => {
+    xtest('subtract to previous hour', () => {
       expect(new Clock(10, 3).minus(30).toString()).toEqual('09:33');
     });
 
-    test('subtract more than an hour', () => {
+    xtest('subtract more than an hour', () => {
       expect(new Clock(10, 3).minus(70).toString()).toEqual('08:53');
     });
 
-    test('subtract across midnight', () => {
+    xtest('subtract across midnight', () => {
       expect(new Clock(0, 3).minus(4).toString()).toEqual('23:59');
     });
 
-    test('subtract more than two hours', () => {
+    xtest('subtract more than two hours', () => {
       expect(new Clock(0, 0).minus(160).toString()).toEqual('21:20');
     });
 
-    test('subtract more than two hours with borrow', () => {
+    xtest('subtract more than two hours with borrow', () => {
       expect(new Clock(6, 15).minus(160).toString()).toEqual('03:35');
     });
 
-    test('subtract more than one day (1500 min = 25 hrs)', () => {
+    xtest('subtract more than one day (1500 min = 25 hrs)', () => {
       expect(new Clock(5, 32).minus(1500).toString()).toEqual('04:32');
     });
 
-    test('subtract more than two days', () => {
+    xtest('subtract more than two days', () => {
       expect(new Clock(2, 20).minus(3000).toString()).toEqual('00:20');
     });
   });
 
   describe('Compare two clocks for equality', () => {
-    test('clocks with same time', () => {
+    xtest('clocks with same time', () => {
       expect(new Clock(15, 37).equals(new Clock(15, 37))).toBe(true);
     });
 
-    test('clocks a minute apart', () => {
+    xtest('clocks a minute apart', () => {
       expect(new Clock(15, 36).equals(new Clock(15, 37))).toBe(false);
     });
 
-    test('clocks an hour apart', () => {
+    xtest('clocks an hour apart', () => {
       expect(new Clock(14, 37).equals(new Clock(15, 37))).toBe(false);
     });
 
-    test('clocks with hour overflow', () => {
+    xtest('clocks with hour overflow', () => {
       expect(new Clock(10, 37).equals(new Clock(34, 37))).toBe(true);
     });
 
-    test('clocks with hour overflow by several days', () => {
+    xtest('clocks with hour overflow by several days', () => {
       expect(new Clock(3, 11).equals(new Clock(99, 11))).toBe(true);
     });
 
-    test('clocks with negative hour', () => {
+    xtest('clocks with negative hour', () => {
       expect(new Clock(22, 40).equals(new Clock(-2, 40))).toBe(true);
     });
 
-    test('clocks with negative hour that wraps', () => {
+    xtest('clocks with negative hour that wraps', () => {
       expect(new Clock(17, 3).equals(new Clock(-31, 3))).toBe(true);
     });
 
-    test('clocks with negative hour that wraps multiple times', () => {
+    xtest('clocks with negative hour that wraps multiple times', () => {
       expect(new Clock(13, 49).equals(new Clock(-83, 49))).toBe(true);
     });
 
-    test('clocks with minute overflow', () => {
+    xtest('clocks with minute overflow', () => {
       expect(new Clock(0, 1).equals(new Clock(0, 1441))).toBe(true);
     });
 
-    test('clocks with minute overflow by several days', () => {
+    xtest('clocks with minute overflow by several days', () => {
       expect(new Clock(2, 2).equals(new Clock(2, 4322))).toBe(true);
     });
 
-    test('clocks with negative minute', () => {
+    xtest('clocks with negative minute', () => {
       expect(new Clock(2, 40).equals(new Clock(3, -20))).toBe(true);
     });
 
-    test('clocks with negative minute that wraps', () => {
+    xtest('clocks with negative minute that wraps', () => {
       expect(new Clock(4, 10).equals(new Clock(5, -1490))).toBe(true);
     });
 
-    test('clocks with negative minute that wraps multiple times', () => {
+    xtest('clocks with negative minute that wraps multiple times', () => {
       expect(new Clock(6, 15).equals(new Clock(6, -4305))).toBe(true);
     });
 
-    test('clocks with negative hours and minutes', () => {
+    xtest('clocks with negative hours and minutes', () => {
       expect(new Clock(7, 32).equals(new Clock(-12, -268))).toBe(true);
     });
 
-    test('clocks with negative hours and minutes that wrap', () => {
+    xtest('clocks with negative hours and minutes that wrap', () => {
       expect(new Clock(18, 7).equals(new Clock(-54, -11513))).toBe(true);
     });
 
-    test('full clock and zeroed clock', () => {
+    xtest('full clock and zeroed clock', () => {
       expect(new Clock(24, 0).equals(new Clock(0, 0))).toBe(true);
     });
   });

test/fixtures/two-fer/error/syntax/two-fer.spec.js

@@ -5,12 +5,12 @@ describe('twoFer()', () => {
     expect(twoFer()).toEqual('One for you, one for me.');
   });
 
-  test('a name given', () => {
+  xtest('a name given', () => {
     const name = 'Alice';
     expect(twoFer(name)).toEqual('One for Alice, one for me.');
   });
 
-  test('another name given', () => {
+  xtest('another name given', () => {
     const name = 'Bob';
     expect(twoFer(name)).toEqual('One for Bob, one for me.');
   });

test/fixtures/two-fer/fail/empty/two-fer.spec.js

@@ -5,12 +5,12 @@ describe('twoFer()', () => {
     expect(twoFer()).toEqual('One for you, one for me.');
   });
 
-  test('a name given', () => {
+  xtest('a name given', () => {
     const name = 'Alice';
     expect(twoFer(name)).toEqual('One for Alice, one for me.');
   });
 
-  test('another name given', () => {
+  xtest('another name given', () => {
     const name = 'Bob';
     expect(twoFer(name)).toEqual('One for Bob, one for me.');
   });

test/fixtures/two-fer/fail/tests/two-fer.spec.js

@@ -5,12 +5,12 @@ describe('twoFer()', () => {
     expect(twoFer()).toEqual('One for you, one for me.');
   });
 
-  test('a name given', () => {
+  xtest('a name given', () => {
     const name = 'Alice';
     expect(twoFer(name)).toEqual('One for Alice, one for me.');
   });
 
-  test('another name given', () => {
+  xtest('another name given', () => {
     const name = 'Bob';
     expect(twoFer(name)).toEqual('One for Bob, one for me.');
   });

test/fixtures/two-fer/pass/two-fer.spec.js

@@ -5,12 +5,12 @@ describe('twoFer()', () => {
     expect(twoFer()).toEqual('One for you, one for me.');
   });
 
-  test('a name given', () => {
+  xtest('a name given', () => {
     const name = 'Alice';
     expect(twoFer(name)).toEqual('One for Alice, one for me.');
   });
 
-  test('another name given', () => {
+  xtest('another name given', () => {
     const name = 'Bob';
     expect(twoFer(name)).toEqual('One for Bob, one for me.');
   });

test/skip.test.ts

@@ -1,3 +1,4 @@
+import { describe, afterEach, test, expect } from '@jest/globals'
 import { spawnSync } from 'child_process'
 import { join, resolve } from 'path'
 import { lstat, mkdtempSync, readFileSync, unlink } from 'fs'
@@ -9,6 +10,8 @@ const bin = resolve(root, 'bin')
 const run = resolve(bin, 'run.sh')
 
 describe('skipping via test.skip', () => {
+  jest.setTimeout(120 * 1000)
+
   describe('passing solution', () => {
     const resultPath = join(
       fixtures,
@@ -63,7 +66,9 @@ describe('skipping via test.skip', () => {
         lstat(resultPath, (err, _) => {
           expect(err).toBeNull()
 
-          const result = JSON.parse(readFileSync(resultPath).toString())
+          const result = JSON.parse(readFileSync(resultPath).toString()) as {
+            status: string
+          }
           expect(result.status).toBe('pass')
 
           if (err) {

test/smoke.test.ts

@@ -1,3 +1,4 @@
+import { describe, afterEach, test, expect } from '@jest/globals'
 import { spawnSync } from 'child_process'
 import { join, resolve } from 'path'
 import { lstat, mkdtempSync, readFileSync, unlink } from 'fs'
@@ -9,6 +10,8 @@ const bin = resolve(root, 'bin')
 const run = resolve(bin, 'run.sh')
 
 describe('javascript-test-runner', () => {
+  jest.setTimeout(120 * 1000)
+
   describe('passing solution', () => {
     const resultPath = join(fixtures, 'two-fer', 'pass', 'results.json')
 
@@ -46,7 +49,9 @@ describe('javascript-test-runner', () => {
         lstat(resultPath, (err, _) => {
           expect(err).toBeNull()
 
-          const result = JSON.parse(readFileSync(resultPath).toString())
+          const result = JSON.parse(readFileSync(resultPath).toString()) as {
+            status: string
+          }
           expect(result.status).toBe('pass')
 
           if (err) {
@@ -139,7 +144,9 @@ describe('javascript-test-runner', () => {
           lstat(resultPath, (err, _) => {
             expect(err).toBeNull()
 
-            const result = JSON.parse(readFileSync(resultPath).toString())
+            const result = JSON.parse(readFileSync(resultPath).toString()) as {
+              status: string
+            }
             expect(result.status).toBe('pass')
 
             if (err) {
@@ -228,7 +235,9 @@ describe('javascript-test-runner', () => {
           lstat(resultPath, (err, _) => {
             expect(err).toBeNull()
 
-            const result = JSON.parse(readFileSync(resultPath).toString())
+            const result = JSON.parse(readFileSync(resultPath).toString()) as {
+              status: string
+            }
             expect(result.status).toBe('fail')
 
             if (err) {
@@ -312,7 +321,10 @@ describe('javascript-test-runner', () => {
           lstat(resultPath, (err, _) => {
             expect(err).toBeNull()
 
-            const result = JSON.parse(readFileSync(resultPath).toString())
+            const result = JSON.parse(readFileSync(resultPath).toString()) as {
+              status: string
+              message: string | undefined
+            }
             expect(result.status).toBe('error')
             expect(result.message).not.toBeUndefined()
 

tsconfig.json

@@ -1,64 +1,5 @@
 {
-  "compilerOptions": {
-    /* Basic Options */
-    "target": "ESNEXT" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017','ES2018' or 'ESNEXT'. */,
-    "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */,
-    "lib": [
-      "esnext"
-    ] /* Specify library files to be included in the compilation. */,
-    // "allowJs": true,                          /* Allow javascript files to be compiled. */
-    // "checkJs": true,                       /* Report errors in .js files. */
-    // "jsx": "preserve",                     /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
-    // "declaration": true,                   /* Generates corresponding '.d.ts' file. */
-    // "declarationMap": true,                /* Generates a sourcemap for each corresponding '.d.ts' file. */
-    // "sourceMap": true,                     /* Generates corresponding '.map' file. */
-    // "outFile": "./",                       /* Concatenate and emit output to single file. */
-    "outDir": "./dist" /* Redirect output structure to the directory. */,
-    "rootDirs": [
-      "./src"
-    ] /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */,
-    // "composite": true,                     /* Enable project compilation */
-    // "removeComments": true,                /* Do not emit comments to output. */
-    // "noEmit": true,                        /* Do not emit outputs. */
-    // "importHelpers": true,                 /* Import emit helpers from 'tslib'. */
-    // "downlevelIteration": true,            /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
-    // "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
-
-    /* Strict Type-Checking Options */
-    "strict": true /* Enable all strict type-checking options. */,
-    // "noImplicitAny": true,                 /* Raise error on expressions and declarations with an implied 'any' type. */
-    // "strictNullChecks": true,              /* Enable strict null checks. */
-    // "strictFunctionTypes": true,           /* Enable strict checking of function types. */
-    // "strictBindCallApply": true,           /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
-    // "strictPropertyInitialization": true,  /* Enable strict checking of property initialization in classes. */
-    // "noImplicitThis": true,                /* Raise error on 'this' expressions with an implied 'any' type. */
-    // "alwaysStrict": true,                  /* Parse in strict mode and emit "use strict" for each source file. */
-
-    /* Additional Checks */
-    // "noUnusedLocals": true,                /* Report errors on unused locals. */
-    "noUnusedParameters": true /* Report errors on unused parameters. */,
-    // "noImplicitReturns": true,             /* Report error when not all code paths in function return a value. */
-    // "noFallthroughCasesInSwitch": true,    /* Report errors for fallthrough cases in switch statement. */
-
-    /* Module Resolution Options */
-    "moduleResolution": "node" /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */,
-    "baseUrl": "./" /* Base directory to resolve non-absolute module names. */,
-    // "paths": { "~src/*": ["./src/*"] },                           /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
-    // "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
-    // "typeRoots": [],                       /* List of folders to include type definitions from. */
-    // "types": [],                           /* Type declaration files to be included in compilation. */
-    "allowSyntheticDefaultImports": true /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */,
-    "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
-    // "preserveSymlinks": true,              /* Do not resolve the real path of symlinks. */
-
-    /* Source Map Options */
-    // "sourceRoot": "",                      /* Specify the location where debugger should locate TypeScript files instead of source locations. */
-    // "mapRoot": "",                         /* Specify the location where debugger should locate map files instead of generated locations. */
-    // "inlineSourceMap": true,               /* Emit a single file with source maps instead of having a separate file. */
-    // "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
-
-    /* Experimental Options */
-    // "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
-    // "emitDecoratorMetadata": true,         /* Enables experimental support for emitting type metadata for decorators. */
-  }
+  "extends": "@tsconfig/node16/tsconfig.json",
+  "include": ["src"],
+  "exclude": ["test", "node_modules"]
 }