Add support for ndjson benchmark output

src/config.ts

@@ -3,7 +3,7 @@ import { promises as fs } from 'fs';
 import * as os from 'os';
 import * as path from 'path';
 
-export type ToolType = 'cargo' | 'go' | 'benchmarkjs' | 'pytest' | 'googlecpp' | 'catch2';
+export type ToolType = 'cargo' | 'go' | 'benchmarkjs' | 'pytest' | 'googlecpp' | 'catch2' | 'ndjson';
 export interface Config {
     name: string;
     tool: ToolType;
@@ -24,7 +24,7 @@ export interface Config {
     maxItemsInChart: number | null;
 }
 
-export const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest', 'googlecpp', 'catch2'];
+export const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest', 'googlecpp', 'catch2', 'ndjson'];
 const RE_UINT = /^\d+$/;
 
 function validateToolType(tool: string): asserts tool is ToolType {

src/extract.ts

@@ -8,6 +8,7 @@ export interface BenchmarkResult {
     range?: string;
     unit: string;
     extra?: string;
+    biggerIsBetter?: boolean;
 }
 
 interface GitHubUser {
@@ -415,6 +416,71 @@ function extractCatch2Result(output: string): BenchmarkResult[] {
     return ret;
 }
 
+function ensureString(json: { [key: string]: unknown }, key: string): string {
+    const value = json[key];
+    if (typeof value === 'string') {
+        return value;
+    }
+    throw new Error(`Expected the JSON to have a string at the key "${key}"`);
+}
+
+function ensureNumber(json: { [key: string]: unknown }, key: string): number {
+    const value = json[key];
+    if (typeof value === 'number') {
+        return value;
+    }
+    throw new Error(`Expected the JSON to have a number at the key "${key}"`);
+}
+
+/**
+ * Process results that use the internal format for this tool, but using newline
+ * delimited json. This format is friendly for appending results onto a file.
+ *
+ * http://ndjson.org/
+ */
+function extractNdjsonResult(output: string): BenchmarkResult[] {
+    // Split the newlines, process the json, and then validate the fields.
+    return output
+        .split(/\r?\n/g)
+        .filter(n => n)
+        .map(line => {
+            // Process the JSON.
+            let json;
+            try {
+                json = JSON.parse(line);
+            } catch (err) {
+                throw new Error(
+                    `A line from the ndjson file could not be parsed.\nLine: ${line}\nError: ${err.message}`,
+                );
+            }
+            if (!json || typeof json !== 'object') {
+                throw new Error(`A line form the ndjson was not an object. Line: ${line}`);
+            }
+
+            const biggerIsBetter = json.biggerIsBetter;
+            if (typeof biggerIsBetter !== 'boolean') {
+                throw new Error(
+                    `ndjson output must include a "biggerIsBetter" field as it cannot be inferred from the test suite.`,
+                );
+            }
+
+            // Validate the data provided in the entry.
+            const result: BenchmarkResult = {
+                name: ensureString(json, 'name'),
+                value: ensureNumber(json, 'value'),
+                unit: ensureString(json, 'unit'),
+                biggerIsBetter,
+            };
+            if ('range' in json) {
+                result.range = ensureString(json, 'range');
+            }
+            if ('extra' in json) {
+                result.extra = ensureString(json, 'extra');
+            }
+            return result;
+        });
+}
+
 export async function extractResult(config: Config): Promise<Benchmark> {
     const output = await fs.readFile(config.outputFilePath, 'utf8');
     const { tool } = config;
@@ -439,6 +505,9 @@ export async function extractResult(config: Config): Promise<Benchmark> {
         case 'catch2':
             benches = extractCatch2Result(output);
             break;
+        case 'ndjson':
+            benches = extractNdjsonResult(output);
+            break;
         default:
             throw new Error(`FATAL: Unexpected tool: '${tool}'`);
     }

src/write.ts

@@ -56,7 +56,7 @@ async function addIndexHtmlIfNeeded(dir: string) {
     console.log('Created default index.html at', indexHtml);
 }
 
-function biggerIsBetter(tool: ToolType): boolean {
+function biggerIsBetter(tool: ToolType, result: BenchmarkResult): boolean {
     switch (tool) {
         case 'cargo':
             return false;
@@ -70,6 +70,13 @@ function biggerIsBetter(tool: ToolType): boolean {
             return false;
         case 'catch2':
             return false;
+        case 'ndjson': {
+            const { biggerIsBetter } = result;
+            if (biggerIsBetter === undefined) {
+                throw new Error('ndjson is assumed to have a biggerIsBetter field.');
+            }
+            return biggerIsBetter;
+        }
     }
 }
 
@@ -90,7 +97,7 @@ function findAlerts(curSuite: Benchmark, prevSuite: Benchmark, threshold: number
             continue;
         }
 
-        const ratio = biggerIsBetter(curSuite.tool)
+        const ratio = biggerIsBetter(curSuite.tool, current)
             ? prev.value / current.value // e.g. current=100, prev=200
             : current.value / prev.value; // e.g. current=200, prev=100
 
@@ -160,7 +167,7 @@ function buildComment(benchName: string, curSuite: Benchmark, prevSuite: Benchma
         const prev = prevSuite.benches.find(i => i.name === current.name);
 
         if (prev) {
-            const ratio = biggerIsBetter(curSuite.tool)
+            const ratio = biggerIsBetter(curSuite.tool, current)
                 ? prev.value / current.value // e.g. current=100, prev=200
                 : current.value / prev.value;
 

test/data/extract/ndjson_output.ndjson

@@ -0,0 +1,2 @@
+{ "extra": "100 samples\n76353 iterations", "name": "Fibonacci 10", "range": "± 0", "unit": "ns", "value": 0, "biggerIsBetter": true }
+{ "extra": "100 samples\n75814 iterations", "name": "Fibonacci 20", "range": "± 0", "unit": "ns", "value": 1, "biggerIsBetter": true }

test/extract.ts

@@ -220,6 +220,28 @@ describe('extractResult()', function() {
                 },
             ],
         },
+        {
+            tool: 'ndjson',
+            file: 'ndjson_output.ndjson',
+            expected: [
+                {
+                    extra: '100 samples\n76353 iterations',
+                    name: 'Fibonacci 10',
+                    range: '± 0',
+                    unit: 'ns',
+                    value: 0,
+                    biggerIsBetter: true,
+                },
+                {
+                    extra: '100 samples\n75814 iterations',
+                    name: 'Fibonacci 20',
+                    range: '± 0',
+                    unit: 'ns',
+                    value: 1,
+                    biggerIsBetter: true,
+                },
+            ],
+        },
     ];
 
     for (const test of normalCases) {