scripts/parser-tests/utils/parser-test-runner.js
"use strict";
const fs = require("fs").promises;
const chalk = require("chalk");
const { parse: parser } = require("../../../packages/babel-parser");
const dot = chalk.gray(".");
class TestRunner {
constructor({
testDir,
allowlist,
logInterval = 1,
shouldUpdate,
getTests,
parse = this.parse,
}) {
this.testDir = testDir;
this.allowlist = allowlist;
this.logInterval = logInterval;
this.shouldUpdate = shouldUpdate;
this.getTests = getTests;
this.parse = parse;
}
async run() {
const allowlistP = this.getAllowlist();
console.log(`Now running tests...`);
const results = [];
for await (const result of this.runTests()) {
results.push(result);
if (results.length % this.logInterval === 0) process.stdout.write(dot);
}
process.stdout.write("\n");
const summary = this.interpret(results, await allowlistP);
await this.output(summary);
}
async *runTests() {
for await (const test of this.getTests()) {
yield this.runTest(test);
}
}
runTest(test) {
try {
this.parse(test, parser);
test.actualError = false;
} catch (err) {
test.actualError = true;
}
test.result = test.expectedError !== test.actualError ? "fail" : "pass";
return test;
}
parse(test, parser) {
parser(test.contents, {
sourceType: test.sourceType,
plugins: test.plugins,
});
}
async getAllowlist() {
const contents = await fs.readFile(this.allowlist, "utf-8");
const table = new Set();
for (const line of contents.split("\n")) {
const filename = line.replace(/#.*$/, "").trim();
if (filename) table.add(filename);
}
return table;
}
async updateAllowlist(summary) {
const contents = await fs.readFile(this.allowlist, "utf-8");
const toRemove = summary.disallowed.success
.concat(summary.disallowed.failure)
.map(test => test.id)
.concat(summary.unrecognized);
const updated = summary.disallowed.falsePositive
.concat(summary.disallowed.falseNegative)
.map(test => test.id);
for (const line of contents.split("\n")) {
const testId = line.replace(/#.*$/, "").trim();
if (!toRemove.includes(testId) && line) {
updated.push(line);
}
}
updated.sort();
await fs.writeFile(this.allowlist, updated.join("\n") + "\n", "utf8");
}
interpret(results, allowlist) {
const summary = {
passed: true,
allowed: {
success: [],
failure: [],
falsePositive: [],
falseNegative: [],
},
disallowed: {
success: [],
failure: [],
falsePositive: [],
falseNegative: [],
},
unrecognized: null,
count: results.length,
};
results.forEach(function (result) {
let classification, isAllowed;
const inAllowlist = allowlist.has(result.id);
allowlist.delete(result.id);
if (!result.expectedError) {
if (!result.actualError) {
classification = "success";
isAllowed = !inAllowlist;
} else {
classification = "falseNegative";
isAllowed = inAllowlist;
}
} else {
if (!result.actualError) {
classification = "falsePositive";
isAllowed = inAllowlist;
} else {
classification = "failure";
isAllowed = !inAllowlist;
}
}
summary.passed &= isAllowed;
summary[isAllowed ? "allowed" : "disallowed"][classification].push(
result
);
});
summary.unrecognized = Array.from(allowlist);
summary.passed = !!summary.passed && summary.unrecognized.length === 0;
return summary;
}
async output(summary) {
const goodnews = [
summary.allowed.success.length + " valid programs parsed without error",
summary.allowed.failure.length +
" invalid programs produced a parsing error",
summary.allowed.falsePositive.length +
" invalid programs did not produce a parsing error" +
" (and allowed by the allowlist file)",
summary.allowed.falseNegative.length +
" valid programs produced a parsing error" +
" (and allowed by the allowlist file)",
];
const badnews = [];
const badnewsDetails = [];
void [
{
tests: summary.disallowed.success,
label:
"valid programs parsed without error" +
" (in violation of the allowlist file)",
},
{
tests: summary.disallowed.failure,
label:
"invalid programs produced a parsing error" +
" (in violation of the allowlist file)",
},
{
tests: summary.disallowed.falsePositive,
label:
"invalid programs did not produce a parsing error" +
" (without a corresponding entry in the allowlist file)",
},
{
tests: summary.disallowed.falseNegative,
label:
"valid programs produced a parsing error" +
" (without a corresponding entry in the allowlist file)",
},
{
tests: summary.unrecognized,
label: "non-existent programs specified in the allowlist file",
},
].forEach(function ({ tests, label }) {
if (!tests.length) {
return;
}
const desc = tests.length + " " + label;
badnews.push(desc);
badnewsDetails.push(desc + ":");
badnewsDetails.push(...tests.map(test => ` ${test.id || test}`));
});
console.log(`Testing complete (${summary.count} tests).`);
console.log("Summary:");
console.log(chalk.green(goodnews.join("\n").replace(/^/gm, " ✔ ")));
if (!summary.passed) {
console.log("");
console.log(chalk.red(badnews.join("\n").replace(/^/gm, " ✘ ")));
console.log("");
console.log("Details:");
console.log(badnewsDetails.join("\n").replace(/^/gm, " "));
}
if (this.shouldUpdate) {
await this.updateAllowlist(summary);
console.log("");
console.log("Allowlist file updated.");
} else {
process.exitCode = summary.passed ? 0 : 1;
}
}
}
module.exports = exports = TestRunner;