tools/tool-testing/run.js
// Represents a test run of the tool (except we also use it in
// tests/old.js to run Node scripts). Typically created through the
// run() method on Sandbox, but can also be created directly, say if
// you want to do something other than invoke the 'meteor' command in
// a nice sandbox.
//
// Options: args, cwd, env
//
// The 'execPath' argument and the 'cwd' option are assumed to be standard
// paths.
//
// Arguments in the 'args' option are not assumed to be standard paths, so
// calling any of the 'files.*' methods on them is not safe.
import { spawn } from 'child_process';
import * as files from '../fs/files';
import {
parse as parseStackParse,
} from '../utils/parse-stack';
import { Console } from '../console/console.js';
import Matcher from './matcher.js';
import OutputLog from './output-log.js';
import { randomPort, timeoutScaleFactor, sleepMs } from '../utils/utils.js';
import TestFailure from './test-failure.js';
import { execFileSync } from '../utils/processes';
let runningTest = null;
export default class Run {
constructor(execPath, options) {
this.execPath = execPath;
this.cwd = options.cwd || files.convertToStandardPath(process.cwd());
// default env variables
this.env = Object.assign({ SELFTEST: "t", METEOR_NO_WORDWRAP: "t" }, options.env);
this._args = [];
this.proc = null;
this.baseTimeout = 20;
this.extraTime = 0;
this.client = options.client;
this.stdoutMatcher = new Matcher(this);
this.stderrMatcher = new Matcher(this);
this.outputLog = new OutputLog(this);
this.matcherEndPromise = null;
this.exitStatus = undefined; // 'null' means failed rather than exited
this.exitPromiseResolvers = [];
const opts = options.args || [];
this.args.apply(this, opts || []);
this.fakeMongoPort = null;
this.fakeMongoConnection = null;
if (options.fakeMongo) {
this.fakeMongoPort = randomPort();
this.env.METEOR_TEST_FAKE_MONGOD_CONTROL_PORT = this.fakeMongoPort;
}
runningTest.onCleanup(() => {
this._stopWithoutWaiting();
});
}
// Set command-line arguments. This may be called multiple times as
// long as the run has not yet started (the run starts after the
// first call to a function that requires it, like match()).
//
// Pass as many arguments as you want. Non-object values will be
// cast to string, and object values will be treated as maps from
// option names to values.
args(...args) {
if (this.proc) {
throw new Error("already started?");
}
args.forEach((a) => {
if (typeof a !== "object") {
this._args.push(`${a}`);
} else {
Object.keys(a).forEach((key) => {
const value = a[key];
this._args.push(`--${key}`);
this._args.push(`${value}`);
});
}
});
}
connectClient() {
if (!this.client) {
throw new Error("Must create Run with a client to use connectClient().");
}
this._ensureStarted();
this.client.connect();
}
// Useful for matching one-time patterns not sensitive to ordering.
matchBeforeExit(pattern) {
return this.stdoutMatcher.matchBeforeEnd(pattern);
}
matchErrBeforeExit(pattern) {
return this.stderrMatcher.matchBeforeEnd(pattern);
}
_exited(status) {
if (this.exitStatus !== undefined) {
throw new Error("already exited?");
}
if (this.client) {
this.client.stop();
}
this.exitStatus = status;
const exitPromiseResolvers = this.exitPromiseResolvers;
this.exitPromiseResolvers = null;
exitPromiseResolvers.forEach((resolve) => {
resolve();
});
this._endMatchers();
}
_endMatchers() {
this.matcherEndPromise =
this.matcherEndPromise || Promise.all([
this.stdoutMatcher.endAsync(),
this.stderrMatcher.endAsync()
]);
return this.matcherEndPromise;
}
_ensureStarted() {
if (this.proc) {
return;
}
const env = Object.assign(Object.create(null), process.env);
Object.assign(env, this.env);
this.proc = spawn(files.convertToOSPath(this.execPath),
this._args, {
cwd: files.convertToOSPath(this.cwd),
env,
});
this.proc.on('close', (code, signal) => {
if (this.exitStatus === undefined) {
this._exited({ code, signal });
}
});
this.proc.on('exit', (code, signal) => {
if (this.exitStatus === undefined) {
this._exited({ code, signal });
}
});
this.proc.on('error', (err) => {
if (this.exitStatus === undefined) {
this._exited(null);
}
});
this.proc.stdout.setEncoding('utf8');
this.proc.stdout.on('data', (data) => {
this.outputLog.write('stdout', data);
this.stdoutMatcher.write(data);
});
this.proc.stderr.setEncoding('utf8');
this.proc.stderr.on('data', (data) => {
this.outputLog.write('stderr', data);
this.stderrMatcher.write(data);
});
}
// Wait until we get text on stdout that matches 'pattern', which
// may be a regular expression or a string. Consume stdout up to
// that point. If this pattern does not appear after a timeout (or
// the program exits before emitting the pattern), fail.
match(pattern, _strict) {
this._ensureStarted();
let timeout = this.baseTimeout + this.extraTime;
timeout *= timeoutScaleFactor;
this.extraTime = 0;
Console.simpleDebug('match', pattern);
return this.stdoutMatcher.match(pattern, timeout, _strict);
}
getMatcherFullBuffer() {
return this.stdoutMatcher.getFullBuffer();
}
// As expect(), but for stderr instead of stdout.
matchErr(pattern, _strict) {
this._ensureStarted();
let timeout = this.baseTimeout + this.extraTime;
timeout *= timeoutScaleFactor;
this.extraTime = 0;
Console.simpleDebug('matchErr', pattern);
return this.stderrMatcher.match(pattern, timeout, _strict);
}
// Like match(), but won't skip ahead looking for a match. It must
// follow immediately after the last thing we matched or read.
read(pattern, strict = true) {
return this.match(pattern, strict);
}
// As read(), but for stderr instead of stdout.
readErr(pattern) {
return this.matchErr(pattern, true);
}
// Assert that 'pattern' (again, a regexp or string) has not
// occurred on stdout at any point so far in this run. Currently
// this works on complete lines, so unlike match() and read(),
// 'pattern' cannot span multiple lines, and furthermore if it is
// called before the end of the program, it may not see text on a
// partially read line. We could lift these restrictions easily, but
// there may not be any benefit since the usual way to use this is
// to call it after expectExit or expectEnd.
//
// Example:
// run = s.run("--help");
// run.expectExit(1); // <<-- important to actually run the command
// run.forbidErr("unwanted string"); // <<-- important to run **after** the
// // command ran the process.
forbid(pattern) {
this._ensureStarted();
this.outputLog.forbid(pattern, 'stdout');
}
// As forbid(), but for stderr instead of stdout.
forbidErr(pattern) {
this._ensureStarted();
this.outputLog.forbid(pattern, 'stderr');
}
// Combination of forbid() and forbidErr(). Forbids the pattern on
// both stdout and stderr.
forbidAll(pattern) {
this._ensureStarted();
this.outputLog.forbid(pattern);
}
// Expect the program to exit without anything further being
// printed on either stdout or stderr.
expectEnd() {
this._ensureStarted();
let timeout = this.baseTimeout + this.extraTime;
timeout *= timeoutScaleFactor;
this.extraTime = 0;
this.expectExit();
this.stdoutMatcher.matchEmpty();
this.stderrMatcher.matchEmpty();
}
// Expect the program to exit with the given (numeric) exit
// status. Fail if the process exits with a different code, or if
// the process does not exit after a timeout. You can also omit the
// argument to simply wait for the program to exit.
expectExit(code) {
this._ensureStarted();
this._endMatchers().await();
if (this.exitStatus === undefined) {
let timeout = this.baseTimeout + this.extraTime;
timeout *= timeoutScaleFactor;
this.extraTime = 0;
let timer;
const failure = new TestFailure('exit-timeout', { run: this });
const promise = new Promise((resolve, reject) => {
this.exitPromiseResolvers.push(resolve);
timer = setTimeout(() => {
this.exitPromiseResolvers =
this.exitPromiseResolvers.filter(r => r !== resolve);
reject(failure);
}, timeout * 1000);
});
try {
promise.await();
} finally {
clearTimeout(timer);
}
}
if (! this.exitStatus) {
throw new TestFailure('spawn-failure', { run: this });
}
if (code !== undefined && this.exitStatus.code !== code) {
throw new TestFailure('wrong-exit-code', {
expected: { code },
actual: this.exitStatus,
run: this,
});
}
}
// Extend the timeout for the next operation by 'secs' seconds.
waitSecs(secs) {
this.extraTime += secs;
}
// Send 'string' to the program on its stdin.
write(string) {
this._ensureStarted();
this.proc.stdin.write(string);
}
// Kill the program and then wait for it to actually exit.
stop() {
if (this.exitStatus === undefined) {
this._ensureStarted();
if (this.client) {
this.client.stop();
}
this._killProcess();
this.expectExit();
}
}
// Like stop, but doesn't wait for it to exit.
_stopWithoutWaiting() {
if (this.exitStatus === undefined && this.proc) {
if (this.client) {
this.client.stop();
}
this._killProcess();
}
}
// Kills the running process and it's child processes
_killProcess() {
if (!this.proc) {
throw new Error("Unexpected: `this.proc` undefined when calling _killProcess");
}
if (process.platform === "win32") {
// looks like in Windows `this.proc.kill()` doesn't kill child
// processes.
execFileSync("taskkill", ["/pid", this.proc.pid, '/f', '/t']);
} else {
this.proc.kill();
}
}
// If the fakeMongo option was set, sent a command to the stub
// mongod. Available commands currently are:
//
// - { stdout: "xyz" } to make fake-mongod write "xyz" to stdout
// - { stderr: "xyz" } likewise for stderr
// - { exit: 123 } to make fake-mongod exit with code 123
//
// Blocks until a connection to fake-mongod can be
// established. Throws a TestFailure if it cannot be established.
tellMongo(command) {
if (! this.fakeMongoPort) {
throw new Error("fakeMongo option on sandbox must be set");
}
this._ensureStarted();
// If it's the first time we've called tellMongo on this sandbox,
// open a connection to fake-mongod. Wait up to 60 seconds for it
// to accept the connection, retrying every 100ms.
//
// XXX we never clean up this connection. Hopefully once
// fake-mongod has dropped its end of the connection, and we hold
// no reference to our end, it will get gc'd. If not, that's not
// great, but it probably doesn't actually create any practical
// problems since this is only for testing.
if (! this.fakeMongoConnection) {
const net = require('net');
let lastStartTime = 0;
for (
let attempts = 0;
!this.fakeMongoConnection && attempts < 600;
attempts++
) {
// Throttle attempts to one every 100ms
sleepMs((lastStartTime + 100) - (+ new Date()));
lastStartTime = +(new Date());
new Promise((resolve) => {
// This is all arranged so that if a previous attempt
// belatedly succeeds, somehow, we ignore it.
const conn = net.connect(this.fakeMongoPort, () => {
if (resolve) {
this.fakeMongoConnection = conn;
resolve(true);
resolve = null;
}
});
conn.setNoDelay();
function fail() {
if (resolve) {
resolve(false);
resolve = null;
}
}
conn.on('error', fail);
setTimeout(fail, 100); // 100ms connection timeout
}).await();
}
if (!this.fakeMongoConnection) {
throw new TestFailure("mongo-not-running", { run: this });
}
}
this.fakeMongoConnection.write(`${JSON.stringify(command)}\n`);
// If we told it to exit, then we should close our end and connect again if
// asked to send more.
if (command.exit) {
this.fakeMongoConnection.end();
this.fakeMongoConnection = null;
}
}
static runTest(testList, test, testRunner, options = {}) {
options.retries = options.retries || 0;
let failure = null;
let startTime;
try {
runningTest = test;
startTime = +(new Date);
// ensure we mark the bottom of the stack each time we start a new test
testRunner();
} catch (e) {
failure = e;
} finally {
runningTest = null;
test.cleanup();
}
test.durationMs = +(new Date) - startTime;
if (failure) {
let checkmark;
if (process.platform === "win32") {
checkmark = 'FAIL';
} else {
checkmark = '\u2717'; // CROSS
}
Console.error(`... fail! (${test.durationMs} ms)`, Console.options({ bulletPoint: `${checkmark} ` }));
if (failure instanceof TestFailure) {
const frames = parseStackParse(failure).outsideFiber;
const toolsDir = files.getCurrentToolsDir();
let pathWithLineNumber;
frames.some(frame => {
// The parsed stack trace will typically include frame.file
// strings of the form "/tools/tests/whatever.js", which can be
// made absolute by joining them with toolsDir. If the resulting
// absPath exists, then we know we interpreted the frame.file
// correctly, and we can normalize away the leading '/'
// character to get a safe relative path.
const absPath = files.pathJoin(toolsDir, frame.file);
if (files.exists(absPath)) {
const relPath = files.pathRelative(toolsDir, absPath);
const parts = relPath.split("/");
if (parts[0] === "tools" &&
parts[1] === "tool-testing") {
// Ignore frames inside the /tools/tool-testing directory,
// like run.js and selftest.js.
return false;
}
pathWithLineNumber = `${relPath}:${frame.line}`;
return true;
}
// If frame.file was not joinable with toolsDir to obtain an
// absolute path that exists, show it to the user without trying
// to interpret what it means.
pathWithLineNumber = `${frame.file}:${frame.line}`;
return true;
});
Console.rawError(
` => Failure Reason: "${failure.reason}" at "${pathWithLineNumber}"\n`);
if (failure.reason === 'no-match' || failure.reason === 'junk-before' ||
failure.reason === 'match-timeout') {
Console.arrowError(`Pattern: "${failure.details.pattern}"`, 2);
}
if (failure.reason === "wrong-exit-code") {
const s = status => `${status.signal || status.code || "???"}`;
Console.rawError(
` => Expected: "${s(failure.details.expected)}"` +
`; actual: "${s(failure.details.actual)}"\n`);
}
if (failure.reason === 'expected-exception') {
}
if (failure.reason === 'not-equal') {
Console.rawError(
` => Expected: "${JSON.stringify(failure.details.expected)}";
actual: "${JSON.stringify(failure.details.actual)}"`);
}
if (failure.details.run) {
failure.details.run.outputLog.end();
const lines = failure.details.run.outputLog.get();
if (! lines.length) {
Console.arrowError("No output", 2);
} else {
const historyLines = options.historyLines || 100;
Console.arrowError(`Last ${historyLines} lines:`, 2);
lines.slice(-historyLines).forEach((line) => {
Console.rawError(" " +
(line.channel === "stderr" ? "2| " : "1| ") +
line.text +
(line.bare ? "%" : "") + "\n");
});
}
}
if (failure.details.messages) {
Console.arrowError("Errors while building:", 2);
Console.rawError(failure.details.messages.formatMessages() + "\n");
}
} else {
Console.rawError(` => Test threw exception: ${failure.stack}\n`);
}
if (options.retries > 0) {
Console.error(
"... retrying (" +
options.retries +
(options.retries === 1 ? " try" : " tries") +
" remaining) ...",
Console.options({ indent: 2 })
);
options.retries--;
return this.runTest(testList, test, testRunner, options);
}
testList.notifyFailed(test, failure);
} else {
Console.success(`... ok! (${test.durationMs} ms)`);
}
}
}
import { markThrowingMethods } from "./test-utils.js";
markThrowingMethods(Run.prototype);