test_runner.js
/**
* Copyright 2015-2016 Unicity International
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
'use strict'
const fs = require('fs')
const request = require('request')
const q = require('q')
const colors = require('chalk')
const path = require('path')
const xml2js = require('xml2js')
const knox = require('knox')
const childProcess = require('child_process')
var xmlChildPath = path.join(__dirname, '/xmlChild')
const self = {}
function main (options) {
self.options = options;
console.log('options', options)
var host = options.host;
var port = options.port;
var verbose = options.verbose;
var diffUrl = options.diffUrl || '';
var testFile = options.testFile;
var basePath = options.basePath || '';
var testFolder = options.testFolder;
var commandLineTests = options.commandLineTests;
var timeStart = Date.now();
var testPromises = [];
var testQueue = [];
var client;
var tests;
testFile = JSON.parse(fs.readFileSync(testFile).toString());
tests = Object.keys(testFile);
if (options.AWSSecret) {
client = knox.createClient({
key : options.AWSKey,
secret : options.AWSSecret,
bucket : options.AWSBucket
});
} // FIXME where no secret, 'client' is not initialized
options.client = client;
function outputTest (test) {
var Text = 'Test: ';
var endPoint = test.endpoint;
var subTest = test.name;
var testPath = endPoint + '.' + subTest;
if (verbose) {
console.log('######################');
console.log('------Headers-------');
console.log(test.headers);
console.log('------Response------');
console.log(test.responseBody);
}
if (test.passed) {
if (test.ignore){
Text += colors.blue(testPath + ' passed but was ignored');
} else if (!test.timedOut && !test.warnOnTime) {
Text += colors.green(testPath + ' passed');
} else if (test.warnOnTime) {
Text += colors.yellow(testPath + ' passed, but exceeded warning threshold (expected time ' + test.ms +' ms. Max run time ' + test.maxTime.toFixed(0)+' ms )');
} else {
Text += colors.red(testPath + ' passed, but took too long (expected time ' + test.ms +' ms. Max run time ' + test.maxTime.toFixed(0)+' ms )');
}
} else if (test.ignore){
Text += colors.blue(testPath + ' failed but was ignored');
} else {
Text += colors.red(testPath + ' failed ' + test.reason).trim();
}
Text += ' in ' + (test.end - test.start) + ' ms';
console.log(Text.trim());
}
function queueTest (test, queue) {
queue.push(test);
}
function processQueue (queue) {
var promise = q.defer();
var tests = queue.slice(); // makes shallow copy
var running = 0;
var max = options.maxWorkers;
console.log('running tests with: ', options.maxWorkers, ' workers');
function fill (){
while (running <= max){
if (!tests.length){
if (running <= 0){
promise.resolve(queue);
}
break;
} else {
const test = tests.pop();
runTest(test).then(function (){
outputTest(test);
next();
}, function (e){
console.log('there was an error process queue');
console.log(e);
})
running++;
}
}
}
function next (){
running -= 1;
if (!tests.length && running <= 0){
promise.resolve(queue);
} else {
fill();
}
}
fill();
return promise.promise;
}
function runTest (test) {
var promise = q.defer();
test.start = Date.now();
request.post(test.requestOptions, function (err, res, body) {
test.end = Date.now();
var comparePromise;
var expectedOutput;
var statusCode = test.status || 200;
if (err){
failTest(test, promise, err);
return;
}
testResponseTime(test);
test.headers = res.headers;
var response = body.toString('utf8');
test.responseBody = response;
if (!test.outputs) {
failTest(test, promise, 'Test failed: no output file given');
return;
}
if (res.statusCode !== statusCode){
failTest(test, promise, `Expected response code '${statusCode}' , but got '${res.statusCode}'`);
return;
}
//I hate exceptions
try {
expectedOutput = fs.readFileSync(path.join(testFolder, test.groupKey, test.name, test.outputs), 'utf8').toString();
}
catch (e) {
failTest(test, promise, 'No output file exists');
return;
}
if (res.headers['content-type'].indexOf('application/json') > -1) {
comparePromise = testJSON(test, expectedOutput, response, options);
}
else if (res.headers['content-type'].indexOf('text/xml') > -1) {
//For some reason xml parsing is messing stuff up going to run it in a child process
var extension = test.outputs.split('.');
extension = extension[extension.length - 1];
if (extension === 'xsd') {
comparePromise = testXMLSchema(test, expectedOutput, response);
}
else {
comparePromise = testXML(test, expectedOutput, response, options);
}
}
else {
//standarize line endings
if (response.trim() === expectedOutput.trim()) {
let testPromise = q.defer();
test.passed = true;
comparePromise = testPromise.promise;
testPromise.resolve();
}
else {
var index = 0;
var differ = false;
while (response[index] || expectedOutput[index]){
if (response[index] !== expectedOutput[index] && !differ){
differ = true;
}
index++;
}
test.passed = false;
comparePromise = getDifferencesUrl(response, expectedOutput, 'txt', diffUrl, client).then(function (url) {
test.reason = 'Outputs do not match ' + url;
return;
})
}
}
comparePromise.then(function () {
promise.resolve(test);
}, function () {
console.log('error');
}).catch(function (e) {
console.log(e, e.stack);
});
});
return promise.promise;
}
tests.forEach(function (testKey) {
//If there is a test to run from command line only run that one
var testSubset;
if (commandLineTests.length) {
var found = false;
commandLineTests.forEach(function (commandLineTest) { // FIXME unnecessarily looping through the entire loop
if (commandLineTest.test === testKey) {
testSubset = commandLineTest.subTest;
found = true;
}
});
if (!found) {
return;
}
}
var parentTest = testFile[testKey];
parentTest.tests.forEach(function (subTest) {
//If we are running a specific set of inputs only run those
if (testSubset && (testSubset !== subTest.name)) {
return;
}
var test = {
endpoint : parentTest.endpoint,
inputs : subTest.files.inputs,
outputs : subTest.files.output,
name : subTest.name,
ignore : subTest.ignore,
groupKey : testKey,
description : subTest.description,
ms : subTest.ms,
status : subTest.status
}
var promise = q.defer();
testPromises.push(promise.promise);
var input = test.inputs;
var qs = '';
if (subTest.files['input-qs']){
qs = fs.readFileSync(path.join(testFolder, testKey, subTest.name, subTest.files['input-qs'])).toString().trim();
}
var requestOptions = {
url : 'http://' + host + ':' + port + '/' + basePath + test.endpoint + qs,
encoding : null
};
// FIXME simplify if/else statement
if (input && (typeof input === 'object')) {
var fieldNames = Object.keys(input);
var formData = {};
fieldNames.forEach(function (fieldName) {
var field = input[fieldName];
//Ignore description field
formData[fieldName] = fs.readFileSync(path.join(testFolder, testKey, subTest.name, field));
});
requestOptions.formData = formData;
}
else {
if (input) {
requestOptions.body = fs.readFileSync(path.join(testFolder, testKey, subTest.name, input));
}
}
//Kill the test if it takes too long
// test.timeout = setTimeout(function() {
// if (promise.promise.inspect().state === "pending") {
// test.reason = "timed out";
// console.log(promise);
// promise.reject(test);
// }
// }, 30 * 1000);
// test.start = Date.now() - 100;
test.requestOptions = requestOptions;
queueTest(test, testQueue);
});
});
processQueue(testQueue).then(function (tests) {
var passed = 0;
var warnings = 0;
var ignored = 0;
var total = tests.length;
var totalTestTime = 0;
var wallClockTime = Date.now() - timeStart;
tests.forEach(function (test) {
totalTestTime += test.end - test.start;
if ((test.passed && !test.timedOut && !test.ignore)) {
passed++;
}
if (test.warnOnTime) {
warnings++;
}
if (test.ignore){
ignored++;
total--;
}
});
var passingText = passed + '/' + total + ' passed';
if (passed !== total) {
console.log(colors.red(passingText));
}
else {
console.log(colors.green(passingText));
}
if (warnings) {
console.log(colors.yellow(warnings + ' warnings'));
}
if (ignored){
console.log(colors.blue(ignored + ' tests ignored'));
}
console.log(colors.cyan('Testing comlete in:', wallClockTime/1000, 'seconds'));
console.log(colors.cyan('Sum of all test runtimes:', totalTestTime/1000, 'seconds'));
console.log(colors.green('Done testing'));
var exit = process.exit;
if (passed === total){
exit(0);
}
else {
exit(1);
}
}).catch((e) => {
console.log(e);
});
}
function testXMLSchema (test, expected, actual) {
var promise = q.defer();
var child = childProcess.fork(xmlChildPath);
child.send({
xml : actual,
schema : expected
});
child.on('message', function (errors) {
if (!errors) {
test.passed = true;
}
else {
test.passed = false;
test.reason = errors[0];
}
promise.resolve();
child.kill();
});
return promise.promise;
}
function testXML (test, expected, actual, options) {
var promise = q.defer();
xml2js.parseString(actual, function (err, actualJSON){
xml2js.parseString(expected, function (err, expectedJSON){
var valid = deepCompare(actualJSON, expectedJSON);
if (valid){
test.passed = true;
promise.resolve();
} else {
test.passed = false;
test.reason = 'Outputs do not match';
getDifferencesUrl(actualJSON, expectedJSON, 'json', options.diffUrl, options.client).then(function (url) {
test.reason += ' ' + url;
promise.resolve();
});
}
});
});
return promise.promise;
}
function testJSON (test, expected, actual, options) {
var promise = q.defer();
var expectedOutput = parseJSON(expected);
var actualOutput = parseJSON(actual);
if (!expectedOutput) {
failTest(test, promise, 'Couldn\'t parse expected output file as json');
}
else if (!actualOutput) {
failTest(test, promise, 'Couldn\'t parse output from server');
}
else {
if (deepCompare(expectedOutput, actualOutput)) {
test.passed = true;
promise.resolve();
}
else {
test.passed = false;
test.reason = 'Outputs do not match';
getDifferencesUrl(actualOutput, expectedOutput, 'json', options.diffUrl, options.client).then(function (url) {
test.reason += ' ' + url;
promise.resolve();
});
}
}
return promise.promise;
}
function testResponseTime (test) {
if (test.ms) {
var totalTime = test.end - test.start;
var maxTime = (100 * test.ms) / 85;
test.maxTime = maxTime;
if (test.ms < totalTime){
if (maxTime <= totalTime){
test.timedOut = true;
}
else {
test.warnOnTime = true;
}
}
}
}
//A nice convenience function for failing a test
function failTest (test, promise, reason) {
test.reason = reason;
test.passed = false;
promise.resolve(test);
}
function getDifferencesUrl (actual, expected, type, diffUrl, client) {
var promise = q.defer();
var left;
var right;
if (!diffUrl) {
promise.resolve('');
}
else {
if (client) {
return sendToS3(actual, 'actual-', type, client).then(function (url) {
right = encodeURI(url);
return sendToS3(expected, 'expected-', type, client);
}).then(function (url) {
left = encodeURI(url);
var finalUrl = diffUrl + '?left=' + left + '&right=' + right;
return shortenUrl(finalUrl);
});
}
else {
left = encodeURI(JSON.stringify(actual));
right = encodeURI(JSON.stringify(expected));
var url = diffUrl + '?left=' + left + '&right=' + right;
return shortenUrl(url);
}
}
return promise.promise;
}
function sendToS3 (obj, name, type, client) {
var string = obj;
var promise = q.defer();
var contentType;
name = name + Math.random().toString(36).replace(/[^a-z]+/g, '').substr(0, 7);
name = name + '.' + type;
if (type === 'json') {
contentType = 'application/json';
string = JSON.stringify(obj, null, 2);
}
else {
contentType = 'text/plain';
}
var req = client.put(name, {
'Content-Length' : Buffer.byteLength(string),
'Content-Type' : contentType
});
req.on('err', function (err) {
console.log('there was an error s3');
console.log(err);
});
req.on('response', function (res) {
if (200 === res.statusCode) {
promise.resolve(req.url);
}
else {
promise.resolve('');
}
});
req.end(string);
return promise.promise;
}
function shortenUrl (url) {
var promise = q.defer();
if (typeof self.options === 'object'
&& typeof self.options.shortener === 'string'
&& fs.existsSync(self.options.shortener)) {
const shortener = require(path.join(process.cwd(), self.options.shortener))
shortener(url, (shortUrl) => {
promise.resolve(shortUrl)
})
return promise.promise
}
return url
}
function parseJSON (string) {
var result;
try {
result = JSON.parse(string);
}
catch (e) {
result = null;
}
return result;
}
exports.main = main;
function deepCompare (ar1, ar2) {
var matches = true;
var type1 = typeof ar1;
var type2 = typeof ar2;
if ((ar1 === null) || (ar2 === null)) {
matches = ar1 === ar2;
return matches;
}
if (type1 !== type2) {
matches = false;
}
else {
switch (typeof ar1) {
case 'object': {
var keys1 = Object.keys(ar1);
var keys2 = Object.keys(ar2);
if (!Array.isArray(ar1)) {
keys1.sort();
keys2.sort();
}
if (keys1.length !== keys2.length) {
matches = false;
}
else {
keys1.every(function (key1, n) {
if (key1 !== keys2[n]) {
matches = false;
return matches;
}
else {
matches = deepCompare(ar1[key1], ar2[key1]);
return matches;
}
});
}
break;
}
case 'string':
case 'boolean':
case 'number': {
matches = ar1 === ar2;
break;
}
default: {
console.log(ar1);
console.log('what happened');
break;
}
}
}
return matches;
}