inventid/iaas

import "babel-polyfill";

import express from "express";
import config from "config";
import bodyParser from "body-parser";
import formidable from "formidable";
import log from "./log";
import urlParameters, {hasFiltersApplied} from "./urlParameters";
import * as imageResponse from "./imageResponse";
import * as token from "./token";
import {areAllDefined, roundedRatio} from "./helper";
import IntegerCounter from "./integerCounter";
import metrics, {metricFromParams, REQUEST_TOKEN, UPLOAD} from "./metrics";
import timingMetric from "./metrics/timingMetric";
import database from './databases';
import robotsTxt, {syncRobotsTxt} from "./robotsTxt";
import startMigrations from './migrations';
import {MAX_IMAGE_IN_MP} from "./sizes";

process.on('uncaughtException', function (err) {
  log('error', err);
  process.exit(1);
});

const hitCounter = IntegerCounter();
const missCounter = IntegerCounter();
const uploadCounter = IntegerCounter();

const startedAt = new Date();

const stats = {
  hits: hitCounter,
  misses: missCounter,
  uploads: uploadCounter,
  get: () => {
    const hits = hitCounter.get();
    const misses = missCounter.get();
    const uploads = uploadCounter.get();
    const total = (hits + misses);
    const datetime = new Date().toISOString();
    const uptimeInSeconds = (new Date() - startedAt) / 1000;
    const generationsPerMinute = roundedRatio(misses, uptimeInSeconds / 60);
    return {
      datetime,
      hits,
      misses,
      uploads,
      uptimeInSeconds,
      generationsPerMinute,
      cacheHitRatio: roundedRatio(hits, total)
    };
  }
};
let statsPrinter;

const promiseUpload = (form, request) => {
  return new Promise((resolve, reject) => form.parse(request, (err, fields, files) => err ? reject(err) : resolve(files)));
};

const getValidDimensionParameter = (value) => {
  // Convert to a number and drop the decimals
  const number = ~~Number(value);
  if (isNaN(number) || number < 0) {
    return undefined;
  }
  return number;
};

const cropParametersOnUpload = (req) => {
  const xOffset = getValidDimensionParameter(req.query.x);
  const yOffset = getValidDimensionParameter(req.query.y);
  const width = getValidDimensionParameter(req.query.width);
  const height = getValidDimensionParameter(req.query.height);
  if (areAllDefined([xOffset, yOffset, width, height])) {
    return {
      xOffset, yOffset, width, height
    };
  }
  return null;
};

const uploadImage = async (req, res) => {
  const sentToken = req.headers['x-token'];
  const name = req.params.name;
  log('info', `Requested image upload for image_id ${name} with token ${sentToken}`);
  const metric = timingMetric(UPLOAD, {fields: {name: name}});

  const canConsumeToken = await token.consumeToken(sentToken, name);
  if (!canConsumeToken) {
    res.status(403).end();
    metric.addTag('status', 403);
    metrics.write(metric);
    return;
  }

  let uploadCompleted = false;
  let uploadCancelled = false;
  // When an uploads gets cancelled in the progress, we want to make the token available again
  const handleCancelledUpload = async () => {
    if (!uploadCompleted && !uploadCancelled) {
      log('info', `Freeing token for ${name} as the upload was aborted prior to completion`);
      await token.deleteTokenForImageId(name);
      uploadCancelled = true;
    }
  };

  try {
    req.once('close', handleCancelledUpload);

    // Valid token
    const form = new formidable.IncomingForm();
    const files = await promiseUpload(form, req);
    if (!files.image || !files.image.path) {
      res.status(400).end();
      metric.addTag('status', 400);
      metrics.write(metric);
      return;
    }

    const isAllowedToHandle = await imageResponse.hasAllowableImageSize(files.image.path, MAX_IMAGE_IN_MP);
    if (!isAllowedToHandle) {
      log('warn', `Image ${name} was too big to handle (max ${MAX_IMAGE_IN_MP} megapixel allowed) and hence rejected`);
      res.status(413).end();
      metric.addTag('status', 413);
      metrics.write(metric);
      await token.deleteTokenForImageId(name);
      return;
    }

    const cropParameters = cropParametersOnUpload(req);
    if (uploadCancelled) {
      // If the image is cancelled at this point, save ourselves the trouble of doing any image work
      return;
    }

    const result = await imageResponse.upload(name, files.image.path, cropParameters);
    if (uploadCancelled) {
      // At this point the image would already be on disk. However as the client cannot get the final OK, we discard
      // saving it to the database, and instead let the client retry. Also there is a tiny race condition otherwise.
      // While the await for the upload is holding the cancel can occur. In that case the upload has not been marked as
      // ok (yet), so the token is freed, whereas the file is persisted to disk already.
      return;
    }

    stats.uploads.incrementAndGet();
    log('info', `Finished writing original file ${name}`);
    await token.markAsCompleted(sentToken, name);
    // From this point on we no longer clear the token as the upload was successful
    uploadCompleted = true;
    res.json({
      status: 'OK',
      id: name,
      original_height: result.originalHeight,
      original_width: result.originalWidth
    });
    metric.addTag('status', 200);
    metrics.write(metric);
  } catch (e) {
    await handleCancelledUpload();
  }
};

const server = express();
server.use(bodyParser.json());
server.use((req, res, next) => {
  // Allow cross origin
  res.set({
    'Access-Control-Allow-Origin': '*',
    'Access-Control-Allow-Headers': 'accept, content-type',
    'Access-Control-Allow-Method': 'GET'
  });
  next();
});
server.get('/_health', async (req, res) => {
  const dbIsOk = await database.isDbAlive();
  if (dbIsOk) {
    res.status(200).end('OK');
  } else {
    res.status(500).end('No database connection');
    log('error', 'Healthcheck FAILED');
  }
});
server.get('/robots.txt', (req, res) => {
  res.status(200).end(robotsTxt());
  log('debug', 'Robots.txt served');
});
server.get('/favicon.ico', (req, res) => {
  res.status(404);
});

function patchConnectionForTermination(req, params) {
  const connectionStart = new Date();
  const description = imageResponse.description(params);
  req.once('close', () => {
    const connectionEnd = new Date();
    // eslint-disable-next-line max-len
    log('warn', `Client disconnected prematurely (${connectionEnd - connectionStart}ms after creating the connection). Terminating stream for ${description}`);
  });
}

const isFreshnessKeyConfigured = config.has('freshness') && config.get('freshness').length > 0;
function isFreshImageRequired(req) {
  if (!isFreshnessKeyConfigured) {
    return false;
  }
  return (req.headers && req.headers['freshness-token'] && req.headers['freshness-token'] === config.get('freshness'));
}

function serveResizedImage(req, res) {
  const params = urlParameters(req);
  patchConnectionForTermination(req, params);
  imageResponse.magic(params, req.method, res, stats, metricFromParams(params), isFreshImageRequired(req));
}

// The actual endpoints for fetching
// Serve a resized image with scaling
server.get('/(:name)_(:width)_(:height)_(:scale)x.(:format)', serveResizedImage);
// Serve a resized image
server.get('/(:name)_(:width)_(:height).(:format)', serveResizedImage);
server.get('/(:name).(:format)', (req, res) => {
  // Serve the original, with optionally filters applied
  const params = urlParameters(req, false);
  patchConnectionForTermination(req, params);
  if (hasFiltersApplied(params)) {
    imageResponse.magic(params, req.method, res, stats, metricFromParams(params), isFreshImageRequired(req));
  } else {
    imageResponse.original(params, req.method, res, metricFromParams(params));
  }
});

const isSecretKeyConfigured = config.has('secret') && config.get('secret').length > 0;

// The upload stuff
server.post('/token', async (req, res) => {
  // If we have a secret key configured, check whether it matches first
  if (isSecretKeyConfigured && config.get('secret') !== req.body.secret) {
    res.status(401).json({error: 'A secret key is required for this server, and it was not supplied or incorrect'});
    return;
  }

  // Create a token
  const image = req.body.id;
  const metric = timingMetric(REQUEST_TOKEN, {fields: {name: image}});
  if (image === null) {
    res.status(400).end();
    metric.addTag('status', 400);
    metrics.write(metric);
    return;
  }
  const newToken = await token.createToken(image);
  if (!newToken) {
    // Duplicate
    res.status(403).json({error: 'The requested image_id is already requested'});
    metric.addTag('status', 403);
    metrics.write(metric);
    return;
  }
  res.json({token: newToken});
  metric.addTag('status', 200);
  metrics.write(metric);
});
server.post('/(:name).(:format)', uploadImage);
server.post('/(:name)', uploadImage);

const slowShutdown = (expressInstance, timeout = 100) => setTimeout(() => {
  if (expressInstance) {
    expressInstance.close();
  }
  if (statsPrinter) {
    clearInterval(statsPrinter);
  }

  // We try to give the database pool back if that is possible. If it does not succeed within 5 seconds we just quit
  // Most likely the database crashed in that case anyway
  const onDone = () => process.exit(2);
  setTimeout(onDone, 5000);
  database.close().then(onDone);
}, timeout);

database.migrate((err) => {
  if (err) {
    log('error', `Error fetching client from pool for migrations: ${err}`);
    slowShutdown(null, 250);
    return;
  }
  const port = process.env.PORT || 1337; //eslint-disable-line no-process-env
  const handler = server.listen(port, () => {
    log('info', `Server started listening on port ${port}`);
    if (isSecretKeyConfigured) {
      log('info', `Server is configured with a secret key for /token`);
    }
  });
  // Sync robots.txt
  syncRobotsTxt();
  // Run any required migrations
  startMigrations();

  // Log the stats every 5 minutes if enabled
  statsPrinter = setInterval(() => log('stats', stats.get()), 5 * 60 * 1000);
  const timeoutDelay = Math.floor(Math.random() * 2500);
  // Delay the checking a bit randomly, as otherwise everyone hugs the connections at the same time
  setTimeout(() => {
    const dbChecker = setInterval(async () => {
      const isAlive = await database.isDbAlive();

      if (!isAlive) {
        clearInterval(dbChecker);
        log('error', 'Database connection went offline! Restarting the application so we can connect to another one');
        // Slight timeout to handle some final requests?
        slowShutdown(handler);
      }
    }, 2500);
  }, timeoutDelay);
});