NodeBB/NodeBB

'use strict';

const cronJob = require('cron').CronJob;
const winston = require('winston');
const nconf = require('nconf');
const crypto = require('crypto');
const util = require('util');
const _ = require('lodash');

const sleep = util.promisify(setTimeout);

const db = require('./database');
const utils = require('./utils');
const plugins = require('./plugins');
const meta = require('./meta');
const pubsub = require('./pubsub');
const cacheCreate = require('./cache/lru');

const Analytics = module.exports;

const secret = nconf.get('secret');

let local = {
    counters: {},
    pageViews: 0,
    pageViewsRegistered: 0,
    pageViewsGuest: 0,
    pageViewsBot: 0,
    uniqueIPCount: 0,
    uniquevisitors: 0,
};
const empty = _.cloneDeep(local);
const total = _.cloneDeep(local);

let ipCache;

const runJobs = nconf.get('runJobs');

Analytics.init = async function () {
    ipCache = cacheCreate({
        max: parseInt(meta.config['analytics:maxCache'], 10) || 500,
        ttl: 0,
    });

    new cronJob('*/10 * * * * *', (async () => {
        publishLocalAnalytics();
        if (runJobs) {
            await sleep(2000);
            await Analytics.writeData();
        }
    }), null, true);

    if (runJobs) {
        pubsub.on('analytics:publish', (data) => {
            incrementProperties(total, data.local);
        });
    }
};

function publishLocalAnalytics() {
    pubsub.publish('analytics:publish', {
        local: local,
    });
    local = _.cloneDeep(empty);
}

function incrementProperties(obj1, obj2) {
    for (const [key, value] of Object.entries(obj2)) {
        if (typeof value === 'object') {
            incrementProperties(obj1[key], value);
        } else if (utils.isNumber(value)) {
            obj1[key] = obj1[key] || 0;
            obj1[key] += obj2[key];
        }
    }
}

Analytics.increment = function (keys, callback) {
    keys = Array.isArray(keys) ? keys : [keys];

    plugins.hooks.fire('action:analytics.increment', { keys: keys });

    keys.forEach((key) => {
        local.counters[key] = local.counters[key] || 0;
        local.counters[key] += 1;
    });

    if (typeof callback === 'function') {
        callback();
    }
};

Analytics.getKeys = async () => db.getSortedSetRange('analyticsKeys', 0, -1);

Analytics.pageView = async function (payload) {
    local.pageViews += 1;

    if (payload.uid > 0) {
        local.pageViewsRegistered += 1;
    } else if (payload.uid < 0) {
        local.pageViewsBot += 1;
    } else {
        local.pageViewsGuest += 1;
    }

    if (payload.ip) {
        // Retrieve hash or calculate if not present
        let hash = ipCache.get(payload.ip + secret);
        if (!hash) {
            hash = crypto.createHash('sha1').update(payload.ip + secret).digest('hex');
            ipCache.set(payload.ip + secret, hash);
        }

        const score = await db.sortedSetScore('ip:recent', hash);
        if (!score) {
            local.uniqueIPCount += 1;
        }
        const today = new Date();
        today.setHours(today.getHours(), 0, 0, 0);
        if (!score || score < today.getTime()) {
            local.uniquevisitors += 1;
            await db.sortedSetAdd('ip:recent', Date.now(), hash);
        }
    }
};

Analytics.writeData = async function () {
    const today = new Date();
    const month = new Date();
    const dbQueue = [];
    const incrByBulk = [];

    // Build list of metrics that were updated
    let metrics = [
        'pageviews',
        'pageviews:month',
    ];
    metrics.forEach((metric) => {
        const toAdd = ['registered', 'guest', 'bot'].map(type => `${metric}:${type}`);
        metrics = [...metrics, ...toAdd];
    });
    metrics.push('uniquevisitors');

    today.setHours(today.getHours(), 0, 0, 0);
    month.setMonth(month.getMonth(), 1);
    month.setHours(0, 0, 0, 0);

    if (total.pageViews > 0) {
        incrByBulk.push(['analytics:pageviews', total.pageViews, today.getTime()]);
        incrByBulk.push(['analytics:pageviews:month', total.pageViews, month.getTime()]);
        total.pageViews = 0;
    }

    if (total.pageViewsRegistered > 0) {
        incrByBulk.push(['analytics:pageviews:registered', total.pageViewsRegistered, today.getTime()]);
        incrByBulk.push(['analytics:pageviews:month:registered', total.pageViewsRegistered, month.getTime()]);
        total.pageViewsRegistered = 0;
    }

    if (total.pageViewsGuest > 0) {
        incrByBulk.push(['analytics:pageviews:guest', total.pageViewsGuest, today.getTime()]);
        incrByBulk.push(['analytics:pageviews:month:guest', total.pageViewsGuest, month.getTime()]);
        total.pageViewsGuest = 0;
    }

    if (total.pageViewsBot > 0) {
        incrByBulk.push(['analytics:pageviews:bot', total.pageViewsBot, today.getTime()]);
        incrByBulk.push(['analytics:pageviews:month:bot', total.pageViewsBot, month.getTime()]);
        total.pageViewsBot = 0;
    }

    if (total.uniquevisitors > 0) {
        incrByBulk.push(['analytics:uniquevisitors', total.uniquevisitors, today.getTime()]);
        total.uniquevisitors = 0;
    }

    if (total.uniqueIPCount > 0) {
        dbQueue.push(db.incrObjectFieldBy('global', 'uniqueIPCount', total.uniqueIPCount));
        total.uniqueIPCount = 0;
    }

    for (const [key, value] of Object.entries(total.counters)) {
        incrByBulk.push([`analytics:${key}`, value, today.getTime()]);
        metrics.push(key);
        delete total.counters[key];
    }

    if (incrByBulk.length) {
        dbQueue.push(db.sortedSetIncrByBulk(incrByBulk));
    }

    // Update list of tracked metrics
    dbQueue.push(db.sortedSetAdd('analyticsKeys', metrics.map(() => +Date.now()), metrics));

    try {
        await Promise.all(dbQueue);
    } catch (err) {
        winston.error(`[analytics] Encountered error while writing analytics to data store\n${err.stack}`);
    }
};

Analytics.getHourlyStatsForSet = async function (set, hour, numHours) {
    // Guard against accidental ommission of `analytics:` prefix
    if (!set.startsWith('analytics:')) {
        set = `analytics:${set}`;
    }

    const terms = {};
    const hoursArr = [];

    hour = new Date(hour);
    hour.setHours(hour.getHours(), 0, 0, 0);

    for (let i = 0, ii = numHours; i < ii; i += 1) {
        hoursArr.push(hour.getTime() - (i * 3600 * 1000));
    }

    const counts = await db.sortedSetScores(set, hoursArr);

    hoursArr.forEach((term, index) => {
        terms[term] = parseInt(counts[index], 10) || 0;
    });

    const termsArr = [];

    hoursArr.reverse();
    hoursArr.forEach((hour) => {
        termsArr.push(terms[hour]);
    });

    return termsArr;
};

Analytics.getDailyStatsForSet = async function (set, day, numDays) {
    // Guard against accidental ommission of `analytics:` prefix
    if (!set.startsWith('analytics:')) {
        set = `analytics:${set}`;
    }

    day = new Date(day);
    // set the date to tomorrow, because getHourlyStatsForSet steps *backwards* 24 hours to sum up the values
    day.setDate(day.getDate() + 1);
    day.setHours(0, 0, 0, 0);

    async function getHourlyStats(hour) {
        const dayData = await Analytics.getHourlyStatsForSet(
            set,
            hour,
            24
        );
        return dayData.reduce((cur, next) => cur + next);
    }
    const hours = [];
    while (numDays > 0) {
        hours.push(day.getTime() - (1000 * 60 * 60 * 24 * (numDays - 1)));
        numDays -= 1;
    }

    return await Promise.all(hours.map(getHourlyStats));
};

Analytics.getUnwrittenPageviews = function () {
    return local.pageViews;
};

Analytics.getSummary = async function () {
    const today = new Date();
    today.setHours(0, 0, 0, 0);

    const [seven, thirty] = await Promise.all([
        Analytics.getDailyStatsForSet('analytics:pageviews', today, 7),
        Analytics.getDailyStatsForSet('analytics:pageviews', today, 30),
    ]);

    return {
        seven: seven.reduce((sum, cur) => sum + cur, 0),
        thirty: thirty.reduce((sum, cur) => sum + cur, 0),
    };
};

Analytics.getCategoryAnalytics = async function (cid) {
    return await utils.promiseParallel({
        'pageviews:hourly': Analytics.getHourlyStatsForSet(`analytics:pageviews:byCid:${cid}`, Date.now(), 24),
        'pageviews:daily': Analytics.getDailyStatsForSet(`analytics:pageviews:byCid:${cid}`, Date.now(), 30),
        'topics:daily': Analytics.getDailyStatsForSet(`analytics:topics:byCid:${cid}`, Date.now(), 7),
        'posts:daily': Analytics.getDailyStatsForSet(`analytics:posts:byCid:${cid}`, Date.now(), 7),
    });
};

Analytics.getErrorAnalytics = async function () {
    return await utils.promiseParallel({
        'not-found': Analytics.getDailyStatsForSet('analytics:errors:404', Date.now(), 7),
        toobusy: Analytics.getDailyStatsForSet('analytics:errors:503', Date.now(), 7),
    });
};

Analytics.getBlacklistAnalytics = async function () {
    return await utils.promiseParallel({
        daily: Analytics.getDailyStatsForSet('analytics:blacklist', Date.now(), 7),
        hourly: Analytics.getHourlyStatsForSet('analytics:blacklist', Date.now(), 24),
    });
};

require('./promisify')(Analytics);