plugins/greylist.js
// Greylisting plugin for Haraka
// version 0.1.4
// node builtins
const net = require('net');
const util = require('util');
// Haraka modules
const DSN = require('haraka-dsn');
const tlds = require('haraka-tld');
const net_utils = require('haraka-net-utils');
const { Address } = require('address-rfc2821');
// External NPM modules
const ipaddr = require('ipaddr.js');
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
exports.register = function (next) {
this.inherits('haraka-plugin-redis');
this.load_config();
this.register_hook('init_master', 'init_redis_plugin');
this.register_hook('init_child', 'init_redis_plugin');
// redundant - using the special hook_ nomenclature
// this.register_hook('rcpt_ok', 'hook_rcpt_ok');
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
exports.load_config = function () {
this.cfg = this.config.get('greylist.ini', {
booleans : [
'+skip.dnswlorg',
'-skip.mailspikewl'
]
}, () => {
this.load_config();
});
this.merge_redis_ini();
this.load_config_lists();
}
// Load various configuration lists
exports.load_config_lists = function () {
const plugin = this;
plugin.whitelist = {};
plugin.list = {};
function load_list (type, file_name) {
plugin.whitelist[type] = {};
const list = Object.keys(plugin.cfg[file_name]);
// toLower when loading spends a fraction of a second at load time
// to save millions of seconds during run time.
for (const element of list) {
plugin.whitelist[type][element.toLowerCase()] = true;
}
plugin.logdebug(`whitelist {${type}} loaded from ${file_name} with ${list.length} entries`);
}
function load_ip_list (type, file_name) {
plugin.whitelist[type] = [];
const list = Object.keys(plugin.cfg[file_name]);
for (const element of list) {
try {
let addr = element;
if (addr.match(/\/\d+$/)) {
addr = ipaddr.parseCIDR(addr);
}
else {
addr = ipaddr.parseCIDR(`${addr}${((net.isIPv6(addr)) ? '/128' : '/32')}`);
}
plugin.whitelist[type].push(addr);
}
catch (e) {}
}
plugin.logdebug(`whitelist {${type}} loaded from ${file_name} with ${plugin.whitelist[type].length} entries`);
}
function load_config_list (type, file_name) {
plugin.list[type] = Object.keys(plugin.cfg[file_name]);
plugin.logdebug(`list {${type}} loaded from ${file_name} with ${plugin.list[type].length} entries`);
}
load_list('mail', 'envelope_whitelist');
load_list('rcpt', 'recipient_whitelist');
load_ip_list('ip', 'ip_whitelist');
load_config_list('dyndom', 'special_dynamic_domains');
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
exports.shutdown = function () {
if (this.db) this.db.quit();
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// We check for IP and envelope whitelist
exports.hook_mail = function (next, connection, params) {
if (!connection.transaction) return next();
const mail_from = params[0];
// whitelist checks
if (this.ip_in_list(connection.remote.ip)) { // check connecting IP
this.loginfo(connection, 'Connecting IP was whitelisted via config');
connection.transaction.results.add(this, { skip : 'config-whitelist(ip)' })
}
else if (this.addr_in_list('mail', mail_from.address().toLowerCase())) { // check envelope (email & domain)
this.loginfo(connection, 'Envelope was whitelisted via config');
connection.transaction.results.add(this, { skip : 'config-whitelist(envelope)' });
}
else {
const why_skip = this.process_skip_rules(connection);
if (why_skip) {
this.loginfo(connection, `Requested to skip the GL because skip rule matched: ${why_skip}`);
connection.transaction.results.add(this, { skip : `requested(${why_skip})` });
}
}
next();
}
//
exports.hook_rcpt_ok = function (next, connection, rcpt) {
if (this.should_skip_check(connection)) return next();
if (this.was_whitelisted_in_session(connection)) {
this.logdebug(connection, 'host already whitelisted in this session');
return next();
}
const { transaction } = connection
const ctr = transaction.results;
const { mail_from } = transaction;
// check rcpt in whitelist (email & domain)
if (this.addr_in_list('rcpt', rcpt.address().toLowerCase())) {
this.loginfo(connection, 'RCPT was whitelisted via config');
ctr.add(this, { skip : 'config-whitelist(recipient)' });
return next();
}
this.check_and_update_white(connection, (err, white_rec) => {
if (err) {
this.logerror(connection, `Got error: ${util.inspect(err)}`);
return next(DENYSOFT, DSN.sec_unspecified('Backend failure. Please, retry later or contact our support.'));
}
if (white_rec) {
this.logdebug(connection, 'host in WHITE zone');
ctr.add(this, { pass : 'whitelisted' });
ctr.push(this, { stats : { rcpt : white_rec }, stage : 'rcpt' });
return next();
}
return this.process_tuple(connection, mail_from.address(), rcpt.address(), (err2, white_promo_rec) => {
if (err2) {
if (err2 instanceof Error && err2.notanerror) {
this.logdebug(connection, 'host in GREY zone');
ctr.add(this, {
fail : 'greylisted'
});
ctr.push(this, {
stats : {
rcpt : err2.record
},
stage : 'rcpt'
});
return this.invoke_outcome_cb(next, false);
}
throw err2;
}
if (!white_promo_rec) {
ctr.add(this, {
fail : 'greylisted',
stage : 'rcpt'
});
return this.invoke_outcome_cb(next, false);
}
else {
this.loginfo(connection, 'host has been promoted to WHITE zone');
ctr.add(this, {
pass : 'whitelisted',
stats : white_promo_rec,
stage : 'rcpt'
});
ctr.add(this, {
pass : 'whitelisted'
});
return this.invoke_outcome_cb(next, true);
}
})
})
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// Main GL engine that accepts tuple and returns matched record or a rejection.
exports.process_tuple = function (connection, sender, rcpt, cb) {
const key = this.craft_grey_key(connection, sender, rcpt);
if (!key) return;
return this.db_lookup(key, (err, record) => {
if (err) {
if (err instanceof Error && err.what == 'db_error')
this.logwarn(connection, `got err from DB: ${util.inspect(err)}`);
throw err;
}
this.logdebug(connection, `got record: ${util.inspect(record)}`);
// { created: TS, updated: TS, lifetime: TTL, tried: Integer }
const now = Date.now() / 1000;
if (record &&
(record.created + this.cfg.period.black < now) &&
(record.created + record.lifetime >= now)) {
// Host passed greylisting
return this.promote_to_white(connection, record, cb);
}
return this.update_grey(key, !record, (err2, created_record) => {
const err3 = new Error('in black zone');
err3.record = created_record || record;
err3.notanerror = true;
return cb(err3, null);
});
});
}
// Checks if host is _white_. Updates stats if so.
exports.check_and_update_white = function (connection, cb) {
const key = this.craft_white_key(connection);
return this.db_lookup(key, (err, record) => {
if (err) {
this.logwarn(connection, `got err from DB: ${util.inspect(err)}`);
throw err;
}
if (record) {
if (record.updated + record.lifetime - 2 < Date.now() / 1000) { // race "prevention".
this.logerror(connection, "Mischief! Race condition triggered.");
return cb(new Error('drunkard'));
}
return this.update_white_record(key, record, cb);
}
return cb(null, false);
});
}
// invokes next() depending on outcome param
exports.invoke_outcome_cb = function (next, is_whitelisted) {
if (is_whitelisted) return next();
const text = this.cfg.main.text || '';
return next(DENYSOFT, DSN.sec_unauthorized(text, '451'));
}
// Should we skip greylisting invokation altogether?
exports.should_skip_check = function (connection) {
const { transaction, relaying, remote } = connection ?? {}
if (!transaction) return true;
const ctr = transaction.results;
if (relaying) {
this.logdebug(connection, 'skipping GL for relaying host');
ctr.add(this, {
skip : 'relaying'
});
return true;
}
if (remote?.is_private) {
connection.logdebug(this, `skipping private IP: ${connection.remote.ip}`);
ctr.add(this, {
skip : 'private-ip'
});
return true;
}
if (ctr) {
if (ctr.has(this, 'skip', /^config-whitelist/)) {
this.loginfo(connection, 'skipping GL for host whitelisted in config');
return true;
}
if (ctr.has(this, 'skip', /^requested/)) {
this.loginfo(connection, 'skipping GL because was asked to previously');
return true;
}
}
return false;
}
// Was whitelisted previously in this session
exports.was_whitelisted_in_session = function (connection) {
if (!connection?.transaction?.results) return false;
return connection.transaction.results.has(this, 'pass', 'whitelisted');
}
exports.process_skip_rules = function (connection) {
const cr = connection.results;
const skip_cfg = this.cfg.skip;
if (skip_cfg) {
if (skip_cfg.dnswlorg && cr.has('dnswl.org', 'pass', /^list\.dnswl\.org\([123]\)$/)) {
return 'dnswl.org(MED)'
}
if (skip_cfg.mailspikewl && cr.has('dnswl.org', 'pass', /^wl\.mailspike\.net\((1[7-9]|20)\)$/)) {
return 'mailspike(H2)'
}
}
return '';
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// Build greylist DB key (originally, a "tuple") off supplied params.
// When _to_ is false, we craft +sender+ key
// When _to_ is String, we craft +rcpt+ key
exports.craft_grey_key = function (connection, from, to) {
const crafted_host_id = this.craft_hostid(connection)
if (!crafted_host_id) return null;
let key = `grey:${crafted_host_id}:${(from || '<>')}`;
if (to != undefined) {
key += `:${(to || '<>')}`;
}
return key;
}
// Build white DB key off supplied params.
exports.craft_white_key = function (connection) {
return `white:${this.craft_hostid(connection)}`;
}
// Return so-called +hostid+.
exports.craft_hostid = function (connection) {
const plugin = this;
const { transaction, remote } = connection ?? {};
if (!transaction || !remote) return null;
if (transaction.notes?.greylist?.hostid) {
return transaction.notes.greylist.hostid; // "caching"
}
function chsit (value, reason) { // cache the return value
if (!value)
plugin.logdebug(connection, `hostid set to IP: ${reason}`);
transaction.results.add(plugin, {
hostid_type : value ? 'domain' : 'ip',
rdns : (value || remote.ip),
msg : reason
}); // !don't move me.
value = value || remote.ip;
return ((transaction.notes.greylist = transaction.notes.greylist || {}).hostid = value);
}
// no rDNS . FIXME: use fcrdns results
if (!remote.host || [ 'Unknown' | 'DNSERROR' ].includes(remote.host))
return chsit(null, 'no rDNS info for this host');
remote.host = remote.host.replace(/\.$/, ''); // strip ending dot, just in case
const fcrdns = connection.results.get('fcrdns');
if (!fcrdns) {
plugin.logwarn(connection, 'No FcrDNS plugin results, fix this.');
return chsit(null, 'no FcrDNS plugin results');
}
if (!connection.results.has('fcrdns', 'pass', 'fcrdns')) // FcrDNS failed
return chsit(null, 'FcrDNS failed');
if (connection.results.get('fcrdns').ptr_names.length > 1) // multiple PTR returned
return chsit(null, 'multiple PTR returned');
if (connection.results.has('fcrdns', 'fail', /^is_generic/)) // generic/dynamic rDNS record
return chsit(null, 'rDNS is a generic record');
if (connection.results.has('fcrdns', 'fail', /^valid_tld/)) // invalid org domain in rDNS
return chsit(null, 'invalid org domain in rDNS');
// strip first label up until the tld boundary.
const decoupled = tlds.split_hostname(!remote.host, 3);
const vardom = decoupled[0]; // "variable" portion of domain
const dom = decoupled[1]; // "static" portion of domain
// we check for special cases where rdns looks custom/static, but really is dynamic
const special_case_info = plugin.check_rdns_for_special_cases(!remote.host, vardom);
if (special_case_info) return chsit(null, special_case_info.why);
let stripped_dom = dom;
if (vardom) {
// check for decimal IP in rDNS
if (vardom.match(String(net_utils.ip_to_long(remote.ip))))
return chsit(null, 'decimal IP');
// craft the +hostid+
const label = vardom.split('.').slice(1).join('.');
if (label)
stripped_dom = `${label}.${stripped_dom}`;
}
return chsit(stripped_dom);
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
// Retrieve _grey_ record
// not implemented
exports.retrieve_grey = function (rcpt_key, sender_key, cb) {
const multi = this.db.multi();
multi.hgetall(rcpt_key);
multi.hgetall(sender_key);
multi.exec((err, result) => {
if (err) {
this.lognotice(`DB error: ${util.inspect(err)}`);
err.what = 'db_error';
throw err;
}
return cb(err, result);
});
}
// Update or create _grey_ record
exports.update_grey = function (key, create, cb) {
const multi = this.db.multi();
const ts_now = Math.round(Date.now() / 1000);
let new_record;
if (create) {
const lifetime = this.cfg.period.grey;
new_record = {
created : ts_now,
updated : ts_now,
lifetime,
tried : 1
};
multi.hmset(key, new_record);
multi.expire(key, lifetime);
}
else {
multi.hincrby(key, 'tried', 1);
multi.hmset(key, {
updated : ts_now
});
}
multi.exec((err, records) => {
if (err) {
this.lognotice(`DB error: ${util.inspect(err)}`);
err.what = 'db_error';
throw err;
}
return cb(null, ((create) ? new_record : false));
});
}
// Promote _grey_ record to _white_.
exports.promote_to_white = function (connection, grey_rec, cb) {
const ts_now = Math.round(Date.now() / 1000);
const white_ttl = this.cfg.period.white;
// { first_connect: TS, whitelisted: TS, updated: TS, lifetime: TTL, tried: Integer, tried_when_greylisted: Integer }
const white_rec = {
first_connect : grey_rec.created,
whitelisted : ts_now,
updated : ts_now,
lifetime : white_ttl,
tried_when_greylisted : grey_rec.tried,
tried : 1
};
const white_key = this.craft_white_key(connection);
if (!white_key) return;
return this.db.hmset(white_key, white_rec, (err, result) => {
if (err) {
this.lognotice(`DB error: ${util.inspect(err)}`);
err.what = 'db_error';
throw err;
}
this.db.expire(white_key, white_ttl, (err2, result2) => {
if (err2) {
this.lognotice(`DB error: ${util.inspect(err2)}`);
}
return cb(err2, result2);
});
});
}
// Update _white_ record
exports.update_white_record = function (key, record, cb) {
const multi = this.db.multi();
const ts_now = Math.round(Date.now() / 1000);
// { first_connect: TS, whitelisted: TS, updated: TS, lifetime: TTL, tried: Integer, tried_when_greylisted: Integer }
multi.hincrby(key, 'tried', 1);
multi.hmset(key, {
updated : ts_now
});
multi.expire(key, record.lifetime);
return multi.exec((err2, record2) => {
if (err2) {
this.lognotice(`DB error: ${util.inspect(err2)}`);
err2.what = 'db_error';
throw err2;
}
return cb(null, record2);
});
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
exports.db_lookup = function (key, cb) {
this.db.hgetall(key, (err, result) => {
if (err) {
this.lognotice(`DB error: ${util.inspect(err)}`, key);
}
if (result && typeof result === 'object') { // groom known-to-be numeric values
['created', 'updated', 'lifetime', 'tried', 'first_connect', 'whitelisted', 'tried_when_greylisted'].forEach(kk => {
const val = result[kk];
if (val !== undefined) {
result[kk] = Number(val);
}
});
}
return cb(null, result);
});
}
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
exports.addr_in_list = function (type, address) {
if (!this.whitelist[type]) {
this.logwarn(`List not defined: ${type}`);
return false;
}
if (this.whitelist[type][address]) {
return true;
}
try {
const addr = new Address(address);
return !!this.whitelist[type][addr.host];
}
catch (err) {
return false;
}
}
exports.ip_in_list = function (ip) {
const ipobj = ipaddr.parse(ip);
const list = this.whitelist.ip;
for (const element of list) {
try {
if (ipobj.match(element)) {
return true;
}
}
catch (e) {}
}
return false;
}
// Match patterns in the list against (end of) domain
exports.domain_in_list = function (list_name, domain) {
const list = this.list[list_name];
if (!list) {
this.logwarn(`List not defined: ${list_name}`);
return false;
}
for (const element of list) {
if (domain.length - domain.lastIndexOf(element) == element.length)
return true;
}
return false;
}
// Check for special rDNS cases
// @return {type: 'dynamic'} if rnds is dynamic (hostid should be IP)
exports.check_rdns_for_special_cases = function (domain, label) {
// ptr for these is in fact dynamic
if (this.domain_in_list('dyndom', domain))
return {
type : 'dynamic',
why : 'rDNS considered dynamic: listed in dynamic.domains config list'
};
return false;
}