Function results has 134 lines of code (exceeds 25 allowed). Consider refactoring.

Open

    const results = _.map(urls, (url, index) => {
        console.log('Registered ' + url.url);
        const processResults = processOne({
            config,
            url,

Function processOneCallback has 122 lines of code (exceeds 25 allowed). Consider refactoring.

Open

        }, function processOneCallback(error, record) {
            const id = urlCount + 1;
            if (!!error || !record) {
                console.error('%d - Error! %s', id, error.message);
                if (!!error && !error.retry) {

Function exports has 106 lines of code (exceeds 25 allowed). Consider refactoring.

Open

module.exports = (data, cb) => {
    const config = data.config;
    const plugins = data.plugins;
    const isRetry = !!data.isRetry;
    const url = data.url;

File app.js has 331 lines of code (exceeds 250 allowed). Consider refactoring.

Open

#!/usr/bin/env node

/**
 * App
 */

Function exports has 76 lines of code (exceeds 25 allowed). Consider refactoring.

Open

module.exports = (config) => {
    const url = config.url;
    const parsedUrl = !url ? null : URL.parse(url);

    if (!parsedUrl || !parsedUrl.hostname) {

Function fetch has 73 lines of code (exceeds 25 allowed). Consider refactoring.

Open

    const fetch = () => {
        const shasum = crypto.createHash('sha1');
        shasum.update(url.url, 'utf8');

        if (action === 'delete') {

Function exports has a Cognitive Complexity of 19 (exceeds 5 allowed). Consider refactoring.

Open

module.exports = (config) => {
    const url = config.url;
    const parsedUrl = !url ? null : URL.parse(url);

    if (!parsedUrl || !parsedUrl.hostname) {

• Read up
  Read up

File process.js has 269 lines of code (exceeds 250 allowed). Consider refactoring.

Open

/**
 * Processes one url
 */

'use strict';

Function exports has 58 lines of code (exceeds 25 allowed). Consider refactoring.

Open

module.exports = (config, cb) => {
    _.each(config.sitemaps, (sitemap) => {
        const urls = [];
        const parsedUrl = URL.parse(sitemap.url);
        const httpOptions = {

Function req has 55 lines of code (exceeds 25 allowed). Consider refactoring.

Open

            var req = client.request(httpOptions, (res) => {
                let data = '';
                const now = new Date();
                const meta = {
                    date: now,

Function send has 55 lines of code (exceeds 25 allowed). Consider refactoring.

Open

    const send = (data, cb) => {
        if (data && data.result && data.action && data.url) {
            const postData =  querystring.stringify({
                result: data.result,
                action: data.action,

Function parse has 51 lines of code (exceeds 25 allowed). Consider refactoring.

Open

const parse = (record, data, config) => {
    const $ = cheerio.load(data);
    
    // Process all selectors
    _.each(config.selectors, (selector) => {

Function exports has a Cognitive Complexity of 14 (exceeds 5 allowed). Consider refactoring.

Open

module.exports = (data, cb) => {
    const config = data.config;
    const plugins = data.plugins;
    const isRetry = !!data.isRetry;
    const url = data.url;

• Read up
  Read up

Function recursiveFindValue has a Cognitive Complexity of 14 (exceeds 5 allowed). Consider refactoring.

Open

const recursiveFindValue = (node, array, attribs) => {
    if (!node || node.type === 'comment') {
        return array;
    }
    

• Read up
  Read up

Function req has 38 lines of code (exceeds 25 allowed). Consider refactoring.

Open

        const req = client.request(httpOptions, (res) => {
            let data = '';
            
            res.setEncoding('utf8');
            

Function displayErrorReport has a Cognitive Complexity of 12 (exceeds 5 allowed). Consider refactoring.

Open

const displayErrorReport = () => {
    console.log();
    if (!errors.length) {
        console.log(chalk.green('No problem were reported during the crawl!'));
    } else {

• Read up
  Read up

Function recursiveFindValue has 33 lines of code (exceeds 25 allowed). Consider refactoring.

Open

const recursiveFindValue = (node, array, attribs) => {
    if (!node || node.type === 'comment') {
        return array;
    }
    

Consider simplifying this complex logical expression.

Open

        if (data && data.result && data.action && data.url) {
            const postData =  querystring.stringify({
                result: data.result,
                action: data.action,
                url: data.url,

Function trimmer has a Cognitive Complexity of 9 (exceeds 5 allowed). Consider refactoring.

Open

const trimmer = (record, config) => {
    const bytes = (s) => ~-encodeURI(s).split(/%..|./).length;
    const jsonSize = (s) => bytes(JSON.stringify(s));
    const limit = config.maxRecordSize;
    const attributes = config.attributesToPop || defaultPoppedAttributes;

• Read up
  Read up

Function exports has a Cognitive Complexity of 9 (exceeds 5 allowed). Consider refactoring.

Open

module.exports = (config, cb) => {
    _.each(config.sitemaps, (sitemap) => {
        const urls = [];
        const parsedUrl = URL.parse(sitemap.url);
        const httpOptions = {

• Read up
  Read up