File helpers.py has 354 lines of code (exceeds 250 allowed). Consider refactoring.

Open

from __future__ import unicode_literals

import re
import logging
import functools

Function cross_db has a Cognitive Complexity of 27 (exceeds 5 allowed). Consider refactoring.

Open

def cross_db(docs, source_db=None, target_db=None, index=None, versions=False, **kwargs):
    """
    Migration to go between
        cassandra > postgres
        postgres > cassandra

• Read up
  Read up

File cassandra.py has 290 lines of code (exceeds 250 allowed). Consider refactoring.

Open

from __future__ import absolute_import

import six
import json
import logging

File tasks.py has 255 lines of code (exceeds 250 allowed). Consider refactoring.

Open

import os
import base64
import logging
import platform
from datetime import date, timedelta

Function logged has a Cognitive Complexity of 11 (exceeds 5 allowed). Consider refactoring.

Open

def logged(event, index=None):
    def _logged(func):
        @wraps(func)
        def wrapped(*args, **kwargs):
            context = extract_context(func, *args, **kwargs)

• Read up
  Read up

Function seperate_provider_object_uris has a Cognitive Complexity of 11 (exceeds 5 allowed). Consider refactoring.

Open

def seperate_provider_object_uris(identifiers):
    object_uris = gather_object_uris(identifiers)
    provider_uris = []
    for item in identifiers:

• Read up
  Read up

Function documents has a Cognitive Complexity of 10 (exceeds 5 allowed). Consider refactoring.

Open

    def documents(self, *sources):
        q = Document.objects.all()
        querysets = (q.filter(source=source) for source in sources) if sources else [q]
        for query in querysets:
            for doc in paginated(query):

• Read up
  Read up

Function strip_empty has a Cognitive Complexity of 10 (exceeds 5 allowed). Consider refactoring.

Open

def strip_empty(document, required=tuple()):
    ''' Removes empty fields from the processed schema
    '''
    new_doc = {}
    for k, v in document.items():

• Read up
  Read up

Function to_normalized has a Cognitive Complexity of 10 (exceeds 5 allowed). Consider refactoring.

Open

    def to_normalized(self, doc):
        # make the new dict actually contain real items
        normed = {}
        do_not_include = ['docID', 'doc', 'filetype', 'timestamps', 'source', 'versions', 'key']
        for key, value in dict(doc).items():

• Read up
  Read up

Function migrate has 7 arguments (exceeds 4 allowed). Consider refactoring.

Open

def migrate(migration, source_db=None, sources=tuple(), async=False, dry=True, group_size=1000, **kwargs):

Function record_or_load_response has 7 arguments (exceeds 4 allowed). Consider refactoring.

Open

def record_or_load_response(method, url, throttle=None, force=False, params=None, expected=(200,), **kwargs):

Function cross_db has 6 arguments (exceeds 4 allowed). Consider refactoring.

Open

def cross_db(docs, source_db=None, target_db=None, index=None, versions=False, **kwargs):

Avoid deeply nested control flow statements.

Open

                        if norm_version:
                            target_processor.process_normalized(raw_version, norm_version)
                        else:
                            logger.info('Not storing migrated normalized version from {} with id {}, document is not in approved set list.'.format(raw.attributes['source'], raw.attributes['docID']))
                else:

Function format_tags has a Cognitive Complexity of 8 (exceeds 5 allowed). Consider refactoring.

Open

def format_tags(all_tags, sep=','):
    tags = []
    if isinstance(all_tags, six.string_types):
        tags = all_tags.split(sep)
    elif isinstance(all_tags, list):

• Read up
  Read up

Function populate has a Cognitive Complexity of 8 (exceeds 5 allowed). Consider refactoring.

Open

def populate(grid_file):
    transformer = JSONTransformer(schema)
    for doc in get_jsons(grid_file):
        transformed = transformer.transform(doc, load=False)
        try:

• Read up
  Read up

Function record_or_load_response has a Cognitive Complexity of 8 (exceeds 5 allowed). Consider refactoring.

Open

def record_or_load_response(method, url, throttle=None, force=False, params=None, expected=(200,), **kwargs):

    resp = _maybe_load_response(method, url)

    if not force and resp and resp.ok:

• Read up
  Read up

Function migrate has a Cognitive Complexity of 8 (exceeds 5 allowed). Consider refactoring.

Open

def migrate(migration, source_db=None, sources=tuple(), async=False, dry=True, group_size=1000, **kwargs):

    source_db = source_db or settings.CANONICAL_PROCESSOR
    documents = processing.get_processor(source_db).documents

• Read up
  Read up

Function migrate has a Cognitive Complexity of 8 (exceeds 5 allowed). Consider refactoring.

Open

def migrate(migration, sources=None, kwargs_string=None, dry=True, async=False, group_size=1000):
    ''' Task to run a migration.

    :param migration: The migration function to run. This is passed in
    as a string then interpreted as a function by the invoke task.

• Read up
  Read up

Function migrate has 6 arguments (exceeds 4 allowed). Consider refactoring.

Open

def migrate(migration, sources=None, kwargs_string=None, dry=True, async=False, group_size=1000):

Function test has 5 arguments (exceeds 4 allowed). Consider refactoring.

Open

def test(cov=True, doctests=True, verbose=False, debug=False, pdb=False):