Function main has a Cognitive Complexity of 64 (exceeds 5 allowed). Consider refactoring.

Open

def main():

    # assign constraints from input
    kwargs, variables, admin, searchargs = assign_constraints()
    # define directory where requests for downloads are stored

• Read up
  Read up

Function compare_instances has a Cognitive Complexity of 48 (exceeds 5 allowed). Consider refactoring.

Open

def compare_instances(db,remote,local,const_keys,admin):
    ''' Compare remote and local search results they're both a list of dictionaries
        :argument db: sqlalchemy local db session 
        :argument remote: each dict has keys version, files (objs), filenames, tracking_ids, dataset_id 
        :argument local:  list of version objects

• Read up
  Read up

Function update_files has a Cognitive Complexity of 33 (exceeds 5 allowed). Consider refactoring.

Open

def update_files(local,remote):
    ''' return urls of files to update '''
    urls=[]
    dataset_info=[]
    # this return too many we need to do it variable by variable

• Read up
  Read up

Function main has a Cognitive Complexity of 31 (exceeds 5 allowed). Consider refactoring.

Open

def main():

    # Calling parse_input() function to build kwargs from external arguments passed by user 
    kwargs=assign_constraints()
    all_versions=kwargs.pop("all_versions")

• Read up
  Read up

Function new_files has a Cognitive Complexity of 27 (exceeds 5 allowed). Consider refactoring.

Open

def new_files(remote):
    ''' return urls of new files to download '''
    urls=[]
    dataset_info=[]
    # this return too many we need to do it variable by variable

• Read up
  Read up

Function compare_files has a Cognitive Complexity of 26 (exceeds 5 allowed). Consider refactoring.

Open

def compare_files(db,rds,v,admin):
    ''' Compare files of remote and local version of a dataset
        :argument rds: dictionary of remote dataset object selected attributes  
        :argument v:  local version object   
        :return: result set, NB updating VerisonFiles object in databse if calculating checksums 

• Read up
  Read up

File compare_helpers.py has 304 lines of code (exceeds 250 allowed). Consider refactoring.

Open

#!/usr/bin/env python
# This check data available on ESGF and on raijin that matches constraints passed on by user and return a summary.
"""
Copyright 2016 ARC Centre of Excellence for Climate Systems Science

Function result_matrix has a Cognitive Complexity of 12 (exceeds 5 allowed). Consider refactoring.

Open

def result_matrix(matrix,exp,var,remote,local):
    ''' Build a matrix of the results to output to csv table '''
    # for each var_mip retrieve_info create a dict{var_mip:[[(mod1,ens1), details list][(mod1,ens2), details list],[..]]}
    # they are added to exp_dict and each key will be column header, (mod1,ens1) will indicate row and details will be cell value
    exp_dict=matrix[exp]

• Read up
  Read up

Function retrieve_ds has a Cognitive Complexity of 11 (exceeds 5 allowed). Consider refactoring.

Open

def retrieve_ds(args):
    ''' Retrieve info from a remote dataset object '''
    (ds, variables) = args
    # skip dataset if from NCI 
    if 'esgf.nci.org.au' in ds.dataset_id:

• Read up
  Read up

Function assign_constraints has a Cognitive Complexity of 10 (exceeds 5 allowed). Consider refactoring.

Open

def assign_constraints():
    ''' Assign default values and input to constraints '''
    kwargs = parse_input()
    admin = kwargs.pop("admin")
    searchargs={}

• Read up
  Read up

Function assign_constraints has a Cognitive Complexity of 9 (exceeds 5 allowed). Consider refactoring.

Open

def assign_constraints():
    ''' Assign default values and input to constraints '''
    kwargs = parse_input()
    admin = kwargs.pop("admin")
    warning = kwargs.pop("warning")[0]

• Read up
  Read up

Avoid deeply nested control flow statements.

Open

                    for s in upd_urls:
                        print(s.split("'")[0])
                        fout.writelines("'" +s + "'\n")
                    fout.close()

Avoid deeply nested control flow statements.

Open

                    if request2: os.system ("cp %s %s" % (outfile, outdir+outfile)) 
        for var in variables:

Avoid deeply nested control flow statements.

Open

                for f in files:
                    checksum=check_hash(v_obj.path+"/"+f,'md5')
                    rows.append(dict(filename=f, md5=checksum, version_id=v_obj.id))
                    add_bulk_items(db, VersionFile, rows)
            else:

Avoid deeply nested control flow statements.

Open

                    if ctype=="None": 
                        urls.append("' '".join([f.filename,f.download_url,"None","None"]))
                        dataset_info.append(",".join([f.filename,f.tracking_id,"None"])+"\n")
                    else:
                        urls.append("' '".join([f.filename,f.download_url,ctype.upper(),f.checksum]))

Avoid deeply nested control flow statements.

Open

                    for s in urls:
                        print(s.split("'")[0])
                        fout.writelines("'" +s + "'\n")
                    fout.close()

Avoid deeply nested control flow statements.

Open

                for f in files:
                    kw_files['filename']=f
                    kw_files['md5']=check_hash(v_obj.path+"/"+f,'md5')
                    insert_unique(db, VersionFile, **kw_files)

Avoid deeply nested control flow statements.

Open

                    if not admin:
                        ds_info=[str(x) for x in ds_instance.items()]
                        write_log(" ".join(["update"]+ds_info+[v.version,v.path,"\n"]))
            # if local dataset_id is the same as remote skip all other checks
            if v.dataset_id==ds['dataset_id']:

Function write_table has a Cognitive Complexity of 8 (exceeds 5 allowed). Consider refactoring.

Open

def write_table(matrix,exp):
    ''' write a csv table to summarise search
        argument matrix:
        argument exp: 
    '''

• Read up
  Read up

Avoid deeply nested control flow statements.

Open

                    if f.filename not in different: continue
                    if ctype=="None":