File download_column_historical_lookups.py has 3065 lines of code (exceeds 250 allowed). Consider refactoring.

Open

"""
Sets up mappings from column names used in downloads to the query paths used to get the data from django.

Not in use while we pull CSV data from the non-historical tables. Until we switch to pulling CSV downloads from the
historical tables TransactionFPDS and TransactionFABS, import download_column_lookups.py instead.

Function matview_search_filter has a Cognitive Complexity of 176 (exceeds 15 allowed). Consider refactoring.

Open

def matview_search_filter(filters, model, for_downloads=False):
    queryset = model.objects.all()

    recipient_scope_q = Q(recipient_location_country_code="USA") | Q(recipient_location_country_name="UNITED STATES")
    pop_scope_q = Q(pop_country_code="USA") | Q(pop_country_name="UNITED STATES")

• Read up
  Read up

File load_transactions_in_delta.py has 1270 lines of code (exceeds 250 allowed). Consider refactoring.

Open

import copy
import logging
import re

from contextlib import contextmanager

File conftest_spark.py has 1200 lines of code (exceeds 250 allowed). Consider refactoring.

Open

import json
import logging
import uuid
from datetime import datetime
from pathlib import Path

Function subaward_filter has a Cognitive Complexity of 144 (exceeds 15 allowed). Consider refactoring.

Open

def subaward_filter(filters, for_downloads=False):
    queryset = SubawardSearch.objects.all()

    recipient_scope_q = Q(sub_legal_entity_country_code="USA") | Q(sub_legal_entity_country_name="UNITED STATES")
    pop_scope_q = Q(sub_place_of_perform_country_co="USA") | Q(sub_place_of_perform_country_name="UNITED STATES")

• Read up
  Read up

File download_annotation_functions.py has 863 lines of code (exceeds 250 allowed). Consider refactoring.

Open

import datetime
from typing import List, Optional

from django.db.models.functions import Cast, Coalesce
from django.db.models import (

Function post has a Cognitive Complexity of 110 (exceeds 15 allowed). Consider refactoring.

Open

    def post(self, request, pk, format=None):
        # create response
        response = {"results": {}}

        # get federal account id from url

• Read up
  Read up

File sqs_work_dispatcher.py has 825 lines of code (exceeds 250 allowed). Consider refactoring.

Open

import logging
import inspect
import json
import os

File query_with_filters.py has 784 lines of code (exceeds 250 allowed). Consider refactoring.

Open

import copy
import itertools
import logging
import re
from datetime import datetime

Function validate_post_request has a Cognitive Complexity of 96 (exceeds 15 allowed). Consider refactoring.

Open

    def validate_post_request(self, request):
        if "filters" in request:
            for filt in request["filters"]:
                if "combine_method" in filt:
                    try:

• Read up
  Read up

File request_validations.py has 723 lines of code (exceeds 250 allowed). Consider refactoring.

Open

import json
from copy import deepcopy
from datetime import datetime, MINYEAR, MAXYEAR
from django.conf import settings
from typing import Optional

File swap_in_new_table.py has 672 lines of code (exceeds 250 allowed). Consider refactoring.

Open

import json
import logging
import re

from datetime import datetime

Function get_business_categories_fpds has a Cognitive Complexity of 83 (exceeds 15 allowed). Consider refactoring.

Open

def get_business_categories_fpds(row):
    # This function is supposed to be invoked as a Spark UDF with a named_struct containing the necessary
    # fields passed to it.

    def row_get(row, col_name):

• Read up
  Read up

File spark.py has 639 lines of code (exceeds 250 allowed). Consider refactoring.

Open

"""
Spark utility functions that could be used as stages or steps of an ETL job (aka "data pipeline")

NOTE: This is distinguished from the usaspending_api.common.helpers.spark_helpers module, which holds mostly boilerplate
functions for setup and configuration of the spark environment

Function build_elasticsearch_result has a Cognitive Complexity of 81 (exceeds 15 allowed). Consider refactoring.

Open

    def build_elasticsearch_result(self, response: dict) -> Dict[str, dict]:
        results = {}
        geo_info_buckets = response.get("group_by_agg_key", {}).get("buckets", [])

        for bucket in geo_info_buckets:

• Read up
  Read up

File download_generation.py has 631 lines of code (exceeds 250 allowed). Consider refactoring.

Open

import json
import logging
import multiprocessing
import os
from copy import deepcopy

File award_search.py has 631 lines of code (exceeds 250 allowed). Consider refactoring.

Open

from usaspending_api.awards.v2.lookups.lookups import award_type_mapping

AWARD_SEARCH_COLUMNS = {
    "treasury_account_identifiers": {"delta": "ARRAY<INTEGER>", "postgres": "INTEGER[]", "gold": False},
    "award_id": {"delta": "LONG NOT NULL", "postgres": "BIGINT NOT NULL", "gold": False},

File federal_accounts_v2.py has 626 lines of code (exceeds 250 allowed). Consider refactoring.

Open

import ast
from collections import OrderedDict
from typing import List

from django.db.models import DecimalField, Exists, F, Func, OuterRef, Q, Subquery, Sum

File delete_data.py has 560 lines of code (exceeds 250 allowed). Consider refactoring.

Open

import logging
from time import perf_counter
from typing import Any, Dict, List, Optional, Union

import pandas as pd

Function type_filter has a Cognitive Complexity of 67 (exceeds 15 allowed). Consider refactoring.

Open

def type_filter(_type, filters, limit=None):
    _types = [
        "budget_function",
        "budget_subfunction",
        "federal_account",

• Read up
  Read up