Function init_emr_config has a Cognitive Complexity of 49 (exceeds 5 allowed). Consider refactoring.

Open

def init_emr_config(run_parallel=False, role=None):
    cfg = ResourcesConfiguration()

    emr_client = aws_common.connect_emr(role=role)

• Read up
  Read up

Function do_add_stats has a Cognitive Complexity of 47 (exceeds 5 allowed). Consider refactoring.

Open

def do_add_stats(nodelist, result_map):

    load_aggregates = ('average', 'sum', 'min', 'max')
    all_aggregates = load_aggregates + ('count', )
    load_metrics = ('cpu', 'mem', 'sysload')

• Read up
  Read up

Function get_cluster_savings has a Cognitive Complexity of 38 (exceeds 5 allowed). Consider refactoring.

Open

def get_cluster_savings(info, baseline_nodes, zone='us-east-1'):
    result = {}
    instance_start_times = {}
    instance_end_times = {}
    instance_types = {}

• Read up
  Read up

Function get_fixed_price has a Cognitive Complexity of 35 (exceeds 5 allowed). Consider refactoring.

Open

def get_fixed_price(zone, type, os='Linux', tenancy='Shared'):
    zone = get_short_zone(zone)
    doc = load_fixed_prices(zone)
    result = None
    for key, product in doc['products'].iteritems():

• Read up
  Read up

File emr_monitoring.py has 356 lines of code (exceeds 250 allowed). Consider refactoring.

Open

import subprocess
import re
import os
import json
import math

Function remove_NaN has a Cognitive Complexity of 26 (exceeds 5 allowed). Consider refactoring.

Open

def remove_NaN(obj, delete_values=True, replacement='NaN', expect_only_numbers=False):
    if isinstance(obj, list):
        i = 0
        while i < len(obj):
            if is_composite(obj[i]):

• Read up
  Read up

Function perform_scaling has a Cognitive Complexity of 25 (exceeds 5 allowed). Consider refactoring.

Open

def perform_scaling(cluster):
    app_config = config.get_config()
    monitoring_interval_secs = int(app_config.general.monitoring_time_window)
    info = cluster.monitoring_data
    LOG.debug('info={}'.format(json.dumps(info, indent=2)))

• Read up
  Read up

Function get_cluster_nodes has a Cognitive Complexity of 22 (exceeds 5 allowed). Consider refactoring.

Open

def get_cluster_nodes(cluster_id, role=None):
    emr_client = connect_emr(role=role)
    result = run_func(emr_client.list_instances, ClusterId=cluster_id,
                      InstanceStates=['AWAITING_FULFILLMENT', 'PROVISIONING', 'BOOTSTRAPPING', 'RUNNING'],
                      cache_duration_secs=QUERY_CACHE_TIMEOUT)

• Read up
  Read up

Function get_ssh_keys has a Cognitive Complexity of 21 (exceeds 5 allowed). Consider refactoring.

Open

def get_ssh_keys():
    keys = re.split(r'\s*,\s*', config.get_value(KEY_SSH_KEYS))
    for i in range(0, len(keys)):
        key = keys[i]
        if key[0] == '$':

• Read up
  Read up

Function collect_info has a Cognitive Complexity of 20 (exceeds 5 allowed). Consider refactoring.

Open

def collect_info(stream, monitoring_interval_secs=600, config=None):
    result = {}
    # check if we need to re-load and save the stream details
    if STREAMS_CHANGED.get(stream.id):
        stream = save_modified_stream(stream)

• Read up
  Read up

File common.py has 281 lines of code (exceeds 250 allowed). Consider refactoring.

Open

import threading
import subprocess32 as subprocess
import os
import re
import time

Function fillup_with_zeros has a Cognitive Complexity of 16 (exceeds 5 allowed). Consider refactoring.

Open

def fillup_with_zeros(datapoints, start_time, time_window, period, statistic='Sum'):
    # first, make sure datapoints are sorted
    datapoints.sort(key=lambda item: item['Timestamp'])

    dates_as_string = False

• Read up
  Read up

Function collect_info has a Cognitive Complexity of 16 (exceeds 5 allowed). Consider refactoring.

Open

def collect_info(cluster, nodes=None, config=None,
        monitoring_interval_secs=MONITORING_INTERVAL_SECS):
    try:
        # LOG.info('Collect monitoring info for cluster %s' % cluster.id)
        result = {}

• Read up
  Read up

Function run_ssh has a Cognitive Complexity of 13 (exceeds 5 allowed). Consider refactoring.

Open

def run_ssh(cmd, host, user=None, keys=None, via_hosts=[], cache_duration_secs=0):
    if not keys:
        keys = get_ssh_keys()

    user = '%s@' % user if user else ''

• Read up
  Read up

Function run has a Cognitive Complexity of 13 (exceeds 5 allowed). Consider refactoring.

Open

def run(cmd, cache_duration_secs=0, log_error=False, retries=0, sleep=2, backoff=1.4):
    def do_run(cmd):
        try:
            mutex_popen.acquire()
            # process = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)

• Read up
  Read up

Function get_termination_candidates_for_market_or_group has a Cognitive Complexity of 13 (exceeds 5 allowed). Consider refactoring.

Open

def get_termination_candidates_for_market_or_group(info, preferred, instance_group_type=INSTANCE_GROUP_TYPE_TASK):
    candidates = []
    cluster_id = info['cluster_id']
    LOG.debug('cluster_id={}'.format(json.dumps(cluster_id)))
    role = emr_monitoring.get_iam_role_for_cluster(cluster_id)

• Read up
  Read up

Function get_config has a Cognitive Complexity of 13 (exceeds 5 allowed). Consider refactoring.

Open

def get_config(force_load=False, config_file_only=False):
    global last_config_load_time, CACHED_CONFIG
    if TEST_CONFIG:
        return TEST_CONFIG

• Read up
  Read up

Function get_node_load_part has a Cognitive Complexity of 13 (exceeds 5 allowed). Consider refactoring.

Open

def get_node_load_part(cluster, host, type, monitoring_interval_secs=MONITORING_INTERVAL_SECS):
    ganglia_data = get_ganglia_datapoints(cluster, host, type, monitoring_interval_secs)
    if not ganglia_data:
        return float('NaN')
    curves_map = {}

• Read up
  Read up

Function refresh has 32 lines of code (exceeds 25 allowed). Consider refactoring.

Open

    $scope.refresh = function() {
      $scope.state.loading = true;
      $scope.history.loading = true;
      $scope.savings.loading = true;
      client.then(function(client) {

Function parallelize has a Cognitive Complexity of 11 (exceeds 5 allowed). Consider refactoring.

Open

def parallelize(array_or_dict, func):
    class MyThread (threading.Thread):
        def __init__(self, item, key=None):
            threading.Thread.__init__(self)
            self.item = item

• Read up
  Read up