Function update_neighbour_index has a Cognitive Complexity of 42 (exceeds 5 allowed). Consider refactoring.

Open

def update_neighbour_index(location_index, neighbour_index, pair_to_merge):
    for side in Side:
        disappearing_pairs = neighbour_index[pair_to_merge][side]
        for disappearing_pair in disappearing_pairs:
            if can_be_concat(disappearing_pair, pair_to_merge, side):

• Read up
  Read up

File wild_bpe.py has 395 lines of code (exceeds 250 allowed). Consider refactoring.

Open

# SPDX-FileCopyrightText: 2020 Hlib Babii <hlibbabii@gmail.com>
#
# SPDX-License-Identifier: Apache-2.0

import logging

Function update_location_index has a Cognitive Complexity of 28 (exceeds 5 allowed). Consider refactoring.

Open

def update_location_index(location_index, neighbour_index, pair_to_merge):
    occurence_changes = []
    disappearing_pairs = neighbour_index[pair_to_merge]
    main_list = location_index[pair_to_merge]
    if pair_to_merge in neighbour_index[pair_to_merge][Side.any()]:

• Read up
  Read up

Function run has a Cognitive Complexity of 16 (exceeds 5 allowed). Consider refactoring.

Open

def run(generator: Generator[str, None, None], n_merges: int=sys.maxsize,
        include_performance_stats_every_n_merges: int = 0) \
        -> Tuple[str, int, Optional[List[BpePerformanceStatsEntry]]]:

    checkpoint = time.time()

• Read up
  Read up

Function merge_lists_both has a Cognitive Complexity of 12 (exceeds 5 allowed). Consider refactoring.

Open

def merge_lists_both(main_list: List[int], list2: List[int], position_shift: Tuple[int, int]) -> Tuple[List[int], List[int]]:
    """
    >>> merge_lists_both([0, 5, 7, 11, 16], [1, 9, 15], (2, -2))
    ([1, 15], [7])

• Read up
  Read up

Function get_char_iterator_for_dir has a Cognitive Complexity of 10 (exceeds 5 allowed). Consider refactoring.

Open

def get_char_iterator_for_dir(path_to_dir: str) -> Generator[str, None, None]:
    for root, dirs, files in os.walk(path_to_dir):
        for file in files:
            if file.endswith('.py'):
                yield from get_char_iterator_for_file(os.path.join(root, file))

• Read up
  Read up

Function __init__ has 7 arguments (exceeds 4 allowed). Consider refactoring.

Open

    def __init__(self, merges_done: int, time_for_last_merge: float,

Avoid deeply nested control flow statements.

Open

                    if can_be_concat(pair_to_merge, cc, op_side.opposite()):
                        cc_concat = concat_pairs(pair_to_merge, cc, op_side.opposite())
                        add_pairs_to_neighbour_index(neighbour_index, appeared_pair, cc_concat, op_side,
                                                     location_index)

Avoid deeply nested control flow statements.

Open

                    for disappeared_pair2 in disappearing_pairs:
                        mm = concat_pairs(pair_to_merge, disappeared_pair2, side)
                        add_pairs_to_neighbour_index(neighbour_index, appeared_pair, mm, side, location_index)
                        if can_be_concat(pair_to_merge, mm, side.opposite()):
                            mm_concat = concat_pairs(pair_to_merge, mm, side.opposite())

Avoid deeply nested control flow statements.

Open

                    if can_be_concat(pair_to_merge, neighbour_of_neighbour, side.opposite()):
                        neighbour_of_neighbour_concat = concat_pairs(pair_to_merge, neighbour_of_neighbour,
                                                                     side.opposite())
                        add_pairs_to_neighbour_index(neighbour_index, appeared_pair, neighbour_of_neighbour_concat,
                                                     side, location_index)

Function add_pairs_to_neighbour_index has 5 arguments (exceeds 4 allowed). Consider refactoring.

Open

def add_pairs_to_neighbour_index(index, pair1, pair2, side, location_index):

Function cleanup_location_index has a Cognitive Complexity of 7 (exceeds 5 allowed). Consider refactoring.

Open

def cleanup_location_index(location_index, most_freq_pair, disappearing_pairs):
    for side in Side:
        for disappearing_pair in disappearing_pairs[side]:
            if len(location_index[disappearing_pair]) == 0:
                del location_index[disappearing_pair]

• Read up
  Read up

Function cleanup_neighbour_index has a Cognitive Complexity of 7 (exceeds 5 allowed). Consider refactoring.

Open

def cleanup_neighbour_index(location_index, neighbour_index, most_freq_pair):
    for side in Side:
        disappearing_pairs = neighbour_index[most_freq_pair][side]
        for disappearing_pair in disappearing_pairs:
            if disappearing_pair not in location_index and disappearing_pair in neighbour_index:

• Read up
  Read up