current/deps/v8/tools/locs.py
#!/usr/bin/env python
# Copyright 2018 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
""" locs.py - Count lines of code before and after preprocessor expansion
Consult --help for more information.
"""
# for py2/py3 compatibility
from __future__ import print_function
import argparse
import json
import multiprocessing
import os
import re
import subprocess
import sys
import tempfile
import time
from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
# for py2/py3 compatibility
try:
FileNotFoundError
except NameError:
FileNotFoundError = IOError
ARGPARSE = argparse.ArgumentParser(
description=("A script that computes LoC for a build dir"),
epilog="""Examples:
Count with default settings for build in out/Default:
locs.py --build-dir out/Default
Count only a custom group of files settings for build in out/Default:
tools/locs.py --build-dir out/Default
--group src-compiler '\.\./\.\./src/compiler'
--only src-compiler
Report the 10 files with the worst expansion:
tools/locs.py --build-dir out/Default --worst 10
Report the 10 files with the worst expansion in src/compiler:
tools/locs.py --build-dir out/Default --worst 10
--group src-compiler '\.\./\.\./src/compiler'
--only src-compiler
Report the 10 largest files after preprocessing:
tools/locs.py --build-dir out/Default --largest 10
Report the 10 smallest input files:
tools/locs.py --build-dir out/Default --smallest 10""",
formatter_class=argparse.RawTextHelpFormatter
)
ARGPARSE.add_argument(
'--json',
action='store_true',
default=False,
help="output json instead of short summary")
ARGPARSE.add_argument(
'--build-dir',
type=str,
help="Use specified build dir and generate necessary files",
required=True)
ARGPARSE.add_argument(
'--echocmd',
action='store_true',
default=False,
help="output command used to compute LoC")
ARGPARSE.add_argument(
'--only',
action='append',
default=[],
help="Restrict counting to report group (can be passed multiple times)")
ARGPARSE.add_argument(
'--not',
action='append',
default=[],
help="Exclude specific group (can be passed multiple times)")
ARGPARSE.add_argument(
'--list-groups',
action='store_true',
default=False,
help="List groups and associated regular expressions")
ARGPARSE.add_argument(
'--group',
nargs=2,
action='append',
default=[],
help="Add a report group (can be passed multiple times)")
ARGPARSE.add_argument(
'--largest',
type=int,
nargs='?',
default=0,
const=3,
help="Output the n largest files after preprocessing")
ARGPARSE.add_argument(
'--worst',
type=int,
nargs='?',
default=0,
const=3,
help="Output the n files with worst expansion by preprocessing")
ARGPARSE.add_argument(
'--smallest',
type=int,
nargs='?',
default=0,
const=3,
help="Output the n smallest input files")
ARGPARSE.add_argument(
'--files',
type=int,
nargs='?',
default=0,
const=3,
help="Output results for each file separately")
ARGPARSE.add_argument(
'--jobs',
type=int,
default=multiprocessing.cpu_count(),
help="Process specified number of files concurrently")
ARGS = vars(ARGPARSE.parse_args())
def MaxWidth(strings):
max_width = 0
for s in strings:
max_width = max(max_width, len(s))
return max_width
def GenerateCompileCommandsAndBuild(build_dir, out):
if not os.path.isdir(build_dir):
print("Error: Specified build dir {} is not a directory.".format(
build_dir), file=sys.stderr)
exit(1)
autoninja = "autoninja -C {}".format(build_dir)
if subprocess.call(autoninja, shell=True, stdout=out) != 0:
print("Error: Building {} failed.".format(build_dir), file=sys.stderr)
exit(1)
compile_commands_file = "{}/compile_commands.json".format(build_dir)
print("Generating compile commands in {}.".format(
compile_commands_file), file=out)
ninja = "ninja -C {} -t compdb cxx cc > {}".format(
build_dir, compile_commands_file)
if subprocess.call(ninja, shell=True, stdout=out) != 0:
print("Error: Cound not generate {} for {}.".format(
compile_commands_file, build_dir), file=sys.stderr)
exit(1)
ninja_deps_file = "{}/ninja-deps.txt".format(build_dir)
print("Generating ninja dependencies in {}.".format(
ninja_deps_file), file=out)
ninja = "ninja -C {} -t deps > {}".format(
build_dir, ninja_deps_file)
if subprocess.call(ninja, shell=True, stdout=out) != 0:
print("Error: Cound not generate {} for {}.".format(
ninja_deps_file, build_dir), file=sys.stderr)
exit(1)
return compile_commands_file, ninja_deps_file
def fmt_bytes(num_bytes):
if num_bytes > 1024*1024*1024:
return int(num_bytes / (1024*1024)), "MB"
elif num_bytes > 1024*1024:
return int(num_bytes / (1024)), "kB"
return int(num_bytes), " B"
class CompilationData:
def __init__(self, loc, in_bytes, expanded, expanded_bytes):
self.loc = loc
self.in_bytes = in_bytes
self.expanded = expanded
self.expanded_bytes = expanded_bytes
def ratio(self):
return self.expanded / (self.loc+1)
def to_string(self):
exp_bytes, exp_unit = fmt_bytes(self.expanded_bytes)
in_bytes, in_unit = fmt_bytes(self.in_bytes)
return "{:>9,} LoC ({:>7,} {}) to {:>12,} LoC ({:>7,} {}) ({:>5.0f}x)".format(
self.loc, in_bytes, in_unit, self.expanded, exp_bytes, exp_unit, self.ratio())
class File(CompilationData):
def __init__(self, file, target, loc, in_bytes, expanded, expanded_bytes):
super().__init__(loc, in_bytes, expanded, expanded_bytes)
self.file = file
self.target = target
def to_string(self):
return "{} {} {}".format(super().to_string(), self.file, self.target)
class Group(CompilationData):
def __init__(self, name, regexp_string):
super().__init__(0, 0, 0, 0)
self.name = name
self.count = 0
self.regexp = re.compile(regexp_string)
def account(self, unit):
if (self.regexp.match(unit.file)):
self.loc += unit.loc
self.in_bytes += unit.in_bytes
self.expanded += unit.expanded
self.expanded_bytes += unit.expanded_bytes
self.count += 1
def to_string(self, name_width):
return "{:<{}} ({:>5} files): {}".format(
self.name, name_width, self.count, super().to_string())
def SetupReportGroups():
default_report_groups = {"total": '.*',
"src": '\\.\\./\\.\\./src',
"test": '\\.\\./\\.\\./test',
"third_party": '\\.\\./\\.\\./third_party',
"gen": 'gen'}
report_groups = default_report_groups.copy()
report_groups.update(dict(ARGS['group']))
if ARGS['only']:
for only_arg in ARGS['only']:
if not only_arg in report_groups.keys():
print("Error: specified report group '{}' is not defined.".format(
ARGS['only']))
exit(1)
else:
report_groups = {
k: v for (k, v) in report_groups.items() if k in ARGS['only']}
if ARGS['not']:
report_groups = {
k: v for (k, v) in report_groups.items() if k not in ARGS['not']}
if ARGS['list_groups']:
print_cat_max_width = MaxWidth(list(report_groups.keys()) + ["Category"])
print(" {:<{}} {}".format("Category",
print_cat_max_width, "Regular expression"))
for cat, regexp_string in report_groups.items():
print(" {:<{}}: {}".format(
cat, print_cat_max_width, regexp_string))
report_groups = {k: Group(k, v) for (k, v) in report_groups.items()}
return report_groups
class Results:
def __init__(self):
self.groups = SetupReportGroups()
self.units = {}
self.source_dependencies = {}
self.header_dependents = {}
def track(self, filename):
is_tracked = False
for group in self.groups.values():
if group.regexp.match(filename):
is_tracked = True
return is_tracked
def recordFile(self, filename, targetname, loc, in_bytes, expanded, expanded_bytes):
unit = File(filename, targetname, loc, in_bytes, expanded, expanded_bytes)
self.units[filename] = unit
for group in self.groups.values():
group.account(unit)
def maxGroupWidth(self):
return MaxWidth([v.name for v in self.groups.values()])
def printGroupResults(self, file):
for key in sorted(self.groups.keys()):
print(self.groups[key].to_string(self.maxGroupWidth()), file=file)
def printSorted(self, key, count, reverse, out):
for unit in sorted(list(self.units.values()), key=key, reverse=reverse)[:count]:
print(unit.to_string(), file=out)
def addHeaderDeps(self, source_dependencies, header_dependents):
self.source_dependencies = source_dependencies
self.header_dependents = header_dependents
class LocsEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, File):
return {"file": o.file, "target": o.target, "loc": o.loc, "in_bytes": o.in_bytes,
"expanded": o.expanded, "expanded_bytes": o.expanded_bytes}
if isinstance(o, Group):
return {"name": o.name, "loc": o.loc, "in_bytes": o.in_bytes,
"expanded": o.expanded, "expanded_bytes": o.expanded_bytes}
if isinstance(o, Results):
return {"groups": o.groups, "units": o.units,
"source_dependencies": o.source_dependencies,
"header_dependents": o.header_dependents}
return json.JSONEncoder.default(self, o)
class StatusLine:
def __init__(self):
self.max_width = 0
def print(self, statusline, end="\r", file=sys.stdout):
self.max_width = max(self.max_width, len(statusline))
print("{0:<{1}}".format(statusline, self.max_width),
end=end, file=file, flush=True)
class CommandSplitter:
def __init__(self):
self.cmd_pattern = re.compile(
"([^\\s]*\\s+)?(?P<clangcmd>[^\\s]*clang.*)"
" -c (?P<infile>.*) -o (?P<outfile>.*)")
def process(self, compilation_unit):
cmd = self.cmd_pattern.match(compilation_unit['command'])
outfilename = cmd.group('outfile')
infilename = cmd.group('infile')
infile = Path(compilation_unit['directory']).joinpath(infilename)
return (cmd.group('clangcmd'), infilename, infile, outfilename)
def parse_ninja_deps(ninja_deps):
source_dependencies = {}
header_dependents = defaultdict(int)
current_target = None
for line in ninja_deps:
line = line.rstrip()
# Ignore empty lines
if not line:
current_target = None
continue
if line[0] == ' ':
# New dependency
if len(line) < 5 or line[0:4] != ' ' or line[5] == ' ':
sys.exit('Lines must have no indentation or exactly four ' +
'spaces.')
dep = line[4:]
if not re.search(r"\.(h|hpp)$", dep):
continue
header_dependents[dep] += 1
continue
# New target
colon_pos = line.find(':')
if colon_pos < 0:
sys.exit('Unindented line must have a colon')
if current_target is not None:
sys.exit('Missing empty line before new target')
current_target = line[0:colon_pos]
match = re.search(r"#deps (\d+)", line)
deps_number = match.group(1)
source_dependencies[current_target] = int(deps_number)
return (source_dependencies, header_dependents)
def Main():
out = sys.stdout
if ARGS['json']:
out = sys.stderr
compile_commands_file, ninja_deps_file = GenerateCompileCommandsAndBuild(
ARGS['build_dir'], out)
result = Results()
status = StatusLine()
try:
with open(compile_commands_file) as file:
compile_commands = json.load(file)
with open(ninja_deps_file) as file:
source_dependencies, header_dependents = parse_ninja_deps(file)
result.addHeaderDeps(source_dependencies, header_dependents)
except FileNotFoundError:
print("Error: Cannot read '{}'. Consult --help to get started.".format(
ninja_deps_file))
exit(1)
cmd_splitter = CommandSplitter()
def count_lines_of_unit(ikey):
i, key = ikey
if not result.track(key['file']):
return
message = "[{}/{}] Counting LoCs of {}".format(
i, len(compile_commands), key['file'])
status.print(message, file=out)
clangcmd, infilename, infile, outfilename = cmd_splitter.process(key)
if not infile.is_file():
return
clangcmd = clangcmd + " -E -P " + \
str(infile) + " -o /dev/stdout | sed '/^\\s*$/d' | wc -lc"
loccmd = ("cat {} | sed '\\;^\\s*//;d' | sed '\\;^/\\*;d'"
" | sed '/^\\*/d' | sed '/^\\s*$/d' | wc -lc")
loccmd = loccmd.format(infile)
runcmd = " {} ; {}".format(clangcmd, loccmd)
if ARGS['echocmd']:
print(runcmd)
process = subprocess.Popen(
runcmd, shell=True, cwd=key['directory'], stdout=subprocess.PIPE)
p = {'process': process, 'infile': infilename, 'outfile': outfilename}
output, _ = p['process'].communicate()
expanded, expanded_bytes, loc, in_bytes = list(map(int, output.split()))
result.recordFile(p['infile'], p['outfile'], loc,
in_bytes, expanded, expanded_bytes)
with tempfile.TemporaryDirectory(dir='/tmp/', prefix="locs.") as temp:
start = time.time()
with ThreadPoolExecutor(max_workers=ARGS['jobs']) as executor:
list(executor.map(count_lines_of_unit, enumerate(compile_commands)))
end = time.time()
if ARGS['json']:
print(json.dumps(result, ensure_ascii=False, cls=LocsEncoder))
status.print("Processed {:,} files in {:,.2f} sec.".format(
len(compile_commands), end-start), end="\n", file=out)
result.printGroupResults(file=out)
if ARGS['largest']:
print("Largest {} files after expansion:".format(ARGS['largest']))
result.printSorted(
lambda v: v.expanded, ARGS['largest'], reverse=True, out=out)
if ARGS['worst']:
print("Worst expansion ({} files):".format(ARGS['worst']))
result.printSorted(
lambda v: v.ratio(), ARGS['worst'], reverse=True, out=out)
if ARGS['smallest']:
print("Smallest {} input files:".format(ARGS['smallest']))
result.printSorted(
lambda v: v.loc, ARGS['smallest'], reverse=False, out=out)
if ARGS['files']:
print("List of input files:")
result.printSorted(
lambda v: v.file, ARGS['files'], reverse=False, out=out)
return 0
if __name__ == '__main__':
sys.exit(Main())