deps/v8/tools/gc-nvp-trace-processor.py
#!/usr/bin/env python
#
# Copyright 2010 the V8 project authors. All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials provided
# with the distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
#
# This is an utility for plotting charts based on GC traces produced by V8 when
# run with flags --trace-gc --trace-gc-nvp. Relies on gnuplot for actual
# plotting.
#
# Usage: gc-nvp-trace-processor.py <GC-trace-filename>
#
from __future__ import with_statement
import sys, types, re, subprocess, math
def flatten(l):
flat = []
for i in l: flat.extend(i)
return flat
def split_nvp(s):
t = {}
for (name, value) in re.findall(r"(\w+)=([-\w]+)", s):
try:
t[name] = int(value)
except ValueError:
t[name] = value
return t
def parse_gc_trace(input):
trace = []
with open(input) as f:
for line in f:
info = split_nvp(line)
if info and 'pause' in info and info['pause'] > 0:
info['i'] = len(trace)
trace.append(info)
return trace
def extract_field_names(script):
fields = { 'data': true, 'in': true }
for m in re.finditer(r"$(\w+)", script):
field_name = m.group(1)
if field_name not in fields:
fields[field] = field_count
field_count = field_count + 1
return fields
def gnuplot(script):
gnuplot = subprocess.Popen(["gnuplot"], stdin=subprocess.PIPE)
gnuplot.stdin.write(script)
gnuplot.stdin.close()
gnuplot.wait()
x1y1 = 'x1y1'
x1y2 = 'x1y2'
x2y1 = 'x2y1'
x2y2 = 'x2y2'
class Item(object):
def __init__(self, title, field, axis = x1y1, **keywords):
self.title = title
self.axis = axis
self.props = keywords
if type(field) is types.ListType:
self.field = field
else:
self.field = [field]
def fieldrefs(self):
return self.field
def to_gnuplot(self, context):
args = ['"%s"' % context.datafile,
'using %s' % context.format_fieldref(self.field),
'title "%s"' % self.title,
'axis %s' % self.axis]
if 'style' in self.props:
args.append('with %s' % self.props['style'])
if 'lc' in self.props:
args.append('lc rgb "%s"' % self.props['lc'])
if 'fs' in self.props:
args.append('fs %s' % self.props['fs'])
return ' '.join(args)
class Plot(object):
def __init__(self, *items):
self.items = items
def fieldrefs(self):
return flatten([item.fieldrefs() for item in self.items])
def to_gnuplot(self, ctx):
return 'plot ' + ', '.join([item.to_gnuplot(ctx) for item in self.items])
class Set(object):
def __init__(self, value):
self.value = value
def to_gnuplot(self, ctx):
return 'set ' + self.value
def fieldrefs(self):
return []
class Context(object):
def __init__(self, datafile, field_to_index):
self.datafile = datafile
self.field_to_index = field_to_index
def format_fieldref(self, fieldref):
return ':'.join([str(self.field_to_index[field]) for field in fieldref])
def collect_fields(plot):
field_to_index = {}
fields = []
def add_field(field):
if field not in field_to_index:
fields.append(field)
field_to_index[field] = len(fields)
for field in flatten([item.fieldrefs() for item in plot]):
add_field(field)
return (fields, field_to_index)
def is_y2_used(plot):
for subplot in plot:
if isinstance(subplot, Plot):
for item in subplot.items:
if item.axis == x1y2 or item.axis == x2y2:
return True
return False
def get_field(trace_line, field):
t = type(field)
if t is types.StringType:
return trace_line[field]
elif t is types.FunctionType:
return field(trace_line)
def generate_datafile(datafile_name, trace, fields):
with open(datafile_name, 'w') as datafile:
for line in trace:
data_line = [str(get_field(line, field)) for field in fields]
datafile.write('\t'.join(data_line))
datafile.write('\n')
def generate_script_and_datafile(plot, trace, datafile, output):
(fields, field_to_index) = collect_fields(plot)
generate_datafile(datafile, trace, fields)
script = [
'set terminal png',
'set output "%s"' % output,
'set autoscale',
'set ytics nomirror',
'set xtics nomirror',
'set key below'
]
if is_y2_used(plot):
script.append('set autoscale y2')
script.append('set y2tics')
context = Context(datafile, field_to_index)
for item in plot:
script.append(item.to_gnuplot(context))
return '\n'.join(script)
def plot_all(plots, trace, prefix):
charts = []
for plot in plots:
outfilename = "%s_%d.png" % (prefix, len(charts))
charts.append(outfilename)
script = generate_script_and_datafile(plot, trace, '~datafile', outfilename)
print 'Plotting %s...' % outfilename
gnuplot(script)
return charts
def reclaimed_bytes(row):
return row['total_size_before'] - row['total_size_after']
def other_scope(r):
if r['gc'] == 's':
# there is no 'other' scope for scavenging collections.
return 0
return r['pause'] - r['mark'] - r['sweep'] - r['external']
def scavenge_scope(r):
if r['gc'] == 's':
return r['pause'] - r['external']
return 0
def real_mutator(r):
return r['mutator'] - r['stepstook']
plots = [
[
Set('style fill solid 0.5 noborder'),
Set('style histogram rowstacked'),
Set('style data histograms'),
Plot(Item('Scavenge', scavenge_scope, lc = 'green'),
Item('Marking', 'mark', lc = 'purple'),
Item('Sweep', 'sweep', lc = 'blue'),
Item('External', 'external', lc = '#489D43'),
Item('Other', other_scope, lc = 'grey'),
Item('IGC Steps', 'stepstook', lc = '#FF6347'))
],
[
Set('style fill solid 0.5 noborder'),
Set('style histogram rowstacked'),
Set('style data histograms'),
Plot(Item('Scavenge', scavenge_scope, lc = 'green'),
Item('Marking', 'mark', lc = 'purple'),
Item('Sweep', 'sweep', lc = 'blue'),
Item('External', 'external', lc = '#489D43'),
Item('Other', other_scope, lc = '#ADD8E6'),
Item('External', 'external', lc = '#D3D3D3'))
],
[
Plot(Item('Mutator', real_mutator, lc = 'black', style = 'lines'))
],
[
Set('style histogram rowstacked'),
Set('style data histograms'),
Plot(Item('Heap Size (before GC)', 'total_size_before', x1y2,
fs = 'solid 0.4 noborder',
lc = 'green'),
Item('Total holes (after GC)', 'holes_size_before', x1y2,
fs = 'solid 0.4 noborder',
lc = 'red'),
Item('GC Time', ['i', 'pause'], style = 'lines', lc = 'red'))
],
[
Set('style histogram rowstacked'),
Set('style data histograms'),
Plot(Item('Heap Size (after GC)', 'total_size_after', x1y2,
fs = 'solid 0.4 noborder',
lc = 'green'),
Item('Total holes (after GC)', 'holes_size_after', x1y2,
fs = 'solid 0.4 noborder',
lc = 'red'),
Item('GC Time', ['i', 'pause'],
style = 'lines',
lc = 'red'))
],
[
Set('style fill solid 0.5 noborder'),
Set('style data histograms'),
Plot(Item('Allocated', 'allocated'),
Item('Reclaimed', reclaimed_bytes),
Item('Promoted', 'promoted', style = 'lines', lc = 'black'))
],
]
def freduce(f, field, trace, init):
return reduce(lambda t,r: f(t, r[field]), trace, init)
def calc_total(trace, field):
return freduce(lambda t,v: t + long(v), field, trace, long(0))
def calc_max(trace, field):
return freduce(lambda t,r: max(t, r), field, trace, 0)
def count_nonzero(trace, field):
return freduce(lambda t,r: t if r == 0 else t + 1, field, trace, 0)
def process_trace(filename):
trace = parse_gc_trace(filename)
marksweeps = filter(lambda r: r['gc'] == 'ms', trace)
scavenges = filter(lambda r: r['gc'] == 's', trace)
globalgcs = filter(lambda r: r['gc'] != 's', trace)
charts = plot_all(plots, trace, filename)
def stats(out, prefix, trace, field):
n = len(trace)
total = calc_total(trace, field)
max = calc_max(trace, field)
if n > 0:
avg = total / n
else:
avg = 0
if n > 1:
dev = math.sqrt(freduce(lambda t,r: t + (r - avg) ** 2, field, trace, 0) /
(n - 1))
else:
dev = 0
out.write('<tr><td>%s</td><td>%d</td><td>%d</td>'
'<td>%d</td><td>%d [dev %f]</td></tr>' %
(prefix, n, total, max, avg, dev))
def HumanReadable(size):
suffixes = ['bytes', 'kB', 'MB', 'GB']
power = 1
for i in range(len(suffixes)):
if size < power*1024:
return "%.1f" % (float(size) / power) + " " + suffixes[i]
power *= 1024
def throughput(name, trace):
total_live_after = calc_total(trace, 'total_size_after')
total_live_before = calc_total(trace, 'total_size_before')
total_gc = calc_total(trace, 'pause')
if total_gc == 0:
return
out.write('GC %s Throughput (after): %s / %s ms = %s/ms<br/>' %
(name,
HumanReadable(total_live_after),
total_gc,
HumanReadable(total_live_after / total_gc)))
out.write('GC %s Throughput (before): %s / %s ms = %s/ms<br/>' %
(name,
HumanReadable(total_live_before),
total_gc,
HumanReadable(total_live_before / total_gc)))
with open(filename + '.html', 'w') as out:
out.write('<html><body>')
out.write('<table>')
out.write('<tr><td>Phase</td><td>Count</td><td>Time (ms)</td>')
out.write('<td>Max</td><td>Avg</td></tr>')
stats(out, 'Total in GC', trace, 'pause')
stats(out, 'Scavenge', scavenges, 'pause')
stats(out, 'MarkSweep', marksweeps, 'pause')
stats(out, 'Mark', filter(lambda r: r['mark'] != 0, trace), 'mark')
stats(out, 'Sweep', filter(lambda r: r['sweep'] != 0, trace), 'sweep')
stats(out,
'External',
filter(lambda r: r['external'] != 0, trace),
'external')
out.write('</table>')
throughput('TOTAL', trace)
throughput('MS', marksweeps)
throughput('OLDSPACE', globalgcs)
out.write('<br/>')
for chart in charts:
out.write('<img src="%s">' % chart)
out.write('</body></html>')
print "%s generated." % (filename + '.html')
if len(sys.argv) != 2:
print "Usage: %s <GC-trace-filename>" % sys.argv[0]
sys.exit(1)
process_trace(sys.argv[1])