# Copyright 2016 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Utility functions for parsing pprof, CWP data and Chrome OS groups files."""
from collections import defaultdict
import csv
import os
import re
SEPARATOR_REGEX = re.compile(r'-+\+-+')
FUNCTION_STATISTIC_REGEX = \
re.compile(r'(\S+)\s+(\S+)%\s+(\S+)%\s+(\S+)\s+(\S+)%')
CHILD_FUNCTION_PERCENTAGE_REGEX = re.compile(r'([0-9.]+)%')
FUNCTION_KEY_SEPARATOR_REGEX = re.compile(r'\|\s+')
# Constants used to identify if a function is common in the pprof and CWP
# files.
COMMON_FUNCTION = 'common'
EXTRA_FUNCTION = 'extra'
PARENT_CHILD_FUNCTIONS_SEPARATOR = ';;'
# List of pairs of strings used for make substitutions in file names to make
# CWP and pprof data consistent.
FILE_NAME_REPLACING_PAIR_STRINGS = [('gnawty', 'BOARD'),
('amd64-generic', 'BOARD'),
(' ../sysdeps', ',sysdeps'),
(' ../nptl', ',nptl'),
(' aes-x86_64.s', ',aes-x86_64.s'),
(' (inline)', ''),
(' (partial-inline)', ''),
(' ../', ','),
('../', '')]
# Separator used to delimit the function from the file name.
FUNCTION_FILE_SEPARATOR = ' /'
def MakeCWPAndPprofFileNamesConsistent(file_name):
"""Makes the CWP and pprof file names consistent.
For the same function, it may happen for some file paths to differ slightly
in the CWP data compared to the pprof output. In a file name, for each tuple
element of the list, we substitute the first element with the second one.
Args:
file_name: A string representing the name of the file.
Returns:
A string representing the modified name of tihe file.
"""
file_name = file_name.replace(', ', '; ')
for replacing_pair_string in FILE_NAME_REPLACING_PAIR_STRINGS:
file_name = file_name.replace(replacing_pair_string[0],
replacing_pair_string[1])
return file_name
def MakePprofFunctionKey(function_and_file_name):
"""Creates the function key from the function and file name.
Parsing the the pprof --top and --tree outputs is difficult due to the fact
that it hard to extract the function and file name (i.e the function names
can have a lot of unexpected charachters such as spaces, operators etc).
For the moment, we used FUNCTION_FILE_SEPARATOR as delimiter between the
function and the file name. However, there are some cases where the file name
does not start with / and we treat this cases separately (i.e ../sysdeps,
../nptl, aes-x86_64.s).
Args:
function_and_file_name: A string representing the function and the file name
as it appears in the pprof output.
Returns:
A string representing the function key, composed from the function and file
name, comma separated.
"""
# TODO(evelinad): Use pprof --topproto instead of pprof --top to parse
# protobuffers instead of text output. Investigate if there is an equivalent
# for pprof --tree that gives protobuffer output.
#
# In the CWP output, we replace the , with ; as a workaround for parsing
# csv files. We do the same for the pprof output.
#
# TODO(evelinad): Use dremel --csv_dialect=excel-tab in the queries for
# replacing the , delimiter with tab.
function_and_file_name = function_and_file_name.replace(', ', '; ')
# If the function and file name sequence contains the FUNCTION_FILE_SEPARATOR,
# we normalize the path name of the file and make the string subtitutions
# to make the CWP and pprof data consistent. The returned key is composed
# from the function name and normalized file path name, separated by a comma.
# If the function and file name does not contain the FUNCTION_FILE_SEPARATOR,
# we just do the strings substitution.
if FUNCTION_FILE_SEPARATOR in function_and_file_name:
function_name, file_name = \
function_and_file_name.split(FUNCTION_FILE_SEPARATOR)
file_name = \
MakeCWPAndPprofFileNamesConsistent(os.path.normpath("/" + file_name))
return ','.join([function_name, file_name])
return MakeCWPAndPprofFileNamesConsistent(function_and_file_name)
def ComputeCWPCummulativeInclusiveStatistics(cwp_inclusive_count_statistics):
"""Computes the cumulative inclusive count value of a function.
A function might appear declared in multiple files or objects. When
computing the fraction of the inclusive count value from a child function to
the parent function, we take into consideration the sum of the
inclusive_count
count values from all the ocurences of that function.
Args:
cwp_inclusive_count_statistics: A dict containing the inclusive count
statistics extracted by the ParseCWPInclusiveCountFile method.
Returns:
A dict having as a ket the name of the function and as a value the sum of
the inclusive count values of the occurences of the functions from all
the files and objects.
"""
cwp_inclusive_count_statistics_cumulative = defaultdict(int)
for function_key, function_statistics \
in cwp_inclusive_count_statistics.iteritems():
function_name, _ = function_key.split(',')
cwp_inclusive_count_statistics_cumulative[function_name] += \
function_statistics[1]
return cwp_inclusive_count_statistics_cumulative
def ComputeCWPChildFunctionsFractions(cwp_inclusive_count_statistics_cumulative,
cwp_pairwise_inclusive_count_statistics):
"""Computes the fractions of the inclusive count values for child functions.
The fraction represents the inclusive count value of a child function over
the one of the parent function.
Args:
cwp_inclusive_count_statistics_cumulative: A dict containing the
cumulative inclusive count values of the CWP functions.
cwp_pairwise_inclusive_count_statistics: A dict containing the inclusive
count statistics for pairs of parent and child functions. The key is the
parent function. The value is a dict with the key the name of the child
function and the file name, comma separated, and the value is the
inclusive count value of the pair of parent and child functions.
Returns:
A dict containing the inclusive count statistics for pairs of parent
and child functions. The key is the parent function. The value is a
dict with the key the name of the child function and the file name,
comma separated, and the value is the inclusive count fraction of the
child function out of the parent function.
"""
pairwise_inclusive_count_fractions = {}
for parent_function_key, child_functions_metrics in \
cwp_pairwise_inclusive_count_statistics.iteritems():
child_functions_fractions = {}
parent_function_inclusive_count = \
cwp_inclusive_count_statistics_cumulative.get(parent_function_key, 0.0)
if parent_function_key in cwp_inclusive_count_statistics_cumulative:
for child_function_key, child_function_inclusive_count \
in child_functions_metrics.iteritems():
child_functions_fractions[child_function_key] = \
child_function_inclusive_count / parent_function_inclusive_count
else:
for child_function_key, child_function_inclusive_count \
in child_functions_metrics.iteritems():
child_functions_fractions[child_function_key] = 0.0
pairwise_inclusive_count_fractions[parent_function_key] = \
child_functions_fractions
return pairwise_inclusive_count_fractions
def ParseFunctionGroups(cwp_function_groups_lines):
"""Parses the contents of the function groups file.
Args:
cwp_function_groups_lines: A list of the lines contained in the CWP
function groups file. A line contains the group name and the file path
that describes the group, separated by a space.
Returns:
A list of tuples containing the group name and the file path.
"""
# The order of the groups mentioned in the cwp_function_groups file
# matters. A function declared in a file will belong to the first
# mentioned group that matches its path to the one of the file.
# It is possible to have multiple paths that belong to the same group.
return [tuple(line.split()) for line in cwp_function_groups_lines]
def ParsePprofTopOutput(file_name):
"""Parses a file that contains the output of the pprof --top command.
Args:
file_name: The name of the file containing the pprof --top output.
Returns:
A dict having as a key the name of the function and the file containing
the declaration of the function, separated by a comma, and as a value
a tuple containing the flat, flat percentage, sum percentage, cummulative
and cummulative percentage values.
"""
pprof_top_statistics = {}
# In the pprof top output, the statistics of the functions start from the
# 6th line.
with open(file_name) as input_file:
pprof_top_content = input_file.readlines()[6:]
for line in pprof_top_content:
function_statistic_match = FUNCTION_STATISTIC_REGEX.search(line)
flat, flat_p, sum_p, cum, cum_p = function_statistic_match.groups()
flat_p = str(float(flat_p) / 100.0)
sum_p = str(float(sum_p) / 100.0)
cum_p = str(float(cum_p) / 100.0)
lookup_index = function_statistic_match.end()
function_and_file_name = line[lookup_index + 2 : -1]
key = MakePprofFunctionKey(function_and_file_name)
pprof_top_statistics[key] = (flat, flat_p, sum_p, cum, cum_p)
return pprof_top_statistics
def ParsePprofTreeOutput(file_name):
"""Parses a file that contains the output of the pprof --tree command.
Args:
file_name: The name of the file containing the pprof --tree output.
Returns:
A dict including the statistics for pairs of parent and child functions.
The key is the name of the parent function and the file where the
function is declared, separated by a comma. The value is a dict having as
a key the name of the child function and the file where the function is
delcared, comma separated and as a value the percentage of time the
parent function spends in the child function.
"""
# In the pprof output, the statistics of the functions start from the 9th
# line.
with open(file_name) as input_file:
pprof_tree_content = input_file.readlines()[9:]
pprof_tree_statistics = defaultdict(lambda: defaultdict(float))
track_child_functions = False
# The statistics of a given function, its parent and child functions are
# included between two separator marks.
# All the parent function statistics are above the line containing the
# statistics of the given function.
# All the statistics of a child function are below the statistics of the
# given function.
# The statistics of a parent or a child function contain the calls, calls
# percentage, the function name and the file where the function is declared.
# The statistics of the given function contain the flat, flat percentage,
# sum percentage, cummulative, cummulative percentage, function name and the
# name of the file containing the declaration of the function.
for line in pprof_tree_content:
separator_match = SEPARATOR_REGEX.search(line)
if separator_match:
track_child_functions = False
continue
parent_function_statistic_match = FUNCTION_STATISTIC_REGEX.search(line)
if parent_function_statistic_match:
track_child_functions = True
lookup_index = parent_function_statistic_match.end()
parent_function_key_match = \
FUNCTION_KEY_SEPARATOR_REGEX.search(line, pos=lookup_index)
lookup_index = parent_function_key_match.end()
parent_function_key = MakePprofFunctionKey(line[lookup_index:-1])
continue
if not track_child_functions:
continue
child_function_statistic_match = \
CHILD_FUNCTION_PERCENTAGE_REGEX.search(line)
child_function_percentage = \
float(child_function_statistic_match.group(1))
lookup_index = child_function_statistic_match.end()
child_function_key_match = \
FUNCTION_KEY_SEPARATOR_REGEX.search(line, pos=lookup_index)
lookup_index = child_function_key_match.end()
child_function_key = MakePprofFunctionKey(line[lookup_index:-1])
pprof_tree_statistics[parent_function_key][child_function_key] += \
child_function_percentage / 100.0
return pprof_tree_statistics
def ParseCWPInclusiveCountFile(file_name):
"""Parses the CWP inclusive count files.
A line should contain the name of the function, the file name with the
declaration, the inclusive count and inclusive count fraction out of the
total extracted inclusive count values.
Args:
file_name: The file containing the inclusive count values of the CWP
functions.
Returns:
A dict containing the inclusive count statistics. The key is the name of
the function and the file name, comma separated. The value represents a
tuple with the object name containing the function declaration, the
inclusive count and inclusive count fraction values, and a marker to
identify if the function is present in one of the benchmark profiles.
"""
cwp_inclusive_count_statistics = defaultdict(lambda: ('', 0, 0.0, 0))
with open(file_name) as input_file:
statistics_reader = csv.DictReader(input_file, delimiter=',')
for statistic in statistics_reader:
function_name = statistic['function']
file_name = MakeCWPAndPprofFileNamesConsistent(
os.path.normpath(statistic['file']))
dso_name = statistic['dso']
inclusive_count = statistic['inclusive_count']
inclusive_count_fraction = statistic['inclusive_count_fraction']
# We ignore the lines that have empty fields(i.e they specify only the
# addresses of the functions and the inclusive counts values).
if all([
function_name, file_name, dso_name, inclusive_count,
inclusive_count_fraction
]):
key = '%s,%s' % (function_name, file_name)
# There might be situations where a function appears in multiple files
# or objects. Such situations can occur when in the Dremel queries there
# are not specified the Chrome OS version and the name of the board (i.e
# the files can belong to different kernel or library versions).
inclusive_count_sum = \
cwp_inclusive_count_statistics[key][1] + int(inclusive_count)
inclusive_count_fraction_sum = \
cwp_inclusive_count_statistics[key][2] + \
float(inclusive_count_fraction)
# All the functions are initially marked as EXTRA_FUNCTION.
value = \
(dso_name, inclusive_count_sum, inclusive_count_fraction_sum,
EXTRA_FUNCTION)
cwp_inclusive_count_statistics[key] = value
return cwp_inclusive_count_statistics
def ParseCWPPairwiseInclusiveCountFile(file_name):
"""Parses the CWP pairwise inclusive count files.
A line of the file should contain a pair of a parent and a child function,
concatenated by the PARENT_CHILD_FUNCTIONS_SEPARATOR, the name of the file
where the child function is declared and the inclusive count fractions of
the pair of functions out of the total amount of inclusive count values.
Args:
file_name: The file containing the pairwise inclusive_count statistics of
the
CWP functions.
Returns:
A dict containing the statistics of the parent functions and each of
their child functions. The key of the dict is the name of the parent
function. The value is a dict having as a key the name of the child
function with its file name separated by a ',' and as a value the
inclusive count value of the parent-child function pair.
"""
pairwise_inclusive_count_statistics = defaultdict(lambda: defaultdict(float))
with open(file_name) as input_file:
statistics_reader = csv.DictReader(input_file, delimiter=',')
for statistic in statistics_reader:
parent_function_name, child_function_name = \
statistic['parent_child_functions'].split(
PARENT_CHILD_FUNCTIONS_SEPARATOR)
child_function_file_name = MakeCWPAndPprofFileNamesConsistent(
os.path.normpath(statistic['child_function_file']))
inclusive_count = statistic['inclusive_count']
# There might be situations where a child function appears in
# multiple files or objects. Such situations can occur when in the
# Dremel queries are not specified the Chrome OS version and the
# name of the board (i.e the files can belong to different kernel or
# library versions), when the child function is a template function
# that is declared in a header file or there are name collisions
# between multiple executable objects.
# If a pair of child and parent functions appears multiple times, we
# add their inclusive count values.
child_function_key = ','.join(
[child_function_name, child_function_file_name])
pairwise_inclusive_count_statistics[parent_function_name] \
[child_function_key] += float(inclusive_count)
return pairwise_inclusive_count_statistics