#!/usr/bin/env python2.7
"""Check CFC - Check Compile Flow Consistency
This is a compiler wrapper for testing that code generation is consistent with
different compilation processes. It checks that code is not unduly affected by
compiler options or other changes which should not have side effects.
To use:
-Ensure that the compiler under test (i.e. clang, clang++) is on the PATH
-On Linux copy this script to the name of the compiler
e.g. cp check_cfc.py clang && cp check_cfc.py clang++
-On Windows use setup.py to generate check_cfc.exe and copy that to clang.exe
and clang++.exe
-Enable the desired checks in check_cfc.cfg (in the same directory as the
wrapper)
e.g.
[Checks]
dash_g_no_change = true
dash_s_no_change = false
-The wrapper can be run using its absolute path or added to PATH before the
compiler under test
e.g. export PATH=<path to check_cfc>:$PATH
-Compile as normal. The wrapper intercepts normal -c compiles and will return
non-zero if the check fails.
e.g.
$ clang -c test.cpp
Code difference detected with -g
--- /tmp/tmp5nv893.o
+++ /tmp/tmp6Vwjnc.o
@@ -1 +1 @@
- 0: 48 8b 05 51 0b 20 00 mov 0x200b51(%rip),%rax
+ 0: 48 39 3d 51 0b 20 00 cmp %rdi,0x200b51(%rip)
-To run LNT with Check CFC specify the absolute path to the wrapper to the --cc
and --cxx options
e.g.
lnt runtest nt --cc <path to check_cfc>/clang \\
--cxx <path to check_cfc>/clang++ ...
To add a new check:
-Create a new subclass of WrapperCheck
-Implement the perform_check() method. This should perform the alternate compile
and do the comparison.
-Add the new check to check_cfc.cfg. The check has the same name as the
subclass.
"""
from __future__ import print_function
import imp
import os
import platform
import shutil
import subprocess
import sys
import tempfile
import ConfigParser
import io
import obj_diff
def is_windows():
"""Returns True if running on Windows."""
return platform.system() == 'Windows'
class WrapperStepException(Exception):
"""Exception type to be used when a step other than the original compile
fails."""
def __init__(self, msg, stdout, stderr):
self.msg = msg
self.stdout = stdout
self.stderr = stderr
class WrapperCheckException(Exception):
"""Exception type to be used when a comparison check fails."""
def __init__(self, msg):
self.msg = msg
def main_is_frozen():
"""Returns True when running as a py2exe executable."""
return (hasattr(sys, "frozen") or # new py2exe
hasattr(sys, "importers") or # old py2exe
imp.is_frozen("__main__")) # tools/freeze
def get_main_dir():
"""Get the directory that the script or executable is located in."""
if main_is_frozen():
return os.path.dirname(sys.executable)
return os.path.dirname(sys.argv[0])
def remove_dir_from_path(path_var, directory):
"""Remove the specified directory from path_var, a string representing
PATH"""
pathlist = path_var.split(os.pathsep)
norm_directory = os.path.normpath(os.path.normcase(directory))
pathlist = filter(lambda x: os.path.normpath(
os.path.normcase(x)) != norm_directory, pathlist)
return os.pathsep.join(pathlist)
def path_without_wrapper():
"""Returns the PATH variable modified to remove the path to this program."""
scriptdir = get_main_dir()
path = os.environ['PATH']
return remove_dir_from_path(path, scriptdir)
def flip_dash_g(args):
"""Search for -g in args. If it exists then return args without. If not then
add it."""
if '-g' in args:
# Return args without any -g
return [x for x in args if x != '-g']
else:
# No -g, add one
return args + ['-g']
def derive_output_file(args):
"""Derive output file from the input file (if just one) or None
otherwise."""
infile = get_input_file(args)
if infile is None:
return None
else:
return '{}.o'.format(os.path.splitext(infile)[0])
def get_output_file(args):
"""Return the output file specified by this command or None if not
specified."""
grabnext = False
for arg in args:
if grabnext:
return arg
if arg == '-o':
# Specified as a separate arg
grabnext = True
elif arg.startswith('-o'):
# Specified conjoined with -o
return arg[2:]
assert grabnext == False
return None
def is_output_specified(args):
"""Return true is output file is specified in args."""
return get_output_file(args) is not None
def replace_output_file(args, new_name):
"""Replaces the specified name of an output file with the specified name.
Assumes that the output file name is specified in the command line args."""
replaceidx = None
attached = False
for idx, val in enumerate(args):
if val == '-o':
replaceidx = idx + 1
attached = False
elif val.startswith('-o'):
replaceidx = idx
attached = True
if replaceidx is None:
raise Exception
replacement = new_name
if attached == True:
replacement = '-o' + new_name
args[replaceidx] = replacement
return args
def add_output_file(args, output_file):
"""Append an output file to args, presuming not already specified."""
return args + ['-o', output_file]
def set_output_file(args, output_file):
"""Set the output file within the arguments. Appends or replaces as
appropriate."""
if is_output_specified(args):
args = replace_output_file(args, output_file)
else:
args = add_output_file(args, output_file)
return args
gSrcFileSuffixes = ('.c', '.cpp', '.cxx', '.c++', '.cp', '.cc')
def get_input_file(args):
"""Return the input file string if it can be found (and there is only
one)."""
inputFiles = list()
for arg in args:
testarg = arg
quotes = ('"', "'")
while testarg.endswith(quotes):
testarg = testarg[:-1]
testarg = os.path.normcase(testarg)
# Test if it is a source file
if testarg.endswith(gSrcFileSuffixes):
inputFiles.append(arg)
if len(inputFiles) == 1:
return inputFiles[0]
else:
return None
def set_input_file(args, input_file):
"""Replaces the input file with that specified."""
infile = get_input_file(args)
if infile:
infile_idx = args.index(infile)
args[infile_idx] = input_file
return args
else:
# Could not find input file
assert False
def is_normal_compile(args):
"""Check if this is a normal compile which will output an object file rather
than a preprocess or link. args is a list of command line arguments."""
compile_step = '-c' in args
# Bitcode cannot be disassembled in the same way
bitcode = '-flto' in args or '-emit-llvm' in args
# Version and help are queries of the compiler and override -c if specified
query = '--version' in args or '--help' in args
# Options to output dependency files for make
dependency = '-M' in args or '-MM' in args
# Check if the input is recognised as a source file (this may be too
# strong a restriction)
input_is_valid = bool(get_input_file(args))
return compile_step and not bitcode and not query and not dependency and input_is_valid
def run_step(command, my_env, error_on_failure):
"""Runs a step of the compilation. Reports failure as exception."""
# Need to use shell=True on Windows as Popen won't use PATH otherwise.
p = subprocess.Popen(command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=my_env, shell=is_windows())
(stdout, stderr) = p.communicate()
if p.returncode != 0:
raise WrapperStepException(error_on_failure, stdout, stderr)
def get_temp_file_name(suffix):
"""Get a temporary file name with a particular suffix. Let the caller be
reponsible for deleting it."""
tf = tempfile.NamedTemporaryFile(suffix=suffix, delete=False)
tf.close()
return tf.name
class WrapperCheck(object):
"""Base class for a check. Subclass this to add a check."""
def __init__(self, output_file_a):
"""Record the base output file that will be compared against."""
self._output_file_a = output_file_a
def perform_check(self, arguments, my_env):
"""Override this to perform the modified compilation and required
checks."""
raise NotImplementedError("Please Implement this method")
class dash_g_no_change(WrapperCheck):
def perform_check(self, arguments, my_env):
"""Check if different code is generated with/without the -g flag."""
output_file_b = get_temp_file_name('.o')
alternate_command = list(arguments)
alternate_command = flip_dash_g(alternate_command)
alternate_command = set_output_file(alternate_command, output_file_b)
run_step(alternate_command, my_env, "Error compiling with -g")
# Compare disassembly (returns first diff if differs)
difference = obj_diff.compare_object_files(self._output_file_a,
output_file_b)
if difference:
raise WrapperCheckException(
"Code difference detected with -g\n{}".format(difference))
# Clean up temp file if comparison okay
os.remove(output_file_b)
class dash_s_no_change(WrapperCheck):
def perform_check(self, arguments, my_env):
"""Check if compiling to asm then assembling in separate steps results
in different code than compiling to object directly."""
output_file_b = get_temp_file_name('.o')
alternate_command = arguments + ['-via-file-asm']
alternate_command = set_output_file(alternate_command, output_file_b)
run_step(alternate_command, my_env,
"Error compiling with -via-file-asm")
# Compare if object files are exactly the same
exactly_equal = obj_diff.compare_exact(self._output_file_a, output_file_b)
if not exactly_equal:
# Compare disassembly (returns first diff if differs)
difference = obj_diff.compare_object_files(self._output_file_a,
output_file_b)
if difference:
raise WrapperCheckException(
"Code difference detected with -S\n{}".format(difference))
# Code is identical, compare debug info
dbgdifference = obj_diff.compare_debug_info(self._output_file_a,
output_file_b)
if dbgdifference:
raise WrapperCheckException(
"Debug info difference detected with -S\n{}".format(dbgdifference))
raise WrapperCheckException("Object files not identical with -S\n")
# Clean up temp file if comparison okay
os.remove(output_file_b)
if __name__ == '__main__':
# Create configuration defaults from list of checks
default_config = """
[Checks]
"""
# Find all subclasses of WrapperCheck
checks = [cls.__name__ for cls in vars()['WrapperCheck'].__subclasses__()]
for c in checks:
default_config += "{} = false\n".format(c)
config = ConfigParser.RawConfigParser()
config.readfp(io.BytesIO(default_config))
scriptdir = get_main_dir()
config_path = os.path.join(scriptdir, 'check_cfc.cfg')
try:
config.read(os.path.join(config_path))
except:
print("Could not read config from {}, "
"using defaults.".format(config_path))
my_env = os.environ.copy()
my_env['PATH'] = path_without_wrapper()
arguments_a = list(sys.argv)
# Prevent infinite loop if called with absolute path.
arguments_a[0] = os.path.basename(arguments_a[0])
# Sanity check
enabled_checks = [check_name
for check_name in checks
if config.getboolean('Checks', check_name)]
checks_comma_separated = ', '.join(enabled_checks)
print("Check CFC, checking: {}".format(checks_comma_separated))
# A - original compilation
output_file_orig = get_output_file(arguments_a)
if output_file_orig is None:
output_file_orig = derive_output_file(arguments_a)
p = subprocess.Popen(arguments_a, env=my_env, shell=is_windows())
p.communicate()
if p.returncode != 0:
sys.exit(p.returncode)
if not is_normal_compile(arguments_a) or output_file_orig is None:
# Bail out here if we can't apply checks in this case.
# Does not indicate an error.
# Maybe not straight compilation (e.g. -S or --version or -flto)
# or maybe > 1 input files.
sys.exit(0)
# Sometimes we generate files which have very long names which can't be
# read/disassembled. This will exit early if we can't find the file we
# expected to be output.
if not os.path.isfile(output_file_orig):
sys.exit(0)
# Copy output file to a temp file
temp_output_file_orig = get_temp_file_name('.o')
shutil.copyfile(output_file_orig, temp_output_file_orig)
# Run checks, if they are enabled in config and if they are appropriate for
# this command line.
current_module = sys.modules[__name__]
for check_name in checks:
if config.getboolean('Checks', check_name):
class_ = getattr(current_module, check_name)
checker = class_(temp_output_file_orig)
try:
checker.perform_check(arguments_a, my_env)
except WrapperCheckException as e:
# Check failure
print("{} {}".format(get_input_file(arguments_a), e.msg), file=sys.stderr)
# Remove file to comply with build system expectations (no
# output file if failed)
os.remove(output_file_orig)
sys.exit(1)
except WrapperStepException as e:
# Compile step failure
print(e.msg, file=sys.stderr)
print("*** stdout ***", file=sys.stderr)
print(e.stdout, file=sys.stderr)
print("*** stderr ***", file=sys.stderr)
print(e.stderr, file=sys.stderr)
# Remove file to comply with build system expectations (no
# output file if failed)
os.remove(output_file_orig)
sys.exit(1)