# Copyright 2018 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import json
import os
import sys
import time
from . import base
from ..local import junit_output
def print_failure_header(test):
if test.output_proc.negative:
negative_marker = '[negative] '
else:
negative_marker = ''
print "=== %(label)s %(negative)s===" % {
'label': test,
'negative': negative_marker,
}
class TestsCounter(base.TestProcObserver):
def __init__(self):
super(TestsCounter, self).__init__()
self.total = 0
def _on_next_test(self, test):
self.total += 1
class ResultsTracker(base.TestProcObserver):
def __init__(self):
super(ResultsTracker, self).__init__()
self._requirement = base.DROP_OUTPUT
self.failed = 0
self.remaining = 0
self.total = 0
def _on_next_test(self, test):
self.total += 1
self.remaining += 1
def _on_result_for(self, test, result):
self.remaining -= 1
if result.has_unexpected_output:
self.failed += 1
class ProgressIndicator(base.TestProcObserver):
def finished(self):
pass
class SimpleProgressIndicator(ProgressIndicator):
def __init__(self):
super(SimpleProgressIndicator, self).__init__()
self._requirement = base.DROP_PASS_OUTPUT
self._failed = []
self._total = 0
def _on_next_test(self, test):
self._total += 1
def _on_result_for(self, test, result):
# TODO(majeski): Support for dummy/grouped results
if result.has_unexpected_output:
self._failed.append((test, result))
def finished(self):
crashed = 0
print
for test, result in self._failed:
print_failure_header(test)
if result.output.stderr:
print "--- stderr ---"
print result.output.stderr.strip()
if result.output.stdout:
print "--- stdout ---"
print result.output.stdout.strip()
print "Command: %s" % result.cmd.to_string()
if result.output.HasCrashed():
print "exit code: %d" % result.output.exit_code
print "--- CRASHED ---"
crashed += 1
if result.output.HasTimedOut():
print "--- TIMEOUT ---"
if len(self._failed) == 0:
print "==="
print "=== All tests succeeded"
print "==="
else:
print
print "==="
print "=== %i tests failed" % len(self._failed)
if crashed > 0:
print "=== %i tests CRASHED" % crashed
print "==="
class VerboseProgressIndicator(SimpleProgressIndicator):
def __init__(self):
super(VerboseProgressIndicator, self).__init__()
self._last_printed_time = time.time()
def _print(self, text):
print text
sys.stdout.flush()
self._last_printed_time = time.time()
def _on_result_for(self, test, result):
super(VerboseProgressIndicator, self)._on_result_for(test, result)
# TODO(majeski): Support for dummy/grouped results
if result.has_unexpected_output:
if result.output.HasCrashed():
outcome = 'CRASH'
else:
outcome = 'FAIL'
else:
outcome = 'pass'
self._print('Done running %s: %s' % (test, outcome))
def _on_heartbeat(self):
if time.time() - self._last_printed_time > 30:
# Print something every 30 seconds to not get killed by an output
# timeout.
self._print('Still working...')
class DotsProgressIndicator(SimpleProgressIndicator):
def __init__(self):
super(DotsProgressIndicator, self).__init__()
self._count = 0
def _on_result_for(self, test, result):
# TODO(majeski): Support for dummy/grouped results
self._count += 1
if self._count > 1 and self._count % 50 == 1:
sys.stdout.write('\n')
if result.has_unexpected_output:
if result.output.HasCrashed():
sys.stdout.write('C')
sys.stdout.flush()
elif result.output.HasTimedOut():
sys.stdout.write('T')
sys.stdout.flush()
else:
sys.stdout.write('F')
sys.stdout.flush()
else:
sys.stdout.write('.')
sys.stdout.flush()
class CompactProgressIndicator(ProgressIndicator):
def __init__(self, templates):
super(CompactProgressIndicator, self).__init__()
self._requirement = base.DROP_PASS_OUTPUT
self._templates = templates
self._last_status_length = 0
self._start_time = time.time()
self._total = 0
self._passed = 0
self._failed = 0
def _on_next_test(self, test):
self._total += 1
def _on_result_for(self, test, result):
# TODO(majeski): Support for dummy/grouped results
if result.has_unexpected_output:
self._failed += 1
else:
self._passed += 1
self._print_progress(str(test))
if result.has_unexpected_output:
output = result.output
stdout = output.stdout.strip()
stderr = output.stderr.strip()
self._clear_line(self._last_status_length)
print_failure_header(test)
if len(stdout):
print self._templates['stdout'] % stdout
if len(stderr):
print self._templates['stderr'] % stderr
print "Command: %s" % result.cmd
if output.HasCrashed():
print "exit code: %d" % output.exit_code
print "--- CRASHED ---"
if output.HasTimedOut():
print "--- TIMEOUT ---"
def finished(self):
self._print_progress('Done')
print
def _print_progress(self, name):
self._clear_line(self._last_status_length)
elapsed = time.time() - self._start_time
if not self._total:
progress = 0
else:
progress = (self._passed + self._failed) * 100 // self._total
status = self._templates['status_line'] % {
'passed': self._passed,
'progress': progress,
'failed': self._failed,
'test': name,
'mins': int(elapsed) / 60,
'secs': int(elapsed) % 60
}
status = self._truncate(status, 78)
self._last_status_length = len(status)
print status,
sys.stdout.flush()
def _truncate(self, string, length):
if length and len(string) > (length - 3):
return string[:(length - 3)] + "..."
else:
return string
def _clear_line(self, last_length):
raise NotImplementedError()
class ColorProgressIndicator(CompactProgressIndicator):
def __init__(self):
templates = {
'status_line': ("[%(mins)02i:%(secs)02i|"
"\033[34m%%%(progress) 4d\033[0m|"
"\033[32m+%(passed) 4d\033[0m|"
"\033[31m-%(failed) 4d\033[0m]: %(test)s"),
'stdout': "\033[1m%s\033[0m",
'stderr': "\033[31m%s\033[0m",
}
super(ColorProgressIndicator, self).__init__(templates)
def _clear_line(self, last_length):
print "\033[1K\r",
class MonochromeProgressIndicator(CompactProgressIndicator):
def __init__(self):
templates = {
'status_line': ("[%(mins)02i:%(secs)02i|%%%(progress) 4d|"
"+%(passed) 4d|-%(failed) 4d]: %(test)s"),
'stdout': '%s',
'stderr': '%s',
}
super(MonochromeProgressIndicator, self).__init__(templates)
def _clear_line(self, last_length):
print ("\r" + (" " * last_length) + "\r"),
class JUnitTestProgressIndicator(ProgressIndicator):
def __init__(self, junitout, junittestsuite):
super(JUnitTestProgressIndicator, self).__init__()
self._requirement = base.DROP_PASS_STDOUT
self.outputter = junit_output.JUnitTestOutput(junittestsuite)
if junitout:
self.outfile = open(junitout, "w")
else:
self.outfile = sys.stdout
def _on_result_for(self, test, result):
# TODO(majeski): Support for dummy/grouped results
fail_text = ""
output = result.output
if result.has_unexpected_output:
stdout = output.stdout.strip()
if len(stdout):
fail_text += "stdout:\n%s\n" % stdout
stderr = output.stderr.strip()
if len(stderr):
fail_text += "stderr:\n%s\n" % stderr
fail_text += "Command: %s" % result.cmd.to_string()
if output.HasCrashed():
fail_text += "exit code: %d\n--- CRASHED ---" % output.exit_code
if output.HasTimedOut():
fail_text += "--- TIMEOUT ---"
self.outputter.HasRunTest(
test_name=str(test),
test_cmd=result.cmd.to_string(relative=True),
test_duration=output.duration,
test_failure=fail_text)
def finished(self):
self.outputter.FinishAndWrite(self.outfile)
if self.outfile != sys.stdout:
self.outfile.close()
class JsonTestProgressIndicator(ProgressIndicator):
def __init__(self, json_test_results, arch, mode):
super(JsonTestProgressIndicator, self).__init__()
# We want to drop stdout/err for all passed tests on the first try, but we
# need to get outputs for all runs after the first one. To accommodate that,
# reruns are set to keep the result no matter what requirement says, i.e.
# keep_output set to True in the RerunProc.
self._requirement = base.DROP_PASS_STDOUT
self.json_test_results = json_test_results
self.arch = arch
self.mode = mode
self.results = []
self.tests = []
def _on_result_for(self, test, result):
if result.is_rerun:
self.process_results(test, result.results)
else:
self.process_results(test, [result])
def process_results(self, test, results):
for run, result in enumerate(results):
# TODO(majeski): Support for dummy/grouped results
output = result.output
# Buffer all tests for sorting the durations in the end.
# TODO(machenbach): Running average + buffer only slowest 20 tests.
self.tests.append((test, output.duration, result.cmd))
# Omit tests that run as expected on the first try.
# Everything that happens after the first run is included in the output
# even if it flakily passes.
if not result.has_unexpected_output and run == 0:
continue
self.results.append({
"name": str(test),
"flags": result.cmd.args,
"command": result.cmd.to_string(relative=True),
"run": run + 1,
"stdout": output.stdout,
"stderr": output.stderr,
"exit_code": output.exit_code,
"result": test.output_proc.get_outcome(output),
"expected": test.expected_outcomes,
"duration": output.duration,
"random_seed": test.random_seed,
"target_name": test.get_shell(),
"variant": test.variant,
})
def finished(self):
complete_results = []
if os.path.exists(self.json_test_results):
with open(self.json_test_results, "r") as f:
# Buildbot might start out with an empty file.
complete_results = json.loads(f.read() or "[]")
duration_mean = None
if self.tests:
# Get duration mean.
duration_mean = (
sum(duration for (_, duration, cmd) in self.tests) /
float(len(self.tests)))
# Sort tests by duration.
self.tests.sort(key=lambda (_, duration, cmd): duration, reverse=True)
slowest_tests = [
{
"name": str(test),
"flags": cmd.args,
"command": cmd.to_string(relative=True),
"duration": duration,
"marked_slow": test.is_slow,
} for (test, duration, cmd) in self.tests[:20]
]
complete_results.append({
"arch": self.arch,
"mode": self.mode,
"results": self.results,
"slowest_tests": slowest_tests,
"duration_mean": duration_mean,
"test_total": len(self.tests),
})
with open(self.json_test_results, "w") as f:
f.write(json.dumps(complete_results))