#!/usr/bin/python
#
# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import datetime, unittest
import mox
import common
# This must come before the import of complete_failures in order to use the
# in memory database.
from autotest_lib.frontend import setup_django_readonly_environment
from autotest_lib.frontend import setup_test_environment
import complete_failures
from autotest_lib.client.common_lib import mail
from autotest_lib.frontend.tko import models
from django import test
GOOD_STATUS_IDX = 6
# See complte_failurs_functional_tests.py for why we need this.
class MockDatetime(datetime.datetime):
"""Used to mock out parts of datetime.datetime."""
pass
class EmailAboutTestFailureTests(mox.MoxTestBase):
"""
Tests that emails are sent about failed tests.
"""
def setUp(self):
super(EmailAboutTestFailureTests, self).setUp()
# We need to mock out the send function in all tests or else the
# emails will be sent out during tests.
self.mox.StubOutWithMock(mail, 'send')
self._orig_too_long = complete_failures._DAYS_TO_BE_FAILING_TOO_LONG
def tearDown(self):
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = self._orig_too_long
super(EmailAboutTestFailureTests, self).tearDown()
def test_email_sent_about_all_failed_tests(self):
"""Test that the email report mentions all the failed_tests."""
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 60
mail.send(
'chromeos-test-health@google.com',
['chromeos-lab-infrastructure@google.com'],
[],
'Long Failing Tests',
'1/1 tests have been failing for at least %d days.\n'
'They are the following:\n\ntest'
% complete_failures._DAYS_TO_BE_FAILING_TOO_LONG)
failures = ['test']
all_tests = set(failures)
self.mox.ReplayAll()
complete_failures.email_about_test_failure(failures, all_tests)
def test_email_has_test_names_sorted_alphabetically(self):
"""Test that the email report has entries sorted alphabetically."""
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 60
mail.send(
'chromeos-test-health@google.com',
['chromeos-lab-infrastructure@google.com'],
[],
'Long Failing Tests',
'2/2 tests have been failing for at least %d days.\n'
'They are the following:\n\ntest1\ntest2'
% complete_failures._DAYS_TO_BE_FAILING_TOO_LONG)
# We use an OrderedDict to gurantee that the elements would be out of
# order if we did a simple traversal.
failures = ['test2', 'test1']
all_tests = set(failures)
self.mox.ReplayAll()
complete_failures.email_about_test_failure(failures, all_tests)
def test_email_count_of_total_number_of_tests(self):
"""Test that the email report displays total number of tests."""
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 60
mail.send(
'chromeos-test-health@google.com',
['chromeos-lab-infrastructure@google.com'],
[],
'Long Failing Tests',
'1/2 tests have been failing for at least %d days.\n'
'They are the following:\n\ntest'
% complete_failures._DAYS_TO_BE_FAILING_TOO_LONG)
failures = ['test']
all_tests = set(failures) | {'not_failure'}
self.mox.ReplayAll()
complete_failures.email_about_test_failure(failures, all_tests)
class IsValidTestNameTests(test.TestCase):
"""Tests the is_valid_test_name function."""
def test_returns_true_for_valid_test_name(self):
"""Test that a valid test name returns True."""
name = 'TestName.TestName'
self.assertTrue(complete_failures.is_valid_test_name(name))
def test_returns_false_if_name_has_slash_in_it(self):
"""Test that a name with a slash in it returns False."""
name = 'path/to/test'
self.assertFalse(complete_failures.is_valid_test_name(name))
def test_returns_false_for_try_new_image_entries(self):
"""Test that a name that starts with try_new_image returns False."""
name = 'try_new_image-blah'
self.assertFalse(complete_failures.is_valid_test_name(name))
class PrepareLastPassesTests(test.TestCase):
"""Tests the prepare_last_passes function."""
def setUp(self):
super(PrepareLastPassesTests, self).setUp()
def tearDown(self):
super(PrepareLastPassesTests, self).tearDown()
def test_does_not_return_invalid_test_names(self):
"""Tests that tests with invalid test names are not returned."""
results = complete_failures.prepare_last_passes(['invalid_test/name'])
self.assertEqual(results, {})
class GetRecentlyRanTestNamesTests(mox.MoxTestBase, test.TestCase):
"""Tests the get_recently_ran_test_names function."""
def setUp(self):
super(GetRecentlyRanTestNamesTests, self).setUp()
self.mox.StubOutWithMock(MockDatetime, 'today')
self.datetime = datetime.datetime
datetime.datetime = MockDatetime
setup_test_environment.set_up()
self._orig_cutoff = complete_failures._DAYS_NOT_RUNNING_CUTOFF
def tearDown(self):
datetime.datetime = self.datetime
complete_failures._DAYS_NOT_RUNNING_CUTOFF = self._orig_cutoff
setup_test_environment.tear_down()
super(GetRecentlyRanTestNamesTests, self).tearDown()
def test_return_all_recently_ran_tests(self):
"""Test that the function does as it says it does."""
job = models.Job(job_idx=1)
kernel = models.Kernel(kernel_idx=1)
machine = models.Machine(machine_idx=1)
success_status = models.Status(status_idx=GOOD_STATUS_IDX)
recent = models.Test(job=job, status=success_status,
kernel=kernel, machine=machine,
test='recent',
started_time=self.datetime(2012, 1, 1))
recent.save()
old = models.Test(job=job, status=success_status,
kernel=kernel, machine=machine,
test='old',
started_time=self.datetime(2011, 1, 2))
old.save()
datetime.datetime.today().AndReturn(self.datetime(2012, 1, 4))
complete_failures._DAYS_NOT_RUNNING_CUTOFF = 60
self.mox.ReplayAll()
results = complete_failures.get_recently_ran_test_names()
self.assertEqual(set(results), set(['recent']))
def test_returns_no_duplicate_names(self):
"""Test that each test name appears only once."""
job = models.Job(job_idx=1)
kernel = models.Kernel(kernel_idx=1)
machine = models.Machine(machine_idx=1)
success_status = models.Status(status_idx=GOOD_STATUS_IDX)
test = models.Test(job=job, status=success_status,
kernel=kernel, machine=machine,
test='test',
started_time=self.datetime(2012, 1, 1))
test.save()
duplicate = models.Test(job=job, status=success_status,
kernel=kernel, machine=machine,
test='test',
started_time=self.datetime(2012, 1, 2))
duplicate.save()
datetime.datetime.today().AndReturn(self.datetime(2012, 1, 3))
complete_failures._DAYS_NOT_RUNNING_CUTOFF = 60
self.mox.ReplayAll()
results = complete_failures.get_recently_ran_test_names()
self.assertEqual(len(results), 1)
class GetTestsToAnalyzeTests(mox.MoxTestBase):
"""Tests the get_tests_to_analyze function."""
def test_returns_recent_test_names(self):
"""Test should return all the test names in the database."""
recent_tests = {'passing_test', 'failing_test'}
last_passes = {'passing_test': datetime.datetime(2012, 1 ,1),
'old_passing_test': datetime.datetime(2011, 1, 1)}
results = complete_failures.get_tests_to_analyze(recent_tests,
last_passes)
self.assertEqual(results,
{'passing_test': datetime.datetime(2012, 1, 1),
'failing_test': datetime.datetime.min})
def test_returns_failing_tests_with_min_datetime(self):
"""Test that never-passed tests are paired with datetime.min."""
recent_tests = {'test'}
last_passes = {}
self.mox.ReplayAll()
results = complete_failures.get_tests_to_analyze(recent_tests,
last_passes)
self.assertEqual(results, {'test': datetime.datetime.min})
class FilterOutGoodTestsTests(mox.MoxTestBase):
"""Tests the filter_our_good_tests function."""
def setUp(self):
super(FilterOutGoodTestsTests, self).setUp()
self.mox.StubOutWithMock(MockDatetime, 'today')
self.datetime = datetime.datetime
datetime.datetime = MockDatetime
self._orig_too_long = complete_failures._DAYS_TO_BE_FAILING_TOO_LONG
def tearDown(self):
datetime.datetime = self.datetime
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = self._orig_too_long
super(FilterOutGoodTestsTests, self).tearDown()
def test_remove_all_tests_that_have_passed_recently_enough(self):
"""Test that all recently passing tests are not returned."""
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 10
datetime.datetime.today().AndReturn(self.datetime(2012, 1, 21))
self.mox.ReplayAll()
result = complete_failures.filter_out_good_tests(
{'test': self.datetime(2012, 1, 20)})
self.assertEqual(result, [])
def test_keep_all_tests_that_have_not_passed_recently_enough(self):
"""Test that the tests that have not recently passed are returned."""
complete_failures._DAYS_TO_BE_FAILING_TOO_LONG = 10
datetime.datetime.today().AndReturn(self.datetime(2012, 1, 21))
self.mox.ReplayAll()
result = complete_failures.filter_out_good_tests(
{'test': self.datetime(2012, 1, 10)})
self.assertEqual(result, ['test'])
if __name__ == '__main__':
unittest.main()