# Copyright 2017 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import json
import logging
import socket
import time
import urllib2
import urlparse
from autotest_lib.client.bin import utils as client_utils
from autotest_lib.client.common_lib import error
from autotest_lib.client.common_lib.cros import dev_server
from autotest_lib.server import hosts
def snippet(text):
"""Returns the text with start/end snip markers around it.
@param text: The snippet text.
@return The text with start/end snip markers around it.
"""
snip = '---8<---' * 10
start = '-- START -'
end = '-- END -'
return ('%s%s\n%s\n%s%s' %
(start, snip[len(start):], text, end, snip[len(end):]))
class OmahaDevserverFailedToStart(error.TestError):
"""Raised when a omaha devserver fails to start."""
class OmahaDevserver(object):
"""Spawns a test-private devserver instance."""
# How long to wait for a devserver to start.
_WAIT_FOR_DEVSERVER_STARTED_SECONDS = 30
# How long to sleep (seconds) between checks to see if a devserver is up.
_WAIT_SLEEP_INTERVAL = 1
# Max devserver execution time (seconds); used with timeout(1) to ensure we
# don't have defunct instances hogging the system.
_DEVSERVER_TIMELIMIT_SECONDS = 12 * 60 * 60
def __init__(self, omaha_host, update_payload_staged_url, max_updates=1,
critical_update=True):
"""Starts a private devserver instance, operating at Omaha capacity.
@param omaha_host: host address where the devserver is spawned.
@param update_payload_staged_url: URL to provision for update requests.
@param max_updates: int number of updates this devserver will handle.
This is passed to src/platform/dev/devserver.py.
@param critical_update: Whether to set a deadline in responses.
"""
self._devserver_dir = '/home/chromeos-test/chromiumos/src/platform/dev'
if not update_payload_staged_url:
raise error.TestError('Missing update payload url')
self._critical_update = critical_update
self._max_updates = max_updates
self._omaha_host = omaha_host
self._devserver_pid = 0
self._devserver_port = 0 # Determined later from devserver portfile.
self._update_payload_staged_url = update_payload_staged_url
self._devserver_ssh = hosts.SSHHost(self._omaha_host,
user='chromeos-test')
# Temporary files for various devserver outputs.
self._devserver_logfile = None
self._devserver_stdoutfile = None
self._devserver_portfile = None
self._devserver_pidfile = None
self._devserver_static_dir = None
def _cleanup_devserver_files(self):
"""Cleans up the temporary devserver files."""
for filename in (self._devserver_logfile, self._devserver_stdoutfile,
self._devserver_portfile, self._devserver_pidfile):
if filename:
self._devserver_ssh.run('rm -f %s' % filename,
ignore_status=True)
if self._devserver_static_dir:
self._devserver_ssh.run('rm -rf %s' % self._devserver_static_dir,
ignore_status=True)
def _create_tempfile_on_devserver(self, label, dir=False):
"""Creates a temporary file/dir on the devserver and returns its path.
@param label: Identifier for the file context (string, no whitespaces).
@param dir: If True, create a directory instead of a file.
@raises test.TestError: If we failed to invoke mktemp on the server.
@raises OmahaDevserverFailedToStart: If tempfile creation failed.
"""
remote_cmd = 'mktemp --tmpdir devserver-%s.XXXXXX' % label
if dir:
remote_cmd += ' --directory'
logging.info(remote_cmd)
try:
result = self._devserver_ssh.run(remote_cmd, ignore_status=True,
ssh_failure_retry_ok=True)
except error.AutoservRunError as e:
self._log_and_raise_remote_ssh_error(e)
if result.exit_status != 0:
logging.info(result)
raise OmahaDevserverFailedToStart(
'Could not create a temporary %s file on the devserver, '
'error output: "%s"' % (label, result.stderr))
return result.stdout.strip()
@staticmethod
def _log_and_raise_remote_ssh_error(e):
"""Logs failure to ssh remote, then raises a TestError."""
logging.debug('Failed to ssh into the devserver: %s', e)
logging.error('If you are running this locally it means you did not '
'configure ssh correctly.')
raise error.TestError('Failed to ssh into the devserver: %s' % e)
def _read_int_from_devserver_file(self, filename):
"""Reads and returns an integer value from a file on the devserver."""
return int(self._get_devserver_file_content(filename).strip())
def _wait_for_devserver_to_start(self):
"""Waits until the devserver starts within the time limit.
Infers and sets the devserver PID and serving port.
Raises:
OmahaDevserverFailedToStart: If the time limit is reached and we
cannot connect to the devserver.
"""
# Compute the overall timeout.
deadline = time.time() + self._WAIT_FOR_DEVSERVER_STARTED_SECONDS
# First, wait for port file to be filled and determine the server port.
logging.warning('Waiting for devserver to start up.')
while time.time() < deadline:
try:
self._devserver_pid = self._read_int_from_devserver_file(
self._devserver_pidfile)
self._devserver_port = self._read_int_from_devserver_file(
self._devserver_portfile)
logging.info('Devserver pid is %d, serving on port %d',
self._devserver_pid, self._devserver_port)
break
except Exception: # Couldn't read file or corrupt content.
time.sleep(self._WAIT_SLEEP_INTERVAL)
else:
try:
self._devserver_ssh.run_output('uptime',
ssh_failure_retry_ok=True)
except error.AutoservRunError as e:
logging.debug('Failed to run uptime on the devserver: %s', e)
raise OmahaDevserverFailedToStart(
'The test failed to find the pid/port of the omaha '
'devserver after %d seconds. Check the dumped devserver '
'logs and devserver load for more information.' %
self._WAIT_FOR_DEVSERVER_STARTED_SECONDS)
# Check that the server is reponsding to network requests.
logging.warning('Waiting for devserver to accept network requests.')
url = 'http://%s' % self.get_netloc()
while time.time() < deadline:
if dev_server.ImageServer.devserver_healthy(url, timeout_min=0.1):
break
# TODO(milleral): Refactor once crbug.com/221626 is resolved.
time.sleep(self._WAIT_SLEEP_INTERVAL)
else:
raise OmahaDevserverFailedToStart(
'The test failed to establish a connection to the omaha '
'devserver it set up on port %d. Check the dumped '
'devserver logs for more information.' %
self._devserver_port)
def start_devserver(self):
"""Starts the devserver and confirms it is up.
Raises:
test.TestError: If we failed to spawn the remote devserver.
OmahaDevserverFailedToStart: If the time limit is reached and we
cannot connect to the devserver.
"""
update_payload_url_base, update_payload_path = self._split_url(
self._update_payload_staged_url)
# Allocate temporary files for various server outputs.
self._devserver_logfile = self._create_tempfile_on_devserver('log')
self._devserver_stdoutfile = self._create_tempfile_on_devserver(
'stdout')
self._devserver_portfile = self._create_tempfile_on_devserver('port')
self._devserver_pidfile = self._create_tempfile_on_devserver('pid')
self._devserver_static_dir = self._create_tempfile_on_devserver(
'static', dir=True)
# Invoke the Omaha/devserver on the remote server. Will attempt to kill
# it with a SIGTERM after a predetermined timeout has elapsed, followed
# by SIGKILL if not dead within 30 seconds from the former signal.
cmdlist = [
'timeout', '-s', 'TERM', '-k', '30',
str(self._DEVSERVER_TIMELIMIT_SECONDS),
'%s/devserver.py' % self._devserver_dir,
'--payload=%s' % update_payload_path,
'--port=0',
'--pidfile=%s' % self._devserver_pidfile,
'--portfile=%s' % self._devserver_portfile,
'--logfile=%s' % self._devserver_logfile,
'--remote_payload',
'--urlbase=%s' % update_payload_url_base,
'--max_updates=%s' % self._max_updates,
'--host_log',
'--static_dir=%s' % self._devserver_static_dir
]
if self._critical_update:
cmdlist.append('--critical_update')
remote_cmd = '( %s ) </dev/null >%s 2>&1 &' % (
' '.join(cmdlist), self._devserver_stdoutfile)
logging.info('Starting devserver with %r', remote_cmd)
try:
self._devserver_ssh.run_output(remote_cmd,
ssh_failure_retry_ok=True)
except error.AutoservRunError as e:
self._log_and_raise_remote_ssh_error(e)
try:
self._wait_for_devserver_to_start()
except OmahaDevserverFailedToStart:
self._kill_remote_process()
self._dump_devserver_log()
self._cleanup_devserver_files()
raise
def _kill_remote_process(self):
"""Kills the devserver and verifies it's down; clears the remote pid."""
def devserver_down():
"""Ensure that the devserver process is down."""
return not self._remote_process_alive()
if devserver_down():
return
for signal in 'SIGTERM', 'SIGKILL':
remote_cmd = 'kill -s %s %s' % (signal, self._devserver_pid)
self._devserver_ssh.run(remote_cmd, ssh_failure_retry_ok=True)
try:
client_utils.poll_for_condition(
devserver_down, sleep_interval=1, desc='devserver down')
break
except client_utils.TimeoutError:
logging.warning('Could not kill devserver with %s.', signal)
else:
logging.warning('Failed to kill devserver, giving up.')
self._devserver_pid = None
def _remote_process_alive(self):
"""Tests whether the remote devserver process is running."""
if not self._devserver_pid:
return False
remote_cmd = 'test -e /proc/%s' % self._devserver_pid
result = self._devserver_ssh.run(remote_cmd, ignore_status=True)
return result.exit_status == 0
def get_netloc(self):
"""Returns the netloc (host:port) of the devserver."""
if not (self._devserver_pid and self._devserver_port):
raise error.TestError('No running omaha/devserver')
return '%s:%s' % (self._omaha_host, self._devserver_port)
def get_update_url(self):
"""Returns the update_url you can use to update via this server."""
return urlparse.urlunsplit(('http', self.get_netloc(), '/update',
'', ''))
def _get_devserver_file_content(self, filename):
"""Returns the content of a file on the devserver."""
return self._devserver_ssh.run_output('cat %s' % filename,
stdout_tee=None,
ssh_failure_retry_ok=True)
def _get_devserver_log(self):
"""Obtain the devserver output."""
return self._get_devserver_file_content(self._devserver_logfile)
def _get_devserver_stdout(self):
"""Obtain the devserver output in stdout and stderr."""
return self._get_devserver_file_content(self._devserver_stdoutfile)
def get_hostlog(self, ip, wait_for_reboot_events=False):
"""Get the update events json (aka hostlog).
@param ip: IP of the DUT to get update info for.
@param wait_for_reboot_events: True if we expect the reboot events.
@return the json dump of the update events for the given IP.
"""
omaha_hostlog_url = urlparse.urlunsplit(
['http', self.get_netloc(), '/api/hostlog',
'ip=' + ip, ''])
# 4 rootfs and 1 post reboot
expected_events_count = 5
while True:
try:
conn = urllib2.urlopen(omaha_hostlog_url)
except urllib2.URLError, e:
logging.warning('Failed to read event log url: %s', e)
return None
except socket.timeout, e:
logging.warning('Timed out reading event log url: %s', e)
return None
event_log_resp = conn.read()
conn.close()
hostlog = json.loads(event_log_resp)
if wait_for_reboot_events and len(hostlog) < expected_events_count:
time.sleep(5)
continue
else:
return hostlog
def _dump_devserver_log(self, logging_level=logging.ERROR):
"""Dump the devserver log to the autotest log.
@param logging_level: logging level (from logging) to log the output.
"""
logging.log(logging_level, "Devserver stdout and stderr:\n" +
snippet(self._get_devserver_stdout()))
logging.log(logging_level, "Devserver log file:\n" +
snippet(self._get_devserver_log()))
@staticmethod
def _split_url(url):
"""Splits a URL into the URL base and path."""
split_url = urlparse.urlsplit(url)
url_base = urlparse.urlunsplit(
(split_url.scheme, split_url.netloc, '', '', ''))
url_path = split_url.path
return url_base, url_path.lstrip('/')
def stop_devserver(self):
"""Kill remote process and wait for it to die, dump its output."""
if not self._devserver_pid:
logging.error('No running omaha/devserver.')
return
logging.info('Killing omaha/devserver')
self._kill_remote_process()
logging.debug('Final devserver log before killing')
self._dump_devserver_log(logging.DEBUG)
self._cleanup_devserver_files()