#!/usr/bin/env python
#
# Copyright 2018 - The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Common operations between managing GCE and Cuttlefish devices.
This module provides the common operations between managing GCE (device_driver)
and Cuttlefish (create_cuttlefish_action) devices. Should not be called
directly.
"""
from __future__ import print_function
import getpass
import logging
import os
import subprocess
from acloud import errors
from acloud.public import avd
from acloud.public import report
from acloud.internal import constants
from acloud.internal.lib import utils
logger = logging.getLogger(__name__)
def CreateSshKeyPairIfNecessary(cfg):
"""Create ssh key pair if necessary.
Args:
cfg: An Acloudconfig instance.
Raises:
error.DriverError: If it falls into an unexpected condition.
"""
if not cfg.ssh_public_key_path:
logger.warning(
"ssh_public_key_path is not specified in acloud config. "
"Project-wide public key will "
"be used when creating AVD instances. "
"Please ensure you have the correct private half of "
"a project-wide public key if you want to ssh into the "
"instances after creation.")
elif cfg.ssh_public_key_path and not cfg.ssh_private_key_path:
logger.warning(
"Only ssh_public_key_path is specified in acloud config, "
"but ssh_private_key_path is missing. "
"Please ensure you have the correct private half "
"if you want to ssh into the instances after creation.")
elif cfg.ssh_public_key_path and cfg.ssh_private_key_path:
utils.CreateSshKeyPairIfNotExist(cfg.ssh_private_key_path,
cfg.ssh_public_key_path)
else:
# Should never reach here.
raise errors.DriverError(
"Unexpected error in CreateSshKeyPairIfNecessary")
class DevicePool(object):
"""A class that manages a pool of virtual devices.
Attributes:
devices: A list of devices in the pool.
"""
def __init__(self, device_factory, devices=None):
"""Constructs a new DevicePool.
Args:
device_factory: A device factory capable of producing a goldfish or
cuttlefish device. The device factory must expose an attribute with
the credentials that can be used to retrieve information from the
constructed device.
devices: List of devices managed by this pool.
"""
self._devices = devices or []
self._device_factory = device_factory
self._compute_client = device_factory.GetComputeClient()
def _CollectAdbLogcats(self, output_dir):
"""Collect Adb logcats.
Args:
output_dir: String, the output file directory to store adb logcats.
Returns:
The file information dictionary with file path and file name.
"""
file_dict = {}
for device in self._devices:
if not device.adb_port:
# If device adb tunnel is not established, do not do adb logcat
continue
file_name = "%s_adb_logcat.log" % device.instance_name
full_file_path = os.path.join(output_dir, file_name)
logger.info("Get adb %s:%s logcat for instance %s",
constants.LOCALHOST, device.adb_port,
device.instance_name)
try:
subprocess.check_call(
["adb -s %s:%s logcat -b all -d > %s" % (
constants.LOCALHOST, device.adb_port, full_file_path)],
shell=True)
file_dict[full_file_path] = file_name
except subprocess.CalledProcessError:
logging.error("Failed to get adb logcat for %s for instance %s",
device.serial_number, device.instance_name)
return file_dict
def CreateDevices(self, num):
"""Creates |num| devices for given build_target and build_id.
Args:
num: Number of devices to create.
"""
# Create host instances for cuttlefish/goldfish device.
# Currently one instance supports only 1 device.
for _ in range(num):
instance = self._device_factory.CreateInstance()
ip = self._compute_client.GetInstanceIP(instance)
self.devices.append(
avd.AndroidVirtualDevice(ip=ip, instance_name=instance))
@utils.TimeExecute(function_description="Waiting for AVD(s) to boot up",
result_evaluator=utils.BootEvaluator)
def WaitForBoot(self):
"""Waits for all devices to boot up.
Returns:
A dictionary that contains all the failures.
The key is the name of the instance that fails to boot,
and the value is an errors.DeviceBootError object.
"""
failures = {}
for device in self._devices:
try:
self._compute_client.WaitForBoot(device.instance_name)
except errors.DeviceBootError as e:
failures[device.instance_name] = e
return failures
def PullLogs(self, source_files, output_dir, user=None, ssh_rsa_path=None):
"""Tar logs from GCE instance into output_dir.
Args:
source_files: List of file names to be pulled.
output_dir: String. The output file dirtory
user: String, the ssh username to access GCE
ssh_rsa_path: String, the ssh rsa key path to access GCE
Returns:
The file dictionary with file_path and file_name
"""
file_dict = {}
for device in self._devices:
if isinstance(source_files, basestring):
source_files = [source_files]
for source_file in source_files:
file_name = "%s_%s" % (device.instance_name,
os.path.basename(source_file))
dst_file = os.path.join(output_dir, file_name)
logger.info("Pull %s for instance %s with user %s to %s",
source_file, device.instance_name, user, dst_file)
try:
utils.ScpPullFile(source_file, dst_file, device.ip,
user_name=user, rsa_key_file=ssh_rsa_path)
file_dict[dst_file] = file_name
except errors.DeviceConnectionError as e:
logger.warning("Failed to pull %s from instance %s: %s",
source_file, device.instance_name, e)
return file_dict
def CollectSerialPortLogs(self, output_file,
port=constants.DEFAULT_SERIAL_PORT):
"""Tar the instance serial logs into specified output_file.
Args:
output_file: String, the output tar file path
port: The serial port number to be collected
"""
# For emulator, the serial log is the virtual host serial log.
# For GCE AVD device, the serial log is the AVD device serial log.
with utils.TempDir() as tempdir:
src_dict = {}
for device in self._devices:
logger.info("Store instance %s serial port %s output to %s",
device.instance_name, port, output_file)
serial_log = self._compute_client.GetSerialPortOutput(
instance=device.instance_name, port=port)
file_name = "%s_serial_%s.log" % (device.instance_name, port)
file_path = os.path.join(tempdir, file_name)
src_dict[file_path] = file_name
with open(file_path, "w") as f:
f.write(serial_log.encode("utf-8"))
utils.MakeTarFile(src_dict, output_file)
def CollectLogcats(self, output_file, ssh_user, ssh_rsa_path):
"""Tar the instances' logcat and other logs into specified output_file.
Args:
output_file: String, the output tar file path
ssh_user: The ssh user name
ssh_rsa_path: The ssh rsa key path
"""
with utils.TempDir() as tempdir:
file_dict = {}
if getattr(self._device_factory, "LOG_FILES", None):
file_dict = self.PullLogs(
self._device_factory.LOG_FILES, tempdir, user=ssh_user,
ssh_rsa_path=ssh_rsa_path)
# If the device is auto-connected, get adb logcat
for file_path, file_name in self._CollectAdbLogcats(
tempdir).items():
file_dict[file_path] = file_name
utils.MakeTarFile(file_dict, output_file)
@property
def devices(self):
"""Returns a list of devices in the pool.
Returns:
A list of devices in the pool.
"""
return self._devices
# TODO: Delete unused-argument when b/119614469 is resolved.
# pylint: disable=unused-argument
# pylint: disable=too-many-locals
def CreateDevices(command, cfg, device_factory, num, avd_type,
report_internal_ip=False, autoconnect=False,
serial_log_file=None, logcat_file=None):
"""Create a set of devices using the given factory.
Main jobs in create devices.
1. Create GCE instance: Launch instance in GCP(Google Cloud Platform).
2. Starting up AVD: Wait device boot up.
Args:
command: The name of the command, used for reporting.
cfg: An AcloudConfig instance.
device_factory: A factory capable of producing a single device.
num: The number of devices to create.
avd_type: String, the AVD type(cuttlefish, goldfish...).
report_internal_ip: Boolean to report the internal ip instead of
external ip.
serial_log_file: String, the file path to tar the serial logs.
logcat_file: String, the file path to tar the logcats.
autoconnect: Boolean, whether to auto connect to device.
Raises:
errors: Create instance fail.
Returns:
A Report instance.
"""
reporter = report.Report(command=command)
try:
CreateSshKeyPairIfNecessary(cfg)
device_pool = DevicePool(device_factory)
device_pool.CreateDevices(num)
failures = device_pool.WaitForBoot()
if failures:
reporter.SetStatus(report.Status.BOOT_FAIL)
else:
reporter.SetStatus(report.Status.SUCCESS)
# Collect logs
if serial_log_file:
device_pool.CollectSerialPortLogs(
serial_log_file, port=constants.DEFAULT_SERIAL_PORT)
# TODO(b/119614469): Refactor CollectLogcats into a utils lib and
# turn it on inside the reporting loop.
# if logcat_file:
# device_pool.CollectLogcats(logcat_file, ssh_user, ssh_rsa_path)
# Write result to report.
for device in device_pool.devices:
ip = (device.ip.internal if report_internal_ip
else device.ip.external)
device_dict = {
"ip": ip,
"instance_name": device.instance_name
}
for attr in ("branch", "build_target", "build_id", "kernel_branch",
"kernel_build_target", "kernel_build_id",
"emulator_branch", "emulator_build_target",
"emulator_build_id"):
if getattr(device_factory, "_%s" % attr, None):
device_dict[attr] = getattr(device_factory, "_%s" % attr)
if autoconnect:
forwarded_ports = utils.AutoConnect(
ip, cfg.ssh_private_key_path,
utils.AVD_PORT_DICT[avd_type].vnc_port,
utils.AVD_PORT_DICT[avd_type].adb_port,
getpass.getuser())
device_dict[constants.VNC_PORT] = forwarded_ports.vnc_port
device_dict[constants.ADB_PORT] = forwarded_ports.adb_port
if device.instance_name in failures:
reporter.AddData(key="devices_failing_boot", value=device_dict)
reporter.AddError(str(failures[device.instance_name]))
else:
reporter.AddData(key="devices", value=device_dict)
except errors.DriverError as e:
reporter.AddError(str(e))
reporter.SetStatus(report.Status.FAIL)
return reporter