#!/usr/bin/env python
#
# Copyright 2016 - The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Base Cloud API Client.
BasicCloudApiCliend does basic setup for a cloud API.
"""
import httplib
import logging
import socket
import ssl
# pylint: disable=import-error
from apiclient import errors as gerrors
from apiclient.discovery import build
import apiclient.http
import httplib2
from oauth2client import client
from acloud import errors
from acloud.internal.lib import utils
logger = logging.getLogger(__name__)
class BaseCloudApiClient(object):
"""A class that does basic setup for a cloud API."""
# To be overriden by subclasses.
API_NAME = ""
API_VERSION = "v1"
SCOPE = ""
# Defaults for retry.
RETRY_COUNT = 5
RETRY_BACKOFF_FACTOR = 1.5
RETRY_SLEEP_MULTIPLIER = 2
RETRY_HTTP_CODES = [
# 403 is to retry the "Rate Limit Exceeded" error.
# We could retry on a finer-grained error message later if necessary.
403,
500, # Internal Server Error
502, # Bad Gateway
503, # Service Unavailable
]
RETRIABLE_ERRORS = (httplib.HTTPException, httplib2.HttpLib2Error,
socket.error, ssl.SSLError)
RETRIABLE_AUTH_ERRORS = (client.AccessTokenRefreshError, )
def __init__(self, oauth2_credentials):
"""Initialize.
Args:
oauth2_credentials: An oauth2client.OAuth2Credentials instance.
"""
self._service = self.InitResourceHandle(oauth2_credentials)
@classmethod
def InitResourceHandle(cls, oauth2_credentials):
"""Authenticate and initialize a Resource object.
Authenticate http and create a Resource object with methods
for interacting with the service.
Args:
oauth2_credentials: An oauth2client.OAuth2Credentials instance.
Returns:
An apiclient.discovery.Resource object
"""
http_auth = oauth2_credentials.authorize(httplib2.Http())
return utils.RetryExceptionType(
exception_types=cls.RETRIABLE_AUTH_ERRORS,
max_retries=cls.RETRY_COUNT,
functor=build,
sleep_multiplier=cls.RETRY_SLEEP_MULTIPLIER,
retry_backoff_factor=cls.RETRY_BACKOFF_FACTOR,
serviceName=cls.API_NAME,
version=cls.API_VERSION,
# This is workaround for a known issue of some veriosn
# of api client.
# https://github.com/google/google-api-python-client/issues/435
cache_discovery=False,
http=http_auth)
@staticmethod
def _ShouldRetry(exception, retry_http_codes,
other_retriable_errors):
"""Check if exception is retriable.
Args:
exception: An instance of Exception.
retry_http_codes: a list of integers, retriable HTTP codes of
HttpError
other_retriable_errors: a tuple of error types to retry other than
HttpError.
Returns:
Boolean, True if retriable, False otherwise.
"""
if isinstance(exception, other_retriable_errors):
return True
if isinstance(exception, errors.HttpError):
if exception.code in retry_http_codes:
return True
else:
logger.debug("_ShouldRetry: Exception code %s not in %s: %s",
exception.code, retry_http_codes, str(exception))
logger.debug("_ShouldRetry: Exception %s is not one of %s: %s",
type(exception),
list(other_retriable_errors) + [errors.HttpError],
str(exception))
return False
@staticmethod
def _TranslateError(exception):
"""Translate the exception to a desired type.
Args:
exception: An instance of Exception.
Returns:
gerrors.HttpError will be translated to errors.HttpError.
If the error code is errors.HTTP_NOT_FOUND_CODE, it will
be translated to errors.ResourceNotFoundError.
Unrecognized error type will not be translated and will
be returned as is.
"""
if isinstance(exception, gerrors.HttpError):
exception = errors.HttpError.CreateFromHttpError(exception)
if exception.code == errors.HTTP_NOT_FOUND_CODE:
exception = errors.ResourceNotFoundError(
exception.code, str(exception))
return exception
def ExecuteOnce(self, api):
"""Execute an api and parse the errors.
Args:
api: An apiclient.http.HttpRequest, representing the api to execute.
Returns:
Execution result of the api.
Raises:
errors.ResourceNotFoundError: For 404 error.
errors.HttpError: For other types of http error.
"""
try:
return api.execute()
except gerrors.HttpError as e:
raise self._TranslateError(e)
def Execute(self,
api,
retry_http_codes=None,
max_retry=None,
sleep=None,
backoff_factor=None,
other_retriable_errors=None):
"""Execute an api with retry.
Call ExecuteOnce and retry on http error with given codes.
Args:
api: An apiclient.http.HttpRequest, representing the api to execute:
retry_http_codes: A list of http codes to retry.
max_retry: See utils.Retry.
sleep: See utils.Retry.
backoff_factor: See utils.Retry.
other_retriable_errors: A tuple of error types that should be retried
other than errors.HttpError.
Returns:
Execution result of the api.
Raises:
See ExecuteOnce.
"""
retry_http_codes = (self.RETRY_HTTP_CODES
if retry_http_codes is None else retry_http_codes)
max_retry = (self.RETRY_COUNT if max_retry is None else max_retry)
sleep = (self.RETRY_SLEEP_MULTIPLIER if sleep is None else sleep)
backoff_factor = (self.RETRY_BACKOFF_FACTOR
if backoff_factor is None else backoff_factor)
other_retriable_errors = (self.RETRIABLE_ERRORS
if other_retriable_errors is None else
other_retriable_errors)
def _Handler(exc):
"""Check if |exc| is a retriable exception.
Args:
exc: An exception.
Returns:
True if exc is an errors.HttpError and code exists in |retry_http_codes|
False otherwise.
"""
if self._ShouldRetry(exc, retry_http_codes,
other_retriable_errors):
logger.debug("Will retry error: %s", str(exc))
return True
return False
return utils.Retry(
_Handler,
max_retries=max_retry,
functor=self.ExecuteOnce,
sleep_multiplier=sleep,
retry_backoff_factor=backoff_factor,
api=api)
def BatchExecuteOnce(self, requests):
"""Execute requests in a batch.
Args:
requests: A dictionary where key is request id and value
is an http request.
Returns:
results, a dictionary in the following format
{request_id: (response, exception)}
request_ids are those from requests; response
is the http response for the request or None on error;
exception is an instance of DriverError or None if no error.
"""
results = {}
def _CallBack(request_id, response, exception):
results[request_id] = (response, self._TranslateError(exception))
batch = apiclient.http.BatchHttpRequest()
for request_id, request in requests.iteritems():
batch.add(
request=request, callback=_CallBack, request_id=request_id)
batch.execute()
return results
def BatchExecute(self,
requests,
retry_http_codes=None,
max_retry=None,
sleep=None,
backoff_factor=None,
other_retriable_errors=None):
"""Batch execute multiple requests with retry.
Call BatchExecuteOnce and retry on http error with given codes.
Args:
requests: A dictionary where key is request id picked by caller,
and value is a apiclient.http.HttpRequest.
retry_http_codes: A list of http codes to retry.
max_retry: See utils.Retry.
sleep: See utils.Retry.
backoff_factor: See utils.Retry.
other_retriable_errors: A tuple of error types that should be retried
other than errors.HttpError.
Returns:
results, a dictionary in the following format
{request_id: (response, exception)}
request_ids are those from requests; response
is the http response for the request or None on error;
exception is an instance of DriverError or None if no error.
"""
executor = utils.BatchHttpRequestExecutor(
self.BatchExecuteOnce,
requests=requests,
retry_http_codes=retry_http_codes or self.RETRY_HTTP_CODES,
max_retry=max_retry or self.RETRY_COUNT,
sleep=sleep or self.RETRY_SLEEP_MULTIPLIER,
backoff_factor=backoff_factor or self.RETRY_BACKOFF_FACTOR,
other_retriable_errors=other_retriable_errors
or self.RETRIABLE_ERRORS)
executor.Execute()
return executor.GetResults()
def ListWithMultiPages(self, api_resource, *args, **kwargs):
"""Call an api that list a type of resource.
Multiple google services support listing a type of
resource (e.g list gce instances, list storage objects).
The querying pattern is similar --
Step 1: execute the api and get a response object like,
{
"items": [..list of resource..],
# The continuation token that can be used
# to get the next page.
"nextPageToken": "A String",
}
Step 2: execute the api again with the nextPageToken to
retrieve more pages and get a response object.
Step 3: Repeat Step 2 until no more page.
This method encapsulates the generic logic of
calling such listing api.
Args:
api_resource: An apiclient.discovery.Resource object
used to create an http request for the listing api.
*args: Arguments used to create the http request.
**kwargs: Keyword based arguments to create the http
request.
Returns:
A list of items.
"""
items = []
next_page_token = None
while True:
api = api_resource(pageToken=next_page_token, *args, **kwargs)
response = self.Execute(api)
items.extend(response.get("items", []))
next_page_token = response.get("nextPageToken")
if not next_page_token:
break
return items
@property
def service(self):
"""Return self._service as a property."""
return self._service