#!/usr/bin/python
#
# Copyright (C) 2012 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Merge master-chromium to master within the Android tree."""
import logging
import optparse
import os
import re
import shutil
import subprocess
import sys
import merge_common
AUTOGEN_MESSAGE = 'This commit was generated by merge_to_master.py.'
WEBVIEW_PROJECT = 'frameworks/webview'
def _GetAbsPath(project):
"""Returns the full path to a given project (either Chromium or Android)."""
if project in merge_common.ALL_PROJECTS:
abs_path = os.path.join(merge_common.REPOSITORY_ROOT, project)
else:
abs_path = os.path.join(os.environ['ANDROID_BUILD_TOP'], project)
if not os.path.exists(abs_path):
raise merge_common.MergeError('Cannot find path ' + abs_path)
return abs_path
def _CheckoutSingleProject(project, target_branch):
"""Checks out the tip of the target_branch into a local branch (merge-to-XXX).
Args:
project: a Chromium project (., third_party/foo) or frameworks/webview.
target_branch: name of the target branch (in the goog remote).
"""
dest_dir = _GetAbsPath(project)
tracking_branch = 'goog/' + target_branch
logging.debug('Check out %-45s at %-16s', project, tracking_branch)
merge_common.GetCommandStdout(['git', 'remote', 'update', 'goog'],
cwd=dest_dir)
merge_common.GetCommandStdout(['git', 'checkout',
'-b', 'merge-to-' + target_branch,
'-t', tracking_branch], cwd=dest_dir)
def _FetchSingleProject(project, remote, remote_ref):
"""Fetches a remote ref for the given project and returns the fetched SHA.
Args:
project: a Chromium project (., third_party/foo) or frameworks/webview.
remote: Git remote name (goog for most projects, history for squashed ones).
remote_ref: the remote ref to fetch (e.g., refs/archive/chromium-XXX).
Returns:
The SHA1 of the FETCH_HEAD.
"""
dest_dir = _GetAbsPath(project)
logging.debug('Fetch %-45s %s:%s', project, remote, remote_ref)
merge_common.GetCommandStdout(['git', 'fetch', remote, remote_ref],
cwd=dest_dir)
return merge_common.GetCommandStdout(['git', 'rev-parse', 'FETCH_HEAD'],
cwd=dest_dir).strip()
def _MergeSingleProject(project, merge_sha, revision, target_branch, flatten):
"""Merges a single project at a given SHA.
Args:
project: a Chromium project (., third_party/foo) or frameworks/webview.
merge_sha: the SHA to merge.
revision: Abbrev. commitish in the main Chromium repository.
target_branch: name of the target branch.
flatten: True: squash history while merging; False: perform a normal merge.
"""
dest_dir = _GetAbsPath(project)
if flatten:
# Make the previous merges into grafts so we can do a correct merge.
old_sha = merge_common.GetCommandStdout(['git', 'rev-parse', 'HEAD'],
cwd=dest_dir).strip()
merge_log = os.path.join(dest_dir, '.merged-revisions')
if os.path.exists(merge_log):
shutil.copyfile(merge_log,
os.path.join(dest_dir, '.git', 'info', 'grafts'))
# Early out if there is nothing to merge.
if not merge_common.GetCommandStdout(['git', 'rev-list', '-1',
'HEAD..' + merge_sha], cwd=dest_dir):
logging.debug('No new commits to merge in project %s', project)
return
logging.debug('Merging project %s (flatten: %s)...', project, flatten)
merge_cmd = ['git', 'merge', '--no-commit']
merge_cmd += ['--squash'] if flatten else ['--no-ff']
merge_cmd += [merge_sha]
# Merge conflicts cause 'git merge' to return 1, so ignore errors
merge_common.GetCommandStdout(merge_cmd, cwd=dest_dir, ignore_errors=True)
if flatten:
dirs_to_prune = merge_common.PRUNE_WHEN_FLATTENING.get(project, [])
if dirs_to_prune:
merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch', '-rf'] +
dirs_to_prune, cwd=dest_dir)
if project in merge_common.ALL_PROJECTS:
commit_msg = 'Merge from Chromium at DEPS revision %s' % revision
else:
commit_msg = 'Merge master-chromium into %s at %s' % (target_branch,
revision)
commit_msg += '\n\n' + AUTOGEN_MESSAGE
merge_common.CheckNoConflictsAndCommitMerge(commit_msg, cwd=dest_dir)
if flatten:
# Generate the new grafts file and commit it on top of the merge.
new_sha = merge_common.GetCommandStdout(['git', 'rev-parse', 'HEAD'],
cwd=dest_dir).strip()
with open(merge_log, 'a+') as f:
f.write('%s %s %s\n' % (new_sha, old_sha, merge_sha))
merge_common.GetCommandStdout(['git', 'add', '.merged-revisions'],
cwd=dest_dir)
merge_common.GetCommandStdout(
['git', 'commit', '-m',
'Record Chromium merge at DEPS revision %s\n\n%s' %
(revision, AUTOGEN_MESSAGE)], cwd=dest_dir)
def _IsAncestor(ref1, ref2, cwd):
"""Checks whether ref1 is a ancestor of ref2 in the given Git repo."""
cmd = ['git', 'merge-base', '--is-ancestor', ref1, ref2]
ret = subprocess.call(cmd, cwd=cwd)
if ret == 0:
return True
elif ret == 1:
return False
else:
raise merge_common.CommandError(ret, ' '.join(cmd), cwd, 'N/A', 'N/A')
def _MergeChromiumProjects(revision, target_branch, repo_shas=None,
force=False):
"""Merges the Chromium projects from master-chromium to target_branch.
The larger projects' histories are flattened in the process.
When repo_shas != None, it checks that the SHAs of the projects in the
archive match exactly the SHAs of the projects in repo.prop.
Args:
revision: Abbrev. commitish in the main Chromium repository.
target_branch: target branch name to merge and push to.
repo_shas: optional dict. of expected revisions (only for --repo-prop).
force: True: merge anyways using the SHAs from repo.prop; False: bail out if
projects mismatch (archive vs repo.prop).
"""
# Sync and checkout ToT for all projects (creating the merge-to-XXX branch)
# and fetch the archive snapshot.
fetched_shas = {}
remote_ref = 'refs/archive/chromium-%s' % revision
for project in merge_common.PROJECTS_WITH_FLAT_HISTORY:
_CheckoutSingleProject(project, target_branch)
fetched_shas[project] = _FetchSingleProject(project, 'history', remote_ref)
for project in merge_common.PROJECTS_WITH_FULL_HISTORY:
_CheckoutSingleProject(project, target_branch)
fetched_shas[project] = _FetchSingleProject(project, 'goog', remote_ref)
if repo_shas:
project_shas_mismatch = False
for project, merge_sha in fetched_shas.items(): # the dict can be modified.
expected_sha = repo_shas.get(project)
if expected_sha != merge_sha:
logging.warn('The SHA for project %s specified in the repo.prop (%s) '
'and the one in the archive (%s) differ.',
project, expected_sha, merge_sha)
dest_dir = _GetAbsPath(project)
if expected_sha is None:
reason = 'cannot find a SHA in the repo.pro for %s' % project
elif _IsAncestor(merge_sha, expected_sha, cwd=dest_dir):
reason = 'the SHA in repo.prop is ahead of the SHA in the archive. '
log_cmd = ['git', 'log', '--oneline', '--graph', '--max-count=10',
'%s..%s' % (merge_sha, expected_sha)]
log_cmd_output = merge_common.GetCommandStdout(log_cmd, cwd=dest_dir)
reason += 'showing partial log (%s): \n %s' % (' '.join(log_cmd),
log_cmd_output)
elif _IsAncestor(expected_sha, merge_sha, cwd=dest_dir):
reason = 'The SHA is already merged in the archive'
else:
reason = 'The project history diverged. Consult your Git historian.'
project_shas_mismatch = True
if force:
logging.debug('Merging the SHA in repo.prop anyways (due to --force)')
fetched_shas[project] = expected_sha
else:
logging.debug('Reason: %s', reason)
if not force and project_shas_mismatch:
raise merge_common.MergeError(
'The revision of some projects in the archive is different from the '
'one provided in build.prop. See the log for more details. Re-run '
'with --force to continue.')
for project in merge_common.PROJECTS_WITH_FLAT_HISTORY:
_MergeSingleProject(project, fetched_shas[project], revision, target_branch,
flatten=True)
for project in merge_common.PROJECTS_WITH_FULL_HISTORY:
_MergeSingleProject(project, fetched_shas[project], revision, target_branch,
flatten=False)
def _GetNearestUpstreamAbbrevSHA(reference='history/master-chromium'):
"""Returns the abbrev. upstream SHA which closest to the given reference."""
logging.debug('Getting upstream SHA for %s...', reference)
merge_common.GetCommandStdout(['git', 'remote', 'update', 'history'])
upstream_commit = merge_common.Abbrev(merge_common.GetCommandStdout([
'git', 'merge-base', 'history/upstream-master', reference]))
# Pedantic check: look for the existence of a merge commit which contains the
# |upstream_commit| in its message and is its children.
merge_parents = merge_common.GetCommandStdout([
'git', 'rev-list', reference, '--grep', upstream_commit, '--merges',
'--parents', '-1'])
if upstream_commit not in merge_parents:
raise merge_common.MergeError(
'Found upstream commit %s, but the merge child (%s) could not be found '
'or is not a parent of the upstream SHA')
logging.debug('Found nearest Chromium revision %s', upstream_commit)
return upstream_commit
def _MergeWithRepoProp(repo_prop_file, target_branch, force):
"""Performs a merge using a repo.prop file (from Android build waterfall).
This does NOT merge (unless forced with force=True) the pinned
revisions in repo.prop, as a repo.prop can snapshot an intermediate state
(between two automerger cycles). Instead, this looks up the archived snapshot
(generated by the chromium->master-chromium auto-merger) which is closest to
the given repo.prop (following the main Chromium project) and merges that one.
If the projects revisions don't match, it fails with detailed error messages.
Args:
repo_prop_file: Path to a downloaded repo.prop file.
target_branch: name of the target branch to merget to.
force: ignores the aforementioned check and merged anyways.
"""
chromium_sha = None
webview_sha = None
repo_shas = {} # 'project/path' -> 'sha'
with open(repo_prop_file) as prop:
for line in prop:
repo, sha = line.split()
# Translate the Android repo paths into the relative project paths used in
# merge_common (e.g., platform/external/chromium_org/foo -> foo).
m = (
re.match(r'^platform/(frameworks/.+)$', repo) or
re.match(r'^platform/external/chromium_org/?(.*?)(-history)?$', repo))
if m:
project = m.group(1) if m.group(1) else '.' # '.' = Main project.
repo_shas[project] = sha
chromium_sha = repo_shas.get('.')
webview_sha = repo_shas.get(WEBVIEW_PROJECT)
if not chromium_sha or not webview_sha:
raise merge_common.MergeError('SHAs for projects not found; '
'invalid build.prop?')
# Check that the revisions in repo.prop and the on in the archive match.
archived_chromium_revision = _GetNearestUpstreamAbbrevSHA(chromium_sha)
logging.info('Merging Chromium at %s and WebView at %s',
archived_chromium_revision, webview_sha)
_MergeChromiumProjects(archived_chromium_revision, target_branch, repo_shas,
force)
_CheckoutSingleProject(WEBVIEW_PROJECT, target_branch)
_MergeSingleProject(WEBVIEW_PROJECT, webview_sha,
archived_chromium_revision, target_branch, flatten=False)
def Push(target_branch):
"""Push the finished snapshot to the Android repository.
Creates first a CL for frameworks/webview (if the merge-to-XXX branch exists)
then wait for user confirmation and pushes the Chromium merges. This is to
give an opportunity to get a +2 for frameworks/webview and then push both
frameworks/webview and the Chromium projects atomically(ish).
Args:
target_branch: name of the target branch (in the goog remote).
"""
merge_branch = 'merge-to-%s' % target_branch
# Create a Gerrit CL for the frameworks/webview project (if needed).
dest_dir = _GetAbsPath(WEBVIEW_PROJECT)
did_upload_webview_cl = False
if merge_common.GetCommandStdout(['git', 'branch', '--list', merge_branch],
cwd=dest_dir):
# Check that there was actually something to merge.
merge_range = 'goog/%s..%s' % (target_branch, merge_branch)
if merge_common.GetCommandStdout(['git', 'rev-list', '-1', merge_range],
cwd=dest_dir):
logging.info('Uploading a merge CL for %s...', WEBVIEW_PROJECT)
refspec = '%s:refs/for/%s' % (merge_branch, target_branch)
upload = merge_common.GetCommandStdout(['git', 'push', 'goog', refspec],
cwd=dest_dir)
logging.info(upload)
did_upload_webview_cl = True
prompt_msg = 'About push the Chromium projects merge. '
if not did_upload_webview_cl:
logging.info('No merge CL needed for %s.', WEBVIEW_PROJECT)
else:
prompt_msg += ('At this point you should have the CL +2-ed and merge it '
'together with this push.')
prompt_msg += '\nPress "y" to continue: '
if raw_input(prompt_msg) != 'y':
logging.warn('Push aborted by the user!')
return
logging.debug('Pushing Chromium projects to %s ...', target_branch)
refspec = '%s:%s' % (merge_branch, target_branch)
for path in merge_common.ALL_PROJECTS:
logging.debug('Pushing %s', path)
dest_dir = _GetAbsPath(path)
# Delete the graft before pushing otherwise git will attempt to push all the
# grafted-in objects to the server as well as the ones we want.
graftfile = os.path.join(dest_dir, '.git', 'info', 'grafts')
if os.path.exists(graftfile):
os.remove(graftfile)
merge_common.GetCommandStdout(['git', 'push', 'goog', refspec],
cwd=dest_dir)
def main():
parser = optparse.OptionParser(usage='%prog [options]')
parser.epilog = ('Takes the current master-chromium branch of the Chromium '
'projects in Android and merges them into master to publish '
'them.')
parser.add_option(
'', '--revision',
default=None,
help=('Merge to the specified archived master-chromium revision (abbrev. '
'SHA or release version) rather than using HEAD. e.g., '
'--revision=a1b2c3d4e5f6 or --revision=38.0.2125.24'))
parser.add_option(
'', '--repo-prop',
default=None, metavar='FILE',
help=('Merge to the revisions specified in this repo.prop file.'))
parser.add_option(
'', '--force',
default=False, action='store_true',
help=('Skip history checks and merged anyways (only for --repo-prop).'))
parser.add_option(
'', '--push',
default=False, action='store_true',
help=('Push the result of a previous merge to the server.'))
parser.add_option(
'', '--target',
default='master', metavar='BRANCH',
help=('Target branch to push to. Defaults to master.'))
(options, args) = parser.parse_args()
if args:
parser.print_help()
return 1
logging.basicConfig(format='%(message)s', level=logging.DEBUG,
stream=sys.stdout)
if options.push:
Push(options.target)
elif options.repo_prop:
_MergeWithRepoProp(os.path.expanduser(options.repo_prop),
options.target, options.force)
elif options.revision:
_MergeChromiumProjects(options.revision, options.target)
else:
first_upstream_sha = _GetNearestUpstreamAbbrevSHA()
_MergeChromiumProjects(first_upstream_sha, options.target)
return 0
if __name__ == '__main__':
sys.exit(main())