#!/usr/bin/env python
# Copyright (C) 2018 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This tool translates a collection of BUILD.gn files into a mostly equivalent
# BUILD file for the Bazel build system. The input to the tool is a
# JSON description of the GN build definition generated with the following
# command:
#
# gn desc out --format=json --all-toolchains "//*" > desc.json
#
# The tool is then given a list of GN labels for which to generate Bazel
# build rules.
from __future__ import print_function
import argparse
import errno
import functools
import json
import os
import re
import shutil
import subprocess
import sys
import textwrap
# Copyright header for generated code.
header = """# Copyright (C) 2019 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This file is automatically generated by {}. Do not edit.
""".format(__file__)
# Arguments for the GN output directory.
# host_os="linux" is to generate the right build files from Mac OS.
gn_args = 'target_os="linux" is_debug=false host_os="linux"'
# Default targets to translate to the blueprint file.
default_targets = [
'//src/protozero:libprotozero',
'//src/trace_processor:trace_processor',
'//src/trace_processor:trace_processor_shell_host(//gn/standalone/toolchain:gcc_like_host)',
'//tools/trace_to_text:trace_to_text_host(//gn/standalone/toolchain:gcc_like_host)',
'//protos/perfetto/config:merged_config_gen',
'//protos/perfetto/trace:merged_trace_gen',
]
# Aliases to add to the BUILD file
alias_targets = {
'//src/protozero:libprotozero': 'libprotozero',
'//src/trace_processor:trace_processor': 'trace_processor',
'//src/trace_processor:trace_processor_shell_host': 'trace_processor_shell',
'//tools/trace_to_text:trace_to_text_host': 'trace_to_text',
}
def enable_sqlite(module):
module.deps.add(Label('//third_party/sqlite'))
module.deps.add(Label('//third_party/sqlite:sqlite_ext_percentile'))
def enable_jsoncpp(module):
module.deps.add(Label('//third_party/perfetto/google:jsoncpp'))
def enable_linenoise(module):
module.deps.add(Label('//third_party/perfetto/google:linenoise'))
def enable_gtest_prod(module):
module.deps.add(Label('//third_party/perfetto/google:gtest_prod'))
def enable_protobuf_full(module):
module.deps.add(Label('//third_party/protobuf:libprotoc'))
module.deps.add(Label('//third_party/protobuf'))
def enable_perfetto_version(module):
module.deps.add(Label('//third_party/perfetto/google:perfetto_version'))
def disable_module(module):
pass
# Internal equivalents for third-party libraries that the upstream project
# depends on.
builtin_deps = {
'//gn:jsoncpp_deps': enable_jsoncpp,
'//buildtools:linenoise': enable_linenoise,
'//buildtools:protobuf_lite': disable_module,
'//buildtools:protobuf_full': enable_protobuf_full,
'//buildtools:protoc': disable_module,
'//buildtools:sqlite': enable_sqlite,
'//gn:default_deps': disable_module,
'//gn:gtest_prod_config': enable_gtest_prod,
'//gn:protoc_lib_deps': enable_protobuf_full,
'//gn/standalone:gen_git_revision': enable_perfetto_version,
}
# ----------------------------------------------------------------------------
# End of configuration.
# ----------------------------------------------------------------------------
def check_output(cmd, cwd):
try:
output = subprocess.check_output(
cmd, stderr=subprocess.STDOUT, cwd=cwd)
except subprocess.CalledProcessError as e:
print('Cmd "{}" failed in {}:'.format(
' '.join(cmd), cwd), file=sys.stderr)
print(e.output)
exit(1)
else:
return output
class Error(Exception):
pass
def repo_root():
"""Returns an absolute path to the repository root."""
return os.path.join(
os.path.realpath(os.path.dirname(__file__)), os.path.pardir)
def create_build_description(repo_root):
"""Creates the JSON build description by running GN."""
out = os.path.join(repo_root, 'out', 'tmp.gen_build')
try:
try:
os.makedirs(out)
except OSError as e:
if e.errno != errno.EEXIST:
raise
check_output(
['gn', 'gen', out, '--args=%s' % gn_args], repo_root)
desc = check_output(
['gn', 'desc', out, '--format=json', '--all-toolchains', '//*'],
repo_root)
return json.loads(desc)
finally:
shutil.rmtree(out)
def label_to_path(label):
"""Turn a GN output label (e.g., //some_dir/file.cc) into a path."""
assert label.startswith('//')
return label[2:]
def label_to_target_name_with_path(label):
"""
Turn a GN label into a target name involving the full path.
e.g., //src/perfetto:tests -> src_perfetto_tests
"""
name = re.sub(r'^//:?', '', label)
name = re.sub(r'[^a-zA-Z0-9_]', '_', name)
return name
def label_without_toolchain(label):
"""Strips the toolchain from a GN label.
Return a GN label (e.g //buildtools:protobuf(//gn/standalone/toolchain:
gcc_like_host) without the parenthesised toolchain part.
"""
return label.split('(')[0]
def is_public_header(label):
"""
Returns if this is a c++ header file that is part of the API.
Args:
label: Label to evaluate
"""
return label.endswith('.h') and label.startswith('//include/perfetto/')
@functools.total_ordering
class Label(object):
"""Represents a label in BUILD file terminology. This class wraps a string
label to allow for correct comparision of labels for sorting.
Args:
label: The string rerepsentation of the label.
"""
def __init__(self, label):
self.label = label
def is_absolute(self):
return self.label.startswith('//')
def dirname(self):
return self.label.split(':')[0] if ':' in self.label else self.label
def basename(self):
return self.label.split(':')[1] if ':' in self.label else ''
def __eq__(self, other):
return self.label == other.label
def __lt__(self, other):
return (
self.is_absolute(),
self.dirname(),
self.basename()
) < (
other.is_absolute(),
other.dirname(),
other.basename()
)
def __str__(self):
return self.label
def __hash__(self):
return hash(self.label)
class Writer(object):
def __init__(self, output, width=79):
self.output = output
self.width = width
def comment(self, text):
for line in textwrap.wrap(text,
self.width - 2,
break_long_words=False,
break_on_hyphens=False):
self.output.write('# {}\n'.format(line))
def newline(self):
self.output.write('\n')
def line(self, s, indent=0):
self.output.write(' ' * indent + s + '\n')
def variable(self, key, value, sort=True):
if value is None:
return
if isinstance(value, set) or isinstance(value, list):
if len(value) == 0:
return
self.line('{} = ['.format(key), indent=1)
for v in sorted(list(value)) if sort else value:
self.line('"{}",'.format(v), indent=2)
self.line('],', indent=1)
elif isinstance(value, basestring):
self.line('{} = "{}",'.format(key, value), indent=1)
else:
self.line('{} = {},'.format(key, value), indent=1)
def header(self):
self.output.write(header)
class Target(object):
"""In-memory representation of a BUILD target."""
def __init__(self, type, name, gn_name=None):
assert type in ('cc_binary', 'cc_library', 'cc_proto_library',
'proto_library', 'filegroup', 'alias',
'pbzero_cc_proto_library', 'genrule', )
self.type = type
self.name = name
self.srcs = set()
self.hdrs = set()
self.deps = set()
self.visibility = set()
self.gn_name = gn_name
self.is_pbzero = False
self.src_proto_library = None
self.outs = set()
self.cmd = None
self.tools = set()
def write(self, writer):
if self.gn_name:
writer.comment('GN target: {}'.format(self.gn_name))
writer.line('{}('.format(self.type))
writer.variable('name', self.name)
writer.variable('srcs', self.srcs)
writer.variable('hdrs', self.hdrs)
if self.type == 'proto_library' and not self.is_pbzero:
if self.srcs:
writer.variable('has_services', 1)
writer.variable('cc_api_version', 2)
if self.srcs:
writer.variable('cc_generic_services', 1)
writer.variable('src_proto_library', self.src_proto_library)
writer.variable('outs', self.outs)
writer.variable('cmd', self.cmd)
writer.variable('tools', self.tools)
# Keep visibility and deps last.
writer.variable('visibility', self.visibility)
if type != 'filegroup':
writer.variable('deps', self.deps)
writer.line(')')
class Build(object):
"""In-memory representation of a BUILD file."""
def __init__(self, public, header_lines=[]):
self.targets = {}
self.public = public
self.header_lines = header_lines
def add_target(self, target):
self.targets[target.name] = target
def write(self, writer):
writer.header()
writer.newline()
for line in self.header_lines:
writer.line(line)
if self.header_lines:
writer.newline()
if self.public:
writer.line(
'package(default_visibility = ["//visibility:public"])')
else:
writer.line(
'package(default_visibility = ["//third_party/perfetto:__subpackages__"])')
writer.newline()
writer.line('licenses(["notice"]) # Apache 2.0')
writer.newline()
writer.line('exports_files(["LICENSE"])')
writer.newline()
sorted_targets = sorted(
self.targets.itervalues(), key=lambda m: m.name)
for target in sorted_targets[:-1]:
target.write(writer)
writer.newline()
# BUILD files shouldn't have a trailing new line.
sorted_targets[-1].write(writer)
class BuildGenerator(object):
def __init__(self, desc):
self.desc = desc
self.action_generated_files = set()
for target in self.desc.itervalues():
if target['type'] == 'action':
self.action_generated_files.update(target['outputs'])
def create_build_for_targets(self, targets):
"""Generate a BUILD for a list of GN targets and aliases."""
self.build = Build(public=True)
proto_cc_import = 'load("//tools/build_defs/proto/cpp:cc_proto_library.bzl", "cc_proto_library")'
pbzero_cc_import = 'load("//third_party/perfetto/google:build_defs.bzl", "pbzero_cc_proto_library")'
self.proto_build = Build(public=False, header_lines=[
proto_cc_import, pbzero_cc_import])
for target in targets:
self.create_target(target)
return (self.build, self.proto_build)
def resolve_dependencies(self, target_name):
"""Return the set of direct dependent-on targets for a GN target.
Args:
desc: JSON GN description.
target_name: Name of target
Returns:
A set of transitive dependencies in the form of GN targets.
"""
if label_without_toolchain(target_name) in builtin_deps:
return set()
target = self.desc[target_name]
resolved_deps = set()
for dep in target.get('deps', []):
resolved_deps.add(dep)
return resolved_deps
def apply_module_sources_to_target(self, target, module_desc):
"""
Args:
target: Module to which dependencies should be added.
module_desc: JSON GN description of the module.
visibility: Whether the module is visible with respect to the target.
"""
for src in module_desc['sources']:
label = Label(label_to_path(src))
if target.type == 'cc_library' and is_public_header(src):
target.hdrs.add(label)
else:
target.srcs.add(label)
def apply_module_dependency(self, target, dep_name):
"""
Args:
build: BUILD instance which is being generated.
proto_build: BUILD instance which is being generated to hold protos.
desc: JSON GN description.
target: Module to which dependencies should be added.
dep_name: GN target of the dependency.
"""
# If the dependency refers to a library which we can replace with an internal
# equivalent, stop recursing and patch the dependency in.
dep_name_no_toolchain = label_without_toolchain(dep_name)
if dep_name_no_toolchain in builtin_deps:
builtin_deps[dep_name_no_toolchain](target)
return
dep_desc = self.desc[dep_name]
if dep_desc['type'] == 'source_set':
for inner_name in self.resolve_dependencies(dep_name):
self.apply_module_dependency(target, inner_name)
# Any source set which has a source generated by an action doesn't need
# to be depended on as we will depend on the action directly.
if any(src in self.action_generated_files for src in dep_desc['sources']):
return
self.apply_module_sources_to_target(target, dep_desc)
elif dep_desc['type'] == 'action':
args = dep_desc['args']
if "gen_merged_sql_metrics" in dep_name:
dep_target = self.create_merged_sql_metrics_target(dep_name)
target.deps.add(Label("//third_party/perfetto:" + dep_target.name))
if target.type == 'cc_library' or target.type == 'cc_binary':
target.srcs.update(dep_target.outs)
elif args[0].endswith('/protoc'):
(proto_target, cc_target) = self.create_proto_target(dep_name)
if target.type == 'proto_library':
dep_target_name = proto_target.name
else:
dep_target_name = cc_target.name
target.deps.add(
Label("//third_party/perfetto/protos:" + dep_target_name))
else:
raise Error('Unsupported action in target %s: %s' % (dep_target_name,
args))
elif dep_desc['type'] == 'static_library':
dep_target = self.create_target(dep_name)
target.deps.add(Label("//third_party/perfetto:" + dep_target.name))
elif dep_desc['type'] == 'group':
for inner_name in self.resolve_dependencies(dep_name):
self.apply_module_dependency(target, inner_name)
elif dep_desc['type'] == 'executable':
# Just create the dep target but don't add it as a dep because it's an
# executable.
self.create_target(dep_name)
else:
raise Error('Unknown target name %s with type: %s' %
(dep_name, dep_desc['type']))
def create_merged_sql_metrics_target(self, gn_target_name):
target_desc = self.desc[gn_target_name]
gn_target_name_no_toolchain = label_without_toolchain(gn_target_name)
target = Target(
'genrule',
'gen_merged_sql_metrics',
gn_name=gn_target_name_no_toolchain,
)
target.outs.update(
Label(src[src.index('gen/') + len('gen/'):])
for src in target_desc.get('outputs', [])
)
target.cmd = '$(location gen_merged_sql_metrics_py) --cpp_out=$@ $(SRCS)'
target.tools.update([
'gen_merged_sql_metrics_py',
])
target.srcs.update(
Label(label_to_path(src))
for src in target_desc.get('inputs', [])
if src not in self.action_generated_files
)
self.build.add_target(target)
return target
def create_proto_target(self, gn_target_name):
target_desc = self.desc[gn_target_name]
args = target_desc['args']
gn_target_name_no_toolchain = label_without_toolchain(gn_target_name)
stripped_path = gn_target_name_no_toolchain.replace("protos/perfetto/", "")
pretty_target_name = label_to_target_name_with_path(stripped_path)
pretty_target_name = pretty_target_name.replace("_lite_gen", "")
pretty_target_name = pretty_target_name.replace("_zero_gen", "_zero")
proto_target = Target(
'proto_library',
pretty_target_name,
gn_name=gn_target_name_no_toolchain
)
proto_target.is_pbzero = any("pbzero" in arg for arg in args)
proto_target.srcs.update([
Label(label_to_path(src).replace('protos/', ''))
for src in target_desc.get('sources', [])
])
if not proto_target.is_pbzero:
proto_target.visibility.add("//visibility:public")
self.proto_build.add_target(proto_target)
for dep_name in self.resolve_dependencies(gn_target_name):
self.apply_module_dependency(proto_target, dep_name)
if proto_target.is_pbzero:
# Remove all the protozero srcs from the proto_library.
proto_target.srcs.difference_update(
[src for src in proto_target.srcs if not src.label.endswith('.proto')])
# Remove all the non-proto deps from the proto_library and add to the cc
# library.
cc_deps = [
dep for dep in proto_target.deps
if not dep.label.startswith('//third_party/perfetto/protos')
]
proto_target.deps.difference_update(cc_deps)
cc_target_name = proto_target.name + "_cc_proto"
cc_target = Target('pbzero_cc_proto_library', cc_target_name,
gn_name=gn_target_name_no_toolchain)
cc_target.deps.add(Label('//third_party/perfetto:libprotozero'))
cc_target.deps.update(cc_deps)
# Add the proto_library to the cc_target.
cc_target.src_proto_library = \
"//third_party/perfetto/protos:" + proto_target.name
self.proto_build.add_target(cc_target)
else:
cc_target_name = proto_target.name + "_cc_proto"
cc_target = Target('cc_proto_library',
cc_target_name, gn_name=gn_target_name_no_toolchain)
cc_target.visibility.add("//visibility:public")
cc_target.deps.add(
Label("//third_party/perfetto/protos:" + proto_target.name))
self.proto_build.add_target(cc_target)
return (proto_target, cc_target)
def create_target(self, gn_target_name):
"""Generate module(s) for a given GN target.
Given a GN target name, generate one or more corresponding modules into a
build file.
Args:
build: Build instance which is being generated.
desc: JSON GN description.
gn_target_name: GN target name for module generation.
"""
target_desc = self.desc[gn_target_name]
if target_desc['type'] == 'action':
args = target_desc['args']
if args[0].endswith('/protoc'):
return self.create_proto_target(gn_target_name)
else:
raise Error('Unsupported action in target %s: %s' % (gn_target_name,
args))
elif target_desc['type'] == 'executable':
target_type = 'cc_binary'
elif target_desc['type'] == 'static_library':
target_type = 'cc_library'
elif target_desc['type'] == 'source_set':
target_type = 'filegroup'
else:
raise Error('Unknown target type: %s' % target_desc['type'])
label_no_toolchain = label_without_toolchain(gn_target_name)
target_name_path = label_to_target_name_with_path(label_no_toolchain)
target_name = alias_targets.get(label_no_toolchain, target_name_path)
target = Target(target_type, target_name, gn_name=label_no_toolchain)
target.srcs.update(
Label(label_to_path(src))
for src in target_desc.get('sources', [])
if src not in self.action_generated_files
)
for dep_name in self.resolve_dependencies(gn_target_name):
self.apply_module_dependency(target, dep_name)
self.build.add_target(target)
return target
def main():
parser = argparse.ArgumentParser(
description='Generate BUILD from a GN description.')
parser.add_argument(
'--desc',
help='GN description (e.g., gn desc out --format=json --all-toolchains "//*"'
)
parser.add_argument(
'--repo-root',
help='Standalone Perfetto repository to generate a GN description',
default=repo_root(),
)
parser.add_argument(
'--extras',
help='Extra targets to include at the end of the BUILD file',
default=os.path.join(repo_root(), 'BUILD.extras'),
)
parser.add_argument(
'--output',
help='BUILD file to create',
default=os.path.join(repo_root(), 'BUILD'),
)
parser.add_argument(
'--output-proto',
help='Proto BUILD file to create',
default=os.path.join(repo_root(), 'protos', 'BUILD'),
)
parser.add_argument(
'targets',
nargs=argparse.REMAINDER,
help='Targets to include in the BUILD file (e.g., "//:perfetto_tests")')
args = parser.parse_args()
if args.desc:
with open(args.desc) as f:
desc = json.load(f)
else:
desc = create_build_description(args.repo_root)
build_generator = BuildGenerator(desc)
build, proto_build = build_generator.create_build_for_targets(
args.targets or default_targets)
with open(args.output, 'w') as f:
writer = Writer(f)
build.write(writer)
writer.newline()
with open(args.extras, 'r') as r:
for line in r:
writer.line(line.rstrip("\n\r"))
with open(args.output_proto, 'w') as f:
proto_build.write(Writer(f))
return 0
if __name__ == '__main__':
sys.exit(main())