// Copyright 2017 Google Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package main import ( "bytes" "context" "flag" "fmt" "io/ioutil" "os" "path/filepath" "runtime" "strings" "sync" "android/soong/ui/build" "android/soong/ui/logger" "android/soong/ui/tracer" ) // We default to number of cpus / 4, which seems to be the sweet spot for my // system. I suspect this is mostly due to memory or disk bandwidth though, and // may depend on the size ofthe source tree, so this probably isn't a great // default. func detectNumJobs() int { if runtime.NumCPU() < 4 { return 1 } return runtime.NumCPU() / 4 } var numJobs = flag.Int("j", detectNumJobs(), "number of parallel kati jobs") var keep = flag.Bool("keep", false, "keep successful output files") var outDir = flag.String("out", "", "path to store output directories (defaults to tmpdir under $OUT when empty)") var onlyConfig = flag.Bool("only-config", false, "Only run product config (not Soong or Kati)") var onlySoong = flag.Bool("only-soong", false, "Only run product config and Soong (not Kati)") type Product struct { ctx build.Context config build.Config } func main() { log := logger.New(os.Stderr) defer log.Cleanup() flag.Parse() ctx, cancel := context.WithCancel(context.Background()) defer cancel() trace := tracer.New(log) defer trace.Close() build.SetupSignals(log, cancel, func() { trace.Close() log.Cleanup() }) buildCtx := build.Context{&build.ContextImpl{ Context: ctx, Logger: log, Tracer: trace, StdioInterface: build.StdioImpl{}, }} failed := false config := build.NewConfig(buildCtx) if *outDir == "" { var err error *outDir, err = ioutil.TempDir(config.OutDir(), "multiproduct") if err != nil { log.Fatalf("Failed to create tempdir: %v", err) } if !*keep { defer func() { if !failed { os.RemoveAll(*outDir) } }() } } config.Environment().Set("OUT_DIR", *outDir) log.Println("Output directory:", *outDir) build.SetupOutDir(buildCtx, config) log.SetOutput(filepath.Join(config.OutDir(), "soong.log")) trace.SetOutput(filepath.Join(config.OutDir(), "build.trace")) vars, err := build.DumpMakeVars(buildCtx, config, nil, nil, []string{"all_named_products"}) if err != nil { log.Fatal(err) } products := strings.Fields(vars["all_named_products"]) log.Verbose("Got product list:", products) var wg sync.WaitGroup errs := make(chan error, len(products)) productConfigs := make(chan Product, len(products)) // Run the product config for every product in parallel for _, product := range products { wg.Add(1) go func(product string) { defer wg.Done() defer logger.Recover(func(err error) { errs <- fmt.Errorf("Error building %s: %v", product, err) }) productOutDir := filepath.Join(config.OutDir(), product) if err := os.MkdirAll(productOutDir, 0777); err != nil { log.Fatalf("Error creating out directory: %v", err) } f, err := os.Create(filepath.Join(productOutDir, "std.log")) if err != nil { log.Fatalf("Error creating std.log: %v", err) } productLog := logger.New(&bytes.Buffer{}) productLog.SetOutput(filepath.Join(productOutDir, "soong.log")) productCtx := build.Context{&build.ContextImpl{ Context: ctx, Logger: productLog, Tracer: trace, StdioInterface: build.NewCustomStdio(nil, f, f), Thread: trace.NewThread(product), }} productConfig := build.NewConfig(productCtx) productConfig.Environment().Set("OUT_DIR", productOutDir) productConfig.Lunch(productCtx, product, "eng") build.Build(productCtx, productConfig, build.BuildProductConfig) productConfigs <- Product{productCtx, productConfig} }(product) } go func() { defer close(productConfigs) wg.Wait() }() var wg2 sync.WaitGroup // Then run up to numJobs worth of Soong and Kati for i := 0; i < *numJobs; i++ { wg2.Add(1) go func() { defer wg2.Done() for product := range productConfigs { func() { defer logger.Recover(func(err error) { errs <- fmt.Errorf("Error building %s: %v", product.config.TargetProduct(), err) }) buildWhat := 0 if !*onlyConfig { buildWhat |= build.BuildSoong if !*onlySoong { buildWhat |= build.BuildKati } } build.Build(product.ctx, product.config, buildWhat) if !*keep { os.RemoveAll(product.config.OutDir()) } log.Println("Finished running for", product.config.TargetProduct()) }() } }() } go func() { wg2.Wait() close(errs) }() for err := range errs { failed = true log.Print(err) } if failed { log.Fatalln("Failed") } }