// Copyright 2015 Google Inc. All rights reserved
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package kati
import (
"fmt"
"path/filepath"
"sort"
"strings"
"github.com/golang/glog"
)
// DepNode represents a makefile rule for an output.
type DepNode struct {
Output string
Cmds []string
Deps []*DepNode
OrderOnlys []*DepNode
Parents []*DepNode
HasRule bool
IsPhony bool
ActualInputs []string
TargetSpecificVars Vars
Filename string
Lineno int
}
func (n *DepNode) String() string {
return fmt.Sprintf("Dep{output=%s cmds=%d deps=%d orders=%d hasRule=%t phony=%t filename=%s lineno=%d}",
n.Output, len(n.Cmds), len(n.Deps), len(n.OrderOnlys), n.HasRule, n.IsPhony, n.Filename, n.Lineno)
}
type depBuilder struct {
rules map[string]*rule
ruleVars map[string]Vars
implicitRules *ruleTrie
suffixRules map[string][]*rule
firstRule *rule
vars Vars
ev *Evaluator
vpaths searchPaths
done map[string]*DepNode
phony map[string]bool
trace []string
nodeCnt int
pickExplicitRuleCnt int
pickImplicitRuleCnt int
pickSuffixRuleCnt int
pickExplicitRuleWithoutCmdCnt int
}
type ruleTrieEntry struct {
rule *rule
suffix string
}
type ruleTrie struct {
rules []ruleTrieEntry
children map[byte]*ruleTrie
}
func newRuleTrie() *ruleTrie {
return &ruleTrie{
children: make(map[byte]*ruleTrie),
}
}
func (rt *ruleTrie) add(name string, r *rule) {
glog.V(1).Infof("rule trie: add %q %v %s", name, r.outputPatterns[0], r)
if name == "" || name[0] == '%' {
glog.V(1).Infof("rule trie: add entry %q %v %s", name, r.outputPatterns[0], r)
rt.rules = append(rt.rules, ruleTrieEntry{
rule: r,
suffix: name,
})
return
}
c, found := rt.children[name[0]]
if !found {
c = newRuleTrie()
rt.children[name[0]] = c
}
c.add(name[1:], r)
}
func (rt *ruleTrie) lookup(name string) []*rule {
glog.V(1).Infof("rule trie: lookup %q", name)
if rt == nil {
return nil
}
var rules []*rule
for _, entry := range rt.rules {
if (entry.suffix == "" && name == "") || strings.HasSuffix(name, entry.suffix[1:]) {
rules = append(rules, entry.rule)
}
}
if name == "" {
return rules
}
rules = append(rules, rt.children[name[0]].lookup(name[1:])...)
glog.V(1).Infof("rule trie: lookup %q => %v", name, rules)
return rules
}
func (rt *ruleTrie) size() int {
if rt == nil {
return 0
}
size := len(rt.rules)
for _, c := range rt.children {
size += c.size()
}
return size
}
func replaceSuffix(s string, newsuf string) string {
// TODO: Factor out the logic around suffix rules and use
// it from substitution references.
// http://www.gnu.org/software/make/manual/make.html#Substitution-Refs
return fmt.Sprintf("%s.%s", stripExt(s), newsuf)
}
func (db *depBuilder) exists(target string) bool {
_, present := db.rules[target]
if present {
return true
}
if db.phony[target] {
return true
}
_, ok := db.vpaths.exists(target)
return ok
}
func (db *depBuilder) canPickImplicitRule(r *rule, output string) bool {
outputPattern := r.outputPatterns[0]
if !outputPattern.match(output) {
return false
}
for _, input := range r.inputs {
input = outputPattern.subst(input, output)
if !db.exists(input) {
return false
}
}
return true
}
func (db *depBuilder) mergeImplicitRuleVars(outputs []string, vars Vars) Vars {
if len(outputs) != 1 {
// TODO(ukai): should return error?
panic(fmt.Sprintf("FIXME: Implicit rule should have only one output but %q", outputs))
}
glog.V(1).Infof("merge? %q", db.ruleVars)
glog.V(1).Infof("merge? %q", outputs[0])
ivars, present := db.ruleVars[outputs[0]]
if !present {
return vars
}
if vars == nil {
return ivars
}
glog.V(1).Info("merge!")
v := make(Vars)
v.Merge(ivars)
v.Merge(vars)
return v
}
func (db *depBuilder) pickRule(output string) (*rule, Vars, bool) {
r, present := db.rules[output]
vars := db.ruleVars[output]
if present {
db.pickExplicitRuleCnt++
if len(r.cmds) > 0 {
return r, vars, true
}
// If none of the explicit rules for a target has commands,
// then `make' searches for an applicable implicit rule to
// find some commands.
db.pickExplicitRuleWithoutCmdCnt++
}
irules := db.implicitRules.lookup(output)
for i := len(irules) - 1; i >= 0; i-- {
irule := irules[i]
if !db.canPickImplicitRule(irule, output) {
glog.Infof("ignore implicit rule %q %s", output, irule)
continue
}
glog.Infof("pick implicit rule %q => %q %s", output, irule.outputPatterns, irule)
db.pickImplicitRuleCnt++
if r != nil {
ir := &rule{}
*ir = *r
ir.outputPatterns = irule.outputPatterns
// implicit rule's prerequisites will be used for $<
ir.inputs = append(irule.inputs, ir.inputs...)
ir.cmds = irule.cmds
// TODO(ukai): filename, lineno?
ir.cmdLineno = irule.cmdLineno
return ir, vars, true
}
if vars != nil {
var outputs []string
for _, op := range irule.outputPatterns {
outputs = append(outputs, op.String())
}
vars = db.mergeImplicitRuleVars(outputs, vars)
}
// TODO(ukai): check len(irule.cmd) ?
return irule, vars, true
}
outputSuffix := filepath.Ext(output)
if !strings.HasPrefix(outputSuffix, ".") {
return r, vars, r != nil
}
rules, present := db.suffixRules[outputSuffix[1:]]
if !present {
return r, vars, r != nil
}
for _, irule := range rules {
if len(irule.inputs) != 1 {
// TODO(ukai): should return error?
panic(fmt.Sprintf("FIXME: unexpected number of input for a suffix rule (%d)", len(irule.inputs)))
}
if !db.exists(replaceSuffix(output, irule.inputs[0])) {
continue
}
db.pickSuffixRuleCnt++
if r != nil {
sr := &rule{}
*sr = *r
// TODO(ukai): input order is correct?
sr.inputs = append([]string{replaceSuffix(output, irule.inputs[0])}, r.inputs...)
sr.cmds = irule.cmds
// TODO(ukai): filename, lineno?
sr.cmdLineno = irule.cmdLineno
return sr, vars, true
}
if vars != nil {
vars = db.mergeImplicitRuleVars(irule.outputs, vars)
}
// TODO(ukai): check len(irule.cmd) ?
return irule, vars, true
}
return r, vars, r != nil
}
func expandInputs(rule *rule, output string) []string {
var inputs []string
for _, input := range rule.inputs {
if len(rule.outputPatterns) > 0 {
if len(rule.outputPatterns) != 1 {
panic(fmt.Sprintf("FIXME: multiple output pattern is not supported yet"))
}
input = intern(rule.outputPatterns[0].subst(input, output))
} else if rule.isSuffixRule {
input = intern(replaceSuffix(output, input))
}
inputs = append(inputs, input)
}
return inputs
}
func (db *depBuilder) buildPlan(output string, neededBy string, tsvs Vars) (*DepNode, error) {
glog.V(1).Infof("Evaluating command: %s", output)
db.nodeCnt++
if db.nodeCnt%100 == 0 {
db.reportStats()
}
if n, present := db.done[output]; present {
return n, nil
}
n := &DepNode{Output: output, IsPhony: db.phony[output]}
db.done[output] = n
// create depnode for phony targets?
rule, vars, present := db.pickRule(output)
if !present {
return n, nil
}
var restores []func()
if vars != nil {
for name, v := range vars {
// TODO: Consider not updating db.vars.
tsv := v.(*targetSpecificVar)
restores = append(restores, db.vars.save(name))
restores = append(restores, tsvs.save(name))
switch tsv.op {
case ":=", "=":
db.vars[name] = tsv
tsvs[name] = v
case "+=":
oldVar, present := db.vars[name]
if !present || oldVar.String() == "" {
db.vars[name] = tsv
} else {
var err error
v, err = oldVar.AppendVar(db.ev, tsv)
if err != nil {
return nil, err
}
db.vars[name] = v
}
tsvs[name] = v
case "?=":
if _, present := db.vars[name]; !present {
db.vars[name] = tsv
tsvs[name] = v
}
}
}
defer func() {
for _, restore := range restores {
restore()
}
}()
}
inputs := expandInputs(rule, output)
glog.Infof("Evaluating command: %s inputs:%q => %q", output, rule.inputs, inputs)
for _, input := range inputs {
db.trace = append(db.trace, input)
ni, err := db.buildPlan(input, output, tsvs)
db.trace = db.trace[0 : len(db.trace)-1]
if err != nil {
return nil, err
}
if ni != nil {
n.Deps = append(n.Deps, ni)
ni.Parents = append(ni.Parents, n)
}
}
for _, input := range rule.orderOnlyInputs {
db.trace = append(db.trace, input)
ni, err := db.buildPlan(input, output, tsvs)
db.trace = db.trace[0 : len(db.trace)-1]
if err != nil {
return nil, err
}
if n != nil {
n.OrderOnlys = append(n.OrderOnlys, ni)
ni.Parents = append(ni.Parents, n)
}
}
n.HasRule = true
n.Cmds = rule.cmds
n.ActualInputs = inputs
n.TargetSpecificVars = make(Vars)
for k, v := range tsvs {
if glog.V(1) {
glog.Infof("output=%s tsv %s=%s", output, k, v)
}
n.TargetSpecificVars[k] = v
}
n.Filename = rule.filename
if len(rule.cmds) > 0 {
if rule.cmdLineno > 0 {
n.Lineno = rule.cmdLineno
} else {
n.Lineno = rule.lineno
}
}
return n, nil
}
func (db *depBuilder) populateSuffixRule(r *rule, output string) bool {
if len(output) == 0 || output[0] != '.' {
return false
}
rest := output[1:]
dotIndex := strings.IndexByte(rest, '.')
// If there is only a single dot or the third dot, this is not a
// suffix rule.
if dotIndex < 0 || strings.IndexByte(rest[dotIndex+1:], '.') >= 0 {
return false
}
// This is a suffix rule.
inputSuffix := rest[:dotIndex]
outputSuffix := rest[dotIndex+1:]
sr := &rule{}
*sr = *r
sr.inputs = []string{inputSuffix}
sr.isSuffixRule = true
db.suffixRules[outputSuffix] = append([]*rule{sr}, db.suffixRules[outputSuffix]...)
return true
}
func mergeRules(oldRule, r *rule, output string, isSuffixRule bool) (*rule, error) {
if oldRule.isDoubleColon != r.isDoubleColon {
return nil, r.errorf("*** target file %q has both : and :: entries.", output)
}
if len(oldRule.cmds) > 0 && len(r.cmds) > 0 && !isSuffixRule && !r.isDoubleColon {
warn(r.cmdpos(), "overriding commands for target %q", output)
warn(oldRule.cmdpos(), "ignoring old commands for target %q", output)
}
mr := &rule{}
*mr = *r
if r.isDoubleColon {
mr.cmds = append(oldRule.cmds, mr.cmds...)
} else if len(oldRule.cmds) > 0 && len(r.cmds) == 0 {
mr.cmds = oldRule.cmds
}
// If the latter rule has a command (regardless of the
// commands in oldRule), inputs in the latter rule has a
// priority.
if len(r.cmds) > 0 {
mr.inputs = append(mr.inputs, oldRule.inputs...)
mr.orderOnlyInputs = append(mr.orderOnlyInputs, oldRule.orderOnlyInputs...)
} else {
mr.inputs = append(oldRule.inputs, mr.inputs...)
mr.orderOnlyInputs = append(oldRule.orderOnlyInputs, mr.orderOnlyInputs...)
}
mr.outputPatterns = append(mr.outputPatterns, oldRule.outputPatterns...)
return mr, nil
}
// expandPattern expands static pattern (target: target-pattern: prereq-pattern).
func expandPattern(r *rule) []*rule {
if len(r.outputs) == 0 {
return []*rule{r}
}
if len(r.outputPatterns) != 1 {
return []*rule{r}
}
var rules []*rule
pat := r.outputPatterns[0]
for _, output := range r.outputs {
nr := new(rule)
*nr = *r
nr.outputs = []string{output}
nr.outputPatterns = nil
nr.inputs = nil
for _, input := range r.inputs {
nr.inputs = append(nr.inputs, intern(pat.subst(input, output)))
}
rules = append(rules, nr)
}
glog.V(1).Infof("expand static pattern: outputs=%q inputs=%q -> %q", r.outputs, r.inputs, rules)
return rules
}
func (db *depBuilder) populateExplicitRule(r *rule) error {
// It seems rules with no outputs are siliently ignored.
if len(r.outputs) == 0 {
return nil
}
for _, output := range r.outputs {
output = trimLeadingCurdir(output)
isSuffixRule := db.populateSuffixRule(r, output)
if oldRule, present := db.rules[output]; present {
mr, err := mergeRules(oldRule, r, output, isSuffixRule)
if err != nil {
return err
}
db.rules[output] = mr
} else {
db.rules[output] = r
if db.firstRule == nil && !strings.HasPrefix(output, ".") {
db.firstRule = r
}
}
}
return nil
}
func (db *depBuilder) populateImplicitRule(r *rule) {
for _, outputPattern := range r.outputPatterns {
ir := &rule{}
*ir = *r
ir.outputPatterns = []pattern{outputPattern}
db.implicitRules.add(outputPattern.String(), ir)
}
}
func (db *depBuilder) populateRules(er *evalResult) error {
for _, r := range er.rules {
for i, input := range r.inputs {
r.inputs[i] = trimLeadingCurdir(input)
}
for i, orderOnlyInput := range r.orderOnlyInputs {
r.orderOnlyInputs[i] = trimLeadingCurdir(orderOnlyInput)
}
for _, r := range expandPattern(r) {
err := db.populateExplicitRule(r)
if err != nil {
return err
}
if len(r.outputs) == 0 {
db.populateImplicitRule(r)
}
}
}
return nil
}
func (db *depBuilder) reportStats() {
if !PeriodicStatsFlag {
return
}
logStats("node=%d explicit=%d implicit=%d suffix=%d explicitWOCmd=%d",
db.nodeCnt, db.pickExplicitRuleCnt, db.pickImplicitRuleCnt, db.pickSuffixRuleCnt, db.pickExplicitRuleWithoutCmdCnt)
if len(db.trace) > 1 {
logStats("trace=%q", db.trace)
}
}
func newDepBuilder(er *evalResult, vars Vars) (*depBuilder, error) {
db := &depBuilder{
rules: make(map[string]*rule),
ruleVars: er.ruleVars,
implicitRules: newRuleTrie(),
suffixRules: make(map[string][]*rule),
vars: vars,
ev: NewEvaluator(vars),
vpaths: er.vpaths,
done: make(map[string]*DepNode),
phony: make(map[string]bool),
}
err := db.populateRules(er)
if err != nil {
return nil, err
}
rule, present := db.rules[".PHONY"]
if present {
for _, input := range rule.inputs {
db.phony[input] = true
}
}
return db, nil
}
func (db *depBuilder) Eval(targets []string) ([]*DepNode, error) {
if len(targets) == 0 {
if db.firstRule == nil {
return nil, fmt.Errorf("*** No targets.")
}
targets = append(targets, db.firstRule.outputs[0])
var phonys []string
for t := range db.phony {
phonys = append(phonys, t)
}
sort.Strings(phonys)
targets = append(targets, phonys...)
}
if StatsFlag {
logStats("%d variables", len(db.vars))
logStats("%d explicit rules", len(db.rules))
logStats("%d implicit rules", db.implicitRules.size())
logStats("%d suffix rules", len(db.suffixRules))
logStats("%d dirs %d files", fsCache.dirs(), fsCache.files())
}
var nodes []*DepNode
for _, target := range targets {
db.trace = []string{target}
n, err := db.buildPlan(target, "", make(Vars))
if err != nil {
return nil, err
}
nodes = append(nodes, n)
}
db.reportStats()
return nodes, nil
}