// Copyright 2015 Google Inc. All rights reserved
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package kati
import (
"bytes"
"crypto/sha1"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"github.com/golang/glog"
)
type fileState int
const (
fileExists fileState = iota
fileNotExists
fileInconsistent // Modified during kati is running.
)
type accessedMakefile struct {
Filename string
Hash [sha1.Size]byte
State fileState
}
type accessCache struct {
mu sync.Mutex
m map[string]*accessedMakefile
}
func newAccessCache() *accessCache {
return &accessCache{
m: make(map[string]*accessedMakefile),
}
}
func (ac *accessCache) update(fn string, hash [sha1.Size]byte, st fileState) string {
if ac == nil {
return ""
}
ac.mu.Lock()
defer ac.mu.Unlock()
rm, present := ac.m[fn]
if present {
switch rm.State {
case fileExists:
if st != fileExists {
return fmt.Sprintf("%s was removed after the previous read", fn)
} else if !bytes.Equal(hash[:], rm.Hash[:]) {
ac.m[fn].State = fileInconsistent
return fmt.Sprintf("%s was modified after the previous read", fn)
}
return ""
case fileNotExists:
if st != fileNotExists {
ac.m[fn].State = fileInconsistent
return fmt.Sprintf("%s was created after the previous read", fn)
}
case fileInconsistent:
return ""
}
return ""
}
ac.m[fn] = &accessedMakefile{
Filename: fn,
Hash: hash,
State: st,
}
return ""
}
func (ac *accessCache) Slice() []*accessedMakefile {
if ac == nil {
return nil
}
ac.mu.Lock()
defer ac.mu.Unlock()
r := []*accessedMakefile{}
for _, v := range ac.m {
r = append(r, v)
}
return r
}
type evalResult struct {
vars Vars
rules []*rule
ruleVars map[string]Vars
accessedMks []*accessedMakefile
exports map[string]bool
vpaths searchPaths
}
type srcpos struct {
filename string
lineno int
}
func (p srcpos) String() string {
return fmt.Sprintf("%s:%d", p.filename, p.lineno)
}
// EvalError is an error in kati evaluation.
type EvalError struct {
Filename string
Lineno int
Err error
}
func (e EvalError) Error() string {
return fmt.Sprintf("%s:%d: %v", e.Filename, e.Lineno, e.Err)
}
func (p srcpos) errorf(f string, args ...interface{}) error {
return EvalError{
Filename: p.filename,
Lineno: p.lineno,
Err: fmt.Errorf(f, args...),
}
}
func (p srcpos) error(err error) error {
if _, ok := err.(EvalError); ok {
return err
}
return EvalError{
Filename: p.filename,
Lineno: p.lineno,
Err: err,
}
}
// Evaluator manages makefile evaluation.
type Evaluator struct {
paramVars []tmpval // $1 => paramVars[1]
outVars Vars
outRules []*rule
outRuleVars map[string]Vars
vars Vars
lastRule *rule
currentScope Vars
cache *accessCache
exports map[string]bool
vpaths []vpath
avoidIO bool
hasIO bool
// delayedOutputs are commands which should run at ninja-time
// (i.e., info, warning, and error).
delayedOutputs []string
srcpos
}
// NewEvaluator creates new Evaluator.
func NewEvaluator(vars map[string]Var) *Evaluator {
return &Evaluator{
outVars: make(Vars),
vars: vars,
outRuleVars: make(map[string]Vars),
exports: make(map[string]bool),
}
}
func (ev *Evaluator) args(buf *evalBuffer, args ...Value) ([][]byte, error) {
pos := make([]int, 0, len(args))
for _, arg := range args {
buf.resetSep()
err := arg.Eval(buf, ev)
if err != nil {
return nil, err
}
pos = append(pos, buf.Len())
}
v := buf.Bytes()
buf.args = buf.args[:0]
s := 0
for _, p := range pos {
buf.args = append(buf.args, v[s:p])
s = p
}
return buf.args, nil
}
func (ev *Evaluator) evalAssign(ast *assignAST) error {
ev.lastRule = nil
lhs, rhs, err := ev.evalAssignAST(ast)
if err != nil {
return err
}
if glog.V(1) {
glog.Infof("ASSIGN: %s=%q (flavor:%q)", lhs, rhs, rhs.Flavor())
}
if lhs == "" {
return ast.errorf("*** empty variable name.")
}
ev.outVars.Assign(lhs, rhs)
return nil
}
func (ev *Evaluator) evalAssignAST(ast *assignAST) (string, Var, error) {
ev.srcpos = ast.srcpos
var lhs string
switch v := ast.lhs.(type) {
case literal:
lhs = string(v)
case tmpval:
lhs = string(v)
default:
buf := newEbuf()
err := v.Eval(buf, ev)
if err != nil {
return "", nil, err
}
lhs = string(trimSpaceBytes(buf.Bytes()))
buf.release()
}
rhs, err := ast.evalRHS(ev, lhs)
if err != nil {
return "", nil, err
}
return lhs, rhs, nil
}
func (ev *Evaluator) setTargetSpecificVar(assign *assignAST, output string) error {
vars, present := ev.outRuleVars[output]
if !present {
vars = make(Vars)
ev.outRuleVars[output] = vars
}
ev.currentScope = vars
lhs, rhs, err := ev.evalAssignAST(assign)
if err != nil {
return err
}
if glog.V(1) {
glog.Infof("rule outputs:%q assign:%q%s%q (flavor:%q)", output, lhs, assign.op, rhs, rhs.Flavor())
}
vars.Assign(lhs, &targetSpecificVar{v: rhs, op: assign.op})
ev.currentScope = nil
return nil
}
func (ev *Evaluator) evalMaybeRule(ast *maybeRuleAST) error {
ev.lastRule = nil
ev.srcpos = ast.srcpos
if glog.V(1) {
glog.Infof("maybe rule %s: %q assign:%v", ev.srcpos, ast.expr, ast.assign)
}
abuf := newEbuf()
aexpr := toExpr(ast.expr)
var rhs expr
semi := ast.semi
for i, v := range aexpr {
var hashFound bool
var buf evalBuffer
buf.resetSep()
switch v.(type) {
case literal, tmpval:
s := v.String()
i := strings.Index(s, "#")
if i >= 0 {
hashFound = true
v = tmpval(trimRightSpaceBytes([]byte(s[:i])))
}
}
err := v.Eval(&buf, ev)
if err != nil {
return err
}
b := buf.Bytes()
if ast.isRule {
abuf.Write(b)
continue
}
eq := findLiteralChar(b, '=', 0, skipVar)
if eq >= 0 {
abuf.Write(b[:eq+1])
if eq+1 < len(b) {
rhs = append(rhs, tmpval(trimLeftSpaceBytes(b[eq+1:])))
}
if i+1 < len(aexpr) {
rhs = append(rhs, aexpr[i+1:]...)
}
if ast.semi != nil {
rhs = append(rhs, literal(';'))
sexpr, _, err := parseExpr(ast.semi, nil, parseOp{})
if err != nil {
return err
}
rhs = append(rhs, toExpr(sexpr)...)
semi = nil
}
break
}
abuf.Write(b)
if hashFound {
break
}
}
line := abuf.Bytes()
r := &rule{srcpos: ast.srcpos}
if glog.V(1) {
glog.Infof("rule? %s: %q assign:%v rhs:%s", r.srcpos, line, ast.assign, rhs)
}
assign, err := r.parse(line, ast.assign, rhs)
if err != nil {
ws := newWordScanner(line)
if ws.Scan() {
if string(ws.Bytes()) == "override" {
warnNoPrefix(ast.srcpos, "invalid `override' directive")
return nil
}
}
return ast.error(err)
}
abuf.release()
if glog.V(1) {
glog.Infof("rule %q assign:%v rhs:%v=> outputs:%q, inputs:%q", ast.expr, ast.assign, rhs, r.outputs, r.inputs)
}
// TODO: Pretty print.
// glog.V(1).Infof("RULE: %s=%s (%d commands)", lhs, rhs, len(cmds))
if assign != nil {
glog.V(1).Infof("target specific var: %#v", assign)
for _, output := range r.outputs {
ev.setTargetSpecificVar(assign, output)
}
for _, output := range r.outputPatterns {
ev.setTargetSpecificVar(assign, output.String())
}
return nil
}
if semi != nil {
r.cmds = append(r.cmds, string(semi))
}
if glog.V(1) {
glog.Infof("rule outputs:%q cmds:%q", r.outputs, r.cmds)
}
ev.lastRule = r
ev.outRules = append(ev.outRules, r)
return nil
}
func (ev *Evaluator) evalCommand(ast *commandAST) error {
ev.srcpos = ast.srcpos
if ev.lastRule == nil || ev.lastRule.outputs == nil {
// This could still be an assignment statement. See
// assign_after_tab.mk.
if strings.IndexByte(ast.cmd, '=') >= 0 {
line := trimLeftSpace(ast.cmd)
mk, err := parseMakefileString(line, ast.srcpos)
if err != nil {
return ast.errorf("parse failed: %q: %v", line, err)
}
if len(mk.stmts) >= 1 && mk.stmts[len(mk.stmts)-1].(*assignAST) != nil {
for _, stmt := range mk.stmts {
err = ev.eval(stmt)
if err != nil {
return err
}
}
}
return nil
}
// Or, a comment is OK.
if strings.TrimSpace(ast.cmd)[0] == '#' {
return nil
}
return ast.errorf("*** commands commence before first target.")
}
ev.lastRule.cmds = append(ev.lastRule.cmds, ast.cmd)
if ev.lastRule.cmdLineno == 0 {
ev.lastRule.cmdLineno = ast.lineno
}
return nil
}
func (ev *Evaluator) paramVar(name string) (Var, error) {
idx, err := strconv.ParseInt(name, 10, 32)
if err != nil {
return nil, fmt.Errorf("param: %s: %v", name, err)
}
i := int(idx)
if i < 0 || i >= len(ev.paramVars) {
return nil, fmt.Errorf("param: %s out of %d", name, len(ev.paramVars))
}
return &automaticVar{value: []byte(ev.paramVars[i])}, nil
}
// LookupVar looks up named variable.
func (ev *Evaluator) LookupVar(name string) Var {
if ev.currentScope != nil {
v := ev.currentScope.Lookup(name)
if v.IsDefined() {
return v
}
}
v := ev.outVars.Lookup(name)
if v.IsDefined() {
return v
}
v, err := ev.paramVar(name)
if err == nil {
return v
}
return ev.vars.Lookup(name)
}
func (ev *Evaluator) lookupVarInCurrentScope(name string) Var {
if ev.currentScope != nil {
v := ev.currentScope.Lookup(name)
return v
}
v := ev.outVars.Lookup(name)
if v.IsDefined() {
return v
}
v, err := ev.paramVar(name)
if err == nil {
return v
}
return ev.vars.Lookup(name)
}
// EvaluateVar evaluates variable named name.
// Only for a few special uses such as getting SHELL and handling
// export/unexport.
func (ev *Evaluator) EvaluateVar(name string) (string, error) {
var buf evalBuffer
buf.resetSep()
err := ev.LookupVar(name).Eval(&buf, ev)
if err != nil {
return "", err
}
return buf.String(), nil
}
func (ev *Evaluator) evalIncludeFile(fname string, mk makefile) error {
te := traceEvent.begin("include", literal(fname), traceEventMain)
defer func() {
traceEvent.end(te)
}()
var err error
makefileList := ev.outVars.Lookup("MAKEFILE_LIST")
makefileList, err = makefileList.Append(ev, mk.filename)
if err != nil {
return err
}
ev.outVars.Assign("MAKEFILE_LIST", makefileList)
for _, stmt := range mk.stmts {
err = ev.eval(stmt)
if err != nil {
return err
}
}
return nil
}
func (ev *Evaluator) evalInclude(ast *includeAST) error {
ev.lastRule = nil
ev.srcpos = ast.srcpos
glog.Infof("%s include %q", ev.srcpos, ast.expr)
v, _, err := parseExpr([]byte(ast.expr), nil, parseOp{})
if err != nil {
return ast.errorf("parse failed: %q: %v", ast.expr, err)
}
var buf evalBuffer
buf.resetSep()
err = v.Eval(&buf, ev)
if err != nil {
return ast.errorf("%v", err)
}
pats := splitSpaces(buf.String())
buf.Reset()
var files []string
for _, pat := range pats {
if strings.Contains(pat, "*") || strings.Contains(pat, "?") {
matched, err := filepath.Glob(pat)
if err != nil {
return ast.errorf("glob error: %s: %v", pat, err)
}
files = append(files, matched...)
} else {
files = append(files, pat)
}
}
for _, fn := range files {
fn = trimLeadingCurdir(fn)
if IgnoreOptionalInclude != "" && ast.op == "-include" && matchPattern(fn, IgnoreOptionalInclude) {
continue
}
mk, hash, err := makefileCache.parse(fn)
if os.IsNotExist(err) {
if ast.op == "include" {
return ev.errorf("%v\nNOTE: kati does not support generating missing makefiles", err)
}
msg := ev.cache.update(fn, hash, fileNotExists)
if msg != "" {
warn(ev.srcpos, "%s", msg)
}
continue
}
msg := ev.cache.update(fn, hash, fileExists)
if msg != "" {
warn(ev.srcpos, "%s", msg)
}
err = ev.evalIncludeFile(fn, mk)
if err != nil {
return err
}
}
return nil
}
func (ev *Evaluator) evalIf(iast *ifAST) error {
var isTrue bool
switch iast.op {
case "ifdef", "ifndef":
expr := iast.lhs
buf := newEbuf()
err := expr.Eval(buf, ev)
if err != nil {
return iast.errorf("%v\n expr:%s", err, expr)
}
v := ev.LookupVar(buf.String())
buf.Reset()
err = v.Eval(buf, ev)
if err != nil {
return iast.errorf("%v\n expr:%s=>%s", err, expr, v)
}
value := buf.String()
val := buf.Len()
buf.release()
isTrue = (val > 0) == (iast.op == "ifdef")
if glog.V(1) {
glog.Infof("%s lhs=%q value=%q => %t", iast.op, iast.lhs, value, isTrue)
}
case "ifeq", "ifneq":
lexpr := iast.lhs
rexpr := iast.rhs
buf := newEbuf()
params, err := ev.args(buf, lexpr, rexpr)
if err != nil {
return iast.errorf("%v\n (%s,%s)", err, lexpr, rexpr)
}
lhs := string(params[0])
rhs := string(params[1])
buf.release()
isTrue = (lhs == rhs) == (iast.op == "ifeq")
if glog.V(1) {
glog.Infof("%s lhs=%q %q rhs=%q %q => %t", iast.op, iast.lhs, lhs, iast.rhs, rhs, isTrue)
}
default:
return iast.errorf("unknown if statement: %q", iast.op)
}
var stmts []ast
if isTrue {
stmts = iast.trueStmts
} else {
stmts = iast.falseStmts
}
for _, stmt := range stmts {
err := ev.eval(stmt)
if err != nil {
return err
}
}
return nil
}
func (ev *Evaluator) evalExport(ast *exportAST) error {
ev.lastRule = nil
ev.srcpos = ast.srcpos
v, _, err := parseExpr(ast.expr, nil, parseOp{})
if err != nil {
return ast.errorf("failed to parse: %q: %v", string(ast.expr), err)
}
var buf evalBuffer
buf.resetSep()
err = v.Eval(&buf, ev)
if err != nil {
return ast.errorf("%v\n expr:%s", err, v)
}
if ast.hasEqual {
ev.exports[string(trimSpaceBytes(buf.Bytes()))] = ast.export
} else {
for _, n := range splitSpacesBytes(buf.Bytes()) {
ev.exports[string(n)] = ast.export
}
}
return nil
}
func (ev *Evaluator) evalVpath(ast *vpathAST) error {
ev.lastRule = nil
ev.srcpos = ast.srcpos
var ebuf evalBuffer
ebuf.resetSep()
err := ast.expr.Eval(&ebuf, ev)
if err != nil {
return ast.errorf("%v\n expr:%s", err, ast.expr)
}
ws := newWordScanner(ebuf.Bytes())
if !ws.Scan() {
ev.vpaths = nil
return nil
}
pat := string(ws.Bytes())
if !ws.Scan() {
vpaths := ev.vpaths
ev.vpaths = nil
for _, v := range vpaths {
if v.pattern == pat {
continue
}
ev.vpaths = append(ev.vpaths, v)
}
return nil
}
// The search path, DIRECTORIES, is a list of directories to be
// searched, separated by colons (semi-colons on MS-DOS and
// MS-Windows) or blanks, just like the search path used in the
// `VPATH' variable.
var dirs []string
for {
for _, dir := range bytes.Split(ws.Bytes(), []byte{':'}) {
dirs = append(dirs, string(dir))
}
if !ws.Scan() {
break
}
}
ev.vpaths = append(ev.vpaths, vpath{
pattern: pat,
dirs: dirs,
})
return nil
}
func (ev *Evaluator) eval(stmt ast) error {
return stmt.eval(ev)
}
func eval(mk makefile, vars Vars, useCache bool) (er *evalResult, err error) {
ev := NewEvaluator(vars)
if useCache {
ev.cache = newAccessCache()
}
makefileList := vars.Lookup("MAKEFILE_LIST")
if !makefileList.IsDefined() {
makefileList = &simpleVar{value: []string{""}, origin: "file"}
}
makefileList, err = makefileList.Append(ev, mk.filename)
if err != nil {
return nil, err
}
ev.outVars.Assign("MAKEFILE_LIST", makefileList)
for _, stmt := range mk.stmts {
err = ev.eval(stmt)
if err != nil {
return nil, err
}
}
vpaths := searchPaths{
vpaths: ev.vpaths,
}
v, found := ev.outVars["VPATH"]
if found {
wb := newWbuf()
err := v.Eval(wb, ev)
if err != nil {
return nil, err
}
// In the 'VPATH' variable, directory names are separated
// by colons or blanks. (on windows, semi-colons)
for _, word := range wb.words {
for _, dir := range bytes.Split(word, []byte{':'}) {
vpaths.dirs = append(vpaths.dirs, string(dir))
}
}
}
glog.Infof("vpaths: %#v", vpaths)
return &evalResult{
vars: ev.outVars,
rules: ev.outRules,
ruleVars: ev.outRuleVars,
accessedMks: ev.cache.Slice(),
exports: ev.exports,
vpaths: vpaths,
}, nil
}