// Copyright 2015 Google Inc. All rights reserved
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build ignore
#include "func.h"
#include <errno.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <algorithm>
#include <iterator>
#include <memory>
#include <unordered_map>
#include "eval.h"
#include "fileutil.h"
#include "find.h"
#include "log.h"
#include "parser.h"
#include "stats.h"
#include "stmt.h"
#include "strutil.h"
#include "symtab.h"
#include "var.h"
namespace {
// TODO: This code is very similar to
// NinjaGenerator::TranslateCommand. Factor them out.
void StripShellComment(string* cmd) {
if (cmd->find('#') == string::npos)
return;
string res;
bool prev_backslash = false;
// Set space as an initial value so the leading comment will be
// stripped out.
char prev_char = ' ';
char quote = 0;
bool done = false;
const char* in = cmd->c_str();
for (; *in && !done; in++) {
switch (*in) {
case '#':
if (quote == 0 && isspace(prev_char)) {
while (in[1] && *in != '\n')
in++;
break;
}
case '\'':
case '"':
case '`':
if (quote) {
if (quote == *in)
quote = 0;
} else if (!prev_backslash) {
quote = *in;
}
res += *in;
break;
case '\\':
res += '\\';
break;
default:
res += *in;
}
if (*in == '\\') {
prev_backslash = !prev_backslash;
} else {
prev_backslash = false;
}
prev_char = *in;
}
cmd->swap(res);
}
void PatsubstFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& pat_str = args[0]->Eval(ev);
const string&& repl = args[1]->Eval(ev);
const string&& str = args[2]->Eval(ev);
WordWriter ww(s);
Pattern pat(pat_str);
for (StringPiece tok : WordScanner(str)) {
ww.MaybeAddWhitespace();
pat.AppendSubst(tok, repl, s);
}
}
void StripFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& str = args[0]->Eval(ev);
WordWriter ww(s);
for (StringPiece tok : WordScanner(str)) {
ww.Write(tok);
}
}
void SubstFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& pat = args[0]->Eval(ev);
const string&& repl = args[1]->Eval(ev);
const string&& str = args[2]->Eval(ev);
if (pat.empty()) {
*s += str;
*s += repl;
return;
}
size_t index = 0;
while (index < str.size()) {
size_t found = str.find(pat, index);
if (found == string::npos)
break;
AppendString(StringPiece(str).substr(index, found - index), s);
AppendString(repl, s);
index = found + pat.size();
}
AppendString(StringPiece(str).substr(index), s);
}
void FindstringFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& find = args[0]->Eval(ev);
const string&& in = args[1]->Eval(ev);
if (in.find(find) != string::npos)
AppendString(find, s);
}
void FilterFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& pat_buf = args[0]->Eval(ev);
const string&& text = args[1]->Eval(ev);
vector<Pattern> pats;
for (StringPiece pat : WordScanner(pat_buf)) {
pats.push_back(Pattern(pat));
}
WordWriter ww(s);
for (StringPiece tok : WordScanner(text)) {
for (const Pattern& pat : pats) {
if (pat.Match(tok)) {
ww.Write(tok);
break;
}
}
}
}
void FilterOutFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& pat_buf = args[0]->Eval(ev);
const string&& text = args[1]->Eval(ev);
vector<Pattern> pats;
for (StringPiece pat : WordScanner(pat_buf)) {
pats.push_back(Pattern(pat));
}
WordWriter ww(s);
for (StringPiece tok : WordScanner(text)) {
bool matched = false;
for (const Pattern& pat : pats) {
if (pat.Match(tok)) {
matched = true;
break;
}
}
if (!matched)
ww.Write(tok);
}
}
void SortFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
string list;
args[0]->Eval(ev, &list);
COLLECT_STATS("func sort time");
// TODO(hamaji): Probably we could use a faster string-specific sort
// algorithm.
vector<StringPiece> toks;
WordScanner(list).Split(&toks);
stable_sort(toks.begin(), toks.end());
WordWriter ww(s);
StringPiece prev;
for (StringPiece tok : toks) {
if (prev != tok) {
ww.Write(tok);
prev = tok;
}
}
}
static int GetNumericValueForFunc(const string& buf) {
StringPiece s = TrimLeftSpace(buf);
char* end;
long n = strtol(s.data(), &end, 10);
if (n < 0 || n == LONG_MAX || s.data() + s.size() != end) {
return -1;
}
return n;
}
void WordFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& n_str = args[0]->Eval(ev);
int n = GetNumericValueForFunc(n_str);
if (n < 0) {
ev->Error(StringPrintf(
"*** non-numeric first argument to `word' function: '%s'.",
n_str.c_str()));
}
if (n == 0) {
ev->Error("*** first argument to `word' function must be greater than 0.");
}
const string&& text = args[1]->Eval(ev);
for (StringPiece tok : WordScanner(text)) {
n--;
if (n == 0) {
AppendString(tok, s);
break;
}
}
}
void WordlistFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& s_str = args[0]->Eval(ev);
int si = GetNumericValueForFunc(s_str);
if (si < 0) {
ev->Error(StringPrintf(
"*** non-numeric first argument to `wordlist' function: '%s'.",
s_str.c_str()));
}
if (si == 0) {
ev->Error(StringPrintf(
"*** invalid first argument to `wordlist' function: %s`",
s_str.c_str()));
}
const string&& e_str = args[1]->Eval(ev);
int ei = GetNumericValueForFunc(e_str);
if (ei < 0) {
ev->Error(StringPrintf(
"*** non-numeric second argument to `wordlist' function: '%s'.",
e_str.c_str()));
}
const string&& text = args[2]->Eval(ev);
int i = 0;
WordWriter ww(s);
for (StringPiece tok : WordScanner(text)) {
i++;
if (si <= i && i <= ei) {
ww.Write(tok);
}
}
}
void WordsFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& text = args[0]->Eval(ev);
WordScanner ws(text);
int n = 0;
for (auto iter = ws.begin(); iter != ws.end(); ++iter)
n++;
char buf[32];
sprintf(buf, "%d", n);
*s += buf;
}
void FirstwordFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& text = args[0]->Eval(ev);
for (StringPiece tok : WordScanner(text)) {
AppendString(tok, s);
return;
}
}
void LastwordFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& text = args[0]->Eval(ev);
StringPiece last;
for (StringPiece tok : WordScanner(text)) {
last = tok;
}
AppendString(last, s);
}
void JoinFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& list1 = args[0]->Eval(ev);
const string&& list2 = args[1]->Eval(ev);
WordScanner ws1(list1);
WordScanner ws2(list2);
WordWriter ww(s);
for (WordScanner::Iterator iter1 = ws1.begin(), iter2 = ws2.begin();
iter1 != ws1.end() && iter2 != ws2.end();
++iter1, ++iter2) {
ww.Write(*iter1);
// Use |AppendString| not to append extra ' '.
AppendString(*iter2, s);
}
}
void WildcardFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& pat = args[0]->Eval(ev);
COLLECT_STATS("func wildcard time");
// Note GNU make does not delay the execution of $(wildcard) so we
// do not need to check avoid_io here.
WordWriter ww(s);
vector<string>* files;
for (StringPiece tok : WordScanner(pat)) {
ScopedTerminator st(tok);
Glob(tok.data(), &files);
sort(files->begin(), files->end());
for (const string& file : *files) {
ww.Write(file);
}
}
}
void DirFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& text = args[0]->Eval(ev);
WordWriter ww(s);
for (StringPiece tok : WordScanner(text)) {
ww.Write(Dirname(tok));
s->push_back('/');
}
}
void NotdirFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& text = args[0]->Eval(ev);
WordWriter ww(s);
for (StringPiece tok : WordScanner(text)) {
if (tok == "/") {
ww.Write(StringPiece(""));
} else {
ww.Write(Basename(tok));
}
}
}
void SuffixFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& text = args[0]->Eval(ev);
WordWriter ww(s);
for (StringPiece tok : WordScanner(text)) {
StringPiece suf = GetExt(tok);
if (!suf.empty())
ww.Write(suf);
}
}
void BasenameFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& text = args[0]->Eval(ev);
WordWriter ww(s);
for (StringPiece tok : WordScanner(text)) {
ww.Write(StripExt(tok));
}
}
void AddsuffixFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& suf = args[0]->Eval(ev);
const string&& text = args[1]->Eval(ev);
WordWriter ww(s);
for (StringPiece tok : WordScanner(text)) {
ww.Write(tok);
*s += suf;
}
}
void AddprefixFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& pre = args[0]->Eval(ev);
const string&& text = args[1]->Eval(ev);
WordWriter ww(s);
for (StringPiece tok : WordScanner(text)) {
ww.Write(pre);
AppendString(tok, s);
}
}
void RealpathFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& text = args[0]->Eval(ev);
if (ev->avoid_io()) {
*s += "$(";
string kati_binary;
GetExecutablePath(&kati_binary);
*s += kati_binary;
*s += " --realpath ";
*s += text;
*s += " 2> /dev/null)";
return;
}
WordWriter ww(s);
for (StringPiece tok : WordScanner(text)) {
ScopedTerminator st(tok);
char buf[PATH_MAX];
if (realpath(tok.data(), buf))
ww.Write(buf);
}
}
void AbspathFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& text = args[0]->Eval(ev);
WordWriter ww(s);
string buf;
for (StringPiece tok : WordScanner(text)) {
AbsPath(tok, &buf);
ww.Write(buf);
}
}
void IfFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& cond = args[0]->Eval(ev);
if (cond.empty()) {
if (args.size() > 2)
args[2]->Eval(ev, s);
} else {
args[1]->Eval(ev, s);
}
}
void AndFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
string cond;
for (Value* a : args) {
cond = a->Eval(ev);
if (cond.empty())
return;
}
if (!cond.empty()) {
*s += cond;
}
}
void OrFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
for (Value* a : args) {
const string&& cond = a->Eval(ev);
if (!cond.empty()) {
*s += cond;
return;
}
}
}
void ValueFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& var_name = args[0]->Eval(ev);
Var* var = ev->LookupVar(Intern(var_name));
AppendString(var->String().as_string(), s);
}
void EvalFunc(const vector<Value*>& args, Evaluator* ev, string*) {
// TODO: eval leaks everything... for now.
//const string text = args[0]->Eval(ev);
string* text = new string;
args[0]->Eval(ev, text);
if ((*text)[0] == '#') {
delete text;
return;
}
if (ev->avoid_io()) {
KATI_WARN("%s:%d: *warning*: $(eval) in a recipe is not recommended: %s",
LOCF(ev->loc()), text->c_str());
}
vector<Stmt*> stmts;
Parse(*text, ev->loc(), &stmts);
for (Stmt* stmt : stmts) {
LOG("%s", stmt->DebugString().c_str());
stmt->Eval(ev);
//delete stmt;
}
}
//#define TEST_FIND_EMULATOR
// A hack for Android build. We need to evaluate things like $((3+4))
// when we emit ninja file, because the result of such expressions
// will be passed to other make functions.
// TODO: Maybe we should introduce a helper binary which evaluate
// make expressions at ninja-time.
static bool HasNoIoInShellScript(const string& cmd) {
if (cmd.empty())
return true;
if (HasPrefix(cmd, "echo $((") && cmd[cmd.size()-1] == ')')
return true;
return false;
}
static void ShellFuncImpl(const string& shell, const string& cmd,
string* s, FindCommand** fc) {
LOG("ShellFunc: %s", cmd.c_str());
#ifdef TEST_FIND_EMULATOR
bool need_check = false;
string out2;
#endif
if (FindEmulator::Get()) {
*fc = new FindCommand();
if ((*fc)->Parse(cmd)) {
#ifdef TEST_FIND_EMULATOR
if (FindEmulator::Get()->HandleFind(cmd, **fc, &out2)) {
need_check = true;
}
#else
if (FindEmulator::Get()->HandleFind(cmd, **fc, s)) {
return;
}
#endif
}
delete *fc;
*fc = NULL;
}
COLLECT_STATS_WITH_SLOW_REPORT("func shell time", cmd.c_str());
RunCommand(shell, cmd, RedirectStderr::NONE, s);
FormatForCommandSubstitution(s);
#ifdef TEST_FIND_EMULATOR
if (need_check) {
if (*s != out2) {
ERROR("FindEmulator is broken: %s\n%s\nvs\n%s",
cmd.c_str(), s->c_str(), out2.c_str());
}
}
#endif
}
static vector<CommandResult*> g_command_results;
bool ShouldStoreCommandResult(StringPiece cmd) {
if (HasWord(cmd, "date") || HasWord(cmd, "echo"))
return false;
Pattern pat(g_flags.ignore_dirty_pattern);
Pattern nopat(g_flags.no_ignore_dirty_pattern);
for (StringPiece tok : WordScanner(cmd)) {
if (pat.Match(tok) && !nopat.Match(tok)) {
return false;
}
}
return true;
}
void ShellFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
string cmd = args[0]->Eval(ev);
if (ev->avoid_io() && !HasNoIoInShellScript(cmd)) {
if (ev->eval_depth() > 1) {
ERROR("%s:%d: kati doesn't support passing results of $(shell) "
"to other make constructs: %s",
LOCF(ev->loc()), cmd.c_str());
}
StripShellComment(&cmd);
*s += "$(";
*s += cmd;
*s += ")";
return;
}
const string&& shell = ev->EvalVar(kShellSym);
string out;
FindCommand* fc = NULL;
ShellFuncImpl(shell, cmd, &out, &fc);
if (ShouldStoreCommandResult(cmd)) {
CommandResult* cr = new CommandResult();
cr->cmd = cmd;
cr->find.reset(fc);
cr->result = out;
g_command_results.push_back(cr);
}
*s += out;
}
void CallFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
static const Symbol tmpvar_names[] = {
Intern("0"), Intern("1"), Intern("2"), Intern("3"), Intern("4"),
Intern("5"), Intern("6"), Intern("7"), Intern("8"), Intern("9")
};
const string&& func_name = args[0]->Eval(ev);
Var* func = ev->LookupVar(Intern(func_name));
if (!func->IsDefined()) {
KATI_WARN("%s:%d: *warning*: undefined user function: %s",
ev->loc(), func_name.c_str());
}
vector<unique_ptr<SimpleVar>> av;
for (size_t i = 1; i < args.size(); i++) {
unique_ptr<SimpleVar> s(
new SimpleVar(args[i]->Eval(ev), VarOrigin::AUTOMATIC));
av.push_back(move(s));
}
vector<unique_ptr<ScopedGlobalVar>> sv;
for (size_t i = 1; ; i++) {
string s;
Symbol tmpvar_name_sym(Symbol::IsUninitialized{});
if (i < sizeof(tmpvar_names)/sizeof(tmpvar_names[0])) {
tmpvar_name_sym = tmpvar_names[i];
} else {
s = StringPrintf("%d", i);
tmpvar_name_sym = Intern(s);
}
if (i < args.size()) {
sv.emplace_back(new ScopedGlobalVar(tmpvar_name_sym, av[i-1].get()));
} else {
// We need to blank further automatic vars
Var *v = ev->LookupVar(tmpvar_name_sym);
if (!v->IsDefined()) break;
if (v->Origin() != VarOrigin::AUTOMATIC) break;
av.emplace_back(new SimpleVar("", VarOrigin::AUTOMATIC));
sv.emplace_back(new ScopedGlobalVar(tmpvar_name_sym, av[i-1].get()));
}
}
ev->DecrementEvalDepth();
func->Eval(ev, s);
ev->IncrementEvalDepth();
}
void ForeachFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& varname = args[0]->Eval(ev);
const string&& list = args[1]->Eval(ev);
ev->DecrementEvalDepth();
WordWriter ww(s);
for (StringPiece tok : WordScanner(list)) {
unique_ptr<SimpleVar> v(new SimpleVar(
tok.as_string(), VarOrigin::AUTOMATIC));
ScopedGlobalVar sv(Intern(varname), v.get());
ww.MaybeAddWhitespace();
args[2]->Eval(ev, s);
}
ev->IncrementEvalDepth();
}
void OriginFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& var_name = args[0]->Eval(ev);
Var* var = ev->LookupVar(Intern(var_name));
*s += GetOriginStr(var->Origin());
}
void FlavorFunc(const vector<Value*>& args, Evaluator* ev, string* s) {
const string&& var_name = args[0]->Eval(ev);
Var* var = ev->LookupVar(Intern(var_name));
*s += var->Flavor();
}
void InfoFunc(const vector<Value*>& args, Evaluator* ev, string*) {
const string&& a = args[0]->Eval(ev);
if (ev->avoid_io()) {
ev->add_delayed_output_command(StringPrintf("echo -e \"%s\"", EchoEscape(a).c_str()));
return;
}
printf("%s\n", a.c_str());
fflush(stdout);
}
void WarningFunc(const vector<Value*>& args, Evaluator* ev, string*) {
const string&& a = args[0]->Eval(ev);
if (ev->avoid_io()) {
ev->add_delayed_output_command(
StringPrintf("echo -e \"%s:%d: %s\" 2>&1", LOCF(ev->loc()), EchoEscape(a).c_str()));
return;
}
printf("%s:%d: %s\n", LOCF(ev->loc()), a.c_str());
fflush(stdout);
}
void ErrorFunc(const vector<Value*>& args, Evaluator* ev, string*) {
const string&& a = args[0]->Eval(ev);
if (ev->avoid_io()) {
ev->add_delayed_output_command(
StringPrintf("echo -e \"%s:%d: *** %s.\" 2>&1 && false",
LOCF(ev->loc()), EchoEscape(a).c_str()));
return;
}
ev->Error(StringPrintf("*** %s.", a.c_str()));
}
FuncInfo g_func_infos[] = {
{ "patsubst", &PatsubstFunc, 3, 3, false, false },
{ "strip", &StripFunc, 1, 1, false, false },
{ "subst", &SubstFunc, 3, 3, false, false },
{ "findstring", &FindstringFunc, 2, 2, false, false },
{ "filter", &FilterFunc, 2, 2, false, false },
{ "filter-out", &FilterOutFunc, 2, 2, false, false },
{ "sort", &SortFunc, 1, 1, false, false },
{ "word", &WordFunc, 2, 2, false, false },
{ "wordlist", &WordlistFunc, 3, 3, false, false },
{ "words", &WordsFunc, 1, 1, false, false },
{ "firstword", &FirstwordFunc, 1, 1, false, false },
{ "lastword", &LastwordFunc, 1, 1, false, false },
{ "join", &JoinFunc, 2, 2, false, false },
{ "wildcard", &WildcardFunc, 1, 1, false, false },
{ "dir", &DirFunc, 1, 1, false, false },
{ "notdir", &NotdirFunc, 1, 1, false, false },
{ "suffix", &SuffixFunc, 1, 1, false, false },
{ "basename", &BasenameFunc, 1, 1, false, false },
{ "addsuffix", &AddsuffixFunc, 2, 2, false, false },
{ "addprefix", &AddprefixFunc, 2, 2, false, false },
{ "realpath", &RealpathFunc, 1, 1, false, false },
{ "abspath", &AbspathFunc, 1, 1, false, false },
{ "if", &IfFunc, 3, 2, false, true },
{ "and", &AndFunc, 0, 0, true, false },
{ "or", &OrFunc, 0, 0, true, false },
{ "value", &ValueFunc, 1, 1, false, false },
{ "eval", &EvalFunc, 1, 1, false, false },
{ "shell", &ShellFunc, 1, 1, false, false },
{ "call", &CallFunc, 0, 0, false, false },
{ "foreach", &ForeachFunc, 3, 3, false, false },
{ "origin", &OriginFunc, 1, 1, false, false },
{ "flavor", &FlavorFunc, 1, 1, false, false },
{ "info", &InfoFunc, 1, 1, false, false },
{ "warning", &WarningFunc, 1, 1, false, false },
{ "error", &ErrorFunc, 1, 1, false, false },
};
unordered_map<StringPiece, FuncInfo*>* g_func_info_map;
} // namespace
void InitFuncTable() {
g_func_info_map = new unordered_map<StringPiece, FuncInfo*>;
for (size_t i = 0; i < sizeof(g_func_infos) / sizeof(g_func_infos[0]); i++) {
FuncInfo* fi = &g_func_infos[i];
bool ok = g_func_info_map->emplace(fi->name, fi).second;
CHECK(ok);
}
}
void QuitFuncTable() {
delete g_func_info_map;
}
FuncInfo* GetFuncInfo(StringPiece name) {
auto found = g_func_info_map->find(name);
if (found == g_func_info_map->end())
return NULL;
return found->second;
}
const vector<CommandResult*>& GetShellCommandResults() {
return g_command_results;
}