// Copyright 2015 Google Inc. All rights reserved // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // +build ignore #include "parser.h" #include <stack> #include <unordered_map> #include "expr.h" #include "file.h" #include "loc.h" #include "log.h" #include "stats.h" #include "stmt.h" #include "string_piece.h" #include "strutil.h" enum struct ParserState { NOT_AFTER_RULE = 0, AFTER_RULE, MAYBE_AFTER_RULE, }; class Parser { struct IfState { IfStmt* stmt; bool is_in_else; int num_nest; }; typedef void (Parser::*DirectiveHandler)(StringPiece line, StringPiece directive); typedef unordered_map<StringPiece, DirectiveHandler> DirectiveMap; public: Parser(StringPiece buf, const char* filename, vector<Stmt*>* stmts) : buf_(buf), state_(ParserState::NOT_AFTER_RULE), stmts_(stmts), out_stmts_(stmts), num_define_nest_(0), num_if_nest_(0), loc_(filename, 0), fixed_lineno_(false) {} Parser(StringPiece buf, const Loc& loc, vector<Stmt*>* stmts) : buf_(buf), state_(ParserState::NOT_AFTER_RULE), stmts_(stmts), out_stmts_(stmts), num_if_nest_(0), loc_(loc), fixed_lineno_(true) {} ~Parser() {} void Parse() { l_ = 0; for (l_ = 0; l_ < buf_.size();) { size_t lf_cnt = 0; size_t e = FindEndOfLine(&lf_cnt); if (!fixed_lineno_) loc_.lineno++; StringPiece line(buf_.data() + l_, e - l_); if (line.get(line.size() - 1) == '\r') line.remove_suffix(1); orig_line_with_directives_ = line; ParseLine(line); if (!fixed_lineno_) loc_.lineno += lf_cnt - 1; if (e == buf_.size()) break; l_ = e + 1; } if (!if_stack_.empty()) ERROR_LOC(Loc(loc_.filename, loc_.lineno + 1), "*** missing `endif'."); if (!define_name_.empty()) ERROR_LOC(Loc(loc_.filename, define_start_line_), "*** missing `endef', unterminated `define'."); } static void Init() { make_directives_ = new DirectiveMap; (*make_directives_)["include"] = &Parser::ParseInclude; (*make_directives_)["-include"] = &Parser::ParseInclude; (*make_directives_)["sinclude"] = &Parser::ParseInclude; (*make_directives_)["define"] = &Parser::ParseDefine; (*make_directives_)["ifdef"] = &Parser::ParseIfdef; (*make_directives_)["ifndef"] = &Parser::ParseIfdef; (*make_directives_)["ifeq"] = &Parser::ParseIfeq; (*make_directives_)["ifneq"] = &Parser::ParseIfeq; (*make_directives_)["else"] = &Parser::ParseElse; (*make_directives_)["endif"] = &Parser::ParseEndif; (*make_directives_)["override"] = &Parser::ParseOverride; (*make_directives_)["export"] = &Parser::ParseExport; (*make_directives_)["unexport"] = &Parser::ParseUnexport; else_if_directives_ = new DirectiveMap; (*else_if_directives_)["ifdef"] = &Parser::ParseIfdef; (*else_if_directives_)["ifndef"] = &Parser::ParseIfdef; (*else_if_directives_)["ifeq"] = &Parser::ParseIfeq; (*else_if_directives_)["ifneq"] = &Parser::ParseIfeq; assign_directives_ = new DirectiveMap; (*assign_directives_)["define"] = &Parser::ParseDefine; (*assign_directives_)["export"] = &Parser::ParseExport; (*assign_directives_)["override"] = &Parser::ParseOverride; shortest_directive_len_ = 9999; longest_directive_len_ = 0; for (auto p : *make_directives_) { size_t len = p.first.size(); shortest_directive_len_ = min(len, shortest_directive_len_); longest_directive_len_ = max(len, longest_directive_len_); } } static void Quit() { delete make_directives_; } void set_state(ParserState st) { state_ = st; } static vector<ParseErrorStmt*> parse_errors; private: void Error(const string& msg) { ParseErrorStmt* stmt = new ParseErrorStmt(); stmt->set_loc(loc_); stmt->msg = msg; out_stmts_->push_back(stmt); parse_errors.push_back(stmt); } size_t FindEndOfLine(size_t* lf_cnt) { return ::FindEndOfLine(buf_, l_, lf_cnt); } Value* ParseExpr(StringPiece s, ParseExprOpt opt = ParseExprOpt::NORMAL) { return ::ParseExpr(loc_, s, opt); } void ParseLine(StringPiece line) { if (!define_name_.empty()) { ParseInsideDefine(line); return; } if (line.empty() || (line.size() == 1 && line[0] == '\r')) return; current_directive_ = AssignDirective::NONE; if (line[0] == '\t' && state_ != ParserState::NOT_AFTER_RULE) { CommandStmt* stmt = new CommandStmt(); stmt->set_loc(loc_); stmt->expr = ParseExpr(line.substr(1), ParseExprOpt::COMMAND); stmt->orig = line; out_stmts_->push_back(stmt); return; } line = TrimLeftSpace(line); if (line[0] == '#') return; if (HandleDirective(line, make_directives_)) { return; } ParseRuleOrAssign(line); } void ParseRuleOrAssign(StringPiece line) { size_t sep = FindThreeOutsideParen(line, ':', '=', ';'); if (sep == string::npos || line[sep] == ';') { ParseRule(line, string::npos); } else if (line[sep] == '=') { ParseAssign(line, sep); } else if (line.get(sep + 1) == '=') { ParseAssign(line, sep + 1); } else if (line[sep] == ':') { ParseRule(line, sep); } else { CHECK(false); } } void ParseRule(StringPiece line, size_t sep) { if (current_directive_ != AssignDirective::NONE) { if (IsInExport()) return; if (sep != string::npos) { sep += orig_line_with_directives_.size() - line.size(); } line = orig_line_with_directives_; } line = TrimLeftSpace(line); if (line.empty()) return; if (orig_line_with_directives_[0] == '\t') { Error("*** commands commence before first target."); return; } const bool is_rule = sep != string::npos && line[sep] == ':'; RuleStmt* rule_stmt = new RuleStmt(); rule_stmt->set_loc(loc_); size_t found = FindTwoOutsideParen(line.substr(sep + 1), '=', ';'); if (found != string::npos) { found += sep + 1; rule_stmt->lhs = ParseExpr(TrimSpace(line.substr(0, found))); if (line[found] == ';') { rule_stmt->sep = RuleStmt::SEP_SEMICOLON; } else if (line[found] == '=') { if (line.size() > (found + 2) && line[found + 1] == '$' && line[found + 2] == '=') { rule_stmt->sep = RuleStmt::SEP_FINALEQ; found += 2; } else { rule_stmt->sep = RuleStmt::SEP_EQ; } } ParseExprOpt opt = rule_stmt->sep == RuleStmt::SEP_SEMICOLON ? ParseExprOpt::COMMAND : ParseExprOpt::NORMAL; rule_stmt->rhs = ParseExpr(TrimLeftSpace(line.substr(found + 1)), opt); } else { rule_stmt->lhs = ParseExpr(line); rule_stmt->sep = RuleStmt::SEP_NULL; rule_stmt->rhs = NULL; } out_stmts_->push_back(rule_stmt); state_ = is_rule ? ParserState::AFTER_RULE : ParserState::MAYBE_AFTER_RULE; } void ParseAssign(StringPiece line, size_t separator_pos) { if (separator_pos == 0) { Error("*** empty variable name ***"); return; } StringPiece lhs; StringPiece rhs; AssignOp op; ParseAssignStatement(line, separator_pos, &lhs, &rhs, &op); // If rhs starts with '$=', this is 'final assignment', // e.g., a combination of the assignment and // .KATI_READONLY := <lhs> // statement. Note that we assume that ParseAssignStatement // trimmed the left bool is_final = (rhs.size() >= 2 && rhs[0] == '$' && rhs[1] == '='); if (is_final) { rhs = TrimLeftSpace(rhs.substr(2)); } AssignStmt* stmt = new AssignStmt(); stmt->set_loc(loc_); stmt->lhs = ParseExpr(lhs); stmt->rhs = ParseExpr(rhs); stmt->orig_rhs = rhs; stmt->op = op; stmt->directive = current_directive_; stmt->is_final = is_final; out_stmts_->push_back(stmt); state_ = ParserState::NOT_AFTER_RULE; } void ParseInclude(StringPiece line, StringPiece directive) { IncludeStmt* stmt = new IncludeStmt(); stmt->set_loc(loc_); stmt->expr = ParseExpr(line); stmt->should_exist = directive[0] == 'i'; out_stmts_->push_back(stmt); state_ = ParserState::NOT_AFTER_RULE; } void ParseDefine(StringPiece line, StringPiece) { if (line.empty()) { Error("*** empty variable name."); return; } define_name_ = line; num_define_nest_ = 1; define_start_ = 0; define_start_line_ = loc_.lineno; state_ = ParserState::NOT_AFTER_RULE; } void ParseInsideDefine(StringPiece line) { line = TrimLeftSpace(line); StringPiece directive = GetDirective(line); if (directive == "define") num_define_nest_++; else if (directive == "endef") num_define_nest_--; if (num_define_nest_ > 0) { if (define_start_ == 0) define_start_ = l_; return; } StringPiece rest = TrimRightSpace( RemoveComment(TrimLeftSpace(line.substr(sizeof("endef"))))); if (!rest.empty()) { WARN_LOC(loc_, "extraneous text after `endef' directive"); } AssignStmt* stmt = new AssignStmt(); stmt->set_loc(Loc(loc_.filename, define_start_line_)); stmt->lhs = ParseExpr(define_name_); StringPiece rhs; if (define_start_) rhs = buf_.substr(define_start_, l_ - define_start_ - 1); stmt->rhs = ParseExpr(rhs, ParseExprOpt::DEFINE); stmt->orig_rhs = rhs; stmt->op = AssignOp::EQ; stmt->directive = current_directive_; out_stmts_->push_back(stmt); define_name_.clear(); } void EnterIf(IfStmt* stmt) { IfState* st = new IfState(); st->stmt = stmt; st->is_in_else = false; st->num_nest = num_if_nest_; if_stack_.push(st); out_stmts_ = &stmt->true_stmts; } void ParseIfdef(StringPiece line, StringPiece directive) { IfStmt* stmt = new IfStmt(); stmt->set_loc(loc_); stmt->op = directive[2] == 'n' ? CondOp::IFNDEF : CondOp::IFDEF; stmt->lhs = ParseExpr(line); stmt->rhs = NULL; out_stmts_->push_back(stmt); EnterIf(stmt); } bool ParseIfEqCond(StringPiece s, IfStmt* stmt) { if (s.empty()) { return false; } if (s[0] == '(' && s[s.size() - 1] == ')') { s = s.substr(1, s.size() - 2); char terms[] = {',', '\0'}; size_t n; stmt->lhs = ParseExprImpl(loc_, s, terms, ParseExprOpt::NORMAL, &n, true); if (s[n] != ',') return false; s = TrimLeftSpace(s.substr(n + 1)); stmt->rhs = ParseExprImpl(loc_, s, NULL, ParseExprOpt::NORMAL, &n); s = TrimLeftSpace(s.substr(n)); } else { for (int i = 0; i < 2; i++) { if (s.empty()) return false; char quote = s[0]; if (quote != '\'' && quote != '"') return false; size_t end = s.find(quote, 1); if (end == string::npos) return false; Value* v = ParseExpr(s.substr(1, end - 1), ParseExprOpt::NORMAL); if (i == 0) stmt->lhs = v; else stmt->rhs = v; s = TrimLeftSpace(s.substr(end + 1)); } } if (!s.empty()) { WARN_LOC(loc_, "extraneous text after `ifeq' directive"); return true; } return true; } void ParseIfeq(StringPiece line, StringPiece directive) { IfStmt* stmt = new IfStmt(); stmt->set_loc(loc_); stmt->op = directive[2] == 'n' ? CondOp::IFNEQ : CondOp::IFEQ; if (!ParseIfEqCond(line, stmt)) { Error("*** invalid syntax in conditional."); return; } out_stmts_->push_back(stmt); EnterIf(stmt); } void ParseElse(StringPiece line, StringPiece) { if (!CheckIfStack("else")) return; IfState* st = if_stack_.top(); if (st->is_in_else) { Error("*** only one `else' per conditional."); return; } st->is_in_else = true; out_stmts_ = &st->stmt->false_stmts; StringPiece next_if = TrimLeftSpace(line); if (next_if.empty()) return; num_if_nest_ = st->num_nest + 1; if (!HandleDirective(next_if, else_if_directives_)) { WARN_LOC(loc_, "extraneous text after `else' directive"); } num_if_nest_ = 0; } void ParseEndif(StringPiece line, StringPiece) { if (!CheckIfStack("endif")) return; if (!line.empty()) { Error("extraneous text after `endif` directive"); return; } IfState st = *if_stack_.top(); for (int t = 0; t <= st.num_nest; t++) { delete if_stack_.top(); if_stack_.pop(); if (if_stack_.empty()) { out_stmts_ = stmts_; } else { IfState* st = if_stack_.top(); if (st->is_in_else) out_stmts_ = &st->stmt->false_stmts; else out_stmts_ = &st->stmt->true_stmts; } } } bool IsInExport() const { return (static_cast<int>(current_directive_) & static_cast<int>(AssignDirective::EXPORT)); } void CreateExport(StringPiece line, bool is_export) { ExportStmt* stmt = new ExportStmt; stmt->set_loc(loc_); stmt->expr = ParseExpr(line); stmt->is_export = is_export; out_stmts_->push_back(stmt); } void ParseOverride(StringPiece line, StringPiece) { current_directive_ = static_cast<AssignDirective>( (static_cast<int>(current_directive_) | static_cast<int>(AssignDirective::OVERRIDE))); if (HandleDirective(line, assign_directives_)) return; if (IsInExport()) { CreateExport(line, true); } ParseRuleOrAssign(line); } void ParseExport(StringPiece line, StringPiece) { current_directive_ = static_cast<AssignDirective>( (static_cast<int>(current_directive_) | static_cast<int>(AssignDirective::EXPORT))); if (HandleDirective(line, assign_directives_)) return; CreateExport(line, true); ParseRuleOrAssign(line); } void ParseUnexport(StringPiece line, StringPiece) { CreateExport(line, false); } bool CheckIfStack(const char* keyword) { if (if_stack_.empty()) { Error(StringPrintf("*** extraneous `%s'.", keyword)); return false; } return true; } StringPiece RemoveComment(StringPiece line) { size_t i = FindOutsideParen(line, '#'); if (i == string::npos) return line; return line.substr(0, i); } StringPiece GetDirective(StringPiece line) { if (line.size() < shortest_directive_len_) return StringPiece(); StringPiece prefix = line.substr(0, longest_directive_len_ + 1); size_t space_index = prefix.find_first_of(" \t#"); return prefix.substr(0, space_index); } bool HandleDirective(StringPiece line, const DirectiveMap* directive_map) { StringPiece directive = GetDirective(line); auto found = directive_map->find(directive); if (found == directive_map->end()) return false; StringPiece rest = TrimRightSpace( RemoveComment(TrimLeftSpace(line.substr(directive.size())))); (this->*found->second)(rest, directive); return true; } StringPiece buf_; size_t l_; ParserState state_; vector<Stmt*>* stmts_; vector<Stmt*>* out_stmts_; StringPiece define_name_; int num_define_nest_; size_t define_start_; int define_start_line_; StringPiece orig_line_with_directives_; AssignDirective current_directive_; int num_if_nest_; stack<IfState*> if_stack_; Loc loc_; bool fixed_lineno_; static DirectiveMap* make_directives_; static DirectiveMap* else_if_directives_; static DirectiveMap* assign_directives_; static size_t shortest_directive_len_; static size_t longest_directive_len_; }; void Parse(Makefile* mk) { COLLECT_STATS("parse file time"); Parser parser(StringPiece(mk->buf()), mk->filename().c_str(), mk->mutable_stmts()); parser.Parse(); } void Parse(StringPiece buf, const Loc& loc, vector<Stmt*>* out_stmts) { COLLECT_STATS("parse eval time"); Parser parser(buf, loc, out_stmts); parser.Parse(); } void ParseNotAfterRule(StringPiece buf, const Loc& loc, vector<Stmt*>* out_stmts) { Parser parser(buf, loc, out_stmts); parser.set_state(ParserState::NOT_AFTER_RULE); parser.Parse(); } void InitParser() { Parser::Init(); } void QuitParser() { Parser::Quit(); } Parser::DirectiveMap* Parser::make_directives_; Parser::DirectiveMap* Parser::else_if_directives_; Parser::DirectiveMap* Parser::assign_directives_; size_t Parser::shortest_directive_len_; size_t Parser::longest_directive_len_; vector<ParseErrorStmt*> Parser::parse_errors; void ParseAssignStatement(StringPiece line, size_t sep, StringPiece* lhs, StringPiece* rhs, AssignOp* op) { CHECK(sep != 0); *op = AssignOp::EQ; size_t lhs_end = sep; switch (line[sep - 1]) { case ':': lhs_end--; *op = AssignOp::COLON_EQ; break; case '+': lhs_end--; *op = AssignOp::PLUS_EQ; break; case '?': lhs_end--; *op = AssignOp::QUESTION_EQ; break; } *lhs = TrimSpace(line.substr(0, lhs_end)); *rhs = TrimLeftSpace(line.substr(sep + 1)); } const vector<ParseErrorStmt*>& GetParseErrors() { return Parser::parse_errors; }