// Copyright 2014 Google Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package pathtools import ( "errors" "fmt" "io/ioutil" "os" "path/filepath" "strings" "github.com/google/blueprint/deptools" ) var GlobMultipleRecursiveErr = errors.New("pattern contains multiple **") var GlobLastRecursiveErr = errors.New("pattern ** as last path element") // Glob returns the list of files and directories that match the given pattern // but do not match the given exclude patterns, along with the list of // directories and other dependencies that were searched to construct the file // list. The supported glob and exclude patterns are equivalent to // filepath.Glob, with an extension that recursive glob (** matching zero or // more complete path entries) is supported. Any directories in the matches // list will have a '/' suffix. // // In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps // should be used instead, as they will automatically set up dependencies // to rerun the primary builder when the list of matching files changes. func Glob(pattern string, excludes []string, follow ShouldFollowSymlinks) (matches, deps []string, err error) { return startGlob(OsFs, pattern, excludes, follow) } func startGlob(fs FileSystem, pattern string, excludes []string, follow ShouldFollowSymlinks) (matches, deps []string, err error) { if filepath.Base(pattern) == "**" { return nil, nil, GlobLastRecursiveErr } else { matches, deps, err = glob(fs, pattern, false, follow) } if err != nil { return nil, nil, err } matches, err = filterExcludes(matches, excludes) if err != nil { return nil, nil, err } // If the pattern has wildcards, we added dependencies on the // containing directories to know about changes. // // If the pattern didn't have wildcards, and didn't find matches, the // most specific found directories were added. // // But if it didn't have wildcards, and did find a match, no // dependencies were added, so add the match itself to detect when it // is removed. if !isWild(pattern) { deps = append(deps, matches...) } for i, match := range matches { isSymlink, err := fs.IsSymlink(match) if err != nil { return nil, nil, err } if !(isSymlink && follow == DontFollowSymlinks) { isDir, err := fs.IsDir(match) if os.IsNotExist(err) { if isSymlink { return nil, nil, fmt.Errorf("%s: dangling symlink", match) } } if err != nil { return nil, nil, fmt.Errorf("%s: %s", match, err.Error()) } if isDir { matches[i] = match + "/" } } } return matches, deps, nil } // glob is a recursive helper function to handle globbing each level of the pattern individually, // allowing searched directories to be tracked. Also handles the recursive glob pattern, **. func glob(fs FileSystem, pattern string, hasRecursive bool, follow ShouldFollowSymlinks) (matches, dirs []string, err error) { if !isWild(pattern) { // If there are no wilds in the pattern, check whether the file exists or not. // Uses filepath.Glob instead of manually statting to get consistent results. pattern = filepath.Clean(pattern) matches, err = fs.glob(pattern) if err != nil { return matches, dirs, err } if len(matches) == 0 { // Some part of the non-wild pattern didn't exist. Add the last existing directory // as a dependency. var matchDirs []string for len(matchDirs) == 0 { pattern, _ = saneSplit(pattern) matchDirs, err = fs.glob(pattern) if err != nil { return matches, dirs, err } } dirs = append(dirs, matchDirs...) } return matches, dirs, err } dir, file := saneSplit(pattern) if file == "**" { if hasRecursive { return matches, dirs, GlobMultipleRecursiveErr } hasRecursive = true } dirMatches, dirs, err := glob(fs, dir, hasRecursive, follow) if err != nil { return nil, nil, err } for _, m := range dirMatches { isDir, err := fs.IsDir(m) if os.IsNotExist(err) { if isSymlink, _ := fs.IsSymlink(m); isSymlink { return nil, nil, fmt.Errorf("dangling symlink: %s", m) } } if err != nil { return nil, nil, fmt.Errorf("unexpected error after glob: %s", err) } if isDir { if file == "**" { recurseDirs, err := fs.ListDirsRecursive(m, follow) if err != nil { return nil, nil, err } matches = append(matches, recurseDirs...) } else { dirs = append(dirs, m) newMatches, err := fs.glob(filepath.Join(MatchEscape(m), file)) if err != nil { return nil, nil, err } if file[0] != '.' { newMatches = filterDotFiles(newMatches) } matches = append(matches, newMatches...) } } } return matches, dirs, nil } // Faster version of dir, file := filepath.Dir(path), filepath.File(path) with no allocations // Similar to filepath.Split, but returns "." if dir is empty and trims trailing slash if dir is // not "/". Returns ".", "" if path is "." func saneSplit(path string) (dir, file string) { if path == "." { return ".", "" } dir, file = filepath.Split(path) switch dir { case "": dir = "." case "/": // Nothing default: dir = dir[:len(dir)-1] } return dir, file } func isWild(pattern string) bool { return strings.ContainsAny(pattern, "*?[") } // Filters the strings in matches based on the glob patterns in excludes. Hierarchical (a/*) and // recursive (**) glob patterns are supported. func filterExcludes(matches []string, excludes []string) ([]string, error) { if len(excludes) == 0 { return matches, nil } var ret []string matchLoop: for _, m := range matches { for _, e := range excludes { exclude, err := Match(e, m) if err != nil { return nil, err } if exclude { continue matchLoop } } ret = append(ret, m) } return ret, nil } // filterDotFiles filters out files that start with '.' func filterDotFiles(matches []string) []string { ret := make([]string, 0, len(matches)) for _, match := range matches { _, name := filepath.Split(match) if name[0] == '.' { continue } ret = append(ret, match) } return ret } // Match returns true if name matches pattern using the same rules as filepath.Match, but supporting // hierarchical patterns (a/*) and recursive globs (**). func Match(pattern, name string) (bool, error) { if filepath.Base(pattern) == "**" { return false, GlobLastRecursiveErr } patternDir := pattern[len(pattern)-1] == '/' nameDir := name[len(name)-1] == '/' if patternDir != nameDir { return false, nil } if nameDir { name = name[:len(name)-1] pattern = pattern[:len(pattern)-1] } for { var patternFile, nameFile string pattern, patternFile = saneSplit(pattern) name, nameFile = saneSplit(name) if patternFile == "**" { return matchPrefix(pattern, filepath.Join(name, nameFile)) } if nameFile == "" && patternFile == "" { return true, nil } else if nameFile == "" || patternFile == "" { return false, nil } match, err := filepath.Match(patternFile, nameFile) if err != nil || !match { return match, err } } } // matchPrefix returns true if the beginning of name matches pattern using the same rules as // filepath.Match, but supporting hierarchical patterns (a/*). Recursive globs (**) are not // supported, they should have been handled in Match(). func matchPrefix(pattern, name string) (bool, error) { if len(pattern) > 0 && pattern[0] == '/' { if len(name) > 0 && name[0] == '/' { pattern = pattern[1:] name = name[1:] } else { return false, nil } } for { var patternElem, nameElem string patternElem, pattern = saneSplitFirst(pattern) nameElem, name = saneSplitFirst(name) if patternElem == "." { patternElem = "" } if nameElem == "." { nameElem = "" } if patternElem == "**" { return false, GlobMultipleRecursiveErr } if patternElem == "" { return true, nil } else if nameElem == "" { return false, nil } match, err := filepath.Match(patternElem, nameElem) if err != nil || !match { return match, err } } } func saneSplitFirst(path string) (string, string) { i := strings.IndexRune(path, filepath.Separator) if i < 0 { return path, "" } return path[:i], path[i+1:] } func GlobPatternList(patterns []string, prefix string) (globedList []string, depDirs []string, err error) { var ( matches []string deps []string ) globedList = make([]string, 0) depDirs = make([]string, 0) for _, pattern := range patterns { if isWild(pattern) { matches, deps, err = Glob(filepath.Join(prefix, pattern), nil, FollowSymlinks) if err != nil { return nil, nil, err } globedList = append(globedList, matches...) depDirs = append(depDirs, deps...) } else { globedList = append(globedList, filepath.Join(prefix, pattern)) } } return globedList, depDirs, nil } // IsGlob returns true if the pattern contains any glob characters (*, ?, or [). func IsGlob(pattern string) bool { return strings.IndexAny(pattern, "*?[") >= 0 } // HasGlob returns true if any string in the list contains any glob characters (*, ?, or [). func HasGlob(in []string) bool { for _, s := range in { if IsGlob(s) { return true } } return false } // GlobWithDepFile finds all files and directories that match glob. Directories // will have a trailing '/'. It compares the list of matches against the // contents of fileListFile, and rewrites fileListFile if it has changed. It // also writes all of the the directories it traversed as dependencies on // fileListFile to depFile. // // The format of glob is either path/*.ext for a single directory glob, or // path/**/*.ext for a recursive glob. // // Returns a list of file paths, and an error. // // In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps // should be used instead, as they will automatically set up dependencies // to rerun the primary builder when the list of matching files changes. func GlobWithDepFile(glob, fileListFile, depFile string, excludes []string) (files []string, err error) { files, deps, err := Glob(glob, excludes, FollowSymlinks) if err != nil { return nil, err } fileList := strings.Join(files, "\n") + "\n" WriteFileIfChanged(fileListFile, []byte(fileList), 0666) deptools.WriteDepFile(depFile, fileListFile, deps) return } // WriteFileIfChanged wraps ioutil.WriteFile, but only writes the file if // the files does not already exist with identical contents. This can be used // along with ninja restat rules to skip rebuilding downstream rules if no // changes were made by a rule. func WriteFileIfChanged(filename string, data []byte, perm os.FileMode) error { var isChanged bool dir := filepath.Dir(filename) err := os.MkdirAll(dir, 0777) if err != nil { return err } info, err := os.Stat(filename) if err != nil { if os.IsNotExist(err) { // The file does not exist yet. isChanged = true } else { return err } } else { if info.Size() != int64(len(data)) { isChanged = true } else { oldData, err := ioutil.ReadFile(filename) if err != nil { return err } if len(oldData) != len(data) { isChanged = true } else { for i := range data { if oldData[i] != data[i] { isChanged = true break } } } } } if isChanged { err = ioutil.WriteFile(filename, data, perm) if err != nil { return err } } return nil } var matchEscaper = strings.NewReplacer( `*`, `\*`, `?`, `\?`, `[`, `\[`, `]`, `\]`, ) // MatchEscape returns its inputs with characters that would be interpreted by func MatchEscape(s string) string { return matchEscaper.Replace(s) }