Colin Cross | 8e0c511 | 2015-01-23 14:15:10 -0800 | [diff] [blame] | 1 | // Copyright 2014 Google Inc. All rights reserved. |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
Jamie Gennis | debef53 | 2014-10-24 10:42:57 -0700 | [diff] [blame] | 15 | package pathtools |
| 16 | |
| 17 | import ( |
Colin Cross | 7bac3c6 | 2015-04-23 15:45:25 -0700 | [diff] [blame] | 18 | "errors" |
Colin Cross | ced59ee | 2016-11-05 08:37:33 -0700 | [diff] [blame] | 19 | "fmt" |
Colin Cross | 127d2ea | 2016-11-01 11:10:51 -0700 | [diff] [blame] | 20 | "io/ioutil" |
Colin Cross | 4b793e5 | 2015-04-23 13:29:28 -0700 | [diff] [blame] | 21 | "os" |
Jamie Gennis | debef53 | 2014-10-24 10:42:57 -0700 | [diff] [blame] | 22 | "path/filepath" |
| 23 | "strings" |
Colin Cross | 127d2ea | 2016-11-01 11:10:51 -0700 | [diff] [blame] | 24 | |
| 25 | "github.com/google/blueprint/deptools" |
Jamie Gennis | debef53 | 2014-10-24 10:42:57 -0700 | [diff] [blame] | 26 | ) |
| 27 | |
Colin Cross | 7bac3c6 | 2015-04-23 15:45:25 -0700 | [diff] [blame] | 28 | var GlobMultipleRecursiveErr = errors.New("pattern contains multiple **") |
| 29 | var GlobLastRecursiveErr = errors.New("pattern ** as last path element") |
| 30 | |
Jamie Gennis | debef53 | 2014-10-24 10:42:57 -0700 | [diff] [blame] | 31 | // Glob returns the list of files that match the given pattern along with the |
| 32 | // list of directories that were searched to construct the file list. |
Colin Cross | 7bac3c6 | 2015-04-23 15:45:25 -0700 | [diff] [blame] | 33 | // The supported glob patterns are equivalent to filepath.Glob, with an |
| 34 | // extension that recursive glob (** matching zero or more complete path |
Colin Cross | 127d2ea | 2016-11-01 11:10:51 -0700 | [diff] [blame] | 35 | // entries) is supported. Glob also returns a list of directories that were |
| 36 | // searched. |
| 37 | // |
| 38 | // In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps |
| 39 | // should be used instead, as they will automatically set up dependencies |
| 40 | // to rerun the primary builder when the list of matching files changes. |
Jamie Gennis | debef53 | 2014-10-24 10:42:57 -0700 | [diff] [blame] | 41 | func Glob(pattern string) (matches, dirs []string, err error) { |
Colin Cross | 5d6d4c7 | 2015-04-23 17:56:11 -0700 | [diff] [blame] | 42 | return GlobWithExcludes(pattern, nil) |
| 43 | } |
| 44 | |
| 45 | // GlobWithExcludes returns the list of files that match the given pattern but |
| 46 | // do not match the given exclude patterns, along with the list of directories |
| 47 | // that were searched to construct the file list. The supported glob and |
| 48 | // exclude patterns are equivalent to filepath.Glob, with an extension that |
| 49 | // recursive glob (** matching zero or more complete path entries) is supported. |
Colin Cross | 08e4954 | 2016-11-14 15:23:33 -0800 | [diff] [blame] | 50 | // GlobWithExcludes also returns a list of directories that were searched. |
Colin Cross | 127d2ea | 2016-11-01 11:10:51 -0700 | [diff] [blame] | 51 | // |
| 52 | // In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps |
| 53 | // should be used instead, as they will automatically set up dependencies |
| 54 | // to rerun the primary builder when the list of matching files changes. |
Colin Cross | 5d6d4c7 | 2015-04-23 17:56:11 -0700 | [diff] [blame] | 55 | func GlobWithExcludes(pattern string, excludes []string) (matches, dirs []string, err error) { |
Colin Cross | 7d2ee0d | 2015-06-18 10:48:15 -0700 | [diff] [blame] | 56 | if filepath.Base(pattern) == "**" { |
Colin Cross | 7bac3c6 | 2015-04-23 15:45:25 -0700 | [diff] [blame] | 57 | return nil, nil, GlobLastRecursiveErr |
| 58 | } else { |
| 59 | matches, dirs, err = glob(pattern, false) |
| 60 | } |
| 61 | |
Colin Cross | 5d6d4c7 | 2015-04-23 17:56:11 -0700 | [diff] [blame] | 62 | if err != nil { |
| 63 | return nil, nil, err |
| 64 | } |
| 65 | |
| 66 | matches, err = filterExcludes(matches, excludes) |
| 67 | if err != nil { |
| 68 | return nil, nil, err |
| 69 | } |
| 70 | |
| 71 | return matches, dirs, nil |
Colin Cross | 7bac3c6 | 2015-04-23 15:45:25 -0700 | [diff] [blame] | 72 | } |
| 73 | |
| 74 | // glob is a recursive helper function to handle globbing each level of the pattern individually, |
| 75 | // allowing searched directories to be tracked. Also handles the recursive glob pattern, **. |
| 76 | func glob(pattern string, hasRecursive bool) (matches, dirs []string, err error) { |
| 77 | if !isWild(pattern) { |
Colin Cross | 7d2ee0d | 2015-06-18 10:48:15 -0700 | [diff] [blame] | 78 | // If there are no wilds in the pattern, check whether the file exists or not. |
| 79 | // Uses filepath.Glob instead of manually statting to get consistent results. |
| 80 | pattern = filepath.Clean(pattern) |
| 81 | matches, err = filepath.Glob(pattern) |
| 82 | if err != nil { |
| 83 | return matches, dirs, err |
| 84 | } |
| 85 | |
| 86 | if len(matches) == 0 { |
| 87 | // Some part of the non-wild pattern didn't exist. Add the last existing directory |
| 88 | // as a dependency. |
| 89 | var matchDirs []string |
| 90 | for len(matchDirs) == 0 { |
| 91 | pattern, _ = saneSplit(pattern) |
| 92 | matchDirs, err = filepath.Glob(pattern) |
| 93 | if err != nil { |
| 94 | return matches, dirs, err |
| 95 | } |
| 96 | } |
| 97 | dirs = append(dirs, matchDirs...) |
| 98 | } |
Colin Cross | 4b793e5 | 2015-04-23 13:29:28 -0700 | [diff] [blame] | 99 | return matches, dirs, err |
Jamie Gennis | debef53 | 2014-10-24 10:42:57 -0700 | [diff] [blame] | 100 | } |
| 101 | |
Colin Cross | 4b793e5 | 2015-04-23 13:29:28 -0700 | [diff] [blame] | 102 | dir, file := saneSplit(pattern) |
Colin Cross | 7bac3c6 | 2015-04-23 15:45:25 -0700 | [diff] [blame] | 103 | |
| 104 | if file == "**" { |
| 105 | if hasRecursive { |
| 106 | return matches, dirs, GlobMultipleRecursiveErr |
| 107 | } |
| 108 | hasRecursive = true |
| 109 | } |
| 110 | |
| 111 | dirMatches, dirs, err := glob(dir, hasRecursive) |
| 112 | if err != nil { |
| 113 | return nil, nil, err |
| 114 | } |
| 115 | |
Colin Cross | 4b793e5 | 2015-04-23 13:29:28 -0700 | [diff] [blame] | 116 | for _, m := range dirMatches { |
Colin Cross | ced59ee | 2016-11-05 08:37:33 -0700 | [diff] [blame] | 117 | info, err := os.Stat(m) |
| 118 | if err != nil { |
| 119 | return nil, nil, fmt.Errorf("unexpected error after glob: %s", err) |
| 120 | } |
| 121 | if info.IsDir() { |
Colin Cross | 7bac3c6 | 2015-04-23 15:45:25 -0700 | [diff] [blame] | 122 | if file == "**" { |
| 123 | recurseDirs, err := walkAllDirs(m) |
| 124 | if err != nil { |
| 125 | return nil, nil, err |
| 126 | } |
| 127 | matches = append(matches, recurseDirs...) |
| 128 | } else { |
| 129 | dirs = append(dirs, m) |
| 130 | newMatches, err := filepath.Glob(filepath.Join(m, file)) |
| 131 | if err != nil { |
| 132 | return nil, nil, err |
| 133 | } |
| 134 | matches = append(matches, newMatches...) |
Jamie Gennis | debef53 | 2014-10-24 10:42:57 -0700 | [diff] [blame] | 135 | } |
| 136 | } |
| 137 | } |
| 138 | |
Colin Cross | 4b793e5 | 2015-04-23 13:29:28 -0700 | [diff] [blame] | 139 | return matches, dirs, nil |
Jamie Gennis | debef53 | 2014-10-24 10:42:57 -0700 | [diff] [blame] | 140 | } |
| 141 | |
Colin Cross | 7bac3c6 | 2015-04-23 15:45:25 -0700 | [diff] [blame] | 142 | // Faster version of dir, file := filepath.Dir(path), filepath.File(path) with no allocations |
Colin Cross | 4b793e5 | 2015-04-23 13:29:28 -0700 | [diff] [blame] | 143 | // Similar to filepath.Split, but returns "." if dir is empty and trims trailing slash if dir is |
Colin Cross | 7bac3c6 | 2015-04-23 15:45:25 -0700 | [diff] [blame] | 144 | // not "/". Returns ".", "" if path is "." |
Colin Cross | 4b793e5 | 2015-04-23 13:29:28 -0700 | [diff] [blame] | 145 | func saneSplit(path string) (dir, file string) { |
Colin Cross | 7bac3c6 | 2015-04-23 15:45:25 -0700 | [diff] [blame] | 146 | if path == "." { |
| 147 | return ".", "" |
| 148 | } |
Colin Cross | 4b793e5 | 2015-04-23 13:29:28 -0700 | [diff] [blame] | 149 | dir, file = filepath.Split(path) |
| 150 | switch dir { |
| 151 | case "": |
| 152 | dir = "." |
| 153 | case "/": |
| 154 | // Nothing |
| 155 | default: |
| 156 | dir = dir[:len(dir)-1] |
Jamie Gennis | debef53 | 2014-10-24 10:42:57 -0700 | [diff] [blame] | 157 | } |
Colin Cross | 4b793e5 | 2015-04-23 13:29:28 -0700 | [diff] [blame] | 158 | return dir, file |
Jamie Gennis | debef53 | 2014-10-24 10:42:57 -0700 | [diff] [blame] | 159 | } |
| 160 | |
| 161 | func isWild(pattern string) bool { |
| 162 | return strings.ContainsAny(pattern, "*?[") |
| 163 | } |
Jean-Francois Dupuis | 418d5c5 | 2015-02-05 20:01:49 -0800 | [diff] [blame] | 164 | |
Colin Cross | 7bac3c6 | 2015-04-23 15:45:25 -0700 | [diff] [blame] | 165 | // Returns a list of all directories under dir |
| 166 | func walkAllDirs(dir string) (dirs []string, err error) { |
| 167 | err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { |
| 168 | if err != nil { |
| 169 | return err |
| 170 | } |
| 171 | |
| 172 | if info.Mode().IsDir() { |
| 173 | dirs = append(dirs, path) |
| 174 | } |
| 175 | return nil |
| 176 | }) |
| 177 | |
| 178 | return dirs, err |
| 179 | } |
| 180 | |
Colin Cross | 5d6d4c7 | 2015-04-23 17:56:11 -0700 | [diff] [blame] | 181 | // Filters the strings in matches based on the glob patterns in excludes. Hierarchical (a/*) and |
| 182 | // recursive (**) glob patterns are supported. |
| 183 | func filterExcludes(matches []string, excludes []string) ([]string, error) { |
| 184 | if len(excludes) == 0 { |
| 185 | return matches, nil |
| 186 | } |
| 187 | |
| 188 | var ret []string |
| 189 | matchLoop: |
| 190 | for _, m := range matches { |
| 191 | for _, e := range excludes { |
| 192 | exclude, err := match(e, m) |
| 193 | if err != nil { |
| 194 | return nil, err |
| 195 | } |
| 196 | if exclude { |
| 197 | continue matchLoop |
| 198 | } |
| 199 | } |
| 200 | ret = append(ret, m) |
| 201 | } |
| 202 | |
| 203 | return ret, nil |
| 204 | } |
| 205 | |
| 206 | // match returns true if name matches pattern using the same rules as filepath.Match, but supporting |
| 207 | // hierarchical patterns (a/*) and recursive globs (**). |
| 208 | func match(pattern, name string) (bool, error) { |
| 209 | if filepath.Base(pattern) == "**" { |
| 210 | return false, GlobLastRecursiveErr |
| 211 | } |
| 212 | |
| 213 | for { |
| 214 | var patternFile, nameFile string |
| 215 | pattern, patternFile = saneSplit(pattern) |
| 216 | name, nameFile = saneSplit(name) |
| 217 | |
| 218 | if patternFile == "**" { |
| 219 | return matchPrefix(pattern, filepath.Join(name, nameFile)) |
| 220 | } |
| 221 | |
| 222 | if nameFile == "" && patternFile == "" { |
| 223 | return true, nil |
| 224 | } else if nameFile == "" || patternFile == "" { |
| 225 | return false, nil |
| 226 | } |
| 227 | |
| 228 | match, err := filepath.Match(patternFile, nameFile) |
| 229 | if err != nil || !match { |
| 230 | return match, err |
| 231 | } |
| 232 | } |
| 233 | } |
| 234 | |
| 235 | // matchPrefix returns true if the beginning of name matches pattern using the same rules as |
| 236 | // filepath.Match, but supporting hierarchical patterns (a/*). Recursive globs (**) are not |
| 237 | // supported, they should have been handled in match(). |
| 238 | func matchPrefix(pattern, name string) (bool, error) { |
| 239 | if len(pattern) > 0 && pattern[0] == '/' { |
| 240 | if len(name) > 0 && name[0] == '/' { |
| 241 | pattern = pattern[1:] |
| 242 | name = name[1:] |
| 243 | } else { |
| 244 | return false, nil |
| 245 | } |
| 246 | } |
| 247 | |
| 248 | for { |
| 249 | var patternElem, nameElem string |
| 250 | patternElem, pattern = saneSplitFirst(pattern) |
| 251 | nameElem, name = saneSplitFirst(name) |
| 252 | |
| 253 | if patternElem == "." { |
| 254 | patternElem = "" |
| 255 | } |
| 256 | if nameElem == "." { |
| 257 | nameElem = "" |
| 258 | } |
| 259 | |
| 260 | if patternElem == "**" { |
| 261 | return false, GlobMultipleRecursiveErr |
| 262 | } |
| 263 | |
| 264 | if patternElem == "" { |
| 265 | return true, nil |
| 266 | } else if nameElem == "" { |
| 267 | return false, nil |
| 268 | } |
| 269 | |
| 270 | match, err := filepath.Match(patternElem, nameElem) |
| 271 | if err != nil || !match { |
| 272 | return match, err |
| 273 | } |
| 274 | } |
| 275 | } |
| 276 | |
| 277 | func saneSplitFirst(path string) (string, string) { |
| 278 | i := strings.IndexRune(path, filepath.Separator) |
| 279 | if i < 0 { |
| 280 | return path, "" |
| 281 | } |
| 282 | return path[:i], path[i+1:] |
| 283 | } |
| 284 | |
Jean-Francois Dupuis | 418d5c5 | 2015-02-05 20:01:49 -0800 | [diff] [blame] | 285 | func GlobPatternList(patterns []string, prefix string) (globedList []string, depDirs []string, err error) { |
| 286 | var ( |
| 287 | matches []string |
Colin Cross | 63d5d4d | 2015-04-20 16:41:55 -0700 | [diff] [blame] | 288 | deps []string |
Jean-Francois Dupuis | 418d5c5 | 2015-02-05 20:01:49 -0800 | [diff] [blame] | 289 | ) |
| 290 | |
| 291 | globedList = make([]string, 0) |
| 292 | depDirs = make([]string, 0) |
| 293 | |
| 294 | for _, pattern := range patterns { |
| 295 | if isWild(pattern) { |
| 296 | matches, deps, err = Glob(filepath.Join(prefix, pattern)) |
| 297 | if err != nil { |
| 298 | return nil, nil, err |
| 299 | } |
| 300 | globedList = append(globedList, matches...) |
| 301 | depDirs = append(depDirs, deps...) |
| 302 | } else { |
| 303 | globedList = append(globedList, filepath.Join(prefix, pattern)) |
| 304 | } |
| 305 | } |
| 306 | return globedList, depDirs, nil |
| 307 | } |
Colin Cross | 127d2ea | 2016-11-01 11:10:51 -0700 | [diff] [blame] | 308 | |
| 309 | // IsGlob returns true if the pattern contains any glob characters (*, ?, or [). |
| 310 | func IsGlob(pattern string) bool { |
| 311 | return strings.IndexAny(pattern, "*?[") >= 0 |
| 312 | } |
| 313 | |
| 314 | // HasGlob returns true if any string in the list contains any glob characters (*, ?, or [). |
| 315 | func HasGlob(in []string) bool { |
| 316 | for _, s := range in { |
| 317 | if IsGlob(s) { |
| 318 | return true |
| 319 | } |
| 320 | } |
| 321 | |
| 322 | return false |
| 323 | } |
| 324 | |
| 325 | // GlobWithDepFile finds all files that match glob. It compares the list of files |
| 326 | // against the contents of fileListFile, and rewrites fileListFile if it has changed. It also |
| 327 | // writes all of the the directories it traversed as a depenencies on fileListFile to depFile. |
| 328 | // |
| 329 | // The format of glob is either path/*.ext for a single directory glob, or path/**/*.ext |
| 330 | // for a recursive glob. |
| 331 | // |
| 332 | // Returns a list of file paths, and an error. |
| 333 | // |
| 334 | // In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps |
| 335 | // should be used instead, as they will automatically set up dependencies |
| 336 | // to rerun the primary builder when the list of matching files changes. |
| 337 | func GlobWithDepFile(glob, fileListFile, depFile string, excludes []string) (files []string, err error) { |
| 338 | files, dirs, err := GlobWithExcludes(glob, excludes) |
| 339 | if err != nil { |
| 340 | return nil, err |
| 341 | } |
| 342 | |
| 343 | fileList := strings.Join(files, "\n") + "\n" |
| 344 | |
| 345 | WriteFileIfChanged(fileListFile, []byte(fileList), 0666) |
| 346 | deptools.WriteDepFile(depFile, fileListFile, dirs) |
| 347 | |
| 348 | return |
| 349 | } |
| 350 | |
| 351 | // WriteFileIfChanged wraps ioutil.WriteFile, but only writes the file if |
| 352 | // the files does not already exist with identical contents. This can be used |
| 353 | // along with ninja restat rules to skip rebuilding downstream rules if no |
| 354 | // changes were made by a rule. |
| 355 | func WriteFileIfChanged(filename string, data []byte, perm os.FileMode) error { |
| 356 | var isChanged bool |
| 357 | |
| 358 | dir := filepath.Dir(filename) |
| 359 | err := os.MkdirAll(dir, 0777) |
| 360 | if err != nil { |
| 361 | return err |
| 362 | } |
| 363 | |
| 364 | info, err := os.Stat(filename) |
| 365 | if err != nil { |
| 366 | if os.IsNotExist(err) { |
| 367 | // The file does not exist yet. |
| 368 | isChanged = true |
| 369 | } else { |
| 370 | return err |
| 371 | } |
| 372 | } else { |
| 373 | if info.Size() != int64(len(data)) { |
| 374 | isChanged = true |
| 375 | } else { |
| 376 | oldData, err := ioutil.ReadFile(filename) |
| 377 | if err != nil { |
| 378 | return err |
| 379 | } |
| 380 | |
| 381 | if len(oldData) != len(data) { |
| 382 | isChanged = true |
| 383 | } else { |
| 384 | for i := range data { |
| 385 | if oldData[i] != data[i] { |
| 386 | isChanged = true |
| 387 | break |
| 388 | } |
| 389 | } |
| 390 | } |
| 391 | } |
| 392 | } |
| 393 | |
| 394 | if isChanged { |
| 395 | err = ioutil.WriteFile(filename, data, perm) |
| 396 | if err != nil { |
| 397 | return err |
| 398 | } |
| 399 | } |
| 400 | |
| 401 | return nil |
| 402 | } |