Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 1 | // Copyright 2015 Google Inc. All rights reserved |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
| 15 | package kati |
| 16 | |
| 17 | import ( |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 18 | "fmt" |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 19 | "os/exec" |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 20 | "strings" |
| 21 | "sync" |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame^] | 22 | |
| 23 | "github.com/golang/glog" |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 24 | ) |
| 25 | |
| 26 | type execContext struct { |
| 27 | shell string |
| 28 | |
| 29 | mu sync.Mutex |
| 30 | ev *Evaluator |
| 31 | output string |
| 32 | inputs []string |
| 33 | } |
| 34 | |
| 35 | func newExecContext(vars Vars, avoidIO bool) *execContext { |
| 36 | ev := NewEvaluator(vars) |
| 37 | ev.avoidIO = avoidIO |
| 38 | |
| 39 | ctx := &execContext{ |
| 40 | ev: ev, |
| 41 | } |
| 42 | av := autoVar{ctx: ctx} |
| 43 | for k, v := range map[string]Var{ |
| 44 | "@": autoAtVar{autoVar: av}, |
| 45 | "<": autoLessVar{autoVar: av}, |
| 46 | "^": autoHatVar{autoVar: av}, |
| 47 | "+": autoPlusVar{autoVar: av}, |
| 48 | "*": autoStarVar{autoVar: av}, |
| 49 | } { |
| 50 | ev.vars[k] = v |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 51 | // $<k>D = $(patsubst %/,%,$(dir $<k>)) |
Fumitoshi Ukai | bbb0db5 | 2015-06-30 16:51:27 +0900 | [diff] [blame] | 52 | ev.vars[k+"D"] = suffixDVar(k) |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 53 | // $<k>F = $(notdir $<k>) |
Fumitoshi Ukai | bbb0db5 | 2015-06-30 16:51:27 +0900 | [diff] [blame] | 54 | ev.vars[k+"F"] = suffixFVar(k) |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 55 | } |
| 56 | |
| 57 | // TODO: We should move this to somewhere around evalCmd so that |
| 58 | // we can handle SHELL in target specific variables. |
| 59 | shell, err := ev.EvaluateVar("SHELL") |
| 60 | if err != nil { |
| 61 | shell = "/bin/sh" |
| 62 | } |
| 63 | ctx.shell = shell |
| 64 | return ctx |
| 65 | } |
| 66 | |
| 67 | func (ec *execContext) uniqueInputs() []string { |
| 68 | var uniqueInputs []string |
| 69 | seen := make(map[string]bool) |
| 70 | for _, input := range ec.inputs { |
| 71 | if !seen[input] { |
| 72 | seen[input] = true |
| 73 | uniqueInputs = append(uniqueInputs, input) |
| 74 | } |
| 75 | } |
| 76 | return uniqueInputs |
| 77 | } |
| 78 | |
| 79 | type autoVar struct{ ctx *execContext } |
| 80 | |
| 81 | func (v autoVar) Flavor() string { return "undefined" } |
| 82 | func (v autoVar) Origin() string { return "automatic" } |
| 83 | func (v autoVar) IsDefined() bool { return true } |
| 84 | func (v autoVar) Append(*Evaluator, string) (Var, error) { |
| 85 | return nil, fmt.Errorf("cannot append to autovar") |
| 86 | } |
| 87 | func (v autoVar) AppendVar(*Evaluator, Value) (Var, error) { |
| 88 | return nil, fmt.Errorf("cannot append to autovar") |
| 89 | } |
| 90 | func (v autoVar) serialize() serializableVar { |
| 91 | return serializableVar{Type: ""} |
| 92 | } |
| 93 | func (v autoVar) dump(d *dumpbuf) { |
| 94 | d.err = fmt.Errorf("cannot dump auto var: %v", v) |
| 95 | } |
| 96 | |
| 97 | type autoAtVar struct{ autoVar } |
| 98 | |
Fumitoshi Ukai | b44b12d | 2015-07-07 14:19:32 +0900 | [diff] [blame] | 99 | func (v autoAtVar) Eval(w evalWriter, ev *Evaluator) error { |
Fumitoshi Ukai | bbb0db5 | 2015-06-30 16:51:27 +0900 | [diff] [blame] | 100 | fmt.Fprint(w, v.String()) |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 101 | return nil |
| 102 | } |
Fumitoshi Ukai | bbb0db5 | 2015-06-30 16:51:27 +0900 | [diff] [blame] | 103 | func (v autoAtVar) String() string { return v.ctx.output } |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 104 | |
| 105 | type autoLessVar struct{ autoVar } |
| 106 | |
Fumitoshi Ukai | b44b12d | 2015-07-07 14:19:32 +0900 | [diff] [blame] | 107 | func (v autoLessVar) Eval(w evalWriter, ev *Evaluator) error { |
Fumitoshi Ukai | bbb0db5 | 2015-06-30 16:51:27 +0900 | [diff] [blame] | 108 | fmt.Fprint(w, v.String()) |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 109 | return nil |
| 110 | } |
Fumitoshi Ukai | bbb0db5 | 2015-06-30 16:51:27 +0900 | [diff] [blame] | 111 | func (v autoLessVar) String() string { |
| 112 | if len(v.ctx.inputs) > 0 { |
| 113 | return v.ctx.inputs[0] |
| 114 | } |
| 115 | return "" |
| 116 | } |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 117 | |
| 118 | type autoHatVar struct{ autoVar } |
| 119 | |
Fumitoshi Ukai | b44b12d | 2015-07-07 14:19:32 +0900 | [diff] [blame] | 120 | func (v autoHatVar) Eval(w evalWriter, ev *Evaluator) error { |
Fumitoshi Ukai | bbb0db5 | 2015-06-30 16:51:27 +0900 | [diff] [blame] | 121 | fmt.Fprint(w, v.String()) |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 122 | return nil |
| 123 | } |
Fumitoshi Ukai | bbb0db5 | 2015-06-30 16:51:27 +0900 | [diff] [blame] | 124 | func (v autoHatVar) String() string { |
| 125 | return strings.Join(v.ctx.uniqueInputs(), " ") |
| 126 | } |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 127 | |
| 128 | type autoPlusVar struct{ autoVar } |
| 129 | |
Fumitoshi Ukai | b44b12d | 2015-07-07 14:19:32 +0900 | [diff] [blame] | 130 | func (v autoPlusVar) Eval(w evalWriter, ev *Evaluator) error { |
Fumitoshi Ukai | bbb0db5 | 2015-06-30 16:51:27 +0900 | [diff] [blame] | 131 | fmt.Fprint(w, v.String()) |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 132 | return nil |
| 133 | } |
Fumitoshi Ukai | bbb0db5 | 2015-06-30 16:51:27 +0900 | [diff] [blame] | 134 | func (v autoPlusVar) String() string { return strings.Join(v.ctx.inputs, " ") } |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 135 | |
| 136 | type autoStarVar struct{ autoVar } |
| 137 | |
Fumitoshi Ukai | b44b12d | 2015-07-07 14:19:32 +0900 | [diff] [blame] | 138 | func (v autoStarVar) Eval(w evalWriter, ev *Evaluator) error { |
Fumitoshi Ukai | bbb0db5 | 2015-06-30 16:51:27 +0900 | [diff] [blame] | 139 | fmt.Fprint(w, v.String()) |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 140 | return nil |
| 141 | } |
| 142 | |
Fumitoshi Ukai | bbb0db5 | 2015-06-30 16:51:27 +0900 | [diff] [blame] | 143 | // TODO: Use currentStem. See auto_stem_var.mk |
| 144 | func (v autoStarVar) String() string { return stripExt(v.ctx.output) } |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 145 | |
Fumitoshi Ukai | bbb0db5 | 2015-06-30 16:51:27 +0900 | [diff] [blame] | 146 | func suffixDVar(k string) Var { |
| 147 | return &recursiveVar{ |
| 148 | expr: expr{ |
| 149 | &funcPatsubst{ |
| 150 | fclosure: fclosure{ |
| 151 | args: []Value{ |
| 152 | literal("(patsubst"), |
| 153 | literal("%/"), |
| 154 | literal("%"), |
| 155 | &funcDir{ |
| 156 | fclosure: fclosure{ |
| 157 | args: []Value{ |
| 158 | literal("(dir"), |
| 159 | &varref{ |
| 160 | varname: literal(k), |
| 161 | }, |
| 162 | }, |
| 163 | }, |
| 164 | }, |
| 165 | }, |
| 166 | }, |
| 167 | }, |
| 168 | }, |
| 169 | origin: "automatic", |
| 170 | } |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 171 | } |
| 172 | |
Fumitoshi Ukai | bbb0db5 | 2015-06-30 16:51:27 +0900 | [diff] [blame] | 173 | func suffixFVar(k string) Var { |
| 174 | return &recursiveVar{ |
| 175 | expr: expr{ |
| 176 | &funcNotdir{ |
| 177 | fclosure: fclosure{ |
| 178 | args: []Value{ |
| 179 | literal("(notdir"), |
| 180 | &varref{varname: literal(k)}, |
| 181 | }, |
| 182 | }, |
| 183 | }, |
| 184 | }, |
| 185 | origin: "automatic", |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 186 | } |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 187 | } |
| 188 | |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 189 | // runner is a single shell command invocation. |
| 190 | type runner struct { |
| 191 | output string |
| 192 | cmd string |
| 193 | echo bool |
| 194 | ignoreError bool |
| 195 | shell string |
| 196 | } |
| 197 | |
| 198 | func (r runner) String() string { |
| 199 | cmd := r.cmd |
| 200 | if !r.echo { |
| 201 | cmd = "@" + cmd |
| 202 | } |
| 203 | if r.ignoreError { |
| 204 | cmd = "-" + cmd |
| 205 | } |
| 206 | return cmd |
| 207 | } |
| 208 | |
| 209 | func (r runner) forCmd(s string) runner { |
| 210 | for { |
| 211 | s = trimLeftSpace(s) |
| 212 | if s == "" { |
| 213 | return runner{} |
| 214 | } |
| 215 | switch s[0] { |
| 216 | case '@': |
| 217 | if !DryRunFlag { |
| 218 | r.echo = false |
| 219 | } |
| 220 | s = s[1:] |
| 221 | continue |
| 222 | case '-': |
| 223 | r.ignoreError = true |
| 224 | s = s[1:] |
| 225 | continue |
| 226 | } |
| 227 | break |
| 228 | } |
| 229 | r.cmd = s |
| 230 | return r |
| 231 | } |
| 232 | |
| 233 | func (r runner) eval(ev *Evaluator, s string) ([]runner, error) { |
| 234 | r = r.forCmd(s) |
| 235 | if strings.IndexByte(r.cmd, '$') < 0 { |
| 236 | // fast path |
| 237 | return []runner{r}, nil |
| 238 | } |
| 239 | // TODO(ukai): parse once more earlier? |
Fumitoshi Ukai | e9aa380 | 2015-07-03 11:33:23 +0900 | [diff] [blame] | 240 | expr, _, err := parseExpr([]byte(r.cmd), nil, parseOp{}) |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 241 | if err != nil { |
| 242 | return nil, ev.errorf("parse cmd %q: %v", r.cmd, err) |
| 243 | } |
Fumitoshi Ukai | a4a0225 | 2015-07-09 14:25:18 +0900 | [diff] [blame] | 244 | buf := newEbuf() |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 245 | err = expr.Eval(buf, ev) |
| 246 | if err != nil { |
| 247 | return nil, err |
| 248 | } |
| 249 | cmds := buf.String() |
Fumitoshi Ukai | a4a0225 | 2015-07-09 14:25:18 +0900 | [diff] [blame] | 250 | buf.release() |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame^] | 251 | glog.V(1).Infof("evalcmd: %q => %q", r.cmd, cmds) |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 252 | var runners []runner |
| 253 | for _, cmd := range strings.Split(cmds, "\n") { |
| 254 | if len(runners) > 0 && strings.HasSuffix(runners[len(runners)-1].cmd, "\\") { |
| 255 | runners[len(runners)-1].cmd += "\n" |
| 256 | runners[len(runners)-1].cmd += cmd |
| 257 | continue |
| 258 | } |
| 259 | runners = append(runners, r.forCmd(cmd)) |
| 260 | } |
| 261 | return runners, nil |
| 262 | } |
| 263 | |
| 264 | func (r runner) run(output string) error { |
| 265 | if r.echo || DryRunFlag { |
| 266 | fmt.Printf("%s\n", r.cmd) |
| 267 | } |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame^] | 268 | s := cmdline(r.cmd) |
| 269 | glog.Infof("sh:%q", s) |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 270 | if DryRunFlag { |
| 271 | return nil |
| 272 | } |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame^] | 273 | args := []string{r.shell, "-c", s} |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 274 | cmd := exec.Cmd{ |
| 275 | Path: args[0], |
| 276 | Args: args, |
| 277 | } |
| 278 | out, err := cmd.CombinedOutput() |
| 279 | fmt.Printf("%s", out) |
| 280 | exit := exitStatus(err) |
| 281 | if r.ignoreError && exit != 0 { |
| 282 | fmt.Printf("[%s] Error %d (ignored)\n", output, exit) |
| 283 | err = nil |
| 284 | } |
| 285 | return err |
| 286 | } |
| 287 | |
| 288 | func createRunners(ctx *execContext, n *DepNode) ([]runner, bool, error) { |
| 289 | var runners []runner |
| 290 | if len(n.Cmds) == 0 { |
| 291 | return runners, false, nil |
| 292 | } |
| 293 | |
| 294 | ctx.mu.Lock() |
| 295 | defer ctx.mu.Unlock() |
| 296 | // For automatic variables. |
| 297 | ctx.output = n.Output |
| 298 | ctx.inputs = n.ActualInputs |
| 299 | for k, v := range n.TargetSpecificVars { |
| 300 | restore := ctx.ev.vars.save(k) |
| 301 | defer restore() |
| 302 | ctx.ev.vars[k] = v |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame^] | 303 | glog.Infof("set tsv: %s=%s", k, v) |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 304 | } |
| 305 | |
| 306 | ctx.ev.filename = n.Filename |
| 307 | ctx.ev.lineno = n.Lineno |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame^] | 308 | glog.Infof("Building: %s cmds:%q", n.Output, n.Cmds) |
Fumitoshi Ukai | a70b4ea | 2015-06-30 15:31:49 +0900 | [diff] [blame] | 309 | r := runner{ |
| 310 | output: n.Output, |
| 311 | echo: true, |
| 312 | shell: ctx.shell, |
| 313 | } |
| 314 | for _, cmd := range n.Cmds { |
| 315 | rr, err := r.eval(ctx.ev, cmd) |
| 316 | if err != nil { |
| 317 | return nil, false, err |
| 318 | } |
| 319 | for _, r := range rr { |
| 320 | if len(r.cmd) != 0 { |
| 321 | runners = append(runners, r) |
| 322 | } |
| 323 | } |
| 324 | } |
| 325 | return runners, ctx.ev.hasIO, nil |
| 326 | } |
| 327 | |
| 328 | func evalCommands(nodes []*DepNode, vars Vars) error { |
| 329 | ioCnt := 0 |
| 330 | ectx := newExecContext(vars, true) |
| 331 | for i, n := range nodes { |
| 332 | runners, hasIO, err := createRunners(ectx, n) |
| 333 | if err != nil { |
| 334 | return err |
| 335 | } |
| 336 | if hasIO { |
| 337 | ioCnt++ |
| 338 | if ioCnt%100 == 0 { |
| 339 | logStats("%d/%d rules have IO", ioCnt, i+1) |
| 340 | } |
| 341 | continue |
| 342 | } |
| 343 | |
| 344 | n.Cmds = []string{} |
| 345 | n.TargetSpecificVars = make(Vars) |
| 346 | for _, r := range runners { |
| 347 | n.Cmds = append(n.Cmds, r.String()) |
| 348 | } |
| 349 | } |
| 350 | logStats("%d/%d rules have IO", ioCnt, len(nodes)) |
| 351 | return nil |
| 352 | } |