Shinichiro Hamaji | b69bf8a | 2015-06-10 14:52:06 +0900 | [diff] [blame] | 1 | // Copyright 2015 Google Inc. All rights reserved |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 15 | package kati |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 16 | |
| 17 | import ( |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 18 | "bytes" |
Shinichiro Hamaji | 4220be3 | 2015-05-26 13:30:18 +0900 | [diff] [blame] | 19 | "crypto/sha1" |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 20 | "encoding/binary" |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 21 | "encoding/gob" |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 22 | "encoding/json" |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 23 | "fmt" |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 24 | "io" |
Fumitoshi Ukai | bc40a46 | 2015-06-02 15:51:16 +0900 | [diff] [blame] | 25 | "io/ioutil" |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 26 | "net/url" |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 27 | "os" |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 28 | "sort" |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 29 | "strconv" |
Fumitoshi Ukai | 7655f18 | 2015-07-15 14:05:41 +0900 | [diff] [blame] | 30 | "strings" |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 31 | "time" |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 32 | |
| 33 | "github.com/golang/glog" |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 34 | ) |
| 35 | |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 36 | const ( |
Fumitoshi Ukai | a78fce0 | 2015-06-25 12:29:09 +0900 | [diff] [blame] | 37 | valueTypeRecursive = 'R' |
| 38 | valueTypeSimple = 'S' |
| 39 | valueTypeTSV = 'T' |
| 40 | valueTypeUndefined = 'U' |
| 41 | valueTypeAssign = 'a' |
| 42 | valueTypeExpr = 'e' |
| 43 | valueTypeFunc = 'f' |
| 44 | valueTypeLiteral = 'l' |
| 45 | valueTypeNop = 'n' |
| 46 | valueTypeParamref = 'p' |
| 47 | valueTypeVarref = 'r' |
| 48 | valueTypeVarsubst = 's' |
| 49 | valueTypeTmpval = 't' |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 50 | ) |
| 51 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 52 | // JSON is a json loader/saver. |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 53 | var JSON LoadSaver |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 54 | |
| 55 | // GOB is a gob loader/saver. |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 56 | var GOB LoadSaver |
| 57 | |
| 58 | func init() { |
| 59 | JSON = jsonLoadSaver{} |
| 60 | GOB = gobLoadSaver{} |
| 61 | } |
| 62 | |
| 63 | type jsonLoadSaver struct{} |
| 64 | type gobLoadSaver struct{} |
| 65 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 66 | type dumpbuf struct { |
| 67 | w bytes.Buffer |
| 68 | err error |
| 69 | } |
| 70 | |
| 71 | func (d *dumpbuf) Int(i int) { |
| 72 | if d.err != nil { |
| 73 | return |
| 74 | } |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 75 | v := int32(i) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 76 | d.err = binary.Write(&d.w, binary.LittleEndian, &v) |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 77 | } |
| 78 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 79 | func (d *dumpbuf) Str(s string) { |
| 80 | if d.err != nil { |
| 81 | return |
Fumitoshi Ukai | 145598a | 2015-06-19 10:08:17 +0900 | [diff] [blame] | 82 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 83 | d.Int(len(s)) |
| 84 | if d.err != nil { |
| 85 | return |
| 86 | } |
| 87 | _, d.err = io.WriteString(&d.w, s) |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 88 | } |
| 89 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 90 | func (d *dumpbuf) Bytes(b []byte) { |
| 91 | if d.err != nil { |
| 92 | return |
Fumitoshi Ukai | 145598a | 2015-06-19 10:08:17 +0900 | [diff] [blame] | 93 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 94 | d.Int(len(b)) |
| 95 | if d.err != nil { |
| 96 | return |
| 97 | } |
| 98 | _, d.err = d.w.Write(b) |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 99 | } |
| 100 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 101 | func (d *dumpbuf) Byte(b byte) { |
| 102 | if d.err != nil { |
| 103 | return |
Fumitoshi Ukai | 145598a | 2015-06-19 10:08:17 +0900 | [diff] [blame] | 104 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 105 | d.err = writeByte(&d.w, b) |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 106 | } |
| 107 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 108 | type serializableVar struct { |
Shinichiro Hamaji | c8bc731 | 2015-04-28 02:48:03 +0900 | [diff] [blame] | 109 | Type string |
| 110 | V string |
| 111 | Origin string |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 112 | Children []serializableVar |
Shinichiro Hamaji | c8bc731 | 2015-04-28 02:48:03 +0900 | [diff] [blame] | 113 | } |
| 114 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 115 | type serializableDepNode struct { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 116 | Output int |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 117 | Cmds []string |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 118 | Deps []int |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 119 | OrderOnlys []int |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 120 | Parents []int |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 121 | HasRule bool |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 122 | IsPhony bool |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 123 | ActualInputs []int |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 124 | TargetSpecificVars []int |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 125 | Filename string |
| 126 | Lineno int |
| 127 | } |
| 128 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 129 | type serializableTargetSpecificVar struct { |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 130 | Name string |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 131 | Value serializableVar |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 132 | } |
| 133 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 134 | type serializableGraph struct { |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 135 | Nodes []*serializableDepNode |
| 136 | Vars map[string]serializableVar |
| 137 | Tsvs []serializableTargetSpecificVar |
| 138 | Targets []string |
| 139 | Roots []string |
| 140 | AccessedMks []*accessedMakefile |
| 141 | Exports map[string]bool |
Shinichiro Hamaji | c8bc731 | 2015-04-28 02:48:03 +0900 | [diff] [blame] | 142 | } |
| 143 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 144 | func encGob(v interface{}) (string, error) { |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 145 | var buf bytes.Buffer |
| 146 | e := gob.NewEncoder(&buf) |
| 147 | err := e.Encode(v) |
| 148 | if err != nil { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 149 | return "", err |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 150 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 151 | return buf.String(), nil |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 152 | } |
| 153 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 154 | func encVar(k string, v Var) (string, error) { |
| 155 | var dump dumpbuf |
| 156 | dump.Str(k) |
| 157 | v.dump(&dump) |
| 158 | return dump.w.String(), dump.err |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 159 | } |
| 160 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 161 | type depNodesSerializer struct { |
| 162 | nodes []*serializableDepNode |
| 163 | tsvs []serializableTargetSpecificVar |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 164 | tsvMap map[string]int |
| 165 | targets []string |
| 166 | targetMap map[string]int |
| 167 | done map[string]bool |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 168 | err error |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 169 | } |
| 170 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 171 | func newDepNodesSerializer() *depNodesSerializer { |
| 172 | return &depNodesSerializer{ |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 173 | tsvMap: make(map[string]int), |
| 174 | targetMap: make(map[string]int), |
| 175 | done: make(map[string]bool), |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 176 | } |
| 177 | } |
| 178 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 179 | func (ns *depNodesSerializer) serializeTarget(t string) int { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 180 | id, present := ns.targetMap[t] |
| 181 | if present { |
| 182 | return id |
| 183 | } |
| 184 | id = len(ns.targets) |
| 185 | ns.targetMap[t] = id |
| 186 | ns.targets = append(ns.targets, t) |
| 187 | return id |
| 188 | } |
| 189 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 190 | func (ns *depNodesSerializer) serializeDepNodes(nodes []*DepNode) { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 191 | if ns.err != nil { |
| 192 | return |
| 193 | } |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 194 | for _, n := range nodes { |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 195 | if ns.done[n.Output] { |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 196 | continue |
| 197 | } |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 198 | ns.done[n.Output] = true |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 199 | |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 200 | var deps []int |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 201 | for _, d := range n.Deps { |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 202 | deps = append(deps, ns.serializeTarget(d.Output)) |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 203 | } |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 204 | var orderonlys []int |
| 205 | for _, d := range n.OrderOnlys { |
| 206 | orderonlys = append(orderonlys, ns.serializeTarget(d.Output)) |
| 207 | } |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 208 | var parents []int |
| 209 | for _, d := range n.Parents { |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 210 | parents = append(parents, ns.serializeTarget(d.Output)) |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 211 | } |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 212 | var actualInputs []int |
| 213 | for _, i := range n.ActualInputs { |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 214 | actualInputs = append(actualInputs, ns.serializeTarget(i)) |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 215 | } |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 216 | |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 217 | // Sort keys for consistent serialization. |
| 218 | var tsvKeys []string |
| 219 | for k := range n.TargetSpecificVars { |
| 220 | tsvKeys = append(tsvKeys, k) |
| 221 | } |
| 222 | sort.Strings(tsvKeys) |
| 223 | |
| 224 | var vars []int |
| 225 | for _, k := range tsvKeys { |
| 226 | v := n.TargetSpecificVars[k] |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 227 | sv := serializableTargetSpecificVar{Name: k, Value: v.serialize()} |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 228 | //gob := encGob(sv) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 229 | gob, err := encVar(k, v) |
| 230 | if err != nil { |
| 231 | ns.err = err |
| 232 | return |
| 233 | } |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 234 | id, present := ns.tsvMap[gob] |
| 235 | if !present { |
| 236 | id = len(ns.tsvs) |
| 237 | ns.tsvMap[gob] = id |
| 238 | ns.tsvs = append(ns.tsvs, sv) |
| 239 | } |
| 240 | vars = append(vars, id) |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 241 | } |
| 242 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 243 | ns.nodes = append(ns.nodes, &serializableDepNode{ |
| 244 | Output: ns.serializeTarget(n.Output), |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 245 | Cmds: n.Cmds, |
| 246 | Deps: deps, |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 247 | OrderOnlys: orderonlys, |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 248 | Parents: parents, |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 249 | HasRule: n.HasRule, |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 250 | IsPhony: n.IsPhony, |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 251 | ActualInputs: actualInputs, |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 252 | TargetSpecificVars: vars, |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 253 | Filename: n.Filename, |
| 254 | Lineno: n.Lineno, |
| 255 | }) |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 256 | ns.serializeDepNodes(n.Deps) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 257 | if ns.err != nil { |
| 258 | return |
| 259 | } |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 260 | ns.serializeDepNodes(n.OrderOnlys) |
| 261 | if ns.err != nil { |
| 262 | return |
| 263 | } |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 264 | } |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 265 | } |
| 266 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 267 | func makeSerializableVars(vars Vars) (r map[string]serializableVar) { |
| 268 | r = make(map[string]serializableVar) |
Shinichiro Hamaji | c8bc731 | 2015-04-28 02:48:03 +0900 | [diff] [blame] | 269 | for k, v := range vars { |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 270 | r[k] = v.serialize() |
Shinichiro Hamaji | c8bc731 | 2015-04-28 02:48:03 +0900 | [diff] [blame] | 271 | } |
| 272 | return r |
| 273 | } |
| 274 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 275 | func makeSerializableGraph(g *DepGraph, roots []string) (serializableGraph, error) { |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 276 | ns := newDepNodesSerializer() |
| 277 | ns.serializeDepNodes(g.nodes) |
| 278 | v := makeSerializableVars(g.vars) |
| 279 | return serializableGraph{ |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 280 | Nodes: ns.nodes, |
| 281 | Vars: v, |
| 282 | Tsvs: ns.tsvs, |
| 283 | Targets: ns.targets, |
| 284 | Roots: roots, |
| 285 | AccessedMks: g.accessedMks, |
| 286 | Exports: g.exports, |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 287 | }, ns.err |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 288 | } |
Shinichiro Hamaji | c8bc731 | 2015-04-28 02:48:03 +0900 | [diff] [blame] | 289 | |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 290 | func (jsonLoadSaver) Save(g *DepGraph, filename string, roots []string) error { |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 291 | startTime := time.Now() |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 292 | sg, err := makeSerializableGraph(g, roots) |
| 293 | if err != nil { |
| 294 | return err |
| 295 | } |
Shinichiro Hamaji | 750ae2e | 2015-05-20 12:59:15 +0900 | [diff] [blame] | 296 | o, err := json.MarshalIndent(sg, " ", " ") |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 297 | if err != nil { |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 298 | return err |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 299 | } |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 300 | f, err := os.Create(filename) |
| 301 | if err != nil { |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 302 | return err |
| 303 | } |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 304 | _, err = f.Write(o) |
| 305 | if err != nil { |
| 306 | f.Close() |
| 307 | return err |
| 308 | } |
| 309 | err = f.Close() |
| 310 | if err != nil { |
| 311 | return err |
| 312 | } |
| 313 | logStats("json serialize time: %q", time.Since(startTime)) |
| 314 | return nil |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 315 | } |
| 316 | |
| 317 | func (gobLoadSaver) Save(g *DepGraph, filename string, roots []string) error { |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 318 | startTime := time.Now() |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 319 | f, err := os.Create(filename) |
| 320 | if err != nil { |
| 321 | return err |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 322 | } |
| 323 | e := gob.NewEncoder(f) |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 324 | var sg serializableGraph |
| 325 | { |
| 326 | startTime := time.Now() |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 327 | sg, err = makeSerializableGraph(g, roots) |
| 328 | if err != nil { |
| 329 | return err |
| 330 | } |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 331 | logStats("gob serialize prepare time: %q", time.Since(startTime)) |
| 332 | } |
| 333 | { |
| 334 | startTime := time.Now() |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 335 | err = e.Encode(sg) |
| 336 | if err != nil { |
| 337 | return err |
| 338 | } |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 339 | logStats("gob serialize output time: %q", time.Since(startTime)) |
| 340 | } |
| 341 | err = f.Close() |
| 342 | if err != nil { |
| 343 | return err |
| 344 | } |
| 345 | logStats("gob serialize time: %q", time.Since(startTime)) |
| 346 | return nil |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 347 | } |
| 348 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 349 | func cacheFilename(mk string, roots []string) string { |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 350 | filename := ".kati_cache." + mk |
| 351 | for _, r := range roots { |
| 352 | filename += "." + r |
| 353 | } |
| 354 | return url.QueryEscape(filename) |
| 355 | } |
| 356 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 357 | func saveCache(g *DepGraph, roots []string) error { |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 358 | if len(g.accessedMks) == 0 { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 359 | return fmt.Errorf("no Makefile is read") |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 360 | } |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 361 | cacheFile := cacheFilename(g.accessedMks[0].Filename, roots) |
| 362 | for _, mk := range g.accessedMks { |
Shinichiro Hamaji | f3ea5d0 | 2015-05-20 17:13:00 +0900 | [diff] [blame] | 363 | // Inconsistent, do not dump this result. |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 364 | if mk.State == fileInconsistent { |
Shinichiro Hamaji | f3ea5d0 | 2015-05-20 17:13:00 +0900 | [diff] [blame] | 365 | if exists(cacheFile) { |
| 366 | os.Remove(cacheFile) |
| 367 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 368 | return nil |
Shinichiro Hamaji | f3ea5d0 | 2015-05-20 17:13:00 +0900 | [diff] [blame] | 369 | } |
| 370 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 371 | return GOB.Save(g, cacheFile, roots) |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 372 | } |
| 373 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 374 | func deserializeSingleChild(sv serializableVar) (Value, error) { |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 375 | if len(sv.Children) != 1 { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 376 | return nil, fmt.Errorf("unexpected number of children: %q", sv) |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 377 | } |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 378 | return deserializeVar(sv.Children[0]) |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 379 | } |
| 380 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 381 | func deserializeVar(sv serializableVar) (r Value, err error) { |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 382 | switch sv.Type { |
| 383 | case "literal": |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 384 | return literal(sv.V), nil |
Shinichiro Hamaji | c8dea2a | 2015-04-28 17:15:30 +0900 | [diff] [blame] | 385 | case "tmpval": |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 386 | return tmpval([]byte(sv.V)), nil |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 387 | case "expr": |
Fumitoshi Ukai | 55c8fa9 | 2015-06-25 15:56:10 +0900 | [diff] [blame] | 388 | var e expr |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 389 | for _, v := range sv.Children { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 390 | dv, err := deserializeVar(v) |
| 391 | if err != nil { |
| 392 | return nil, err |
| 393 | } |
| 394 | e = append(e, dv) |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 395 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 396 | return e, nil |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 397 | case "varref": |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 398 | dv, err := deserializeSingleChild(sv) |
| 399 | if err != nil { |
| 400 | return nil, err |
| 401 | } |
Fumitoshi Ukai | bbb0db5 | 2015-06-30 16:51:27 +0900 | [diff] [blame] | 402 | return &varref{varname: dv, paren: sv.V[0]}, nil |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 403 | case "paramref": |
| 404 | v, err := strconv.Atoi(sv.V) |
| 405 | if err != nil { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 406 | return nil, err |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 407 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 408 | return paramref(v), nil |
Shinichiro Hamaji | c8dea2a | 2015-04-28 17:15:30 +0900 | [diff] [blame] | 409 | case "varsubst": |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 410 | varname, err := deserializeVar(sv.Children[0]) |
| 411 | if err != nil { |
| 412 | return nil, err |
Shinichiro Hamaji | c8dea2a | 2015-04-28 17:15:30 +0900 | [diff] [blame] | 413 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 414 | pat, err := deserializeVar(sv.Children[1]) |
| 415 | if err != nil { |
| 416 | return nil, err |
| 417 | } |
| 418 | subst, err := deserializeVar(sv.Children[2]) |
| 419 | if err != nil { |
| 420 | return nil, err |
| 421 | } |
| 422 | return varsubst{ |
| 423 | varname: varname, |
| 424 | pat: pat, |
| 425 | subst: subst, |
Fumitoshi Ukai | bbb0db5 | 2015-06-30 16:51:27 +0900 | [diff] [blame] | 426 | paren: sv.V[0], |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 427 | }, nil |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 428 | |
| 429 | case "func": |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 430 | dv, err := deserializeVar(sv.Children[0]) |
| 431 | if err != nil { |
| 432 | return nil, err |
| 433 | } |
| 434 | name, ok := dv.(literal) |
| 435 | if !ok { |
| 436 | return nil, fmt.Errorf("func name is not literal %s: %T", dv, dv) |
| 437 | } |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 438 | f := funcMap[string(name[1:])]() |
| 439 | f.AddArg(name) |
| 440 | for _, a := range sv.Children[1:] { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 441 | dv, err := deserializeVar(a) |
| 442 | if err != nil { |
| 443 | return nil, err |
| 444 | } |
| 445 | f.AddArg(dv) |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 446 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 447 | return f, nil |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 448 | case "funcEvalAssign": |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 449 | rhs, err := deserializeVar(sv.Children[2]) |
| 450 | if err != nil { |
| 451 | return nil, err |
| 452 | } |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 453 | return &funcEvalAssign{ |
| 454 | lhs: sv.Children[0].V, |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 455 | op: sv.Children[1].V, |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 456 | rhs: rhs, |
| 457 | }, nil |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 458 | case "funcNop": |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 459 | return &funcNop{expr: sv.V}, nil |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 460 | |
| 461 | case "simple": |
Fumitoshi Ukai | 7bf992d | 2015-06-25 12:42:19 +0900 | [diff] [blame] | 462 | return &simpleVar{ |
Fumitoshi Ukai | 7655f18 | 2015-07-15 14:05:41 +0900 | [diff] [blame] | 463 | value: strings.Split(sv.V, " "), |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 464 | origin: sv.Origin, |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 465 | }, nil |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 466 | case "recursive": |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 467 | expr, err := deserializeSingleChild(sv) |
| 468 | if err != nil { |
| 469 | return nil, err |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 470 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 471 | return &recursiveVar{ |
| 472 | expr: expr, |
| 473 | origin: sv.Origin, |
| 474 | }, nil |
Shinichiro Hamaji | f8f075c | 2015-04-28 04:25:37 +0900 | [diff] [blame] | 475 | |
| 476 | case ":=", "=", "+=", "?=": |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 477 | dv, err := deserializeSingleChild(sv) |
| 478 | if err != nil { |
| 479 | return nil, err |
Shinichiro Hamaji | f8f075c | 2015-04-28 04:25:37 +0900 | [diff] [blame] | 480 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 481 | v, ok := dv.(Var) |
| 482 | if !ok { |
| 483 | return nil, fmt.Errorf("not var: target specific var %s %T", dv, dv) |
| 484 | } |
| 485 | return &targetSpecificVar{ |
| 486 | v: v, |
| 487 | op: sv.Type, |
| 488 | }, nil |
Shinichiro Hamaji | f8f075c | 2015-04-28 04:25:37 +0900 | [diff] [blame] | 489 | |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 490 | default: |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 491 | return nil, fmt.Errorf("unknown serialized variable type: %q", sv) |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 492 | } |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 493 | } |
| 494 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 495 | func deserializeVars(vars map[string]serializableVar) (Vars, error) { |
Shinichiro Hamaji | 79a5693 | 2015-04-28 03:53:57 +0900 | [diff] [blame] | 496 | r := make(Vars) |
| 497 | for k, v := range vars { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 498 | dv, err := deserializeVar(v) |
| 499 | if err != nil { |
| 500 | return nil, err |
| 501 | } |
| 502 | vv, ok := dv.(Var) |
| 503 | if !ok { |
| 504 | return nil, fmt.Errorf("not var: %s: %T", dv, dv) |
| 505 | } |
| 506 | r[k] = vv |
Shinichiro Hamaji | 79a5693 | 2015-04-28 03:53:57 +0900 | [diff] [blame] | 507 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 508 | return r, nil |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 509 | } |
| 510 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 511 | func deserializeNodes(g serializableGraph) (r []*DepNode, err error) { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 512 | nodes := g.Nodes |
| 513 | tsvs := g.Tsvs |
| 514 | targets := g.Targets |
Shinichiro Hamaji | 7a65e68 | 2015-04-28 18:00:12 +0900 | [diff] [blame] | 515 | // Deserialize all TSVs first so that multiple rules can share memory. |
| 516 | var tsvValues []Var |
| 517 | for _, sv := range tsvs { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 518 | dv, err := deserializeVar(sv.Value) |
| 519 | if err != nil { |
| 520 | return nil, err |
| 521 | } |
| 522 | vv, ok := dv.(Var) |
| 523 | if !ok { |
| 524 | return nil, fmt.Errorf("not var: %s %T", dv, dv) |
| 525 | } |
| 526 | tsvValues = append(tsvValues, vv) |
Shinichiro Hamaji | 7a65e68 | 2015-04-28 18:00:12 +0900 | [diff] [blame] | 527 | } |
| 528 | |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 529 | nodeMap := make(map[string]*DepNode) |
| 530 | for _, n := range nodes { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 531 | var actualInputs []string |
| 532 | for _, i := range n.ActualInputs { |
| 533 | actualInputs = append(actualInputs, targets[i]) |
| 534 | } |
| 535 | |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 536 | d := &DepNode{ |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 537 | Output: targets[n.Output], |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 538 | Cmds: n.Cmds, |
| 539 | HasRule: n.HasRule, |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 540 | IsPhony: n.IsPhony, |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 541 | ActualInputs: actualInputs, |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 542 | Filename: n.Filename, |
| 543 | Lineno: n.Lineno, |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 544 | TargetSpecificVars: make(Vars), |
| 545 | } |
| 546 | |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 547 | for _, id := range n.TargetSpecificVars { |
| 548 | sv := tsvs[id] |
Shinichiro Hamaji | 7a65e68 | 2015-04-28 18:00:12 +0900 | [diff] [blame] | 549 | d.TargetSpecificVars[sv.Name] = tsvValues[id] |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 550 | } |
| 551 | |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 552 | nodeMap[targets[n.Output]] = d |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 553 | r = append(r, d) |
| 554 | } |
| 555 | |
| 556 | for _, n := range nodes { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 557 | d := nodeMap[targets[n.Output]] |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 558 | for _, o := range n.Deps { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 559 | c, present := nodeMap[targets[o]] |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 560 | if !present { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 561 | return nil, fmt.Errorf("unknown target: %d (%s)", o, targets[o]) |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 562 | } |
| 563 | d.Deps = append(d.Deps, c) |
| 564 | } |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 565 | for _, o := range n.OrderOnlys { |
| 566 | c, present := nodeMap[targets[o]] |
| 567 | if !present { |
| 568 | return nil, fmt.Errorf("unknown target: %d (%s)", o, targets[o]) |
| 569 | } |
| 570 | d.OrderOnlys = append(d.OrderOnlys, c) |
| 571 | } |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 572 | for _, o := range n.Parents { |
| 573 | c, present := nodeMap[targets[o]] |
| 574 | if !present { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 575 | return nil, fmt.Errorf("unknown target: %d (%s)", o, targets[o]) |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 576 | } |
| 577 | d.Parents = append(d.Parents, c) |
| 578 | } |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 579 | } |
| 580 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 581 | return r, nil |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 582 | } |
| 583 | |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 584 | func human(n int) string { |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 585 | if n >= 10*1000*1000*1000 { |
| 586 | return fmt.Sprintf("%.2fGB", float32(n)/1000/1000/1000) |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 587 | } |
Fumitoshi Ukai | 936de10 | 2015-06-08 11:21:16 +0900 | [diff] [blame] | 588 | if n >= 10*1000*1000 { |
| 589 | return fmt.Sprintf("%.2fMB", float32(n)/1000/1000) |
| 590 | } |
| 591 | if n >= 10*1000 { |
| 592 | return fmt.Sprintf("%.2fkB", float32(n)/1000) |
| 593 | } |
| 594 | return fmt.Sprintf("%dB", n) |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 595 | } |
| 596 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 597 | func showSerializedNodesStats(nodes []*serializableDepNode) { |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 598 | outputSize := 0 |
| 599 | cmdSize := 0 |
| 600 | depsSize := 0 |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 601 | orderOnlysSize := 0 |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 602 | actualInputSize := 0 |
| 603 | tsvSize := 0 |
| 604 | filenameSize := 0 |
| 605 | linenoSize := 0 |
| 606 | for _, n := range nodes { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 607 | outputSize += 4 |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 608 | for _, c := range n.Cmds { |
| 609 | cmdSize += len(c) |
| 610 | } |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 611 | depsSize += 4 * len(n.Deps) |
| 612 | orderOnlysSize += 4 * len(n.OrderOnlys) |
| 613 | actualInputSize += 4 * len(n.ActualInputs) |
| 614 | tsvSize += 4 * len(n.TargetSpecificVars) |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 615 | filenameSize += len(n.Filename) |
| 616 | linenoSize += 4 |
| 617 | } |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 618 | size := outputSize + cmdSize + depsSize + orderOnlysSize + actualInputSize + tsvSize + filenameSize + linenoSize |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 619 | logStats("%d nodes %s", len(nodes), human(size)) |
| 620 | logStats(" output %s", human(outputSize)) |
| 621 | logStats(" command %s", human(cmdSize)) |
| 622 | logStats(" deps %s", human(depsSize)) |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 623 | logStats(" orderonlys %s", human(orderOnlysSize)) |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 624 | logStats(" inputs %s", human(actualInputSize)) |
| 625 | logStats(" tsv %s", human(tsvSize)) |
| 626 | logStats(" filename %s", human(filenameSize)) |
| 627 | logStats(" lineno %s", human(linenoSize)) |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 628 | } |
| 629 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 630 | func (v serializableVar) size() int { |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 631 | size := 0 |
| 632 | size += len(v.Type) |
| 633 | size += len(v.V) |
| 634 | size += len(v.Origin) |
| 635 | for _, c := range v.Children { |
| 636 | size += c.size() |
| 637 | } |
| 638 | return size |
| 639 | } |
| 640 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 641 | func showSerializedVarsStats(vars map[string]serializableVar) { |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 642 | nameSize := 0 |
| 643 | valueSize := 0 |
| 644 | for k, v := range vars { |
| 645 | nameSize += len(k) |
| 646 | valueSize += v.size() |
| 647 | } |
| 648 | size := nameSize + valueSize |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 649 | logStats("%d vars %s", len(vars), human(size)) |
| 650 | logStats(" name %s", human(nameSize)) |
| 651 | logStats(" value %s", human(valueSize)) |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 652 | } |
| 653 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 654 | func showSerializedTsvsStats(vars []serializableTargetSpecificVar) { |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 655 | nameSize := 0 |
| 656 | valueSize := 0 |
| 657 | for _, v := range vars { |
| 658 | nameSize += len(v.Name) |
| 659 | valueSize += v.Value.size() |
| 660 | } |
| 661 | size := nameSize + valueSize |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 662 | logStats("%d tsvs %s", len(vars), human(size)) |
| 663 | logStats(" name %s", human(nameSize)) |
| 664 | logStats(" value %s", human(valueSize)) |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 665 | } |
| 666 | |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 667 | func showSerializedTargetsStats(targets []string) { |
| 668 | size := 0 |
| 669 | for _, t := range targets { |
| 670 | size += len(t) |
| 671 | } |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 672 | logStats("%d targets %s", len(targets), human(size)) |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 673 | } |
| 674 | |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 675 | func showSerializedAccessedMksStats(accessedMks []*accessedMakefile) { |
Shinichiro Hamaji | 006adbf | 2015-05-25 18:42:12 +0900 | [diff] [blame] | 676 | size := 0 |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 677 | for _, rm := range accessedMks { |
Shinichiro Hamaji | 4220be3 | 2015-05-26 13:30:18 +0900 | [diff] [blame] | 678 | size += len(rm.Filename) + len(rm.Hash) + 4 |
Shinichiro Hamaji | 006adbf | 2015-05-25 18:42:12 +0900 | [diff] [blame] | 679 | } |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 680 | logStats("%d makefiles %s", len(accessedMks), human(size)) |
Shinichiro Hamaji | 006adbf | 2015-05-25 18:42:12 +0900 | [diff] [blame] | 681 | } |
| 682 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 683 | func showSerializedGraphStats(g serializableGraph) { |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 684 | showSerializedNodesStats(g.Nodes) |
| 685 | showSerializedVarsStats(g.Vars) |
| 686 | showSerializedTsvsStats(g.Tsvs) |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 687 | showSerializedTargetsStats(g.Targets) |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 688 | showSerializedAccessedMksStats(g.AccessedMks) |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 689 | } |
| 690 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 691 | func deserializeGraph(g serializableGraph) (*DepGraph, error) { |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 692 | if StatsFlag { |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 693 | showSerializedGraphStats(g) |
| 694 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 695 | nodes, err := deserializeNodes(g) |
| 696 | if err != nil { |
| 697 | return nil, err |
| 698 | } |
| 699 | vars, err := deserializeVars(g.Vars) |
| 700 | if err != nil { |
| 701 | return nil, err |
| 702 | } |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 703 | return &DepGraph{ |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 704 | nodes: nodes, |
| 705 | vars: vars, |
| 706 | accessedMks: g.AccessedMks, |
| 707 | exports: g.Exports, |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 708 | }, nil |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 709 | } |
| 710 | |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 711 | func (jsonLoadSaver) Load(filename string) (*DepGraph, error) { |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 712 | startTime := time.Now() |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 713 | f, err := os.Open(filename) |
| 714 | if err != nil { |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 715 | return nil, err |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 716 | } |
Fumitoshi Ukai | ed9a615 | 2015-06-02 15:28:55 +0900 | [diff] [blame] | 717 | defer f.Close() |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 718 | |
| 719 | d := json.NewDecoder(f) |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 720 | g := serializableGraph{Vars: make(map[string]serializableVar)} |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 721 | err = d.Decode(&g) |
| 722 | if err != nil { |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 723 | return nil, err |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 724 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 725 | dg, err := deserializeGraph(g) |
| 726 | if err != nil { |
| 727 | return nil, err |
| 728 | } |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 729 | logStats("gob deserialize time: %q", time.Since(startTime)) |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 730 | return dg, nil |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 731 | } |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 732 | |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 733 | func (gobLoadSaver) Load(filename string) (*DepGraph, error) { |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 734 | startTime := time.Now() |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 735 | f, err := os.Open(filename) |
| 736 | if err != nil { |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 737 | return nil, err |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 738 | } |
Fumitoshi Ukai | ed9a615 | 2015-06-02 15:28:55 +0900 | [diff] [blame] | 739 | defer f.Close() |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 740 | |
| 741 | d := gob.NewDecoder(f) |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 742 | g := serializableGraph{Vars: make(map[string]serializableVar)} |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 743 | err = d.Decode(&g) |
| 744 | if err != nil { |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 745 | return nil, err |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 746 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 747 | dg, err := deserializeGraph(g) |
| 748 | if err != nil { |
| 749 | return nil, err |
| 750 | } |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 751 | logStats("json deserialize time: %q", time.Since(startTime)) |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 752 | return dg, nil |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 753 | } |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 754 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 755 | func loadCache(makefile string, roots []string) (*DepGraph, error) { |
Shinichiro Hamaji | 82ecff9 | 2015-05-25 18:36:04 +0900 | [diff] [blame] | 756 | startTime := time.Now() |
| 757 | defer func() { |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 758 | logStats("Cache lookup time: %q", time.Since(startTime)) |
Shinichiro Hamaji | 82ecff9 | 2015-05-25 18:36:04 +0900 | [diff] [blame] | 759 | }() |
| 760 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 761 | filename := cacheFilename(makefile, roots) |
Shinichiro Hamaji | 39e6240 | 2015-05-20 19:34:58 +0900 | [diff] [blame] | 762 | if !exists(filename) { |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 763 | glog.Warningf("Cache not found %q", filename) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 764 | return nil, fmt.Errorf("cache not found: %s", filename) |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 765 | } |
Shinichiro Hamaji | 39e6240 | 2015-05-20 19:34:58 +0900 | [diff] [blame] | 766 | |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 767 | g, err := GOB.Load(filename) |
| 768 | if err != nil { |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 769 | glog.Warning("Cache load error %q: %v", filename, err) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 770 | return nil, err |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 771 | } |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 772 | for _, mk := range g.accessedMks { |
| 773 | if mk.State != fileExists && mk.State != fileNotExists { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 774 | return nil, fmt.Errorf("internal error: broken state: %d", mk.State) |
Shinichiro Hamaji | 0ba75a9 | 2015-05-20 19:42:43 +0900 | [diff] [blame] | 775 | } |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 776 | if mk.State == fileNotExists { |
Shinichiro Hamaji | 45cde1d | 2015-05-25 18:21:23 +0900 | [diff] [blame] | 777 | if exists(mk.Filename) { |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 778 | glog.Infof("Cache expired: %s", mk.Filename) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 779 | return nil, fmt.Errorf("cache expired: %s", mk.Filename) |
Shinichiro Hamaji | 45cde1d | 2015-05-25 18:21:23 +0900 | [diff] [blame] | 780 | } |
| 781 | } else { |
Fumitoshi Ukai | bc40a46 | 2015-06-02 15:51:16 +0900 | [diff] [blame] | 782 | c, err := ioutil.ReadFile(mk.Filename) |
Shinichiro Hamaji | 4220be3 | 2015-05-26 13:30:18 +0900 | [diff] [blame] | 783 | if err != nil { |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 784 | glog.Infof("Cache expired: %s", mk.Filename) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 785 | return nil, fmt.Errorf("cache expired: %s", mk.Filename) |
Shinichiro Hamaji | 4220be3 | 2015-05-26 13:30:18 +0900 | [diff] [blame] | 786 | } |
| 787 | h := sha1.Sum(c) |
| 788 | if !bytes.Equal(h[:], mk.Hash[:]) { |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 789 | glog.Infof("Cache expired: %s", mk.Filename) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 790 | return nil, fmt.Errorf("cache expired: %s", mk.Filename) |
Shinichiro Hamaji | 45cde1d | 2015-05-25 18:21:23 +0900 | [diff] [blame] | 791 | } |
Shinichiro Hamaji | 39e6240 | 2015-05-20 19:34:58 +0900 | [diff] [blame] | 792 | } |
| 793 | } |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 794 | glog.Info("Cache found in %q", filename) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 795 | return g, nil |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 796 | } |