Shinichiro Hamaji | b69bf8a | 2015-06-10 14:52:06 +0900 | [diff] [blame] | 1 | // Copyright 2015 Google Inc. All rights reserved |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
Fumitoshi Ukai | 744bb2b | 2015-06-25 00:10:52 +0900 | [diff] [blame] | 15 | package kati |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 16 | |
| 17 | import ( |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 18 | "bytes" |
Shinichiro Hamaji | 4220be3 | 2015-05-26 13:30:18 +0900 | [diff] [blame] | 19 | "crypto/sha1" |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 20 | "encoding/binary" |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 21 | "encoding/gob" |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 22 | "encoding/json" |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 23 | "fmt" |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 24 | "io" |
Fumitoshi Ukai | bc40a46 | 2015-06-02 15:51:16 +0900 | [diff] [blame] | 25 | "io/ioutil" |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 26 | "net/url" |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 27 | "os" |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 28 | "sort" |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 29 | "strconv" |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 30 | "time" |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame^] | 31 | |
| 32 | "github.com/golang/glog" |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 33 | ) |
| 34 | |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 35 | const ( |
Fumitoshi Ukai | a78fce0 | 2015-06-25 12:29:09 +0900 | [diff] [blame] | 36 | valueTypeRecursive = 'R' |
| 37 | valueTypeSimple = 'S' |
| 38 | valueTypeTSV = 'T' |
| 39 | valueTypeUndefined = 'U' |
| 40 | valueTypeAssign = 'a' |
| 41 | valueTypeExpr = 'e' |
| 42 | valueTypeFunc = 'f' |
| 43 | valueTypeLiteral = 'l' |
| 44 | valueTypeNop = 'n' |
| 45 | valueTypeParamref = 'p' |
| 46 | valueTypeVarref = 'r' |
| 47 | valueTypeVarsubst = 's' |
| 48 | valueTypeTmpval = 't' |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 49 | ) |
| 50 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 51 | // JSON is a json loader/saver. |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 52 | var JSON LoadSaver |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 53 | |
| 54 | // GOB is a gob loader/saver. |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 55 | var GOB LoadSaver |
| 56 | |
| 57 | func init() { |
| 58 | JSON = jsonLoadSaver{} |
| 59 | GOB = gobLoadSaver{} |
| 60 | } |
| 61 | |
| 62 | type jsonLoadSaver struct{} |
| 63 | type gobLoadSaver struct{} |
| 64 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 65 | type dumpbuf struct { |
| 66 | w bytes.Buffer |
| 67 | err error |
| 68 | } |
| 69 | |
| 70 | func (d *dumpbuf) Int(i int) { |
| 71 | if d.err != nil { |
| 72 | return |
| 73 | } |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 74 | v := int32(i) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 75 | d.err = binary.Write(&d.w, binary.LittleEndian, &v) |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 76 | } |
| 77 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 78 | func (d *dumpbuf) Str(s string) { |
| 79 | if d.err != nil { |
| 80 | return |
Fumitoshi Ukai | 145598a | 2015-06-19 10:08:17 +0900 | [diff] [blame] | 81 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 82 | d.Int(len(s)) |
| 83 | if d.err != nil { |
| 84 | return |
| 85 | } |
| 86 | _, d.err = io.WriteString(&d.w, s) |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 87 | } |
| 88 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 89 | func (d *dumpbuf) Bytes(b []byte) { |
| 90 | if d.err != nil { |
| 91 | return |
Fumitoshi Ukai | 145598a | 2015-06-19 10:08:17 +0900 | [diff] [blame] | 92 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 93 | d.Int(len(b)) |
| 94 | if d.err != nil { |
| 95 | return |
| 96 | } |
| 97 | _, d.err = d.w.Write(b) |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 98 | } |
| 99 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 100 | func (d *dumpbuf) Byte(b byte) { |
| 101 | if d.err != nil { |
| 102 | return |
Fumitoshi Ukai | 145598a | 2015-06-19 10:08:17 +0900 | [diff] [blame] | 103 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 104 | d.err = writeByte(&d.w, b) |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 105 | } |
| 106 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 107 | type serializableVar struct { |
Shinichiro Hamaji | c8bc731 | 2015-04-28 02:48:03 +0900 | [diff] [blame] | 108 | Type string |
| 109 | V string |
| 110 | Origin string |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 111 | Children []serializableVar |
Shinichiro Hamaji | c8bc731 | 2015-04-28 02:48:03 +0900 | [diff] [blame] | 112 | } |
| 113 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 114 | type serializableDepNode struct { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 115 | Output int |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 116 | Cmds []string |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 117 | Deps []int |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 118 | OrderOnlys []int |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 119 | Parents []int |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 120 | HasRule bool |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 121 | IsPhony bool |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 122 | ActualInputs []int |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 123 | TargetSpecificVars []int |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 124 | Filename string |
| 125 | Lineno int |
| 126 | } |
| 127 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 128 | type serializableTargetSpecificVar struct { |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 129 | Name string |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 130 | Value serializableVar |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 131 | } |
| 132 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 133 | type serializableGraph struct { |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 134 | Nodes []*serializableDepNode |
| 135 | Vars map[string]serializableVar |
| 136 | Tsvs []serializableTargetSpecificVar |
| 137 | Targets []string |
| 138 | Roots []string |
| 139 | AccessedMks []*accessedMakefile |
| 140 | Exports map[string]bool |
Shinichiro Hamaji | c8bc731 | 2015-04-28 02:48:03 +0900 | [diff] [blame] | 141 | } |
| 142 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 143 | func encGob(v interface{}) (string, error) { |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 144 | var buf bytes.Buffer |
| 145 | e := gob.NewEncoder(&buf) |
| 146 | err := e.Encode(v) |
| 147 | if err != nil { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 148 | return "", err |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 149 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 150 | return buf.String(), nil |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 151 | } |
| 152 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 153 | func encVar(k string, v Var) (string, error) { |
| 154 | var dump dumpbuf |
| 155 | dump.Str(k) |
| 156 | v.dump(&dump) |
| 157 | return dump.w.String(), dump.err |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 158 | } |
| 159 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 160 | type depNodesSerializer struct { |
| 161 | nodes []*serializableDepNode |
| 162 | tsvs []serializableTargetSpecificVar |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 163 | tsvMap map[string]int |
| 164 | targets []string |
| 165 | targetMap map[string]int |
| 166 | done map[string]bool |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 167 | err error |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 168 | } |
| 169 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 170 | func newDepNodesSerializer() *depNodesSerializer { |
| 171 | return &depNodesSerializer{ |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 172 | tsvMap: make(map[string]int), |
| 173 | targetMap: make(map[string]int), |
| 174 | done: make(map[string]bool), |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 175 | } |
| 176 | } |
| 177 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 178 | func (ns *depNodesSerializer) serializeTarget(t string) int { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 179 | id, present := ns.targetMap[t] |
| 180 | if present { |
| 181 | return id |
| 182 | } |
| 183 | id = len(ns.targets) |
| 184 | ns.targetMap[t] = id |
| 185 | ns.targets = append(ns.targets, t) |
| 186 | return id |
| 187 | } |
| 188 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 189 | func (ns *depNodesSerializer) serializeDepNodes(nodes []*DepNode) { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 190 | if ns.err != nil { |
| 191 | return |
| 192 | } |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 193 | for _, n := range nodes { |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 194 | if ns.done[n.Output] { |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 195 | continue |
| 196 | } |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 197 | ns.done[n.Output] = true |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 198 | |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 199 | var deps []int |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 200 | for _, d := range n.Deps { |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 201 | deps = append(deps, ns.serializeTarget(d.Output)) |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 202 | } |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 203 | var orderonlys []int |
| 204 | for _, d := range n.OrderOnlys { |
| 205 | orderonlys = append(orderonlys, ns.serializeTarget(d.Output)) |
| 206 | } |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 207 | var parents []int |
| 208 | for _, d := range n.Parents { |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 209 | parents = append(parents, ns.serializeTarget(d.Output)) |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 210 | } |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 211 | var actualInputs []int |
| 212 | for _, i := range n.ActualInputs { |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 213 | actualInputs = append(actualInputs, ns.serializeTarget(i)) |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 214 | } |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 215 | |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 216 | // Sort keys for consistent serialization. |
| 217 | var tsvKeys []string |
| 218 | for k := range n.TargetSpecificVars { |
| 219 | tsvKeys = append(tsvKeys, k) |
| 220 | } |
| 221 | sort.Strings(tsvKeys) |
| 222 | |
| 223 | var vars []int |
| 224 | for _, k := range tsvKeys { |
| 225 | v := n.TargetSpecificVars[k] |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 226 | sv := serializableTargetSpecificVar{Name: k, Value: v.serialize()} |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 227 | //gob := encGob(sv) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 228 | gob, err := encVar(k, v) |
| 229 | if err != nil { |
| 230 | ns.err = err |
| 231 | return |
| 232 | } |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 233 | id, present := ns.tsvMap[gob] |
| 234 | if !present { |
| 235 | id = len(ns.tsvs) |
| 236 | ns.tsvMap[gob] = id |
| 237 | ns.tsvs = append(ns.tsvs, sv) |
| 238 | } |
| 239 | vars = append(vars, id) |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 240 | } |
| 241 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 242 | ns.nodes = append(ns.nodes, &serializableDepNode{ |
| 243 | Output: ns.serializeTarget(n.Output), |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 244 | Cmds: n.Cmds, |
| 245 | Deps: deps, |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 246 | OrderOnlys: orderonlys, |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 247 | Parents: parents, |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 248 | HasRule: n.HasRule, |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 249 | IsPhony: n.IsPhony, |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 250 | ActualInputs: actualInputs, |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 251 | TargetSpecificVars: vars, |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 252 | Filename: n.Filename, |
| 253 | Lineno: n.Lineno, |
| 254 | }) |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 255 | ns.serializeDepNodes(n.Deps) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 256 | if ns.err != nil { |
| 257 | return |
| 258 | } |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 259 | ns.serializeDepNodes(n.OrderOnlys) |
| 260 | if ns.err != nil { |
| 261 | return |
| 262 | } |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 263 | } |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 264 | } |
| 265 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 266 | func makeSerializableVars(vars Vars) (r map[string]serializableVar) { |
| 267 | r = make(map[string]serializableVar) |
Shinichiro Hamaji | c8bc731 | 2015-04-28 02:48:03 +0900 | [diff] [blame] | 268 | for k, v := range vars { |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 269 | r[k] = v.serialize() |
Shinichiro Hamaji | c8bc731 | 2015-04-28 02:48:03 +0900 | [diff] [blame] | 270 | } |
| 271 | return r |
| 272 | } |
| 273 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 274 | func makeSerializableGraph(g *DepGraph, roots []string) (serializableGraph, error) { |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 275 | ns := newDepNodesSerializer() |
| 276 | ns.serializeDepNodes(g.nodes) |
| 277 | v := makeSerializableVars(g.vars) |
| 278 | return serializableGraph{ |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 279 | Nodes: ns.nodes, |
| 280 | Vars: v, |
| 281 | Tsvs: ns.tsvs, |
| 282 | Targets: ns.targets, |
| 283 | Roots: roots, |
| 284 | AccessedMks: g.accessedMks, |
| 285 | Exports: g.exports, |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 286 | }, ns.err |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 287 | } |
Shinichiro Hamaji | c8bc731 | 2015-04-28 02:48:03 +0900 | [diff] [blame] | 288 | |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 289 | func (jsonLoadSaver) Save(g *DepGraph, filename string, roots []string) error { |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 290 | startTime := time.Now() |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 291 | sg, err := makeSerializableGraph(g, roots) |
| 292 | if err != nil { |
| 293 | return err |
| 294 | } |
Shinichiro Hamaji | 750ae2e | 2015-05-20 12:59:15 +0900 | [diff] [blame] | 295 | o, err := json.MarshalIndent(sg, " ", " ") |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 296 | if err != nil { |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 297 | return err |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 298 | } |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 299 | f, err := os.Create(filename) |
| 300 | if err != nil { |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 301 | return err |
| 302 | } |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 303 | _, err = f.Write(o) |
| 304 | if err != nil { |
| 305 | f.Close() |
| 306 | return err |
| 307 | } |
| 308 | err = f.Close() |
| 309 | if err != nil { |
| 310 | return err |
| 311 | } |
| 312 | logStats("json serialize time: %q", time.Since(startTime)) |
| 313 | return nil |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 314 | } |
| 315 | |
| 316 | func (gobLoadSaver) Save(g *DepGraph, filename string, roots []string) error { |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 317 | startTime := time.Now() |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 318 | f, err := os.Create(filename) |
| 319 | if err != nil { |
| 320 | return err |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 321 | } |
| 322 | e := gob.NewEncoder(f) |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 323 | var sg serializableGraph |
| 324 | { |
| 325 | startTime := time.Now() |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 326 | sg, err = makeSerializableGraph(g, roots) |
| 327 | if err != nil { |
| 328 | return err |
| 329 | } |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 330 | logStats("gob serialize prepare time: %q", time.Since(startTime)) |
| 331 | } |
| 332 | { |
| 333 | startTime := time.Now() |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 334 | err = e.Encode(sg) |
| 335 | if err != nil { |
| 336 | return err |
| 337 | } |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 338 | logStats("gob serialize output time: %q", time.Since(startTime)) |
| 339 | } |
| 340 | err = f.Close() |
| 341 | if err != nil { |
| 342 | return err |
| 343 | } |
| 344 | logStats("gob serialize time: %q", time.Since(startTime)) |
| 345 | return nil |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 346 | } |
| 347 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 348 | func cacheFilename(mk string, roots []string) string { |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 349 | filename := ".kati_cache." + mk |
| 350 | for _, r := range roots { |
| 351 | filename += "." + r |
| 352 | } |
| 353 | return url.QueryEscape(filename) |
| 354 | } |
| 355 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 356 | func saveCache(g *DepGraph, roots []string) error { |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 357 | if len(g.accessedMks) == 0 { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 358 | return fmt.Errorf("no Makefile is read") |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 359 | } |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 360 | cacheFile := cacheFilename(g.accessedMks[0].Filename, roots) |
| 361 | for _, mk := range g.accessedMks { |
Shinichiro Hamaji | f3ea5d0 | 2015-05-20 17:13:00 +0900 | [diff] [blame] | 362 | // Inconsistent, do not dump this result. |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 363 | if mk.State == fileInconsistent { |
Shinichiro Hamaji | f3ea5d0 | 2015-05-20 17:13:00 +0900 | [diff] [blame] | 364 | if exists(cacheFile) { |
| 365 | os.Remove(cacheFile) |
| 366 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 367 | return nil |
Shinichiro Hamaji | f3ea5d0 | 2015-05-20 17:13:00 +0900 | [diff] [blame] | 368 | } |
| 369 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 370 | return GOB.Save(g, cacheFile, roots) |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 371 | } |
| 372 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 373 | func deserializeSingleChild(sv serializableVar) (Value, error) { |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 374 | if len(sv.Children) != 1 { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 375 | return nil, fmt.Errorf("unexpected number of children: %q", sv) |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 376 | } |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 377 | return deserializeVar(sv.Children[0]) |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 378 | } |
| 379 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 380 | func deserializeVar(sv serializableVar) (r Value, err error) { |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 381 | switch sv.Type { |
| 382 | case "literal": |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 383 | return literal(sv.V), nil |
Shinichiro Hamaji | c8dea2a | 2015-04-28 17:15:30 +0900 | [diff] [blame] | 384 | case "tmpval": |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 385 | return tmpval([]byte(sv.V)), nil |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 386 | case "expr": |
Fumitoshi Ukai | 55c8fa9 | 2015-06-25 15:56:10 +0900 | [diff] [blame] | 387 | var e expr |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 388 | for _, v := range sv.Children { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 389 | dv, err := deserializeVar(v) |
| 390 | if err != nil { |
| 391 | return nil, err |
| 392 | } |
| 393 | e = append(e, dv) |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 394 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 395 | return e, nil |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 396 | case "varref": |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 397 | dv, err := deserializeSingleChild(sv) |
| 398 | if err != nil { |
| 399 | return nil, err |
| 400 | } |
Fumitoshi Ukai | bbb0db5 | 2015-06-30 16:51:27 +0900 | [diff] [blame] | 401 | return &varref{varname: dv, paren: sv.V[0]}, nil |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 402 | case "paramref": |
| 403 | v, err := strconv.Atoi(sv.V) |
| 404 | if err != nil { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 405 | return nil, err |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 406 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 407 | return paramref(v), nil |
Shinichiro Hamaji | c8dea2a | 2015-04-28 17:15:30 +0900 | [diff] [blame] | 408 | case "varsubst": |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 409 | varname, err := deserializeVar(sv.Children[0]) |
| 410 | if err != nil { |
| 411 | return nil, err |
Shinichiro Hamaji | c8dea2a | 2015-04-28 17:15:30 +0900 | [diff] [blame] | 412 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 413 | pat, err := deserializeVar(sv.Children[1]) |
| 414 | if err != nil { |
| 415 | return nil, err |
| 416 | } |
| 417 | subst, err := deserializeVar(sv.Children[2]) |
| 418 | if err != nil { |
| 419 | return nil, err |
| 420 | } |
| 421 | return varsubst{ |
| 422 | varname: varname, |
| 423 | pat: pat, |
| 424 | subst: subst, |
Fumitoshi Ukai | bbb0db5 | 2015-06-30 16:51:27 +0900 | [diff] [blame] | 425 | paren: sv.V[0], |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 426 | }, nil |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 427 | |
| 428 | case "func": |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 429 | dv, err := deserializeVar(sv.Children[0]) |
| 430 | if err != nil { |
| 431 | return nil, err |
| 432 | } |
| 433 | name, ok := dv.(literal) |
| 434 | if !ok { |
| 435 | return nil, fmt.Errorf("func name is not literal %s: %T", dv, dv) |
| 436 | } |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 437 | f := funcMap[string(name[1:])]() |
| 438 | f.AddArg(name) |
| 439 | for _, a := range sv.Children[1:] { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 440 | dv, err := deserializeVar(a) |
| 441 | if err != nil { |
| 442 | return nil, err |
| 443 | } |
| 444 | f.AddArg(dv) |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 445 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 446 | return f, nil |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 447 | case "funcEvalAssign": |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 448 | rhs, err := deserializeVar(sv.Children[2]) |
| 449 | if err != nil { |
| 450 | return nil, err |
| 451 | } |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 452 | return &funcEvalAssign{ |
| 453 | lhs: sv.Children[0].V, |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 454 | op: sv.Children[1].V, |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 455 | rhs: rhs, |
| 456 | }, nil |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 457 | case "funcNop": |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 458 | return &funcNop{expr: sv.V}, nil |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 459 | |
| 460 | case "simple": |
Fumitoshi Ukai | 7bf992d | 2015-06-25 12:42:19 +0900 | [diff] [blame] | 461 | return &simpleVar{ |
Fumitoshi Ukai | dd248f2 | 2015-06-18 23:04:28 +0900 | [diff] [blame] | 462 | value: sv.V, |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 463 | origin: sv.Origin, |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 464 | }, nil |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 465 | case "recursive": |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 466 | expr, err := deserializeSingleChild(sv) |
| 467 | if err != nil { |
| 468 | return nil, err |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 469 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 470 | return &recursiveVar{ |
| 471 | expr: expr, |
| 472 | origin: sv.Origin, |
| 473 | }, nil |
Shinichiro Hamaji | f8f075c | 2015-04-28 04:25:37 +0900 | [diff] [blame] | 474 | |
| 475 | case ":=", "=", "+=", "?=": |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 476 | dv, err := deserializeSingleChild(sv) |
| 477 | if err != nil { |
| 478 | return nil, err |
Shinichiro Hamaji | f8f075c | 2015-04-28 04:25:37 +0900 | [diff] [blame] | 479 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 480 | v, ok := dv.(Var) |
| 481 | if !ok { |
| 482 | return nil, fmt.Errorf("not var: target specific var %s %T", dv, dv) |
| 483 | } |
| 484 | return &targetSpecificVar{ |
| 485 | v: v, |
| 486 | op: sv.Type, |
| 487 | }, nil |
Shinichiro Hamaji | f8f075c | 2015-04-28 04:25:37 +0900 | [diff] [blame] | 488 | |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 489 | default: |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 490 | return nil, fmt.Errorf("unknown serialized variable type: %q", sv) |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 491 | } |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 492 | } |
| 493 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 494 | func deserializeVars(vars map[string]serializableVar) (Vars, error) { |
Shinichiro Hamaji | 79a5693 | 2015-04-28 03:53:57 +0900 | [diff] [blame] | 495 | r := make(Vars) |
| 496 | for k, v := range vars { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 497 | dv, err := deserializeVar(v) |
| 498 | if err != nil { |
| 499 | return nil, err |
| 500 | } |
| 501 | vv, ok := dv.(Var) |
| 502 | if !ok { |
| 503 | return nil, fmt.Errorf("not var: %s: %T", dv, dv) |
| 504 | } |
| 505 | r[k] = vv |
Shinichiro Hamaji | 79a5693 | 2015-04-28 03:53:57 +0900 | [diff] [blame] | 506 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 507 | return r, nil |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 508 | } |
| 509 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 510 | func deserializeNodes(g serializableGraph) (r []*DepNode, err error) { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 511 | nodes := g.Nodes |
| 512 | tsvs := g.Tsvs |
| 513 | targets := g.Targets |
Shinichiro Hamaji | 7a65e68 | 2015-04-28 18:00:12 +0900 | [diff] [blame] | 514 | // Deserialize all TSVs first so that multiple rules can share memory. |
| 515 | var tsvValues []Var |
| 516 | for _, sv := range tsvs { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 517 | dv, err := deserializeVar(sv.Value) |
| 518 | if err != nil { |
| 519 | return nil, err |
| 520 | } |
| 521 | vv, ok := dv.(Var) |
| 522 | if !ok { |
| 523 | return nil, fmt.Errorf("not var: %s %T", dv, dv) |
| 524 | } |
| 525 | tsvValues = append(tsvValues, vv) |
Shinichiro Hamaji | 7a65e68 | 2015-04-28 18:00:12 +0900 | [diff] [blame] | 526 | } |
| 527 | |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 528 | nodeMap := make(map[string]*DepNode) |
| 529 | for _, n := range nodes { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 530 | var actualInputs []string |
| 531 | for _, i := range n.ActualInputs { |
| 532 | actualInputs = append(actualInputs, targets[i]) |
| 533 | } |
| 534 | |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 535 | d := &DepNode{ |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 536 | Output: targets[n.Output], |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 537 | Cmds: n.Cmds, |
| 538 | HasRule: n.HasRule, |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 539 | IsPhony: n.IsPhony, |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 540 | ActualInputs: actualInputs, |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 541 | Filename: n.Filename, |
| 542 | Lineno: n.Lineno, |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 543 | TargetSpecificVars: make(Vars), |
| 544 | } |
| 545 | |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 546 | for _, id := range n.TargetSpecificVars { |
| 547 | sv := tsvs[id] |
Shinichiro Hamaji | 7a65e68 | 2015-04-28 18:00:12 +0900 | [diff] [blame] | 548 | d.TargetSpecificVars[sv.Name] = tsvValues[id] |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 549 | } |
| 550 | |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 551 | nodeMap[targets[n.Output]] = d |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 552 | r = append(r, d) |
| 553 | } |
| 554 | |
| 555 | for _, n := range nodes { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 556 | d := nodeMap[targets[n.Output]] |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 557 | for _, o := range n.Deps { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 558 | c, present := nodeMap[targets[o]] |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 559 | if !present { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 560 | return nil, fmt.Errorf("unknown target: %d (%s)", o, targets[o]) |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 561 | } |
| 562 | d.Deps = append(d.Deps, c) |
| 563 | } |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 564 | for _, o := range n.OrderOnlys { |
| 565 | c, present := nodeMap[targets[o]] |
| 566 | if !present { |
| 567 | return nil, fmt.Errorf("unknown target: %d (%s)", o, targets[o]) |
| 568 | } |
| 569 | d.OrderOnlys = append(d.OrderOnlys, c) |
| 570 | } |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 571 | for _, o := range n.Parents { |
| 572 | c, present := nodeMap[targets[o]] |
| 573 | if !present { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 574 | return nil, fmt.Errorf("unknown target: %d (%s)", o, targets[o]) |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 575 | } |
| 576 | d.Parents = append(d.Parents, c) |
| 577 | } |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 578 | } |
| 579 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 580 | return r, nil |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 581 | } |
| 582 | |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 583 | func human(n int) string { |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 584 | if n >= 10*1000*1000*1000 { |
| 585 | return fmt.Sprintf("%.2fGB", float32(n)/1000/1000/1000) |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 586 | } |
Fumitoshi Ukai | 936de10 | 2015-06-08 11:21:16 +0900 | [diff] [blame] | 587 | if n >= 10*1000*1000 { |
| 588 | return fmt.Sprintf("%.2fMB", float32(n)/1000/1000) |
| 589 | } |
| 590 | if n >= 10*1000 { |
| 591 | return fmt.Sprintf("%.2fkB", float32(n)/1000) |
| 592 | } |
| 593 | return fmt.Sprintf("%dB", n) |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 594 | } |
| 595 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 596 | func showSerializedNodesStats(nodes []*serializableDepNode) { |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 597 | outputSize := 0 |
| 598 | cmdSize := 0 |
| 599 | depsSize := 0 |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 600 | orderOnlysSize := 0 |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 601 | actualInputSize := 0 |
| 602 | tsvSize := 0 |
| 603 | filenameSize := 0 |
| 604 | linenoSize := 0 |
| 605 | for _, n := range nodes { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 606 | outputSize += 4 |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 607 | for _, c := range n.Cmds { |
| 608 | cmdSize += len(c) |
| 609 | } |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 610 | depsSize += 4 * len(n.Deps) |
| 611 | orderOnlysSize += 4 * len(n.OrderOnlys) |
| 612 | actualInputSize += 4 * len(n.ActualInputs) |
| 613 | tsvSize += 4 * len(n.TargetSpecificVars) |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 614 | filenameSize += len(n.Filename) |
| 615 | linenoSize += 4 |
| 616 | } |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 617 | size := outputSize + cmdSize + depsSize + orderOnlysSize + actualInputSize + tsvSize + filenameSize + linenoSize |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 618 | logStats("%d nodes %s", len(nodes), human(size)) |
| 619 | logStats(" output %s", human(outputSize)) |
| 620 | logStats(" command %s", human(cmdSize)) |
| 621 | logStats(" deps %s", human(depsSize)) |
Fumitoshi Ukai | c916ea2 | 2015-06-30 09:52:13 +0900 | [diff] [blame] | 622 | logStats(" orderonlys %s", human(orderOnlysSize)) |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 623 | logStats(" inputs %s", human(actualInputSize)) |
| 624 | logStats(" tsv %s", human(tsvSize)) |
| 625 | logStats(" filename %s", human(filenameSize)) |
| 626 | logStats(" lineno %s", human(linenoSize)) |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 627 | } |
| 628 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 629 | func (v serializableVar) size() int { |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 630 | size := 0 |
| 631 | size += len(v.Type) |
| 632 | size += len(v.V) |
| 633 | size += len(v.Origin) |
| 634 | for _, c := range v.Children { |
| 635 | size += c.size() |
| 636 | } |
| 637 | return size |
| 638 | } |
| 639 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 640 | func showSerializedVarsStats(vars map[string]serializableVar) { |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 641 | nameSize := 0 |
| 642 | valueSize := 0 |
| 643 | for k, v := range vars { |
| 644 | nameSize += len(k) |
| 645 | valueSize += v.size() |
| 646 | } |
| 647 | size := nameSize + valueSize |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 648 | logStats("%d vars %s", len(vars), human(size)) |
| 649 | logStats(" name %s", human(nameSize)) |
| 650 | logStats(" value %s", human(valueSize)) |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 651 | } |
| 652 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 653 | func showSerializedTsvsStats(vars []serializableTargetSpecificVar) { |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 654 | nameSize := 0 |
| 655 | valueSize := 0 |
| 656 | for _, v := range vars { |
| 657 | nameSize += len(v.Name) |
| 658 | valueSize += v.Value.size() |
| 659 | } |
| 660 | size := nameSize + valueSize |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 661 | logStats("%d tsvs %s", len(vars), human(size)) |
| 662 | logStats(" name %s", human(nameSize)) |
| 663 | logStats(" value %s", human(valueSize)) |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 664 | } |
| 665 | |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 666 | func showSerializedTargetsStats(targets []string) { |
| 667 | size := 0 |
| 668 | for _, t := range targets { |
| 669 | size += len(t) |
| 670 | } |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 671 | logStats("%d targets %s", len(targets), human(size)) |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 672 | } |
| 673 | |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 674 | func showSerializedAccessedMksStats(accessedMks []*accessedMakefile) { |
Shinichiro Hamaji | 006adbf | 2015-05-25 18:42:12 +0900 | [diff] [blame] | 675 | size := 0 |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 676 | for _, rm := range accessedMks { |
Shinichiro Hamaji | 4220be3 | 2015-05-26 13:30:18 +0900 | [diff] [blame] | 677 | size += len(rm.Filename) + len(rm.Hash) + 4 |
Shinichiro Hamaji | 006adbf | 2015-05-25 18:42:12 +0900 | [diff] [blame] | 678 | } |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 679 | logStats("%d makefiles %s", len(accessedMks), human(size)) |
Shinichiro Hamaji | 006adbf | 2015-05-25 18:42:12 +0900 | [diff] [blame] | 680 | } |
| 681 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 682 | func showSerializedGraphStats(g serializableGraph) { |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 683 | showSerializedNodesStats(g.Nodes) |
| 684 | showSerializedVarsStats(g.Vars) |
| 685 | showSerializedTsvsStats(g.Tsvs) |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 686 | showSerializedTargetsStats(g.Targets) |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 687 | showSerializedAccessedMksStats(g.AccessedMks) |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 688 | } |
| 689 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 690 | func deserializeGraph(g serializableGraph) (*DepGraph, error) { |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame^] | 691 | if StatsFlag { |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 692 | showSerializedGraphStats(g) |
| 693 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 694 | nodes, err := deserializeNodes(g) |
| 695 | if err != nil { |
| 696 | return nil, err |
| 697 | } |
| 698 | vars, err := deserializeVars(g.Vars) |
| 699 | if err != nil { |
| 700 | return nil, err |
| 701 | } |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 702 | return &DepGraph{ |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 703 | nodes: nodes, |
| 704 | vars: vars, |
| 705 | accessedMks: g.AccessedMks, |
| 706 | exports: g.Exports, |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 707 | }, nil |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 708 | } |
| 709 | |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 710 | func (jsonLoadSaver) Load(filename string) (*DepGraph, error) { |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 711 | startTime := time.Now() |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 712 | f, err := os.Open(filename) |
| 713 | if err != nil { |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 714 | return nil, err |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 715 | } |
Fumitoshi Ukai | ed9a615 | 2015-06-02 15:28:55 +0900 | [diff] [blame] | 716 | defer f.Close() |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 717 | |
| 718 | d := json.NewDecoder(f) |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 719 | g := serializableGraph{Vars: make(map[string]serializableVar)} |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 720 | err = d.Decode(&g) |
| 721 | if err != nil { |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 722 | return nil, err |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 723 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 724 | dg, err := deserializeGraph(g) |
| 725 | if err != nil { |
| 726 | return nil, err |
| 727 | } |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 728 | logStats("gob deserialize time: %q", time.Since(startTime)) |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 729 | return dg, nil |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 730 | } |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 731 | |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 732 | func (gobLoadSaver) Load(filename string) (*DepGraph, error) { |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 733 | startTime := time.Now() |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 734 | f, err := os.Open(filename) |
| 735 | if err != nil { |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 736 | return nil, err |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 737 | } |
Fumitoshi Ukai | ed9a615 | 2015-06-02 15:28:55 +0900 | [diff] [blame] | 738 | defer f.Close() |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 739 | |
| 740 | d := gob.NewDecoder(f) |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 741 | g := serializableGraph{Vars: make(map[string]serializableVar)} |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 742 | err = d.Decode(&g) |
| 743 | if err != nil { |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 744 | return nil, err |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 745 | } |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 746 | dg, err := deserializeGraph(g) |
| 747 | if err != nil { |
| 748 | return nil, err |
| 749 | } |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 750 | logStats("json deserialize time: %q", time.Since(startTime)) |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 751 | return dg, nil |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 752 | } |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 753 | |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 754 | func loadCache(makefile string, roots []string) (*DepGraph, error) { |
Shinichiro Hamaji | 82ecff9 | 2015-05-25 18:36:04 +0900 | [diff] [blame] | 755 | startTime := time.Now() |
| 756 | defer func() { |
Fumitoshi Ukai | 49599e5 | 2015-06-26 10:10:24 +0900 | [diff] [blame] | 757 | logStats("Cache lookup time: %q", time.Since(startTime)) |
Shinichiro Hamaji | 82ecff9 | 2015-05-25 18:36:04 +0900 | [diff] [blame] | 758 | }() |
| 759 | |
Fumitoshi Ukai | a045ccb | 2015-06-25 12:57:25 +0900 | [diff] [blame] | 760 | filename := cacheFilename(makefile, roots) |
Shinichiro Hamaji | 39e6240 | 2015-05-20 19:34:58 +0900 | [diff] [blame] | 761 | if !exists(filename) { |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame^] | 762 | glog.Warningf("Cache not found %q", filename) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 763 | return nil, fmt.Errorf("cache not found: %s", filename) |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 764 | } |
Shinichiro Hamaji | 39e6240 | 2015-05-20 19:34:58 +0900 | [diff] [blame] | 765 | |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 766 | g, err := GOB.Load(filename) |
| 767 | if err != nil { |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame^] | 768 | glog.Warning("Cache load error %q: %v", filename, err) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 769 | return nil, err |
Fumitoshi Ukai | 4bb4cd5 | 2015-06-25 18:07:21 +0900 | [diff] [blame] | 770 | } |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 771 | for _, mk := range g.accessedMks { |
| 772 | if mk.State != fileExists && mk.State != fileNotExists { |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 773 | return nil, fmt.Errorf("internal error: broken state: %d", mk.State) |
Shinichiro Hamaji | 0ba75a9 | 2015-05-20 19:42:43 +0900 | [diff] [blame] | 774 | } |
Fumitoshi Ukai | 0af4452 | 2015-06-25 15:26:08 +0900 | [diff] [blame] | 775 | if mk.State == fileNotExists { |
Shinichiro Hamaji | 45cde1d | 2015-05-25 18:21:23 +0900 | [diff] [blame] | 776 | if exists(mk.Filename) { |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame^] | 777 | glog.Infof("Cache expired: %s", mk.Filename) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 778 | return nil, fmt.Errorf("cache expired: %s", mk.Filename) |
Shinichiro Hamaji | 45cde1d | 2015-05-25 18:21:23 +0900 | [diff] [blame] | 779 | } |
| 780 | } else { |
Fumitoshi Ukai | bc40a46 | 2015-06-02 15:51:16 +0900 | [diff] [blame] | 781 | c, err := ioutil.ReadFile(mk.Filename) |
Shinichiro Hamaji | 4220be3 | 2015-05-26 13:30:18 +0900 | [diff] [blame] | 782 | if err != nil { |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame^] | 783 | glog.Infof("Cache expired: %s", mk.Filename) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 784 | return nil, fmt.Errorf("cache expired: %s", mk.Filename) |
Shinichiro Hamaji | 4220be3 | 2015-05-26 13:30:18 +0900 | [diff] [blame] | 785 | } |
| 786 | h := sha1.Sum(c) |
| 787 | if !bytes.Equal(h[:], mk.Hash[:]) { |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame^] | 788 | glog.Infof("Cache expired: %s", mk.Filename) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 789 | return nil, fmt.Errorf("cache expired: %s", mk.Filename) |
Shinichiro Hamaji | 45cde1d | 2015-05-25 18:21:23 +0900 | [diff] [blame] | 790 | } |
Shinichiro Hamaji | 39e6240 | 2015-05-20 19:34:58 +0900 | [diff] [blame] | 791 | } |
| 792 | } |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame^] | 793 | glog.Info("Cache found in %q", filename) |
Fumitoshi Ukai | 65c7233 | 2015-06-26 21:32:50 +0900 | [diff] [blame] | 794 | return g, nil |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 795 | } |