Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 1 | package main |
| 2 | |
| 3 | import ( |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 4 | "bytes" |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 5 | "encoding/binary" |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 6 | "encoding/gob" |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 7 | "encoding/json" |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 8 | "fmt" |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 9 | "io" |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 10 | "net/url" |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 11 | "os" |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 12 | "sort" |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 13 | "strconv" |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 14 | "time" |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 15 | ) |
| 16 | |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 17 | const ( |
| 18 | VALUE_TYPE_RECURSIVE = 'R' |
| 19 | VALUE_TYPE_SIMPLE = 'S' |
| 20 | VALUE_TYPE_TSV = 'T' |
| 21 | VALUE_TYPE_UNDEFINED = 'U' |
| 22 | VALUE_TYPE_ASSIGN = 'a' |
| 23 | VALUE_TYPE_EXPR = 'e' |
| 24 | VALUE_TYPE_FUNC = 'f' |
| 25 | VALUE_TYPE_LITERAL = 'l' |
| 26 | VALUE_TYPE_NOP = 'n' |
| 27 | VALUE_TYPE_PARAMREF = 'p' |
| 28 | VALUE_TYPE_VARREF = 'r' |
| 29 | VALUE_TYPE_VARSUBST = 's' |
| 30 | VALUE_TYPE_TMPVAL = 't' |
| 31 | ) |
| 32 | |
| 33 | func dumpData(w io.Writer, data []byte) { |
| 34 | for len(data) != 0 { |
| 35 | written, err := w.Write(data) |
| 36 | if err == io.EOF { |
| 37 | return |
| 38 | } |
| 39 | if err != nil { |
| 40 | panic(err) |
| 41 | } |
| 42 | data = data[written:] |
| 43 | } |
| 44 | } |
| 45 | |
| 46 | func dumpInt(w io.Writer, i int) { |
| 47 | v := int32(i) |
| 48 | binary.Write(w, binary.LittleEndian, &v) |
| 49 | } |
| 50 | |
| 51 | func dumpString(w io.Writer, s string) { |
| 52 | dumpInt(w, len(s)) |
| 53 | dumpData(w, []byte(s)) |
| 54 | } |
| 55 | |
| 56 | func dumpBytes(w io.Writer, b []byte) { |
| 57 | dumpInt(w, len(b)) |
| 58 | dumpData(w, b) |
| 59 | } |
| 60 | |
| 61 | func dumpByte(w io.Writer, b byte) { |
| 62 | w.Write([]byte{b}) |
| 63 | } |
| 64 | |
Shinichiro Hamaji | c8bc731 | 2015-04-28 02:48:03 +0900 | [diff] [blame] | 65 | type SerializableVar struct { |
| 66 | Type string |
| 67 | V string |
| 68 | Origin string |
| 69 | Children []SerializableVar |
| 70 | } |
| 71 | |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 72 | type SerializableDepNode struct { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 73 | Output int |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 74 | Cmds []string |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 75 | Deps []int |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 76 | Parents []int |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 77 | HasRule bool |
| 78 | IsOrderOnly bool |
| 79 | IsPhony bool |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 80 | ActualInputs []int |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 81 | TargetSpecificVars []int |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 82 | Filename string |
| 83 | Lineno int |
| 84 | } |
| 85 | |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 86 | type SerializableTargetSpecificVar struct { |
| 87 | Name string |
| 88 | Value SerializableVar |
| 89 | } |
| 90 | |
Shinichiro Hamaji | c8bc731 | 2015-04-28 02:48:03 +0900 | [diff] [blame] | 91 | type SerializableGraph struct { |
Shinichiro Hamaji | f3ea5d0 | 2015-05-20 17:13:00 +0900 | [diff] [blame] | 92 | Nodes []*SerializableDepNode |
| 93 | Vars map[string]SerializableVar |
| 94 | Tsvs []SerializableTargetSpecificVar |
| 95 | Targets []string |
| 96 | Roots []string |
Shinichiro Hamaji | 45cde1d | 2015-05-25 18:21:23 +0900 | [diff] [blame] | 97 | ReadMks []*ReadMakefile |
Shinichiro Hamaji | c8bc731 | 2015-04-28 02:48:03 +0900 | [diff] [blame] | 98 | } |
| 99 | |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 100 | func encGob(v interface{}) string { |
| 101 | var buf bytes.Buffer |
| 102 | e := gob.NewEncoder(&buf) |
| 103 | err := e.Encode(v) |
| 104 | if err != nil { |
| 105 | panic(err) |
| 106 | } |
| 107 | return buf.String() |
| 108 | } |
| 109 | |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 110 | func encVar(k string, v Var) string { |
| 111 | var buf bytes.Buffer |
| 112 | dumpString(&buf, k) |
| 113 | v.Dump(&buf) |
| 114 | return buf.String() |
| 115 | } |
| 116 | |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 117 | type DepNodesSerializer struct { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 118 | nodes []*SerializableDepNode |
| 119 | tsvs []SerializableTargetSpecificVar |
| 120 | tsvMap map[string]int |
| 121 | targets []string |
| 122 | targetMap map[string]int |
| 123 | done map[string]bool |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 124 | } |
| 125 | |
| 126 | func NewDepNodesSerializer() *DepNodesSerializer { |
| 127 | return &DepNodesSerializer{ |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 128 | tsvMap: make(map[string]int), |
| 129 | targetMap: make(map[string]int), |
| 130 | done: make(map[string]bool), |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 131 | } |
| 132 | } |
| 133 | |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 134 | func (ns *DepNodesSerializer) SerializeTarget(t string) int { |
| 135 | id, present := ns.targetMap[t] |
| 136 | if present { |
| 137 | return id |
| 138 | } |
| 139 | id = len(ns.targets) |
| 140 | ns.targetMap[t] = id |
| 141 | ns.targets = append(ns.targets, t) |
| 142 | return id |
| 143 | } |
| 144 | |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 145 | func (ns *DepNodesSerializer) SerializeDepNodes(nodes []*DepNode) { |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 146 | for _, n := range nodes { |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 147 | if ns.done[n.Output] { |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 148 | continue |
| 149 | } |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 150 | ns.done[n.Output] = true |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 151 | |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 152 | var deps []int |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 153 | for _, d := range n.Deps { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 154 | deps = append(deps, ns.SerializeTarget(d.Output)) |
| 155 | } |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 156 | var parents []int |
| 157 | for _, d := range n.Parents { |
| 158 | parents = append(parents, ns.SerializeTarget(d.Output)) |
| 159 | } |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 160 | var actualInputs []int |
| 161 | for _, i := range n.ActualInputs { |
| 162 | actualInputs = append(actualInputs, ns.SerializeTarget(i)) |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 163 | } |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 164 | |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 165 | // Sort keys for consistent serialization. |
| 166 | var tsvKeys []string |
| 167 | for k := range n.TargetSpecificVars { |
| 168 | tsvKeys = append(tsvKeys, k) |
| 169 | } |
| 170 | sort.Strings(tsvKeys) |
| 171 | |
| 172 | var vars []int |
| 173 | for _, k := range tsvKeys { |
| 174 | v := n.TargetSpecificVars[k] |
| 175 | sv := SerializableTargetSpecificVar{Name: k, Value: v.Serialize()} |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 176 | //gob := encGob(sv) |
| 177 | gob := encVar(k, v) |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 178 | id, present := ns.tsvMap[gob] |
| 179 | if !present { |
| 180 | id = len(ns.tsvs) |
| 181 | ns.tsvMap[gob] = id |
| 182 | ns.tsvs = append(ns.tsvs, sv) |
| 183 | } |
| 184 | vars = append(vars, id) |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 185 | } |
| 186 | |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 187 | ns.nodes = append(ns.nodes, &SerializableDepNode{ |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 188 | Output: ns.SerializeTarget(n.Output), |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 189 | Cmds: n.Cmds, |
| 190 | Deps: deps, |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 191 | Parents: parents, |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 192 | HasRule: n.HasRule, |
| 193 | IsOrderOnly: n.IsOrderOnly, |
| 194 | IsPhony: n.IsPhony, |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 195 | ActualInputs: actualInputs, |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 196 | TargetSpecificVars: vars, |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 197 | Filename: n.Filename, |
| 198 | Lineno: n.Lineno, |
| 199 | }) |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 200 | ns.SerializeDepNodes(n.Deps) |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 201 | } |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 202 | } |
| 203 | |
Shinichiro Hamaji | c8bc731 | 2015-04-28 02:48:03 +0900 | [diff] [blame] | 204 | func MakeSerializableVars(vars Vars) (r map[string]SerializableVar) { |
| 205 | r = make(map[string]SerializableVar) |
| 206 | for k, v := range vars { |
| 207 | r[k] = v.Serialize() |
| 208 | } |
| 209 | return r |
| 210 | } |
| 211 | |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 212 | func MakeSerializableGraph(g *DepGraph, roots []string) SerializableGraph { |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 213 | ns := NewDepNodesSerializer() |
Shinichiro Hamaji | 750ae2e | 2015-05-20 12:59:15 +0900 | [diff] [blame] | 214 | ns.SerializeDepNodes(g.nodes) |
| 215 | v := MakeSerializableVars(g.vars) |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 216 | return SerializableGraph{ |
Shinichiro Hamaji | f3ea5d0 | 2015-05-20 17:13:00 +0900 | [diff] [blame] | 217 | Nodes: ns.nodes, |
| 218 | Vars: v, |
| 219 | Tsvs: ns.tsvs, |
| 220 | Targets: ns.targets, |
| 221 | Roots: roots, |
| 222 | ReadMks: g.readMks, |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 223 | } |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 224 | } |
Shinichiro Hamaji | c8bc731 | 2015-04-28 02:48:03 +0900 | [diff] [blame] | 225 | |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 226 | func DumpDepGraphAsJson(g *DepGraph, filename string, roots []string) { |
Shinichiro Hamaji | 750ae2e | 2015-05-20 12:59:15 +0900 | [diff] [blame] | 227 | sg := MakeSerializableGraph(g, roots) |
| 228 | o, err := json.MarshalIndent(sg, " ", " ") |
Shinichiro Hamaji | 0df24ec | 2015-04-27 21:29:53 +0900 | [diff] [blame] | 229 | if err != nil { |
| 230 | panic(err) |
| 231 | } |
| 232 | f, err2 := os.Create(filename) |
| 233 | if err2 != nil { |
| 234 | panic(err2) |
| 235 | } |
| 236 | f.Write(o) |
| 237 | } |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 238 | |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 239 | func DumpDepGraph(g *DepGraph, filename string, roots []string) { |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 240 | f, err := os.Create(filename) |
| 241 | if err != nil { |
| 242 | panic(err) |
| 243 | } |
| 244 | e := gob.NewEncoder(f) |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 245 | startTime := time.Now() |
Shinichiro Hamaji | 750ae2e | 2015-05-20 12:59:15 +0900 | [diff] [blame] | 246 | sg := MakeSerializableGraph(g, roots) |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 247 | LogStats("serialize prepare time: %q", time.Now().Sub(startTime)) |
| 248 | startTime = time.Now() |
Shinichiro Hamaji | 750ae2e | 2015-05-20 12:59:15 +0900 | [diff] [blame] | 249 | e.Encode(sg) |
Shinichiro Hamaji | 723f56a | 2015-05-15 17:12:55 +0900 | [diff] [blame] | 250 | LogStats("serialize output time: %q", time.Now().Sub(startTime)) |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 251 | } |
| 252 | |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 253 | func GetCacheFilename(mk string, roots []string) string { |
| 254 | filename := ".kati_cache." + mk |
| 255 | for _, r := range roots { |
| 256 | filename += "." + r |
| 257 | } |
| 258 | return url.QueryEscape(filename) |
| 259 | } |
| 260 | |
| 261 | func DumpDepGraphCache(g *DepGraph, roots []string) { |
| 262 | if len(g.readMks) == 0 { |
| 263 | panic("No Makefile is read") |
| 264 | } |
Shinichiro Hamaji | f3ea5d0 | 2015-05-20 17:13:00 +0900 | [diff] [blame] | 265 | cacheFile := GetCacheFilename(g.readMks[0].Filename, roots) |
| 266 | for _, mk := range g.readMks { |
| 267 | // Inconsistent, do not dump this result. |
Shinichiro Hamaji | 45cde1d | 2015-05-25 18:21:23 +0900 | [diff] [blame] | 268 | if mk.State == 2 { |
Shinichiro Hamaji | f3ea5d0 | 2015-05-20 17:13:00 +0900 | [diff] [blame] | 269 | if exists(cacheFile) { |
| 270 | os.Remove(cacheFile) |
| 271 | } |
| 272 | return |
| 273 | } |
| 274 | } |
| 275 | DumpDepGraph(g, cacheFile, roots) |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 276 | } |
| 277 | |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 278 | func DeserializeSingleChild(sv SerializableVar) Value { |
| 279 | if len(sv.Children) != 1 { |
| 280 | panic(fmt.Sprintf("unexpected number of children: %q", sv)) |
| 281 | } |
| 282 | return DeserializeVar(sv.Children[0]) |
| 283 | } |
| 284 | |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 285 | func DeserializeVar(sv SerializableVar) (r Value) { |
| 286 | switch sv.Type { |
| 287 | case "literal": |
| 288 | return literal(sv.V) |
Shinichiro Hamaji | c8dea2a | 2015-04-28 17:15:30 +0900 | [diff] [blame] | 289 | case "tmpval": |
| 290 | return tmpval([]byte(sv.V)) |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 291 | case "expr": |
| 292 | var e Expr |
| 293 | for _, v := range sv.Children { |
| 294 | e = append(e, DeserializeVar(v)) |
| 295 | } |
| 296 | return e |
| 297 | case "varref": |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 298 | return varref{varname: DeserializeSingleChild(sv)} |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 299 | case "paramref": |
| 300 | v, err := strconv.Atoi(sv.V) |
| 301 | if err != nil { |
| 302 | panic(err) |
| 303 | } |
| 304 | return paramref(v) |
Shinichiro Hamaji | c8dea2a | 2015-04-28 17:15:30 +0900 | [diff] [blame] | 305 | case "varsubst": |
| 306 | return varsubst{ |
| 307 | varname: DeserializeVar(sv.Children[0]), |
| 308 | pat: DeserializeVar(sv.Children[1]), |
| 309 | subst: DeserializeVar(sv.Children[2]), |
| 310 | } |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 311 | |
| 312 | case "func": |
| 313 | name := DeserializeVar(sv.Children[0]).(literal) |
| 314 | f := funcMap[string(name[1:])]() |
| 315 | f.AddArg(name) |
| 316 | for _, a := range sv.Children[1:] { |
| 317 | f.AddArg(DeserializeVar(a)) |
| 318 | } |
| 319 | return f |
| 320 | case "funcEvalAssign": |
| 321 | return &funcEvalAssign{ |
| 322 | lhs: sv.Children[0].V, |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 323 | op: sv.Children[1].V, |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 324 | rhs: DeserializeVar(sv.Children[2]), |
| 325 | } |
| 326 | case "funcNop": |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 327 | return &funcNop{expr: sv.V} |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 328 | |
| 329 | case "simple": |
| 330 | return SimpleVar{ |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 331 | value: []byte(sv.V), |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 332 | origin: sv.Origin, |
| 333 | } |
| 334 | case "recursive": |
| 335 | return RecursiveVar{ |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 336 | expr: DeserializeSingleChild(sv), |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 337 | origin: sv.Origin, |
| 338 | } |
Shinichiro Hamaji | f8f075c | 2015-04-28 04:25:37 +0900 | [diff] [blame] | 339 | |
| 340 | case ":=", "=", "+=", "?=": |
| 341 | return TargetSpecificVar{ |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 342 | v: DeserializeSingleChild(sv).(Var), |
Shinichiro Hamaji | f8f075c | 2015-04-28 04:25:37 +0900 | [diff] [blame] | 343 | op: sv.Type, |
| 344 | } |
| 345 | |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 346 | default: |
| 347 | panic(fmt.Sprintf("unknown serialized variable type: %q", sv)) |
| 348 | } |
| 349 | return UndefinedVar{} |
| 350 | } |
| 351 | |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 352 | func DeserializeVars(vars map[string]SerializableVar) Vars { |
Shinichiro Hamaji | 79a5693 | 2015-04-28 03:53:57 +0900 | [diff] [blame] | 353 | r := make(Vars) |
| 354 | for k, v := range vars { |
| 355 | r[k] = DeserializeVar(v).(Var) |
| 356 | } |
| 357 | return r |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 358 | } |
| 359 | |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 360 | func DeserializeNodes(g SerializableGraph) (r []*DepNode) { |
| 361 | nodes := g.Nodes |
| 362 | tsvs := g.Tsvs |
| 363 | targets := g.Targets |
Shinichiro Hamaji | 7a65e68 | 2015-04-28 18:00:12 +0900 | [diff] [blame] | 364 | // Deserialize all TSVs first so that multiple rules can share memory. |
| 365 | var tsvValues []Var |
| 366 | for _, sv := range tsvs { |
| 367 | tsvValues = append(tsvValues, DeserializeVar(sv.Value).(Var)) |
| 368 | } |
| 369 | |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 370 | nodeMap := make(map[string]*DepNode) |
| 371 | for _, n := range nodes { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 372 | var actualInputs []string |
| 373 | for _, i := range n.ActualInputs { |
| 374 | actualInputs = append(actualInputs, targets[i]) |
| 375 | } |
| 376 | |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 377 | d := &DepNode{ |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 378 | Output: targets[n.Output], |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 379 | Cmds: n.Cmds, |
| 380 | HasRule: n.HasRule, |
| 381 | IsOrderOnly: n.IsOrderOnly, |
| 382 | IsPhony: n.IsPhony, |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 383 | ActualInputs: actualInputs, |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 384 | Filename: n.Filename, |
| 385 | Lineno: n.Lineno, |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 386 | TargetSpecificVars: make(Vars), |
| 387 | } |
| 388 | |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 389 | for _, id := range n.TargetSpecificVars { |
| 390 | sv := tsvs[id] |
Shinichiro Hamaji | 7a65e68 | 2015-04-28 18:00:12 +0900 | [diff] [blame] | 391 | d.TargetSpecificVars[sv.Name] = tsvValues[id] |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 392 | } |
| 393 | |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 394 | nodeMap[targets[n.Output]] = d |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 395 | r = append(r, d) |
| 396 | } |
| 397 | |
| 398 | for _, n := range nodes { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 399 | d := nodeMap[targets[n.Output]] |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 400 | for _, o := range n.Deps { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 401 | c, present := nodeMap[targets[o]] |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 402 | if !present { |
| 403 | panic(fmt.Sprintf("unknown target: %s", o)) |
| 404 | } |
| 405 | d.Deps = append(d.Deps, c) |
| 406 | } |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 407 | for _, o := range n.Parents { |
| 408 | c, present := nodeMap[targets[o]] |
| 409 | if !present { |
| 410 | panic(fmt.Sprintf("unknown target: %s", o)) |
| 411 | } |
| 412 | d.Parents = append(d.Parents, c) |
| 413 | } |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 414 | } |
| 415 | |
| 416 | return r |
| 417 | } |
| 418 | |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 419 | func human(n int) string { |
Shinichiro Hamaji | 744b139 | 2015-05-14 23:29:14 +0900 | [diff] [blame] | 420 | if n >= 10*1000*1000*1000 { |
| 421 | return fmt.Sprintf("%.2fGB", float32(n)/1000/1000/1000) |
| 422 | } else if n >= 10*1000*1000 { |
| 423 | return fmt.Sprintf("%.2fMB", float32(n)/1000/1000) |
| 424 | } else if n >= 10*1000 { |
| 425 | return fmt.Sprintf("%.2fkB", float32(n)/1000) |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 426 | } else { |
| 427 | return fmt.Sprintf("%dB", n) |
| 428 | } |
| 429 | } |
| 430 | |
| 431 | func showSerializedNodesStats(nodes []*SerializableDepNode) { |
| 432 | outputSize := 0 |
| 433 | cmdSize := 0 |
| 434 | depsSize := 0 |
| 435 | actualInputSize := 0 |
| 436 | tsvSize := 0 |
| 437 | filenameSize := 0 |
| 438 | linenoSize := 0 |
| 439 | for _, n := range nodes { |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 440 | outputSize += 4 |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 441 | for _, c := range n.Cmds { |
| 442 | cmdSize += len(c) |
| 443 | } |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 444 | for _ = range n.Deps { |
| 445 | depsSize += 4 |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 446 | } |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 447 | for _ = range n.ActualInputs { |
| 448 | actualInputSize += 4 |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 449 | } |
| 450 | for _ = range n.TargetSpecificVars { |
| 451 | tsvSize += 4 |
| 452 | } |
| 453 | filenameSize += len(n.Filename) |
| 454 | linenoSize += 4 |
| 455 | } |
| 456 | size := outputSize + cmdSize + depsSize + actualInputSize + tsvSize + filenameSize + linenoSize |
| 457 | LogStats("%d nodes %s", len(nodes), human(size)) |
| 458 | LogStats(" output %s", human(outputSize)) |
| 459 | LogStats(" command %s", human(cmdSize)) |
| 460 | LogStats(" deps %s", human(depsSize)) |
| 461 | LogStats(" inputs %s", human(actualInputSize)) |
| 462 | LogStats(" tsv %s", human(tsvSize)) |
| 463 | LogStats(" filename %s", human(filenameSize)) |
| 464 | LogStats(" lineno %s", human(linenoSize)) |
| 465 | } |
| 466 | |
| 467 | func (v SerializableVar) size() int { |
| 468 | size := 0 |
| 469 | size += len(v.Type) |
| 470 | size += len(v.V) |
| 471 | size += len(v.Origin) |
| 472 | for _, c := range v.Children { |
| 473 | size += c.size() |
| 474 | } |
| 475 | return size |
| 476 | } |
| 477 | |
| 478 | func showSerializedVarsStats(vars map[string]SerializableVar) { |
| 479 | nameSize := 0 |
| 480 | valueSize := 0 |
| 481 | for k, v := range vars { |
| 482 | nameSize += len(k) |
| 483 | valueSize += v.size() |
| 484 | } |
| 485 | size := nameSize + valueSize |
| 486 | LogStats("%d vars %s", len(vars), human(size)) |
| 487 | LogStats(" name %s", human(nameSize)) |
| 488 | LogStats(" value %s", human(valueSize)) |
| 489 | } |
| 490 | |
| 491 | func showSerializedTsvsStats(vars []SerializableTargetSpecificVar) { |
| 492 | nameSize := 0 |
| 493 | valueSize := 0 |
| 494 | for _, v := range vars { |
| 495 | nameSize += len(v.Name) |
| 496 | valueSize += v.Value.size() |
| 497 | } |
| 498 | size := nameSize + valueSize |
| 499 | LogStats("%d tsvs %s", len(vars), human(size)) |
| 500 | LogStats(" name %s", human(nameSize)) |
| 501 | LogStats(" value %s", human(valueSize)) |
| 502 | } |
| 503 | |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 504 | func showSerializedTargetsStats(targets []string) { |
| 505 | size := 0 |
| 506 | for _, t := range targets { |
| 507 | size += len(t) |
| 508 | } |
| 509 | LogStats("%d targets %s", len(targets), human(size)) |
| 510 | } |
| 511 | |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 512 | func showSerializedGraphStats(g SerializableGraph) { |
| 513 | showSerializedNodesStats(g.Nodes) |
| 514 | showSerializedVarsStats(g.Vars) |
| 515 | showSerializedTsvsStats(g.Tsvs) |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 516 | showSerializedTargetsStats(g.Targets) |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 517 | } |
| 518 | |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 519 | func DeserializeGraph(g SerializableGraph) *DepGraph { |
Shinichiro Hamaji | 0962e86 | 2015-04-28 23:58:57 +0900 | [diff] [blame] | 520 | if katiLogFlag || katiStatsFlag { |
| 521 | showSerializedGraphStats(g) |
| 522 | } |
Shinichiro Hamaji | f013e4b | 2015-04-29 00:21:54 +0900 | [diff] [blame] | 523 | nodes := DeserializeNodes(g) |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 524 | vars := DeserializeVars(g.Vars) |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 525 | return &DepGraph{ |
Shinichiro Hamaji | f3ea5d0 | 2015-05-20 17:13:00 +0900 | [diff] [blame] | 526 | nodes: nodes, |
| 527 | vars: vars, |
| 528 | readMks: g.ReadMks, |
Shinichiro Hamaji | 750ae2e | 2015-05-20 12:59:15 +0900 | [diff] [blame] | 529 | } |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 530 | } |
| 531 | |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 532 | func LoadDepGraphFromJson(filename string) *DepGraph { |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 533 | f, err := os.Open(filename) |
| 534 | if err != nil { |
| 535 | panic(err) |
| 536 | } |
| 537 | |
| 538 | d := json.NewDecoder(f) |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 539 | g := SerializableGraph{Vars: make(map[string]SerializableVar)} |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 540 | err = d.Decode(&g) |
| 541 | if err != nil { |
| 542 | panic(err) |
| 543 | } |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 544 | return DeserializeGraph(g) |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 545 | } |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 546 | |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 547 | func LoadDepGraph(filename string) *DepGraph { |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 548 | f, err := os.Open(filename) |
| 549 | if err != nil { |
| 550 | panic(err) |
| 551 | } |
| 552 | |
| 553 | d := gob.NewDecoder(f) |
Shinichiro Hamaji | 3d6d0aa | 2015-04-28 16:18:44 +0900 | [diff] [blame] | 554 | g := SerializableGraph{Vars: make(map[string]SerializableVar)} |
Shinichiro Hamaji | 1833c28 | 2015-04-28 05:02:27 +0900 | [diff] [blame] | 555 | err = d.Decode(&g) |
| 556 | if err != nil { |
| 557 | panic(err) |
| 558 | } |
Shinichiro Hamaji | 530136b | 2015-04-28 17:12:48 +0900 | [diff] [blame] | 559 | return DeserializeGraph(g) |
Shinichiro Hamaji | 8c9d0e3 | 2015-04-28 03:30:19 +0900 | [diff] [blame] | 560 | } |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 561 | |
Shinichiro Hamaji | f3ea5d0 | 2015-05-20 17:13:00 +0900 | [diff] [blame] | 562 | func LoadDepGraphCache(makefile string, roots []string) *DepGraph { |
Shinichiro Hamaji | 82ecff9 | 2015-05-25 18:36:04 +0900 | [diff] [blame^] | 563 | startTime := time.Now() |
| 564 | defer func() { |
| 565 | LogStats("Cache lookup time: %q", time.Now().Sub(startTime)) |
| 566 | }() |
| 567 | |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 568 | filename := GetCacheFilename(makefile, roots) |
Shinichiro Hamaji | 39e6240 | 2015-05-20 19:34:58 +0900 | [diff] [blame] | 569 | if !exists(filename) { |
| 570 | LogAlways("Cache not found") |
| 571 | return nil |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 572 | } |
Shinichiro Hamaji | 39e6240 | 2015-05-20 19:34:58 +0900 | [diff] [blame] | 573 | |
| 574 | g := LoadDepGraph(filename) |
| 575 | for _, mk := range g.readMks { |
Shinichiro Hamaji | 45cde1d | 2015-05-25 18:21:23 +0900 | [diff] [blame] | 576 | if mk.State != FILE_EXISTS && mk.State != FILE_NOT_EXISTS { |
| 577 | panic(fmt.Sprintf("Internal error: broken state: %d", mk.State)) |
Shinichiro Hamaji | 0ba75a9 | 2015-05-20 19:42:43 +0900 | [diff] [blame] | 578 | } |
Shinichiro Hamaji | 45cde1d | 2015-05-25 18:21:23 +0900 | [diff] [blame] | 579 | if mk.State == FILE_NOT_EXISTS { |
| 580 | if exists(mk.Filename) { |
| 581 | LogAlways("Cache expired: %s", mk.Filename) |
| 582 | return nil |
| 583 | } |
| 584 | } else { |
| 585 | c, err := readFile(mk.Filename) |
| 586 | if err != nil || !bytes.Equal(c, mk.Content) { |
| 587 | LogAlways("Cache expired: %s", mk.Filename) |
| 588 | return nil |
| 589 | } |
Shinichiro Hamaji | 39e6240 | 2015-05-20 19:34:58 +0900 | [diff] [blame] | 590 | } |
| 591 | } |
| 592 | g.isCached = true |
| 593 | LogAlways("Cache found!") |
| 594 | return g |
Shinichiro Hamaji | b0d2e2f | 2015-05-20 16:42:59 +0900 | [diff] [blame] | 595 | } |