blob: 4a938db45aaeea0c458ab9b1f74bf4298126836e [file] [log] [blame]
Colin Cross8e0c5112015-01-23 14:15:10 -08001// Copyright 2014 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
Jamie Gennis1bc967e2014-05-27 16:34:41 -070015package bootstrap
16
17import (
Colin Crossde7afaa2019-01-23 13:23:00 -080018 "bufio"
Jamie Gennis1bc967e2014-05-27 16:34:41 -070019 "flag"
20 "fmt"
Colin Crossde7afaa2019-01-23 13:23:00 -080021 "io"
Jamie Gennis1bc967e2014-05-27 16:34:41 -070022 "io/ioutil"
23 "os"
Jamie Gennisc4ed7092014-11-09 11:58:40 -080024 "path/filepath"
Colin Cross5bdb4ca2015-04-14 17:22:19 -070025 "runtime"
Colin Cross2ecec572016-05-17 13:56:21 -070026 "runtime/debug"
Colin Cross63d5d4d2015-04-20 16:41:55 -070027 "runtime/pprof"
Colin Crossaec881d2016-08-05 13:59:43 -070028 "runtime/trace"
Colin Cross5bdb4ca2015-04-14 17:22:19 -070029
30 "github.com/google/blueprint"
31 "github.com/google/blueprint/deptools"
Jamie Gennis1bc967e2014-05-27 16:34:41 -070032)
33
Lukacs T. Berki7ea1c162021-03-16 08:54:33 +010034type Args struct {
Lukacs T. Berkib3b9cb62021-03-16 09:11:10 +010035 OutFile string
36 GlobFile string
37 DepFile string
38 DocFile string
39 Cpuprofile string
40 Memprofile string
Lukacs T. Berki77ef79b2021-04-12 12:07:02 +020041 DelveListen string
42 DelvePath string
Lukacs T. Berkib3b9cb62021-03-16 09:11:10 +010043 TraceFile string
44 RunGoTests bool
45 UseValidations bool
46 NoGC bool
47 EmptyNinjaFile bool
48 BuildDir string
49 ModuleListFile string
50 NinjaBuildDir string
51 TopFile string
52 GeneratingPrimaryBuilder bool
Lukacs T. Berki78df8532021-04-14 10:28:54 +020053
54 PrimaryBuilderInvocations []PrimaryBuilderInvocation
Lukacs T. Berki7ea1c162021-03-16 08:54:33 +010055}
56
57var (
Lukacs T. Berki98e0efb2021-04-14 13:47:52 +020058 CmdlineArgs Args
59 absSrcDir string
Jamie Gennis7d5b2f82014-09-24 17:51:52 -070060)
Jamie Gennis1bc967e2014-05-27 16:34:41 -070061
Jamie Gennis1bc967e2014-05-27 16:34:41 -070062func init() {
Lukacs T. Berki98e0efb2021-04-14 13:47:52 +020063 flag.StringVar(&CmdlineArgs.OutFile, "o", "build.ninja", "the Ninja file to output")
64 flag.StringVar(&CmdlineArgs.GlobFile, "globFile", "build-globs.ninja", "the Ninja file of globs to output")
65 flag.StringVar(&CmdlineArgs.BuildDir, "b", ".", "the build output directory")
66 flag.StringVar(&CmdlineArgs.NinjaBuildDir, "n", "", "the ninja builddir directory")
67 flag.StringVar(&CmdlineArgs.DepFile, "d", "", "the dependency file to output")
68 flag.StringVar(&CmdlineArgs.DocFile, "docs", "", "build documentation file to output")
69 flag.StringVar(&CmdlineArgs.Cpuprofile, "cpuprofile", "", "write cpu profile to file")
70 flag.StringVar(&CmdlineArgs.TraceFile, "trace", "", "write trace to file")
71 flag.StringVar(&CmdlineArgs.Memprofile, "memprofile", "", "write memory profile to file")
72 flag.BoolVar(&CmdlineArgs.NoGC, "nogc", false, "turn off GC for debugging")
73 flag.BoolVar(&CmdlineArgs.RunGoTests, "t", false, "build and run go tests during bootstrap")
74 flag.BoolVar(&CmdlineArgs.UseValidations, "use-validations", false, "use validations to depend on go tests")
75 flag.StringVar(&CmdlineArgs.ModuleListFile, "l", "", "file that lists filepaths to parse")
76 flag.BoolVar(&CmdlineArgs.EmptyNinjaFile, "empty-ninja-file", false, "write out a 0-byte ninja file")
Jamie Gennis1bc967e2014-05-27 16:34:41 -070077}
78
Lukacs T. Berkif4d43ac2021-04-15 15:05:26 +020079func Main(ctx *blueprint.Context, config interface{}, generatingPrimaryBuilder bool) {
Jamie Gennis1bc967e2014-05-27 16:34:41 -070080 if !flag.Parsed() {
81 flag.Parse()
82 }
83
Lukacs T. Berki7ea1c162021-03-16 08:54:33 +010084 if flag.NArg() != 1 {
85 fatalf("no Blueprints file specified")
86 }
87
Lukacs T. Berki98e0efb2021-04-14 13:47:52 +020088 CmdlineArgs.TopFile = flag.Arg(0)
89 CmdlineArgs.GeneratingPrimaryBuilder = generatingPrimaryBuilder
Lukacs T. Berkif4d43ac2021-04-15 15:05:26 +020090 ninjaDeps := RunBlueprint(CmdlineArgs, ctx, config)
91 err := deptools.WriteDepFile(CmdlineArgs.DepFile, CmdlineArgs.OutFile, ninjaDeps)
Lukacs T. Berki98e0efb2021-04-14 13:47:52 +020092 if err != nil {
93 fatalf("Cannot write depfile '%s': %s", CmdlineArgs.DepFile, err)
94 }
Lukacs T. Berki7ea1c162021-03-16 08:54:33 +010095}
96
Lukacs T. Berki78df8532021-04-14 10:28:54 +020097func PrimaryBuilderExtraFlags(args Args, globFile, mainNinjaFile string) []string {
Lukacs T. Berki77ef79b2021-04-12 12:07:02 +020098 result := make([]string, 0)
99
100 if args.RunGoTests {
101 result = append(result, "-t")
102 }
103
104 result = append(result, "-l", args.ModuleListFile)
105 result = append(result, "-globFile", globFile)
106 result = append(result, "-o", mainNinjaFile)
107
108 if args.EmptyNinjaFile {
109 result = append(result, "--empty-ninja-file")
110 }
111
112 if args.DelveListen != "" {
113 result = append(result, "--delve_listen", args.DelveListen)
114 }
115
116 if args.DelvePath != "" {
117 result = append(result, "--delve_path", args.DelvePath)
118 }
119
Lukacs T. Berki77ef79b2021-04-12 12:07:02 +0200120 return result
121}
122
123func writeEmptyGlobFile(path string) {
124 err := os.MkdirAll(filepath.Dir(path), 0777)
125 if err != nil {
126 fatalf("Failed to create parent directories of empty ninja glob file '%s': %s", path, err)
127 }
128
129 if _, err := os.Stat(path); os.IsNotExist(err) {
130 err = ioutil.WriteFile(path, nil, 0666)
131 if err != nil {
132 fatalf("Failed to create empty ninja glob file '%s': %s", path, err)
133 }
134 }
135}
Lukacs T. Berkif4d43ac2021-04-15 15:05:26 +0200136
137// Returns the list of dependencies the emitted Ninja files has. These can be
138// written to the .d file for the output so that it is correctly rebuilt when
139// needed in case Blueprint is itself invoked from Ninja
140func RunBlueprint(args Args, ctx *blueprint.Context, config interface{}) []string {
Colin Cross5bdb4ca2015-04-14 17:22:19 -0700141 runtime.GOMAXPROCS(runtime.NumCPU())
142
Lukacs T. Berki7ea1c162021-03-16 08:54:33 +0100143 if args.NoGC {
Colin Cross2ecec572016-05-17 13:56:21 -0700144 debug.SetGCPercent(-1)
145 }
146
Colin Crossc5fa50e2019-12-17 13:12:35 -0800147 absSrcDir = ctx.SrcDir()
148
Lukacs T. Berki7ea1c162021-03-16 08:54:33 +0100149 if args.Cpuprofile != "" {
150 f, err := os.Create(absolutePath(args.Cpuprofile))
Jamie Gennis7d5b2f82014-09-24 17:51:52 -0700151 if err != nil {
152 fatalf("error opening cpuprofile: %s", err)
153 }
154 pprof.StartCPUProfile(f)
155 defer f.Close()
156 defer pprof.StopCPUProfile()
157 }
158
Lukacs T. Berki7ea1c162021-03-16 08:54:33 +0100159 if args.TraceFile != "" {
160 f, err := os.Create(absolutePath(args.TraceFile))
Colin Crossaec881d2016-08-05 13:59:43 -0700161 if err != nil {
162 fatalf("error opening trace: %s", err)
163 }
164 trace.Start(f)
165 defer f.Close()
166 defer trace.Stop()
167 }
168
Lukacs T. Berki7ea1c162021-03-16 08:54:33 +0100169 srcDir := filepath.Dir(args.TopFile)
Jamie Gennis1bc967e2014-05-27 16:34:41 -0700170
Lukacs T. Berkif4d43ac2021-04-15 15:05:26 +0200171 ninjaDeps := make([]string, 0)
172
Lukacs T. Berki7ea1c162021-03-16 08:54:33 +0100173 if args.ModuleListFile != "" {
174 ctx.SetModuleListFile(args.ModuleListFile)
Lukacs T. Berkif4d43ac2021-04-15 15:05:26 +0200175 ninjaDeps = append(ninjaDeps, args.ModuleListFile)
Jeff Gastonc3e28442017-08-09 15:13:12 -0700176 } else {
177 fatalf("-l <moduleListFile> is required and must be nonempty")
178 }
Lukacs T. Berki7ea1c162021-03-16 08:54:33 +0100179 filesToParse, err := ctx.ListModulePaths(srcDir)
Jeff Gastonc3e28442017-08-09 15:13:12 -0700180 if err != nil {
181 fatalf("could not enumerate files: %v\n", err.Error())
182 }
183
Lukacs T. Berkif646afb2021-04-15 15:46:35 +0200184 buildDir := config.(BootstrapConfig).BuildDir()
Dan Willemsencd4e0ce2017-07-19 22:43:30 -0700185
Dan Willemsen91a657e2015-07-22 17:05:59 -0700186 stage := StageMain
Lukacs T. Berkib3b9cb62021-03-16 09:11:10 +0100187 if args.GeneratingPrimaryBuilder {
188 stage = StagePrimary
Dan Willemsen30a80c32015-06-24 19:21:21 -0700189 }
Jamie Gennis1bc967e2014-05-27 16:34:41 -0700190
Lukacs T. Berki77ef79b2021-04-12 12:07:02 +0200191 primaryBuilderNinjaGlobFile := absolutePath(filepath.Join(args.BuildDir, bootstrapSubDir, "build-globs.ninja"))
192 mainNinjaFile := filepath.Join("$buildDir", "build.ninja")
193
194 writeEmptyGlobFile(primaryBuilderNinjaGlobFile)
195
Lukacs T. Berki78df8532021-04-14 10:28:54 +0200196 var invocations []PrimaryBuilderInvocation
197
198 if args.PrimaryBuilderInvocations != nil {
199 invocations = args.PrimaryBuilderInvocations
200 } else {
201 primaryBuilderArgs := PrimaryBuilderExtraFlags(args, primaryBuilderNinjaGlobFile, mainNinjaFile)
202 primaryBuilderArgs = append(primaryBuilderArgs, args.TopFile)
203
204 invocations = []PrimaryBuilderInvocation{{
205 Inputs: []string{args.TopFile},
206 Outputs: []string{mainNinjaFile},
207 Args: primaryBuilderArgs,
208 }}
209 }
210
Dan Willemsen30a80c32015-06-24 19:21:21 -0700211 bootstrapConfig := &Config{
Dan Willemsen91a657e2015-07-22 17:05:59 -0700212 stage: stage,
Dan Willemsendac90d32018-10-25 22:02:10 -0700213
Lukacs T. Berki78df8532021-04-14 10:28:54 +0200214 topLevelBlueprintsFile: args.TopFile,
215 globFile: primaryBuilderNinjaGlobFile,
216 runGoTests: args.RunGoTests,
217 useValidations: args.UseValidations,
218 primaryBuilderInvocations: invocations,
Dan Willemsen30a80c32015-06-24 19:21:21 -0700219 }
220
Dan Willemsenfdeb7242015-07-24 16:53:27 -0700221 ctx.RegisterBottomUpMutator("bootstrap_plugin_deps", pluginDeps)
Dan Willemsen30a80c32015-06-24 19:21:21 -0700222 ctx.RegisterModuleType("bootstrap_go_package", newGoPackageModuleFactory(bootstrapConfig))
Dan Willemsenaf456ea2017-07-19 19:22:34 -0700223 ctx.RegisterModuleType("bootstrap_go_binary", newGoBinaryModuleFactory(bootstrapConfig, false))
224 ctx.RegisterModuleType("blueprint_go_binary", newGoBinaryModuleFactory(bootstrapConfig, true))
Dan Willemsen30a80c32015-06-24 19:21:21 -0700225 ctx.RegisterSingletonType("bootstrap", newSingletonFactory(bootstrapConfig))
226
Colin Cross25236982021-04-05 17:20:34 -0700227 ctx.RegisterSingletonType("glob", globSingletonFactory(bootstrapConfig, ctx))
Colin Cross127d2ea2016-11-01 11:10:51 -0700228
Lukacs T. Berkif4d43ac2021-04-15 15:05:26 +0200229 blueprintFiles, errs := ctx.ParseFileList(filepath.Dir(args.TopFile), filesToParse, config)
Jamie Gennis1bc967e2014-05-27 16:34:41 -0700230 if len(errs) > 0 {
231 fatalErrors(errs)
232 }
233
Christopher Horvath3159cb72014-06-26 15:34:06 -0700234 // Add extra ninja file dependencies
Lukacs T. Berkif4d43ac2021-04-15 15:05:26 +0200235 ninjaDeps = append(ninjaDeps, blueprintFiles...)
Christopher Horvath3159cb72014-06-26 15:34:06 -0700236
Colin Cross874a3462017-07-31 17:26:06 -0700237 extraDeps, errs := ctx.ResolveDependencies(config)
Colin Cross4572edd2015-05-13 14:36:24 -0700238 if len(errs) > 0 {
239 fatalErrors(errs)
240 }
Lukacs T. Berkif4d43ac2021-04-15 15:05:26 +0200241 ninjaDeps = append(ninjaDeps, extraDeps...)
Colin Cross4572edd2015-05-13 14:36:24 -0700242
Lukacs T. Berki7ea1c162021-03-16 08:54:33 +0100243 if args.DocFile != "" {
244 err := writeDocs(ctx, config, absolutePath(args.DocFile))
Colin Cross4572edd2015-05-13 14:36:24 -0700245 if err != nil {
246 fatalErrors([]error{err})
247 }
Lukacs T. Berki98e0efb2021-04-14 13:47:52 +0200248 return nil
Colin Cross4572edd2015-05-13 14:36:24 -0700249 }
250
Colin Cross28b28432017-12-11 15:03:11 -0800251 if c, ok := config.(ConfigStopBefore); ok {
252 if c.StopBefore() == StopBeforePrepareBuildActions {
Lukacs T. Berkif4d43ac2021-04-15 15:05:26 +0200253 return ninjaDeps
Colin Cross28b28432017-12-11 15:03:11 -0800254 }
255 }
256
Colin Cross874a3462017-07-31 17:26:06 -0700257 extraDeps, errs = ctx.PrepareBuildActions(config)
Jamie Gennis1bc967e2014-05-27 16:34:41 -0700258 if len(errs) > 0 {
259 fatalErrors(errs)
260 }
Lukacs T. Berkif4d43ac2021-04-15 15:05:26 +0200261 ninjaDeps = append(ninjaDeps, extraDeps...)
Mathias Agopian5b8477d2014-06-25 17:21:54 -0700262
Chris Parsons5e834262020-11-05 11:17:32 -0500263 if c, ok := config.(ConfigStopBefore); ok {
264 if c.StopBefore() == StopBeforeWriteNinja {
Lukacs T. Berkif4d43ac2021-04-15 15:05:26 +0200265 return ninjaDeps
Chris Parsons5e834262020-11-05 11:17:32 -0500266 }
267 }
268
Jamie Genniscbc6f862014-06-05 20:00:22 -0700269 const outFilePermissions = 0666
Colin Cross0335e092021-01-21 15:26:21 -0800270 var out io.StringWriter
Colin Crossde7afaa2019-01-23 13:23:00 -0800271 var f *os.File
272 var buf *bufio.Writer
273
Lukacs T. Berki7ea1c162021-03-16 08:54:33 +0100274 if args.EmptyNinjaFile {
275 if err := ioutil.WriteFile(absolutePath(args.OutFile), []byte(nil), outFilePermissions); err != nil {
Dan Willemsen6c6c1032020-04-19 12:50:48 -0700276 fatalf("error writing empty Ninja file: %s", err)
277 }
278 }
279
Lukacs T. Berki7ea1c162021-03-16 08:54:33 +0100280 if stage != StageMain || !args.EmptyNinjaFile {
281 f, err = os.OpenFile(absolutePath(args.OutFile), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, outFilePermissions)
Colin Crossde7afaa2019-01-23 13:23:00 -0800282 if err != nil {
283 fatalf("error opening Ninja file: %s", err)
284 }
Colin Crossc8b9e552021-01-21 15:28:04 -0800285 buf = bufio.NewWriterSize(f, 16*1024*1024)
Colin Crossde7afaa2019-01-23 13:23:00 -0800286 out = buf
287 } else {
Colin Cross0335e092021-01-21 15:26:21 -0800288 out = ioutil.Discard.(io.StringWriter)
Colin Crossde7afaa2019-01-23 13:23:00 -0800289 }
290
Lukacs T. Berki7ea1c162021-03-16 08:54:33 +0100291 if args.GlobFile != "" {
Colin Cross25236982021-04-05 17:20:34 -0700292 buffer, errs := generateGlobNinjaFile(bootstrapConfig, config, ctx.Globs)
Dan Willemsenab223a52018-07-05 21:56:59 -0700293 if len(errs) > 0 {
294 fatalErrors(errs)
295 }
296
Lukacs T. Berki7ea1c162021-03-16 08:54:33 +0100297 err = ioutil.WriteFile(absolutePath(args.GlobFile), buffer, outFilePermissions)
Dan Willemsenab223a52018-07-05 21:56:59 -0700298 if err != nil {
Lukacs T. Berki7ea1c162021-03-16 08:54:33 +0100299 fatalf("error writing %s: %s", args.GlobFile, err)
Dan Willemsenab223a52018-07-05 21:56:59 -0700300 }
301 }
302
Colin Cross1cda3fd2020-01-16 10:32:35 -0800303 err = ctx.WriteBuildFile(out)
304 if err != nil {
305 fatalf("error writing Ninja file contents: %s", err)
306 }
307
308 if buf != nil {
309 err = buf.Flush()
310 if err != nil {
311 fatalf("error flushing Ninja file contents: %s", err)
312 }
313 }
314
315 if f != nil {
316 err = f.Close()
317 if err != nil {
318 fatalf("error closing Ninja file: %s", err)
319 }
320 }
321
Dan Willemsencd4e0ce2017-07-19 22:43:30 -0700322 if c, ok := config.(ConfigRemoveAbandonedFilesUnder); ok {
Lukacs T. Berki7ea1c162021-03-16 08:54:33 +0100323 under, except := c.RemoveAbandonedFilesUnder(buildDir)
324 err := removeAbandonedFilesUnder(ctx, srcDir, buildDir, under, except)
Colin Cross6d529f02015-11-17 16:16:58 -0800325 if err != nil {
326 fatalf("error removing abandoned files: %s", err)
327 }
Jamie Gennisaf435562014-10-27 22:34:56 -0700328 }
Colin Cross2ecec572016-05-17 13:56:21 -0700329
Lukacs T. Berki7ea1c162021-03-16 08:54:33 +0100330 if args.Memprofile != "" {
331 f, err := os.Create(absolutePath(args.Memprofile))
Colin Cross2ecec572016-05-17 13:56:21 -0700332 if err != nil {
333 fatalf("error opening memprofile: %s", err)
334 }
335 defer f.Close()
336 pprof.WriteHeapProfile(f)
337 }
Lukacs T. Berki98e0efb2021-04-14 13:47:52 +0200338
Lukacs T. Berkif4d43ac2021-04-15 15:05:26 +0200339 return ninjaDeps
Jamie Gennis1bc967e2014-05-27 16:34:41 -0700340}
341
342func fatalf(format string, args ...interface{}) {
Jamie Genniscbc6f862014-06-05 20:00:22 -0700343 fmt.Printf(format, args...)
George Kulakowski4904c8a2015-08-21 14:47:06 -0700344 fmt.Print("\n")
Jamie Gennis1bc967e2014-05-27 16:34:41 -0700345 os.Exit(1)
346}
347
348func fatalErrors(errs []error) {
Colin Crossea599542016-01-07 11:16:48 -0800349 red := "\x1b[31m"
350 unred := "\x1b[0m"
351
Jamie Gennis1bc967e2014-05-27 16:34:41 -0700352 for _, err := range errs {
Colin Crossea599542016-01-07 11:16:48 -0800353 switch err := err.(type) {
Colin Cross2c628442016-10-07 17:13:10 -0700354 case *blueprint.BlueprintError,
355 *blueprint.ModuleError,
356 *blueprint.PropertyError:
Colin Crossea599542016-01-07 11:16:48 -0800357 fmt.Printf("%serror:%s %s\n", red, unred, err.Error())
Jamie Gennis1bc967e2014-05-27 16:34:41 -0700358 default:
Colin Crossea599542016-01-07 11:16:48 -0800359 fmt.Printf("%sinternal error:%s %s\n", red, unred, err)
Jamie Gennis1bc967e2014-05-27 16:34:41 -0700360 }
361 }
362 os.Exit(1)
363}
Colin Crossc5fa50e2019-12-17 13:12:35 -0800364
365func absolutePath(path string) string {
366 if filepath.IsAbs(path) {
367 return path
368 }
369 return filepath.Join(absSrcDir, path)
370}