Move proto compilation to RuleBuilder
Using blueprint.Rule for protoc commands was causing code duplication
because there was no good way to run the same protoc for cc, java and
python but then run custom source packaging steps for java and python.
Move most of the code into a common function that returns a
RuleBuilder, and then let java and python add their own commands at
the end of the rule.
Bug: 70706119
Test: All Soong tests
Test: m checkbuild
Change-Id: Ic692136775d273bcc4f4de99620ab4878667c83a
diff --git a/Android.bp b/Android.bp
index 7465341..a4e6e7d 100644
--- a/Android.bp
+++ b/Android.bp
@@ -193,6 +193,7 @@
"cc/gen_test.go",
"cc/genrule_test.go",
"cc/library_test.go",
+ "cc/proto_test.go",
"cc/test_data_test.go",
"cc/util_test.go",
],
diff --git a/android/proto.go b/android/proto.go
index 801837e..83dc32a 100644
--- a/android/proto.go
+++ b/android/proto.go
@@ -14,6 +14,12 @@
package android
+import (
+ "strings"
+
+ "github.com/google/blueprint/proptools"
+)
+
// TODO(ccross): protos are often used to communicate between multiple modules. If the only
// way to convert a proto to source is to reference it as a source file, and external modules cannot
// reference source files in other modules, then every module that owns a proto file will need to
@@ -22,9 +28,17 @@
// and then external modules could depend on the proto module but use their own settings to
// generate the source.
-func ProtoFlags(ctx ModuleContext, p *ProtoProperties) []string {
- protoFlags := []string{}
+type ProtoFlags struct {
+ Flags []string
+ CanonicalPathFromRoot bool
+ Dir ModuleGenPath
+ SubDir ModuleGenPath
+ OutTypeFlag string
+ OutParams []string
+}
+func GetProtoFlags(ctx ModuleContext, p *ProtoProperties) ProtoFlags {
+ var protoFlags []string
if len(p.Proto.Local_include_dirs) > 0 {
localProtoIncludeDirs := PathsForModuleSrc(ctx, p.Proto.Local_include_dirs)
protoFlags = append(protoFlags, JoinWithPrefix(localProtoIncludeDirs.Strings(), "-I"))
@@ -34,24 +48,12 @@
protoFlags = append(protoFlags, JoinWithPrefix(rootProtoIncludeDirs.Strings(), "-I"))
}
- return protoFlags
-}
-
-func ProtoCanonicalPathFromRoot(ctx ModuleContext, p *ProtoProperties) bool {
- if p.Proto.Canonical_path_from_root == nil {
- return true
+ return ProtoFlags{
+ Flags: protoFlags,
+ CanonicalPathFromRoot: proptools.BoolDefault(p.Proto.Canonical_path_from_root, true),
+ Dir: PathForModuleGen(ctx, "proto"),
+ SubDir: PathForModuleGen(ctx, "proto", ctx.ModuleDir()),
}
- return *p.Proto.Canonical_path_from_root
-}
-
-// ProtoDir returns the module's "gen/proto" directory
-func ProtoDir(ctx ModuleContext) ModuleGenPath {
- return PathForModuleGen(ctx, "proto")
-}
-
-// ProtoSubDir returns the module's "gen/proto/path/to/module" directory
-func ProtoSubDir(ctx ModuleContext) ModuleGenPath {
- return PathForModuleGen(ctx, "proto", ctx.ModuleDir())
}
type ProtoProperties struct {
@@ -76,3 +78,28 @@
Canonical_path_from_root *bool
} `android:"arch_variant"`
}
+
+func ProtoRule(ctx ModuleContext, rule *RuleBuilder, protoFile Path, flags ProtoFlags, deps Paths,
+ outDir WritablePath, depFile WritablePath, outputs WritablePaths) {
+
+ var protoBase string
+ if flags.CanonicalPathFromRoot {
+ protoBase = "."
+ } else {
+ rel := protoFile.Rel()
+ protoBase = strings.TrimSuffix(protoFile.String(), rel)
+ }
+
+ rule.Command().
+ Tool(ctx.Config().HostToolPath(ctx, "aprotoc")).
+ FlagWithArg(flags.OutTypeFlag+"=", strings.Join(flags.OutParams, ",")+":"+outDir.String()).
+ FlagWithDepFile("--dependency_out=", depFile).
+ FlagWithArg("-I ", protoBase).
+ Flags(flags.Flags).
+ Input(protoFile).
+ Implicits(deps).
+ ImplicitOutputs(outputs)
+
+ rule.Command().
+ Tool(ctx.Config().HostToolPath(ctx, "dep_fixer")).Flag(depFile.String())
+}
diff --git a/cc/builder.go b/cc/builder.go
index 6dd7c05..6b139f4 100644
--- a/cc/builder.go
+++ b/cc/builder.go
@@ -261,12 +261,9 @@
stripUseGnuStrip bool
protoDeps android.Paths
- protoFlags string
- protoOutTypeFlag string
- protoOutParams string
+ proto android.ProtoFlags
protoC bool
protoOptionsFile bool
- protoRoot bool
}
type Objects struct {
diff --git a/cc/cc.go b/cc/cc.go
index f5a1567..8e55553 100644
--- a/cc/cc.go
+++ b/cc/cc.go
@@ -162,13 +162,10 @@
GroupStaticLibs bool
+ proto android.ProtoFlags
protoDeps android.Paths
- protoFlags []string // Flags that apply to proto source files
- protoOutTypeFlag string // The output type, --cpp_out for example
- protoOutParams []string // Flags that modify the output of proto generated files
- protoC bool // Whether to use C instead of C++
- protoOptionsFile bool // Whether to look for a .options file next to the .proto
- ProtoRoot bool
+ protoC bool // Whether to use C instead of C++
+ protoOptionsFile bool // Whether to look for a .options file next to the .proto
}
type ObjectLinkerProperties struct {
diff --git a/cc/library.go b/cc/library.go
index cab75ac..5556fd4 100644
--- a/cc/library.go
+++ b/cc/library.go
@@ -848,10 +848,10 @@
if Bool(library.Properties.Proto.Export_proto_headers) {
if library.baseCompiler.hasSrcExt(".proto") {
includes := []string{}
- if flags.ProtoRoot {
- includes = append(includes, "-I"+android.ProtoSubDir(ctx).String())
+ if flags.proto.CanonicalPathFromRoot {
+ includes = append(includes, "-I"+flags.proto.SubDir.String())
}
- includes = append(includes, "-I"+android.ProtoDir(ctx).String())
+ includes = append(includes, "-I"+flags.proto.Dir.String())
library.reexportFlags(includes)
library.reuseExportedFlags = append(library.reuseExportedFlags, includes...)
library.reexportDeps(library.baseCompiler.pathDeps) // TODO: restrict to proto deps
diff --git a/cc/proto.go b/cc/proto.go
index ce8a30e..5a96ae7 100644
--- a/cc/proto.go
+++ b/cc/proto.go
@@ -15,47 +15,26 @@
package cc
import (
- "strings"
-
- "github.com/google/blueprint"
"github.com/google/blueprint/pathtools"
"android/soong/android"
)
-func init() {
- pctx.HostBinToolVariable("protocCmd", "aprotoc")
- pctx.HostBinToolVariable("depFixCmd", "dep_fixer")
-}
-
-var (
- proto = pctx.AndroidStaticRule("protoc",
- blueprint.RuleParams{
- Command: "$protocCmd $protoOut=$protoOutParams:$outDir --dependency_out=$out.d -I $protoBase $protoFlags $in && " +
- `$depFixCmd $out.d`,
- CommandDeps: []string{"$protocCmd", "$depFixCmd"},
- Depfile: "${out}.d",
- Deps: blueprint.DepsGCC,
- }, "protoFlags", "protoOut", "protoOutParams", "protoBase", "outDir")
-)
-
// genProto creates a rule to convert a .proto file to generated .pb.cc and .pb.h files and returns
// the paths to the generated files.
-func genProto(ctx android.ModuleContext, protoFile android.Path, flags builderFlags) (ccFile, headerFile android.WritablePath) {
+func genProto(ctx android.ModuleContext, protoFile android.Path, flags builderFlags) (cc, header android.WritablePath) {
+ var ccFile, headerFile android.ModuleGenPath
srcSuffix := ".cc"
if flags.protoC {
srcSuffix = ".c"
}
- var protoBase string
- if flags.protoRoot {
- protoBase = "."
+ if flags.proto.CanonicalPathFromRoot {
ccFile = android.GenPathWithExt(ctx, "proto", protoFile, "pb"+srcSuffix)
headerFile = android.GenPathWithExt(ctx, "proto", protoFile, "pb.h")
} else {
rel := protoFile.Rel()
- protoBase = strings.TrimSuffix(protoFile.String(), rel)
ccFile = android.PathForModuleGen(ctx, "proto", pathtools.ReplaceExtension(rel, "pb"+srcSuffix))
headerFile = android.PathForModuleGen(ctx, "proto", pathtools.ReplaceExtension(rel, "pb.h"))
}
@@ -63,27 +42,19 @@
protoDeps := flags.protoDeps
if flags.protoOptionsFile {
optionsFile := pathtools.ReplaceExtension(protoFile.String(), "options")
- optionsPath := android.ExistentPathForSource(ctx, optionsFile)
- if optionsPath.Valid() {
- protoDeps = append(android.Paths{optionsPath.Path()}, protoDeps...)
- }
+ optionsPath := android.PathForSource(ctx, optionsFile)
+ protoDeps = append(android.Paths{optionsPath}, protoDeps...)
}
- ctx.Build(pctx, android.BuildParams{
- Rule: proto,
- Description: "protoc " + protoFile.Rel(),
- Output: ccFile,
- ImplicitOutput: headerFile,
- Input: protoFile,
- Implicits: protoDeps,
- Args: map[string]string{
- "outDir": android.ProtoDir(ctx).String(),
- "protoFlags": flags.protoFlags,
- "protoOut": flags.protoOutTypeFlag,
- "protoOutParams": flags.protoOutParams,
- "protoBase": protoBase,
- },
- })
+ outDir := flags.proto.Dir
+ depFile := ccFile.ReplaceExtension(ctx, "d")
+ outputs := android.WritablePaths{ccFile, headerFile}
+
+ rule := android.NewRuleBuilder()
+
+ android.ProtoRule(ctx, rule, protoFile, flags.proto, protoDeps, outDir, depFile, outputs)
+
+ rule.Build(pctx, ctx, "protoc_"+protoFile.Rel(), "protoc "+protoFile.Rel())
return ccFile, headerFile
}
@@ -143,13 +114,11 @@
func protoFlags(ctx ModuleContext, flags Flags, p *android.ProtoProperties) Flags {
flags.CFlags = append(flags.CFlags, "-DGOOGLE_PROTOBUF_NO_RTTI")
- flags.ProtoRoot = android.ProtoCanonicalPathFromRoot(ctx, p)
- if flags.ProtoRoot {
- flags.GlobalFlags = append(flags.GlobalFlags, "-I"+android.ProtoSubDir(ctx).String())
+ flags.proto = android.GetProtoFlags(ctx, p)
+ if flags.proto.CanonicalPathFromRoot {
+ flags.GlobalFlags = append(flags.GlobalFlags, "-I"+flags.proto.SubDir.String())
}
- flags.GlobalFlags = append(flags.GlobalFlags, "-I"+android.ProtoDir(ctx).String())
-
- flags.protoFlags = android.ProtoFlags(ctx, p)
+ flags.GlobalFlags = append(flags.GlobalFlags, "-I"+flags.proto.Dir.String())
var plugin string
@@ -157,18 +126,18 @@
case "nanopb-c", "nanopb-c-enable_malloc", "nanopb-c-16bit", "nanopb-c-enable_malloc-16bit", "nanopb-c-32bit", "nanopb-c-enable_malloc-32bit":
flags.protoC = true
flags.protoOptionsFile = true
- flags.protoOutTypeFlag = "--nanopb_out"
+ flags.proto.OutTypeFlag = "--nanopb_out"
plugin = "protoc-gen-nanopb"
case "full":
- flags.protoOutTypeFlag = "--cpp_out"
+ flags.proto.OutTypeFlag = "--cpp_out"
case "lite":
- flags.protoOutTypeFlag = "--cpp_out"
- flags.protoOutParams = append(flags.protoOutParams, "lite")
+ flags.proto.OutTypeFlag = "--cpp_out"
+ flags.proto.OutParams = append(flags.proto.OutParams, "lite")
case "":
// TODO(b/119714316): this should be equivalent to "lite" in
// order to match protoDeps, but some modules are depending on
// this behavior
- flags.protoOutTypeFlag = "--cpp_out"
+ flags.proto.OutTypeFlag = "--cpp_out"
default:
ctx.PropertyErrorf("proto.type", "unknown proto type %q",
String(p.Proto.Type))
@@ -177,7 +146,7 @@
if plugin != "" {
path := ctx.Config().HostToolPath(ctx, plugin)
flags.protoDeps = append(flags.protoDeps, path)
- flags.protoFlags = append(flags.protoFlags, "--plugin="+path.String())
+ flags.proto.Flags = append(flags.proto.Flags, "--plugin="+path.String())
}
return flags
diff --git a/cc/proto_test.go b/cc/proto_test.go
new file mode 100644
index 0000000..6fee924
--- /dev/null
+++ b/cc/proto_test.go
@@ -0,0 +1,36 @@
+// Copyright 2016 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cc
+
+import (
+ "strings"
+ "testing"
+)
+
+func TestProto(t *testing.T) {
+ t.Run("simple", func(t *testing.T) {
+ ctx := testCc(t, `
+ cc_library_shared {
+ name: "libfoo",
+ srcs: ["a.proto"],
+ }`)
+
+ proto := ctx.ModuleForTests("libfoo", "android_arm_armv7-a-neon_core_shared").Output("proto/a.pb.cc")
+
+ if cmd := proto.RuleParams.Command; !strings.Contains(cmd, "--cpp_out=") {
+ t.Errorf("expected '--cpp_out' in %q", cmd)
+ }
+ })
+}
diff --git a/cc/util.go b/cc/util.go
index 782bf61..30a77a4 100644
--- a/cc/util.go
+++ b/cc/util.go
@@ -85,12 +85,9 @@
groupStaticLibs: in.GroupStaticLibs,
protoDeps: in.protoDeps,
- protoFlags: strings.Join(in.protoFlags, " "),
- protoOutTypeFlag: in.protoOutTypeFlag,
- protoOutParams: strings.Join(in.protoOutParams, ","),
+ proto: in.proto,
protoC: in.protoC,
protoOptionsFile: in.protoOptionsFile,
- protoRoot: in.ProtoRoot,
}
}
diff --git a/java/builder.go b/java/builder.go
index d8b303e..44767a5 100644
--- a/java/builder.go
+++ b/java/builder.go
@@ -153,10 +153,7 @@
kotlincFlags string
kotlincClasspath classpath
- protoFlags []string
- protoOutTypeFlag string // The flag itself: --java_out
- protoOutParams string // Parameters to that flag: --java_out=$protoOutParams:$outDir
- protoRoot bool
+ proto android.ProtoFlags
}
func TransformJavaToClasses(ctx android.ModuleContext, outputFile android.WritablePath, shardIdx int,
diff --git a/java/proto.go b/java/proto.go
index 8028039..8de9e26 100644
--- a/java/proto.go
+++ b/java/proto.go
@@ -15,57 +15,34 @@
package java
import (
- "strings"
-
- "github.com/google/blueprint"
-
"android/soong/android"
)
-func init() {
- pctx.HostBinToolVariable("protocCmd", "aprotoc")
- pctx.HostBinToolVariable("depFixCmd", "dep_fixer")
-}
-
-var (
- proto = pctx.AndroidStaticRule("protoc",
- blueprint.RuleParams{
- Command: `rm -rf $out.tmp && mkdir -p $out.tmp && ` +
- `$protocCmd $protoOut=$protoOutParams:$out.tmp --dependency_out=$out.d -I $protoBase $protoFlags $in && ` +
- `$depFixCmd $out.d && ` +
- `${config.SoongZipCmd} -jar -o $out -C $out.tmp -D $out.tmp && rm -rf $out.tmp`,
- CommandDeps: []string{
- "$protocCmd",
- "$depFixCmd",
- "${config.SoongZipCmd}",
- },
- Depfile: "${out}.d",
- Deps: blueprint.DepsGCC,
- }, "protoBase", "protoFlags", "protoOut", "protoOutParams")
-)
-
func genProto(ctx android.ModuleContext, protoFile android.Path, flags javaBuilderFlags) android.Path {
srcJarFile := android.GenPathWithExt(ctx, "proto", protoFile, "srcjar")
- var protoBase string
- if flags.protoRoot {
- protoBase = "."
- } else {
- protoBase = strings.TrimSuffix(protoFile.String(), protoFile.Rel())
- }
+ outDir := srcJarFile.ReplaceExtension(ctx, "tmp")
+ depFile := srcJarFile.ReplaceExtension(ctx, "srcjar.d")
- ctx.Build(pctx, android.BuildParams{
- Rule: proto,
- Description: "protoc " + protoFile.Rel(),
- Output: srcJarFile,
- Input: protoFile,
- Args: map[string]string{
- "protoBase": protoBase,
- "protoOut": flags.protoOutTypeFlag,
- "protoOutParams": flags.protoOutParams,
- "protoFlags": strings.Join(flags.protoFlags, " "),
- },
- })
+ rule := android.NewRuleBuilder()
+
+ rule.Command().Text("rm -rf").Flag(outDir.String())
+ rule.Command().Text("mkdir -p").Flag(outDir.String())
+
+ android.ProtoRule(ctx, rule, protoFile, flags.proto, nil, outDir, depFile, nil)
+
+ // Proto generated java files have an unknown package name in the path, so package the entire output directory
+ // into a srcjar.
+ rule.Command().
+ Tool(ctx.Config().HostToolPath(ctx, "soong_zip")).
+ Flag("-jar").
+ FlagWithOutput("-o ", srcJarFile).
+ FlagWithArg("-C ", outDir.String()).
+ FlagWithArg("-D ", outDir.String())
+
+ rule.Command().Text("rm -rf").Flag(outDir.String())
+
+ rule.Build(pctx, ctx, "protoc_"+protoFile.Rel(), "protoc "+protoFile.Rel())
return srcJarFile
}
@@ -93,30 +70,24 @@
func protoFlags(ctx android.ModuleContext, j *CompilerProperties, p *android.ProtoProperties,
flags javaBuilderFlags) javaBuilderFlags {
+ flags.proto = android.GetProtoFlags(ctx, p)
+
switch String(p.Proto.Type) {
case "micro":
- flags.protoOutTypeFlag = "--javamicro_out"
+ flags.proto.OutTypeFlag = "--javamicro_out"
case "nano":
- flags.protoOutTypeFlag = "--javanano_out"
+ flags.proto.OutTypeFlag = "--javanano_out"
case "lite":
- flags.protoOutTypeFlag = "--java_out"
- flags.protoOutParams = "lite"
+ flags.proto.OutTypeFlag = "--java_out"
+ flags.proto.OutParams = append(flags.proto.OutParams, "lite")
case "full", "":
- flags.protoOutTypeFlag = "--java_out"
+ flags.proto.OutTypeFlag = "--java_out"
default:
ctx.PropertyErrorf("proto.type", "unknown proto type %q",
String(p.Proto.Type))
}
- if len(j.Proto.Output_params) > 0 {
- if flags.protoOutParams != "" {
- flags.protoOutParams += ","
- }
- flags.protoOutParams += strings.Join(j.Proto.Output_params, ",")
- }
-
- flags.protoFlags = android.ProtoFlags(ctx, p)
- flags.protoRoot = android.ProtoCanonicalPathFromRoot(ctx, p)
+ flags.proto.OutParams = append(flags.proto.OutParams, j.Proto.Output_params...)
return flags
}
diff --git a/python/proto.go b/python/proto.go
index 2370cd2..813e91d 100644
--- a/python/proto.go
+++ b/python/proto.go
@@ -16,58 +16,35 @@
import (
"android/soong/android"
- "strings"
-
- "github.com/google/blueprint"
)
-func init() {
- pctx.HostBinToolVariable("protocCmd", "aprotoc")
-}
+func genProto(ctx android.ModuleContext, protoFile android.Path, flags android.ProtoFlags, pkgPath string) android.Path {
+ srcsZipFile := android.PathForModuleGen(ctx, protoFile.Base()+".srcszip")
-var (
- proto = pctx.AndroidStaticRule("protoc",
- blueprint.RuleParams{
- Command: `rm -rf $out.tmp && mkdir -p $out.tmp && ` +
- `$protocCmd --python_out=$out.tmp --dependency_out=$out.d -I $protoBase $protoFlags $in && ` +
- `$parCmd -o $out $pkgPathArgs -C $out.tmp -D $out.tmp && rm -rf $out.tmp`,
- CommandDeps: []string{
- "$protocCmd",
- "$parCmd",
- },
- Depfile: "${out}.d",
- Deps: blueprint.DepsGCC,
- }, "protoBase", "protoFlags", "pkgPathArgs")
-)
+ outDir := srcsZipFile.ReplaceExtension(ctx, "tmp")
+ depFile := srcsZipFile.ReplaceExtension(ctx, "srcszip.d")
-func genProto(ctx android.ModuleContext, p *android.ProtoProperties,
- protoFile android.Path, protoFlags []string, pkgPath string) android.Path {
- srcJarFile := android.PathForModuleGen(ctx, protoFile.Base()+".srcszip")
+ rule := android.NewRuleBuilder()
- protoRoot := android.ProtoCanonicalPathFromRoot(ctx, p)
+ rule.Command().Text("rm -rf").Flag(outDir.String())
+ rule.Command().Text("mkdir -p").Flag(outDir.String())
- var protoBase string
- if protoRoot {
- protoBase = "."
- } else {
- protoBase = strings.TrimSuffix(protoFile.String(), protoFile.Rel())
- }
+ android.ProtoRule(ctx, rule, protoFile, flags, nil, outDir, depFile, nil)
- var pkgPathArgs string
+ // Proto generated python files have an unknown package name in the path, so package the entire output directory
+ // into a srcszip.
+ zipCmd := rule.Command().
+ Tool(ctx.Config().HostToolPath(ctx, "soong_zip")).
+ FlagWithOutput("-o ", srcsZipFile).
+ FlagWithArg("-C ", outDir.String()).
+ FlagWithArg("-D ", outDir.String())
if pkgPath != "" {
- pkgPathArgs = "-P " + pkgPath
+ zipCmd.FlagWithArg("-P ", pkgPath)
}
- ctx.Build(pctx, android.BuildParams{
- Rule: proto,
- Description: "protoc " + protoFile.Rel(),
- Output: srcJarFile,
- Input: protoFile,
- Args: map[string]string{
- "protoBase": protoBase,
- "protoFlags": strings.Join(protoFlags, " "),
- "pkgPathArgs": pkgPathArgs,
- },
- })
- return srcJarFile
+ rule.Command().Text("rm -rf").Flag(outDir.String())
+
+ rule.Build(pctx, ctx, "protoc_"+protoFile.Rel(), "protoc "+protoFile.Rel())
+
+ return srcsZipFile
}
diff --git a/python/python.go b/python/python.go
index 6eb9b6e..3c3c396 100644
--- a/python/python.go
+++ b/python/python.go
@@ -516,9 +516,11 @@
}
var zips android.Paths
if len(protoSrcs) > 0 {
+ protoFlags := android.GetProtoFlags(ctx, &p.protoProperties)
+ protoFlags.OutTypeFlag = "--python_out"
+
for _, srcFile := range protoSrcs {
- zip := genProto(ctx, &p.protoProperties, srcFile,
- android.ProtoFlags(ctx, &p.protoProperties), pkgPath)
+ zip := genProto(ctx, srcFile, protoFlags, pkgPath)
zips = append(zips, zip)
}
}