Support java compilation sharding for target side.
Add "LOCAL_JAVAC_SHARD_SIZE" to represent the number of java
source path entries in each shard.
Sharding is not allowed when "LOCAL_JAVAC_SHARD_SIZE" and
"LOCAL_JAR_PROCESSORS" are both enabled.
Limitation:
1. 0 <= LOCAL_JAVAC_SHARD_SIZE <= 8192
1. 0 < NUM(java_sources) <= 8192
2. 0 <= NUM(shards) <= 100
Performance Compare:
<Unsharded Build------------------------------------------------>
1. Build from clean state
rm -r -f out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/
&& time m
out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/classes-full-debug.jar
real 1m2.720s user 5m26.604s sys 0m39.552s
2. Incremental build
m out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/classes-full-debug.jar &&
touch frameworks/base/core/java/android/net/http/HttpResponseCache.java && time m
out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/classes-full-debug.jar
real 0m37.586s user 5m47.804s sys 0m50.388s
<Sharded Build-------------------------------------------------->
1. Build from clean state
rm -r -f out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/
&& time m
out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/classes-full-debug.jar
Javac Shard Size: 50----real 1m10.163s user 25m59.008s sys 1m58.460s
Javac Shard Size: 100---real 1m2.115s user 21m3.600s sys 1m15.964s
Javac Shard Size: 150---real 0m59.520s user 18m10.544s sys 1m12.628s
Javac Shard Size: 200---real 0m56.894s user 15m39.244s sys 1m11.608s
Javac Shard Size: 250---real 0m55.991s user 14m38.716s sys 1m2.292s
Javac Shard Size: 300---real 0m55.114s user 13m6.568s sys 1m8.200s
Javac Shard Size: 350---real 0m53.144s user 12m7.740s sys 1m3.836s
Javac Shard Size: 400---real 0m54.929s user 12m9.324s sys 1m4.340s
Javac Shard Size: 450---real 1m30.194s user 25m31.468s sys 1m52.416s
Javac Shard Size: 500---real 0m53.976s user 10m35.500s sys 0m55.160s
2. Incremental build
m out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/classes-full-debug.jar &&
touch frameworks/base/core/java/android/net/http/HttpResponseCache.java && time m
out/target/common/obj/JAVA_LIBRARIES/framework_intermediates/classes-full-debug.jar
Javac Shard Size: 50-----real 0m16.322s user 1m8.648s sys 0m31.700s
Javac Shard Size: 100----real 0m16.163s user 1m22.932s sys 0m29.440s
Javac Shard Size: 150----real 0m16.611s user 1m37.828s sys 0m21.168s
Javac Shard Size: 200----real 0m16.936s user 1m49.248s sys 0m28.636s
Javac Shard Size: 250----real 0m17.509s user 1m54.944s sys 0m32.768s
Javac Shard Size: 300----real 0m18.868s user 1m54.088s sys 0m28.824s
Javac Shard Size: 350----real 0m17.629s user 1m54.108s sys 0m31.056s
Javac Shard Size: 400----real 0m18.658s user 2m7.712s sys 0m30.636s
Javac Shard Size: 450----real 0m18.874s user 2m8.808s sys 0m33.540s
Javac Shard Size: 500----real 0m19.432s user 2m24.400s sys 0m30.368s
time m nothing:
real 0m5.799s user 0m7.236s sys 0m3.068s
Test: m clean && m -j checkbuild
Bug: b/67424047
Change-Id: Id0766d2b7de7c4546d29bbc7f8a0dd0e4b9ad45b
diff --git a/core/clear_vars.mk b/core/clear_vars.mk
index c38059c..b5bb963 100644
--- a/core/clear_vars.mk
+++ b/core/clear_vars.mk
@@ -120,6 +120,7 @@
LOCAL_JAR_PROCESSOR:=
LOCAL_JAR_PROCESSOR_ARGS:=
LOCAL_JAVACFLAGS:=
+LOCAL_JAVAC_SHARD_SIZE:=
LOCAL_JAVA_LANGUAGE_VERSION:=
LOCAL_JAVA_LAYERS_FILE:=
LOCAL_JAVA_LIBRARIES:=
diff --git a/core/definitions.mk b/core/definitions.mk
index cb0a6bb..1f9891e 100644
--- a/core/definitions.mk
+++ b/core/definitions.mk
@@ -2205,6 +2205,17 @@
@<(find $(1) -type f | sort | $(JAR_ARGS) $(1); echo "-C $(EMPTY_DIRECTORY) .")
endef
+# append additional Java sources(resources/Proto sources, and etc) to $(1).
+define fetch-additional-java-source
+$(hide) if [ -d "$(PRIVATE_SOURCE_INTERMEDIATES_DIR)" ]; then \
+ find $(PRIVATE_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $(1); \
+fi
+$(if $(PRIVATE_HAS_PROTO_SOURCES), \
+ $(hide) find $(PRIVATE_PROTO_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $(1))
+$(if $(PRIVATE_HAS_RS_SOURCES), \
+ $(hide) find $(PRIVATE_RS_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $(1))
+endef
+
# Some historical notes:
# - below we write the list of java files to java-source-list to avoid argument
# list length problems with Cygwin
@@ -2214,16 +2225,20 @@
@echo "$($(PRIVATE_PREFIX)DISPLAY) Java source list: $(PRIVATE_MODULE)"
$(hide) rm -f $@
$(call dump-words-to-file,$(sort $(PRIVATE_JAVA_SOURCES)),$@.tmp)
-$(hide) if [ -d "$(PRIVATE_SOURCE_INTERMEDIATES_DIR)" ]; then \
- find $(PRIVATE_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $@.tmp; \
-fi
-$(if $(PRIVATE_HAS_PROTO_SOURCES), \
- $(hide) find $(PRIVATE_PROTO_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $@.tmp)
-$(if $(PRIVATE_HAS_RS_SOURCES), \
- $(hide) find $(PRIVATE_RS_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $@.tmp)
+$(call fetch-additional-java-source,$@.tmp)
$(hide) tr ' ' '\n' < $@.tmp | $(NORMALIZE_PATH) | sort -u > $@
endef
+# $(1): sharding number.
+# $(2): Java source files paths.
+define save-sharded-java-source-list
+$(java_source_list_file).shard.$(1): $(2) $$(NORMALIZE_PATH)
+ @echo "shard java source list: $$@"
+ rm -f $$@
+ $$(call dump-words-to-file,$(2),$$@.tmp)
+ $(hide) tr ' ' '\n' < $$@.tmp | $$(NORMALIZE_PATH) | sort -u > $$@
+endef
+
# Common definition to invoke javac on the host and target.
#
# $(1): javac
@@ -2265,9 +2280,33 @@
$(if $(PRIVATE_EXTRA_JAR_ARGS),$(call add-java-resources-to,$@))
endef
-define transform-java-to-classes.jar
-@echo "$($(PRIVATE_PREFIX)DISPLAY) Java: $(PRIVATE_MODULE) ($(PRIVATE_CLASS_INTERMEDIATES_DIR))"
-$(call compile-java,$(TARGET_JAVAC),$(PRIVATE_ALL_JAVA_HEADER_LIBRARIES))
+# $(1): Javac output jar name.
+# $(2): Java source list file.
+# $(3): Java header libs.
+# $(4): Javac sharding number.
+# $(5): Javac sources deps (the arg may neeed $$ in case of containing '#')
+define create-classes-full-debug.jar
+$(1): PRIVATE_JAVACFLAGS := $$(LOCAL_JAVACFLAGS) $$(annotation_processor_flags)
+$(1): PRIVATE_JAR_EXCLUDE_FILES := $$(LOCAL_JAR_EXCLUDE_FILES)
+$(1): PRIVATE_JAR_PACKAGES := $$(LOCAL_JAR_PACKAGES)
+$(1): PRIVATE_JAR_EXCLUDE_PACKAGES := $$(LOCAL_JAR_EXCLUDE_PACKAGES)
+$(1): PRIVATE_DONT_DELETE_JAR_META_INF := $$(LOCAL_DONT_DELETE_JAR_META_INF)
+$(1): PRIVATE_JAVA_SOURCE_LIST := $(2)
+$(1): PRIVATE_ALL_JAVA_HEADER_LIBRARIES := $(3)
+$(1): PRIVATE_CLASS_INTERMEDIATES_DIR := $(intermediates.COMMON)/classes$(4)
+$(1): PRIVATE_ANNO_INTERMEDIATES_DIR := $(intermediates.COMMON)/anno$(4)
+$(1): \
+ $(2) \
+ $(3) \
+ $(5) \
+ $$(full_java_bootclasspath_libs) \
+ $$(layers_file) \
+ $$(annotation_processor_deps) \
+ $$(NORMALIZE_PATH) \
+ $$(JAR_ARGS) \
+ | $$(SOONG_JAVAC_WRAPPER)
+ @echo "Target Java: $$@ ($$(PRIVATE_CLASS_INTERMEDIATES_DIR))"
+ $$(call compile-java,$$(TARGET_JAVAC),$$(PRIVATE_ALL_JAVA_HEADER_LIBRARIES))
endef
define transform-java-to-header.jar
diff --git a/core/java.mk b/core/java.mk
index 5eddb0e..cf49994 100644
--- a/core/java.mk
+++ b/core/java.mk
@@ -352,7 +352,36 @@
##########################################
java_sources := $(addprefix $(LOCAL_PATH)/, $(filter %.java,$(LOCAL_SRC_FILES))) $(aidl_java_sources) $(logtags_java_sources) \
$(filter %.java,$(LOCAL_GENERATED_SOURCES))
-all_java_sources := $(java_sources) $(addprefix $(TARGET_OUT_COMMON_INTERMEDIATES)/, $(filter %.java,$(LOCAL_INTERMEDIATE_SOURCES)))
+java_intermediate_sources := $(addprefix $(TARGET_OUT_COMMON_INTERMEDIATES)/, $(filter %.java,$(LOCAL_INTERMEDIATE_SOURCES)))
+all_java_sources := $(java_sources) $(java_intermediate_sources)
+
+enable_sharding :=
+ifneq ($(TURBINE_ENABLED),false)
+ifneq ($(LOCAL_JAVAC_SHARD_SIZE),)
+ifneq ($(LOCAL_JAR_PROCESSOR),)
+$(call pretty-error,Cannot set both LOCAL_JAVAC_SHARD_SIZE and LOCAL_JAR_PROCESSOR!)
+endif # LOCAL_JAR_PROCESSOR is not empty
+enable_sharding := true
+
+num_shards := $(call int_divide,$(words $(java_sources)),$(LOCAL_JAVAC_SHARD_SIZE))
+ifneq ($(words $(java_sources)),$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(num_shards)))
+# increment number of shards by 1.
+num_shards := $(call int_plus,$(num_shards),1)
+endif
+
+shard_idx_list := $(call int_range_list,1,$(num_shards))
+sharded_java_source_list_files += $(foreach x,$(shard_idx_list),$(java_source_list_file).shard.$(x))
+sharded_jar_list += $(foreach x,$(shard_idx_list),$(full_classes_compiled_jar).shard.$(x))
+
+# always put dynamically-located .java files (generated by Proto/resource, etc) in a new final shard.
+# increment number of shards by 1.
+num_shards := $(call int_plus,$(num_shards),1)
+sharded_java_source_list_files += $(java_source_list_file).shard.$(num_shards)
+sharded_jar_list += $(full_classes_compiled_jar).shard.$(num_shards)
+LOCAL_INTERMEDIATE_TARGETS += $(sharded_java_source_list_files)
+LOCAL_INTERMEDIATE_TARGETS += $(sharded_jar_list)
+endif # LOCAL_JAVAC_SHARD_SIZE is not empty
+endif # TURBINE_ENABLED != false
include $(BUILD_SYSTEM)/java_common.mk
@@ -419,23 +448,47 @@
$(java_source_list_file): $(java_sources_deps)
$(write-java-source-list)
-$(full_classes_compiled_jar): PRIVATE_JAVACFLAGS := $(LOCAL_JAVACFLAGS) $(annotation_processor_flags)
-$(full_classes_compiled_jar): PRIVATE_JAR_EXCLUDE_FILES := $(LOCAL_JAR_EXCLUDE_FILES)
-$(full_classes_compiled_jar): PRIVATE_JAR_PACKAGES := $(LOCAL_JAR_PACKAGES)
-$(full_classes_compiled_jar): PRIVATE_JAR_EXCLUDE_PACKAGES := $(LOCAL_JAR_EXCLUDE_PACKAGES)
-$(full_classes_compiled_jar): PRIVATE_DONT_DELETE_JAR_META_INF := $(LOCAL_DONT_DELETE_JAR_META_INF)
-$(full_classes_compiled_jar): PRIVATE_JAVA_SOURCE_LIST := $(java_source_list_file)
-$(full_classes_compiled_jar): \
- $(java_source_list_file) \
- $(java_sources_deps) \
- $(full_java_header_libs) \
- $(full_java_bootclasspath_libs) \
- $(layers_file) \
- $(annotation_processor_deps) \
- $(NORMALIZE_PATH) \
- $(JAR_ARGS) \
- | $(SOONG_JAVAC_WRAPPER)
- $(transform-java-to-classes.jar)
+ifdef enable_sharding
+$(foreach x,$(shard_idx_list),\
+ $(eval $(call save-sharded-java-source-list,$(x),\
+ $(wordlist $(call int_plus,1,$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(call int_subtract,$(x),1))),\
+ $(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(x)),$(sort $(java_sources))))))
+
+# always put dynamically-located .java files (generated by Proto/resource, etc) in a new final shard.
+$(java_source_list_file).shard.$(num_shards): PRIVATE_JAVA_INTERMEDIATE_SOURCES := $(java_intermediate_sources)
+$(java_source_list_file).shard.$(num_shards): $(java_resource_sources) \
+ $(RenderScript_file_stamp) \
+ $(proto_java_sources_file_stamp) \
+ $(LOCAL_ADDITIONAL_DEPENDENCIES) \
+ $(NORMALIZE_PATH)
+ $(hide) rm -f $@
+ $(call dump-words-to-file,$(PRIVATE_JAVA_INTERMEDIATE_SOURCES),$@.tmp)
+ $(call fetch-additional-java-source,$@.tmp)
+ $(hide) tr ' ' '\n' < $@.tmp | $(NORMALIZE_PATH) | sort -u > $@
+
+# Javac sharding with header libs including its own header jar as one of dependency.
+$(foreach x,$(shard_idx_list),\
+ $(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar).shard.$(x),\
+ $(java_source_list_file).shard.$(x),\
+ $(full_java_header_libs) $(full_classes_header_jar),$(x),\
+ $(wordlist $(call int_plus,1,$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(call int_subtract,$(x),1))),\
+ $(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(x)),$(sort $(java_sources))))))
+
+# Javac sharding for last shard with additional Java dependencies.
+$(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar).shard.$(num_shards),\
+ $(java_source_list_file).shard.$(num_shards),$(full_java_header_libs) $(full_classes_header_jar),$(strip \
+ $(num_shards)),$$(java_resource_sources) $$(RenderScript_file_stamp) \
+ $$(proto_java_sources_file_stamp) $$(LOCAL_ADDITIONAL_DEPENDENCIES)))
+
+$(full_classes_compiled_jar): PRIVATE_SHARDED_JAR_LIST := $(sharded_jar_list)
+$(full_classes_compiled_jar): $(sharded_jar_list) | $(MERGE_ZIPS)
+ $(MERGE_ZIPS) -j $@ $(PRIVATE_SHARDED_JAR_LIST)
+else
+# we can't use single $ for java_sources_deps since it may contain hash '#' sign.
+$(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar),\
+ $(java_source_list_file),$(full_java_header_libs),,$$(java_sources_deps)))
+
+endif # ifdef enable_sharding
ifneq ($(TURBINE_ENABLED),false)
diff --git a/core/math.mk b/core/math.mk
index 047d046..57cb681 100644
--- a/core/math.mk
+++ b/core/math.mk
@@ -42,6 +42,11 @@
$(if $(call math_is_number,$(1)),,$(error Only positive integers <= 100 are supported (not $(1))))
endef
+# return a list containing integers ranging from [$(1),$(2)]
+define int_range_list
+$(call _math_check_valid,$(1))$(call _math_check_valid,$(2))$(wordlist $(1),$(2),$(__MATH_NUMBERS))
+endef
+
#$(call _math_check_valid,0)
#$(call _math_check_valid,1)
#$(call _math_check_valid,100)
@@ -75,3 +80,60 @@
define inc_and_print
$(strip $(eval $(1) := $($(1)) .)$(words $($(1))))
endef
+
+_INT_LIMIT_WORDS := $(foreach a,x x,$(foreach b,x x x x x x x x x x x x x x x x,\
+ $(foreach c,x x x x x x x x x x x x x x x x,x x x x x x x x x x x x x x x x)))
+
+define _int_encode
+$(if $(filter $(words x $(_INT_LIMIT_WORDS)),$(words $(wordlist 1,$(1),x $(_INT_LIMIT_WORDS)))),\
+ $(call pretty-error,integer greater than $(words $(_INT_LIMIT_WORDS)) is not supported!),\
+ $(wordlist 1,$(1),$(_INT_LIMIT_WORDS)))
+endef
+
+# _int_max returns the maximum of the two arguments
+# input: two (x) lists; output: one (x) list
+# integer cannot be passed in directly. It has to be converted using _int_encode.
+define _int_max
+$(subst xx,x,$(join $(1),$(2)))
+endef
+
+# first argument is greater than second argument
+# output: non-empty if true
+# integer cannot be passed in directly. It has to be converted using _int_encode.
+define _int_greater-than
+$(filter-out $(words $(2)),$(words $(call _int_max,$(1),$(2))))
+endef
+
+# first argument equals to second argument
+# output: non-empty if true
+# integer cannot be passed in directly. It has to be converted using _int_encode.
+define _int_equal
+$(filter $(words $(1)),$(words $(2)))
+endef
+
+# first argument is greater than or equal to second argument
+# output: non-empty if true
+# integer cannot be passed in directly. It has to be converted using _int_encode.
+define _int_greater-or-equal
+$(call _int_greater-than,$(1),$(2))$(call _int_equal,$(1),$(2))
+endef
+
+define int_plus
+$(words $(call _int_encode,$(1)) $(call _int_encode,$(2)))
+endef
+
+define int_subtract
+$(if $(call _int_greater-or-equal,$(call _int_encode,$(1)),$(call _int_encode,$(2))),\
+ $(words $(filter-out xx,$(join $(call _int_encode,$(1)),$(call _int_encode,$(2))))),\
+ $(call pretty-error,$(1) subtract underflow $(2)))
+endef
+
+define int_multiply
+$(words $(foreach a,$(call _int_encode,$(1)),$(call _int_encode,$(2))))
+endef
+
+define int_divide
+$(if $(filter 0,$(2)),$(call pretty-error,division by zero is not allowed!),$(strip \
+ $(if $(call _int_greater-or-equal,$(call _int_encode,$(1)),$(call _int_encode,$(2))), \
+ $(call int_plus,$(call int_divide,$(call int_subtract,$(1),$(2)),$(2)),1),0)))
+endef
diff --git a/core/package_internal.mk b/core/package_internal.mk
index 5a1ef05..01e2463 100644
--- a/core/package_internal.mk
+++ b/core/package_internal.mk
@@ -333,6 +333,7 @@
# Make sure the data-binding process happens before javac and generation of R.java.
$(R_file_stamp): $(data_binding_stamp)
$(java_source_list_file): $(data_binding_stamp)
+$(foreach x,$(sharded_java_source_list_files),$(eval $(x): $(data_binding_stamp)))
$(full_classes_compiled_jar): $(data_binding_stamp)
endif # LOCAL_DATA_BINDING
@@ -426,6 +427,7 @@
# The R.java file must exist by the time the java source
# list is generated
$(java_source_list_file): $(R_file_stamp)
+$(foreach x,$(sharded_java_source_list_files),$(eval $(x): $(R_file_stamp)))
endif # need_compile_res
diff --git a/core/static_java_library.mk b/core/static_java_library.mk
index ca3b071..c402dce 100644
--- a/core/static_java_library.mk
+++ b/core/static_java_library.mk
@@ -168,6 +168,7 @@
$(LOCAL_BUILT_MODULE): $(R_file_stamp)
$(java_source_list_file): $(R_file_stamp)
+$(foreach x,$(sharded_java_source_list_files),$(eval $(x): $(R_file_stamp)))
$(full_classes_compiled_jar): $(R_file_stamp)
$(full_classes_turbine_jar): $(R_file_stamp)