tequilaOS/platform_build
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge "Support java compilation sharding for target side."

Browse source
This commit is contained in:
Treehugger Robot 2017-10-13 00:40:37 +00:00 committed by Gerrit Code Review
commit 60e1f8ee27
6 changed files with 186 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
core/clear_vars.mk
View file

@ -120,6 +120,7 @@ LOCAL_JAR_PACKAGES:=
LOCAL_JAR_PROCESSOR:=
LOCAL_JAR_PROCESSOR_ARGS:=
LOCAL_JAVACFLAGS:=
LOCAL_JAVAC_SHARD_SIZE:=
LOCAL_JAVA_LANGUAGE_VERSION:=
LOCAL_JAVA_LAYERS_FILE:=
LOCAL_JAVA_LIBRARIES:=

59
core/definitions.mk
View file

@ -2205,6 +2205,17 @@ define jar-args-sorted-files-in-directory
@<(find $(1) -type f | sort | $(JAR_ARGS) $(1); echo "-C $(EMPTY_DIRECTORY) .")
endef
# append additional Java sources(resources/Proto sources, and etc) to $(1).
define fetch-additional-java-source
$(hide) if [ -d "$(PRIVATE_SOURCE_INTERMEDIATES_DIR)" ]; then \
find $(PRIVATE_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $(1); \
fi
$(if $(PRIVATE_HAS_PROTO_SOURCES), \
$(hide) find $(PRIVATE_PROTO_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $(1))
$(if $(PRIVATE_HAS_RS_SOURCES), \
$(hide) find $(PRIVATE_RS_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $(1))
endef
# Some historical notes:
# - below we write the list of java files to java-source-list to avoid argument
# list length problems with Cygwin
@ -2214,16 +2225,20 @@ define write-java-source-list
@echo "$($(PRIVATE_PREFIX)DISPLAY) Java source list: $(PRIVATE_MODULE)"
$(hide) rm -f $@
$(call dump-words-to-file,$(sort $(PRIVATE_JAVA_SOURCES)),$@.tmp)
$(hide) if [ -d "$(PRIVATE_SOURCE_INTERMEDIATES_DIR)" ]; then \
find $(PRIVATE_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $@.tmp; \
fi
$(if $(PRIVATE_HAS_PROTO_SOURCES), \
$(hide) find $(PRIVATE_PROTO_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $@.tmp)
$(if $(PRIVATE_HAS_RS_SOURCES), \
$(hide) find $(PRIVATE_RS_SOURCE_INTERMEDIATES_DIR) -name '*.java' -and -not -name '.*' >> $@.tmp)
$(call fetch-additional-java-source,$@.tmp)
$(hide) tr ' ' '\n' < $@.tmp | $(NORMALIZE_PATH) | sort -u > $@
endef
# $(1): sharding number.
# $(2): Java source files paths.
define save-sharded-java-source-list
$(java_source_list_file).shard.$(1): $(2) $$(NORMALIZE_PATH)
@echo "shard java source list: $$@"
rm -f $$@
$$(call dump-words-to-file,$(2),$$@.tmp)
$(hide) tr ' ' '\n' < $$@.tmp | $$(NORMALIZE_PATH) | sort -u > $$@
endef
# Common definition to invoke javac on the host and target.
#
# $(1): javac
@ -2265,9 +2280,33 @@ $(hide) $(JAR) -cf $@ $(call jar-args-sorted-files-in-directory,$(PRIVATE_CLASS_
$(if $(PRIVATE_EXTRA_JAR_ARGS),$(call add-java-resources-to,$@))
endef
define transform-java-to-classes.jar
@echo "$($(PRIVATE_PREFIX)DISPLAY) Java: $(PRIVATE_MODULE) ($(PRIVATE_CLASS_INTERMEDIATES_DIR))"
$(call compile-java,$(TARGET_JAVAC),$(PRIVATE_ALL_JAVA_HEADER_LIBRARIES))
# $(1): Javac output jar name.
# $(2): Java source list file.
# $(3): Java header libs.
# $(4): Javac sharding number.
# $(5): Javac sources deps (the arg may neeed $$ in case of containing '#')
define create-classes-full-debug.jar
$(1): PRIVATE_JAVACFLAGS := $$(LOCAL_JAVACFLAGS) $$(annotation_processor_flags)
$(1): PRIVATE_JAR_EXCLUDE_FILES := $$(LOCAL_JAR_EXCLUDE_FILES)
$(1): PRIVATE_JAR_PACKAGES := $$(LOCAL_JAR_PACKAGES)
$(1): PRIVATE_JAR_EXCLUDE_PACKAGES := $$(LOCAL_JAR_EXCLUDE_PACKAGES)
$(1): PRIVATE_DONT_DELETE_JAR_META_INF := $$(LOCAL_DONT_DELETE_JAR_META_INF)
$(1): PRIVATE_JAVA_SOURCE_LIST := $(2)
$(1): PRIVATE_ALL_JAVA_HEADER_LIBRARIES := $(3)
$(1): PRIVATE_CLASS_INTERMEDIATES_DIR := $(intermediates.COMMON)/classes$(4)
$(1): PRIVATE_ANNO_INTERMEDIATES_DIR := $(intermediates.COMMON)/anno$(4)
$(1): \
$(2) \
$(3) \
$(5) \
$$(full_java_bootclasspath_libs) \
$$(layers_file) \
$$(annotation_processor_deps) \
$$(NORMALIZE_PATH) \
$$(JAR_ARGS) \
| $$(SOONG_JAVAC_WRAPPER)
@echo "Target Java: $$@ ($$(PRIVATE_CLASS_INTERMEDIATES_DIR))"
$$(call compile-java,$$(TARGET_JAVAC),$$(PRIVATE_ALL_JAVA_HEADER_LIBRARIES))
endef
define transform-java-to-header.jar

89
core/java.mk
View file

@ -352,7 +352,36 @@ endif
##########################################
java_sources := $(addprefix $(LOCAL_PATH)/, $(filter %.java,$(LOCAL_SRC_FILES))) $(aidl_java_sources) $(logtags_java_sources) \
$(filter %.java,$(LOCAL_GENERATED_SOURCES))
all_java_sources := $(java_sources) $(addprefix $(TARGET_OUT_COMMON_INTERMEDIATES)/, $(filter %.java,$(LOCAL_INTERMEDIATE_SOURCES)))
java_intermediate_sources := $(addprefix $(TARGET_OUT_COMMON_INTERMEDIATES)/, $(filter %.java,$(LOCAL_INTERMEDIATE_SOURCES)))
all_java_sources := $(java_sources) $(java_intermediate_sources)
enable_sharding :=
ifneq ($(TURBINE_ENABLED),false)
ifneq ($(LOCAL_JAVAC_SHARD_SIZE),)
ifneq ($(LOCAL_JAR_PROCESSOR),)
$(call pretty-error,Cannot set both LOCAL_JAVAC_SHARD_SIZE and LOCAL_JAR_PROCESSOR!)
endif # LOCAL_JAR_PROCESSOR is not empty
enable_sharding := true
num_shards := $(call int_divide,$(words $(java_sources)),$(LOCAL_JAVAC_SHARD_SIZE))
ifneq ($(words $(java_sources)),$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(num_shards)))
# increment number of shards by 1.
num_shards := $(call int_plus,$(num_shards),1)
endif
shard_idx_list := $(call int_range_list,1,$(num_shards))
sharded_java_source_list_files += $(foreach x,$(shard_idx_list),$(java_source_list_file).shard.$(x))
sharded_jar_list += $(foreach x,$(shard_idx_list),$(full_classes_compiled_jar).shard.$(x))
# always put dynamically-located .java files (generated by Proto/resource, etc) in a new final shard.
# increment number of shards by 1.
num_shards := $(call int_plus,$(num_shards),1)
sharded_java_source_list_files += $(java_source_list_file).shard.$(num_shards)
sharded_jar_list += $(full_classes_compiled_jar).shard.$(num_shards)
LOCAL_INTERMEDIATE_TARGETS += $(sharded_java_source_list_files)
LOCAL_INTERMEDIATE_TARGETS += $(sharded_jar_list)
endif # LOCAL_JAVAC_SHARD_SIZE is not empty
endif # TURBINE_ENABLED != false
include $(BUILD_SYSTEM)/java_common.mk
@ -419,23 +448,47 @@ java_sources_deps := \
$(java_source_list_file): $(java_sources_deps)
$(write-java-source-list)
$(full_classes_compiled_jar): PRIVATE_JAVACFLAGS := $(LOCAL_JAVACFLAGS) $(annotation_processor_flags)
$(full_classes_compiled_jar): PRIVATE_JAR_EXCLUDE_FILES := $(LOCAL_JAR_EXCLUDE_FILES)
$(full_classes_compiled_jar): PRIVATE_JAR_PACKAGES := $(LOCAL_JAR_PACKAGES)
$(full_classes_compiled_jar): PRIVATE_JAR_EXCLUDE_PACKAGES := $(LOCAL_JAR_EXCLUDE_PACKAGES)
$(full_classes_compiled_jar): PRIVATE_DONT_DELETE_JAR_META_INF := $(LOCAL_DONT_DELETE_JAR_META_INF)
$(full_classes_compiled_jar): PRIVATE_JAVA_SOURCE_LIST := $(java_source_list_file)
$(full_classes_compiled_jar): \
$(java_source_list_file) \
$(java_sources_deps) \
$(full_java_header_libs) \
$(full_java_bootclasspath_libs) \
$(layers_file) \
$(annotation_processor_deps) \
$(NORMALIZE_PATH) \
$(JAR_ARGS) \
| $(SOONG_JAVAC_WRAPPER)
$(transform-java-to-classes.jar)
ifdef enable_sharding
$(foreach x,$(shard_idx_list),\
$(eval $(call save-sharded-java-source-list,$(x),\
$(wordlist $(call int_plus,1,$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(call int_subtract,$(x),1))),\
$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(x)),$(sort $(java_sources))))))
# always put dynamically-located .java files (generated by Proto/resource, etc) in a new final shard.
$(java_source_list_file).shard.$(num_shards): PRIVATE_JAVA_INTERMEDIATE_SOURCES := $(java_intermediate_sources)
$(java_source_list_file).shard.$(num_shards): $(java_resource_sources) \
$(RenderScript_file_stamp) \
$(proto_java_sources_file_stamp) \
$(LOCAL_ADDITIONAL_DEPENDENCIES) \
$(NORMALIZE_PATH)
$(hide) rm -f $@
$(call dump-words-to-file,$(PRIVATE_JAVA_INTERMEDIATE_SOURCES),$@.tmp)
$(call fetch-additional-java-source,$@.tmp)
$(hide) tr ' ' '\n' < $@.tmp | $(NORMALIZE_PATH) | sort -u > $@
# Javac sharding with header libs including its own header jar as one of dependency.
$(foreach x,$(shard_idx_list),\
$(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar).shard.$(x),\
$(java_source_list_file).shard.$(x),\
$(full_java_header_libs) $(full_classes_header_jar),$(x),\
$(wordlist $(call int_plus,1,$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(call int_subtract,$(x),1))),\
$(call int_multiply,$(LOCAL_JAVAC_SHARD_SIZE),$(x)),$(sort $(java_sources))))))
# Javac sharding for last shard with additional Java dependencies.
$(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar).shard.$(num_shards),\
$(java_source_list_file).shard.$(num_shards),$(full_java_header_libs) $(full_classes_header_jar),$(strip \
$(num_shards)),$$(java_resource_sources) $$(RenderScript_file_stamp) \
$$(proto_java_sources_file_stamp) $$(LOCAL_ADDITIONAL_DEPENDENCIES)))
$(full_classes_compiled_jar): PRIVATE_SHARDED_JAR_LIST := $(sharded_jar_list)
$(full_classes_compiled_jar): $(sharded_jar_list) | $(MERGE_ZIPS)
$(MERGE_ZIPS) -j $@ $(PRIVATE_SHARDED_JAR_LIST)
else
# we can't use single $ for java_sources_deps since it may contain hash '#' sign.
$(eval $(call create-classes-full-debug.jar,$(full_classes_compiled_jar),\
$(java_source_list_file),$(full_java_header_libs),,$$(java_sources_deps)))
endif # ifdef enable_sharding
ifneq ($(TURBINE_ENABLED),false)

62
core/math.mk
View file

@ -42,6 +42,11 @@ define _math_check_valid
$(if $(call math_is_number,$(1)),,$(error Only positive integers <= 100 are supported (not $(1))))
endef
# return a list containing integers ranging from [$(1),$(2)]
define int_range_list
$(call _math_check_valid,$(1))$(call _math_check_valid,$(2))$(wordlist $(1),$(2),$(__MATH_NUMBERS))
endef
#$(call _math_check_valid,0)
#$(call _math_check_valid,1)
#$(call _math_check_valid,100)
@ -75,3 +80,60 @@ endef
define inc_and_print
$(strip $(eval $(1) := $($(1)) .)$(words $($(1))))
endef
_INT_LIMIT_WORDS := $(foreach a,x x,$(foreach b,x x x x x x x x x x x x x x x x,\
$(foreach c,x x x x x x x x x x x x x x x x,x x x x x x x x x x x x x x x x)))
define _int_encode
$(if $(filter $(words x $(_INT_LIMIT_WORDS)),$(words $(wordlist 1,$(1),x $(_INT_LIMIT_WORDS)))),\
$(call pretty-error,integer greater than $(words $(_INT_LIMIT_WORDS)) is not supported!),\
$(wordlist 1,$(1),$(_INT_LIMIT_WORDS)))
endef
# _int_max returns the maximum of the two arguments
# input: two (x) lists; output: one (x) list
# integer cannot be passed in directly. It has to be converted using _int_encode.
define _int_max
$(subst xx,x,$(join $(1),$(2)))
endef
# first argument is greater than second argument
# output: non-empty if true
# integer cannot be passed in directly. It has to be converted using _int_encode.
define _int_greater-than
$(filter-out $(words $(2)),$(words $(call _int_max,$(1),$(2))))
endef
# first argument equals to second argument
# output: non-empty if true
# integer cannot be passed in directly. It has to be converted using _int_encode.
define _int_equal
$(filter $(words $(1)),$(words $(2)))
endef
# first argument is greater than or equal to second argument
# output: non-empty if true
# integer cannot be passed in directly. It has to be converted using _int_encode.
define _int_greater-or-equal
$(call _int_greater-than,$(1),$(2))$(call _int_equal,$(1),$(2))
endef
define int_plus
$(words $(call _int_encode,$(1)) $(call _int_encode,$(2)))
endef
define int_subtract
$(if $(call _int_greater-or-equal,$(call _int_encode,$(1)),$(call _int_encode,$(2))),\
$(words $(filter-out xx,$(join $(call _int_encode,$(1)),$(call _int_encode,$(2))))),\
$(call pretty-error,$(1) subtract underflow $(2)))
endef
define int_multiply
$(words $(foreach a,$(call _int_encode,$(1)),$(call _int_encode,$(2))))
endef
define int_divide
$(if $(filter 0,$(2)),$(call pretty-error,division by zero is not allowed!),$(strip \
$(if $(call _int_greater-or-equal,$(call _int_encode,$(1)),$(call _int_encode,$(2))), \
$(call int_plus,$(call int_divide,$(call int_subtract,$(1),$(2)),$(2)),1),0)))
endef

2
core/package_internal.mk
View file

@ -333,6 +333,7 @@ $(data_binding_stamp) : $(all_res_assets) $(full_android_manifest) \
# Make sure the data-binding process happens before javac and generation of R.java.
$(R_file_stamp): $(data_binding_stamp)
$(java_source_list_file): $(data_binding_stamp)
$(foreach x,$(sharded_java_source_list_files),$(eval $(x): $(data_binding_stamp)))
$(full_classes_compiled_jar): $(data_binding_stamp)
endif # LOCAL_DATA_BINDING
@ -426,6 +427,7 @@ $(LOCAL_BUILT_MODULE): $(R_file_stamp)
# The R.java file must exist by the time the java source
# list is generated
$(java_source_list_file): $(R_file_stamp)
$(foreach x,$(sharded_java_source_list_files),$(eval $(x): $(R_file_stamp)))
endif # need_compile_res

1
core/static_java_library.mk
View file

@ -168,6 +168,7 @@ endif # LOCAL_USE_AAPT2
$(LOCAL_BUILT_MODULE): $(R_file_stamp)
$(java_source_list_file): $(R_file_stamp)
$(foreach x,$(sharded_java_source_list_files),$(eval $(x): $(R_file_stamp)))
$(full_classes_compiled_jar): $(R_file_stamp)
$(full_classes_turbine_jar): $(R_file_stamp)