From a052ddbb7e4ca73685480b131fcac9f727df19d4 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Mon, 25 Sep 2023 21:46:58 -0700 Subject: [PATCH] Shard srcjars when sharding javac compilation java_library rules with javac_shard_size set split the sources into shards to invoke javac multiple times, but were using a single javac invocation for all srcjars. For fraemwork-minus-apex, this srcjar shard was the long pole at 15.7 seconds, containing 266 srcjars with 1542 java files with a total of 614593 lines. Use a rough approximation of 5 sources per srcjar to determine the number of shards to split the srcjars into based on javac_shard_size. This results in splitting the srcjars for frameworks-minus-apex into 8 shards, with the longest taking 10.5 seconds to compile. The longest shard contains most of the aidl srcjars, which have been generated by sharded groups of 50 aidl files and have a much higher average number of sources per srcjar (a mean and median of 27). A future improvement could be to shard those separately assuming a higher number of sources per srcjar. Bug: 302033097 Test: USE_RBE=false m frameworks-minus-apex Change-Id: I85e740c7fcf5651cf18c0cdc90ab8c6ee39cb47b --- java/base.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/java/base.go b/java/base.go index 2c0b3ea7c..653323ab1 100644 --- a/java/base.go +++ b/java/base.go @@ -1343,10 +1343,17 @@ func (j *Module) compile(ctx android.ModuleContext, extraSrcJars, extraClasspath jars = append(jars, classes) } } + // Assume approximately 5 sources per srcjar. + // For framework-minus-apex in AOSP at the time this was written, there are 266 srcjars, with a mean + // of 5.8 sources per srcjar, but a median of 1, a standard deviation of 10, and a max of 48 source files. if len(srcJars) > 0 { - classes := j.compileJavaClasses(ctx, jarName, len(shardSrcs), - nil, srcJars, flags, extraJarDeps) - jars = append(jars, classes) + startIdx := len(shardSrcs) + shardSrcJarsList := android.ShardPaths(srcJars, shardSize/5) + for idx, shardSrcJars := range shardSrcJarsList { + classes := j.compileJavaClasses(ctx, jarName, startIdx+idx, + nil, shardSrcJars, flags, extraJarDeps) + jars = append(jars, classes) + } } } else { classes := j.compileJavaClasses(ctx, jarName, -1, uniqueJavaFiles, srcJars, flags, extraJarDeps)