Multithread symlink forest removal.

This is a second attempt at aosp/2273288 which got rolled back because
it turned out that:

1. We make ~120K symlinks in AOSP (!), all of which need to be deleted
2. System calls are sometimes slow
3. Golang spawns a new OS-level thread for each blocking system calls to
   keep cores busy

All this together means that we sometimes had 10K system calls in
flight, which meant 10K OS-level threads, which is when Go gives up and
says "I created too many threads, please help".

The fix is to move the system calls into a pool of goroutines, which
soon end up on a pool of threads (since they mostly do blocking system
calls)

Test: Presubmits.
Change-Id: Ia9aefff3b0ed373f09bb6c8b2ec1d8b0f00b213b
This commit is contained in:
Lukacs T. Berki
2022-10-31 09:01:34 +00:00
parent 2b8462997a
commit bc5f731791
2 changed files with 105 additions and 5 deletions

View File

@@ -33,7 +33,7 @@ func runJavaBinaryHostTestCase(t *testing.T, tc Bp2buildTestCase) {
}, tc)
}
var fs = map[string]string{
var testFs = map[string]string{
"test.mf": "Main-Class: com.android.test.MainClass",
"other/Android.bp": `cc_library_host_shared {
name: "jni-lib-1",
@@ -44,7 +44,7 @@ var fs = map[string]string{
func TestJavaBinaryHost(t *testing.T) {
runJavaBinaryHostTestCase(t, Bp2buildTestCase{
Description: "java_binary_host with srcs, exclude_srcs, jni_libs, javacflags, and manifest.",
Filesystem: fs,
Filesystem: testFs,
Blueprint: `java_binary_host {
name: "java-binary-host-1",
srcs: ["a.java", "b.java"],
@@ -77,7 +77,7 @@ func TestJavaBinaryHost(t *testing.T) {
func TestJavaBinaryHostRuntimeDeps(t *testing.T) {
runJavaBinaryHostTestCase(t, Bp2buildTestCase{
Description: "java_binary_host with srcs, exclude_srcs, jni_libs, javacflags, and manifest.",
Filesystem: fs,
Filesystem: testFs,
Blueprint: `java_binary_host {
name: "java-binary-host-1",
static_libs: ["java-dep-1"],
@@ -107,7 +107,7 @@ java_library {
func TestJavaBinaryHostLibs(t *testing.T) {
runJavaBinaryHostTestCase(t, Bp2buildTestCase{
Description: "java_binary_host with srcs, libs.",
Filesystem: fs,
Filesystem: testFs,
Blueprint: `java_binary_host {
name: "java-binary-host-libs",
libs: ["java-lib-dep-1"],