Reland "use symlink for bundled APEX"

This reverts commit 31c65d4fe4.

Bug: 144533348

Test: checkout master-art-host and run
ALLOW_MISSING_DEPENDENCIES=true DIST_DIR=out/dist /art/tools/dist_linux_bionic.sh -j80 com.android.art.host
the result is successful

Change-Id: Ica11eec9b64867088b16720a41c6d83905976ec5

apex/androidmk.go

@@ -52,13 +52,40 @@ func (a *apexBundle) androidMkForFiles(w io.Writer, apexName, moduleDir string)
 		return moduleNames
 	}
 
+	var postInstallCommands []string
+	for _, fi := range a.filesInfo {
+		if a.linkToSystemLib && fi.transitiveDep && fi.AvailableToPlatform() {
+			// TODO(jiyong): pathOnDevice should come from fi.module, not being calculated here
+			linkTarget := filepath.Join("/system", fi.Path())
+			linkPath := filepath.Join(a.installDir.ToMakePath().String(), apexName, fi.Path())
+			mkdirCmd := "mkdir -p " + filepath.Dir(linkPath)
+			linkCmd := "ln -sfn " + linkTarget + " " + linkPath
+			postInstallCommands = append(postInstallCommands, mkdirCmd, linkCmd)
+		}
+	}
+	postInstallCommands = append(postInstallCommands, a.compatSymlinks...)
+
 	for _, fi := range a.filesInfo {
 		if cc, ok := fi.module.(*cc.Module); ok && cc.Properties.HideFromMake {
 			continue
 		}
 
-		if !android.InList(fi.moduleName, moduleNames) {
+		linkToSystemLib := a.linkToSystemLib && fi.transitiveDep && fi.AvailableToPlatform()
-			moduleNames = append(moduleNames, fi.moduleName)
+
+		var moduleName string
+		if linkToSystemLib {
+			moduleName = fi.moduleName
+		} else {
+			moduleName = fi.moduleName + "." + apexName + a.suffix
+		}
+
+		if !android.InList(moduleName, moduleNames) {
+			moduleNames = append(moduleNames, moduleName)
+		}
+
+		if linkToSystemLib {
+			// No need to copy the file since it's linked to the system file
+			continue
 		}
 
 		fmt.Fprintln(w, "\ninclude $(CLEAR_VARS)")
@@ -67,7 +94,7 @@ func (a *apexBundle) androidMkForFiles(w io.Writer, apexName, moduleDir string)
 		} else {
 			fmt.Fprintln(w, "LOCAL_PATH :=", moduleDir)
 		}
-		fmt.Fprintln(w, "LOCAL_MODULE :=", fi.moduleName)
+		fmt.Fprintln(w, "LOCAL_MODULE :=", moduleName)
 		// /apex/<apex_name>/{lib|framework|...}
 		pathWhenActivated := filepath.Join("$(PRODUCT_OUT)", "apex", apexName, fi.installDir)
 		if apexType == flattenedApex {
@@ -160,9 +187,9 @@ func (a *apexBundle) androidMkForFiles(w io.Writer, apexName, moduleDir string)
 				}
 				fmt.Fprintln(w, "LOCAL_OVERRIDES_MODULES :=", strings.Join(patterns, " "))
 
-				if len(a.compatSymlinks) > 0 {
+				if apexType == flattenedApex && len(postInstallCommands) > 0 {
 					// For flattened apexes, compat symlinks are attached to apex_manifest.json which is guaranteed for every apex
-					fmt.Fprintln(w, "LOCAL_POST_INSTALL_CMD :=", strings.Join(a.compatSymlinks, " && "))
+					fmt.Fprintln(w, "LOCAL_POST_INSTALL_CMD :=", strings.Join(postInstallCommands, " && "))
 				}
 			}
 			fmt.Fprintln(w, "include $(BUILD_PREBUILT)")

apex/apex.go

@@ -733,6 +733,30 @@ func (af *apexFile) Ok() bool {
 	return af.builtFile != nil && af.builtFile.String() != ""
 }
 
+// Path() returns path of this apex file relative to the APEX root
+func (af *apexFile) Path() string {
+	return filepath.Join(af.installDir, af.builtFile.Base())
+}
+
+// SymlinkPaths() returns paths of the symlinks (if any) relative to the APEX root
+func (af *apexFile) SymlinkPaths() []string {
+	var ret []string
+	for _, symlink := range af.symlinks {
+		ret = append(ret, filepath.Join(af.installDir, symlink))
+	}
+	return ret
+}
+
+func (af *apexFile) AvailableToPlatform() bool {
+	if af.module == nil {
+		return false
+	}
+	if am, ok := af.module.(android.ApexModule); ok {
+		return am.AvailableFor(android.AvailableToPlatform)
+	}
+	return false
+}
+
 type apexBundle struct {
 	android.ModuleBase
 	android.DefaultableModuleBase
@@ -790,6 +814,10 @@ type apexBundle struct {
 	suffix string
 
 	installedFilesFile android.WritablePath
+
+	// Whether to create symlink to the system file instead of having a file
+	// inside the apex or not
+	linkToSystemLib bool
 }
 
 func addDependenciesForNativeModules(ctx android.BottomUpMutatorContext,
@@ -1414,7 +1442,8 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) {
 						// of the original test module (`depName`, shared by all `test_per_src`
 						// variations of that module).
 						af.moduleName = filepath.Base(af.builtFile.String())
-						af.transitiveDep = true
+						// these are not considered transitive dep
+						af.transitiveDep = false
 						filesInfo = append(filesInfo, af)
 						return true // track transitive dependencies
 					}
@@ -1452,15 +1481,22 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) {
 
 	// remove duplicates in filesInfo
 	removeDup := func(filesInfo []apexFile) []apexFile {
-		encountered := make(map[string]bool)
+		encountered := make(map[string]apexFile)
-		result := []apexFile{}
 		for _, f := range filesInfo {
 			dest := filepath.Join(f.installDir, f.builtFile.Base())
-			if !encountered[dest] {
+			if e, ok := encountered[dest]; !ok {
-				encountered[dest] = true
+				encountered[dest] = f
-				result = append(result, f)
+			} else {
+				// If a module is directly included and also transitively depended on
+				// consider it as directly included.
+				e.transitiveDep = e.transitiveDep && f.transitiveDep
+				encountered[dest] = e
 			}
 		}
+		var result []apexFile
+		for _, v := range encountered {
+			result = append(result, v)
+		}
 		return result
 	}
 	filesInfo = removeDup(filesInfo)
@@ -1487,12 +1523,6 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) {
 		}
 	}
 
-	// prepend the name of this APEX to the module names. These names will be the names of
-	// modules that will be defined if the APEX is flattened.
-	for i := range filesInfo {
-		filesInfo[i].moduleName = filesInfo[i].moduleName + "." + a.Name() + a.suffix
-	}
-
 	a.installDir = android.PathForModuleInstall(ctx, "apex")
 	a.filesInfo = filesInfo
 
@@ -1512,6 +1542,14 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) {
 			return
 		}
 	}
+	// Optimization. If we are building bundled APEX, for the files that are gathered due to the
+	// transitive dependencies, don't place them inside the APEX, but place a symlink pointing
+	// the same library in the system partition, thus effectively sharing the same libraries
+	// across the APEX boundary. For unbundled APEX, all the gathered files are actually placed
+	// in the APEX.
+	a.linkToSystemLib = !ctx.Config().UnbundledBuild() &&
+		a.installable() &&
+		!proptools.Bool(a.properties.Use_vendor)
 
 	// prepare apex_manifest.json
 	a.buildManifest(ctx, provideNativeLibs, requireNativeLibs)

apex/apex_test.go

@@ -91,6 +91,10 @@ func withBinder32bit(fs map[string][]byte, config android.Config) {
 	config.TestProductVariables.Binder32bit = proptools.BoolPtr(true)
 }
 
+func withUnbundledBuild(fs map[string][]byte, config android.Config) {
+	config.TestProductVariables.Unbundled_build = proptools.BoolPtr(true)
+}
+
 func testApexContext(t *testing.T, bp string, handlers ...testCustomizer) (*android.TestContext, android.Config) {
 	android.ClearApexDependency()
 
@@ -517,7 +521,7 @@ func TestBasicApex(t *testing.T) {
 	found_foo_link_64 := false
 	found_foo := false
 	for _, cmd := range strings.Split(copyCmds, " && ") {
-		if strings.HasPrefix(cmd, "ln -s foo64") {
+		if strings.HasPrefix(cmd, "ln -sfn foo64") {
 			if strings.HasSuffix(cmd, "bin/foo") {
 				found_foo = true
 			} else if strings.HasSuffix(cmd, "bin/foo_link_64") {
@@ -1598,46 +1602,68 @@ func TestHeaderLibsDependency(t *testing.T) {
 	ensureContains(t, cFlags, "-Imy_include")
 }
 
-func ensureExactContents(t *testing.T, ctx *android.TestContext, moduleName string, files []string) {
+type fileInApex struct {
+	path   string // path in apex
+	isLink bool
+}
+
+func getFiles(t *testing.T, ctx *android.TestContext, moduleName string) []fileInApex {
 	t.Helper()
 	apexRule := ctx.ModuleForTests(moduleName, "android_common_"+moduleName+"_image").Rule("apexRule")
 	copyCmds := apexRule.Args["copy_commands"]
 	imageApexDir := "/image.apex/"
-	var failed bool
+	var ret []fileInApex
-	var surplus []string
-	filesMatched := make(map[string]bool)
-	addContent := func(content string) {
-		for _, expected := range files {
-			if matched, _ := path.Match(expected, content); matched {
-				filesMatched[expected] = true
-				return
-			}
-		}
-		surplus = append(surplus, content)
-	}
 	for _, cmd := range strings.Split(copyCmds, "&&") {
 		cmd = strings.TrimSpace(cmd)
 		if cmd == "" {
 			continue
 		}
 		terms := strings.Split(cmd, " ")
+		var dst string
+		var isLink bool
 		switch terms[0] {
 		case "mkdir":
 		case "cp":
-			if len(terms) != 3 {
+			if len(terms) != 3 && len(terms) != 4 {
 				t.Fatal("copyCmds contains invalid cp command", cmd)
 			}
-			dst := terms[2]
+			dst = terms[len(terms)-1]
+			isLink = false
+		case "ln":
+			if len(terms) != 3 && len(terms) != 4 {
+				// ln LINK TARGET or ln -s LINK TARGET
+				t.Fatal("copyCmds contains invalid ln command", cmd)
+			}
+			dst = terms[len(terms)-1]
+			isLink = true
+		default:
+			t.Fatalf("copyCmds should contain mkdir/cp commands only: %q", cmd)
+		}
+		if dst != "" {
 			index := strings.Index(dst, imageApexDir)
 			if index == -1 {
 				t.Fatal("copyCmds should copy a file to image.apex/", cmd)
 			}
 			dstFile := dst[index+len(imageApexDir):]
-			addContent(dstFile)
+			ret = append(ret, fileInApex{path: dstFile, isLink: isLink})
-		default:
-			t.Fatalf("copyCmds should contain mkdir/cp commands only: %q", cmd)
 		}
 	}
+	return ret
+}
+
+func ensureExactContents(t *testing.T, ctx *android.TestContext, moduleName string, files []string) {
+	var failed bool
+	var surplus []string
+	filesMatched := make(map[string]bool)
+	for _, file := range getFiles(t, ctx, moduleName) {
+		for _, expected := range files {
+			if matched, _ := path.Match(expected, file.path); matched {
+				filesMatched[expected] = true
+				return
+			}
+		}
+		surplus = append(surplus, file.path)
+	}
 
 	if len(surplus) > 0 {
 		sort.Strings(surplus)
@@ -3482,6 +3508,106 @@ func TestCarryRequiredModuleNames(t *testing.T) {
 	ensureContains(t, androidMk, "LOCAL_TARGET_REQUIRED_MODULES += e f\n")
 }
 
+func TestSymlinksFromApexToSystem(t *testing.T) {
+	bp := `
+		apex {
+			name: "myapex",
+			key: "myapex.key",
+			native_shared_libs: ["mylib"],
+			java_libs: ["myjar"],
+		}
+
+		apex_key {
+			name: "myapex.key",
+			public_key: "testkey.avbpubkey",
+			private_key: "testkey.pem",
+		}
+
+		cc_library {
+			name: "mylib",
+			srcs: ["mylib.cpp"],
+			shared_libs: ["myotherlib"],
+			system_shared_libs: [],
+			stl: "none",
+			apex_available: [
+				"myapex",
+				"//apex_available:platform",
+			],
+		}
+
+		cc_library {
+			name: "myotherlib",
+			srcs: ["mylib.cpp"],
+			system_shared_libs: [],
+			stl: "none",
+			apex_available: [
+				"myapex",
+				"//apex_available:platform",
+			],
+		}
+
+		java_library {
+			name: "myjar",
+			srcs: ["foo/bar/MyClass.java"],
+			sdk_version: "none",
+			system_modules: "none",
+			libs: ["myotherjar"],
+			compile_dex: true,
+			apex_available: [
+				"myapex",
+				"//apex_available:platform",
+			],
+		}
+
+		java_library {
+			name: "myotherjar",
+			srcs: ["foo/bar/MyClass.java"],
+			sdk_version: "none",
+			system_modules: "none",
+			apex_available: [
+				"myapex",
+				"//apex_available:platform",
+			],
+		}
+	`
+
+	ensureRealfileExists := func(t *testing.T, files []fileInApex, file string) {
+		for _, f := range files {
+			if f.path == file {
+				if f.isLink {
+					t.Errorf("%q is not a real file", file)
+				}
+				return
+			}
+		}
+		t.Errorf("%q is not found", file)
+	}
+
+	ensureSymlinkExists := func(t *testing.T, files []fileInApex, file string) {
+		for _, f := range files {
+			if f.path == file {
+				if !f.isLink {
+					t.Errorf("%q is not a symlink", file)
+				}
+				return
+			}
+		}
+		t.Errorf("%q is not found", file)
+	}
+
+	ctx, _ := testApex(t, bp, withUnbundledBuild)
+	files := getFiles(t, ctx, "myapex")
+	ensureRealfileExists(t, files, "javalib/myjar.jar")
+	ensureRealfileExists(t, files, "lib64/mylib.so")
+	ensureRealfileExists(t, files, "lib64/myotherlib.so")
+
+	ctx, _ = testApex(t, bp)
+	files = getFiles(t, ctx, "myapex")
+	ensureRealfileExists(t, files, "javalib/myjar.jar")
+	ensureRealfileExists(t, files, "lib64/mylib.so")
+	ensureSymlinkExists(t, files, "lib64/myotherlib.so") // this is symlink
+}
+
 func TestMain(m *testing.M) {
 	run := func() int {
 		setUp()

apex/builder.go

@@ -258,34 +258,40 @@ func (a *apexBundle) buildUnflattenedApex(ctx android.ModuleContext) {
 
 	apexType := a.properties.ApexType
 	suffix := apexType.suffix()
+	var implicitInputs []android.Path
 	unsignedOutputFile := android.PathForModuleOut(ctx, a.Name()+suffix+".unsigned")
 
-	filesToCopy := []android.Path{}
+	// TODO(jiyong): construct the copy rules using RuleBuilder
-	for _, f := range a.filesInfo {
+	var copyCommands []string
-		filesToCopy = append(filesToCopy, f.builtFile)
+	for _, fi := range a.filesInfo {
+		destPath := android.PathForModuleOut(ctx, "image"+suffix, fi.Path()).String()
+		copyCommands = append(copyCommands, "mkdir -p "+filepath.Dir(destPath))
+		if a.linkToSystemLib && fi.transitiveDep && fi.AvailableToPlatform() {
+			// TODO(jiyong): pathOnDevice should come from fi.module, not being calculated here
+			pathOnDevice := filepath.Join("/system", fi.Path())
+			copyCommands = append(copyCommands, "ln -sfn "+pathOnDevice+" "+destPath)
+		} else {
+			copyCommands = append(copyCommands, "cp -f "+fi.builtFile.String()+" "+destPath)
+			implicitInputs = append(implicitInputs, fi.builtFile)
+		}
+		// create additional symlinks pointing the file inside the APEX
+		for _, symlinkPath := range fi.SymlinkPaths() {
+			symlinkDest := android.PathForModuleOut(ctx, "image"+suffix, symlinkPath).String()
+			copyCommands = append(copyCommands, "ln -sfn "+filepath.Base(destPath)+" "+symlinkDest)
+		}
 	}
 
-	copyCommands := []string{}
+	// TODO(jiyong): use RuleBuilder
-	emitCommands := []string{}
+	var emitCommands []string
-	imageContentFile := android.PathForModuleOut(ctx, a.Name()+"-content.txt")
+	imageContentFile := android.PathForModuleOut(ctx, "content.txt")
 	emitCommands = append(emitCommands, "echo ./apex_manifest.pb >> "+imageContentFile.String())
 	if proptools.Bool(a.properties.Legacy_android10_support) {
 		emitCommands = append(emitCommands, "echo ./apex_manifest.json >> "+imageContentFile.String())
 	}
-	for i, src := range filesToCopy {
+	for _, fi := range a.filesInfo {
-		dest := filepath.Join(a.filesInfo[i].installDir, src.Base())
+		emitCommands = append(emitCommands, "echo './"+fi.Path()+"' >> "+imageContentFile.String())
-		emitCommands = append(emitCommands, "echo './"+dest+"' >> "+imageContentFile.String())
-		dest_path := filepath.Join(android.PathForModuleOut(ctx, "image"+suffix).String(), dest)
-		copyCommands = append(copyCommands, "mkdir -p "+filepath.Dir(dest_path))
-		copyCommands = append(copyCommands, "cp "+src.String()+" "+dest_path)
-		for _, sym := range a.filesInfo[i].symlinks {
-			symlinkDest := filepath.Join(filepath.Dir(dest_path), sym)
-			copyCommands = append(copyCommands, "ln -s "+filepath.Base(dest)+" "+symlinkDest)
-		}
 	}
 	emitCommands = append(emitCommands, "sort -o "+imageContentFile.String()+" "+imageContentFile.String())
-
-	implicitInputs := append(android.Paths(nil), filesToCopy...)
 	implicitInputs = append(implicitInputs, a.manifestPbOut)
 
 	if a.properties.Whitelisted_files != nil {
@@ -530,7 +536,7 @@ func (a *apexBundle) buildFilesInfo(ctx android.ModuleContext) {
 	if a.installable() {
 		// For flattened APEX, do nothing but make sure that APEX manifest and apex_pubkey are also copied along
 		// with other ordinary files.
-		a.filesInfo = append(a.filesInfo, newApexFile(ctx, a.manifestPbOut, "apex_manifest.pb."+a.Name()+a.suffix, ".", etc, nil))
+		a.filesInfo = append(a.filesInfo, newApexFile(ctx, a.manifestPbOut, "apex_manifest.pb", ".", etc, nil))
 
 		// rename to apex_pubkey
 		copiedPubkey := android.PathForModuleOut(ctx, "apex_pubkey")
@@ -539,7 +545,7 @@ func (a *apexBundle) buildFilesInfo(ctx android.ModuleContext) {
 			Input:  a.public_key_file,
 			Output: copiedPubkey,
 		})
-		a.filesInfo = append(a.filesInfo, newApexFile(ctx, copiedPubkey, "apex_pubkey."+a.Name()+a.suffix, ".", etc, nil))
+		a.filesInfo = append(a.filesInfo, newApexFile(ctx, copiedPubkey, "apex_pubkey", ".", etc, nil))
 
 		if a.properties.ApexType == flattenedApex {
 			apexName := proptools.StringDefault(a.properties.Apex_name, a.Name())