Merge changes from topic "uses-libs-5" am: 8fab5bcce3

Original change: https://android-review.googlesource.com/c/platform/build/soong/+/1353554

Change-Id: I11201716a66279597781fbe9595e01fee32a84a5

dexpreopt/config.go

@@ -383,7 +383,7 @@ func createGlobalSoongConfig(ctx android.ModuleContext) *GlobalSoongConfig {
 		SoongZip:         ctx.Config().HostToolPath(ctx, "soong_zip"),
 		Zip2zip:          ctx.Config().HostToolPath(ctx, "zip2zip"),
 		ManifestCheck:    ctx.Config().HostToolPath(ctx, "manifest_check"),
-		ConstructContext: android.PathForSource(ctx, "build/soong/scripts/construct_context.sh"),
+		ConstructContext: ctx.Config().HostToolPath(ctx, "construct_context"),
 	}
 }
 
@@ -574,7 +574,7 @@ func GlobalSoongConfigForTests(config android.Config) *GlobalSoongConfig {
 			SoongZip:         android.PathForTesting("soong_zip"),
 			Zip2zip:          android.PathForTesting("zip2zip"),
 			ManifestCheck:    android.PathForTesting("manifest_check"),
-			ConstructContext: android.PathForTesting("construct_context.sh"),
+			ConstructContext: android.PathForTesting("construct_context"),
 		}
 	}).(*GlobalSoongConfig)
 }

dexpreopt/dexpreopt.go

@@ -208,7 +208,7 @@ type classLoaderContext struct {
 // targetSdkVersion in the manifest or APK is less than that API version.
 type classLoaderContextMap map[int]*classLoaderContext
 
-const anySdkVersion int = -1
+const anySdkVersion int = 9999 // should go last in class loader context
 
 func (m classLoaderContextMap) getSortedKeys() []int {
 	keys := make([]int, 0, len(m))
@@ -276,14 +276,30 @@ func dexpreoptCommand(ctx android.PathContext, globalSoong *GlobalSoongConfig, g
 
 	invocationPath := odexPath.ReplaceExtension(ctx, "invocation")
 
+	classLoaderContexts := make(classLoaderContextMap)
 	systemServerJars := NonUpdatableSystemServerJars(ctx, global)
 
-	classLoaderContexts := make(classLoaderContextMap)
+	rule.Command().FlagWithArg("mkdir -p ", filepath.Dir(odexPath.String()))
+	rule.Command().FlagWithOutput("rm -f ", odexPath)
 
-	// A flag indicating if the '&' class loader context is used.
+	if jarIndex := android.IndexList(module.Name, systemServerJars); jarIndex >= 0 {
-	unknownClassLoaderContext := false
+		// System server jars should be dexpreopted together: class loader context of each jar
+		// should include all preceding jars on the system server classpath.
+		classLoaderContexts.addSystemServerLibs(anySdkVersion, ctx, module, systemServerJars[:jarIndex]...)
 
-	if module.EnforceUsesLibraries {
+		// Copy the system server jar to a predefined location where dex2oat will find it.
+		dexPathHost := SystemServerDexJarHostPath(ctx, module.Name)
+		rule.Command().Text("mkdir -p").Flag(filepath.Dir(dexPathHost.String()))
+		rule.Command().Text("cp -f").Input(module.DexPath).Output(dexPathHost)
+
+		checkSystemServerOrder(ctx, jarIndex)
+
+		clc := classLoaderContexts[anySdkVersion]
+		rule.Command().
+			Text("class_loader_context_arg=--class-loader-context=PCL[" + strings.Join(clc.Host.Strings(), ":") + "]").
+			Implicits(clc.Host).
+			Text("stored_class_loader_context_arg=--stored-class-loader-context=PCL[" + strings.Join(clc.Target, ":") + "]")
+	} else if module.EnforceUsesLibraries {
 		// Unconditional class loader context.
 		usesLibs := append(copyOf(module.UsesLibraries), module.OptionalUsesLibraries...)
 		classLoaderContexts.addLibs(anySdkVersion, module, usesLibs...)
@@ -306,41 +322,8 @@ func dexpreoptCommand(ctx android.PathContext, globalSoong *GlobalSoongConfig, g
 		if !contains(usesLibs, testBase) {
 			classLoaderContexts.addLibs(30, module, testBase)
 		}
-	} else if jarIndex := android.IndexList(module.Name, systemServerJars); jarIndex >= 0 {
-		// System server jars should be dexpreopted together: class loader context of each jar
-		// should include all preceding jars on the system server classpath.
-		classLoaderContexts.addSystemServerLibs(anySdkVersion, ctx, module, systemServerJars[:jarIndex]...)
 
-		// Copy the system server jar to a predefined location where dex2oat will find it.
+		// Generate command that saves target SDK version in a shell variable.
-		dexPathHost := SystemServerDexJarHostPath(ctx, module.Name)
-		rule.Command().Text("mkdir -p").Flag(filepath.Dir(dexPathHost.String()))
-		rule.Command().Text("cp -f").Input(module.DexPath).Output(dexPathHost)
-
-		checkSystemServerOrder(ctx, jarIndex)
-	} else {
-		// Pass special class loader context to skip the classpath and collision check.
-		// This will get removed once LOCAL_USES_LIBRARIES is enforced.
-		// Right now LOCAL_USES_LIBRARIES is opt in, for the case where it's not specified we still default
-		// to the &.
-		unknownClassLoaderContext = true
-	}
-
-	rule.Command().FlagWithArg("mkdir -p ", filepath.Dir(odexPath.String()))
-	rule.Command().FlagWithOutput("rm -f ", odexPath)
-	// Set values in the environment of the rule.  These may be modified by construct_context.sh.
-	if unknownClassLoaderContext {
-		rule.Command().
-			Text(`class_loader_context_arg=--class-loader-context=\&`).
-			Text(`stored_class_loader_context_arg=""`)
-	} else {
-		clc := classLoaderContexts[anySdkVersion]
-		rule.Command().
-			Text("class_loader_context_arg=--class-loader-context=PCL[" + strings.Join(clc.Host.Strings(), ":") + "]").
-			Implicits(clc.Host).
-			Text("stored_class_loader_context_arg=--stored-class-loader-context=PCL[" + strings.Join(clc.Target, ":") + "]")
-	}
-
-	if module.EnforceUsesLibraries {
 		if module.ManifestPath != nil {
 			rule.Command().Text(`target_sdk_version="$(`).
 				Tool(globalSoong.ManifestCheck).
@@ -356,20 +339,30 @@ func dexpreoptCommand(ctx android.PathContext, globalSoong *GlobalSoongConfig, g
 				Text(`| grep "targetSdkVersion" | sed -n "s/targetSdkVersion:'\(.*\)'/\1/p"`).
 				Text(`)"`)
 		}
+
+		// Generate command that saves host and target class loader context in shell variables.
+		cmd := rule.Command().
+			Text(`eval "$(`).Tool(globalSoong.ConstructContext).
+			Text(` --target-sdk-version ${target_sdk_version}`)
 		for _, ver := range classLoaderContexts.getSortedKeys() {
 			clc := classLoaderContexts.getValue(ver)
-			var varHost, varTarget string
+			verString := fmt.Sprintf("%d", ver)
 			if ver == anySdkVersion {
-				varHost = "dex_preopt_host_libraries"
+				verString = "any" // a special keyword that means any SDK version
-				varTarget = "dex_preopt_target_libraries"
+			}
+			cmd.Textf(`--host-classpath-for-sdk %s %s`, verString, strings.Join(clc.Host.Strings(), ":")).
+				Implicits(clc.Host).
+				Textf(`--target-classpath-for-sdk %s %s`, verString, strings.Join(clc.Target, ":"))
+		}
+		cmd.Text(`)"`)
 	} else {
-				varHost = fmt.Sprintf("conditional_host_libs_%d", ver)
+		// Pass special class loader context to skip the classpath and collision check.
-				varTarget = fmt.Sprintf("conditional_target_libs_%d", ver)
+		// This will get removed once LOCAL_USES_LIBRARIES is enforced.
-			}
+		// Right now LOCAL_USES_LIBRARIES is opt in, for the case where it's not specified we still default
-			rule.Command().Textf(varHost+`="%s"`, strings.Join(clc.Host.Strings(), " ")).Implicits(clc.Host)
+		// to the &.
-			rule.Command().Textf(varTarget+`="%s"`, strings.Join(clc.Target, " "))
+		rule.Command().
-		}
+			Text(`class_loader_context_arg=--class-loader-context=\&`).
-		rule.Command().Text("source").Tool(globalSoong.ConstructContext).Input(module.DexPath)
+			Text(`stored_class_loader_context_arg=""`)
 	}
 
 	// Devices that do not have a product partition use a symlink from /product to /system/product.

java/app.go

@@ -1865,9 +1865,8 @@ func (u *usesLibrary) deps(ctx android.BottomUpMutatorContext, hasFrameworkLibs
 		// creating a cyclic dependency:
 		//     e.g. framework-res -> org.apache.http.legacy -> ... -> framework-res.
 		if hasFrameworkLibs {
-			// dexpreopt/dexpreopt.go needs the paths to the dex jars of these libraries in case construct_context.sh needs
+			// Dexpreopt needs paths to the dex jars of these libraries in order to construct
-			// to pass them to dex2oat.  Add them as a dependency so we can determine the path to the dex jar of each
+			// class loader context for dex2oat. Add them as a dependency with a special tag.
-			// library to dexpreopt.
 			ctx.AddVariationDependencies(nil, usesLibTag,
 				"org.apache.http.legacy",
 				"android.hidl.base-V1.0-java",

java/app_test.go

@@ -2590,13 +2590,13 @@ func TestUsesLibraries(t *testing.T) {
 	// Test that only present libraries are preopted
 	cmd = app.Rule("dexpreopt").RuleParams.Command
 
-	if w := `dex_preopt_target_libraries="/system/framework/foo.jar /system/framework/bar.jar"`; !strings.Contains(cmd, w) {
+	if w := `--target-classpath-for-sdk any /system/framework/foo.jar:/system/framework/bar.jar`; !strings.Contains(cmd, w) {
 		t.Errorf("wanted %q in %q", w, cmd)
 	}
 
 	cmd = prebuilt.Rule("dexpreopt").RuleParams.Command
 
-	if w := `dex_preopt_target_libraries="/system/framework/foo.jar /system/framework/bar.jar"`; !strings.Contains(cmd, w) {
+	if w := `--target-classpath-for-sdk any /system/framework/foo.jar:/system/framework/bar.jar`; !strings.Contains(cmd, w) {
 		t.Errorf("wanted %q in %q", w, cmd)
 	}
 }

scripts/Android.bp

@@ -149,6 +149,46 @@ python_test_host {
     test_suites: ["general-tests"],
 }
 
+python_binary_host {
+    name: "construct_context",
+    main: "construct_context.py",
+    srcs: [
+        "construct_context.py",
+    ],
+    version: {
+        py2: {
+            enabled: true,
+        },
+        py3: {
+            enabled: false,
+        },
+    },
+    libs: [
+        "manifest_utils",
+    ],
+}
+
+python_test_host {
+    name: "construct_context_test",
+    main: "construct_context_test.py",
+    srcs: [
+        "construct_context_test.py",
+        "construct_context.py",
+    ],
+    version: {
+        py2: {
+            enabled: true,
+        },
+        py3: {
+            enabled: false,
+        },
+    },
+    libs: [
+        "manifest_utils",
+    ],
+    test_suites: ["general-tests"],
+}
+
 python_binary_host {
     name: "lint-project-xml",
     main: "lint-project-xml.py",

scripts/OWNERS

@@ -1,4 +1,4 @@
 per-file system-clang-format,system-clang-format-2 = enh@google.com,smoreland@google.com
 per-file build-mainline-modules.sh = ngeoffray@google.com,paulduffin@google.com,mast@google.com
 per-file build-aml-prebuilts.sh = ngeoffray@google.com,paulduffin@google.com,mast@google.com
-per-file construct_context.sh = ngeoffray@google.com,calin@google.com,mathieuc@google.com,skvadrik@google.com
+per-file construct_context.py = ngeoffray@google.com,calin@google.com,mathieuc@google.com,skvadrik@google.com

scripts/construct_context.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""A tool for constructing class loader context."""
+
+from __future__ import print_function
+
+import argparse
+import sys
+
+from manifest import compare_version_gt
+
+
+def parse_args(args):
+  """Parse commandline arguments."""
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--target-sdk-version', default='', dest='sdk',
+    help='specify target SDK version (as it appears in the manifest)')
+  parser.add_argument('--host-classpath-for-sdk', dest='host_classpaths',
+    action='append', nargs=2, metavar=('sdk','classpath'),
+    help='specify classpath on host for a given SDK version or "any" version')
+  parser.add_argument('--target-classpath-for-sdk', dest='target_classpaths',
+    action='append', nargs=2, metavar=('sdk','classpath'),
+    help='specify classpath on target for a given SDK version or "any" version')
+  return parser.parse_args(args)
+
+# The hidl.manager shared library has a dependency on hidl.base. We manually
+# add that information to the class loader context if we see those libraries.
+HIDL_MANAGER = 'android.hidl.manager-V1.0-java'
+HIDL_BASE    = 'android.hidl.base-V1.0-java'
+
+# Special keyword that means that the classpath should be added to class loader
+# context regardless of the target SDK version.
+any_sdk = 'any'
+
+# We assume that the order of classpath arguments passed to this script is
+# correct (matches the order computed by package manager). It is possible to
+# sort them here, but Soong needs to use deterministic order anyway, so it can
+# as well use the correct order.
+def construct_context(versioned_classpaths, target_sdk):
+  context = []
+  for [sdk, classpath] in versioned_classpaths:
+    if sdk == any_sdk or compare_version_gt(sdk, target_sdk):
+      for jar in classpath.split(':'):
+        pcl = 'PCL[%s]' % jar
+        if HIDL_MANAGER in jar:
+          pcl += '{PCL[%s]}' % jar.replace(HIDL_MANAGER, HIDL_BASE, 1)
+        context.append(pcl)
+  return context
+
+def construct_contexts(args):
+  host_context = construct_context(args.host_classpaths, args.sdk)
+  target_context = construct_context(args.target_classpaths, args.sdk)
+  context_sep = '#'
+  return ('class_loader_context_arg=--class-loader-context=PCL[]{%s} ; ' % context_sep.join(host_context) +
+    'stored_class_loader_context_arg=--stored-class-loader-context=PCL[]{%s}' % context_sep.join(target_context))
+
+def main():
+  """Program entry point."""
+  try:
+    args = parse_args(sys.argv[1:])
+    if not args.sdk:
+      raise SystemExit('target sdk version is not set')
+    if not args.host_classpaths:
+      raise SystemExit('host classpath is not set')
+    if not args.target_classpaths:
+      raise SystemExit('target classpath is not set')
+
+    print(construct_contexts(args))
+
+  # pylint: disable=broad-except
+  except Exception as err:
+    print('error: ' + str(err), file=sys.stderr)
+    sys.exit(-1)
+
+if __name__ == '__main__':
+  main()

scripts/construct_context.sh

@@ -1,78 +0,0 @@
-#!/bin/bash
-#
-# Copyright (C) 2018 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-# target_sdk_version: parsed from manifest
-#
-# outputs
-# class_loader_context_arg: final class loader conext arg
-# stored_class_loader_context_arg: final stored class loader context arg
-
-if [ -z "${target_sdk_version}" ]; then
-    echo "ERROR: target_sdk_version not set"
-    exit 2
-fi
-
-# The hidl.manager shared library has a dependency on hidl.base. We'll manually
-# add that information to the class loader context if we see those libraries.
-hidl_manager="android.hidl.manager-V1.0-java"
-hidl_base="android.hidl.base-V1.0-java"
-
-function add_to_contexts {
-  for i in $1; do
-    if [[ -z "${class_loader_context}" ]]; then
-      export class_loader_context="PCL[$i]"
-    else
-      export class_loader_context+="#PCL[$i]"
-    fi
-    if [[ $i == *"$hidl_manager"* ]]; then
-      export class_loader_context+="{PCL[${i/$hidl_manager/$hidl_base}]}"
-    fi
-  done
-
-  for i in $2; do
-    if [[ -z "${stored_class_loader_context}" ]]; then
-      export stored_class_loader_context="PCL[$i]"
-    else
-      export stored_class_loader_context+="#PCL[$i]"
-    fi
-    if [[ $i == *"$hidl_manager"* ]]; then
-      export stored_class_loader_context+="{PCL[${i/$hidl_manager/$hidl_base}]}"
-    fi
-  done
-}
-
-# The order below must match what the package manager also computes for
-# class loader context.
-
-if [[ "${target_sdk_version}" -lt "28" ]]; then
-  add_to_contexts "${conditional_host_libs_28}" "${conditional_target_libs_28}"
-fi
-
-if [[ "${target_sdk_version}" -lt "29" ]]; then
-  add_to_contexts "${conditional_host_libs_29}" "${conditional_target_libs_29}"
-fi
-
-if [[ "${target_sdk_version}" -lt "30" ]]; then
-  add_to_contexts "${conditional_host_libs_30}" "${conditional_target_libs_30}"
-fi
-
-add_to_contexts "${dex_preopt_host_libraries}" "${dex_preopt_target_libraries}"
-
-# Generate the actual context string.
-export class_loader_context_arg="--class-loader-context=PCL[]{${class_loader_context}}"
-export stored_class_loader_context_arg="--stored-class-loader-context=PCL[]{${stored_class_loader_context}}"

scripts/construct_context_test.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2020 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Unit tests for construct_context.py."""
+
+import sys
+import unittest
+
+import construct_context as cc
+
+sys.dont_write_bytecode = True
+
+def construct_contexts(arglist):
+  args = cc.parse_args(arglist)
+  return cc.construct_contexts(args)
+
+classpaths = [
+  '--host-classpath-for-sdk', '28', 'out/zdir/z.jar',
+  '--target-classpath-for-sdk', '28', '/system/z.jar',
+  '--host-classpath-for-sdk', '29', 'out/xdir/x.jar:out/ydir/y.jar',
+  '--target-classpath-for-sdk', '29', '/system/x.jar:/product/y.jar',
+  '--host-classpath-for-sdk', 'any', 'out/adir/a.jar:out/android.hidl.manager-V1.0-java.jar:out/bdir/b.jar',
+  '--target-classpath-for-sdk', 'any', '/system/a.jar:/system/android.hidl.manager-V1.0-java.jar:/product/b.jar',
+]
+
+class ConstructContextTest(unittest.TestCase):
+  def test_construct_context_28(self):
+    args = ['--target-sdk-version', '28'] + classpaths
+    result = construct_contexts(args)
+    expect = ('class_loader_context_arg=--class-loader-context=PCL[]{PCL[out/xdir/x.jar]'
+      '#PCL[out/ydir/y.jar]'
+      '#PCL[out/adir/a.jar]'
+      '#PCL[out/android.hidl.manager-V1.0-java.jar]{PCL[out/android.hidl.base-V1.0-java.jar]}'
+      '#PCL[out/bdir/b.jar]}'
+      ' ; '
+      'stored_class_loader_context_arg=--stored-class-loader-context=PCL[]{PCL[/system/x.jar]'
+      '#PCL[/product/y.jar]'
+      '#PCL[/system/a.jar]'
+      '#PCL[/system/android.hidl.manager-V1.0-java.jar]{PCL[/system/android.hidl.base-V1.0-java.jar]}'
+      '#PCL[/product/b.jar]}')
+    self.assertEqual(result, expect)
+
+  def test_construct_context_29(self):
+    args = ['--target-sdk-version', '29'] + classpaths
+    result = construct_contexts(args)
+    expect = ('class_loader_context_arg=--class-loader-context=PCL[]{PCL[out/adir/a.jar]'
+      '#PCL[out/android.hidl.manager-V1.0-java.jar]{PCL[out/android.hidl.base-V1.0-java.jar]}'
+      '#PCL[out/bdir/b.jar]}'
+      ' ; '
+      'stored_class_loader_context_arg=--stored-class-loader-context=PCL[]{PCL[/system/a.jar]'
+      '#PCL[/system/android.hidl.manager-V1.0-java.jar]{PCL[/system/android.hidl.base-V1.0-java.jar]}'
+      '#PCL[/product/b.jar]}')
+    self.assertEqual(result, expect)
+
+  def test_construct_context_S(self):
+    args = ['--target-sdk-version', 'S'] + classpaths
+    result = construct_contexts(args)
+    expect = ('class_loader_context_arg=--class-loader-context=PCL[]{PCL[out/adir/a.jar]'
+      '#PCL[out/android.hidl.manager-V1.0-java.jar]{PCL[out/android.hidl.base-V1.0-java.jar]}'
+      '#PCL[out/bdir/b.jar]}'
+      ' ; '
+      'stored_class_loader_context_arg=--stored-class-loader-context=PCL[]{PCL[/system/a.jar]'
+      '#PCL[/system/android.hidl.manager-V1.0-java.jar]{PCL[/system/android.hidl.base-V1.0-java.jar]}'
+      '#PCL[/product/b.jar]}')
+    self.assertEqual(result, expect)
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)