From 3bfabf262613c95bc1c9ad7f77d3373f0e3b3e1b Mon Sep 17 00:00:00 2001 From: Ulya Trafimovich Date: Fri, 20 Nov 2020 17:28:51 +0000 Subject: [PATCH] Documenting java/dexpreopt_bootjars.go Test: m nothing Bug: 173092919 Change-Id: I20018e1df6ad5e21172bb91435ae2c9cad7024e6 --- java/dexpreopt_bootjars.go | 246 +++++++++++++++++++++++++++++++++---- 1 file changed, 219 insertions(+), 27 deletions(-) diff --git a/java/dexpreopt_bootjars.go b/java/dexpreopt_bootjars.go index f9975ba5c..5df58452c 100644 --- a/java/dexpreopt_bootjars.go +++ b/java/dexpreopt_bootjars.go @@ -25,11 +25,177 @@ import ( "github.com/google/blueprint/proptools" ) +// This comment describes: +// 1. ART boot images in general (their types, structure, file layout, etc.) +// 2. build system support for boot images +// +// 1. ART boot images +// ------------------ +// +// A boot image in ART is a set of files that contain AOT-compiled native code and a heap snapshot +// of AOT-initialized classes for the bootclasspath Java libraries. A boot image is compiled from a +// set of DEX jars by the dex2oat compiler. A boot image is used for two purposes: 1) it is +// installed on device and loaded at runtime, and 2) other Java libraries and apps are compiled +// against it (compilation may take place either on host, known as "dexpreopt", or on device, known +// as "dexopt"). +// +// A boot image is not a single file, but a collection of interrelated files. Each boot image has a +// number of components that correspond to the Java libraries that constitute it. For each component +// there are multiple files: +// - *.oat or *.odex file with native code (architecture-specific, one per instruction set) +// - *.art file with pre-initialized Java classes (architecture-specific, one per instruction set) +// - *.vdex file with verification metadata for the DEX bytecode (architecture independent) +// +// *.vdex files for the boot images do not contain the DEX bytecode itself, because the +// bootclasspath DEX files are stored on disk in uncompressed and aligned form. Consequently a boot +// image is not self-contained and cannot be used without its DEX files. To simplify the management +// of boot image files, ART uses a certain naming scheme and associates the following metadata with +// each boot image: +// - A stem, which is a symbolic name that is prepended to boot image file names. +// - A location (on-device path to the boot image files). +// - A list of boot image locations (on-device paths to dependency boot images). +// - A set of DEX locations (on-device paths to the DEX files, one location for one DEX file used +// to compile the boot image). +// +// There are two kinds of boot images: +// - primary boot images +// - boot image extensions +// +// 1.1. Primary boot images +// ------------------------ +// +// A primary boot image is compiled for a core subset of bootclasspath Java libraries. It does not +// depend on any other images, and other boot images may depend on it. +// +// For example, assuming that the stem is "boot", the location is /apex/com.android.art/javalib/, +// the set of core bootclasspath libraries is A B C, and the boot image is compiled for ARM targets +// (32 and 64 bits), it will have three components with the following files: +// - /apex/com.android.art/javalib/{arm,arm64}/boot.{art,oat,vdex} +// - /apex/com.android.art/javalib/{arm,arm64}/boot-B.{art,oat,vdex} +// - /apex/com.android.art/javalib/{arm,arm64}/boot-C.{art,oat,vdex} +// +// The files of the first component are special: they do not have the component name appended after +// the stem. This naming convention dates back to the times when the boot image was not split into +// components, and there were just boot.oat and boot.art. The decision to split was motivated by +// licensing reasons for one of the bootclasspath libraries. +// +// As of November 2020 the only primary boot image in Android is the image in the ART APEX +// com.android.art. The primary ART boot image contains the Core libraries that are part of the ART +// module. When the ART module gets updated, the primary boot image will be updated with it, and all +// dependent images will get invalidated (the checksum of the primary image stored in dependent +// images will not match), unless they are updated in sync with the ART module. +// +// 1.2. Boot image extensions +// -------------------------- +// +// A boot image extension is compiled for a subset of bootclasspath Java libraries (in particular, +// this subset does not include the Core bootclasspath libraries that go into the primary boot +// image). A boot image extension depends on the primary boot image and optionally some other boot +// image extensions. Other images may depend on it. In other words, boot image extensions can form +// acyclic dependency graphs. +// +// The motivation for boot image extensions comes from the Mainline project. Consider a situation +// when the list of bootclasspath libraries is A B C, and both A and B are parts of the Android +// platform, but C is part of an updatable APEX com.android.C. When the APEX is updated, the Java +// code for C might have changed compared to the code that was used to compile the boot image. +// Consequently, the whole boot image is obsolete and invalidated (even though the code for A and B +// that does not depend on C is up to date). To avoid this, the original monolithic boot image is +// split in two parts: the primary boot image that contains A B, and the boot image extension that +// contains C and depends on the primary boot image (extends it). +// +// For example, assuming that the stem is "boot", the location is /system/framework, the set of +// bootclasspath libraries is D E (where D is part of the platform and is located in +// /system/framework, and E is part of a non-updatable APEX com.android.E and is located in +// /apex/com.android.E/javalib), and the boot image is compiled for ARM targets (32 and 64 bits), +// it will have two components with the following files: +// - /system/framework/{arm,arm64}/boot-D.{art,oat,vdex} +// - /system/framework/{arm,arm64}/boot-E.{art,oat,vdex} +// +// As of November 2020 the only boot image extension in Android is the Framework boot image +// extension. It extends the primary ART boot image and contains Framework libraries and other +// bootclasspath libraries from the platform and non-updatable APEXes that are not included in the +// ART image. The Framework boot image extension is updated together with the platform. In the +// future other boot image extensions may be added for some updatable modules. +// +// +// 2. Build system support for boot images +// --------------------------------------- +// +// The primary ART boot image needs to be compiled with one dex2oat invocation that depends on DEX +// jars for the core libraries. Framework boot image extension needs to be compiled with one dex2oat +// invocation that depends on the primary ART boot image and all bootclasspath DEX jars except the +// Core libraries. +// +// 2.1. Libraries that go in the boot images +// ----------------------------------------- +// +// The contents of each boot image are determined by the PRODUCT variables. The primary ART APEX +// boot image contains libraries listed in the ART_APEX_JARS variable in the AOSP makefiles. The +// Framework boot image extension contains libraries specified in the PRODUCT_BOOT_JARS and +// PRODUCT_BOOT_JARS_EXTRA variables. The AOSP makefiles specify some common Framework libraries, +// but more product-specific libraries can be added in the product makefiles. +// +// Each component of the PRODUCT_BOOT_JARS and PRODUCT_BOOT_JARS_EXTRA variables is either a simple +// name (if the library is a part of the Platform), or a colon-separated pair (if the +// library is a part of a non-updatable APEX). +// +// A related variable PRODUCT_UPDATABLE_BOOT_JARS contains bootclasspath libraries that are in +// updatable APEXes. They are not included in the boot image. +// +// One exception to the above rules are "coverage" builds (a special build flavor which requires +// setting environment variable EMMA_INSTRUMENT_FRAMEWORK=true). In coverage builds the Java code in +// boot image libraries is instrumented, which means that the instrumentation library (jacocoagent) +// needs to be added to the list of bootclasspath DEX jars. +// +// In general, there is a requirement that the source code for a boot image library must be +// available at build time (e.g. it cannot be a stub that has a separate implementation library). +// +// 2.2. Static configs +// ------------------- +// +// Because boot images are used to dexpreopt other Java modules, the paths to boot image files must +// be known by the time dexpreopt build rules for the dependent modules are generated. Boot image +// configs are constructed very early during the build, before build rule generation. The configs +// provide predefined paths to boot image files (these paths depend only on static build +// configuration, such as PRODUCT variables, and use hard-coded directory names). +// +// 2.3. Singleton +// -------------- +// +// Build rules for the boot images are generated with a Soong singleton. Because a singleton has no +// dependencies on other modules, it has to find the modules for the DEX jars using VisitAllModules. +// Soong loops through all modules and compares each module against a list of bootclasspath library +// names. Then it generates build rules that copy DEX jars from their intermediate module-specific +// locations to the hard-coded locations predefined in the boot image configs. +// +// It would be possible to use a module with proper dependencies instead, but that would require +// changes in the way Soong generates variables for Make: a singleton can use one MakeVars() method +// that writes variables to out/soong/make_vars-*.mk, which is included early by the main makefile, +// but module(s) would have to use out/soong/Android-*.mk which has a group of LOCAL_* variables +// for each module, and is included later. +// +// 2.4. Install rules +// ------------------ +// +// The primary boot image and the Framework extension are installed in different ways. The primary +// boot image is part of the ART APEX: it is copied into the APEX intermediate files, packaged +// together with other APEX contents, extracted and mounted on device. The Framework boot image +// extension is installed by the rules defined in makefiles (make/core/dex_preopt_libart.mk). Soong +// writes out a few DEXPREOPT_IMAGE_* variables for Make; these variables contain boot image names, +// paths and so on. +// +// 2.5. JIT-Zygote configuration +// ----------------------------- +// +// One special configuration is JIT-Zygote build, when the primary ART image is used for compiling +// apps instead of the Framework boot image extension (see DEXPREOPT_USE_ART_IMAGE and UseArtImage). +// + func init() { RegisterDexpreoptBootJarsComponents(android.InitRegistrationContext) } -// Target-independent description of pre-compiled boot image. +// Target-independent description of a boot image. type bootImageConfig struct { // If this image is an extension, the image that it extends. extends *bootImageConfig @@ -66,7 +232,7 @@ type bootImageConfig struct { variants []*bootImageVariant } -// Target-dependent description of pre-compiled boot image. +// Target-dependent description of a boot image. type bootImageVariant struct { *bootImageConfig @@ -90,6 +256,7 @@ type bootImageVariant struct { unstrippedInstalls android.RuleBuilderInstalls } +// Get target-specific boot image variant for the given boot image config and target. func (image bootImageConfig) getVariant(target android.Target) *bootImageVariant { for _, variant := range image.variants { if variant.target.Os == target.Os && variant.target.Arch.ArchType == target.Arch.ArchType { @@ -99,7 +266,7 @@ func (image bootImageConfig) getVariant(target android.Target) *bootImageVariant return nil } -// Return any (the first) variant which is for the device (as opposed to for the host) +// Return any (the first) variant which is for the device (as opposed to for the host). func (image bootImageConfig) getAnyAndroidVariant() *bootImageVariant { for _, variant := range image.variants { if variant.target.Os == android.Android { @@ -109,10 +276,12 @@ func (image bootImageConfig) getAnyAndroidVariant() *bootImageVariant { return nil } +// Return the name of a boot image module given a boot image config and a component (module) index. +// A module name is a combination of the Java library name, and the boot image stem (that is stored +// in the config). func (image bootImageConfig) moduleName(ctx android.PathContext, idx int) string { - // Dexpreopt on the boot class path produces multiple files. The first dex file - // is converted into 'name'.art (to match the legacy assumption that 'name'.art - // exists), and the rest are converted to 'name'-.art. + // The first module of the primary boot image is special: its module name has only the stem, but + // not the library name. All other module names are of the form - m := image.modules.Jar(idx) name := image.stem if idx != 0 || image.extends != nil { @@ -121,6 +290,7 @@ func (image bootImageConfig) moduleName(ctx android.PathContext, idx int) string return name } +// Return the name of the first boot image module, or stem if the list of modules is empty. func (image bootImageConfig) firstModuleNameOrStem(ctx android.PathContext) string { if image.modules.Len() > 0 { return image.moduleName(ctx, 0) @@ -129,6 +299,8 @@ func (image bootImageConfig) firstModuleNameOrStem(ctx android.PathContext) stri } } +// Return filenames for the given boot image component, given the output directory and a list of +// extensions. func (image bootImageConfig) moduleFiles(ctx android.PathContext, dir android.OutputPath, exts ...string) android.OutputPaths { ret := make(android.OutputPaths, 0, image.modules.Len()*len(exts)) for i := 0; i < image.modules.Len(); i++ { @@ -140,17 +312,26 @@ func (image bootImageConfig) moduleFiles(ctx android.PathContext, dir android.Ou return ret } +// Return boot image locations (as a list of symbolic paths). +// // The image "location" is a symbolic path that, with multiarchitecture support, doesn't really // exist on the device. Typically it is /apex/com.android.art/javalib/boot.art and should be the // same for all supported architectures on the device. The concrete architecture specific files // actually end up in architecture-specific sub-directory such as arm, arm64, x86, or x86_64. // -// For example a physical file -// "/apex/com.android.art/javalib/x86/boot.art" has "image location" -// "/apex/com.android.art/javalib/boot.art" (which is not an actual file). +// For example a physical file /apex/com.android.art/javalib/x86/boot.art has "image location" +// /apex/com.android.art/javalib/boot.art (which is not an actual file). +// +// For a primary boot image the list of locations has a single element. +// +// For a boot image extension the list of locations contains a location for all dependency images +// (including the primary image) and the location of the extension itself. For example, for the +// Framework boot image extension that depends on the primary ART boot image the list contains two +// elements. // // The location is passed as an argument to the ART tools like dex2oat instead of the real path. // ART tools will then reconstruct the architecture-specific real path. +// func (image *bootImageVariant) imageLocations() (imageLocations []string) { if image.extends != nil { imageLocations = image.extends.getVariant(image.target).imageLocations() @@ -158,18 +339,6 @@ func (image *bootImageVariant) imageLocations() (imageLocations []string) { return append(imageLocations, dexpreopt.PathToLocation(image.images, image.target.Arch.ArchType)) } -func concat(lists ...[]string) []string { - var size int - for _, l := range lists { - size += len(l) - } - ret := make([]string, 0, size) - for _, l := range lists { - ret = append(ret, l...) - } - return ret -} - func dexpreoptBootJarsFactory() android.Singleton { return &dexpreoptBootJars{} } @@ -182,10 +351,21 @@ func skipDexpreoptBootJars(ctx android.PathContext) bool { return dexpreopt.GetGlobalConfig(ctx).DisablePreopt } +// Singleton for generating boot image build rules. type dexpreoptBootJars struct { + // Default boot image config (currently always the Framework boot image extension). It should be + // noted that JIT-Zygote builds use ART APEX image instead of the Framework boot image extension, + // but the switch is handled not here, but in the makefiles (triggered with + // DEXPREOPT_USE_ART_IMAGE=true). defaultBootImage *bootImageConfig - otherImages []*bootImageConfig + // Other boot image configs (currently the list contains only the primary ART APEX image. It + // used to contain an experimental JIT-Zygote image (now replaced with the ART APEX image). In + // the future other boot image extensions may be added. + otherImages []*bootImageConfig + + // Build path to a config file that Soong writes for Make (to be used in makefiles that install + // the default boot image). dexpreoptConfigForMake android.WritablePath } @@ -205,7 +385,7 @@ func DexpreoptedArtApexJars(ctx android.BuilderContext) map[android.ArchType]and return files } -// dexpreoptBoot singleton rules +// Generate build rules for boot images. func (d *dexpreoptBootJars) GenerateBuildActions(ctx android.SingletonContext) { if skipDexpreoptBootJars(ctx) { return @@ -334,9 +514,10 @@ func buildBootImage(ctx android.SingletonContext, image *bootImageConfig) *bootI } } - // The path to bootclasspath dex files needs to be known at module GenerateAndroidBuildAction time, before - // the bootclasspath modules have been compiled. Copy the dex jars there so the module rules that have - // already been set up can find them. + // The paths to bootclasspath DEX files need to be known at module GenerateAndroidBuildAction + // time, before the boot images are built (these paths are used in dexpreopt rule generation for + // Java libraries and apps). Generate rules that copy bootclasspath DEX jars to the predefined + // paths. for i := range bootDexJars { ctx.Build(pctx, android.BuildParams{ Rule: android.Cp, @@ -371,6 +552,7 @@ func buildBootImage(ctx android.SingletonContext, image *bootImageConfig) *bootI return image } +// Generate boot image build rules for a specific target. func buildBootImageVariant(ctx android.SingletonContext, image *bootImageVariant, profile android.Path, missingDeps []string) android.WritablePaths { @@ -428,12 +610,15 @@ func buildBootImageVariant(ctx android.SingletonContext, image *bootImageVariant } if image.extends != nil { + // It is a boot image extension, so it needs the boot image it depends on (in this case the + // primary ART APEX image). artImage := image.primaryImages cmd. Flag("--runtime-arg").FlagWithInputList("-Xbootclasspath:", image.dexPathsDeps.Paths(), ":"). Flag("--runtime-arg").FlagWithList("-Xbootclasspath-locations:", image.dexLocationsDeps, ":"). FlagWithArg("--boot-image=", dexpreopt.PathToLocation(artImage, arch)).Implicit(artImage) } else { + // It is a primary image, so it needs a base address. cmd.FlagWithArg("--base=", ctx.Config().LibartImgDeviceBaseAddress()) } @@ -717,7 +902,9 @@ func writeGlobalConfigForMake(ctx android.SingletonContext, path android.Writabl android.WriteFileRule(ctx, path, string(data)) } -// Export paths for default boot image to Make +// Define Make variables for boot image names, paths, etc. These variables are used in makefiles +// (make/core/dex_preopt_libart.mk) to generate install rules that copy boot image files to the +// correct output directories. func (d *dexpreoptBootJars) MakeVars(ctx android.MakeVarsContext) { if d.dexpreoptConfigForMake != nil { ctx.Strict("DEX_PREOPT_CONFIG_FOR_MAKE", d.dexpreoptConfigForMake.String()) @@ -731,6 +918,11 @@ func (d *dexpreoptBootJars) MakeVars(ctx android.MakeVarsContext) { ctx.Strict("DEXPREOPT_BOOTCLASSPATH_DEX_LOCATIONS", strings.Join(image.getAnyAndroidVariant().dexLocationsDeps, " ")) var imageNames []string + // TODO: the primary ART boot image should not be exposed to Make, as it is installed in a + // different way as a part of the ART APEX. However, there is a special JIT-Zygote build + // configuration which uses the primary ART image instead of the Framework boot image + // extension, and it relies on the ART image being exposed to Make. To fix this, it is + // necessary to rework the logic in makefiles. for _, current := range append(d.otherImages, image) { imageNames = append(imageNames, current.name) for _, variant := range current.variants {