Merge "Move boot jars package check from make"

This commit is contained in:
Paul Duffin
2020-10-30 10:06:46 +00:00
committed by Gerrit Code Review
9 changed files with 505 additions and 4 deletions

View File

@@ -67,6 +67,15 @@ func (i ApexInfo) IsForPlatform() bool {
return i.ApexVariationName == ""
}
func (i ApexInfo) InApex(apex string) bool {
for _, a := range i.InApexes {
if a == apex {
return true
}
}
return false
}
// ApexTestForInfo stores the contents of APEXes for which this module is a test and thus has
// access to APEX internals.
type ApexTestForInfo struct {

View File

@@ -793,6 +793,11 @@ func (c *config) AlwaysUsePrebuiltSdks() bool {
return Bool(c.productVariables.Always_use_prebuilt_sdks)
}
// Returns true if the boot jars check should be skipped.
func (c *config) SkipBootJarsCheck() bool {
return Bool(c.productVariables.Skip_boot_jars_check)
}
func (c *config) Fuchsia() bool {
return Bool(c.productVariables.Fuchsia)
}
@@ -1341,6 +1346,11 @@ func (l *ConfiguredJarList) Jar(idx int) string {
return l.jars[idx]
}
// Apex component of idx-th pair on the list.
func (l *ConfiguredJarList) Apex(idx int) string {
return l.apexes[idx]
}
// If the list contains a pair with the given jar.
func (l *ConfiguredJarList) ContainsJar(jar string) bool {
return InList(jar, l.jars)
@@ -1538,3 +1548,11 @@ func (c *config) BootJars() []string {
return list
}).([]string)
}
func (c *config) NonUpdatableBootJars() ConfiguredJarList {
return c.productVariables.BootJars
}
func (c *config) UpdatableBootJars() ConfiguredJarList {
return c.productVariables.UpdatableBootJars
}

View File

@@ -224,6 +224,7 @@ type productVariables struct {
Unbundled_build *bool `json:",omitempty"`
Unbundled_build_apps *bool `json:",omitempty"`
Always_use_prebuilt_sdks *bool `json:",omitempty"`
Skip_boot_jars_check *bool `json:",omitempty"`
Malloc_not_svelte *bool `json:",omitempty"`
Malloc_zero_contents *bool `json:",omitempty"`
Malloc_pattern_fill_contents *bool `json:",omitempty"`

View File

@@ -22,6 +22,7 @@ bootstrap_go_package {
"androidmk.go",
"app_builder.go",
"app.go",
"boot_jars.go",
"builder.go",
"device_host_converter.go",
"dex.go",

View File

@@ -217,10 +217,6 @@ func (prebuilt *DexImport) AndroidMkEntries() []android.AndroidMkEntries {
func(entries *android.AndroidMkEntries) {
if prebuilt.dexJarFile != nil {
entries.SetPath("LOCAL_SOONG_DEX_JAR", prebuilt.dexJarFile)
// TODO(b/125517186): export the dex jar as a classes jar to match some mis-uses in Make until
// boot_jars_package_check.mk can check dex jars.
entries.SetPath("LOCAL_SOONG_HEADER_JAR", prebuilt.dexJarFile)
entries.SetPath("LOCAL_SOONG_CLASSES_JAR", prebuilt.dexJarFile)
}
if len(prebuilt.dexpreopter.builtInstalled) > 0 {
entries.SetString("LOCAL_SOONG_BUILT_INSTALLED", prebuilt.dexpreopter.builtInstalled)

123
java/boot_jars.go Normal file
View File

@@ -0,0 +1,123 @@
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package java
import (
"android/soong/android"
)
func init() {
android.RegisterSingletonType("boot_jars", bootJarsSingletonFactory)
}
func bootJarsSingletonFactory() android.Singleton {
return &bootJarsSingleton{}
}
type bootJarsSingleton struct{}
func populateMapFromConfiguredJarList(ctx android.SingletonContext, moduleToApex map[string]string, list android.ConfiguredJarList, name string) bool {
for i := 0; i < list.Len(); i++ {
module := list.Jar(i)
// Ignore jacocoagent it is only added when instrumenting and so has no impact on
// app compatibility.
if module == "jacocoagent" {
continue
}
apex := list.Apex(i)
if existing, ok := moduleToApex[module]; ok {
ctx.Errorf("Configuration property %q is invalid as it contains multiple references to module (%s) in APEXes (%s and %s)",
module, existing, apex)
return false
}
moduleToApex[module] = apex
}
return true
}
func (b *bootJarsSingleton) GenerateBuildActions(ctx android.SingletonContext) {
config := ctx.Config()
if config.SkipBootJarsCheck() {
return
}
// Populate a map from module name to APEX from the boot jars. If there is a problem
// such as duplicate modules then fail and return immediately.
moduleToApex := make(map[string]string)
if !populateMapFromConfiguredJarList(ctx, moduleToApex, config.NonUpdatableBootJars(), "BootJars") ||
!populateMapFromConfiguredJarList(ctx, moduleToApex, config.UpdatableBootJars(), "UpdatableBootJars") {
return
}
// Map from module name to the correct apex variant.
nameToApexVariant := make(map[string]android.Module)
// Scan all the modules looking for the module/apex variants corresponding to the
// boot jars.
ctx.VisitAllModules(func(module android.Module) {
name := ctx.ModuleName(module)
if apex, ok := moduleToApex[name]; ok {
apexInfo := ctx.ModuleProvider(module, android.ApexInfoProvider).(android.ApexInfo)
if (apex == "platform" && apexInfo.IsForPlatform()) || apexInfo.InApex(apex) {
// The module name/apex variant should be unique in the system but double check
// just in case something has gone wrong.
if existing, ok := nameToApexVariant[name]; ok {
ctx.Errorf("found multiple variants matching %s:%s: %q and %q", apex, name, existing, module)
}
nameToApexVariant[name] = module
}
}
})
timestamp := android.PathForOutput(ctx, "boot-jars-package-check/stamp")
rule := android.NewRuleBuilder()
checkBootJars := rule.Command().BuiltTool(ctx, "check_boot_jars").
Input(android.PathForSource(ctx, "build/soong/scripts/check_boot_jars/package_allowed_list.txt"))
// If this is not an unbundled build and missing dependencies are not allowed
// then all the boot jars listed must have been found.
strict := !config.UnbundledBuild() && !config.AllowMissingDependencies()
// Iterate over the module names on the boot classpath in order
for _, name := range android.SortedStringKeys(moduleToApex) {
if apexVariant, ok := nameToApexVariant[name]; ok {
if dep, ok := apexVariant.(Dependency); ok {
// Add the implementation jars for the module to be checked. This uses implementation
// and resources jar as that is what the previous make based check uses.
for _, jar := range dep.ImplementationAndResourcesJars() {
checkBootJars.Input(jar)
}
} else if _, ok := apexVariant.(*DexImport); ok {
// TODO(b/171479578): ignore deximport when doing package check until boot_jars.go can check dex jars.
} else {
ctx.Errorf("module %q is of type %q which is not supported as a boot jar", name, ctx.ModuleType(apexVariant))
}
} else if strict {
ctx.Errorf("boot jars package check failed as it could not find module %q for apex %q", name, moduleToApex[name])
}
}
checkBootJars.Text("&& touch").Output(timestamp)
rule.Build(pctx, ctx, "boot_jars_package_check", "check boot jar packages")
// The check-boot-jars phony target depends on the timestamp created if the check succeeds.
ctx.Phony("check-boot-jars", timestamp)
// The droidcore phony target depends on the check-boot-jars phony target
ctx.Phony("droidcore", android.PathForPhony(ctx, "check-boot-jars"))
}

View File

@@ -1,3 +1,19 @@
python_binary_host {
name: "check_boot_jars",
main: "check_boot_jars/check_boot_jars.py",
srcs: [
"check_boot_jars/check_boot_jars.py",
],
version: {
py2: {
enabled: true,
},
py3: {
enabled: false,
},
},
}
python_binary_host {
name: "manifest_fixer",
main: "manifest_fixer.py",

View File

@@ -0,0 +1,89 @@
#!/usr/bin/env python
"""
Check boot jars.
Usage: check_boot_jars.py <package_allow_list_file> <jar1> <jar2> ...
"""
import logging
import os.path
import re
import subprocess
import sys
# The compiled allow list RE.
allow_list_re = None
def LoadAllowList(filename):
""" Load and compile allow list regular expressions from filename.
"""
lines = []
with open(filename, 'r') as f:
for line in f:
line = line.strip()
if not line or line.startswith('#'):
continue
lines.append(line)
combined_re = r'^(%s)$' % '|'.join(lines)
global allow_list_re
try:
allow_list_re = re.compile(combined_re)
except re.error:
logging.exception(
'Cannot compile package allow list regular expression: %r',
combined_re)
allow_list_re = None
return False
return True
def CheckJar(allow_list_path, jar):
"""Check a jar file.
"""
# Get the list of files inside the jar file.
p = subprocess.Popen(args='jar tf %s' % jar,
stdout=subprocess.PIPE, shell=True)
stdout, _ = p.communicate()
if p.returncode != 0:
return False
items = stdout.split()
classes = 0
for f in items:
if f.endswith('.class'):
classes += 1
package_name = os.path.dirname(f)
package_name = package_name.replace('/', '.')
if not package_name or not allow_list_re.match(package_name):
print >> sys.stderr, ('Error: %s contains class file %s, whose package name %s is empty or'
' not in the allow list %s of packages allowed on the bootclasspath.'
% (jar, f, package_name, allow_list_path))
return False
if classes == 0:
print >> sys.stderr, ('Error: %s does not contain any class files.' % jar)
return False
return True
def main(argv):
if len(argv) < 2:
print __doc__
return 1
allow_list_path = argv[0]
if not LoadAllowList(allow_list_path):
return 1
passed = True
for jar in argv[1:]:
if not CheckJar(allow_list_path, jar):
passed = False
if not passed:
return 1
return 0
if __name__ == '__main__':
sys.exit(main(sys.argv[1:]))

View File

@@ -0,0 +1,248 @@
# Boot jar package name allowed list.
# Each line is interpreted as a regular expression.
###################################################
# core-libart.jar & core-oj.jar
java\.awt\.font
java\.beans
java\.io
java\.lang
java\.lang\.annotation
java\.lang\.invoke
java\.lang\.ref
java\.lang\.reflect
java\.math
java\.net
java\.nio
java\.nio\.file
java\.nio\.file\.spi
java\.nio\.file\.attribute
java\.nio\.channels
java\.nio\.channels\.spi
java\.nio\.charset
java\.nio\.charset\.spi
java\.security
java\.security\.acl
java\.security\.cert
java\.security\.interfaces
java\.security\.spec
java\.sql
java\.text
java\.text\.spi
java\.time
java\.time\.chrono
java\.time\.format
java\.time\.temporal
java\.time\.zone
java\.util
java\.util\.concurrent
java\.util\.concurrent\.atomic
java\.util\.concurrent\.locks
java\.util\.function
java\.util\.jar
java\.util\.logging
java\.util\.prefs
java\.util\.regex
java\.util\.spi
java\.util\.stream
java\.util\.zip
# TODO: Remove javax.annotation.processing if possible, see http://b/132338110:
javax\.annotation\.processing
javax\.crypto
javax\.crypto\.interfaces
javax\.crypto\.spec
javax\.net
javax\.net\.ssl
javax\.security\.auth
javax\.security\.auth\.callback
javax\.security\.auth\.login
javax\.security\.auth\.x500
javax\.security\.cert
javax\.sql
javax\.xml
javax\.xml\.datatype
javax\.xml\.namespace
javax\.xml\.parsers
javax\.xml\.transform
javax\.xml\.transform\.dom
javax\.xml\.transform\.sax
javax\.xml\.transform\.stream
javax\.xml\.validation
javax\.xml\.xpath
jdk\.internal\.util
jdk\.internal\.vm\.annotation
jdk\.net
org\.w3c\.dom
org\.w3c\.dom\.ls
org\.w3c\.dom\.traversal
# OpenJdk internal implementation.
sun\.invoke\.util
sun\.invoke\.empty
sun\.misc
sun\.util.*
sun\.text.*
sun\.security.*
sun\.reflect.*
sun\.nio.*
sun\.net.*
com\.sun\..*
# TODO: Move these internal org.apache.harmony classes to libcore.*
org\.apache\.harmony\.crypto\.internal
org\.apache\.harmony\.dalvik
org\.apache\.harmony\.dalvik\.ddmc
org\.apache\.harmony\.luni\.internal\.util
org\.apache\.harmony\.security
org\.apache\.harmony\.security\.asn1
org\.apache\.harmony\.security\.fortress
org\.apache\.harmony\.security\.pkcs10
org\.apache\.harmony\.security\.pkcs7
org\.apache\.harmony\.security\.pkcs8
org\.apache\.harmony\.security\.provider\.crypto
org\.apache\.harmony\.security\.utils
org\.apache\.harmony\.security\.x501
org\.apache\.harmony\.security\.x509
org\.apache\.harmony\.security\.x509\.tsp
org\.apache\.harmony\.xml
org\.apache\.harmony\.xml\.dom
org\.apache\.harmony\.xml\.parsers
org\.json
org\.xmlpull\.v1
org\.xmlpull\.v1\.sax2
# TODO: jarjar org.kxml2.io to com.android org\.kxml2\.io
org\.kxml2\.io
org\.xml
org\.xml\.sax
org\.xml\.sax\.ext
org\.xml\.sax\.helpers
dalvik\..*
libcore\..*
android\..*
com\.android\..*
###################################################
# android.test.base.jar
junit\.extensions
junit\.framework
android\.test
android\.test\.suitebuilder\.annotation
###################################################
# ext.jar
# TODO: jarjar javax.sip to com.android
javax\.sip
javax\.sip\.address
javax\.sip\.header
javax\.sip\.message
# TODO: jarjar org.apache.commons to com.android
org\.apache\.commons\.codec
org\.apache\.commons\.codec\.binary
org\.apache\.commons\.codec\.language
org\.apache\.commons\.codec\.net
org\.apache\.commons\.logging
org\.apache\.commons\.logging\.impl
org\.apache\.http
org\.apache\.http\.auth
org\.apache\.http\.auth\.params
org\.apache\.http\.client
org\.apache\.http\.client\.entity
org\.apache\.http\.client\.methods
org\.apache\.http\.client\.params
org\.apache\.http\.client\.protocol
org\.apache\.http\.client\.utils
org\.apache\.http\.conn
org\.apache\.http\.conn\.params
org\.apache\.http\.conn\.routing
org\.apache\.http\.conn\.scheme
org\.apache\.http\.conn\.ssl
org\.apache\.http\.conn\.util
org\.apache\.http\.cookie
org\.apache\.http\.cookie\.params
org\.apache\.http\.entity
org\.apache\.http\.impl
org\.apache\.http\.impl\.auth
org\.apache\.http\.impl\.client
org\.apache\.http\.impl\.client
org\.apache\.http\.impl\.conn
org\.apache\.http\.impl\.conn\.tsccm
org\.apache\.http\.impl\.cookie
org\.apache\.http\.impl\.entity
org\.apache\.http\.impl\.io
org\.apache\.http\.impl\.io
org\.apache\.http\.io
org\.apache\.http\.message
org\.apache\.http\.params
org\.apache\.http\.protocol
org\.apache\.http\.util
# TODO: jarjar gov.nist to com.android
gov\.nist\.core
gov\.nist\.core\.net
gov\.nist\.javax\.sip
gov\.nist\.javax\.sip\.address
gov\.nist\.javax\.sip\.clientauthutils
gov\.nist\.javax\.sip\.header
gov\.nist\.javax\.sip\.header\.extensions
gov\.nist\.javax\.sip\.header\.ims
gov\.nist\.javax\.sip\.message
gov\.nist\.javax\.sip\.parser
gov\.nist\.javax\.sip\.parser\.extensions
gov\.nist\.javax\.sip\.parser\.ims
gov\.nist\.javax\.sip\.stack
org\.ccil\.cowan\.tagsoup
org\.ccil\.cowan\.tagsoup\.jaxp
###################################################
# framework.jar
javax\.microedition\.khronos\.opengles
javax\.microedition\.khronos\.egl
android
###################################################
# apache-xml.jar
org\.apache\.xml\.res
org\.apache\.xml\.utils
org\.apache\.xml\.utils\.res
org\.apache\.xml\.dtm
org\.apache\.xml\.dtm\.ref
org\.apache\.xml\.dtm\.ref\.dom2dtm
org\.apache\.xml\.dtm\.ref\.sax2dtm
org\.apache\.xml\.serializer
org\.apache\.xml\.serializer\.utils
org\.apache\.xml\.serializer\.dom3
org\.apache\.xpath
org\.apache\.xpath\.operations
org\.apache\.xpath\.domapi
org\.apache\.xpath\.functions
org\.apache\.xpath\.res
org\.apache\.xpath\.axes
org\.apache\.xpath\.objects
org\.apache\.xpath\.patterns
org\.apache\.xpath\.jaxp
org\.apache\.xpath\.compiler
org\.apache\.xalan
org\.apache\.xalan\.res
org\.apache\.xalan\.templates
org\.apache\.xalan\.serialize
org\.apache\.xalan\.extensions
org\.apache\.xalan\.processor
org\.apache\.xalan\.transformer
org\.apache\.xalan\.xslt
###################################################
# Packages in the google namespace across all bootclasspath jars.
com\.google\.android\..*
com\.google\.vr\.platform.*
com\.google\.i18n\.phonenumbers\..*
com\.google\.i18n\.phonenumbers
###################################################
# Packages used for Android in Chrome OS
org\.chromium\.arc
org\.chromium\.arc\..*