diff --git a/android/notices.go b/android/notices.go index 194a734d3..2a4c17cd8 100644 --- a/android/notices.go +++ b/android/notices.go @@ -15,93 +15,9 @@ package android import ( - "path/filepath" "strings" - - "github.com/google/blueprint" ) -func init() { - pctx.SourcePathVariable("merge_notices", "build/soong/scripts/mergenotice.py") - pctx.SourcePathVariable("generate_notice", "build/soong/scripts/generate-notice-files.py") - - pctx.HostBinToolVariable("minigzip", "minigzip") -} - -type NoticeOutputs struct { - Merged OptionalPath - TxtOutput OptionalPath - HtmlOutput OptionalPath - HtmlGzOutput OptionalPath -} - -var ( - mergeNoticesRule = pctx.AndroidStaticRule("mergeNoticesRule", blueprint.RuleParams{ - Command: `${merge_notices} --output $out $in`, - CommandDeps: []string{"${merge_notices}"}, - Description: "merge notice files into $out", - }) - - generateNoticeRule = pctx.AndroidStaticRule("generateNoticeRule", blueprint.RuleParams{ - Command: `rm -rf $$(dirname $txtOut) $$(dirname $htmlOut) $$(dirname $out) && ` + - `mkdir -p $$(dirname $txtOut) $$(dirname $htmlOut) $$(dirname $out) && ` + - `${generate_notice} --text-output $txtOut --html-output $htmlOut -t "$title" -s $inputDir && ` + - `${minigzip} -c $htmlOut > $out`, - CommandDeps: []string{"${generate_notice}", "${minigzip}"}, - Description: "produce notice file $out", - }, "txtOut", "htmlOut", "title", "inputDir") -) - -func MergeNotices(ctx ModuleContext, mergedNotice WritablePath, noticePaths []Path) { - ctx.Build(pctx, BuildParams{ - Rule: mergeNoticesRule, - Description: "merge notices", - Inputs: noticePaths, - Output: mergedNotice, - }) -} - -func BuildNoticeOutput(ctx ModuleContext, installPath InstallPath, installFilename string, - noticePaths []Path) NoticeOutputs { - // Merge all NOTICE files into one. - // TODO(jungjw): We should just produce a well-formatted NOTICE.html file in a single pass. - // - // generate-notice-files.py, which processes the merged NOTICE file, has somewhat strict rules - // about input NOTICE file paths. - // 1. Their relative paths to the src root become their NOTICE index titles. We want to use - // on-device paths as titles, and so output the merged NOTICE file the corresponding location. - // 2. They must end with .txt extension. Otherwise, they're ignored. - noticeRelPath := InstallPathToOnDevicePath(ctx, installPath.Join(ctx, installFilename+".txt")) - mergedNotice := PathForModuleOut(ctx, filepath.Join("NOTICE_FILES/src", noticeRelPath)) - MergeNotices(ctx, mergedNotice, noticePaths) - - // Transform the merged NOTICE file into a gzipped HTML file. - txtOuptut := PathForModuleOut(ctx, "NOTICE_txt", "NOTICE.txt") - htmlOutput := PathForModuleOut(ctx, "NOTICE_html", "NOTICE.html") - htmlGzOutput := PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz") - title := "Notices for " + ctx.ModuleName() - ctx.Build(pctx, BuildParams{ - Rule: generateNoticeRule, - Description: "generate notice output", - Input: mergedNotice, - Output: htmlGzOutput, - ImplicitOutputs: WritablePaths{txtOuptut, htmlOutput}, - Args: map[string]string{ - "txtOut": txtOuptut.String(), - "htmlOut": htmlOutput.String(), - "title": title, - "inputDir": PathForModuleOut(ctx, "NOTICE_FILES/src").String(), - }, - }) - - return NoticeOutputs{ - Merged: OptionalPathForPath(mergedNotice), - TxtOutput: OptionalPathForPath(txtOuptut), - HtmlOutput: OptionalPathForPath(htmlOutput), - HtmlGzOutput: OptionalPathForPath(htmlGzOutput), - } -} - // BuildNoticeTextOutputFromLicenseMetadata writes out a notice text file based on the module's // generated license metadata file. func BuildNoticeTextOutputFromLicenseMetadata(ctx ModuleContext, outputFile WritablePath) { @@ -112,5 +28,18 @@ func BuildNoticeTextOutputFromLicenseMetadata(ctx ModuleContext, outputFile Writ FlagWithOutput("-o ", outputFile). FlagWithDepFile("-d ", depsFile). Input(ctx.Module().base().licenseMetadataFile) - rule.Build("container_notice", "container notice file") + rule.Build("text_notice", "container notice file") +} + +// BuildNoticeHtmlOutputFromLicenseMetadata writes out a notice text file based on the module's +// generated license metadata file. +func BuildNoticeHtmlOutputFromLicenseMetadata(ctx ModuleContext, outputFile WritablePath) { + depsFile := outputFile.ReplaceExtension(ctx, strings.TrimPrefix(outputFile.Ext()+".d", ".")) + rule := NewRuleBuilder(pctx, ctx) + rule.Command(). + BuiltTool("htmlnotice"). + FlagWithOutput("-o ", outputFile). + FlagWithDepFile("-d ", depsFile). + Input(ctx.Module().base().licenseMetadataFile) + rule.Build("html_notice", "container notice file") } diff --git a/apex/androidmk.go b/apex/androidmk.go index 059b4d76c..e094a1276 100644 --- a/apex/androidmk.go +++ b/apex/androidmk.go @@ -396,10 +396,6 @@ func (a *apexBundle) androidMkForType() android.AndroidMkData { } a.writeRequiredModules(w, moduleNames) - if a.mergedNotices.Merged.Valid() { - fmt.Fprintln(w, "LOCAL_NOTICE_FILE :=", a.mergedNotices.Merged.Path().String()) - } - fmt.Fprintln(w, "include $(BUILD_PREBUILT)") if apexType == imageApex { diff --git a/apex/apex.go b/apex/apex.go index 6d8a67a5c..cb88f02f5 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -414,8 +414,8 @@ type apexBundle struct { // Processed file_contexts files fileContexts android.WritablePath - // Struct holding the merged notice file paths in different formats - mergedNotices android.NoticeOutputs + // Path to notice file in html.gz format. + htmlGzNotice android.WritablePath // The built APEX file. This is the main product. // Could be .apex or .capex @@ -487,11 +487,10 @@ const ( // for each of the files in case when the APEX is flattened. type apexFile struct { // buildFile is put in the installDir inside the APEX. - builtFile android.Path - noticeFiles android.Paths - installDir string - customStem string - symlinks []string // additional symlinks + builtFile android.Path + installDir string + customStem string + symlinks []string // additional symlinks // Info for Android.mk Module name of `module` in AndroidMk. Note the generated AndroidMk // module for apexFile is named something like .[ 0 { - for _, path := range paths { - noticePathSet[path] = true - } - } - return true - }) - - // If the app has one, add it too. - if len(a.NoticeFiles()) > 0 { - for _, path := range a.NoticeFiles() { - noticePathSet[path] = true - } - } - - if len(noticePathSet) == 0 { - return - } - var noticePaths []android.Path - for path := range noticePathSet { - noticePaths = append(noticePaths, path) - } - sort.Slice(noticePaths, func(i, j int) bool { - return noticePaths[i].String() < noticePaths[j].String() - }) - - a.noticeOutputs = android.BuildNoticeOutput(ctx, a.installDir, a.installApkName+".apk", noticePaths) -} - // Reads and prepends a main cert from the default cert dir if it hasn't been set already, i.e. it // isn't a cert module reference. Also checks and enforces system cert restriction if applicable. func processMainCert(m android.ModuleBase, certPropValue string, certificates []Certificate, ctx android.ModuleContext) []Certificate { @@ -636,9 +586,10 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { } a.onDeviceDir = android.InstallPathToOnDevicePath(ctx, a.installDir) - a.noticeBuildActions(ctx) + noticeFile := android.PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz") + android.BuildNoticeHtmlOutputFromLicenseMetadata(ctx, noticeFile) if Bool(a.appProperties.Embed_notices) || ctx.Config().IsEnvTrue("ALWAYS_EMBED_NOTICES") { - a.aapt.noticeFile = a.noticeOutputs.HtmlGzOutput + a.aapt.noticeFile = android.OptionalPathForPath(noticeFile) } a.classLoaderContexts = a.usesLibrary.classLoaderContextForUsesLibDeps(ctx) diff --git a/java/app_test.go b/java/app_test.go index 16bbec158..08baf5434 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -27,7 +27,6 @@ import ( "android/soong/android" "android/soong/cc" "android/soong/dexpreopt" - "android/soong/genrule" ) // testApp runs tests using the prepareForJavaTest @@ -2722,116 +2721,6 @@ func TestCodelessApp(t *testing.T) { } } -func TestEmbedNotice(t *testing.T) { - result := android.GroupFixturePreparers( - PrepareForTestWithJavaDefaultModules, - cc.PrepareForTestWithCcDefaultModules, - genrule.PrepareForTestWithGenRuleBuildComponents, - android.MockFS{ - "APP_NOTICE": nil, - "GENRULE_NOTICE": nil, - "LIB_NOTICE": nil, - "TOOL_NOTICE": nil, - }.AddToFixture(), - ).RunTestWithBp(t, ` - android_app { - name: "foo", - srcs: ["a.java"], - static_libs: ["javalib"], - jni_libs: ["libjni"], - notice: "APP_NOTICE", - embed_notices: true, - sdk_version: "current", - } - - // No embed_notice flag - android_app { - name: "bar", - srcs: ["a.java"], - jni_libs: ["libjni"], - notice: "APP_NOTICE", - sdk_version: "current", - } - - // No NOTICE files - android_app { - name: "baz", - srcs: ["a.java"], - embed_notices: true, - sdk_version: "current", - } - - cc_library { - name: "libjni", - system_shared_libs: [], - stl: "none", - notice: "LIB_NOTICE", - sdk_version: "current", - } - - java_library { - name: "javalib", - srcs: [ - ":gen", - ], - sdk_version: "current", - } - - genrule { - name: "gen", - tools: ["gentool"], - out: ["gen.java"], - notice: "GENRULE_NOTICE", - } - - java_binary_host { - name: "gentool", - srcs: ["b.java"], - notice: "TOOL_NOTICE", - } - `) - - // foo has NOTICE files to process, and embed_notices is true. - foo := result.ModuleForTests("foo", "android_common") - // verify merge notices rule. - mergeNotices := foo.Rule("mergeNoticesRule") - noticeInputs := mergeNotices.Inputs.Strings() - // TOOL_NOTICE should be excluded as it's a host module. - if len(mergeNotices.Inputs) != 3 { - t.Errorf("number of input notice files: expected = 3, actual = %q", noticeInputs) - } - if !inList("APP_NOTICE", noticeInputs) { - t.Errorf("APP_NOTICE is missing from notice files, %q", noticeInputs) - } - if !inList("LIB_NOTICE", noticeInputs) { - t.Errorf("LIB_NOTICE is missing from notice files, %q", noticeInputs) - } - if !inList("GENRULE_NOTICE", noticeInputs) { - t.Errorf("GENRULE_NOTICE is missing from notice files, %q", noticeInputs) - } - // aapt2 flags should include -A so that its contents are put in the APK's /assets. - res := foo.Output("package-res.apk") - aapt2Flags := res.Args["flags"] - e := "-A out/soong/.intermediates/foo/android_common/NOTICE" - android.AssertStringDoesContain(t, "expected.apkPath", aapt2Flags, e) - - // bar has NOTICE files to process, but embed_notices is not set. - bar := result.ModuleForTests("bar", "android_common") - res = bar.Output("package-res.apk") - aapt2Flags = res.Args["flags"] - e = "-A out/soong/.intermediates/bar/android_common/NOTICE" - android.AssertStringDoesNotContain(t, "bar shouldn't have the asset dir flag for NOTICE", aapt2Flags, e) - - // baz's embed_notice is true, but it doesn't have any NOTICE files. - baz := result.ModuleForTests("baz", "android_common") - res = baz.Output("package-res.apk") - aapt2Flags = res.Args["flags"] - e = "-A out/soong/.intermediates/baz/android_common/NOTICE" - if strings.Contains(aapt2Flags, e) { - t.Errorf("baz shouldn't have the asset dir flag for NOTICE: %q", e) - } -} - func TestUncompressDex(t *testing.T) { testCases := []struct { name string diff --git a/scripts/generate-notice-files.py b/scripts/generate-notice-files.py deleted file mode 100755 index 1b4acfaaf..000000000 --- a/scripts/generate-notice-files.py +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2012 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -Usage: generate-notice-files --text-output [plain text output file] \ - --html-output [html output file] \ - --xml-output [xml output file] \ - -t [file title] -s [directory of notices] - -Generate the Android notice files, including both text and html files. - --h to display this usage message and exit. -""" -from collections import defaultdict -import argparse -import hashlib -import itertools -import os -import os.path -import re -import struct -import sys - -MD5_BLOCKSIZE = 1024 * 1024 -HTML_ESCAPE_TABLE = { - b"&": b"&", - b'"': b""", - b"'": b"'", - b">": b">", - b"<": b"<", - } - -def md5sum(filename): - """Calculate an MD5 of the file given by FILENAME, - and return hex digest as a string. - Output should be compatible with md5sum command""" - - f = open(filename, "rb") - sum = hashlib.md5() - while 1: - block = f.read(MD5_BLOCKSIZE) - if not block: - break - sum.update(block) - f.close() - return sum.hexdigest() - - -def html_escape(text): - """Produce entities within text.""" - # Using for i in text doesn't work since i will be an int, not a byte. - # There are multiple ways to solve this, but the most performant way - # to iterate over a byte array is to use unpack. Using the - # for i in range(len(text)) and using that to get a byte using array - # slices is twice as slow as this method. - return b"".join(HTML_ESCAPE_TABLE.get(i,i) for i in struct.unpack(str(len(text)) + 'c', text)) - -HTML_OUTPUT_CSS=b""" - - -""" - -def combine_notice_files_html(file_hash, input_dir, output_filename): - """Combine notice files in FILE_HASH and output a HTML version to OUTPUT_FILENAME.""" - - SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt") - - # Set up a filename to row id table (anchors inside tables don't work in - # most browsers, but href's to table row ids do) - id_table = {} - id_count = 0 - for value in file_hash: - for filename in value: - id_table[filename] = id_count - id_count += 1 - - # Open the output file, and output the header pieces - output_file = open(output_filename, "wb") - - output_file.write(b"\n") - output_file.write(HTML_OUTPUT_CSS) - output_file.write(b'\n') - - # Output our table of contents - output_file.write(b'
\n') - output_file.write(b"
    \n") - - # Flatten the list of lists into a single list of filenames - sorted_filenames = sorted(itertools.chain.from_iterable(file_hash)) - - # Print out a nice table of contents - for filename in sorted_filenames: - stripped_filename = SRC_DIR_STRIP_RE.sub(r"\1", filename) - output_file.write(('
  • %s
    • \n' % (id_table.get(filename), stripped_filename)).encode()) - - output_file.write(b"
\n") - output_file.write(b"
\n") - # Output the individual notice file lists - output_file.write(b'\n') - for value in file_hash: - output_file.write(('\n\n\n\n") - - # Finish off the file output - output_file.write(b"
\n' % id_table.get(value[0])).encode()) - output_file.write(b'
Notices for file(s):
\n') - output_file.write(b'
\n') - for filename in value: - output_file.write(("%s
\n" % (SRC_DIR_STRIP_RE.sub(r"\1", filename))).encode()) - output_file.write(b"
\n\n") - output_file.write(b'
\n')
-        with open(value[0], "rb") as notice_file:
-            output_file.write(html_escape(notice_file.read()))
-        output_file.write(b"\n
\n") - output_file.write(b"
\n") - output_file.write(b"\n") - output_file.close() - -def combine_notice_files_text(file_hash, input_dir, output_filename, file_title): - """Combine notice files in FILE_HASH and output a text version to OUTPUT_FILENAME.""" - - SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt") - output_file = open(output_filename, "wb") - output_file.write(file_title.encode()) - output_file.write(b"\n") - for value in file_hash: - output_file.write(b"============================================================\n") - output_file.write(b"Notices for file(s):\n") - for filename in value: - output_file.write(SRC_DIR_STRIP_RE.sub(r"\1", filename).encode()) - output_file.write(b"\n") - output_file.write(b"------------------------------------------------------------\n") - with open(value[0], "rb") as notice_file: - output_file.write(notice_file.read()) - output_file.write(b"\n") - output_file.close() - -def combine_notice_files_xml(files_with_same_hash, input_dir, output_filename): - """Combine notice files in FILE_HASH and output a XML version to OUTPUT_FILENAME.""" - - SRC_DIR_STRIP_RE = re.compile(input_dir + "(/.*).txt") - - # Set up a filename to row id table (anchors inside tables don't work in - # most browsers, but href's to table row ids do) - id_table = {} - for file_key, files in files_with_same_hash.items(): - for filename in files: - id_table[filename] = file_key - - # Open the output file, and output the header pieces - output_file = open(output_filename, "wb") - - output_file.write(b'\n') - output_file.write(b"\n") - - # Flatten the list of lists into a single list of filenames - sorted_filenames = sorted(list(id_table)) - - # Print out a nice table of contents - for filename in sorted_filenames: - stripped_filename = SRC_DIR_STRIP_RE.sub(r"\1", filename) - output_file.write(('%s\n' % (id_table.get(filename), stripped_filename)).encode()) - output_file.write(b"\n\n") - - processed_file_keys = [] - # Output the individual notice file lists - for filename in sorted_filenames: - file_key = id_table.get(filename) - if file_key in processed_file_keys: - continue - processed_file_keys.append(file_key) - - output_file.write(('\n\n") - - # Finish off the file output - output_file.write(b"\n") - output_file.close() - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - '--text-output', required=True, - help='The text output file path.') - parser.add_argument( - '--html-output', - help='The html output file path.') - parser.add_argument( - '--xml-output', - help='The xml output file path.') - parser.add_argument( - '-t', '--title', required=True, - help='The file title.') - parser.add_argument( - '-s', '--source-dir', required=True, - help='The directory containing notices.') - parser.add_argument( - '-i', '--included-subdirs', action='append', - help='The sub directories which should be included.') - parser.add_argument( - '-e', '--excluded-subdirs', action='append', - help='The sub directories which should be excluded.') - return parser.parse_args() - -def main(argv): - args = get_args() - - txt_output_file = args.text_output - html_output_file = args.html_output - xml_output_file = args.xml_output - file_title = args.title - included_subdirs = [] - excluded_subdirs = [] - if args.included_subdirs is not None: - included_subdirs = args.included_subdirs - if args.excluded_subdirs is not None: - excluded_subdirs = args.excluded_subdirs - - # Find all the notice files and md5 them - input_dir = os.path.normpath(args.source_dir) - files_with_same_hash = defaultdict(list) - for root, dir, files in os.walk(input_dir): - for file in files: - matched = True - if len(included_subdirs) > 0: - matched = False - for subdir in included_subdirs: - if (root == (input_dir + '/' + subdir) or - root.startswith(input_dir + '/' + subdir + '/')): - matched = True - break - elif len(excluded_subdirs) > 0: - for subdir in excluded_subdirs: - if (root == (input_dir + '/' + subdir) or - root.startswith(input_dir + '/' + subdir + '/')): - matched = False - break - if matched and file.endswith(".txt"): - filename = os.path.join(root, file) - file_md5sum = md5sum(filename) - files_with_same_hash[file_md5sum].append(filename) - - filesets = [sorted(files_with_same_hash[md5]) for md5 in sorted(list(files_with_same_hash))] - - combine_notice_files_text(filesets, input_dir, txt_output_file, file_title) - - if html_output_file is not None: - combine_notice_files_html(filesets, input_dir, html_output_file) - - if xml_output_file is not None: - combine_notice_files_xml(files_with_same_hash, input_dir, xml_output_file) - -if __name__ == "__main__": - main(sys.argv) diff --git a/scripts/mergenotice.py b/scripts/mergenotice.py deleted file mode 100755 index fe990735b..000000000 --- a/scripts/mergenotice.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2019 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -""" -Merges input notice files to the output file while ignoring duplicated files -This script shouldn't be confused with build/soong/scripts/generate-notice-files.py -which is responsible for creating the final notice file for all artifacts -installed. This script has rather limited scope; it is meant to create a merged -notice file for a set of modules that are packaged together, e.g. in an APEX. -The merged notice file does not reveal the individual files in the package. -""" - -import sys -import argparse - -def get_args(): - parser = argparse.ArgumentParser(description='Merge notice files.') - parser.add_argument('--output', help='output file path.') - parser.add_argument('inputs', metavar='INPUT', nargs='+', - help='input notice file') - return parser.parse_args() - -def main(argv): - args = get_args() - - processed = set() - with open(args.output, 'w+') as output: - for input in args.inputs: - with open(input, 'r') as f: - data = f.read().strip() - if data not in processed: - processed.add(data) - output.write('%s\n\n' % data) - -if __name__ == '__main__': - main(sys.argv)