From 770ab053b097f6943e14ac7d3330dbf22cc83199 Mon Sep 17 00:00:00 2001 From: Yifan Hong Date: Fri, 2 Nov 2018 13:43:30 -0700 Subject: [PATCH] Add kernel info to compatibility.zip Add kernel configs / version to verified_assembled_vendor_manifest.xml so that the kernel of the incoming package can be checked against the framework. Previously, the running kernel was used instead. Bug: 111125947 Test: test_extract_kernel Test: manual OTA on Pixel 3 from build: Android P (kernel version 4.9.96) to ToT build: device kernel version = (manually modified) framework requirement = latest, PRODUCT_OTA_ENFORCE_VINTF_KERNEL_REQUIREMENTS = true Change-Id: Id524a58e94bdb6bba348ca461c9d33614ce451a9 --- core/Makefile | 61 +++++++++++ core/config.mk | 2 + tools/extract_kernel.py | 196 +++++++++++++++++++++++++++++++++++ tools/test_extract_kernel.py | 30 ++++++ 4 files changed, 289 insertions(+) create mode 100755 tools/extract_kernel.py create mode 100644 tools/test_extract_kernel.py diff --git a/core/Makefile b/core/Makefile index 0bbaf12177..776c1cba85 100644 --- a/core/Makefile +++ b/core/Makefile @@ -2568,10 +2568,71 @@ $(BUILT_ASSEMBLED_VENDOR_MANIFEST): $(HOST_OUT_EXECUTABLES)/assemble_vintf $(BUILT_ASSEMBLED_VENDOR_MANIFEST): $(BUILT_SYSTEM_MATRIX) $(BUILT_ASSEMBLED_VENDOR_MANIFEST): $(BUILT_VENDOR_MANIFEST) $(BUILT_ASSEMBLED_VENDOR_MANIFEST): $(INTERNAL_VENDORIMAGE_FILES) + +$(BUILT_ASSEMBLED_VENDOR_MANIFEST): PRIVATE_FLAGS := + +# -- Kernel version and configurations. +ifeq ($(PRODUCT_OTA_ENFORCE_VINTF_KERNEL_REQUIREMENTS),true) + +# BOARD_KERNEL_CONFIG_FILE and BOARD_KERNEL_VERSION can be used to override the values extracted +# from INSTALLED_KERNEL_TARGET. +ifdef BOARD_KERNEL_CONFIG_FILE +ifdef BOARD_KERNEL_VERSION +$(BUILT_ASSEMBLED_VENDOR_MANIFEST): $(BOARD_KERNEL_CONFIG_FILE) +$(BUILT_ASSEMBLED_VENDOR_MANIFEST): PRIVATE_FLAGS += --kernel $(BOARD_KERNEL_VERSION):$(BOARD_KERNEL_CONFIG_FILE) +my_board_extracted_kernel := true +endif # BOARD_KERNEL_VERSION +endif # BOARD_KERNEL_CONFIG_FILE + +ifneq ($(my_board_extracted_kernel),true) +ifndef INSTALLED_KERNEL_TARGET +$(warning No INSTALLED_KERNEL_TARGET is defined when PRODUCT_OTA_ENFORCE_VINTF_KERNEL_REQUIREMENTS \ + is true. Information about the updated kernel cannot be built into OTA update package. \ + You can fix this by: (1) setting TARGET_NO_KERNEL to false and installing the built kernel \ + to $(PRODUCT_OUT)/kernel, so that kernel information will be extracted from the built kernel; \ + or (2) extracting kernel configuration and defining BOARD_KERNEL_CONFIG_FILE and \ + BOARD_KERNEL_VERSION manually; or (3) unsetting PRODUCT_OTA_ENFORCE_VINTF_KERNEL_REQUIREMENTS \ + manually.) +else +intermediates := $(call intermediates-dir-for,ETC,$(notdir $(BUILT_ASSEMBLED_VENDOR_MANIFEST))) + +# Tools for decompression that is not in PATH. +# Check $(EXTRACT_KERNEL) for decompression algorithms supported by the script. +# Algorithms that are in the script but not in this list will be found in PATH. +my_decompress_tools := \ + lz4:$(HOST_OUT_EXECUTABLES)/lz4 \ + +my_kernel_configs := $(intermediates)/kernel_configs.txt +my_kernel_version := $(intermediates)/kernel_version.txt +$(my_kernel_configs): .KATI_IMPLICIT_OUTPUTS := $(my_kernel_version) +$(my_kernel_configs): PRIVATE_KERNEL_VERSION_FILE := $(my_kernel_version) +$(my_kernel_configs): PRIVATE_DECOMPRESS_TOOLS := $(my_decompress_tools) +$(my_kernel_configs): $(foreach pair,$(my_decompress_tools),$(call word-colon,2,$(pair))) +$(my_kernel_configs): $(EXTRACT_KERNEL) $(INSTALLED_KERNEL_TARGET) + $< --tools $(PRIVATE_DECOMPRESS_TOOLS) --input $(INSTALLED_KERNEL_TARGET) \ + --output-configs $@ \ + --output-version $(PRIVATE_KERNEL_VERSION_FILE) + +$(BUILT_ASSEMBLED_VENDOR_MANIFEST): $(my_kernel_configs) $(my_kernel_version) +$(BUILT_ASSEMBLED_VENDOR_MANIFEST): PRIVATE_FLAGS += --kernel $$(cat $(my_kernel_version)):$(my_kernel_configs) + +intermediates := +my_kernel_configs := +my_kernel_version := +my_decompress_tools := + +endif # my_board_extracted_kernel +my_board_extracted_kernel := + +endif # INSTALLED_KERNEL_TARGET +endif # PRODUCT_OTA_ENFORCE_VINTF_KERNEL_REQUIREMENTS + +$(BUILT_ASSEMBLED_VENDOR_MANIFEST): @echo "Verifying vendor VINTF manifest." PRODUCT_ENFORCE_VINTF_MANIFEST=$(PRODUCT_ENFORCE_VINTF_MANIFEST) \ $(PRIVATE_SYSTEM_ASSEMBLE_VINTF_ENV_VARS) \ $(HOST_OUT_EXECUTABLES)/assemble_vintf \ + $(PRIVATE_FLAGS) \ -c $(BUILT_SYSTEM_MATRIX) \ -i $(BUILT_VENDOR_MANIFEST) \ $$([ -d $(TARGET_OUT_VENDOR)/etc/vintf/manifest ] && \ diff --git a/core/config.mk b/core/config.mk index ab0ec6d6d0..fe7b84c592 100644 --- a/core/config.mk +++ b/core/config.mk @@ -730,6 +730,8 @@ FINDBUGS := $(FINDBUGS_DIR)/findbugs JETIFIER := prebuilts/sdk/tools/jetifier/jetifier-standalone/bin/jetifier-standalone +EXTRACT_KERNEL := build/make/tools/extract_kernel.py + COLUMN:= column USE_OPENJDK9 := true diff --git a/tools/extract_kernel.py b/tools/extract_kernel.py new file mode 100755 index 0000000000..16ccb22d47 --- /dev/null +++ b/tools/extract_kernel.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A tool to extract kernel information from a kernel image. +""" + +import argparse +import subprocess +import sys +import re + +CONFIG_PREFIX = b'IKCFG_ST' +GZIP_HEADER = b'\037\213\010' +COMPRESSION_ALGO = ( + (["gzip", "-d"], GZIP_HEADER), + (["xz", "-d"], b'\3757zXZ\000'), + (["bzip2", "-d"], b'BZh'), + (["lz4", "-d", "-l"], b'\002\041\114\030'), + + # These are not supported in the build system yet. + # (["unlzma"], b'\135\0\0\0'), + # (["lzop", "-d"], b'\211\114\132'), +) + +# "Linux version " UTS_RELEASE " (" LINUX_COMPILE_BY "@" +# LINUX_COMPILE_HOST ") (" LINUX_COMPILER ") " UTS_VERSION "\n"; +LINUX_BANNER_PREFIX = b'Linux version ' +LINUX_BANNER_REGEX = LINUX_BANNER_PREFIX + \ + r'([0-9]+[.][0-9]+[.][0-9]+).* \(.*@.*\) \(.*\) .*\n' + + +def get_version(input_bytes, start_idx): + null_idx = input_bytes.find('\x00', start_idx) + if null_idx < 0: + return None + linux_banner = input_bytes[start_idx:null_idx].decode() + mo = re.match(LINUX_BANNER_REGEX, linux_banner) + if mo: + return mo.group(1) + return None + + +def dump_version(input_bytes): + idx = 0 + while True: + idx = input_bytes.find(LINUX_BANNER_PREFIX, idx) + if idx < 0: + return None + + version = get_version(input_bytes, idx) + if version: + return version + + idx += len(LINUX_BANNER_PREFIX) + + +def dump_configs(input_bytes): + """ + Dump kernel configuration from input_bytes. This can be done when + CONFIG_IKCONFIG is enabled, which is a requirement on Treble devices. + + The kernel configuration is archived in GZip format right after the magic + string 'IKCFG_ST' in the built kernel. + """ + + # Search for magic string + GZip header + idx = input_bytes.find(CONFIG_PREFIX + GZIP_HEADER) + if idx < 0: + return None + + # Seek to the start of the archive + idx += len(CONFIG_PREFIX) + + sp = subprocess.Popen(["gzip", "-d", "-c"], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + o, _ = sp.communicate(input=input_bytes[idx:]) + if sp.returncode == 1: # error + return None + + # success or trailing garbage warning + assert sp.returncode in (0, 2), sp.returncode + + return o + + +def try_decompress(cmd, search_bytes, input_bytes): + idx = input_bytes.find(search_bytes) + if idx < 0: + return None + + idx = 0 + sp = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + o, _ = sp.communicate(input=input_bytes[idx:]) + # ignore errors + return o + + +def decompress_dump(func, input_bytes): + """ + Run func(input_bytes) first; and if that fails (returns value evaluates to + False), then try different decompression algorithm before running func. + """ + o = func(input_bytes) + if o: + return o + for cmd, search_bytes in COMPRESSION_ALGO: + decompressed = try_decompress(cmd, search_bytes, input_bytes) + if decompressed: + o = func(decompressed) + if o: + return o + # Force decompress the whole file even if header doesn't match + decompressed = try_decompress(cmd, b"", input_bytes) + if decompressed: + o = func(decompressed) + if o: + return o + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter, + description=__doc__ + + "\nThese algorithms are tried when decompressing the image:\n " + + " ".join(tup[0][0] for tup in COMPRESSION_ALGO)) + parser.add_argument('--input', + help='Input kernel image. If not specified, use stdin', + metavar='FILE', + type=argparse.FileType('rb'), + default=sys.stdin) + parser.add_argument('--output-configs', + help='If specified, write configs. Use stdout if no file ' + 'is specified.', + metavar='FILE', + nargs='?', + type=argparse.FileType('wb'), + const=sys.stdout) + parser.add_argument('--output-version', + help='If specified, write version. Use stdout if no file ' + 'is specified.', + metavar='FILE', + nargs='?', + type=argparse.FileType('wb'), + const=sys.stdout) + parser.add_argument('--tools', + help='Decompression tools to use. If not specified, PATH ' + 'is searched.', + metavar='ALGORITHM:EXECUTABLE', + nargs='*') + args = parser.parse_args() + + tools = {pair[0]: pair[1] + for pair in (token.split(':') for token in args.tools or [])} + for cmd, _ in COMPRESSION_ALGO: + if cmd[0] in tools: + cmd[0] = tools[cmd[0]] + + input_bytes = args.input.read() + + ret = 0 + if args.output_configs is not None: + o = decompress_dump(dump_configs, input_bytes) + if o: + args.output_configs.write(o) + else: + sys.stderr.write( + "Cannot extract kernel configs in {}".format(args.input.name)) + ret = 1 + if args.output_version is not None: + o = decompress_dump(dump_version, input_bytes) + if o: + args.output_version.write(o) + else: + sys.stderr.write( + "Cannot extract kernel versions in {}".format(args.input.name)) + ret = 1 + + return ret + + +if __name__ == '__main__': + exit(main()) diff --git a/tools/test_extract_kernel.py b/tools/test_extract_kernel.py new file mode 100644 index 0000000000..1a1cfcbdad --- /dev/null +++ b/tools/test_extract_kernel.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from extract_kernel import get_version, dump_version + +class ExtractKernelTest(unittest.TestCase): + def test_extract_version(self): + self.assertEqual("4.9.100", get_version( + b'Linux version 4.9.100-a123 (a@a) (a) a\n\x00', 0)) + self.assertEqual("4.9.123", get_version( + b'Linux version 4.9.123 (@) () \n\x00', 0)) + + def test_dump_self(self): + self.assertEqual("4.9.1", dump_version( + b"trash\x00Linux version 4.8.8\x00trash\x00" + "other trash Linux version 4.9.1-g3 (2@s) (2) a\n\x00"))