From 780f595fac7e4979848a5a461c8332badb3a2cda Mon Sep 17 00:00:00 2001 From: Mark Salyzyn Date: Fri, 19 Oct 2018 13:44:36 -0700 Subject: [PATCH] build_image: improvements to right size for dynamic partitions If partition_reserved_size is 0 or undefined, and use_dynamic_partition_size is true, we should approach no space and no free inodes automatically. Estimate the space and number of inodes required, then do a first pass build to see how much space actually used, and use those values to refine the estimate. Depends on tune2fs to report the characteristics of the filesystem, so only support for ext filesystems. In the future if there has to be a more generic ability, either a tool per a filesystem has to be found, or we will need root capabilities to mount the filesystem to acquire the characteristics live from the host system. Test: manual + python -m unittest test_build_image Bug: 111302946 Change-Id: I933a388be43516b6de7b5007b296765bd5556fde --- core/Makefile | 3 +- tools/releasetools/build_image.py | 106 ++++++++++++++++++++++--- tools/releasetools/test_build_image.py | 24 +++++- 3 files changed, 119 insertions(+), 14 deletions(-) diff --git a/core/Makefile b/core/Makefile index c21c517fdd..9dbf6e412a 100644 --- a/core/Makefile +++ b/core/Makefile @@ -1206,7 +1206,7 @@ endif ifeq ($(INTERNAL_USERIMAGES_USE_EXT),true) INTERNAL_USERIMAGES_DEPS := $(SIMG2IMG) -INTERNAL_USERIMAGES_DEPS += $(MKEXTUSERIMG) $(MAKE_EXT4FS) $(E2FSCK) +INTERNAL_USERIMAGES_DEPS += $(MKEXTUSERIMG) $(MAKE_EXT4FS) $(E2FSCK) $(TUNE2FS) ifeq ($(TARGET_USERIMAGES_USE_F2FS),true) INTERNAL_USERIMAGES_DEPS += $(MKF2FSUSERIMG) $(MAKE_F2FS) endif @@ -3099,6 +3099,7 @@ OTATOOLS := $(HOST_OUT_EXECUTABLES)/minigzip \ $(HOST_OUT_EXECUTABLES)/mke2fs \ $(HOST_OUT_EXECUTABLES)/mkuserimg_mke2fs \ $(HOST_OUT_EXECUTABLES)/e2fsdroid \ + $(HOST_OUT_EXECUTABLES)/tune2fs \ $(HOST_OUT_EXECUTABLES)/mksquashfsimage.sh \ $(HOST_OUT_EXECUTABLES)/mksquashfs \ $(HOST_OUT_EXECUTABLES)/mkf2fsuserimg.sh \ diff --git a/tools/releasetools/build_image.py b/tools/releasetools/build_image.py index 4a013c2c3c..d7318427bf 100755 --- a/tools/releasetools/build_image.py +++ b/tools/releasetools/build_image.py @@ -54,23 +54,69 @@ def GetDiskUsage(path): """Returns the number of bytes that "path" occupies on host. Args: - path: The directory or file to calculate size on + path: The directory or file to calculate size on. Returns: - The number of bytes. + The number of bytes based on a 1K block_size. Raises: BuildImageError: On error. """ - env_copy = os.environ.copy() - env_copy["POSIXLY_CORRECT"] = "1" - cmd = ["du", "-s", path] + cmd = ["du", "-k", "-s", path] try: - output = common.RunAndCheckOutput(cmd, verbose=False, env=env_copy) + output = common.RunAndCheckOutput(cmd, verbose=False) except common.ExternalError: raise BuildImageError("Failed to get disk usage:\n{}".format(output)) - # POSIX du returns number of blocks with block size 512 - return int(output.split()[0]) * 512 + return int(output.split()[0]) * 1024 + + +def GetInodeUsage(path): + """Returns the number of inodes that "path" occupies on host. + + Args: + path: The directory or file to calculate inode number on. + + Returns: + The number of inodes used. + + Raises: + BuildImageError: On error. + """ + cmd = ["find", path, "-print"] + try: + output = common.RunAndCheckOutput(cmd, verbose=False) + except common.ExternalError: + raise BuildImageError("Failed to get disk inode usage:\n{}".format(output)) + # increase by > 4% as number of files and directories is not whole picture. + return output.count('\n') * 25 // 24 + + +def GetFilesystemCharacteristics(sparse_image_path): + """Returns various filesystem characteristics of "sparse_image_path". + + Args: + sparse_image_path: The file to analyze. + + Returns: + The characteristics dictionary. + + Raises: + BuildImageError: On error. + """ + unsparse_image_path = UnsparseImage(sparse_image_path, replace=False) + + cmd = ["tune2fs", "-l", unsparse_image_path] + try: + output = common.RunAndCheckOutput(cmd, verbose=False) + except common.ExternalError: + raise BuildImageError("Failed to get tune2fs usage:\n{}".format(output)) + os.remove(unsparse_image_path) + fs_dict = { } + for line in output.splitlines(): + fields = line.split(":") + if len(fields) == 2: + fs_dict[fields[0].strip()] = fields[1].strip() + return fs_dict def UnsparseImage(sparse_image_path, replace=True): @@ -121,6 +167,10 @@ def SetUpInDirAndFsConfig(origin_in, prop_dict): if prop_dict["mount_point"] != "system": return origin_in, fs_config + if "first_pass" in prop_dict: + prop_dict["mount_point"] = "/" + return prop_dict["first_pass"] + # Construct a staging directory of the root file system. in_dir = common.MakeTempDir() root_dir = prop_dict.get("root_dir") @@ -144,6 +194,7 @@ def SetUpInDirAndFsConfig(origin_in, prop_dict): with open(fs_config) as fr: fw.writelines(fr.readlines()) fs_config = merged_fs_config + prop_dict["first_pass"] = (in_dir, fs_config) return in_dir, fs_config @@ -175,7 +226,7 @@ def CheckHeadroom(ext4fs_output, prop_dict): m = ext4fs_stats.match(last_line) used_blocks = int(m.groupdict().get('used_blocks')) total_blocks = int(m.groupdict().get('total_blocks')) - headroom_blocks = int(prop_dict['partition_headroom']) / BLOCK_SIZE + headroom_blocks = int(prop_dict['partition_headroom']) // BLOCK_SIZE adjusted_blocks = total_blocks - headroom_blocks if used_blocks > adjusted_blocks: mount_point = prop_dict["mount_point"] @@ -202,6 +253,7 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None): Raises: BuildImageError: On build image failures. """ + original_mount_point = prop_dict["mount_point"] in_dir, fs_config = SetUpInDirAndFsConfig(in_dir, prop_dict) build_command = [] @@ -233,7 +285,8 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None): size = GetDiskUsage(in_dir) logger.info( "The tree size of %s is %d MB.", in_dir, size // BYTES_IN_MB) - size += int(prop_dict.get("partition_reserved_size", 0)) + # If not specified, give us 16MB margin for GetDiskUsage error ... + size += int(prop_dict.get("partition_reserved_size", BYTES_IN_MB * 16)) # Round this up to a multiple of 4K so that avbtool works size = common.RoundUpTo4K(size) # Adjust partition_size to add more space for AVB footer, to prevent @@ -244,6 +297,35 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None): lambda x: verity_utils.AVBCalcMaxImageSize( avbtool, avb_footer_type, x, avb_signing_args)) prop_dict["partition_size"] = str(size) + if fs_type.startswith("ext"): + if "extfs_inode_count" not in prop_dict: + prop_dict["extfs_inode_count"] = str(GetInodeUsage(in_dir)) + logger.info( + "First Pass based on estimates of %d MB and %s inodes.", + size // BYTES_IN_MB, prop_dict["extfs_inode_count"]) + prop_dict["mount_point"] = original_mount_point + BuildImage(in_dir, prop_dict, out_file, target_out) + fs_dict = GetFilesystemCharacteristics(out_file) + block_size = int(fs_dict.get("Block size", "4096")) + free_size = int(fs_dict.get("Free blocks", "0")) * block_size + reserved_size = int(prop_dict.get("partition_reserved_size", 0)) + if free_size <= reserved_size: + logger.info( + "Not worth reducing image %d <= %d.", free_size, reserved_size) + else: + size -= free_size + size += reserved_size + if block_size <= 4096: + size = common.RoundUpTo4K(size) + else: + size = ((size + block_size - 1) // block_size) * block_size + extfs_inode_count = prop_dict["extfs_inode_count"] + inodes = int(fs_dict.get("Inode count", extfs_inode_count)) + inodes -= int(fs_dict.get("Free inodes", "0")) + prop_dict["extfs_inode_count"] = str(inodes) + prop_dict["partition_size"] = str(size) + logger.info( + "Allocating %d Inodes for %s.", inodes, out_file) logger.info( "Allocating %d MB for %s.", size // BYTES_IN_MB, out_file) @@ -363,7 +445,7 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None): int(prop_dict.get("partition_reserved_size", 0)), int(prop_dict.get("partition_reserved_size", 0)) // BYTES_IN_MB)) print( - "The max image size for filsystem files is {} bytes ({} MB), out of a " + "The max image size for filesystem files is {} bytes ({} MB), out of a " "total partition size of {} bytes ({} MB).".format( int(prop_dict["image_size"]), int(prop_dict["image_size"]) // BYTES_IN_MB, @@ -677,7 +759,7 @@ def main(argv): glob_dict = LoadGlobalDict(glob_dict_file) if "mount_point" in glob_dict: - # The caller knows the mount point and provides a dictionay needed by + # The caller knows the mount point and provides a dictionary needed by # BuildImage(). image_properties = glob_dict else: diff --git a/tools/releasetools/test_build_image.py b/tools/releasetools/test_build_image.py index 634c6b1913..1cebd0c095 100644 --- a/tools/releasetools/test_build_image.py +++ b/tools/releasetools/test_build_image.py @@ -19,7 +19,7 @@ import os.path import common from build_image import ( - BuildImageError, CheckHeadroom, SetUpInDirAndFsConfig) + BuildImageError, CheckHeadroom, GetFilesystemCharacteristics, SetUpInDirAndFsConfig) from test_utils import ReleaseToolsTestCase @@ -176,3 +176,25 @@ class BuildImageTest(ReleaseToolsTestCase): self.assertIn('fs-config-system\n', fs_config_data) self.assertIn('fs-config-root\n', fs_config_data) self.assertEqual('/', prop_dict['mount_point']) + + def test_GetFilesystemCharacteristics(self): + input_dir = common.MakeTempDir() + output_image = common.MakeTempFile(suffix='.img') + command = ['mkuserimg_mke2fs', input_dir, output_image, 'ext4', + '/system', '409600', '-j', '0'] + proc = common.Run(command) + ext4fs_output, _ = proc.communicate() + self.assertEqual(0, proc.returncode) + + output_file = common.MakeTempFile(suffix='.img') + cmd = ["img2simg", output_image, output_file] + p = common.Run(cmd) + p.communicate() + self.assertEqual(0, p.returncode) + + fs_dict = GetFilesystemCharacteristics(output_file) + self.assertEqual(int(fs_dict['Block size']), 4096) + self.assertGreaterEqual(int(fs_dict['Free blocks']), 0) # expect ~88 + self.assertGreater(int(fs_dict['Inode count']), 0) # expect ~64 + self.assertGreaterEqual(int(fs_dict['Free inodes']), 0) # expect ~53 + self.assertGreater(int(fs_dict['Inode count']), int(fs_dict['Free inodes']))