diff --git a/core/Makefile b/core/Makefile index c21c517fdd..9dbf6e412a 100644 --- a/core/Makefile +++ b/core/Makefile @@ -1206,7 +1206,7 @@ endif ifeq ($(INTERNAL_USERIMAGES_USE_EXT),true) INTERNAL_USERIMAGES_DEPS := $(SIMG2IMG) -INTERNAL_USERIMAGES_DEPS += $(MKEXTUSERIMG) $(MAKE_EXT4FS) $(E2FSCK) +INTERNAL_USERIMAGES_DEPS += $(MKEXTUSERIMG) $(MAKE_EXT4FS) $(E2FSCK) $(TUNE2FS) ifeq ($(TARGET_USERIMAGES_USE_F2FS),true) INTERNAL_USERIMAGES_DEPS += $(MKF2FSUSERIMG) $(MAKE_F2FS) endif @@ -3099,6 +3099,7 @@ OTATOOLS := $(HOST_OUT_EXECUTABLES)/minigzip \ $(HOST_OUT_EXECUTABLES)/mke2fs \ $(HOST_OUT_EXECUTABLES)/mkuserimg_mke2fs \ $(HOST_OUT_EXECUTABLES)/e2fsdroid \ + $(HOST_OUT_EXECUTABLES)/tune2fs \ $(HOST_OUT_EXECUTABLES)/mksquashfsimage.sh \ $(HOST_OUT_EXECUTABLES)/mksquashfs \ $(HOST_OUT_EXECUTABLES)/mkf2fsuserimg.sh \ diff --git a/tools/releasetools/build_image.py b/tools/releasetools/build_image.py index 4a013c2c3c..d7318427bf 100755 --- a/tools/releasetools/build_image.py +++ b/tools/releasetools/build_image.py @@ -54,23 +54,69 @@ def GetDiskUsage(path): """Returns the number of bytes that "path" occupies on host. Args: - path: The directory or file to calculate size on + path: The directory or file to calculate size on. Returns: - The number of bytes. + The number of bytes based on a 1K block_size. Raises: BuildImageError: On error. """ - env_copy = os.environ.copy() - env_copy["POSIXLY_CORRECT"] = "1" - cmd = ["du", "-s", path] + cmd = ["du", "-k", "-s", path] try: - output = common.RunAndCheckOutput(cmd, verbose=False, env=env_copy) + output = common.RunAndCheckOutput(cmd, verbose=False) except common.ExternalError: raise BuildImageError("Failed to get disk usage:\n{}".format(output)) - # POSIX du returns number of blocks with block size 512 - return int(output.split()[0]) * 512 + return int(output.split()[0]) * 1024 + + +def GetInodeUsage(path): + """Returns the number of inodes that "path" occupies on host. + + Args: + path: The directory or file to calculate inode number on. + + Returns: + The number of inodes used. + + Raises: + BuildImageError: On error. + """ + cmd = ["find", path, "-print"] + try: + output = common.RunAndCheckOutput(cmd, verbose=False) + except common.ExternalError: + raise BuildImageError("Failed to get disk inode usage:\n{}".format(output)) + # increase by > 4% as number of files and directories is not whole picture. + return output.count('\n') * 25 // 24 + + +def GetFilesystemCharacteristics(sparse_image_path): + """Returns various filesystem characteristics of "sparse_image_path". + + Args: + sparse_image_path: The file to analyze. + + Returns: + The characteristics dictionary. + + Raises: + BuildImageError: On error. + """ + unsparse_image_path = UnsparseImage(sparse_image_path, replace=False) + + cmd = ["tune2fs", "-l", unsparse_image_path] + try: + output = common.RunAndCheckOutput(cmd, verbose=False) + except common.ExternalError: + raise BuildImageError("Failed to get tune2fs usage:\n{}".format(output)) + os.remove(unsparse_image_path) + fs_dict = { } + for line in output.splitlines(): + fields = line.split(":") + if len(fields) == 2: + fs_dict[fields[0].strip()] = fields[1].strip() + return fs_dict def UnsparseImage(sparse_image_path, replace=True): @@ -121,6 +167,10 @@ def SetUpInDirAndFsConfig(origin_in, prop_dict): if prop_dict["mount_point"] != "system": return origin_in, fs_config + if "first_pass" in prop_dict: + prop_dict["mount_point"] = "/" + return prop_dict["first_pass"] + # Construct a staging directory of the root file system. in_dir = common.MakeTempDir() root_dir = prop_dict.get("root_dir") @@ -144,6 +194,7 @@ def SetUpInDirAndFsConfig(origin_in, prop_dict): with open(fs_config) as fr: fw.writelines(fr.readlines()) fs_config = merged_fs_config + prop_dict["first_pass"] = (in_dir, fs_config) return in_dir, fs_config @@ -175,7 +226,7 @@ def CheckHeadroom(ext4fs_output, prop_dict): m = ext4fs_stats.match(last_line) used_blocks = int(m.groupdict().get('used_blocks')) total_blocks = int(m.groupdict().get('total_blocks')) - headroom_blocks = int(prop_dict['partition_headroom']) / BLOCK_SIZE + headroom_blocks = int(prop_dict['partition_headroom']) // BLOCK_SIZE adjusted_blocks = total_blocks - headroom_blocks if used_blocks > adjusted_blocks: mount_point = prop_dict["mount_point"] @@ -202,6 +253,7 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None): Raises: BuildImageError: On build image failures. """ + original_mount_point = prop_dict["mount_point"] in_dir, fs_config = SetUpInDirAndFsConfig(in_dir, prop_dict) build_command = [] @@ -233,7 +285,8 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None): size = GetDiskUsage(in_dir) logger.info( "The tree size of %s is %d MB.", in_dir, size // BYTES_IN_MB) - size += int(prop_dict.get("partition_reserved_size", 0)) + # If not specified, give us 16MB margin for GetDiskUsage error ... + size += int(prop_dict.get("partition_reserved_size", BYTES_IN_MB * 16)) # Round this up to a multiple of 4K so that avbtool works size = common.RoundUpTo4K(size) # Adjust partition_size to add more space for AVB footer, to prevent @@ -244,6 +297,35 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None): lambda x: verity_utils.AVBCalcMaxImageSize( avbtool, avb_footer_type, x, avb_signing_args)) prop_dict["partition_size"] = str(size) + if fs_type.startswith("ext"): + if "extfs_inode_count" not in prop_dict: + prop_dict["extfs_inode_count"] = str(GetInodeUsage(in_dir)) + logger.info( + "First Pass based on estimates of %d MB and %s inodes.", + size // BYTES_IN_MB, prop_dict["extfs_inode_count"]) + prop_dict["mount_point"] = original_mount_point + BuildImage(in_dir, prop_dict, out_file, target_out) + fs_dict = GetFilesystemCharacteristics(out_file) + block_size = int(fs_dict.get("Block size", "4096")) + free_size = int(fs_dict.get("Free blocks", "0")) * block_size + reserved_size = int(prop_dict.get("partition_reserved_size", 0)) + if free_size <= reserved_size: + logger.info( + "Not worth reducing image %d <= %d.", free_size, reserved_size) + else: + size -= free_size + size += reserved_size + if block_size <= 4096: + size = common.RoundUpTo4K(size) + else: + size = ((size + block_size - 1) // block_size) * block_size + extfs_inode_count = prop_dict["extfs_inode_count"] + inodes = int(fs_dict.get("Inode count", extfs_inode_count)) + inodes -= int(fs_dict.get("Free inodes", "0")) + prop_dict["extfs_inode_count"] = str(inodes) + prop_dict["partition_size"] = str(size) + logger.info( + "Allocating %d Inodes for %s.", inodes, out_file) logger.info( "Allocating %d MB for %s.", size // BYTES_IN_MB, out_file) @@ -363,7 +445,7 @@ def BuildImage(in_dir, prop_dict, out_file, target_out=None): int(prop_dict.get("partition_reserved_size", 0)), int(prop_dict.get("partition_reserved_size", 0)) // BYTES_IN_MB)) print( - "The max image size for filsystem files is {} bytes ({} MB), out of a " + "The max image size for filesystem files is {} bytes ({} MB), out of a " "total partition size of {} bytes ({} MB).".format( int(prop_dict["image_size"]), int(prop_dict["image_size"]) // BYTES_IN_MB, @@ -677,7 +759,7 @@ def main(argv): glob_dict = LoadGlobalDict(glob_dict_file) if "mount_point" in glob_dict: - # The caller knows the mount point and provides a dictionay needed by + # The caller knows the mount point and provides a dictionary needed by # BuildImage(). image_properties = glob_dict else: diff --git a/tools/releasetools/test_build_image.py b/tools/releasetools/test_build_image.py index 634c6b1913..1cebd0c095 100644 --- a/tools/releasetools/test_build_image.py +++ b/tools/releasetools/test_build_image.py @@ -19,7 +19,7 @@ import os.path import common from build_image import ( - BuildImageError, CheckHeadroom, SetUpInDirAndFsConfig) + BuildImageError, CheckHeadroom, GetFilesystemCharacteristics, SetUpInDirAndFsConfig) from test_utils import ReleaseToolsTestCase @@ -176,3 +176,25 @@ class BuildImageTest(ReleaseToolsTestCase): self.assertIn('fs-config-system\n', fs_config_data) self.assertIn('fs-config-root\n', fs_config_data) self.assertEqual('/', prop_dict['mount_point']) + + def test_GetFilesystemCharacteristics(self): + input_dir = common.MakeTempDir() + output_image = common.MakeTempFile(suffix='.img') + command = ['mkuserimg_mke2fs', input_dir, output_image, 'ext4', + '/system', '409600', '-j', '0'] + proc = common.Run(command) + ext4fs_output, _ = proc.communicate() + self.assertEqual(0, proc.returncode) + + output_file = common.MakeTempFile(suffix='.img') + cmd = ["img2simg", output_image, output_file] + p = common.Run(cmd) + p.communicate() + self.assertEqual(0, p.returncode) + + fs_dict = GetFilesystemCharacteristics(output_file) + self.assertEqual(int(fs_dict['Block size']), 4096) + self.assertGreaterEqual(int(fs_dict['Free blocks']), 0) # expect ~88 + self.assertGreater(int(fs_dict['Inode count']), 0) # expect ~64 + self.assertGreaterEqual(int(fs_dict['Free inodes']), 0) # expect ~53 + self.assertGreater(int(fs_dict['Inode count']), int(fs_dict['Free inodes']))