Merge "fs_config: introduce TARGET_FS_CONFIG_GEN"

am: 54cb05c

* commit '54cb05c01a464d0efb2cfc43c3379eb15d6a2bf0':
  fs_config: introduce TARGET_FS_CONFIG_GEN

Change-Id: I16bf8c0f489a4cbc7d55c2383f63cdefd2b18719

tools/fs_config/Android.mk

@@ -23,20 +23,31 @@ LOCAL_CFLAGS := -Werror
 
 include $(BUILD_HOST_EXECUTABLE)
 
-# To Build the custom target binary for the host to generate the fs_config
+# One can override the default android_filesystem_config.h file in one of two ways:
-# override files. The executable is hard coded to include the
+#
-# $(TARGET_ANDROID_FILESYSTEM_CONFIG_H) file if it exists.
+# 1. The old way:
-# Expectations:
+#   To Build the custom target binary for the host to generate the fs_config
-#    device/<vendor>/<device>/android_filesystem_config.h
+#   override files. The executable is hard coded to include the
-#        fills in struct fs_path_config android_device_dirs[] and
+#   $(TARGET_ANDROID_FILESYSTEM_CONFIG_H) file if it exists.
-#                 struct fs_path_config android_device_files[]
+#   Expectations:
-#    device/<vendor>/<device>/device.mk
+#      device/<vendor>/<device>/android_filesystem_config.h
-#        PRODUCT_PACKAGES += fs_config_dirs fs_config_files
+#          fills in struct fs_path_config android_device_dirs[] and
-
+#                   struct fs_path_config android_device_files[]
-# If not specified, check if default one to be found
+#      device/<vendor>/<device>/device.mk
+#          PRODUCT_PACKAGES += fs_config_dirs fs_config_files
+#   If not specified, check if default one to be found
+#
+# 2. The new way:
+#   set TARGET_FS_CONFIG_GEN to contain a list of intermediate format files
+#   for generating the android_filesystem_config.h file.
+#
+# More information can be found in the README
 ANDROID_FS_CONFIG_H := android_filesystem_config.h
 
 ifneq ($(TARGET_ANDROID_FILESYSTEM_CONFIG_H),)
+ifneq ($(TARGET_FS_CONFIG_GEN),)
+$(error Cannot set TARGET_ANDROID_FILESYSTEM_CONFIG_H and TARGET_FS_CONFIG_GEN simultaneously)
+endif
 
 # One and only one file can be specified.
 ifneq ($(words $(TARGET_ANDROID_FILESYSTEM_CONFIG_H)),1)
@@ -51,20 +62,43 @@ endif
 
 my_fs_config_h := $(TARGET_ANDROID_FILESYSTEM_CONFIG_H)
 else ifneq ($(wildcard $(TARGET_DEVICE_DIR)/$(ANDROID_FS_CONFIG_H)),)
+
+ifneq ($(TARGET_FS_CONFIG_GEN),)
+$(error Cannot provide $(TARGET_DEVICE_DIR)/$(ANDROID_FS_CONFIG_H) and set TARGET_FS_CONFIG_GEN simultaneously)
+endif
 my_fs_config_h := $(TARGET_DEVICE_DIR)/$(ANDROID_FS_CONFIG_H)
+
 else
 my_fs_config_h := $(LOCAL_PATH)/default/$(ANDROID_FS_CONFIG_H)
 endif
 
+##################################
 include $(CLEAR_VARS)
 LOCAL_SRC_FILES := fs_config_generate.c
 LOCAL_MODULE := fs_config_generate_$(TARGET_DEVICE)
+LOCAL_MODULE_CLASS := EXECUTABLES
 LOCAL_SHARED_LIBRARIES := libcutils
 LOCAL_CFLAGS := -Werror -Wno-error=\#warnings
+
+ifneq ($(TARGET_FS_CONFIG_GEN),)
+gen := $(local-generated-sources-dir)/$(ANDROID_FS_CONFIG_H)
+$(gen): PRIVATE_LOCAL_PATH := $(LOCAL_PATH)
+$(gen): PRIVATE_TARGET_FS_CONFIG_GEN := $(TARGET_FS_CONFIG_GEN)
+$(gen): PRIVATE_CUSTOM_TOOL = $(PRIVATE_LOCAL_PATH)/fs_config_generator.py $(PRIVATE_TARGET_FS_CONFIG_GEN) > $@
+$(gen): $(TARGET_FS_CONFIG_GEN) $(LOCAL_PATH)/fs_config_generator.py
+	$(transform-generated-source)
+
+LOCAL_GENERATED_SOURCES := $(gen)
+my_fs_config_h := $(gen)
+gen :=
+endif
+
 LOCAL_C_INCLUDES := $(dir $(my_fs_config_h))
+
 include $(BUILD_HOST_EXECUTABLE)
 fs_config_generate_bin := $(LOCAL_INSTALLED_MODULE)
 
+##################################
 # Generate the system/etc/fs_config_dirs binary file for the target
 # Add fs_config_dirs to PRODUCT_PACKAGES in the device make file to enable
 include $(CLEAR_VARS)
@@ -76,6 +110,7 @@ $(LOCAL_BUILT_MODULE): $(fs_config_generate_bin)
 	@mkdir -p $(dir $@)
 	$< -D -o $@
 
+##################################
 # Generate the system/etc/fs_config_files binary file for the target
 # Add fs_config_files to PRODUCT_PACKAGES in the device make file to enable
 include $(CLEAR_VARS)

tools/fs_config/README

@@ -0,0 +1,116 @@
+ _____  _____  _____  _____  __  __  _____
+/  _  \/   __\/  _  \|  _  \/  \/  \/   __\
+|  _  <|   __||  _  ||  |  ||  \/  ||   __|
+\__|\_/\_____/\__|__/|_____/\__ \__/\_____/
+
+
+Generating the android_filesystem_config.h
+
+To generate the android_filesystem_config.h file, one can choose from
+one of two methods. The first method, is to declare
+TARGET_ANDROID_FILESYSTEM_CONFIG_H in the device BoardConfig.mk file. This
+variable can only have one item in it, and it is used directly as the
+android_filesystem_config.h header when building
+fs_config_generate_$(TARGET_DEVICE) which is used to generate fs_config_files
+and fs_config_dirs target executable.
+
+The limitation with this, is that it can only be set once, thus if the device
+has a make hierarchy, then each device needs its own file, and cannot share
+from a common source or that common source needs to include everything from
+both devices.
+
+The other way is to set TARGET_FS_CONFIG_GEN, which can be a list of
+intermediate fs configuration files. It is a build error on any one
+these conditions:
+ * Specify TARGET_FS_CONFIG_GEN and TARGET_ANDROID_FILESYSTEM_CONFIG_H
+ * Specify TARGET_FS_CONFIG_GEN and provide
+   $(TARGET_DEVICE_DIR)/android_filesystem_config.h
+
+The parsing of the config file follows the Python ConfigParser specification,
+with the sections and fields as defined below. There are two types of sections,
+both sections require all options to be specified. The first section type is
+the "caps" section.
+
+The "caps" section follows the following syntax:
+
+[path]
+mode: Octal file mode
+user: AID_<user>
+group: AID_<group>
+caps: cap*
+
+Where:
+
+[path]
+  The filesystem path to configure. A path ending in / is considered a dir,
+  else its a file.
+
+mode:
+  A valid octal file mode of at least 3 digits. If 3 is specified, it is
+  prefixed with a 0, else mode is used as is.
+
+user:
+  The exact, C define for a valid AID. Note custom AIDs can be defined in the
+  AID section documented below.
+
+group:
+  The exact, C define for a valid AID. Note custom AIDs can be defined in the
+  AID section documented below.
+
+caps:
+  The name as declared in
+  system/core/include/private/android_filesystem_capability.h without the
+  leading CAP_. Mixed case is allowed. Caps can also be the raw:
+   * binary (0b0101)
+   * octal (0455)
+   * int (42)
+   * hex (0xFF)
+  For multiple caps, just separate by whitespace.
+
+It is an error to specify multiple sections with the same [path]. Per the ini
+specifications enforced by Pythons ConfigParser.
+
+
+The next section type is the "AID" section, for specifying OEM specific AIDS.
+
+The AID section follows the following syntax:
+
+[AID_<name>]
+value: <number>
+
+Where:
+
+[AID_<name>]
+  The <name> can be any valid character for a #define identifier in C.
+
+value:
+  A valid C style number string. Hex, octal, binary and decimal are supported. See "caps"
+  above for more details on number formatting.
+
+It is an error to specify multiple sections with the same [AID_<name>]. Per the ini
+specifications enforced by Pythons ConfigParser. It is also an error to specify
+multiple sections with the same value option. It is also an error to specify a value
+that is outside of the OEM range AID_OEM_RESERVED_START(2900) and AID_OEM_RESERVED_END(2999)
+as defined by system/core/include/private/android_filesystem_config.h.
+
+Ordering within the TARGET_FS_CONFIG_GEN files is not relevant. The paths for files are sorted
+like so within their respective array definition:
+ * specified path before prefix match
+ ** ie foo before f*
+ * lexicographical less than before other
+ ** ie boo before foo
+
+Given these paths:
+
+paths=['ac', 'a', 'acd', 'an', 'a*', 'aa', 'ac*']
+
+The sort order would be:
+paths=['a', 'aa', 'ac', 'acd', 'an', 'ac*', 'a*']
+
+Thus the fs_config tools will match on specified paths before attempting prefix, and match on the
+longest matching prefix.
+
+The declared AIDS are sorted in ascending numerical order based on the option "value". The string
+representation of value is preserved. Both choices were made for maximum readability of the generated
+file and to line up files. Sync lines are placed with the source file as comments in the generated
+header file.

tools/fs_config/fs_config_generator.py

@@ -0,0 +1,277 @@
+#!/usr/bin/env python
+
+import ConfigParser
+import re
+import sys
+
+
+GENERATED = '''
+/*
+ * THIS IS AN AUTOGENERATED FILE! DO NOT MODIFY
+ */
+'''
+
+INCLUDE = '#include <private/android_filesystem_config.h>'
+
+DEFINE_NO_DIRS = '#define NO_ANDROID_FILESYSTEM_CONFIG_DEVICE_DIRS\n'
+DEFINE_NO_FILES = '#define NO_ANDROID_FILESYSTEM_CONFIG_DEVICE_FILES\n'
+
+DEFAULT_WARNING = '#warning No device-supplied android_filesystem_config.h, using empty default.'
+
+NO_ANDROID_FILESYSTEM_CONFIG_DEVICE_DIRS_ENTRY = '{ 00000, AID_ROOT,      AID_ROOT,      0, "system/etc/fs_config_dirs" },'
+NO_ANDROID_FILESYSTEM_CONFIG_DEVICE_FILES_ENTRY = '{ 00000, AID_ROOT,      AID_ROOT,      0, "system/etc/fs_config_files" },'
+
+IFDEF_ANDROID_FILESYSTEM_CONFIG_DEVICE_DIRS = '#ifdef NO_ANDROID_FILESYSTEM_CONFIG_DEVICE_DIRS'
+ENDIF = '#endif'
+
+OPEN_FILE_STRUCT = 'static const struct fs_path_config android_device_files[] = {'
+OPEN_DIR_STRUCT = 'static const struct fs_path_config android_device_dirs[] = {'
+CLOSE_FILE_STRUCT = '};'
+
+GENERIC_DEFINE = "#define %s\t%s"
+
+FILE_COMMENT = '// Defined in file: \"%s\"'
+
+# from system/core/include/private/android_filesystem_config.h
+AID_OEM_RESERVED_START = 2900
+AID_OEM_RESERVED_END = 2999
+
+
+AID_MATCH = re.compile('AID_[a-zA-Z]+')
+
+def handle_aid(file_name, section_name, config, aids, seen_aids):
+    value = config.get(section_name, 'value')
+
+    errmsg = '%s for: \"' + section_name + '" file: \"' + file_name + '\"'
+
+    if not value:
+        raise Exception(errmsg % 'Found specified but unset "value"')
+
+    v = convert_int(value)
+    if not v:
+        raise Exception(errmsg % ('Invalid "value", not a number, got: \"%s\"' % value))
+
+    # Values must be within OEM range
+    if (v < AID_OEM_RESERVED_START) or (v > AID_OEM_RESERVED_END):
+        s = '"value" not in valid range %d - %d, got: %s'
+        s = s % (AID_OEM_RESERVED_START, AID_OEM_RESERVED_END, value)
+        raise Exception(errmsg % s)
+
+    # use the normalized int value in the dict and detect
+    # duplicate definitions of the same vallue
+    v = str(v)
+    if v in seen_aids[1]:
+        # map of value to aid name
+        a = seen_aids[1][v]
+
+        # aid name to file
+        f = seen_aids[0][a]
+
+        s = 'Duplicate AID value "%s" found on AID: "%s".' % (value, seen_aids[1][v])
+        s += ' Previous found in file: "%s."' % f
+        raise Exception(errmsg % s)
+
+    seen_aids[1][v] = section_name
+
+    # Append a tuple of (AID_*, base10(value), str(value))
+    # We keep the str version of value so we can print that out in the
+    # generated header so investigating parties can identify parts.
+    # We store the base10 value for sorting, so everything is ascending
+    # later.
+    aids.append((file_name, section_name, v, value))
+
+def convert_int(num):
+
+        try:
+            if num.startswith('0x'):
+                return int(num, 16)
+            elif num.startswith('0b'):
+                return int(num, 2)
+            elif num.startswith('0'):
+                return int(num, 8)
+            else:
+                return int(num, 10)
+        except ValueError:
+            pass
+        return None
+
+def handle_path(file_name, section_name, config, files, dirs):
+
+            mode = config.get(section_name, 'mode')
+            user = config.get(section_name, 'user')
+            group = config.get(section_name, 'group')
+            caps = config.get(section_name, 'caps')
+
+            errmsg = 'Found specified but unset option: \"%s" in file: \"' + file_name + '\"'
+
+            if not mode:
+                raise Exception(errmsg % 'mode')
+
+            if not user:
+                raise Exception(errmsg % 'user')
+
+            if not group:
+                raise Exception(errmsg % 'group')
+
+            if not caps:
+                raise Exception(errmsg % 'caps')
+
+            caps = caps.split()
+
+            tmp = []
+            for x in caps:
+                if convert_int(x):
+                    tmp.append('(' + x + ')')
+                else:
+                    tmp.append('(1ULL << CAP_' + x.upper() + ')')
+
+            caps = tmp
+
+            path = '"' + section_name + '"'
+
+            if len(mode) == 3:
+                mode = '0' + mode
+
+            try:
+                int(mode, 8)
+            except:
+                raise Exception('Mode must be octal characters, got: "' + mode + '"')
+
+            if len(mode) != 4:
+                raise Exception('Mode must be 3 or 4 characters, got: "' + mode + '"')
+
+
+            caps = '|'.join(caps)
+
+            x = [ mode, user, group, caps, section_name ]
+            if section_name[-1] == '/':
+                dirs.append((file_name, x))
+            else:
+                files.append((file_name, x))
+
+def handle_dup(name, file_name, section_name, seen):
+        if section_name in seen:
+            dups = '"' + seen[section_name] + '" and '
+            dups += file_name
+            raise Exception('Duplicate ' + name + ' "' + section_name + '" found in files: ' + dups)
+
+def parse(file_name, files, dirs, aids, seen_paths, seen_aids):
+
+        config = ConfigParser.ConfigParser()
+        config.read(file_name)
+
+        for s in config.sections():
+
+            if AID_MATCH.match(s) and config.has_option(s, 'value'):
+                handle_dup('AID', file_name, s, seen_aids[0])
+                seen_aids[0][s] = file_name
+                handle_aid(file_name, s, config, aids, seen_aids)
+            else:
+                handle_dup('path', file_name, s, seen_paths)
+                seen_paths[s] = file_name
+                handle_path(file_name, s, config, files, dirs)
+
+def generate(files, dirs, aids):
+    print GENERATED
+    print INCLUDE
+    print
+
+    are_dirs = len(dirs) > 0
+    are_files = len(files) > 0
+    are_aids = len(aids) > 0
+
+    if are_aids:
+        # sort on value of (file_name, name, value, strvalue)
+        aids.sort(key=lambda x: x[2])
+        for a in aids:
+            # use the preserved str value
+            print FILE_COMMENT % a[0]
+            print GENERIC_DEFINE % (a[1], a[2])
+
+        print
+
+    if not are_dirs:
+        print DEFINE_NO_DIRS
+
+    if not are_files:
+        print DEFINE_NO_FILES
+
+    if not are_files and not are_dirs and not are_aids:
+        print DEFAULT_WARNING
+        return
+
+    if are_files:
+        print OPEN_FILE_STRUCT
+        for tup in files:
+            f = tup[0]
+            c = tup[1]
+            c[4] = '"' + c[4] + '"'
+            c = '{ ' + '    ,'.join(c) + ' },'
+            print FILE_COMMENT % f
+            print '    ' + c
+
+        if not are_dirs:
+            print IFDEF_ANDROID_FILESYSTEM_CONFIG_DEVICE_DIRS
+            print '    ' + NO_ANDROID_FILESYSTEM_CONFIG_DEVICE_DIRS_ENTRY
+            print ENDIF
+        print CLOSE_FILE_STRUCT
+
+    if are_dirs:
+        print OPEN_DIR_STRUCT
+        for d in dirs:
+            f[4] = '"' + f[4] + '"'
+            d = '{ ' + '    ,'.join(d) + ' },'
+            print '    ' + d
+
+        print CLOSE_FILE_STRUCT
+
+def file_key(x):
+
+    # Wrapper class for custom prefix matching strings
+    class S(object):
+        def __init__(self, str):
+
+            self.orig = str
+            self.is_prefix = str[-1] == '*'
+            if self.is_prefix:
+                self.str = str[:-1]
+            else:
+                self.str = str
+
+        def __lt__(self, other):
+
+            # if were both suffixed the smallest string
+            # is 'bigger'
+            if self.is_prefix and other.is_prefix:
+                b = len(self.str) > len(other.str)
+            # If I am an the suffix match, im bigger
+            elif self.is_prefix:
+                b = False
+            # If other is the suffix match, he's bigger
+            elif other.is_prefix:
+                b = True
+            # Alphabetical
+            else:
+                b = self.str < other.str
+            return b
+
+    return S(x[4])
+
+def main():
+
+    files = []
+    dirs = []
+    aids = []
+    seen_paths = {}
+
+    # (name to file, value to aid)
+    seen_aids = ({}, {})
+
+    for x in sys.argv[1:]:
+        parse(x, files, dirs, aids, seen_paths, seen_aids)
+
+    files.sort(key= lambda x: file_key(x[1]))
+    generate(files, dirs, aids)
+
+if __name__ == '__main__':
+    main()