Shard arch property structs

Arch property struct types are created at runtime.  Go has a limit
of 2**16 bytes for the name of a type, and the type of a struct
created at runtime is a string containing all the names and types
of its fields.  To avoid going over the limit, split the runtime
created structs into multiple shards.

Also undo MoreBaseLinkerProperties now that it is no longer
required.

Bug: 80437643
Test: m checkbuild
Test: no change to out/soong/build.ninja
Change-Id: I035b20332ec63f3d4b1696855c5b0b0a810597b7

android/arch.go

@@ -421,16 +421,9 @@ func decodeMultilib(base *ModuleBase, class OsClass) (multilib, extraMultilib st
 	}
 }
 
-func filterArchStruct(prop reflect.Type) (reflect.Type, bool) {
+func filterArchStructFields(fields []reflect.StructField) []reflect.StructField {
-	var fields []reflect.StructField
+	var ret []reflect.StructField
-
+	for _, field := range fields {
-	ptr := prop.Kind() == reflect.Ptr
-	if ptr {
-		prop = prop.Elem()
-	}
-
-	for i := 0; i < prop.NumField(); i++ {
-		field := prop.Field(i)
 		if !proptools.HasTag(field, "android", "arch_variant") {
 			continue
 		}
@@ -466,8 +459,26 @@ func filterArchStruct(prop reflect.Type) (reflect.Type, bool) {
 			panic("Interfaces are not supported in arch_variant properties")
 		}
 
-		fields = append(fields, field)
+		ret = append(ret, field)
 	}
+
+	return ret
+}
+
+func filterArchStruct(prop reflect.Type) (reflect.Type, bool) {
+	var fields []reflect.StructField
+
+	ptr := prop.Kind() == reflect.Ptr
+	if ptr {
+		prop = prop.Elem()
+	}
+
+	for i := 0; i < prop.NumField(); i++ {
+		fields = append(fields, prop.Field(i))
+	}
+
+	fields = filterArchStructFields(fields)
+
 	if len(fields) == 0 {
 		return nil, false
 	}
@@ -476,15 +487,63 @@ func filterArchStruct(prop reflect.Type) (reflect.Type, bool) {
 	if ptr {
 		ret = reflect.PtrTo(ret)
 	}
+
 	return ret, true
 }
 
-func createArchType(props reflect.Type) reflect.Type {
+func filterArchStructSharded(prop reflect.Type) ([]reflect.Type, bool) {
-	props, ok := filterArchStruct(props)
+	var fields []reflect.StructField
+
+	ptr := prop.Kind() == reflect.Ptr
+	if ptr {
+		prop = prop.Elem()
+	}
+
+	for i := 0; i < prop.NumField(); i++ {
+		fields = append(fields, prop.Field(i))
+	}
+
+	fields = filterArchStructFields(fields)
+
+	if len(fields) == 0 {
+		return nil, false
+	}
+
+	shards := shardFields(fields, 10)
+
+	var ret []reflect.Type
+	for _, shard := range shards {
+		s := reflect.StructOf(shard)
+		if ptr {
+			s = reflect.PtrTo(s)
+		}
+		ret = append(ret, s)
+	}
+
+	return ret, true
+}
+
+func shardFields(fields []reflect.StructField, shardSize int) [][]reflect.StructField {
+	ret := make([][]reflect.StructField, 0, (len(fields)+shardSize-1)/shardSize)
+	for len(fields) > shardSize {
+		ret = append(ret, fields[0:shardSize])
+		fields = fields[shardSize:]
+	}
+	if len(fields) > 0 {
+		ret = append(ret, fields)
+	}
+	return ret
+}
+
+func createArchType(props reflect.Type) []reflect.Type {
+	propShards, ok := filterArchStructSharded(props)
 	if !ok {
 		return nil
 	}
 
+	var ret []reflect.Type
+	for _, props := range propShards {
+
 		variantFields := func(names []string) []reflect.StructField {
 			ret := make([]reflect.StructField, len(names))
 
@@ -511,7 +570,7 @@ func createArchType(props reflect.Type) reflect.Type {
 
 			fields := variantFields(variants)
 
-		fields = append([]reflect.StructField{reflect.StructField{
+			fields = append([]reflect.StructField{{
 				Name:      "BlueprintEmbed",
 				Type:      props,
 				Anonymous: true,
@@ -558,20 +617,22 @@ func createArchType(props reflect.Type) reflect.Type {
 		}
 
 		targetType := reflect.StructOf(variantFields(targets))
-	return reflect.StructOf([]reflect.StructField{
+		ret = append(ret, reflect.StructOf([]reflect.StructField{
-		reflect.StructField{
+			{
 				Name: "Arch",
 				Type: archType,
 			},
-		reflect.StructField{
+			{
 				Name: "Multilib",
 				Type: multilibType,
 			},
-		reflect.StructField{
+			{
 				Name: "Target",
 				Type: targetType,
 			},
-	})
+		}))
+	}
+	return ret
 }
 
 var archPropTypeMap OncePer
@@ -596,21 +657,16 @@ func InitArchModule(m Module) {
 				propertiesValue.Interface()))
 		}
 
-		archPropType := archPropTypeMap.Once(t, func() interface{} {
+		archPropTypes := archPropTypeMap.Once(t, func() interface{} {
 			return createArchType(t)
-		})
+		}).([]reflect.Type)
 
-		if archPropType != nil {
+		var archProperties []interface{}
-			base.archProperties = append(base.archProperties, reflect.New(archPropType.(reflect.Type)).Interface())
+		for _, t := range archPropTypes {
-		} else {
+			archProperties = append(archProperties, reflect.New(t).Interface())
-			base.archProperties = append(base.archProperties, nil)
-		}
-	}
-
-	for _, asp := range base.archProperties {
-		if asp != nil {
-			m.AddProperties(asp)
 		}
+		base.archProperties = append(base.archProperties, archProperties)
+		m.AddProperties(archProperties...)
 	}
 
 	base.customizableProperties = m.GetProperties()
@@ -665,11 +721,12 @@ func (a *ModuleBase) setArchProperties(ctx BottomUpMutatorContext) {
 		if a.archProperties[i] == nil {
 			continue
 		}
-		archProps := reflect.ValueOf(a.archProperties[i]).Elem()
+		for _, archProperties := range a.archProperties[i] {
+			archPropValues := reflect.ValueOf(archProperties).Elem()
 
-		archProp := archProps.FieldByName("Arch")
+			archProp := archPropValues.FieldByName("Arch")
-		multilibProp := archProps.FieldByName("Multilib")
+			multilibProp := archPropValues.FieldByName("Multilib")
-		targetProp := archProps.FieldByName("Target")
+			targetProp := archPropValues.FieldByName("Target")
 
 			var field string
 			var prefix string
@@ -865,6 +922,7 @@ func (a *ModuleBase) setArchProperties(ctx BottomUpMutatorContext) {
 				}
 			}
 		}
+	}
 }
 
 func forEachInterface(v reflect.Value, f func(reflect.Value)) {

android/module.go

@@ -435,7 +435,7 @@ type ModuleBase struct {
 	variableProperties      variableProperties
 	hostAndDeviceProperties hostAndDeviceProperties
 	generalProperties       []interface{}
-	archProperties          []interface{}
+	archProperties          [][]interface{}
 	customizableProperties  []interface{}
 
 	noAddressSanitizer bool

cc/cc.go

@@ -1610,7 +1610,6 @@ func DefaultsFactory(props ...interface{}) android.Module {
 		&VendorProperties{},
 		&BaseCompilerProperties{},
 		&BaseLinkerProperties{},
-		&MoreBaseLinkerProperties{},
 		&LibraryProperties{},
 		&FlagExporterProperties{},
 		&BinaryLinkerProperties{},

cc/linker.go

@@ -118,6 +118,9 @@ type BaseLinkerProperties struct {
 			// list of runtime libs that should not be installed along with the vendor
 			// variant of the C/C++ module.
 			Exclude_runtime_libs []string
+
+			// version script for this vendor variant
+			Version_script *string `android:"arch_variant"`
 		}
 		Recovery struct {
 			// list of shared libs that only should be used to build the recovery
@@ -140,23 +143,12 @@ type BaseLinkerProperties struct {
 
 	// make android::build:GetBuildNumber() available containing the build ID.
 	Use_version_lib *bool `android:"arch_variant"`
-}
 
-// TODO(http://b/80437643): BaseLinkerProperties is getting too big,
-// more than 2^16 bytes. New properties are defined in MoreBaseLinkerProperties.
-type MoreBaseLinkerProperties struct {
 	// Generate compact dynamic relocation table, default true.
 	Pack_relocations *bool `android:"arch_variant"`
 
 	// local file name to pass to the linker as --version_script
 	Version_script *string `android:"arch_variant"`
-
-	Target struct {
-		Vendor struct {
-			// version script for this vendor variant
-			Version_script *string `android:"arch_variant"`
-		}
-	}
 }
 
 func NewBaseLinker(sanitize *sanitize) *baseLinker {
@@ -166,7 +158,6 @@ func NewBaseLinker(sanitize *sanitize) *baseLinker {
 // baseLinker provides support for shared_libs, static_libs, and whole_static_libs properties
 type baseLinker struct {
 	Properties        BaseLinkerProperties
-	MoreProperties    MoreBaseLinkerProperties
 	dynamicProperties struct {
 		RunPaths   []string `blueprint:"mutated"`
 		BuildStubs bool     `blueprint:"mutated"`
@@ -188,7 +179,7 @@ func (linker *baseLinker) linkerInit(ctx BaseModuleContext) {
 }
 
 func (linker *baseLinker) linkerProps() []interface{} {
-	return []interface{}{&linker.Properties, &linker.MoreProperties, &linker.dynamicProperties}
+	return []interface{}{&linker.Properties, &linker.dynamicProperties}
 }
 
 func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps {
@@ -283,9 +274,9 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps {
 	// Version_script is not needed when linking stubs lib where the version
 	// script is created from the symbol map file.
 	if !linker.dynamicProperties.BuildStubs {
-		android.ExtractSourceDeps(ctx, linker.MoreProperties.Version_script)
+		android.ExtractSourceDeps(ctx, linker.Properties.Version_script)
 		android.ExtractSourceDeps(ctx,
-			linker.MoreProperties.Target.Vendor.Version_script)
+			linker.Properties.Target.Vendor.Version_script)
 	}
 
 	return deps
@@ -319,7 +310,7 @@ func (linker *baseLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags {
 
 	if linker.useClangLld(ctx) {
 		flags.LdFlags = append(flags.LdFlags, fmt.Sprintf("${config.%sGlobalLldflags}", hod))
-		if !BoolDefault(linker.MoreProperties.Pack_relocations, true) {
+		if !BoolDefault(linker.Properties.Pack_relocations, true) {
 			flags.LdFlags = append(flags.LdFlags, "-Wl,--pack-dyn-relocs=none")
 		}
 	} else {
@@ -394,11 +385,11 @@ func (linker *baseLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags {
 	// script is created from the symbol map file.
 	if !linker.dynamicProperties.BuildStubs {
 		versionScript := ctx.ExpandOptionalSource(
-			linker.MoreProperties.Version_script, "version_script")
+			linker.Properties.Version_script, "version_script")
 
-		if ctx.useVndk() && linker.MoreProperties.Target.Vendor.Version_script != nil {
+		if ctx.useVndk() && linker.Properties.Target.Vendor.Version_script != nil {
 			versionScript = ctx.ExpandOptionalSource(
-				linker.MoreProperties.Target.Vendor.Version_script,
+				linker.Properties.Target.Vendor.Version_script,
 				"target.vendor.version_script")
 		}