StrixOS/build_soong
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Shard arch property structs

Browse Source 
Arch property struct types are created at runtime.  Go has a limit
of 2**16 bytes for the name of a type, and the type of a struct
created at runtime is a string containing all the names and types
of its fields.  To avoid going over the limit, split the runtime
created structs into multiple shards.

Also undo MoreBaseLinkerProperties now that it is no longer
required.

Bug: 80437643
Test: m checkbuild
Test: no change to out/soong/build.ninja
Change-Id: I035b20332ec63f3d4b1696855c5b0b0a810597b7
This commit is contained in:
Colin Cross
2018-10-24 12:42:09 -07:00
parent 4e81d709ab
commit c17727d06b
4 changed files with 347 additions and 299 deletions
Download Patch File Download Diff File Expand all files Collapse all files

614
android/arch.go
View File

@@ -421,16 +421,9 @@ func decodeMultilib(base *ModuleBase, class OsClass) (multilib, extraMultilib st
}
}
func filterArchStruct(prop reflect.Type) (reflect.Type, bool) {
var fields []reflect.StructField
ptr := prop.Kind() == reflect.Ptr
if ptr {
prop = prop.Elem()
}
for i := 0; i < prop.NumField(); i++ {
field := prop.Field(i)
func filterArchStructFields(fields []reflect.StructField) []reflect.StructField {
var ret []reflect.StructField
for _, field := range fields {
if !proptools.HasTag(field, "android", "arch_variant") {
continue
}
@@ -466,8 +459,26 @@ func filterArchStruct(prop reflect.Type) (reflect.Type, bool) {
panic("Interfaces are not supported in arch_variant properties")
}
fields = append(fields, field)
ret = append(ret, field)
}
return ret
}
func filterArchStruct(prop reflect.Type) (reflect.Type, bool) {
var fields []reflect.StructField
ptr := prop.Kind() == reflect.Ptr
if ptr {
prop = prop.Elem()
}
for i := 0; i < prop.NumField(); i++ {
fields = append(fields, prop.Field(i))
}
fields = filterArchStructFields(fields)
if len(fields) == 0 {
return nil, false
}
@@ -476,102 +487,152 @@ func filterArchStruct(prop reflect.Type) (reflect.Type, bool) {
if ptr {
ret = reflect.PtrTo(ret)
}
return ret, true
}
func createArchType(props reflect.Type) reflect.Type {
props, ok := filterArchStruct(props)
func filterArchStructSharded(prop reflect.Type) ([]reflect.Type, bool) {
var fields []reflect.StructField
ptr := prop.Kind() == reflect.Ptr
if ptr {
prop = prop.Elem()
}
for i := 0; i < prop.NumField(); i++ {
fields = append(fields, prop.Field(i))
}
fields = filterArchStructFields(fields)
if len(fields) == 0 {
return nil, false
}
shards := shardFields(fields, 10)
var ret []reflect.Type
for _, shard := range shards {
s := reflect.StructOf(shard)
if ptr {
s = reflect.PtrTo(s)
}
ret = append(ret, s)
}
return ret, true
}
func shardFields(fields []reflect.StructField, shardSize int) [][]reflect.StructField {
ret := make([][]reflect.StructField, 0, (len(fields)+shardSize-1)/shardSize)
for len(fields) > shardSize {
ret = append(ret, fields[0:shardSize])
fields = fields[shardSize:]
}
if len(fields) > 0 {
ret = append(ret, fields)
}
return ret
}
func createArchType(props reflect.Type) []reflect.Type {
propShards, ok := filterArchStructSharded(props)
if !ok {
return nil
}
variantFields := func(names []string) []reflect.StructField {
ret := make([]reflect.StructField, len(names))
var ret []reflect.Type
for _, props := range propShards {
for i, name := range names {
ret[i].Name = name
ret[i].Type = props
}
variantFields := func(names []string) []reflect.StructField {
ret := make([]reflect.StructField, len(names))
return ret
}
archFields := make([]reflect.StructField, len(archTypeList))
for i, arch := range archTypeList {
variants := []string{}
for _, archVariant := range archVariants[arch] {
archVariant := variantReplacer.Replace(archVariant)
variants = append(variants, proptools.FieldNameForProperty(archVariant))
}
for _, feature := range archFeatures[arch] {
feature := variantReplacer.Replace(feature)
variants = append(variants, proptools.FieldNameForProperty(feature))
}
fields := variantFields(variants)
fields = append([]reflect.StructField{reflect.StructField{
Name: "BlueprintEmbed",
Type: props,
Anonymous: true,
}}, fields...)
archFields[i] = reflect.StructField{
Name: arch.Field,
Type: reflect.StructOf(fields),
}
}
archType := reflect.StructOf(archFields)
multilibType := reflect.StructOf(variantFields([]string{"Lib32", "Lib64"}))
targets := []string{
"Host",
"Android64",
"Android32",
"Bionic",
"Linux",
"Not_windows",
"Arm_on_x86",
"Arm_on_x86_64",
}
for _, os := range osTypeList {
targets = append(targets, os.Field)
for _, archType := range osArchTypeMap[os] {
targets = append(targets, os.Field+"_"+archType.Name)
if os.Linux() {
target := "Linux_" + archType.Name
if !InList(target, targets) {
targets = append(targets, target)
}
for i, name := range names {
ret[i].Name = name
ret[i].Type = props
}
if os.Bionic() {
target := "Bionic_" + archType.Name
if !InList(target, targets) {
targets = append(targets, target)
return ret
}
archFields := make([]reflect.StructField, len(archTypeList))
for i, arch := range archTypeList {
variants := []string{}
for _, archVariant := range archVariants[arch] {
archVariant := variantReplacer.Replace(archVariant)
variants = append(variants, proptools.FieldNameForProperty(archVariant))
}
for _, feature := range archFeatures[arch] {
feature := variantReplacer.Replace(feature)
variants = append(variants, proptools.FieldNameForProperty(feature))
}
fields := variantFields(variants)
fields = append([]reflect.StructField{{
Name: "BlueprintEmbed",
Type: props,
Anonymous: true,
}}, fields...)
archFields[i] = reflect.StructField{
Name: arch.Field,
Type: reflect.StructOf(fields),
}
}
archType := reflect.StructOf(archFields)
multilibType := reflect.StructOf(variantFields([]string{"Lib32", "Lib64"}))
targets := []string{
"Host",
"Android64",
"Android32",
"Bionic",
"Linux",
"Not_windows",
"Arm_on_x86",
"Arm_on_x86_64",
}
for _, os := range osTypeList {
targets = append(targets, os.Field)
for _, archType := range osArchTypeMap[os] {
targets = append(targets, os.Field+"_"+archType.Name)
if os.Linux() {
target := "Linux_" + archType.Name
if !InList(target, targets) {
targets = append(targets, target)
}
}
if os.Bionic() {
target := "Bionic_" + archType.Name
if !InList(target, targets) {
targets = append(targets, target)
}
}
}
}
}
targetType := reflect.StructOf(variantFields(targets))
return reflect.StructOf([]reflect.StructField{
reflect.StructField{
Name: "Arch",
Type: archType,
},
reflect.StructField{
Name: "Multilib",
Type: multilibType,
},
reflect.StructField{
Name: "Target",
Type: targetType,
},
})
targetType := reflect.StructOf(variantFields(targets))
ret = append(ret, reflect.StructOf([]reflect.StructField{
{
Name: "Arch",
Type: archType,
},
{
Name: "Multilib",
Type: multilibType,
},
{
Name: "Target",
Type: targetType,
},
}))
}
return ret
}
var archPropTypeMap OncePer
@@ -596,21 +657,16 @@ func InitArchModule(m Module) {
propertiesValue.Interface()))
}
archPropType := archPropTypeMap.Once(t, func() interface{} {
archPropTypes := archPropTypeMap.Once(t, func() interface{} {
return createArchType(t)
})
}).([]reflect.Type)
if archPropType != nil {
base.archProperties = append(base.archProperties, reflect.New(archPropType.(reflect.Type)).Interface())
} else {
base.archProperties = append(base.archProperties, nil)
}
}
for _, asp := range base.archProperties {
if asp != nil {
m.AddProperties(asp)
var archProperties []interface{}
for _, t := range archPropTypes {
archProperties = append(archProperties, reflect.New(t).Interface())
}
base.archProperties = append(base.archProperties, archProperties)
m.AddProperties(archProperties...)
}
base.customizableProperties = m.GetProperties()
@@ -665,203 +721,205 @@ func (a *ModuleBase) setArchProperties(ctx BottomUpMutatorContext) {
if a.archProperties[i] == nil {
continue
}
archProps := reflect.ValueOf(a.archProperties[i]).Elem()
for _, archProperties := range a.archProperties[i] {
archPropValues := reflect.ValueOf(archProperties).Elem()
archProp := archProps.FieldByName("Arch")
multilibProp := archProps.FieldByName("Multilib")
targetProp := archProps.FieldByName("Target")
archProp := archPropValues.FieldByName("Arch")
multilibProp := archPropValues.FieldByName("Multilib")
targetProp := archPropValues.FieldByName("Target")
var field string
var prefix string
var field string
var prefix string
// Handle arch-specific properties in the form:
// arch: {
// arm64: {
// key: value,
// },
// },
t := arch.ArchType
if arch.ArchType != Common {
field := proptools.FieldNameForProperty(t.Name)
prefix := "arch." + t.Name
archStruct := a.appendProperties(ctx, genProps, archProp, field, prefix)
// Handle arch-variant-specific properties in the form:
// Handle arch-specific properties in the form:
// arch: {
// variant: {
// arm64: {
// key: value,
// },
// },
v := variantReplacer.Replace(arch.ArchVariant)
if v != "" {
field := proptools.FieldNameForProperty(v)
prefix := "arch." + t.Name + "." + v
a.appendProperties(ctx, genProps, archStruct, field, prefix)
t := arch.ArchType
if arch.ArchType != Common {
field := proptools.FieldNameForProperty(t.Name)
prefix := "arch." + t.Name
archStruct := a.appendProperties(ctx, genProps, archProp, field, prefix)
// Handle arch-variant-specific properties in the form:
// arch: {
// variant: {
// key: value,
// },
// },
v := variantReplacer.Replace(arch.ArchVariant)
if v != "" {
field := proptools.FieldNameForProperty(v)
prefix := "arch." + t.Name + "." + v
a.appendProperties(ctx, genProps, archStruct, field, prefix)
}
// Handle cpu-variant-specific properties in the form:
// arch: {
// variant: {
// key: value,
// },
// },
if arch.CpuVariant != arch.ArchVariant {
c := variantReplacer.Replace(arch.CpuVariant)
if c != "" {
field := proptools.FieldNameForProperty(c)
prefix := "arch." + t.Name + "." + c
a.appendProperties(ctx, genProps, archStruct, field, prefix)
}
}
// Handle arch-feature-specific properties in the form:
// arch: {
// feature: {
// key: value,
// },
// },
for _, feature := range arch.ArchFeatures {
field := proptools.FieldNameForProperty(feature)
prefix := "arch." + t.Name + "." + feature
a.appendProperties(ctx, genProps, archStruct, field, prefix)
}
// Handle multilib-specific properties in the form:
// multilib: {
// lib32: {
// key: value,
// },
// },
field = proptools.FieldNameForProperty(t.Multilib)
prefix = "multilib." + t.Multilib
a.appendProperties(ctx, genProps, multilibProp, field, prefix)
}
// Handle cpu-variant-specific properties in the form:
// arch: {
// variant: {
// Handle host-specific properties in the form:
// target: {
// host: {
// key: value,
// },
// },
if arch.CpuVariant != arch.ArchVariant {
c := variantReplacer.Replace(arch.CpuVariant)
if c != "" {
field := proptools.FieldNameForProperty(c)
prefix := "arch." + t.Name + "." + c
a.appendProperties(ctx, genProps, archStruct, field, prefix)
if os.Class == Host || os.Class == HostCross {
field = "Host"
prefix = "target.host"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
}
// Handle target OS generalities of the form:
// target: {
// bionic: {
// key: value,
// },
// bionic_x86: {
// key: value,
// },
// }
if os.Linux() {
field = "Linux"
prefix = "target.linux"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
if arch.ArchType != Common {
field = "Linux_" + arch.ArchType.Name
prefix = "target.linux_" + arch.ArchType.Name
a.appendProperties(ctx, genProps, targetProp, field, prefix)
}
}
// Handle arch-feature-specific properties in the form:
// arch: {
// feature: {
if os.Bionic() {
field = "Bionic"
prefix = "target.bionic"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
if arch.ArchType != Common {
field = "Bionic_" + t.Name
prefix = "target.bionic_" + t.Name
a.appendProperties(ctx, genProps, targetProp, field, prefix)
}
}
// Handle target OS properties in the form:
// target: {
// linux_glibc: {
// key: value,
// },
// not_windows: {
// key: value,
// },
// linux_glibc_x86: {
// key: value,
// },
// linux_glibc_arm: {
// key: value,
// },
// android {
// key: value,
// },
// android_arm {
// key: value,
// },
// android_x86 {
// key: value,
// },
// },
for _, feature := range arch.ArchFeatures {
field := proptools.FieldNameForProperty(feature)
prefix := "arch." + t.Name + "." + feature
a.appendProperties(ctx, genProps, archStruct, field, prefix)
field = os.Field
prefix = "target." + os.Name
a.appendProperties(ctx, genProps, targetProp, field, prefix)
if arch.ArchType != Common {
field = os.Field + "_" + t.Name
prefix = "target." + os.Name + "_" + t.Name
a.appendProperties(ctx, genProps, targetProp, field, prefix)
}
// Handle multilib-specific properties in the form:
// multilib: {
// lib32: {
if (os.Class == Host || os.Class == HostCross) && os != Windows {
field := "Not_windows"
prefix := "target.not_windows"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
}
// Handle 64-bit device properties in the form:
// target {
// android64 {
// key: value,
// },
// android32 {
// key: value,
// },
// },
field = proptools.FieldNameForProperty(t.Multilib)
prefix = "multilib." + t.Multilib
a.appendProperties(ctx, genProps, multilibProp, field, prefix)
}
// WARNING: this is probably not what you want to use in your blueprints file, it selects
// options for all targets on a device that supports 64-bit binaries, not just the targets
// that are being compiled for 64-bit. Its expected use case is binaries like linker and
// debuggerd that need to know when they are a 32-bit process running on a 64-bit device
if os.Class == Device {
if ctx.Config().Android64() {
field := "Android64"
prefix := "target.android64"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
} else {
field := "Android32"
prefix := "target.android32"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
}
// Handle host-specific properties in the form:
// target: {
// host: {
// key: value,
// },
// },
if os.Class == Host || os.Class == HostCross {
field = "Host"
prefix = "target.host"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
}
// Handle target OS generalities of the form:
// target: {
// bionic: {
// key: value,
// },
// bionic_x86: {
// key: value,
// },
// }
if os.Linux() {
field = "Linux"
prefix = "target.linux"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
if arch.ArchType != Common {
field = "Linux_" + arch.ArchType.Name
prefix = "target.linux_" + arch.ArchType.Name
a.appendProperties(ctx, genProps, targetProp, field, prefix)
}
}
if os.Bionic() {
field = "Bionic"
prefix = "target.bionic"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
if arch.ArchType != Common {
field = "Bionic_" + t.Name
prefix = "target.bionic_" + t.Name
a.appendProperties(ctx, genProps, targetProp, field, prefix)
}
}
// Handle target OS properties in the form:
// target: {
// linux_glibc: {
// key: value,
// },
// not_windows: {
// key: value,
// },
// linux_glibc_x86: {
// key: value,
// },
// linux_glibc_arm: {
// key: value,
// },
// android {
// key: value,
// },
// android_arm {
// key: value,
// },
// android_x86 {
// key: value,
// },
// },
field = os.Field
prefix = "target." + os.Name
a.appendProperties(ctx, genProps, targetProp, field, prefix)
if arch.ArchType != Common {
field = os.Field + "_" + t.Name
prefix = "target." + os.Name + "_" + t.Name
a.appendProperties(ctx, genProps, targetProp, field, prefix)
}
if (os.Class == Host || os.Class == HostCross) && os != Windows {
field := "Not_windows"
prefix := "target.not_windows"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
}
// Handle 64-bit device properties in the form:
// target {
// android64 {
// key: value,
// },
// android32 {
// key: value,
// },
// },
// WARNING: this is probably not what you want to use in your blueprints file, it selects
// options for all targets on a device that supports 64-bit binaries, not just the targets
// that are being compiled for 64-bit. Its expected use case is binaries like linker and
// debuggerd that need to know when they are a 32-bit process running on a 64-bit device
if os.Class == Device {
if ctx.Config().Android64() {
field := "Android64"
prefix := "target.android64"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
} else {
field := "Android32"
prefix := "target.android32"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
}
if (arch.ArchType == X86 && (hasArmAbi(arch) ||
hasArmAndroidArch(ctx.Config().Targets[Android]))) ||
(arch.ArchType == Arm &&
hasX86AndroidArch(ctx.Config().Targets[Android])) {
field := "Arm_on_x86"
prefix := "target.arm_on_x86"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
}
if (arch.ArchType == X86_64 && (hasArmAbi(arch) ||
hasArmAndroidArch(ctx.Config().Targets[Android]))) ||
(arch.ArchType == Arm &&
hasX8664AndroidArch(ctx.Config().Targets[Android])) {
field := "Arm_on_x86_64"
prefix := "target.arm_on_x86_64"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
if (arch.ArchType == X86 && (hasArmAbi(arch) ||
hasArmAndroidArch(ctx.Config().Targets[Android]))) ||
(arch.ArchType == Arm &&
hasX86AndroidArch(ctx.Config().Targets[Android])) {
field := "Arm_on_x86"
prefix := "target.arm_on_x86"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
}
if (arch.ArchType == X86_64 && (hasArmAbi(arch) ||
hasArmAndroidArch(ctx.Config().Targets[Android]))) ||
(arch.ArchType == Arm &&
hasX8664AndroidArch(ctx.Config().Targets[Android])) {
field := "Arm_on_x86_64"
prefix := "target.arm_on_x86_64"
a.appendProperties(ctx, genProps, targetProp, field, prefix)
}
}
}
}

2
android/module.go
View File

@@ -435,7 +435,7 @@ type ModuleBase struct {
variableProperties variableProperties
hostAndDeviceProperties hostAndDeviceProperties
generalProperties []interface{}
archProperties []interface{}
archProperties [][]interface{}
customizableProperties []interface{}
noAddressSanitizer bool

1
cc/cc.go
View File

@@ -1610,7 +1610,6 @@ func DefaultsFactory(props ...interface{}) android.Module {
&VendorProperties{},
&BaseCompilerProperties{},
&BaseLinkerProperties{},
&MoreBaseLinkerProperties{},
&LibraryProperties{},
&FlagExporterProperties{},
&BinaryLinkerProperties{},

29
cc/linker.go
View File

@@ -118,6 +118,9 @@ type BaseLinkerProperties struct {
// list of runtime libs that should not be installed along with the vendor
// variant of the C/C++ module.
Exclude_runtime_libs []string
// version script for this vendor variant
Version_script *string `android:"arch_variant"`
}
Recovery struct {
// list of shared libs that only should be used to build the recovery
@@ -140,23 +143,12 @@ type BaseLinkerProperties struct {
// make android::build:GetBuildNumber() available containing the build ID.
Use_version_lib *bool `android:"arch_variant"`
}
// TODO(http://b/80437643): BaseLinkerProperties is getting too big,
// more than 2^16 bytes. New properties are defined in MoreBaseLinkerProperties.
type MoreBaseLinkerProperties struct {
// Generate compact dynamic relocation table, default true.
Pack_relocations *bool `android:"arch_variant"`
// local file name to pass to the linker as --version_script
Version_script *string `android:"arch_variant"`
Target struct {
Vendor struct {
// version script for this vendor variant
Version_script *string `android:"arch_variant"`
}
}
}
func NewBaseLinker(sanitize *sanitize) *baseLinker {
@@ -166,7 +158,6 @@ func NewBaseLinker(sanitize *sanitize) *baseLinker {
// baseLinker provides support for shared_libs, static_libs, and whole_static_libs properties
type baseLinker struct {
Properties BaseLinkerProperties
MoreProperties MoreBaseLinkerProperties
dynamicProperties struct {
RunPaths []string `blueprint:"mutated"`
BuildStubs bool `blueprint:"mutated"`
@@ -188,7 +179,7 @@ func (linker *baseLinker) linkerInit(ctx BaseModuleContext) {
}
func (linker *baseLinker) linkerProps() []interface{} {
return []interface{}{&linker.Properties, &linker.MoreProperties, &linker.dynamicProperties}
return []interface{}{&linker.Properties, &linker.dynamicProperties}
}
func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps {
@@ -283,9 +274,9 @@ func (linker *baseLinker) linkerDeps(ctx DepsContext, deps Deps) Deps {
// Version_script is not needed when linking stubs lib where the version
// script is created from the symbol map file.
if !linker.dynamicProperties.BuildStubs {
android.ExtractSourceDeps(ctx, linker.MoreProperties.Version_script)
android.ExtractSourceDeps(ctx, linker.Properties.Version_script)
android.ExtractSourceDeps(ctx,
linker.MoreProperties.Target.Vendor.Version_script)
linker.Properties.Target.Vendor.Version_script)
}
return deps
@@ -319,7 +310,7 @@ func (linker *baseLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags {
if linker.useClangLld(ctx) {
flags.LdFlags = append(flags.LdFlags, fmt.Sprintf("${config.%sGlobalLldflags}", hod))
if !BoolDefault(linker.MoreProperties.Pack_relocations, true) {
if !BoolDefault(linker.Properties.Pack_relocations, true) {
flags.LdFlags = append(flags.LdFlags, "-Wl,--pack-dyn-relocs=none")
}
} else {
@@ -394,11 +385,11 @@ func (linker *baseLinker) linkerFlags(ctx ModuleContext, flags Flags) Flags {
// script is created from the symbol map file.
if !linker.dynamicProperties.BuildStubs {
versionScript := ctx.ExpandOptionalSource(
linker.MoreProperties.Version_script, "version_script")
linker.Properties.Version_script, "version_script")
if ctx.useVndk() && linker.MoreProperties.Target.Vendor.Version_script != nil {
if ctx.useVndk() && linker.Properties.Target.Vendor.Version_script != nil {
versionScript = ctx.ExpandOptionalSource(
linker.MoreProperties.Target.Vendor.Version_script,
linker.Properties.Target.Vendor.Version_script,
"target.vendor.version_script")
}