Hybrid soong/mk build using prebuilts

Change-Id: I96daa69af27bd19c8bfbe327a1318f51c738fc03

Android.bp

@@ -89,6 +89,7 @@ bootstrap_go_package {
         "soong-glob",
     ],
     srcs: [
+        "common/androidmk.go",
         "common/arch.go",
         "common/config.go",
         "common/defaults.go",
@@ -114,6 +115,7 @@ bootstrap_go_package {
         "soong-genrule",
     ],
     srcs: [
+        "cc/androidmk.go",
         "cc/builder.go",
         "cc/cc.go",
         "cc/clang.go",
@@ -171,6 +173,7 @@ bootstrap_go_package {
         "soong-genrule",
     ],
     srcs: [
+        "java/androidmk.go",
         "java/app_builder.go",
         "java/app.go",
         "java/builder.go",

cc/androidmk.go

@@ -0,0 +1,83 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cc
+
+import (
+	"io"
+	"path/filepath"
+	"strings"
+
+	"android/soong/common"
+)
+
+func (c *CCLibrary) AndroidMk() (ret common.AndroidMkData) {
+	if c.static() {
+		ret.Class = "STATIC_LIBRARIES"
+	} else {
+		ret.Class = "SHARED_LIBRARIES"
+	}
+	ret.OutputFile = c.outputFile()
+	ret.Extra = func(name, prefix, outputFile string, arch common.Arch) (ret []string) {
+		exportedIncludes := c.exportedFlags()
+		for i := range exportedIncludes {
+			exportedIncludes[i] = strings.TrimPrefix(exportedIncludes[i], "-I")
+		}
+		if len(exportedIncludes) > 0 {
+			ret = append(ret, "LOCAL_EXPORT_C_INCLUDE_DIRS := "+strings.Join(exportedIncludes, " "))
+		}
+
+		ret = append(ret, "LOCAL_MODULE_SUFFIX := "+filepath.Ext(outputFile))
+		ret = append(ret, "LOCAL_SHARED_LIBRARIES_"+arch.ArchType.String()+" := "+strings.Join(c.savedDepNames.SharedLibs, " "))
+
+		if c.Properties.Relative_install_path != "" {
+			ret = append(ret, "LOCAL_MODULE_RELATIVE_PATH := "+c.Properties.Relative_install_path)
+		}
+
+		// These are already included in LOCAL_SHARED_LIBRARIES
+		ret = append(ret, "LOCAL_CXX_STL := none")
+		ret = append(ret, "LOCAL_SYSTEM_SHARED_LIBRARIES :=")
+
+		return
+	}
+	return
+}
+
+func (c *ccObject) AndroidMk() (ret common.AndroidMkData) {
+	ret.OutputFile = c.outputFile()
+	ret.Custom = func(w io.Writer, name, prefix string) {
+		out := c.outputFile()
+
+		io.WriteString(w, "$("+prefix+"TARGET_OUT_INTERMEDIATE_LIBRARIES)/"+name+objectExtension+": "+out+" | $(ACP)\n")
+		io.WriteString(w, "\t$(copy-file-to-target)\n")
+	}
+	return
+}
+
+func (c *CCBinary) AndroidMk() (ret common.AndroidMkData) {
+	ret.Class = "EXECUTABLES"
+	ret.Extra = func(name, prefix, outputFile string, arch common.Arch) []string {
+		ret := []string{
+			"LOCAL_CXX_STL := none",
+			"LOCAL_SYSTEM_SHARED_LIBRARIES :=",
+			"LOCAL_SHARED_LIBRARIES_" + arch.ArchType.String() + " += " + strings.Join(c.savedDepNames.SharedLibs, " "),
+		}
+		if c.Properties.Relative_install_path != "" {
+			ret = append(ret, "LOCAL_MODULE_RELATIVE_PATH_"+arch.ArchType.String()+" := "+c.Properties.Relative_install_path)
+		}
+		return ret
+	}
+	ret.OutputFile = c.outputFile()
+	return
+}

cc/cc.go

@@ -1078,6 +1078,7 @@ type CCLibrary struct {
 	objFiles      []string
 	exportFlags   []string
 	out           string
+	systemLibs    []string
 
 	LibraryProperties CCLibraryProperties
 }
@@ -1142,6 +1143,8 @@ func (c *CCLibrary) depNames(ctx common.AndroidBaseContext, depNames CCDeps) CCD
 		depNames.SharedLibs = append(depNames.SharedLibs, c.LibraryProperties.Shared.Shared_libs...)
 	}
 
+	c.systemLibs = c.systemSharedLibs(ctx)
+
 	return depNames
 }
 
@@ -1764,8 +1767,8 @@ func CCDefaultsFactory() (blueprint.Module, []interface{}) {
 		&CCUnusedProperties{},
 	}
 
-	_, propertyStructs = common.InitAndroidArchModule(module, common.HostOrDeviceSupported(0),
+	_, propertyStructs = common.InitAndroidArchModule(module, common.HostAndDeviceDefault,
-		common.Multilib(""), propertyStructs...)
+		common.MultilibDefault, propertyStructs...)
 
 	return common.InitDefaultsModule(module, module, propertyStructs...)
 }

common/androidmk.go

@@ -0,0 +1,268 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"android/soong"
+
+	"github.com/google/blueprint"
+)
+
+func init() {
+	soong.RegisterSingletonType("androidmk", AndroidMkSingleton)
+}
+
+type AndroidMkDataProvider interface {
+	AndroidMk() AndroidMkData
+}
+
+type AndroidMkData struct {
+	Class      string
+	OutputFile string
+
+	Custom func(w io.Writer, name, prefix string)
+
+	Extra func(name, prefix, outputFile string, arch Arch) []string
+}
+
+func AndroidMkSingleton() blueprint.Singleton {
+	return &androidMkSingleton{}
+}
+
+type androidMkSingleton struct{}
+
+func (c *androidMkSingleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
+	fileModules := make(map[string][]blueprint.Module)
+	hasBPFile := make(map[string]bool)
+	bpFiles := []string{}
+
+	ctx.SetNinjaBuildDir(pctx, filepath.Join(ctx.Config().(Config).BuildDir(), ".."))
+
+	ctx.VisitAllModules(func(module blueprint.Module) {
+		if _, ok := module.(AndroidModule); ok {
+			bpFile := ctx.BlueprintFile(module)
+
+			if !hasBPFile[bpFile] {
+				hasBPFile[bpFile] = true
+				bpFiles = append(bpFiles, bpFile)
+			}
+
+			fileModules[bpFile] = append(fileModules[bpFile], module)
+		}
+	})
+
+	// Gather list of eligible Android modules for translation
+	androidMkModules := make(map[blueprint.Module]bool)
+	var validBpFiles []string
+	srcDir := ctx.Config().(Config).SrcDir()
+	intermediatesDir := filepath.Join(ctx.Config().(Config).IntermediatesDir(), "androidmk")
+	sort.Strings(bpFiles)
+	for _, origBp := range bpFiles {
+		mkFile := filepath.Join(srcDir, filepath.Dir(origBp), "Android.mk")
+
+		files, err := Glob(ctx, intermediatesDir, mkFile, nil)
+		if err != nil {
+			ctx.Errorf("glob: %s", err.Error())
+			continue
+		}
+
+		// Existing Android.mk file, use that instead
+		if len(files) > 0 {
+			for _, file := range files {
+				ctx.AddNinjaFileDeps(file)
+			}
+			continue
+		}
+
+		validBpFiles = append(validBpFiles, origBp)
+
+		for _, mod := range fileModules[origBp] {
+			androidMkModules[mod] = true
+		}
+	}
+
+	// Validate that all modules have proper dependencies
+	androidMkModulesList := make([]AndroidModule, 0, len(androidMkModules))
+	for mod := range androidMkModules {
+		ctx.VisitDepsDepthFirstIf(mod, isAndroidModule, func(module blueprint.Module) {
+			if !androidMkModules[module] {
+				ctx.Errorf("Module %q missing dependency for Android.mk: %q", ctx.ModuleName(mod), ctx.ModuleName(module))
+			}
+		})
+		if amod, ok := mod.(AndroidModule); ok {
+			androidMkModulesList = append(androidMkModulesList, amod)
+		}
+	}
+
+	transMk := filepath.Join(ctx.Config().(Config).BuildDir(), "Android.mk")
+
+	err := translateAndroidMk(ctx, transMk, androidMkModulesList)
+	if err != nil {
+		ctx.Errorf(err.Error())
+	}
+
+	ctx.Build(pctx, blueprint.BuildParams{
+		Rule:     blueprint.Phony,
+		Outputs:  []string{transMk},
+		Optional: true,
+	})
+}
+
+func translateAndroidMk(ctx blueprint.SingletonContext, mkFile string, mods []AndroidModule) error {
+	buf := &bytes.Buffer{}
+
+	io.WriteString(buf, "LOCAL_PATH := $(TOP)\n")
+	io.WriteString(buf, "LOCAL_MODULE_MAKEFILE := $(lastword $(MAKEFILE_LIST))\n")
+
+	for _, mod := range mods {
+		err := translateAndroidMkModule(ctx, buf, mod)
+		if err != nil {
+			os.Remove(mkFile)
+			return err
+		}
+	}
+
+	// Don't write to the file if it hasn't changed
+	if _, err := os.Stat(mkFile); !os.IsNotExist(err) {
+		if data, err := ioutil.ReadFile(mkFile); err == nil {
+			matches := buf.Len() == len(data)
+
+			if matches {
+				for i, value := range buf.Bytes() {
+					if value != data[i] {
+						matches = false
+						break
+					}
+				}
+			}
+
+			if matches {
+				return nil
+			}
+		}
+	}
+
+	return ioutil.WriteFile(mkFile, buf.Bytes(), 0666)
+}
+
+func translateAndroidMkModule(ctx blueprint.SingletonContext, w io.Writer, mod blueprint.Module) error {
+	if mod != ctx.PrimaryModule(mod) {
+		// These will be handled by the primary module
+		return nil
+	}
+
+	name := ctx.ModuleName(mod)
+
+	type hostClass struct {
+		host     bool
+		class    string
+		multilib string
+	}
+
+	type archSrc struct {
+		arch  Arch
+		src   string
+		extra []string
+	}
+
+	srcs := make(map[hostClass][]archSrc)
+	var modules []hostClass
+
+	ctx.VisitAllModuleVariants(mod, func(m blueprint.Module) {
+		provider, ok := m.(AndroidMkDataProvider)
+		if !ok {
+			return
+		}
+
+		amod := m.(AndroidModule).base()
+		data := provider.AndroidMk()
+
+		arch := amod.commonProperties.CompileArch
+
+		prefix := ""
+		if amod.HostOrDevice() == Host {
+			if arch.ArchType != ctx.Config().(Config).HostArches[amod.HostType()][0].ArchType {
+				prefix = "2ND_"
+			}
+		} else {
+			if arch.ArchType != ctx.Config().(Config).DeviceArches[0].ArchType {
+				prefix = "2ND_"
+			}
+		}
+
+		if data.Custom != nil {
+			data.Custom(w, name, prefix)
+			return
+		}
+
+		hC := hostClass{
+			host:     amod.HostOrDevice() == Host,
+			class:    data.Class,
+			multilib: amod.commonProperties.Compile_multilib,
+		}
+
+		src := archSrc{
+			arch: arch,
+			src:  data.OutputFile,
+		}
+
+		if data.Extra != nil {
+			src.extra = data.Extra(name, prefix, src.src, arch)
+		}
+
+		if srcs[hC] == nil {
+			modules = append(modules, hC)
+		}
+		srcs[hC] = append(srcs[hC], src)
+	})
+
+	for _, hC := range modules {
+		archSrcs := srcs[hC]
+
+		io.WriteString(w, "\ninclude $(CLEAR_VARS)\n")
+		io.WriteString(w, "LOCAL_MODULE := "+name+"\n")
+		io.WriteString(w, "LOCAL_MODULE_CLASS := "+hC.class+"\n")
+		io.WriteString(w, "LOCAL_MULTILIB := "+hC.multilib+"\n")
+
+		printed := make(map[string]bool)
+		for _, src := range archSrcs {
+			io.WriteString(w, "LOCAL_SRC_FILES_"+src.arch.ArchType.String()+" := "+src.src+"\n")
+
+			for _, extra := range src.extra {
+				if !printed[extra] {
+					printed[extra] = true
+					io.WriteString(w, extra+"\n")
+				}
+			}
+		}
+
+		if hC.host {
+			// TODO: this isn't true for every module
+			io.WriteString(w, "LOCAL_ACP_UNAVAILABLE := true\n")
+
+			io.WriteString(w, "LOCAL_IS_HOST_MODULE := true\n")
+		}
+		io.WriteString(w, "include $(BUILD_PREBUILT)\n")
+	}
+
+	return nil
+}

common/arch.go

@@ -335,6 +335,7 @@ const (
 	HostSupported
 	DeviceSupported
 	HostAndDeviceSupported
+	HostAndDeviceDefault
 )
 
 type HostOrDevice int
@@ -505,16 +506,11 @@ func ArchMutator(mctx AndroidBottomUpMutatorContext) {
 		return
 	}
 
-	hostArches, deviceArches, err := decodeArchProductVariables(mctx.Config().(Config).ProductVariables)
-	if err != nil {
-		mctx.ModuleErrorf("%s", err.Error())
-	}
-
 	moduleArches := []Arch{}
 	multilib := module.base().commonProperties.Compile_multilib
 
 	if module.base().HostSupported() && module.base().HostOrDevice().Host() {
-		hostModuleArches, err := decodeMultilib(multilib, hostArches[module.base().HostType()])
+		hostModuleArches, err := decodeMultilib(multilib, mctx.Config().(Config).HostArches[module.base().HostType()])
 		if err != nil {
 			mctx.ModuleErrorf("%s", err.Error())
 		}
@@ -523,7 +519,7 @@ func ArchMutator(mctx AndroidBottomUpMutatorContext) {
 	}
 
 	if module.base().DeviceSupported() && module.base().HostOrDevice().Device() {
-		deviceModuleArches, err := decodeMultilib(multilib, deviceArches)
+		deviceModuleArches, err := decodeMultilib(multilib, mctx.Config().(Config).DeviceArches)
 		if err != nil {
 			mctx.ModuleErrorf("%s", err.Error())
 		}

common/config.go

@@ -38,9 +38,11 @@ func (f *FileConfigurableOptions) SetDefaultConfig() {
 
 type Config struct {
 	*config
+
+	dontCreateNinjaFile bool
 }
 
-// A config object represents the entire build configuration for Blue.
+// A config object represents the entire build configuration for Android.
 type config struct {
 	FileConfigurableOptions
 	ProductVariables productVariables
@@ -48,6 +50,9 @@ type config struct {
 	ConfigFileName           string
 	ProductVariablesFileName string
 
+	DeviceArches []Arch
+	HostArches   map[HostType][]Arch
+
 	srcDir   string // the path of the root source directory
 	buildDir string // the path of the build output directory
 
@@ -143,6 +148,14 @@ func NewConfig(srcDir, buildDir string) (Config, error) {
 		return Config{}, err
 	}
 
+	hostArches, deviceArches, err := decodeArchProductVariables(config.ProductVariables)
+	if err != nil {
+		return Config{}, err
+	}
+
+	config.HostArches = hostArches
+	config.DeviceArches = deviceArches
+
 	return config, nil
 }
 
@@ -158,6 +171,10 @@ func (c *config) IntermediatesDir() string {
 	return filepath.Join(c.BuildDir(), ".intermediates")
 }
 
+func (c *config) RemoveAbandonedFiles() bool {
+	return false
+}
+
 // PrebuiltOS returns the name of the host OS used in prebuilts directories
 func (c *config) PrebuiltOS() string {
 	switch runtime.GOOS {

common/module.go

@@ -108,6 +108,7 @@ const (
 	MultilibBoth    Multilib = "both"
 	MultilibFirst   Multilib = "first"
 	MultilibCommon  Multilib = "common"
+	MultilibDefault Multilib = ""
 )
 
 func InitAndroidModule(m AndroidModule,
@@ -130,10 +131,13 @@ func InitAndroidArchModule(m AndroidModule, hod HostOrDeviceSupported, defaultMu
 	base.commonProperties.HostOrDeviceSupported = hod
 	base.commonProperties.Compile_multilib = string(defaultMultilib)
 
-	if hod == HostAndDeviceSupported {
+	switch hod {
+	case HostAndDeviceSupported:
 		// Default to module to device supported, host not supported, can override in module
 		// properties
 		base.hostAndDeviceProperties.Device_supported = true
+		fallthrough
+	case HostAndDeviceDefault:
 		propertyStructs = append(propertyStructs, &base.hostAndDeviceProperties)
 	}
 
@@ -312,7 +316,7 @@ func (a *AndroidModuleBase) generateModuleTarget(ctx blueprint.ModuleContext) {
 	if len(deps) > 0 {
 		ctx.Build(pctx, blueprint.BuildParams{
 			Rule:      blueprint.Phony,
-			Outputs:   []string{ctx.ModuleName()},
+			Outputs:   []string{ctx.ModuleName() + "-soong"},
 			Implicits: deps,
 			Optional:  true,
 		})
@@ -525,15 +529,12 @@ func (c *buildTargetSingleton) GenerateBuildActions(ctx blueprint.SingletonConte
 	checkbuildDeps := []string{}
 
 	dirModules := make(map[string][]string)
-	hasBPFile := make(map[string]bool)
-	bpFiles := []string{}
 
 	ctx.VisitAllModules(func(module blueprint.Module) {
 		if a, ok := module.(AndroidModule); ok {
 			blueprintDir := a.base().blueprintDir
 			installTarget := a.base().installTarget
 			checkbuildTarget := a.base().checkbuildTarget
-			bpFile := ctx.BlueprintFile(module)
 
 			if checkbuildTarget != "" {
 				checkbuildDeps = append(checkbuildDeps, checkbuildTarget)
@@ -543,21 +544,15 @@ func (c *buildTargetSingleton) GenerateBuildActions(ctx blueprint.SingletonConte
 			if installTarget != "" {
 				dirModules[blueprintDir] = append(dirModules[blueprintDir], installTarget)
 			}
-
-			if !hasBPFile[bpFile] {
-				hasBPFile[bpFile] = true
-				bpFiles = append(bpFiles, bpFile)
-			}
 		}
 	})
 
 	// Create a top-level checkbuild target that depends on all modules
 	ctx.Build(pctx, blueprint.BuildParams{
 		Rule:      blueprint.Phony,
-		Outputs:   []string{"checkbuild"},
+		Outputs:   []string{"checkbuild-soong"},
 		Implicits: checkbuildDeps,
-		// HACK: checkbuild should be an optional build, but force it enabled for now
+		Optional:  true,
-		//Optional:  true,
 	})
 
 	// Create a mm/<directory> target that depends on all modules in a directory

java/androidmk.go

@@ -0,0 +1,31 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package java
+
+import (
+	"android/soong/common"
+)
+
+func (*JavaLibrary) AndroidMk() (ret common.AndroidMkData) {
+	ret.Class = "JAVA_LIBRARIES"
+	// TODO
+	return
+}
+
+func (*JavaPrebuilt) AndroidMk() (ret common.AndroidMkData) {
+	ret.Class = "JAVA_LIBRARIES"
+	// TODO
+	return
+}