aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Android.bp10
-rw-r--r--bootstrap/command.go32
-rw-r--r--bootstrap/glob.go6
-rw-r--r--context.go231
-rw-r--r--context_test.go74
-rw-r--r--go.mod2
-rw-r--r--optional/optional.go58
-rw-r--r--parser/parser.go1
-rw-r--r--proptools/configurable.go114
-rw-r--r--proptools/hash_provider.go45
-rw-r--r--proptools/hash_provider_test.go84
-rw-r--r--proptools/utils.go42
-rw-r--r--provider.go2
-rw-r--r--transition_test.go32
14 files changed, 468 insertions, 265 deletions
diff --git a/Android.bp b/Android.bp
index f3b9e71..ee0ede0 100644
--- a/Android.bp
+++ b/Android.bp
@@ -116,6 +116,7 @@ bootstrap_go_package {
pkgPath: "github.com/google/blueprint/proptools",
deps: [
"blueprint-parser",
+ "blueprint-optional",
],
srcs: [
"proptools/clone.go",
@@ -128,6 +129,7 @@ bootstrap_go_package {
"proptools/tag.go",
"proptools/typeequal.go",
"proptools/unpack.go",
+ "proptools/utils.go",
],
testSrcs: [
"proptools/clone_test.go",
@@ -142,6 +144,14 @@ bootstrap_go_package {
}
bootstrap_go_package {
+ name: "blueprint-optional",
+ pkgPath: "github.com/google/blueprint/optional",
+ srcs: [
+ "optional/optional.go",
+ ],
+}
+
+bootstrap_go_package {
name: "blueprint-bootstrap",
deps: [
"blueprint",
diff --git a/bootstrap/command.go b/bootstrap/command.go
index 580907c..3071e3e 100644
--- a/bootstrap/command.go
+++ b/bootstrap/command.go
@@ -20,7 +20,6 @@ import (
"fmt"
"io"
"os"
- "path/filepath"
"runtime"
"runtime/debug"
"runtime/pprof"
@@ -28,6 +27,7 @@ import (
"strings"
"github.com/google/blueprint"
+ "github.com/google/blueprint/proptools"
)
type Args struct {
@@ -63,7 +63,7 @@ func RunBlueprint(args Args, stopBefore StopBefore, ctx *blueprint.Context, conf
}
if args.Cpuprofile != "" {
- f, err := os.Create(joinPath(ctx.SrcDir(), args.Cpuprofile))
+ f, err := os.Create(blueprint.JoinPath(ctx.SrcDir(), args.Cpuprofile))
if err != nil {
return nil, fmt.Errorf("error opening cpuprofile: %s", err)
}
@@ -73,7 +73,7 @@ func RunBlueprint(args Args, stopBefore StopBefore, ctx *blueprint.Context, conf
}
if args.TraceFile != "" {
- f, err := os.Create(joinPath(ctx.SrcDir(), args.TraceFile))
+ f, err := os.Create(blueprint.JoinPath(ctx.SrcDir(), args.TraceFile))
if err != nil {
return nil, fmt.Errorf("error opening trace: %s", err)
}
@@ -102,7 +102,6 @@ func RunBlueprint(args Args, stopBefore StopBefore, ctx *blueprint.Context, conf
ctx.RegisterBottomUpMutator("bootstrap_plugin_deps", pluginDeps)
ctx.RegisterSingletonType("bootstrap", newSingletonFactory(), false)
RegisterGoModuleTypes(ctx)
- blueprint.RegisterPackageIncludesModuleType(ctx)
ctx.BeginEvent("parse_bp")
if blueprintFiles, errs := ctx.ParseFileList(".", filesToParse, config); len(errs) > 0 {
@@ -151,7 +150,6 @@ func RunBlueprint(args Args, stopBefore StopBefore, ctx *blueprint.Context, conf
providersValidationChan <- nil
}
- const outFilePermissions = 0666
var out blueprint.StringWriterWriter
var f *os.File
var buf *bufio.Writer
@@ -159,12 +157,12 @@ func RunBlueprint(args Args, stopBefore StopBefore, ctx *blueprint.Context, conf
ctx.BeginEvent("write_files")
defer ctx.EndEvent("write_files")
if args.EmptyNinjaFile {
- if err := os.WriteFile(joinPath(ctx.SrcDir(), args.OutFile), []byte(nil), outFilePermissions); err != nil {
+ if err := os.WriteFile(blueprint.JoinPath(ctx.SrcDir(), args.OutFile), []byte(nil), blueprint.OutFilePermissions); err != nil {
return nil, fmt.Errorf("error writing empty Ninja file: %s", err)
}
out = io.Discard.(blueprint.StringWriterWriter)
} else {
- f, err := os.OpenFile(joinPath(ctx.SrcDir(), args.OutFile), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, outFilePermissions)
+ f, err := os.OpenFile(blueprint.JoinPath(ctx.SrcDir(), args.OutFile), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, blueprint.OutFilePermissions)
if err != nil {
return nil, fmt.Errorf("error opening Ninja file: %s", err)
}
@@ -173,7 +171,7 @@ func RunBlueprint(args Args, stopBefore StopBefore, ctx *blueprint.Context, conf
out = buf
}
- if err := ctx.WriteBuildFile(out); err != nil {
+ if err := ctx.WriteBuildFile(out, !strings.Contains(args.OutFile, "bootstrap.ninja") && !args.EmptyNinjaFile, args.OutFile); err != nil {
return nil, fmt.Errorf("error writing Ninja file contents: %s", err)
}
@@ -191,18 +189,11 @@ func RunBlueprint(args Args, stopBefore StopBefore, ctx *blueprint.Context, conf
providerValidationErrors := <-providersValidationChan
if providerValidationErrors != nil {
- var sb strings.Builder
- for i, err := range providerValidationErrors {
- if i != 0 {
- sb.WriteString("\n")
- }
- sb.WriteString(err.Error())
- }
- return nil, errors.New(sb.String())
+ return nil, proptools.MergeErrors(providerValidationErrors)
}
if args.Memprofile != "" {
- f, err := os.Create(joinPath(ctx.SrcDir(), args.Memprofile))
+ f, err := os.Create(blueprint.JoinPath(ctx.SrcDir(), args.Memprofile))
if err != nil {
return nil, fmt.Errorf("error opening memprofile: %s", err)
}
@@ -230,10 +221,3 @@ func fatalErrors(errs []error) error {
return errors.New("fatal errors encountered")
}
-
-func joinPath(base, path string) string {
- if filepath.IsAbs(path) {
- return path
- }
- return filepath.Join(base, path)
-}
diff --git a/bootstrap/glob.go b/bootstrap/glob.go
index b2c3a2d..4611cef 100644
--- a/bootstrap/glob.go
+++ b/bootstrap/glob.go
@@ -196,7 +196,7 @@ func (s *GlobSingleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
// We don't need to write the depfile because we're guaranteed that ninja
// will run the command at least once (to record it into the ninja_log), so
// the depfile will be loaded from that execution.
- absoluteFileListFile := joinPath(s.SrcDir, fileListFile)
+ absoluteFileListFile := blueprint.JoinPath(s.SrcDir, fileListFile)
err := pathtools.WriteFileIfChanged(absoluteFileListFile, globs.FileList(), 0666)
if err != nil {
panic(fmt.Errorf("error writing %s: %s", fileListFile, err))
@@ -217,7 +217,7 @@ func WriteBuildGlobsNinjaFile(glob *GlobSingleton, config interface{}) error {
}
const outFilePermissions = 0666
- err := ioutil.WriteFile(joinPath(glob.SrcDir, glob.GlobFile), buffer, outFilePermissions)
+ err := ioutil.WriteFile(blueprint.JoinPath(glob.SrcDir, glob.GlobFile), buffer, outFilePermissions)
if err != nil {
return fmt.Errorf("error writing %s: %s", glob.GlobFile, err)
}
@@ -251,7 +251,7 @@ func generateGlobNinjaFile(glob *GlobSingleton, config interface{}) ([]byte, []e
}
buf := bytes.NewBuffer(nil)
- err := ctx.WriteBuildFile(buf)
+ err := ctx.WriteBuildFile(buf, false, "")
if err != nil {
return nil, []error{err}
}
diff --git a/context.go b/context.go
index 7932c74..1591b3c 100644
--- a/context.go
+++ b/context.go
@@ -15,6 +15,7 @@
package blueprint
import (
+ "bufio"
"bytes"
"cmp"
"context"
@@ -50,6 +51,8 @@ var ErrBuildActionsNotReady = errors.New("build actions are not ready")
const maxErrors = 10
const MockModuleListFile = "bplist"
+const OutFilePermissions = 0666
+
// A Context contains all the state needed to parse a set of Blueprints files
// and generate a Ninja file. The process of generating a Ninja file proceeds
// through a series of four phases. Each phase corresponds with a some methods
@@ -809,43 +812,10 @@ type shouldVisitFileInfo struct {
// This should be processed before adding any modules to the build graph
func shouldVisitFile(c *Context, file *parser.File) shouldVisitFileInfo {
skippedModules := []string{}
- var blueprintPackageIncludes *PackageIncludes
for _, def := range file.Defs {
switch def := def.(type) {
case *parser.Module:
skippedModules = append(skippedModules, def.Name())
- if def.Type != "blueprint_package_includes" {
- continue
- }
- module, errs := processModuleDef(def, file.Name, c.moduleFactories, nil, c.ignoreUnknownModuleTypes)
- if len(errs) > 0 {
- // This file contains errors in blueprint_package_includes
- // Visit anyways so that we can report errors on other modules in the file
- return shouldVisitFileInfo{
- shouldVisitFile: true,
- errs: errs,
- }
- }
- logicModule, _ := c.cloneLogicModule(module)
- blueprintPackageIncludes = logicModule.(*PackageIncludes)
- }
- }
-
- if blueprintPackageIncludes != nil {
- packageMatches, err := blueprintPackageIncludes.matchesIncludeTags(c)
- if err != nil {
- return shouldVisitFileInfo{
- errs: []error{err},
- }
- } else if !packageMatches {
- return shouldVisitFileInfo{
- shouldVisitFile: false,
- skippedModules: skippedModules,
- reasonForSkip: fmt.Sprintf(
- "module is defined in %q which contains a blueprint_package_includes module with unsatisfied tags",
- file.Name,
- ),
- }
}
}
@@ -1877,7 +1847,7 @@ func (c *Context) findReverseDependency(module *moduleInfo, config any, destName
// modify the requested variation. It finds a variant that existed before the TransitionMutator ran that is
// a subset of the requested variant to use as the module context for IncomingTransition.
func (c *Context) applyTransitions(config any, module *moduleInfo, group *moduleGroup, variant variationMap,
- requestedVariations []Variation) {
+ requestedVariations []Variation) variationMap {
for _, transitionMutator := range c.transitionMutators {
// Apply the outgoing transition if it was not explicitly requested.
explicitlyRequested := slices.ContainsFunc(requestedVariations, func(variation Variation) bool {
@@ -1903,6 +1873,9 @@ func (c *Context) applyTransitions(config any, module *moduleInfo, group *module
}
finalVariation := transitionMutator.mutator.IncomingTransition(ctx, outgoingVariation)
+ if variant == nil {
+ variant = make(variationMap)
+ }
variant[transitionMutator.name] = finalVariation
appliedIncomingTransition = true
break
@@ -1914,6 +1887,8 @@ func (c *Context) applyTransitions(config any, module *moduleInfo, group *module
delete(variant, transitionMutator.name)
}
}
+
+ return variant
}
func (c *Context) findVariant(module *moduleInfo, config any,
@@ -1940,7 +1915,7 @@ func (c *Context) findVariant(module *moduleInfo, config any,
newVariant[v.Mutator] = v.Variation
}
- c.applyTransitions(config, module, possibleDeps, newVariant, requestedVariations)
+ newVariant = c.applyTransitions(config, module, possibleDeps, newVariant, requestedVariations)
check := func(variant variationMap) bool {
if far {
@@ -4135,23 +4110,50 @@ func (c *Context) VerifyProvidersWereUnchanged() []error {
if !c.buildActionsReady {
return []error{ErrBuildActionsNotReady}
}
- var errors []error
- for _, m := range c.modulesSorted {
- for i, provider := range m.providers {
- if provider != nil {
- hash, err := proptools.HashProvider(provider)
- if err != nil {
- errors = append(errors, fmt.Errorf("provider %q on module %q was modified after being set, and no longer hashable afterwards: %s", providerRegistry[i].typ, m.Name(), err.Error()))
- continue
- }
- if provider != nil && m.providerInitialValueHashes[i] != hash {
- errors = append(errors, fmt.Errorf("provider %q on module %q was modified after being set", providerRegistry[i].typ, m.Name()))
+ toProcess := make(chan *moduleInfo)
+ errorCh := make(chan []error)
+ var wg sync.WaitGroup
+ go func() {
+ for _, m := range c.modulesSorted {
+ toProcess <- m
+ }
+ close(toProcess)
+ }()
+ for i := 0; i < 1000; i++ {
+ wg.Add(1)
+ go func() {
+ var errors []error
+ for m := range toProcess {
+ for i, provider := range m.providers {
+ if provider != nil {
+ hash, err := proptools.CalculateHash(provider)
+ if err != nil {
+ errors = append(errors, fmt.Errorf("provider %q on module %q was modified after being set, and no longer hashable afterwards: %s", providerRegistry[i].typ, m.Name(), err.Error()))
+ continue
+ }
+ if m.providerInitialValueHashes[i] != hash {
+ errors = append(errors, fmt.Errorf("provider %q on module %q was modified after being set", providerRegistry[i].typ, m.Name()))
+ }
+ } else if m.providerInitialValueHashes[i] != 0 {
+ // This should be unreachable, because in setProvider we check if the provider has already been set.
+ errors = append(errors, fmt.Errorf("provider %q on module %q was unset somehow, this is an internal error", providerRegistry[i].typ, m.Name()))
+ }
}
- } else if m.providerInitialValueHashes[i] != 0 {
- // This should be unreachable, because in setProvider we check if the provider has already been set.
- errors = append(errors, fmt.Errorf("provider %q on module %q was unset somehow, this is an internal error", providerRegistry[i].typ, m.Name()))
}
- }
+ if errors != nil {
+ errorCh <- errors
+ }
+ wg.Done()
+ }()
+ }
+ go func() {
+ wg.Wait()
+ close(errorCh)
+ }()
+
+ var errors []error
+ for newErrors := range errorCh {
+ errors = append(errors, newErrors...)
}
return errors
}
@@ -4159,7 +4161,7 @@ func (c *Context) VerifyProvidersWereUnchanged() []error {
// WriteBuildFile writes the Ninja manifest text for the generated build
// actions to w. If this is called before PrepareBuildActions successfully
// completes then ErrBuildActionsNotReady is returned.
-func (c *Context) WriteBuildFile(w StringWriterWriter) error {
+func (c *Context) WriteBuildFile(w StringWriterWriter, shardNinja bool, ninjaFileName string) error {
var err error
pprof.Do(c.Context, pprof.Labels("blueprint", "WriteBuildFile"), func(ctx context.Context) {
if !c.buildActionsReady {
@@ -4199,7 +4201,7 @@ func (c *Context) WriteBuildFile(w StringWriterWriter) error {
return
}
- if err = c.writeAllModuleActions(nw); err != nil {
+ if err = c.writeAllModuleActions(nw, shardNinja, ninjaFileName); err != nil {
return
}
@@ -4468,14 +4470,19 @@ func (s moduleSorter) Swap(i, j int) {
s.modules[i], s.modules[j] = s.modules[j], s.modules[i]
}
-func (c *Context) writeAllModuleActions(nw *ninjaWriter) error {
+func GetNinjaShardFiles(ninjaFile string) []string {
+ ninjaShardCnt := 10
+ fileNames := make([]string, ninjaShardCnt)
+
+ for i := 0; i < ninjaShardCnt; i++ {
+ fileNames[i] = fmt.Sprintf("%s.%d", ninjaFile, i)
+ }
+ return fileNames
+}
+
+func (c *Context) writeAllModuleActions(nw *ninjaWriter, shardNinja bool, ninjaFileName string) error {
c.BeginEvent("modules")
defer c.EndEvent("modules")
- headerTemplate := template.New("moduleHeader")
- if _, err := headerTemplate.Parse(moduleHeaderTemplate); err != nil {
- // This is a programming error.
- panic(err)
- }
modules := make([]*moduleInfo, 0, len(c.moduleInfo))
for _, module := range c.moduleInfo {
@@ -4488,6 +4495,67 @@ func (c *Context) writeAllModuleActions(nw *ninjaWriter) error {
return err
}
+ headerTemplate := template.New("moduleHeader")
+ if _, err := headerTemplate.Parse(moduleHeaderTemplate); err != nil {
+ // This is a programming error.
+ panic(err)
+ }
+
+ if shardNinja {
+ var wg sync.WaitGroup
+ errorCh := make(chan error)
+ files := GetNinjaShardFiles(ninjaFileName)
+ shardedModules := proptools.ShardByCount(modules, len(files))
+ for i, batchModules := range shardedModules {
+ file := files[i]
+ wg.Add(1)
+ go func(file string, batchModules []*moduleInfo) {
+ defer wg.Done()
+ f, err := os.OpenFile(JoinPath(c.SrcDir(), file), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, OutFilePermissions)
+ if err != nil {
+ errorCh <- fmt.Errorf("error opening Ninja file shard: %s", err)
+ return
+ }
+ defer func() {
+ err := f.Close()
+ if err != nil {
+ errorCh <- err
+ }
+ }()
+ buf := bufio.NewWriterSize(f, 16*1024*1024)
+ defer func() {
+ err := buf.Flush()
+ if err != nil {
+ errorCh <- err
+ }
+ }()
+ writer := newNinjaWriter(buf)
+ err = c.writeModuleAction(batchModules, writer, headerTemplate)
+ if err != nil {
+ errorCh <- err
+ }
+ }(file, batchModules)
+ nw.Subninja(file)
+ }
+ go func() {
+ wg.Wait()
+ close(errorCh)
+ }()
+
+ var errors []error
+ for newErrors := range errorCh {
+ errors = append(errors, newErrors)
+ }
+ if len(errors) > 0 {
+ return proptools.MergeErrors(errors)
+ }
+ return nil
+ } else {
+ return c.writeModuleAction(modules, nw, headerTemplate)
+ }
+}
+
+func (c *Context) writeModuleAction(modules []*moduleInfo, nw *ninjaWriter, headerTemplate *template.Template) error {
buf := bytes.NewBuffer(nil)
for _, module := range modules {
@@ -5113,48 +5181,9 @@ Singleton: {{.name}}
Factory: {{.goFactory}}
`
-// Blueprint module type that can be used to gate blueprint files beneath this directory
-type PackageIncludes struct {
- properties struct {
- // Package will be included if all include tags in this list are set
- Match_all []string
- }
- name *string `blueprint:"mutated"`
-}
-
-func (pi *PackageIncludes) Name() string {
- return proptools.String(pi.name)
-}
-
-// This module type does not have any build actions
-func (pi *PackageIncludes) GenerateBuildActions(ctx ModuleContext) {
-}
-
-func newPackageIncludesFactory() (Module, []interface{}) {
- module := &PackageIncludes{}
- AddLoadHook(module, func(ctx LoadHookContext) {
- module.name = proptools.StringPtr(ctx.ModuleDir() + "_includes") // Generate a synthetic name
- })
- return module, []interface{}{&module.properties}
-}
-
-func RegisterPackageIncludesModuleType(ctx *Context) {
- ctx.RegisterModuleType("blueprint_package_includes", newPackageIncludesFactory)
-}
-
-func (pi *PackageIncludes) MatchAll() []string {
- return pi.properties.Match_all
-}
-
-// Returns true if all requested include tags are set in the Context object
-func (pi *PackageIncludes) matchesIncludeTags(ctx *Context) (bool, error) {
- if len(pi.MatchAll()) == 0 {
- return false, ctx.ModuleErrorf(pi, "Match_all must be a non-empty list")
- }
- for _, includeTag := range pi.MatchAll() {
- if !ctx.ContainsIncludeTag(includeTag) {
- return false, nil
- }
+func JoinPath(base, path string) string {
+ if filepath.IsAbs(path) {
+ return path
}
- return true, nil
+ return filepath.Join(base, path)
}
diff --git a/context_test.go b/context_test.go
index e69f5da..d43b243 100644
--- a/context_test.go
+++ b/context_test.go
@@ -19,7 +19,6 @@ import (
"errors"
"fmt"
"hash/fnv"
- "path/filepath"
"reflect"
"strconv"
"strings"
@@ -1089,78 +1088,6 @@ func Test_parallelVisit(t *testing.T) {
})
}
-func TestPackageIncludes(t *testing.T) {
- dir1_foo_bp := `
- blueprint_package_includes {
- match_all: ["use_dir1"],
- }
- foo_module {
- name: "foo",
- }
- `
- dir2_foo_bp := `
- blueprint_package_includes {
- match_all: ["use_dir2"],
- }
- foo_module {
- name: "foo",
- }
- `
- mockFs := map[string][]byte{
- "dir1/Android.bp": []byte(dir1_foo_bp),
- "dir2/Android.bp": []byte(dir2_foo_bp),
- }
- testCases := []struct {
- desc string
- includeTags []string
- expectedDir string
- expectedErr string
- }{
- {
- desc: "use_dir1 is set, use dir1 foo",
- includeTags: []string{"use_dir1"},
- expectedDir: "dir1",
- },
- {
- desc: "use_dir2 is set, use dir2 foo",
- includeTags: []string{"use_dir2"},
- expectedDir: "dir2",
- },
- {
- desc: "duplicate module error if both use_dir1 and use_dir2 are set",
- includeTags: []string{"use_dir1", "use_dir2"},
- expectedDir: "",
- expectedErr: `module "foo" already defined`,
- },
- }
- for _, tc := range testCases {
- t.Run(tc.desc, func(t *testing.T) {
- ctx := NewContext()
- // Register mock FS
- ctx.MockFileSystem(mockFs)
- // Register module types
- ctx.RegisterModuleType("foo_module", newFooModule)
- RegisterPackageIncludesModuleType(ctx)
- // Add include tags for test case
- ctx.AddIncludeTags(tc.includeTags...)
- // Run test
- _, actualErrs := ctx.ParseFileList(".", []string{"dir1/Android.bp", "dir2/Android.bp"}, nil)
- // Evaluate
- if !strings.Contains(fmt.Sprintf("%s", actualErrs), fmt.Sprintf("%s", tc.expectedErr)) {
- t.Errorf("Expected errors: %s, got errors: %s\n", tc.expectedErr, actualErrs)
- }
- if tc.expectedErr != "" {
- return // expectedDir check not necessary
- }
- actualBpFile := ctx.moduleGroupFromName("foo", nil).modules.firstModule().relBlueprintsFile
- if tc.expectedDir != filepath.Dir(actualBpFile) {
- t.Errorf("Expected foo from %s, got %s\n", tc.expectedDir, filepath.Dir(actualBpFile))
- }
- })
- }
-
-}
-
func TestDeduplicateOrderOnlyDeps(t *testing.T) {
b := func(output string, inputs []string, orderOnlyDeps []string) *buildDef {
return &buildDef{
@@ -1532,7 +1459,6 @@ func TestSourceRootDirs(t *testing.T) {
ctx.RegisterModuleType("foo_module", newFooModule)
ctx.RegisterBottomUpMutator("deps", depsMutator)
ctx.AddSourceRootDirs(tc.sourceRootDirs...)
- RegisterPackageIncludesModuleType(ctx)
ctx.ParseFileList(".", fileList, nil)
_, actualErrs := ctx.ResolveDependencies(nil)
diff --git a/go.mod b/go.mod
index 7b8869e..9caab2c 100644
--- a/go.mod
+++ b/go.mod
@@ -1,3 +1,3 @@
module github.com/google/blueprint
-go 1.20
+go 1.22
diff --git a/optional/optional.go b/optional/optional.go
new file mode 100644
index 0000000..0d7ef52
--- /dev/null
+++ b/optional/optional.go
@@ -0,0 +1,58 @@
+// Copyright 2023 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package optional
+
+// ShallowOptional is an optional type that can be constructed from a pointer.
+// It will not copy the pointer, and its size is the same size as a single pointer.
+// It can be used to prevent a downstream consumer from modifying the value through
+// the pointer, but is not suitable for an Optional type with stronger value semantics
+// like you would expect from C++ or Rust Optionals.
+type ShallowOptional[T any] struct {
+ inner *T
+}
+
+// NewShallowOptional creates a new ShallowOptional from a pointer.
+// The pointer will not be copied, the object could be changed by the calling
+// code after the optional was created.
+func NewShallowOptional[T any](inner *T) ShallowOptional[T] {
+ return ShallowOptional[T]{inner: inner}
+}
+
+// IsPresent returns true if the optional contains a value
+func (o *ShallowOptional[T]) IsPresent() bool {
+ return o.inner != nil
+}
+
+// IsEmpty returns true if the optional does not have a value
+func (o *ShallowOptional[T]) IsEmpty() bool {
+ return o.inner == nil
+}
+
+// Get() returns the value inside the optional. It panics if IsEmpty() returns true
+func (o *ShallowOptional[T]) Get() T {
+ if o.inner == nil {
+ panic("tried to get an empty optional")
+ }
+ return *o.inner
+}
+
+// GetOrDefault() returns the value inside the optional if IsPresent() returns true,
+// or the provided value otherwise.
+func (o *ShallowOptional[T]) GetOrDefault(other T) T {
+ if o.inner == nil {
+ return other
+ }
+ return *o.inner
+}
diff --git a/parser/parser.go b/parser/parser.go
index eaed054..81dafc9 100644
--- a/parser/parser.go
+++ b/parser/parser.go
@@ -390,6 +390,7 @@ func (p *parser) evaluateOperator(value1, value2 Expression, operator rune,
Cases: []*SelectCase{{
Value: e1,
}},
+ ExpressionType: e1.Type(),
}
}
}
diff --git a/proptools/configurable.go b/proptools/configurable.go
index 06b39a5..031d965 100644
--- a/proptools/configurable.go
+++ b/proptools/configurable.go
@@ -19,8 +19,37 @@ import (
"slices"
"strconv"
"strings"
+
+ "github.com/google/blueprint/optional"
)
+// ConfigurableOptional is the same as ShallowOptional, but we use this separate
+// name to reserve the ability to switch to an alternative implementation later.
+type ConfigurableOptional[T any] struct {
+ shallowOptional optional.ShallowOptional[T]
+}
+
+// IsPresent returns true if the optional contains a value
+func (o *ConfigurableOptional[T]) IsPresent() bool {
+ return o.shallowOptional.IsPresent()
+}
+
+// IsEmpty returns true if the optional does not have a value
+func (o *ConfigurableOptional[T]) IsEmpty() bool {
+ return o.shallowOptional.IsEmpty()
+}
+
+// Get() returns the value inside the optional. It panics if IsEmpty() returns true
+func (o *ConfigurableOptional[T]) Get() T {
+ return o.shallowOptional.Get()
+}
+
+// GetOrDefault() returns the value inside the optional if IsPresent() returns true,
+// or the provided value otherwise.
+func (o *ConfigurableOptional[T]) GetOrDefault(other T) T {
+ return o.shallowOptional.GetOrDefault(other)
+}
+
type ConfigurableElements interface {
string | bool | []string
}
@@ -381,10 +410,9 @@ func NewConfigurable[T ConfigurableElements](conditions []ConfigurableCondition,
// Get returns the final value for the configurable property.
// A configurable property may be unset, in which case Get will return nil.
-func (c *Configurable[T]) Get(evaluator ConfigurableEvaluator) *T {
+func (c *Configurable[T]) Get(evaluator ConfigurableEvaluator) ConfigurableOptional[T] {
result := c.inner.evaluate(c.propertyName, evaluator)
- // Copy the result so that it can't be changed from soong
- return copyConfiguredValue(result)
+ return configuredValuePtrToOptional(result)
}
// GetOrDefault is the same as Get, but will return the provided default value if the property was unset.
@@ -439,6 +467,7 @@ func (c *singleConfigurable[T]) evaluateNonTransitive(propertyName string, evalu
values[i] = evaluator.EvaluateConfiguration(condition, propertyName)
}
foundMatch := false
+ nonMatchingIndex := 0
var result *T
for _, case_ := range c.cases {
allMatch := true
@@ -449,6 +478,7 @@ func (c *singleConfigurable[T]) evaluateNonTransitive(propertyName string, evalu
}
if !pat.matchesValue(values[i]) {
allMatch = false
+ nonMatchingIndex = i
break
}
}
@@ -460,7 +490,8 @@ func (c *singleConfigurable[T]) evaluateNonTransitive(propertyName string, evalu
if foundMatch {
return result
}
- evaluator.PropertyErrorf(propertyName, "%s had value %s, which was not handled by the select statement", c.conditions, values)
+
+ evaluator.PropertyErrorf(propertyName, "%s had value %s, which was not handled by the select statement", c.conditions[nonMatchingIndex].String(), values[nonMatchingIndex].String())
return nil
}
@@ -526,6 +557,7 @@ type configurableReflection interface {
configuredType() reflect.Type
clone() any
isEmpty() bool
+ printfInto(value string) error
}
// Same as configurableReflection, but since initialize needs to take a pointer
@@ -596,6 +628,60 @@ func (c *configurableInner[T]) setAppend(append *configurableInner[T], replace b
}
}
+func (c Configurable[T]) printfInto(value string) error {
+ return c.inner.printfInto(value)
+}
+
+func (c *configurableInner[T]) printfInto(value string) error {
+ for c != nil {
+ if err := c.single.printfInto(value); err != nil {
+ return err
+ }
+ c = c.next
+ }
+ return nil
+}
+
+func (c *singleConfigurable[T]) printfInto(value string) error {
+ for _, c := range c.cases {
+ if c.value == nil {
+ continue
+ }
+ switch v := any(c.value).(type) {
+ case *string:
+ if err := printfIntoString(v, value); err != nil {
+ return err
+ }
+ case *[]string:
+ for i := range *v {
+ if err := printfIntoString(&((*v)[i]), value); err != nil {
+ return err
+ }
+ }
+ }
+ }
+ return nil
+}
+
+func printfIntoString(s *string, configValue string) error {
+ count := strings.Count(*s, "%")
+ if count == 0 {
+ return nil
+ }
+
+ if count > 1 {
+ return fmt.Errorf("list/value variable properties only support a single '%%'")
+ }
+
+ if !strings.Contains(*s, "%s") {
+ return fmt.Errorf("unsupported %% in value variable property")
+ }
+
+ *s = fmt.Sprintf(*s, configValue)
+
+ return nil
+}
+
func (c Configurable[T]) clone() any {
return Configurable[T]{
propertyName: c.propertyName,
@@ -682,6 +768,19 @@ func copyConfiguredValue[T ConfigurableElements](t *T) *T {
}
}
+func configuredValuePtrToOptional[T ConfigurableElements](t *T) ConfigurableOptional[T] {
+ if t == nil {
+ return ConfigurableOptional[T]{optional.NewShallowOptional(t)}
+ }
+ switch t2 := any(*t).(type) {
+ case []string:
+ result := any(slices.Clone(t2)).(T)
+ return ConfigurableOptional[T]{optional.NewShallowOptional(&result)}
+ default:
+ return ConfigurableOptional[T]{optional.NewShallowOptional(t)}
+ }
+}
+
func copyAndDereferenceConfiguredValue[T ConfigurableElements](t *T) T {
switch t2 := any(*t).(type) {
case []string:
@@ -690,3 +789,10 @@ func copyAndDereferenceConfiguredValue[T ConfigurableElements](t *T) T {
return *t
}
}
+
+// PrintfIntoConfigurable replaces %s occurrences in strings in Configurable properties
+// with the provided string value. It's intention is to support soong config value variables
+// on Configurable properties.
+func PrintfIntoConfigurable(c any, value string) error {
+ return c.(configurableReflection).printfInto(value)
+}
diff --git a/proptools/hash_provider.go b/proptools/hash_provider.go
index 6205bc7..c75bb7f 100644
--- a/proptools/hash_provider.go
+++ b/proptools/hash_provider.go
@@ -18,32 +18,31 @@ import (
"cmp"
"encoding/binary"
"fmt"
- "hash/maphash"
+ "hash"
+ "hash/fnv"
"math"
"reflect"
"sort"
+ "unsafe"
)
-var seed maphash.Seed = maphash.MakeSeed()
-
// byte to insert between elements of lists, fields of structs/maps, etc in order
// to try and make sure the hash is different when values are moved around between
// elements. 36 is arbitrary, but it's the ascii code for a record separator
var recordSeparator []byte = []byte{36}
-func HashProvider(provider interface{}) (uint64, error) {
- hasher := maphash.Hash{}
- hasher.SetSeed(seed)
+func CalculateHash(value interface{}) (uint64, error) {
+ hasher := fnv.New64()
ptrs := make(map[uintptr]bool)
- v := reflect.ValueOf(provider)
+ v := reflect.ValueOf(value)
var err error
if v.IsValid() {
- err = hashProviderInternal(&hasher, v, ptrs)
+ err = calculateHashInternal(hasher, v, ptrs)
}
return hasher.Sum64(), err
}
-func hashProviderInternal(hasher *maphash.Hash, v reflect.Value, ptrs map[uintptr]bool) error {
+func calculateHashInternal(hasher hash.Hash64, v reflect.Value, ptrs map[uintptr]bool) error {
var int64Array [8]byte
int64Buf := int64Array[:]
binary.LittleEndian.PutUint64(int64Buf, uint64(v.Kind()))
@@ -55,7 +54,7 @@ func hashProviderInternal(hasher *maphash.Hash, v reflect.Value, ptrs map[uintpt
hasher.Write(int64Buf)
for i := 0; i < v.NumField(); i++ {
hasher.Write(recordSeparator)
- err := hashProviderInternal(hasher, v.Field(i), ptrs)
+ err := calculateHashInternal(hasher, v.Field(i), ptrs)
if err != nil {
return fmt.Errorf("in field %s: %s", v.Type().Field(i).Name, err.Error())
}
@@ -77,12 +76,12 @@ func hashProviderInternal(hasher *maphash.Hash, v reflect.Value, ptrs map[uintpt
})
for i := 0; i < v.Len(); i++ {
hasher.Write(recordSeparator)
- err := hashProviderInternal(hasher, keys[indexes[i]], ptrs)
+ err := calculateHashInternal(hasher, keys[indexes[i]], ptrs)
if err != nil {
return fmt.Errorf("in map: %s", err.Error())
}
hasher.Write(recordSeparator)
- err = hashProviderInternal(hasher, keys[indexes[i]], ptrs)
+ err = calculateHashInternal(hasher, keys[indexes[i]], ptrs)
if err != nil {
return fmt.Errorf("in map: %s", err.Error())
}
@@ -92,7 +91,7 @@ func hashProviderInternal(hasher *maphash.Hash, v reflect.Value, ptrs map[uintpt
hasher.Write(int64Buf)
for i := 0; i < v.Len(); i++ {
hasher.Write(recordSeparator)
- err := hashProviderInternal(hasher, v.Index(i), ptrs)
+ err := calculateHashInternal(hasher, v.Index(i), ptrs)
if err != nil {
return fmt.Errorf("in %s at index %d: %s", v.Kind().String(), i, err.Error())
}
@@ -103,15 +102,16 @@ func hashProviderInternal(hasher *maphash.Hash, v reflect.Value, ptrs map[uintpt
hasher.Write(int64Buf[:1])
return nil
}
- addr := v.Pointer()
- binary.LittleEndian.PutUint64(int64Buf, uint64(addr))
+ // Hardcoded value to indicate it is a pointer
+ binary.LittleEndian.PutUint64(int64Buf, uint64(0x55))
hasher.Write(int64Buf)
+ addr := v.Pointer()
if _, ok := ptrs[addr]; ok {
// We could make this an error if we want to disallow pointer cycles in the future
return nil
}
ptrs[addr] = true
- err := hashProviderInternal(hasher, v.Elem(), ptrs)
+ err := calculateHashInternal(hasher, v.Elem(), ptrs)
if err != nil {
return fmt.Errorf("in pointer: %s", err.Error())
}
@@ -122,13 +122,20 @@ func hashProviderInternal(hasher *maphash.Hash, v reflect.Value, ptrs map[uintpt
} else {
// The only way get the pointer out of an interface to hash it or check for cycles
// would be InterfaceData(), but that's deprecated and seems like it has undefined behavior.
- err := hashProviderInternal(hasher, v.Elem(), ptrs)
+ err := calculateHashInternal(hasher, v.Elem(), ptrs)
if err != nil {
return fmt.Errorf("in interface: %s", err.Error())
}
}
case reflect.String:
- hasher.WriteString(v.String())
+ strLen := len(v.String())
+ if strLen == 0 {
+ // unsafe.StringData is unspecified in this case
+ int64Buf[0] = 0
+ hasher.Write(int64Buf[:1])
+ return nil
+ }
+ hasher.Write(unsafe.Slice(unsafe.StringData(v.String()), strLen))
case reflect.Bool:
if v.Bool() {
int64Buf[0] = 1
@@ -146,7 +153,7 @@ func hashProviderInternal(hasher *maphash.Hash, v reflect.Value, ptrs map[uintpt
binary.LittleEndian.PutUint64(int64Buf, math.Float64bits(v.Float()))
hasher.Write(int64Buf)
default:
- return fmt.Errorf("providers may only contain primitives, strings, arrays, slices, structs, maps, and pointers, found: %s", v.Kind().String())
+ return fmt.Errorf("data may only contain primitives, strings, arrays, slices, structs, maps, and pointers, found: %s", v.Kind().String())
}
return nil
}
diff --git a/proptools/hash_provider_test.go b/proptools/hash_provider_test.go
index 1c97aec..338c6e4 100644
--- a/proptools/hash_provider_test.go
+++ b/proptools/hash_provider_test.go
@@ -5,9 +5,9 @@ import (
"testing"
)
-func mustHash(t *testing.T, provider interface{}) uint64 {
+func mustHash(t *testing.T, data interface{}) uint64 {
t.Helper()
- result, err := HashProvider(provider)
+ result, err := CalculateHash(data)
if err != nil {
t.Fatal(err)
}
@@ -15,11 +15,11 @@ func mustHash(t *testing.T, provider interface{}) uint64 {
}
func TestHashingMapGetsSameResults(t *testing.T) {
- provider := map[string]string{"foo": "bar", "baz": "qux"}
- first := mustHash(t, provider)
- second := mustHash(t, provider)
- third := mustHash(t, provider)
- fourth := mustHash(t, provider)
+ data := map[string]string{"foo": "bar", "baz": "qux"}
+ first := mustHash(t, data)
+ second := mustHash(t, data)
+ third := mustHash(t, data)
+ fourth := mustHash(t, data)
if first != second || second != third || third != fourth {
t.Fatal("Did not get the same result every time for a map")
}
@@ -27,29 +27,29 @@ func TestHashingMapGetsSameResults(t *testing.T) {
func TestHashingNonSerializableTypesFails(t *testing.T) {
testCases := []struct {
- name string
- provider interface{}
+ name string
+ data interface{}
}{
{
- name: "function pointer",
- provider: []func(){nil},
+ name: "function pointer",
+ data: []func(){nil},
},
{
- name: "channel",
- provider: []chan int{make(chan int)},
+ name: "channel",
+ data: []chan int{make(chan int)},
},
{
- name: "list with non-serializable type",
- provider: []interface{}{"foo", make(chan int)},
+ name: "list with non-serializable type",
+ data: []interface{}{"foo", make(chan int)},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
- _, err := HashProvider(testCase)
+ _, err := CalculateHash(testCase)
if err == nil {
t.Fatal("Expected hashing error but didn't get one")
}
- expected := "providers may only contain primitives, strings, arrays, slices, structs, maps, and pointers"
+ expected := "data may only contain primitives, strings, arrays, slices, structs, maps, and pointers"
if !strings.Contains(err.Error(), expected) {
t.Fatalf("Expected %q, got %q", expected, err.Error())
}
@@ -59,32 +59,32 @@ func TestHashingNonSerializableTypesFails(t *testing.T) {
func TestHashSuccessful(t *testing.T) {
testCases := []struct {
- name string
- provider interface{}
+ name string
+ data interface{}
}{
{
- name: "int",
- provider: 5,
+ name: "int",
+ data: 5,
},
{
- name: "string",
- provider: "foo",
+ name: "string",
+ data: "foo",
},
{
- name: "*string",
- provider: StringPtr("foo"),
+ name: "*string",
+ data: StringPtr("foo"),
},
{
- name: "array",
- provider: [3]string{"foo", "bar", "baz"},
+ name: "array",
+ data: [3]string{"foo", "bar", "baz"},
},
{
- name: "slice",
- provider: []string{"foo", "bar", "baz"},
+ name: "slice",
+ data: []string{"foo", "bar", "baz"},
},
{
name: "struct",
- provider: struct {
+ data: struct {
foo string
bar int
}{
@@ -94,19 +94,35 @@ func TestHashSuccessful(t *testing.T) {
},
{
name: "map",
- provider: map[string]int{
+ data: map[string]int{
"foo": 3,
"bar": 4,
},
},
{
- name: "list of interfaces with different types",
- provider: []interface{}{"foo", 3, []string{"bar", "baz"}},
+ name: "list of interfaces with different types",
+ data: []interface{}{"foo", 3, []string{"bar", "baz"}},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
- mustHash(t, testCase.provider)
+ mustHash(t, testCase.data)
})
}
}
+
+func TestHashingDereferencePointers(t *testing.T) {
+ str1 := "this is a hash test for pointers"
+ str2 := "this is a hash test for pointers"
+ data := []struct {
+ content *string
+ }{
+ {content: &str1},
+ {content: &str2},
+ }
+ first := mustHash(t, data[0])
+ second := mustHash(t, data[1])
+ if first != second {
+ t.Fatal("Got different results for the same string")
+ }
+}
diff --git a/proptools/utils.go b/proptools/utils.go
new file mode 100644
index 0000000..0afb5a4
--- /dev/null
+++ b/proptools/utils.go
@@ -0,0 +1,42 @@
+package proptools
+
+import (
+ "errors"
+ "math"
+ "strings"
+)
+
+func ShardBySize[T ~[]E, E any](toShard T, shardSize int) []T {
+ if len(toShard) == 0 {
+ return nil
+ }
+
+ ret := make([]T, 0, (len(toShard)+shardSize-1)/shardSize)
+ for len(toShard) > shardSize {
+ ret = append(ret, toShard[0:shardSize])
+ toShard = toShard[shardSize:]
+ }
+ if len(toShard) > 0 {
+ ret = append(ret, toShard)
+ }
+ return ret
+}
+
+func ShardByCount[T ~[]E, E any](toShard T, shardCount int) []T {
+ return ShardBySize(toShard, int(math.Ceil(float64(len(toShard))/float64(shardCount))))
+}
+
+// MergeErrors merges a list of errors into a single error.
+func MergeErrors(errs []error) error {
+ if errs != nil {
+ var sb strings.Builder
+ for i, err := range errs {
+ if i != 0 {
+ sb.WriteString("\n")
+ }
+ sb.WriteString(err.Error())
+ }
+ return errors.New(sb.String())
+ }
+ return nil
+}
diff --git a/provider.go b/provider.go
index 297861e..b2e0876 100644
--- a/provider.go
+++ b/provider.go
@@ -158,7 +158,7 @@ func (c *Context) setProvider(m *moduleInfo, provider *providerKey, value any) {
if m.providerInitialValueHashes == nil {
m.providerInitialValueHashes = make([]uint64, len(providerRegistry))
}
- hash, err := proptools.HashProvider(value)
+ hash, err := proptools.CalculateHash(value)
if err != nil {
panic(fmt.Sprintf("Can't set value of provider %s: %s", provider.typ, err.Error()))
}
diff --git a/transition_test.go b/transition_test.go
index a380b86..7c6e1f4 100644
--- a/transition_test.go
+++ b/transition_test.go
@@ -86,7 +86,18 @@ const testTransitionBp = `
}
transition_module {
- name: "F"
+ name: "F",
+ }
+
+ transition_module {
+ name: "G",
+ outgoing: "h",
+ %s
+ }
+
+ transition_module {
+ name: "H",
+ split: ["h"],
}
`
@@ -128,7 +139,7 @@ func checkTransitionMutate(t *testing.T, m *transitionModule, variant string) {
}
func TestTransition(t *testing.T) {
- ctx, errs := testTransition(fmt.Sprintf(testTransitionBp, ""))
+ ctx, errs := testTransition(fmt.Sprintf(testTransitionBp, "", ""))
assertNoErrors(t, errs)
// Module A uses Split to create a and b variants
@@ -154,6 +165,8 @@ func TestTransition(t *testing.T) {
D_d := getTransitionModule(ctx, "D", "d")
E_d := getTransitionModule(ctx, "E", "d")
F := getTransitionModule(ctx, "F", "")
+ G := getTransitionModule(ctx, "G", "")
+ H_h := getTransitionModule(ctx, "H", "h")
checkTransitionDeps(t, ctx, A_a, "B(a)", "C(a)")
checkTransitionDeps(t, ctx, A_b, "B(b)", "C(b)")
@@ -165,6 +178,8 @@ func TestTransition(t *testing.T) {
checkTransitionDeps(t, ctx, D_d, "E(d)")
checkTransitionDeps(t, ctx, E_d)
checkTransitionDeps(t, ctx, F)
+ checkTransitionDeps(t, ctx, G)
+ checkTransitionDeps(t, ctx, H_h)
checkTransitionMutate(t, A_a, "a")
checkTransitionMutate(t, A_b, "b")
@@ -176,11 +191,14 @@ func TestTransition(t *testing.T) {
checkTransitionMutate(t, D_d, "d")
checkTransitionMutate(t, E_d, "d")
checkTransitionMutate(t, F, "")
+ checkTransitionMutate(t, G, "")
+ checkTransitionMutate(t, H_h, "h")
}
func TestPostTransitionDeps(t *testing.T) {
ctx, errs := testTransition(fmt.Sprintf(testTransitionBp,
- `post_transition_deps: ["C", "D:late", "E:d", "F"],`))
+ `post_transition_deps: ["C", "D:late", "E:d", "F"],`,
+ `post_transition_deps: ["H"],`))
assertNoErrors(t, errs)
// Module A uses Split to create a and b variants
@@ -206,6 +224,8 @@ func TestPostTransitionDeps(t *testing.T) {
D_d := getTransitionModule(ctx, "D", "d")
E_d := getTransitionModule(ctx, "E", "d")
F := getTransitionModule(ctx, "F", "")
+ G := getTransitionModule(ctx, "G", "")
+ H_h := getTransitionModule(ctx, "H", "h")
checkTransitionDeps(t, ctx, A_a, "B(a)", "C(a)")
checkTransitionDeps(t, ctx, A_b, "B(b)", "C(b)")
@@ -222,6 +242,8 @@ func TestPostTransitionDeps(t *testing.T) {
checkTransitionDeps(t, ctx, D_d, "E(d)")
checkTransitionDeps(t, ctx, E_d)
checkTransitionDeps(t, ctx, F)
+ checkTransitionDeps(t, ctx, G, "H(h)")
+ checkTransitionDeps(t, ctx, H_h)
checkTransitionMutate(t, A_a, "a")
checkTransitionMutate(t, A_b, "b")
@@ -233,12 +255,14 @@ func TestPostTransitionDeps(t *testing.T) {
checkTransitionMutate(t, D_d, "d")
checkTransitionMutate(t, E_d, "d")
checkTransitionMutate(t, F, "")
+ checkTransitionMutate(t, G, "")
+ checkTransitionMutate(t, H_h, "h")
}
func TestPostTransitionDepsMissingVariant(t *testing.T) {
// TODO: eventually this will create the missing variant on demand
_, errs := testTransition(fmt.Sprintf(testTransitionBp,
- `post_transition_deps: ["E:missing"],`))
+ `post_transition_deps: ["E:missing"],`, ""))
expectedError := `Android.bp:8:4: dependency "E" of "B" missing variant:
transition:missing
available variants:
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-09 19:43:31 +0000