Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion docs/user/reference/cli/azldev_component_build.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion docs/user/reference/cli/azldev_component_diff-sources.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions docs/user/reference/cli/azldev_component_list.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion docs/user/reference/cli/azldev_component_query.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion docs/user/reference/cli/azldev_component_render.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 5 additions & 3 deletions internal/app/azldev/cmds/component/changed.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,11 +119,13 @@ const (
func ChangedComponents(
env *azldev.Env, options *ChangedComponentOptions,
) ([]ChangedResult, error) {
// Changed compares lock files between git refs — skip validation since
// the current working-tree locks may legitimately be stale.
options.ComponentFilter.SkipLockValidation = true
// Changed compares lock files between git refs — skip both validation and population
// since it reads locks directly from git commits (ReadAllAtCommit).
options.ComponentFilter.LockMode = components.LockModeSkipBoth

resolver := components.NewResolver(env)
// Skip lock population since changed reads locks directly from git commits
resolver.SkipLockPopulation = true
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We have parmap, and in theory everything in the lock store should be thread-safe.
Could you quickly check what the perf is of using that?

diff --git a/internal/app/azldev/core/components/resolver.go b/internal/app/azldev/core/components/resolver.go
index 41ab031..c752513 100644
--- a/internal/app/azldev/core/components/resolver.go
+++ b/internal/app/azldev/core/components/resolver.go
@@ -4,6 +4,7 @@
 package components
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"log/slog"
@@ -16,6 +17,7 @@ import (
 	"github.com/microsoft/azure-linux-dev-tools/internal/fingerprint"
 	"github.com/microsoft/azure-linux-dev-tools/internal/projectconfig"
 	"github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils"
+	"github.com/microsoft/azure-linux-dev-tools/internal/utils/parmap"
 )
 
 // Well-known errors.
@@ -103,6 +105,8 @@ func (r *Resolver) FindComponents(filter *ComponentFilter) (components *Componen
 		}
 	}
 
+	r.populateLocksParallel(components)
+
 	r.warnOnLockDrift(components)
 
 	return components, r.validateLockFiles(components, false, skipValidation)
@@ -162,6 +166,8 @@ func (r *Resolver) FindAllComponents() (components *ComponentSet, err error) {
 		components.Add(comp)
 	}
 
+	r.populateLocksParallel(components)
+
 	return components, nil
 }
 
@@ -497,17 +503,23 @@ func (r *Resolver) createComponentFromConfig(componentConfig *projectconfig.Comp
 		componentConfig.Autospec.ReleaseCalculation = projectconfig.ReleaseCalculationAuto
 	}
 
-	// Populate locked state onto the component config. This makes lock data
-	// available to all downstream consumers (render, build, prepare-sources,
-	// diff-sources) without each needing lock-file awareness.
-	r.populateFromLock(componentConfig)
-
 	return &resolvedComponent{
 		env:    r.env,
 		config: *componentConfig,
 	}, nil
 }
 
+func (r *Resolver) populateLocksParallel(set *ComponentSet) {
+	comps := set.Components()
+
+	_ = parmap.Map(r.env, r.env.IOBoundConcurrency(), comps, nil,
+		func(_ context.Context, c Component) error {
+			r.populateFromLock(c.GetConfig())
+
+			return nil
+		})
+}
+
 // populateFromLock reads lock file data and attaches it to the component config
 // as [projectconfig.ComponentLockData]. This centralizes lock-file consumption so
 // all downstream commands (render, build, prepare-sources, diff-sources) get

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With parallel populate cold list -a (no cache effect) time was around 21s and hot was around 10s. There is definitely an improvement in the cold start time. It is still too much to make a difference in interactive sessions.

I had tried a parallel file read implementation which did worse and conclusion I came to was that the fs iops was so low the resulting pileup was negatively impacting performance gains from the parallelism.


comps, err := resolver.FindComponents(&options.ComponentFilter)
if err != nil {
Expand Down
25 changes: 17 additions & 8 deletions internal/app/azldev/cmds/component/list.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ import (
type ListComponentOptions struct {
// Standard filter for selecting components.
ComponentFilter components.ComponentFilter
// SkipLockPopulation controls whether lock files are read during resolution.
// Only applicable when component list is explicitly requested by the user.
SkipLockPopulation bool
}

func listOnAppInit(_ *azldev.App, parentCmd *cobra.Command) {
Expand Down Expand Up @@ -57,26 +60,32 @@ Component name patterns support glob syntax (*, ?, []).`,

components.AddComponentFilterOptionsToCommand(cmd, &options.ComponentFilter)

// List always skips lock validation (read-only), so the flag is
// meaningless here. Hide it to avoid confusion.
_ = cmd.Flags().MarkHidden("skip-lock-validation")
// Add skip-lock-population flag for list command
cmd.Flags().BoolVar(&options.SkipLockPopulation, "skip-lock-population",
true,
"skip lock file population (default: true; set to false to read lock data)")
Comment on lines +64 to +66

return cmd
}

// ListComponentConfigs lists components in the env, in accordance with options.
// Lock validation is always skipped regardless of the caller's SkipLockValidation
// value — list is read-only.
// Lock validation is always skipped since list is read-only.
func ListComponentConfigs(
env *azldev.Env, options *ListComponentOptions,
) (results []projectconfig.ComponentConfig, err error) {
var comps *components.ComponentSet

// List is read-only — skip lock validation so it works even when
// locks are missing or stale.
options.ComponentFilter.SkipLockValidation = true
// List is read-only — always skip lock validation.
// Determine lock mode based on user preference for population.
if options.SkipLockPopulation {
options.ComponentFilter.LockMode = components.LockModeSkipBoth
} else {
options.ComponentFilter.LockMode = components.LockModeSkipValidationPopulate
}

resolver := components.NewResolver(env)
// Set the resolver's SkipLockPopulation to match the filter's LockMode
resolver.SkipLockPopulation = options.SkipLockPopulation

comps, err = resolver.FindComponents(&options.ComponentFilter)
if err != nil {
Expand Down
10 changes: 5 additions & 5 deletions internal/app/azldev/cmds/component/update.go
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,7 @@ type UpdateResult struct {

// UpdateComponents resolves source identities for all selected components and
// writes the results to per-component lock files under locks/.
// Lock validation is always skipped regardless of the caller's SkipLockValidation
// value — update is the lock writer.
// Lock validation is always skipped — update is the lock writer.
func UpdateComponents(env *azldev.Env, options *UpdateComponentOptions) ([]UpdateResult, error) {
if options.Bump && options.CheckOnly {
return nil, fmt.Errorf("%w: --bump and --check-only are mutually exclusive", azldev.ErrInvalidUsage)
Expand All @@ -161,9 +160,10 @@ func UpdateComponents(env *azldev.Env, options *UpdateComponentOptions) ([]Updat
// Suppress staleness warnings — we're about to refresh the locks ourselves,
// so warning the user to "run component update" would be self-referential noise.
resolver.SuppressLockWarnings = true
// Skip lock validation — update is the lock file writer, so missing or
// stale locks are expected and will be fixed by this command.
options.ComponentFilter.SkipLockValidation = true
// Skip lock validation but keep population — update is the lock file writer,
// so missing or stale locks are expected and will be fixed by this command.
// We still need to populate locks to compute freshness.
options.ComponentFilter.LockMode = components.LockModeSkipValidationPopulate
// Enable freshness checking so the resolver computes FreshnessStatus for
// each component. This lets resolveSourceIdentitiesParallel skip
// re-resolution for components whose resolution inputs haven't changed.
Expand Down
42 changes: 35 additions & 7 deletions internal/app/azldev/core/components/filter.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,35 @@ import (
"github.com/spf13/cobra"
)

// LockMode defines how lock files should be handled during component resolution.
type LockMode int

const (
// LockModeValidateAndPopulate validates lock file consistency and populates lock data on components (default).
LockModeValidateAndPopulate LockMode = iota
// LockModeSkipValidationPopulate skips lock file validation but still populates lock data on components.
// Used by commands that build/render components and consume lock data as-is.
LockModeSkipValidationPopulate
// LockModeSkipBoth skips both lock file validation and population of lock data.
// Lock population can only be skipped if validation is also skipped.
// Used by read-only commands (list, changed) that don't consume lock data.
LockModeSkipBoth
)

// String returns a string representation of the LockMode.
func (m LockMode) String() string {
switch m {
case LockModeValidateAndPopulate:
return "validateAndPopulate"
case LockModeSkipValidationPopulate:
return "skipValidationPopulate"
case LockModeSkipBoth:
return "skipBoth"
default:
return "unknown"
}
}

// Describes a filter that selects a subset of components known within the environment's
// loaded configuration.
type ComponentFilter struct {
Expand All @@ -22,10 +51,10 @@ type ComponentFilter struct {
SpecPaths []string
// If true, then *all* known components are included in the result set.
IncludeAllComponents bool
// SkipLockValidation disables lock file consistency checks for this
// filter's resolution. Commands that write lock files (update) or are
// read-only (list) set this to true.
SkipLockValidation bool
// LockMode defines how lock file validation and population should be handled during resolution.
// Default is LockModeValidateAndPopulate.
// Lock population can only be skipped if validation is also skipped (LockModeSkipBoth).
LockMode LockMode
}

// HasNoCriteria returns true if the filter has no criteria set, meaning that it will never
Expand Down Expand Up @@ -53,9 +82,8 @@ func AddComponentFilterOptionsToCommand(cmd *cobra.Command, filter *ComponentFil
cmd.Flags().StringArrayVarP(&filter.SpecPaths, "spec-path", "s", []string{}, "Spec path")
_ = cmd.MarkFlagFilename("spec-path", ".spec")

cmd.Flags().BoolVar(&filter.SkipLockValidation, "skip-lock-validation",
false,
"skip lock file consistency checks")
// Note: skip-lock-validation flag is added per-command, not here, to allow customization
// (e.g., hide for commands that always skip, or pair with skip-lock-population for list)
}

// Function suitable for use as a [cobra.ValidArgsFunction] in a [cobra.Command]. Intended for use
Expand Down
58 changes: 47 additions & 11 deletions internal/app/azldev/core/components/resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package components

import (
"context"
"errors"
"fmt"
"log/slog"
Expand All @@ -16,6 +17,7 @@ import (
"github.com/microsoft/azure-linux-dev-tools/internal/fingerprint"
"github.com/microsoft/azure-linux-dev-tools/internal/projectconfig"
"github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils"
"github.com/microsoft/azure-linux-dev-tools/internal/utils/parmap"
)

// Well-known errors.
Expand All @@ -41,6 +43,11 @@ type Resolver struct {
// Only 'component update' enables this — it uses the freshness status to
// decide which components need re-resolution and which can be skipped.
CheckFreshness bool
// SkipLockPopulation skips reading lock files and attaching lock data to
// resolved components. When true, component Locked fields remain nil.
// Set this when the resolver's lock mode requires skipping population
// (e.g., when called from a command with LockModeSkipBoth).
SkipLockPopulation bool
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are a bunch of weird combos we can get now with the flags, might be better to use an enum. For example, you can have SkipLockPopulation=true and SkipLockValidation=false which I think still hits the perf issue, and doesn't really make sense to me.

}

// NewResolver constructs a new [Resolver] for the given environment.
Expand All @@ -52,11 +59,20 @@ func NewResolver(env *azldev.Env) *Resolver {

// Given a component filter, finds all components defined in the environment that match the filter.
func (r *Resolver) FindComponents(filter *ComponentFilter) (components *ComponentSet, err error) {
// The filter's SkipLockValidation field is the primary control. When
// created via AddComponentFilterOptionsToCommand, its default is false
// (validation on). Commands that write locks (update) or are read-only
// (list) explicitly set it to true.
skipValidation := filter.SkipLockValidation
// Determine whether to skip lock validation based on the filter's LockMode.
skipValidation := filter.LockMode != LockModeValidateAndPopulate
// Also check the resolver's SkipLockPopulation field (set programmatically by callers)
skipPopulation := filter.LockMode == LockModeSkipBoth || r.SkipLockPopulation

// Temporarily set r.SkipLockPopulation during this resolution to ensure
// FindAllComponents and other methods respect the skip requirement.
oldSkipLockPopulation := r.SkipLockPopulation
if skipPopulation {
r.SkipLockPopulation = true
}
defer func() {
r.SkipLockPopulation = oldSkipLockPopulation
}()
Comment on lines +67 to +75

// For usability's sake, detect if the caller/user forgot to specify *any* criteria.
if filter.HasNoCriteria() {
Expand All @@ -72,7 +88,13 @@ func (r *Resolver) FindComponents(filter *ComponentFilter) (components *Componen
return allComps, findErr
}

r.warnOnLockDrift(allComps)
if !skipPopulation {
r.populateLocksParallel(allComps)
}

if !r.SuppressLockWarnings {
r.warnOnLockDrift(allComps)
}

return allComps, r.validateLockFiles(allComps, true, skipValidation)
}
Expand Down Expand Up @@ -103,6 +125,10 @@ func (r *Resolver) FindComponents(filter *ComponentFilter) (components *Componen
}
}

if !skipPopulation {
r.populateLocksParallel(components)
}

r.warnOnLockDrift(components)

return components, r.validateLockFiles(components, false, skipValidation)
Comment on lines +128 to 134
Expand Down Expand Up @@ -162,6 +188,10 @@ func (r *Resolver) FindAllComponents() (components *ComponentSet, err error) {
components.Add(comp)
}

if !r.SkipLockPopulation {
r.populateLocksParallel(components)
}

return components, nil
}

Expand Down Expand Up @@ -497,17 +527,23 @@ func (r *Resolver) createComponentFromConfig(componentConfig *projectconfig.Comp
componentConfig.Release.Calculation = projectconfig.ReleaseCalculationAuto
}

// Populate locked state onto the component config. This makes lock data
// available to all downstream consumers (render, build, prepare-sources,
// diff-sources) without each needing lock-file awareness.
r.populateFromLock(componentConfig)

return &resolvedComponent{
env: r.env,
config: *componentConfig,
}, nil
}

func (r *Resolver) populateLocksParallel(set *ComponentSet) {
comps := set.Components()

_ = parmap.Map(r.env, r.env.IOBoundConcurrency(), comps, nil,
func(_ context.Context, c Component) error {
r.populateFromLock(c.GetConfig())

return nil
})
}

// populateFromLock reads lock file data and attaches it to the component config
// as [projectconfig.ComponentLockData]. This centralizes lock-file consumption so
// all downstream commands (render, build, prepare-sources, diff-sources) get
Expand Down
Loading
Loading