-
Notifications
You must be signed in to change notification settings - Fork 20
perf(comp list): skip lock file reads (non-mutating) #218
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,6 +4,7 @@ | |
| package components | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "fmt" | ||
| "log/slog" | ||
|
|
@@ -16,6 +17,7 @@ import ( | |
| "github.com/microsoft/azure-linux-dev-tools/internal/fingerprint" | ||
| "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" | ||
| "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" | ||
| "github.com/microsoft/azure-linux-dev-tools/internal/utils/parmap" | ||
| ) | ||
|
|
||
| // Well-known errors. | ||
|
|
@@ -41,6 +43,11 @@ type Resolver struct { | |
| // Only 'component update' enables this — it uses the freshness status to | ||
| // decide which components need re-resolution and which can be skipped. | ||
| CheckFreshness bool | ||
| // SkipLockPopulation skips reading lock files and attaching lock data to | ||
| // resolved components. When true, component Locked fields remain nil. | ||
| // Set this when the resolver's lock mode requires skipping population | ||
| // (e.g., when called from a command with LockModeSkipBoth). | ||
| SkipLockPopulation bool | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There are a bunch of weird combos we can get now with the flags, might be better to use an enum. For example, you can have |
||
| } | ||
|
|
||
| // NewResolver constructs a new [Resolver] for the given environment. | ||
|
|
@@ -52,11 +59,20 @@ func NewResolver(env *azldev.Env) *Resolver { | |
|
|
||
| // Given a component filter, finds all components defined in the environment that match the filter. | ||
| func (r *Resolver) FindComponents(filter *ComponentFilter) (components *ComponentSet, err error) { | ||
| // The filter's SkipLockValidation field is the primary control. When | ||
| // created via AddComponentFilterOptionsToCommand, its default is false | ||
| // (validation on). Commands that write locks (update) or are read-only | ||
| // (list) explicitly set it to true. | ||
| skipValidation := filter.SkipLockValidation | ||
| // Determine whether to skip lock validation based on the filter's LockMode. | ||
| skipValidation := filter.LockMode != LockModeValidateAndPopulate | ||
| // Also check the resolver's SkipLockPopulation field (set programmatically by callers) | ||
| skipPopulation := filter.LockMode == LockModeSkipBoth || r.SkipLockPopulation | ||
|
|
||
| // Temporarily set r.SkipLockPopulation during this resolution to ensure | ||
| // FindAllComponents and other methods respect the skip requirement. | ||
| oldSkipLockPopulation := r.SkipLockPopulation | ||
| if skipPopulation { | ||
| r.SkipLockPopulation = true | ||
| } | ||
| defer func() { | ||
| r.SkipLockPopulation = oldSkipLockPopulation | ||
| }() | ||
|
Comment on lines
+67
to
+75
|
||
|
|
||
| // For usability's sake, detect if the caller/user forgot to specify *any* criteria. | ||
| if filter.HasNoCriteria() { | ||
|
|
@@ -72,7 +88,13 @@ func (r *Resolver) FindComponents(filter *ComponentFilter) (components *Componen | |
| return allComps, findErr | ||
| } | ||
|
|
||
| r.warnOnLockDrift(allComps) | ||
| if !skipPopulation { | ||
| r.populateLocksParallel(allComps) | ||
| } | ||
|
|
||
| if !r.SuppressLockWarnings { | ||
| r.warnOnLockDrift(allComps) | ||
| } | ||
|
|
||
| return allComps, r.validateLockFiles(allComps, true, skipValidation) | ||
| } | ||
|
|
@@ -103,6 +125,10 @@ func (r *Resolver) FindComponents(filter *ComponentFilter) (components *Componen | |
| } | ||
| } | ||
|
|
||
| if !skipPopulation { | ||
| r.populateLocksParallel(components) | ||
| } | ||
|
|
||
| r.warnOnLockDrift(components) | ||
|
|
||
| return components, r.validateLockFiles(components, false, skipValidation) | ||
|
Comment on lines
+128
to
134
|
||
|
|
@@ -162,6 +188,10 @@ func (r *Resolver) FindAllComponents() (components *ComponentSet, err error) { | |
| components.Add(comp) | ||
| } | ||
|
|
||
| if !r.SkipLockPopulation { | ||
| r.populateLocksParallel(components) | ||
| } | ||
|
|
||
| return components, nil | ||
| } | ||
|
|
||
|
|
@@ -497,17 +527,23 @@ func (r *Resolver) createComponentFromConfig(componentConfig *projectconfig.Comp | |
| componentConfig.Release.Calculation = projectconfig.ReleaseCalculationAuto | ||
| } | ||
|
|
||
| // Populate locked state onto the component config. This makes lock data | ||
| // available to all downstream consumers (render, build, prepare-sources, | ||
| // diff-sources) without each needing lock-file awareness. | ||
| r.populateFromLock(componentConfig) | ||
|
|
||
| return &resolvedComponent{ | ||
| env: r.env, | ||
| config: *componentConfig, | ||
| }, nil | ||
| } | ||
|
|
||
| func (r *Resolver) populateLocksParallel(set *ComponentSet) { | ||
| comps := set.Components() | ||
|
|
||
| _ = parmap.Map(r.env, r.env.IOBoundConcurrency(), comps, nil, | ||
| func(_ context.Context, c Component) error { | ||
| r.populateFromLock(c.GetConfig()) | ||
|
|
||
| return nil | ||
| }) | ||
| } | ||
|
|
||
| // populateFromLock reads lock file data and attaches it to the component config | ||
| // as [projectconfig.ComponentLockData]. This centralizes lock-file consumption so | ||
| // all downstream commands (render, build, prepare-sources, diff-sources) get | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We have
parmap, and in theory everything in the lock store should be thread-safe.Could you quickly check what the perf is of using that?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
With parallel populate cold list -a (no cache effect) time was around 21s and hot was around 10s. There is definitely an improvement in the cold start time. It is still too much to make a difference in interactive sessions.
I had tried a parallel file read implementation which did worse and conclusion I came to was that the fs iops was so low the resulting pileup was negatively impacting performance gains from the parallelism.