From ff555094d2ee43fb2a9f4d0c2aa7cab0ebcac0d3 Mon Sep 17 00:00:00 2001 From: Wei Hu Date: Thu, 11 Jun 2026 14:50:14 +0000 Subject: [PATCH 01/13] Profile C# post-processing benchmarks --- .../perf/FullGenerationBenchmark.cs | 157 +++++++++++++++ .../src/CSharpGen.cs | 162 +++++++++++---- .../PostProcessing/GeneratedCodeWorkspace.cs | 112 +++++++++-- ...ratedCodeWorkspacePostProcessingProfile.cs | 64 ++++++ .../src/PostProcessing/PostProcessor.cs | 186 +++++++++++++----- 5 files changed, 572 insertions(+), 109 deletions(-) create mode 100644 packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/perf/FullGenerationBenchmark.cs create mode 100644 packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspacePostProcessingProfile.cs diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/perf/FullGenerationBenchmark.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/perf/FullGenerationBenchmark.cs new file mode 100644 index 00000000000..66553e08c38 --- /dev/null +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/perf/FullGenerationBenchmark.cs @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; + +namespace Microsoft.TypeSpec.Generator.Perf +{ + public class FullGenerationBenchmark + { + private const string ProfileEnvironmentVariable = "POSTPROCESSING_BENCHMARK_PROFILE_STEPS"; + private const string ProfileOutputDirectoryEnvironmentVariable = "POSTPROCESSING_BENCHMARK_PROFILE_DIR"; + + private bool _profileSteps; + + [GlobalSetup] + public void GlobalSetup() + { + _profileSteps = string.Equals( + Environment.GetEnvironmentVariable(ProfileEnvironmentVariable), + "true", + StringComparison.OrdinalIgnoreCase); + } + + [Benchmark] + public async Task GenerateSampleTypeSpecProject() + { + var postProcessingProfile = _profileSteps ? new GeneratedCodeWorkspacePostProcessingProfile() : null; + var generationProfile = _profileSteps ? new GeneratedCodeWorkspacePostProcessingProfile() : null; + GeneratedCodeWorkspace.PostProcessingProfile = postProcessingProfile; + CSharpGen.GenerationProfile = generationProfile; + + var benchmarkDirectory = CreateBenchmarkInputDirectory(); + var stopwatch = Stopwatch.StartNew(); + try + { + CodeModelGenerator.Instance = new BenchmarkCodeModelGenerator(benchmarkDirectory); + CodeModelGenerator.Instance.Configure(); + + var csharpGen = new CSharpGen(); + await csharpGen.ExecuteAsync(); + + return Directory.GetFiles(benchmarkDirectory, "*", SearchOption.AllDirectories) + .Where(static path => !path.EndsWith("tspCodeModel.json", StringComparison.Ordinal) && + !path.EndsWith("Configuration.json", StringComparison.Ordinal)) + .Sum(static path => (int)new FileInfo(path).Length); + } + finally + { + stopwatch.Stop(); + if (generationProfile != null) + { + WriteProfile( + generationProfile, + $"full-generation-profile-{DateTime.UtcNow:yyyyMMddHHmmssfff}.csv", + $"Full generation invocation elapsed ms: {stopwatch.Elapsed.TotalMilliseconds:F3}{Environment.NewLine}" + + $"Input directory: {benchmarkDirectory}{Environment.NewLine}"); + } + + if (postProcessingProfile != null) + { + WriteProfile( + postProcessingProfile, + $"full-generation-post-processing-profile-{DateTime.UtcNow:yyyyMMddHHmmssfff}.csv", + $"Full generation post-processing profile{Environment.NewLine}" + + $"Input directory: {benchmarkDirectory}{Environment.NewLine}"); + } + + CSharpGen.GenerationProfile = null; + GeneratedCodeWorkspace.PostProcessingProfile = null; + TryDeleteDirectory(benchmarkDirectory); + } + } + + private static void WriteProfile(GeneratedCodeWorkspacePostProcessingProfile profile, string fileName, string header) + { + var profileDirectory = GetProfileOutputDirectory(); + Directory.CreateDirectory(profileDirectory); + File.WriteAllText(Path.Combine(profileDirectory, fileName), header + profile.GetSummary()); + } + + private static string CreateBenchmarkInputDirectory() + { + var sourceDirectory = FindFullGenerationInputDirectory(); + var benchmarkDirectory = Path.Combine(Path.GetTempPath(), "typespec-full-generation-benchmark", Guid.NewGuid().ToString("N")); + CopyDirectory(sourceDirectory, benchmarkDirectory); + return benchmarkDirectory; + } + + private static string FindFullGenerationInputDirectory() + { + const string relativePath = "packages/http-client-csharp/generator/TestProjects/Local/Sample-TypeSpec"; + + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory != null) + { + var inputDirectory = Path.Combine(directory.FullName, relativePath); + if (File.Exists(Path.Combine(inputDirectory, "tspCodeModel.json")) && + File.Exists(Path.Combine(inputDirectory, "Configuration.json"))) + { + return inputDirectory; + } + + directory = directory.Parent; + } + + throw new DirectoryNotFoundException($"Could not find '{relativePath}' from '{AppContext.BaseDirectory}'."); + } + + private static void CopyDirectory(string sourceDirectory, string destinationDirectory) + { + Directory.CreateDirectory(destinationDirectory); + foreach (var sourceFile in Directory.GetFiles(sourceDirectory, "*", SearchOption.AllDirectories)) + { + var relativePath = Path.GetRelativePath(sourceDirectory, sourceFile); + var destinationFile = Path.Combine(destinationDirectory, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(destinationFile)!); + File.Copy(sourceFile, destinationFile, overwrite: true); + } + } + + private static void TryDeleteDirectory(string directory) + { + try + { + if (Directory.Exists(directory)) + { + Directory.Delete(directory, recursive: true); + } + } + catch + { + // Best-effort cleanup for benchmark temp output. + } + } + + private static string GetProfileOutputDirectory() + { + var configuredPath = Environment.GetEnvironmentVariable(ProfileOutputDirectoryEnvironmentVariable); + return string.IsNullOrWhiteSpace(configuredPath) + ? Path.Combine(Path.GetTempPath(), "typespec-post-processing-profiles") + : Path.GetFullPath(configuredPath); + } + + private sealed class BenchmarkCodeModelGenerator : CodeModelGenerator + { + public BenchmarkCodeModelGenerator(string outputPath) + : base(new GeneratorContext(Configuration.Load(outputPath))) + { + } + } + } +} diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/CSharpGen.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/CSharpGen.cs index 9948fcff594..238b16b9766 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/CSharpGen.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/CSharpGen.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Threading.Tasks; @@ -20,6 +21,8 @@ internal sealed class CSharpGen private static readonly string[] _filesToKeep = [ConfigurationFileName, CodeModelFileName]; + internal static GeneratedCodeWorkspacePostProcessingProfile? GenerationProfile { get; set; } + /// /// Executes the generator task with the instance. /// @@ -33,19 +36,21 @@ public async Task ExecuteAsync() // Resolve PackageReference items from the .csproj so custom code referencing // external NuGet types (e.g., Azure.Storage.Common) compiles correctly. - await GeneratedCodeWorkspace.AddPackageReferencesFromProject(); + await MeasureGenerationStepAsync("Generation.AddPackageReferencesFromProject", GeneratedCodeWorkspace.AddPackageReferencesFromProject); // Pre-walk the input library and resolve any external types that point at NuGet packages. // This populates ExternalTypeReferenceResolver's cache and registers each resolved assembly // as an additional metadata reference *before* the generated/custom code workspaces are // constructed, so their cached Roslyn projects pick the references up. - await ExternalTypeReferenceResolver.ResolveAllAsync(); + await MeasureGenerationStepAsync("Generation.ResolveExternalTypeReferences", ExternalTypeReferenceResolver.ResolveAllAsync); // Initialize the workspace project AFTER all metadata references have been added so the // eagerly-cached project sees them. GeneratedCodeWorkspace.Initialize(); - GeneratedCodeWorkspace customCodeWorkspace = await GeneratedCodeWorkspace.Create(isCustomCodeProject: true); + GeneratedCodeWorkspace customCodeWorkspace = await MeasureGenerationStepAsync( + "Generation.CreateCustomCodeWorkspace", + () => GeneratedCodeWorkspace.Create(isCustomCodeProject: true)); // The generated attributes need to be added into the workspace before loading the custom code. Otherwise, // Roslyn doesn't load the attributes completely and we are unable to get the attribute arguments. @@ -55,88 +60,167 @@ public async Task ExecuteAsync() generateAttributeTasks.Add(customCodeWorkspace.AddInMemoryFile(attributeProvider)); } - await Task.WhenAll(generateAttributeTasks); + await MeasureGenerationStepAsync("Generation.AddCustomizationAttributeProviders", () => Task.WhenAll(generateAttributeTasks)); - CodeModelGenerator.Instance.SourceInputModel = new SourceInputModel( - await customCodeWorkspace.GetCompilationAsync(), - await GeneratedCodeWorkspace.LoadBaselineContract()); + CodeModelGenerator.Instance.SourceInputModel = await MeasureGenerationStepAsync( + "Generation.CreateSourceInputModel", + async () => new SourceInputModel( + await customCodeWorkspace.GetCompilationAsync(), + await GeneratedCodeWorkspace.LoadBaselineContract())); - GeneratedCodeWorkspace generatedCodeWorkspace = await GeneratedCodeWorkspace.Create(isCustomCodeProject: false); + GeneratedCodeWorkspace generatedCodeWorkspace = await MeasureGenerationStepAsync( + "Generation.CreateGeneratedCodeWorkspace", + () => GeneratedCodeWorkspace.Create(isCustomCodeProject: false)); - var output = CodeModelGenerator.Instance.OutputLibrary; + var output = MeasureGenerationStep("Generation.GetOutputLibrary", () => CodeModelGenerator.Instance.OutputLibrary); Directory.CreateDirectory(Path.Combine(generatedSourceOutputPath, "Models")); List generateFilesTasks = new(); // Build all TypeProviders - foreach (var type in output.TypeProviders) + MeasureGenerationStep("Generation.BuildTypeProviders", () => { - type.EnsureBuilt(); - } + foreach (var type in output.TypeProviders) + { + type.EnsureBuilt(); + } + }); LoggingHelpers.LogElapsedTime("All generated type providers built"); // visit the entire library before generating files - foreach (var visitor in CodeModelGenerator.Instance.Visitors) + MeasureGenerationStep("Generation.ApplyVisitors", () => { - visitor.VisitLibrary(output); - } + foreach (var visitor in CodeModelGenerator.Instance.Visitors) + { + visitor.VisitLibrary(output); + } + }); - FilterAllCustomizedMembers(output); + MeasureGenerationStep("Generation.FilterCustomizedMembers", () => FilterAllCustomizedMembers(output)); LoggingHelpers.LogElapsedTime("All visitors have been applied"); - foreach (var outputType in output.TypeProviders) + MeasureGenerationStep("Generation.WriteTypeProviders", () => { - // Ensure back-compatibility processing is done after all visitors have run - outputType.ProcessTypeForBackCompatibility(); - - var writer = CodeModelGenerator.Instance.GetWriter(outputType); - generateFilesTasks.Add(generatedCodeWorkspace.AddGeneratedFile(writer.Write())); - - foreach (var serialization in outputType.SerializationProviders) + foreach (var outputType in output.TypeProviders) { - writer = CodeModelGenerator.Instance.GetWriter(serialization); + // Ensure back-compatibility processing is done after all visitors have run + outputType.ProcessTypeForBackCompatibility(); + + var writer = CodeModelGenerator.Instance.GetWriter(outputType); generateFilesTasks.Add(generatedCodeWorkspace.AddGeneratedFile(writer.Write())); + + foreach (var serialization in outputType.SerializationProviders) + { + writer = CodeModelGenerator.Instance.GetWriter(serialization); + generateFilesTasks.Add(generatedCodeWorkspace.AddGeneratedFile(writer.Write())); + } } - } + }); // Add all the generated files to the workspace - await Task.WhenAll(generateFilesTasks); + await MeasureGenerationStepAsync("Generation.AddGeneratedFilesToWorkspace", () => Task.WhenAll(generateFilesTasks)); LoggingHelpers.LogElapsedTime("All generated types have been written into memory"); // Delete any old generated files - DeleteDirectory(generatedSourceOutputPath, _filesToKeep); + MeasureGenerationStep("Generation.DeleteOldGeneratedFiles", () => DeleteDirectory(generatedSourceOutputPath, _filesToKeep)); LoggingHelpers.LogElapsedTime("All old generated files have been deleted"); - await generatedCodeWorkspace.PostProcessAsync(); + await MeasureGenerationStepAsync("Generation.PostProcessAsync", generatedCodeWorkspace.PostProcessAsync); // Write the generated files to the output directory - await foreach (var file in generatedCodeWorkspace.GetGeneratedFilesAsync()) + await MeasureGenerationStepAsync("Generation.WriteGeneratedFilesToDisk", async () => { - if (string.IsNullOrEmpty(file.Text)) + await foreach (var file in generatedCodeWorkspace.GetGeneratedFilesAsync()) { - continue; + if (string.IsNullOrEmpty(file.Text)) + { + continue; + } + var filename = Path.Combine(outputPath, file.Name); + CodeModelGenerator.Instance.Emitter.Info($"Writing {Path.GetFullPath(filename)}"); + Directory.CreateDirectory(Path.GetDirectoryName(filename)!); + await File.WriteAllTextAsync(filename, file.Text); } - var filename = Path.Combine(outputPath, file.Name); - CodeModelGenerator.Instance.Emitter.Info($"Writing {Path.GetFullPath(filename)}"); - Directory.CreateDirectory(Path.GetDirectoryName(filename)!); - await File.WriteAllTextAsync(filename, file.Text); - } + }); // Write additional output files (e.g. configuration schemas, .targets files) - await CodeModelGenerator.Instance.WriteAdditionalFiles(outputPath); + await MeasureGenerationStepAsync("Generation.WriteAdditionalFiles", () => CodeModelGenerator.Instance.WriteAdditionalFiles(outputPath)); // Write project scaffolding files (after additional files so schema existence can be checked) if (CodeModelGenerator.Instance.IsNewProject) { - await CodeModelGenerator.Instance.TypeFactory.CreateNewProjectScaffolding().Execute(); + await MeasureGenerationStepAsync( + "Generation.WriteProjectScaffolding", + () => CodeModelGenerator.Instance.TypeFactory.CreateNewProjectScaffolding().Execute()); } LoggingHelpers.LogElapsedTime("All files have been written to disk"); } + private static void MeasureGenerationStep(string stepName, Action action) + { + MeasureGenerationStep( + stepName, + () => + { + action(); + return 0; + }); + } + + private static T MeasureGenerationStep(string stepName, Func action) + { + var profile = GenerationProfile; + if (profile == null) + { + return action(); + } + + var allocatedBytes = GC.GetTotalAllocatedBytes(precise: false); + var stopwatch = Stopwatch.StartNew(); + try + { + return action(); + } + finally + { + stopwatch.Stop(); + profile.Add(stepName, stopwatch.Elapsed, GC.GetTotalAllocatedBytes(precise: false) - allocatedBytes); + } + } + + private static Task MeasureGenerationStepAsync(string stepName, Func action) => MeasureGenerationStepAsync( + stepName, + async () => + { + await action(); + return 0; + }); + + private static async Task MeasureGenerationStepAsync(string stepName, Func> action) + { + var profile = GenerationProfile; + if (profile == null) + { + return await action(); + } + + var allocatedBytes = GC.GetTotalAllocatedBytes(precise: false); + var stopwatch = Stopwatch.StartNew(); + try + { + return await action(); + } + finally + { + stopwatch.Stop(); + profile.Add(stepName, stopwatch.Elapsed, GC.GetTotalAllocatedBytes(precise: false) - allocatedBytes); + } + } + internal static void FilterAllCustomizedMembers(OutputLibrary output) { foreach (var typeProvider in output.TypeProviders) diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspace.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspace.cs index 4588b3c4839..281d4ec2e5e 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspace.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspace.cs @@ -37,6 +37,8 @@ internal class GeneratedCodeWorkspace private static readonly Lazy _metadataReferenceResolver = new(() => new WorkspaceMetadataReferenceResolver()); private static Task? _cachedProject; + internal static GeneratedCodeWorkspacePostProcessingProfile? PostProcessingProfile { get; set; } + private static readonly string[] _generatedFolders = [GeneratedFolder]; private static readonly string[] _sharedFolders = [SharedFolder]; @@ -102,7 +104,7 @@ internal static SyntaxTree GetTree(TypeProvider provider) public async IAsyncEnumerable<(string Name, string Text)> GetGeneratedFilesAsync() { - List> documents = new List>(); + List docs = new List(); var memberRemover = new MemberRemoverRewriter(); foreach (Document document in _project.Documents) { @@ -111,9 +113,12 @@ internal static SyntaxTree GetTree(TypeProvider provider) continue; } - documents.Add(ProcessDocument(document, memberRemover)); + docs.Add(document); } - var docs = await Task.WhenAll(documents); + + docs = PostProcessingProfile == null + ? [.. await Task.WhenAll(docs.Select(document => ProcessDocument(document, memberRemover)))] + : await ProcessDocumentsSequentiallyAsync(docs, memberRemover); LoggingHelpers.LogElapsedTime("Roslyn post processing complete"); @@ -129,36 +134,101 @@ internal static SyntaxTree GetTree(TypeProvider provider) } } - private async Task ProcessDocument(Document document, MemberRemoverRewriter memberRemover) + private async Task> ProcessDocumentsSequentiallyAsync(List documents, MemberRemoverRewriter memberRemover) { - var root = await document.GetSyntaxRootAsync(); - var semanticModel = await document.GetSemanticModelAsync(); + List processedDocuments = new(documents.Count); + foreach (var document in documents) + { + processedDocuments.Add(await ProcessDocument(document, memberRemover)); + } - if (semanticModel == null || root == null) + return processedDocuments; + } + + private async Task ProcessDocument(Document document, MemberRemoverRewriter memberRemover) + { + var totalStopwatch = PostProcessingProfile == null ? null : Stopwatch.StartNew(); + try { + var root = await MeasurePostProcessingStepAsync("GetSyntaxRootAsync", () => document.GetSyntaxRootAsync()); + var semanticModel = await MeasurePostProcessingStepAsync("GetSemanticModelAsync", () => document.GetSemanticModelAsync()); + + if (semanticModel == null || root == null) + { + return document; + } + + root = MeasurePostProcessingStep("MemberRemoverRewriter", () => memberRemover.Visit(root)); + + foreach (var rewriter in CodeModelGenerator.Instance.Rewriters) + { + rewriter.SemanticModel = semanticModel; + root = MeasurePostProcessingStep($"CustomRewriter.{rewriter.GetType().Name}", () => rewriter.Visit(root)); + } + document = document.WithSyntaxRoot(root); + + if (!CodeModelGenerator.Instance.Configuration.DisableRoslynReduce) + { + document = await MeasurePostProcessingStepAsync("Roslyn.Simplifier.ReduceAsync", () => Simplifier.ReduceAsync(document)); + } + + // Reformat if any custom rewriters have been applied + if (CodeModelGenerator.Instance.Rewriters.Count > 0) + { + document = await MeasurePostProcessingStepAsync("Formatter.FormatAsync", () => Formatter.FormatAsync(document)); + } return document; } + finally + { + if (totalStopwatch != null) + { + totalStopwatch.Stop(); + PostProcessingProfile?.Add("ProcessDocument.Total", totalStopwatch.Elapsed, 0); + } + } + } - root = memberRemover.Visit(root); + private static T MeasurePostProcessingStep(string stepName, Func action) + { + var profile = PostProcessingProfile; + if (profile == null) + { + return action(); + } - foreach (var rewriter in CodeModelGenerator.Instance.Rewriters) + var allocatedBytes = GC.GetTotalAllocatedBytes(precise: false); + var stopwatch = Stopwatch.StartNew(); + try { - rewriter.SemanticModel = semanticModel; - root = rewriter.Visit(root); + return action(); } - document = document.WithSyntaxRoot(root); + finally + { + stopwatch.Stop(); + profile.Add(stepName, stopwatch.Elapsed, GC.GetTotalAllocatedBytes(precise: false) - allocatedBytes); + } + } - if (!CodeModelGenerator.Instance.Configuration.DisableRoslynReduce) + private static async Task MeasurePostProcessingStepAsync(string stepName, Func> action) + { + var profile = PostProcessingProfile; + if (profile == null) { - document = await Simplifier.ReduceAsync(document); + return await action(); } - // Reformat if any custom rewriters have been applied - if (CodeModelGenerator.Instance.Rewriters.Count > 0) + var allocatedBytes = GC.GetTotalAllocatedBytes(precise: false); + var stopwatch = Stopwatch.StartNew(); + try + { + return await action(); + } + finally { - document = await Formatter.FormatAsync(document); + stopwatch.Stop(); + profile.Add(stepName, stopwatch.Elapsed, GC.GetTotalAllocatedBytes(precise: false) - allocatedBytes); } - return document; } public static bool IsGeneratedDocument(Document document) => document.Folders.Contains(GeneratedFolder); @@ -275,11 +345,11 @@ public async Task PostProcessAsync() case Configuration.UnreferencedTypesHandlingOption.KeepAll: break; case Configuration.UnreferencedTypesHandlingOption.Internalize: - _project = await postProcessor.InternalizeAsync(_project); + _project = await MeasurePostProcessingStepAsync("PostProcess.InternalizeAsync", () => postProcessor.InternalizeAsync(_project)); break; case Configuration.UnreferencedTypesHandlingOption.RemoveOrInternalize: - _project = await postProcessor.InternalizeAsync(_project); - _project = await postProcessor.RemoveAsync(_project); + _project = await MeasurePostProcessingStepAsync("PostProcess.InternalizeAsync", () => postProcessor.InternalizeAsync(_project)); + _project = await MeasurePostProcessingStepAsync("PostProcess.RemoveAsync", () => postProcessor.RemoveAsync(_project)); break; } } diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspacePostProcessingProfile.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspacePostProcessingProfile.cs new file mode 100644 index 00000000000..74a9a58168a --- /dev/null +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspacePostProcessingProfile.cs @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text; + +namespace Microsoft.TypeSpec.Generator +{ + internal sealed class GeneratedCodeWorkspacePostProcessingProfile + { + private readonly object _syncRoot = new(); + private readonly Dictionary _steps = new(StringComparer.Ordinal); + + public void Add(string stepName, TimeSpan elapsed, long allocatedBytes) + { + lock (_syncRoot) + { + ref var summary = ref CollectionsMarshal.GetValueRefOrAddDefault(_steps, stepName, out _); + summary.Count++; + summary.ElapsedTicks += elapsed.Ticks; + summary.AllocatedBytes += allocatedBytes; + } + } + + public string GetSummary() + { + KeyValuePair[] steps; + lock (_syncRoot) + { + steps = _steps.ToArray(); + } + + var totalTicks = steps + .Where(static step => step.Key != "ProcessDocument.Total") + .Sum(static step => step.Value.ElapsedTicks); + var builder = new StringBuilder(); + builder.AppendLine("Post-processing step profile:"); + builder.AppendLine("Step, Count, Total ms, Avg ms, Percent of measured steps, Allocated bytes, Avg allocated bytes"); + + foreach (var step in steps.OrderByDescending(static step => step.Value.ElapsedTicks)) + { + var elapsedMs = TimeSpan.FromTicks(step.Value.ElapsedTicks).TotalMilliseconds; + var averageMs = elapsedMs / step.Value.Count; + var averageAllocatedBytes = step.Value.AllocatedBytes / step.Value.Count; + var percentage = totalTicks == 0 || step.Key == "ProcessDocument.Total" + ? 0 + : step.Value.ElapsedTicks * 100.0 / totalTicks; + builder.AppendLine($"{step.Key}, {step.Value.Count}, {elapsedMs:F3}, {averageMs:F3}, {percentage:F1}%, {step.Value.AllocatedBytes}, {averageAllocatedBytes}"); + } + + return builder.ToString(); + } + + private struct StepSummary + { + public int Count; + public long ElapsedTicks; + public long AllocatedBytes; + } + } +} diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs index be96e11df59..d5db7e47298 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs @@ -20,6 +20,8 @@ internal class PostProcessor private readonly HashSet _typesToKeep; private INamedTypeSymbol? _modelFactorySymbol; + private static GeneratedCodeWorkspacePostProcessingProfile? Profile => GeneratedCodeWorkspace.PostProcessingProfile; + public PostProcessor( HashSet typesToKeep, string? modelFactoryFullName = null, @@ -125,40 +127,55 @@ protected virtual bool ShouldIncludeDocument(Document document) => /// The processed . is immutable, therefore this should usually be a new instance public async Task InternalizeAsync(Project project) { - var compilation = await project.GetCompilationAsync(); + var compilation = await MeasureAsync("PostProcessor.Internalize.GetCompilationAsync", () => project.GetCompilationAsync()); if (compilation == null) return project; // first get all the declared symbols - var definitions = await GetTypeSymbolsAsync(compilation, project, true); + var definitions = await MeasureAsync("PostProcessor.Internalize.GetTypeSymbolsAsync", () => GetTypeSymbolsAsync(compilation, project, true)); // build the reference map var referenceMap = - await new ReferenceMapBuilder(compilation, project).BuildPublicReferenceMapAsync( - definitions.DeclaredSymbols, definitions.DeclaredNodesCache); + await MeasureAsync( + "PostProcessor.Internalize.BuildPublicReferenceMapAsync", + () => new ReferenceMapBuilder(compilation, project).BuildPublicReferenceMapAsync( + definitions.DeclaredSymbols, definitions.DeclaredNodesCache)); // get the root symbols - var rootSymbols = await GetRootSymbolsAsync(project, definitions); + var rootSymbols = await MeasureAsync("PostProcessor.Internalize.GetRootSymbolsAsync", () => GetRootSymbolsAsync(project, definitions)); // traverse all the root and recursively add all the things we met - var publicSymbols = VisitSymbolsFromRootAsync(rootSymbols, referenceMap); + var publicSymbols = Measure("PostProcessor.Internalize.VisitSymbolsFromRoot", () => VisitSymbolsFromRootAsync(rootSymbols, referenceMap).ToArray()); var symbolsToInternalize = definitions.DeclaredSymbols.Except(publicSymbols); - var nodesToInternalize = new Dictionary(); - foreach (var symbol in symbolsToInternalize) + var nodesToInternalize = Measure("PostProcessor.Internalize.CollectNodes", () => { - foreach (var node in definitions.DeclaredNodesCache[symbol]) + var nodes = new Dictionary(); + foreach (var symbol in symbolsToInternalize) { - nodesToInternalize[node] = project.GetDocumentId(node.SyntaxTree)!; + foreach (var node in definitions.DeclaredNodesCache[symbol]) + { + nodes[node] = project.GetDocumentId(node.SyntaxTree)!; + } } - } - foreach (var (model, documentId) in nodesToInternalize) + return nodes; + }); + + project = Measure("PostProcessor.Internalize.MarkInternal", () => { - project = MarkInternal(project, model, documentId); - } + var updatedProject = project; + foreach (var (model, documentId) in nodesToInternalize) + { + updatedProject = MarkInternal(updatedProject, model, documentId); + } + + return updatedProject; + }); var modelNamesToRemove = nodesToInternalize.Keys.Select(item => item.Identifier.Text); - project = await RemoveMethodsFromModelFactoryAsync(project, definitions, modelNamesToRemove.ToHashSet()); + project = await MeasureAsync( + "PostProcessor.Internalize.RemoveMethodsFromModelFactoryAsync", + () => RemoveMethodsFromModelFactoryAsync(project, definitions, modelNamesToRemove.ToHashSet())); return project; } @@ -232,42 +249,49 @@ private async Task RemoveMethodsFromModelFactoryAsync(Project project, /// The processed . is immutable, therefore this should usually be a new instance public async Task RemoveAsync(Project project) { - var compilation = await project.GetCompilationAsync(); + var compilation = await MeasureAsync("PostProcessor.Remove.GetCompilationAsync", () => project.GetCompilationAsync()); if (compilation == null) return project; // find all the declarations, including non-public declared - var definitions = await GetTypeSymbolsAsync(compilation, project, false); + var definitions = await MeasureAsync("PostProcessor.Remove.GetTypeSymbolsAsync", () => GetTypeSymbolsAsync(compilation, project, false)); // build reference map var referenceMap = - await new ReferenceMapBuilder(compilation, project).BuildAllReferenceMapAsync( - definitions.DeclaredSymbols, definitions.DocumentsCache); + await MeasureAsync( + "PostProcessor.Remove.BuildAllReferenceMapAsync", + () => new ReferenceMapBuilder(compilation, project).BuildAllReferenceMapAsync( + definitions.DeclaredSymbols, definitions.DocumentsCache)); // get root symbols - var rootSymbols = await GetRootSymbolsAsync(project, definitions); + var rootSymbols = await MeasureAsync("PostProcessor.Remove.GetRootSymbolsAsync", () => GetRootSymbolsAsync(project, definitions)); // include model factory as a root symbol when doing the remove pass so that we are sure to include any internal // helpers that are required by the model factory. if (_modelFactorySymbol != null) rootSymbols.Add(_modelFactorySymbol); // traverse the map to determine the declarations that we are about to remove, starting from root nodes - var referencedSymbols = VisitSymbolsFromRootAsync(rootSymbols, referenceMap); + var referencedSymbols = Measure("PostProcessor.Remove.VisitSymbolsFromRoot", () => VisitSymbolsFromRootAsync(rootSymbols, referenceMap).ToArray().AsEnumerable()); - referencedSymbols = AddSampleSymbols(referencedSymbols, definitions.DeclaredSymbols); - var referencedSet = new HashSet(referencedSymbols, SymbolEqualityComparer.Default); + referencedSymbols = Measure("PostProcessor.Remove.AddSampleSymbols", () => AddSampleSymbols(referencedSymbols, definitions.DeclaredSymbols)); + var referencedSet = Measure("PostProcessor.Remove.BuildReferencedSet", () => new HashSet(referencedSymbols, SymbolEqualityComparer.Default)); var symbolsToRemove = definitions.DeclaredSymbols.Except(referencedSet); - var nodesToRemove = new List(); - foreach (var symbol in symbolsToRemove) + var nodesToRemove = Measure("PostProcessor.Remove.CollectNodes", () => { - if (referencedSet.Contains(GetBase(symbol))) + var nodes = new List(); + foreach (var symbol in symbolsToRemove) { - continue; + if (referencedSet.Contains(GetBase(symbol))) + { + continue; + } + nodes.AddRange(definitions.DeclaredNodesCache[symbol]); } - nodesToRemove.AddRange(definitions.DeclaredNodesCache[symbol]); - } + + return nodes; + }); // remove them one by one - project = await RemoveModelsAsync(project, nodesToRemove); + project = await MeasureAsync("PostProcessor.Remove.RemoveModelsAsync", () => RemoveModelsAsync(project, nodesToRemove)); return project; } @@ -349,25 +373,35 @@ private async Task RemoveModelsAsync(Project project, IEnumerable unusedModels) { // accumulate the definitions from the same document together - var documents = new Dictionary>(); - - foreach (var model in unusedModels) + var documents = Measure("PostProcessor.Remove.RemoveModelsAsync.GroupByDocument", () => { - var document = project.GetDocument(model.SyntaxTree); - Debug.Assert(document != null); - if (!documents.ContainsKey(document)) - documents.Add(document, new HashSet()); + var groupedDocuments = new Dictionary>(); + foreach (var model in unusedModels) + { + var document = project.GetDocument(model.SyntaxTree); + Debug.Assert(document != null); + if (!groupedDocuments.ContainsKey(document)) + groupedDocuments.Add(document, new HashSet()); - documents[document].Add(model); - } + groupedDocuments[document].Add(model); + } - foreach (var models in documents.Values) + return groupedDocuments; + }); + + project = await MeasureAsync("PostProcessor.Remove.RemoveModelsAsync.RemoveModelsFromDocuments", async () => { - project = await RemoveModelsFromDocumentAsync(project, models); - } + var updatedProject = project; + foreach (var models in documents.Values) + { + updatedProject = await RemoveModelsFromDocumentAsync(updatedProject, models); + } + + return updatedProject; + }); // remove what are now invalid references due to the models being removed - project = await RemoveInvalidRefs(project); + project = await MeasureAsync("PostProcessor.Remove.RemoveModelsAsync.RemoveInvalidRefs", () => RemoveInvalidRefs(project)); return project; } @@ -418,16 +452,28 @@ private async Task RemoveInvalidRefs(Project project) var solution = project.Solution; // Process each document for invalid usings - foreach (var documentId in project.DocumentIds) + solution = await MeasureAsync("PostProcessor.Remove.RemoveInvalidRefs.RemoveInvalidUsings", async () => { - solution = await RemoveInvalidUsings(solution, documentId); - } + var updatedSolution = solution; + foreach (var documentId in project.DocumentIds) + { + updatedSolution = await RemoveInvalidUsings(updatedSolution, documentId); + } + + return updatedSolution; + }); // Process each document for invalid attributes (with fresh semantic models) - foreach (var documentId in project.DocumentIds) + solution = await MeasureAsync("PostProcessor.Remove.RemoveInvalidRefs.RemoveInvalidAttributes", async () => { - solution = await RemoveInvalidAttributes(solution, documentId); - } + var updatedSolution = solution; + foreach (var documentId in project.DocumentIds) + { + updatedSolution = await RemoveInvalidAttributes(updatedSolution, documentId); + } + + return updatedSolution; + }); return solution.GetProject(project.Id)!; } @@ -533,6 +579,48 @@ arg.Expression is TypeOfExpressionSyntax typeOfExpr && return solution; } + private static T Measure(string stepName, Func action) + { + var profile = Profile; + if (profile == null) + { + return action(); + } + + var allocatedBytes = GC.GetTotalAllocatedBytes(precise: false); + var stopwatch = Stopwatch.StartNew(); + try + { + return action(); + } + finally + { + stopwatch.Stop(); + profile.Add(stepName, stopwatch.Elapsed, GC.GetTotalAllocatedBytes(precise: false) - allocatedBytes); + } + } + + private static async Task MeasureAsync(string stepName, Func> action) + { + var profile = Profile; + if (profile == null) + { + return await action(); + } + + var allocatedBytes = GC.GetTotalAllocatedBytes(precise: false); + var stopwatch = Stopwatch.StartNew(); + try + { + return await action(); + } + finally + { + stopwatch.Stop(); + profile.Add(stepName, stopwatch.Elapsed, GC.GetTotalAllocatedBytes(precise: false) - allocatedBytes); + } + } + private async Task> GetRootSymbolsAsync(Project project, TypeSymbols modelSymbols) { var result = new HashSet(SymbolEqualityComparer.Default); From f022e78e26c3b645cdd3cf506cba931303429179 Mon Sep 17 00:00:00 2001 From: Wei Hu Date: Fri, 12 Jun 2026 04:22:34 +0000 Subject: [PATCH 02/13] Add provider reference map shadow replacement --- .../src/CSharpGen.cs | 2 + .../PostProcessing/GeneratedCodeWorkspace.cs | 5 + .../src/PostProcessing/PostProcessor.cs | 110 +++- .../ProviderReferenceMapShadowAnalyzer.cs | 616 ++++++++++++++++++ .../ProviderReferenceMapShadowResult.cs | 14 + 5 files changed, 716 insertions(+), 31 deletions(-) create mode 100644 packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapShadowAnalyzer.cs create mode 100644 packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapShadowResult.cs diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/CSharpGen.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/CSharpGen.cs index 238b16b9766..fdd3ae2191f 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/CSharpGen.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/CSharpGen.cs @@ -121,6 +121,8 @@ await customCodeWorkspace.GetCompilationAsync(), // Add all the generated files to the workspace await MeasureGenerationStepAsync("Generation.AddGeneratedFilesToWorkspace", () => Task.WhenAll(generateFilesTasks)); + MeasureGenerationStep("Generation.ProviderReferenceMapShadowAnalysis", () => generatedCodeWorkspace.AnalyzeProviderReferenceMap(output.TypeProviders)); + LoggingHelpers.LogElapsedTime("All generated types have been written into memory"); // Delete any old generated files diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspace.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspace.cs index 281d4ec2e5e..07e8595be08 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspace.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspace.cs @@ -85,6 +85,11 @@ public async Task AddInMemoryFile(TypeProvider type) await UpdateProject(document); } + internal void AnalyzeProviderReferenceMap(IReadOnlyList providers) + { + ProviderReferenceMapShadowAnalyzer.Analyze(providers, _project); + } + private async Task UpdateProject(Document document) { var root = await document.GetSyntaxRootAsync(); diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs index d5db7e47298..57136754da1 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs @@ -133,18 +133,35 @@ public async Task InternalizeAsync(Project project) // first get all the declared symbols var definitions = await MeasureAsync("PostProcessor.Internalize.GetTypeSymbolsAsync", () => GetTypeSymbolsAsync(compilation, project, true)); - // build the reference map - var referenceMap = - await MeasureAsync( - "PostProcessor.Internalize.BuildPublicReferenceMapAsync", - () => new ReferenceMapBuilder(compilation, project).BuildPublicReferenceMapAsync( - definitions.DeclaredSymbols, definitions.DeclaredNodesCache)); - // get the root symbols - var rootSymbols = await MeasureAsync("PostProcessor.Internalize.GetRootSymbolsAsync", () => GetRootSymbolsAsync(project, definitions)); - // traverse all the root and recursively add all the things we met - var publicSymbols = Measure("PostProcessor.Internalize.VisitSymbolsFromRoot", () => VisitSymbolsFromRootAsync(rootSymbols, referenceMap).ToArray()); - - var symbolsToInternalize = definitions.DeclaredSymbols.Except(publicSymbols); + IEnumerable symbolsToInternalize; + if (ProviderReferenceMapShadowAnalyzer.UseShadowMap && ProviderReferenceMapShadowAnalyzer.LatestResult is { } useShadowResult) + { + symbolsToInternalize = Measure("PostProcessor.Internalize.UseShadowCandidates", () => + GetSymbolsByName(definitions.DeclaredSymbols, useShadowResult.InternalizeCandidates).ToArray()); + } + else + { + // build the reference map + var referenceMap = + await MeasureAsync( + "PostProcessor.Internalize.BuildPublicReferenceMapAsync", + () => new ReferenceMapBuilder(compilation, project).BuildPublicReferenceMapAsync( + definitions.DeclaredSymbols, definitions.DeclaredNodesCache)); + // get the root symbols + var rootSymbols = await MeasureAsync("PostProcessor.Internalize.GetRootSymbolsAsync", () => GetRootSymbolsAsync(project, definitions)); + // traverse all the root and recursively add all the things we met + var publicSymbols = Measure("PostProcessor.Internalize.VisitSymbolsFromRoot", () => VisitSymbolsFromRootAsync(rootSymbols, referenceMap).ToArray()); + + symbolsToInternalize = definitions.DeclaredSymbols.Except(publicSymbols); + } + + if (ProviderReferenceMapShadowAnalyzer.LatestResult is { } shadowResult) + { + ProviderReferenceMapShadowAnalyzer.WriteComparisonReport( + "internalize", + symbolsToInternalize.Select(static symbol => symbol.GetFullyQualifiedName()), + shadowResult.InternalizeCandidates); + } var nodesToInternalize = Measure("PostProcessor.Internalize.CollectNodes", () => { @@ -255,25 +272,45 @@ public async Task RemoveAsync(Project project) // find all the declarations, including non-public declared var definitions = await MeasureAsync("PostProcessor.Remove.GetTypeSymbolsAsync", () => GetTypeSymbolsAsync(compilation, project, false)); - // build reference map - var referenceMap = - await MeasureAsync( - "PostProcessor.Remove.BuildAllReferenceMapAsync", - () => new ReferenceMapBuilder(compilation, project).BuildAllReferenceMapAsync( - definitions.DeclaredSymbols, definitions.DocumentsCache)); - // get root symbols - var rootSymbols = await MeasureAsync("PostProcessor.Remove.GetRootSymbolsAsync", () => GetRootSymbolsAsync(project, definitions)); - // include model factory as a root symbol when doing the remove pass so that we are sure to include any internal - // helpers that are required by the model factory. - if (_modelFactorySymbol != null) - rootSymbols.Add(_modelFactorySymbol); - // traverse the map to determine the declarations that we are about to remove, starting from root nodes - var referencedSymbols = Measure("PostProcessor.Remove.VisitSymbolsFromRoot", () => VisitSymbolsFromRootAsync(rootSymbols, referenceMap).ToArray().AsEnumerable()); - - referencedSymbols = Measure("PostProcessor.Remove.AddSampleSymbols", () => AddSampleSymbols(referencedSymbols, definitions.DeclaredSymbols)); - var referencedSet = Measure("PostProcessor.Remove.BuildReferencedSet", () => new HashSet(referencedSymbols, SymbolEqualityComparer.Default)); - - var symbolsToRemove = definitions.DeclaredSymbols.Except(referencedSet); + IEnumerable symbolsToRemove; + HashSet referencedSet; + if (ProviderReferenceMapShadowAnalyzer.UseShadowMap && ProviderReferenceMapShadowAnalyzer.LatestResult is { } useShadowResult) + { + symbolsToRemove = Measure("PostProcessor.Remove.UseShadowCandidates", () => + GetSymbolsByName(definitions.DeclaredSymbols, useShadowResult.RemoveCandidates).ToArray()); + referencedSet = Measure("PostProcessor.Remove.BuildShadowReferencedSet", () => + new HashSet(definitions.DeclaredSymbols.Except(symbolsToRemove), SymbolEqualityComparer.Default)); + } + else + { + // build reference map + var referenceMap = + await MeasureAsync( + "PostProcessor.Remove.BuildAllReferenceMapAsync", + () => new ReferenceMapBuilder(compilation, project).BuildAllReferenceMapAsync( + definitions.DeclaredSymbols, definitions.DocumentsCache)); + // get root symbols + var rootSymbols = await MeasureAsync("PostProcessor.Remove.GetRootSymbolsAsync", () => GetRootSymbolsAsync(project, definitions)); + // include model factory as a root symbol when doing the remove pass so that we are sure to include any internal + // helpers that are required by the model factory. + if (_modelFactorySymbol != null) + rootSymbols.Add(_modelFactorySymbol); + // traverse the map to determine the declarations that we are about to remove, starting from root nodes + var referencedSymbols = Measure("PostProcessor.Remove.VisitSymbolsFromRoot", () => VisitSymbolsFromRootAsync(rootSymbols, referenceMap).ToArray().AsEnumerable()); + + referencedSymbols = Measure("PostProcessor.Remove.AddSampleSymbols", () => AddSampleSymbols(referencedSymbols, definitions.DeclaredSymbols)); + referencedSet = Measure("PostProcessor.Remove.BuildReferencedSet", () => new HashSet(referencedSymbols, SymbolEqualityComparer.Default)); + + symbolsToRemove = definitions.DeclaredSymbols.Except(referencedSet); + } + + if (ProviderReferenceMapShadowAnalyzer.LatestResult is { } shadowResult) + { + ProviderReferenceMapShadowAnalyzer.WriteComparisonReport( + "remove", + symbolsToRemove.Select(static symbol => symbol.GetFullyQualifiedName()), + shadowResult.RemoveCandidates); + } var nodesToRemove = Measure("PostProcessor.Remove.CollectNodes", () => { @@ -358,6 +395,17 @@ private static IEnumerable GetReferencedTypes(T definition, return Enumerable.Empty(); } + private static IEnumerable GetSymbolsByName(IEnumerable symbols, HashSet names) + { + foreach (var symbol in symbols) + { + if (names.Contains(symbol.GetFullyQualifiedName())) + { + yield return symbol; + } + } + } + private Project MarkInternal(Project project, BaseTypeDeclarationSyntax declarationNode, DocumentId documentId) { var newNode = ChangeModifier(declarationNode, SyntaxKind.PublicKeyword, SyntaxKind.InternalKeyword); diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapShadowAnalyzer.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapShadowAnalyzer.cs new file mode 100644 index 00000000000..7d549afb5a1 --- /dev/null +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapShadowAnalyzer.cs @@ -0,0 +1,616 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Microsoft.TypeSpec.Generator.Primitives; +using Microsoft.TypeSpec.Generator.Providers; +using Microsoft.TypeSpec.Generator.Statements; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; + +namespace Microsoft.TypeSpec.Generator +{ + internal static class ProviderReferenceMapShadowAnalyzer + { + private const string EnableEnvironmentVariable = "TYPESPEC_PROVIDER_REFERENCE_MAP_SHADOW"; + private const string UseShadowEnvironmentVariable = "TYPESPEC_PROVIDER_REFERENCE_MAP_USE_SHADOW"; + private const string OutputDirectoryEnvironmentVariable = "TYPESPEC_PROVIDER_REFERENCE_MAP_SHADOW_DIR"; + + private static ProviderReferenceMapShadowResult? _latestResult; + + public static bool IsEnabled => string.Equals( + Environment.GetEnvironmentVariable(EnableEnvironmentVariable), + "true", + StringComparison.OrdinalIgnoreCase); + + public static ProviderReferenceMapShadowResult? LatestResult => _latestResult; + + public static bool UseShadowMap => string.Equals( + Environment.GetEnvironmentVariable(UseShadowEnvironmentVariable), + "true", + StringComparison.OrdinalIgnoreCase); + + public static void Analyze(IReadOnlyList providers, Project project) + { + if (!IsEnabled) + { + _latestResult = null; + return; + } + + var graph = BuildGraph(providers); + var customRoots = GetCustomCodeGeneratedTypeRoots(project, graph.Nodes); + var internalizeRoots = GetRootNames(providers, graph.Nodes, helperRoots: [], includeModelFactory: false); + internalizeRoots.UnionWith(customRoots); + var internalizeReachableWithoutHelpers = GetReachableTypes(internalizeRoots, graph.References); + var internalizeHelperRoots = GetHelperRootNames(providers, graph.Nodes, internalizeReachableWithoutHelpers); + internalizeRoots.UnionWith(internalizeHelperRoots); + var internalizeReachable = GetReachableTypes(internalizeRoots, graph.References); + var internalizeDeclaredNodes = GetPostProcessorDeclaredNodes(providers, graph.Nodes, publicOnly: true); + var internalizeCandidates = internalizeDeclaredNodes.Except(internalizeReachable, StringComparer.Ordinal).OrderBy(static name => name, StringComparer.Ordinal).ToArray(); + + var removeRoots = GetRootNames(providers, graph.Nodes, helperRoots: [], includeModelFactory: true); + removeRoots.UnionWith(customRoots); + var removeReachableWithoutHelpers = GetReachableTypes(removeRoots, graph.References); + var removeHelperRoots = GetHelperRootNames(providers, graph.Nodes, removeReachableWithoutHelpers); + removeRoots.UnionWith(removeHelperRoots); + var removeReachable = GetReachableTypes(removeRoots, graph.References); + var removeDeclaredNodes = GetPostProcessorDeclaredNodes(providers, graph.Nodes, publicOnly: false); + var removeCandidates = removeDeclaredNodes.Except(removeReachable, StringComparer.Ordinal).OrderBy(static name => name, StringComparer.Ordinal).ToArray(); + + var helperRoots = internalizeHelperRoots.Concat(removeHelperRoots).ToHashSet(StringComparer.Ordinal); + + _latestResult = new ProviderReferenceMapShadowResult( + internalizeCandidates.ToHashSet(StringComparer.Ordinal), + removeCandidates.ToHashSet(StringComparer.Ordinal)); + + WriteReport(graph, customRoots, helperRoots, internalizeRoots, internalizeReachable, internalizeCandidates, removeRoots, removeReachable, removeCandidates); + } + + private static HashSet GetCustomCodeGeneratedTypeRoots(Project project, HashSet generatedTypeNames) + { + var roots = new HashSet(StringComparer.Ordinal); + var compilation = project.GetCompilationAsync().GetAwaiter().GetResult(); + if (compilation == null) + { + return roots; + } + + foreach (var document in project.Documents) + { + if (GeneratedCodeWorkspace.IsGeneratedDocument(document) || GeneratedCodeWorkspace.IsGeneratedTestDocument(document)) + { + continue; + } + + var root = document.GetSyntaxRootAsync().GetAwaiter().GetResult(); + if (root == null) + { + continue; + } + + var model = compilation.GetSemanticModel(root.SyntaxTree); + foreach (var declaration in root.DescendantNodes().OfType()) + { + AddSymbolRoot(roots, model.GetDeclaredSymbol(declaration) as ITypeSymbol, generatedTypeNames); + } + + foreach (var typeSyntax in root.DescendantNodes().OfType()) + { + AddSymbolRoot(roots, model.GetTypeInfo(typeSyntax).Type, generatedTypeNames); + } + + foreach (var objectCreation in root.DescendantNodes().OfType()) + { + AddSymbolRoot(roots, model.GetSymbolInfo(objectCreation).Symbol?.ContainingType, generatedTypeNames); + } + + foreach (var invocation in root.DescendantNodes().OfType()) + { + AddSymbolRoot(roots, model.GetSymbolInfo(invocation).Symbol?.ContainingType, generatedTypeNames); + } + } + + return roots; + } + + private static void AddSymbolRoot(HashSet roots, ITypeSymbol? symbol, HashSet generatedTypeNames) + { + if (symbol is not INamedTypeSymbol namedType) + { + return; + } + + AddMatchingName(roots, namedType.GetFullyQualifiedName(), generatedTypeNames); + foreach (var typeArgument in namedType.TypeArguments) + { + AddSymbolRoot(roots, typeArgument, generatedTypeNames); + } + } + + public static void WriteComparisonReport(string passName, IEnumerable roslynCandidates, IEnumerable providerCandidates) + { + if (!IsEnabled) + { + return; + } + + var roslynSet = roslynCandidates.ToHashSet(StringComparer.Ordinal); + var providerSet = providerCandidates.ToHashSet(StringComparer.Ordinal); + var missingFromProvider = roslynSet.Except(providerSet, StringComparer.Ordinal).OrderBy(static name => name, StringComparer.Ordinal).ToArray(); + var extraInProvider = providerSet.Except(roslynSet, StringComparer.Ordinal).OrderBy(static name => name, StringComparer.Ordinal).ToArray(); + + var directory = GetOutputDirectory(); + Directory.CreateDirectory(directory); + var path = Path.Combine(directory, $"provider-reference-map-shadow-comparison-{passName}-{DateTime.UtcNow:yyyyMMddHHmmssfff}.txt"); + var builder = new StringBuilder(); + builder.AppendLine($"Provider reference map shadow comparison: {passName}"); + builder.AppendLine($"Roslyn candidates: {roslynSet.Count}"); + builder.AppendLine($"Provider candidates: {providerSet.Count}"); + builder.AppendLine($"Missing from provider: {missingFromProvider.Length}"); + builder.AppendLine($"Extra in provider: {extraInProvider.Length}"); + builder.AppendLine(); + builder.AppendLine("Missing from provider:"); + foreach (var item in missingFromProvider) + { + builder.AppendLine($" {item}"); + } + + builder.AppendLine(); + builder.AppendLine("Extra in provider:"); + foreach (var item in extraInProvider) + { + builder.AppendLine($" {item}"); + } + + File.WriteAllText(path, builder.ToString()); + CodeModelGenerator.Instance.Emitter.Debug($"Provider reference map shadow comparison written to {path}"); + } + + private static ProviderReferenceGraph BuildGraph(IReadOnlyList providers) + { + var nodes = providers + .Select(static provider => GetProviderTypeName(provider.Type)) + .ToHashSet(StringComparer.Ordinal); + var references = nodes.ToDictionary(static name => name, _ => new HashSet(StringComparer.Ordinal), StringComparer.Ordinal); + + foreach (var provider in providers) + { + var current = GetProviderTypeName(provider.Type); + AddTypeReference(references[current], provider.Type, nodes); + AddTypeReference(references[current], provider.BaseType, nodes); + AddTypeReference(references[current], provider.DeclaringTypeProvider?.Type, nodes); + + if (IsModelFactoryProvider(provider)) + { + continue; + } + + foreach (var implementedType in provider.Implements) + { + AddTypeReference(references[current], implementedType, nodes); + } + + foreach (var nestedType in provider.NestedTypes) + { + AddTypeReference(references[current], nestedType.Type, nodes); + } + + foreach (var serializationProvider in provider.SerializationProviders) + { + AddTypeReference(references[current], serializationProvider.Type, nodes); + } + + foreach (var property in provider.Properties) + { + AddTypeReference(references[current], property.Type, nodes); + AddTypeReference(references[current], property.ExplicitInterface, nodes); + AddAttributes(references[current], property.Attributes, nodes); + } + + foreach (var field in provider.Fields) + { + AddTypeReference(references[current], field.Type, nodes); + AddAttributes(references[current], field.Attributes, nodes); + } + + foreach (var constructor in provider.Constructors) + { + AddSignatureReferences(references[current], constructor.Signature, nodes); + } + + foreach (var method in provider.Methods) + { + AddSignatureReferences(references[current], method.Signature, nodes); + } + } + + return new ProviderReferenceGraph(nodes, references); + } + + private static HashSet GetRootNames(IReadOnlyList providers, HashSet nodes, HashSet helperRoots, bool includeModelFactory) + { + var generator = CodeModelGenerator.Instance; + var roots = new HashSet(StringComparer.Ordinal); + var modelFactoryName = GetProviderTypeName(generator.OutputLibrary.ModelFactory.Value.Type); + + foreach (var provider in providers) + { + var name = GetProviderTypeName(provider.Type); + if (provider.Name.EndsWith("Client", StringComparison.Ordinal) || + IsKept(provider.Type, generator.AdditionalRootTypes, nodes) || + includeModelFactory && string.Equals(name, modelFactoryName, StringComparison.Ordinal) || + includeModelFactory && helperRoots.Contains(name)) + { + roots.Add(name); + } + } + + foreach (var root in generator.TypeFactory.UnionVariantTypesToKeep) + { + AddMatchingName(roots, root, nodes); + } + + foreach (var root in generator.AdditionalRootTypes) + { + AddMatchingName(roots, root, nodes); + } + + return roots; + } + + private static HashSet GetPostProcessorDeclaredNodes(IReadOnlyList providers, HashSet nodes, bool publicOnly) + { + var generator = CodeModelGenerator.Instance; + var excludedNames = generator.NonRootTypes; + return providers + .Where(provider => !IsModelFactoryProvider(provider)) + .Where(provider => !publicOnly || provider.DeclarationModifiers.HasFlag(TypeSignatureModifiers.Public)) + .Select(provider => GetProviderTypeName(provider.Type)) + .Where(name => nodes.Contains(name)) + .Where(name => !excludedNames.Contains(name) && !excludedNames.Contains(GetSimpleName(name))) + .ToHashSet(StringComparer.Ordinal); + } + + private static bool IsKept(CSharpType type, HashSet roots, HashSet nodes) => + roots.Contains(type.Name) || roots.Contains(GetProviderTypeName(type)) && nodes.Contains(GetProviderTypeName(type)); + + private static bool IsModelFactoryProvider(TypeProvider provider) => provider.GetType().Name == "ModelFactoryProvider"; + + private static HashSet GetHelperRootNames(IReadOnlyList providers, HashSet nodes, HashSet reachableTypes) + { + var roots = new HashSet(StringComparer.Ordinal); + foreach (var provider in providers) + { + var providerName = GetProviderTypeName(provider.Type); + var isModelFactory = IsModelFactoryProvider(provider); + if (!reachableTypes.Contains(providerName) && !isModelFactory) + { + continue; + } + + foreach (var property in provider.Properties) + { + AddInitializationHelperRoot(roots, property.Type, nodes); + AddParameterValidationHelperRoot(roots, property.AsParameter, nodes); + } + + foreach (var field in provider.Fields) + { + AddParameterValidationHelperRoot(roots, field.AsParameter, nodes); + } + + foreach (var constructor in provider.Constructors) + { + foreach (var parameter in constructor.Signature.Parameters) + { + AddParameterValidationHelperRoot(roots, parameter, nodes); + } + } + + foreach (var method in provider.Methods) + { + if (isModelFactory && + (method.Signature.ReturnType == null || !reachableTypes.Contains(GetProviderTypeName(method.Signature.ReturnType)))) + { + continue; + } + + foreach (var parameter in method.Signature.Parameters) + { + AddParameterValidationHelperRoot(roots, parameter, nodes); + if (isModelFactory) + { + AddModelFactoryCollectionInitializationHelperRoot(roots, parameter.Type, nodes); + } + } + } + } + + return roots; + } + + private static void AddParameterValidationHelperRoot(HashSet roots, ParameterProvider parameter, HashSet nodes) + { + if (parameter.Validation != ParameterValidationType.None) + { + AddMatchingName(roots, "Argument", nodes); + } + } + + private static void AddInitializationHelperRoot(HashSet roots, CSharpType? type, HashSet nodes) + { + if (type == null) + { + return; + } + + var initializationType = type.PropertyInitializationType; + if (!string.Equals(initializationType.FullyQualifiedName, type.FullyQualifiedName, StringComparison.Ordinal)) + { + AddMatchingName(roots, initializationType.Name, nodes); + } + + if (type is { IsList: true, IsReadOnlyMemory: false }) + { + AddMatchingName(roots, "ChangeTrackingList", nodes); + } + + foreach (var argument in type.Arguments) + { + AddInitializationHelperRoot(roots, argument, nodes); + } + } + + private static void AddModelFactoryCollectionInitializationHelperRoot(HashSet roots, CSharpType? type, HashSet nodes) + { + if (type == null) + { + return; + } + + if (type is { IsList: true, IsReadOnlyMemory: false }) + { + AddMatchingName(roots, "ChangeTrackingList", nodes); + } + + if (type.IsDictionary) + { + AddMatchingName(roots, "ChangeTrackingDictionary", nodes); + } + + foreach (var argument in type.Arguments) + { + AddModelFactoryCollectionInitializationHelperRoot(roots, argument, nodes); + } + } + + private static void AddMatchingName(HashSet target, string name, HashSet nodes) + { + if (nodes.Contains(name)) + { + target.Add(name); + return; + } + + foreach (var node in nodes) + { + if (string.Equals(StripGenericArity(GetSimpleName(node)), name, StringComparison.Ordinal)) + { + target.Add(node); + } + } + } + + private static HashSet GetReachableTypes(HashSet roots, IReadOnlyDictionary> references) + { + var reachable = new HashSet(StringComparer.Ordinal); + var queue = new Queue(roots); + while (queue.Count > 0) + { + var current = queue.Dequeue(); + if (!reachable.Add(current)) + { + continue; + } + + if (!references.TryGetValue(current, out var children)) + { + continue; + } + + foreach (var child in children) + { + queue.Enqueue(child); + } + } + + return reachable; + } + + private static void AddSignatureReferences(HashSet references, MethodSignatureBase signature, HashSet nodes) + { + AddTypeReference(references, signature.ReturnType, nodes); + AddAttributes(references, signature.Attributes, nodes); + + foreach (var parameter in signature.Parameters) + { + AddTypeReference(references, parameter.Type, nodes); + AddAttributes(references, parameter.Attributes, nodes); + } + + if (signature is MethodSignature methodSignature) + { + AddTypeReference(references, methodSignature.ExplicitInterface, nodes); + if (methodSignature.GenericArguments != null) + { + foreach (var genericArgument in methodSignature.GenericArguments) + { + AddTypeReference(references, genericArgument, nodes); + } + } + + if (methodSignature.GenericParameterConstraints != null) + { + foreach (var constraint in methodSignature.GenericParameterConstraints) + { + AddTypeReference(references, constraint.Type, nodes); + } + } + } + + if (signature is ConstructorSignature constructorSignature) + { + AddTypeReference(references, constructorSignature.Type, nodes); + } + } + + private static void AddAttributes(HashSet references, IReadOnlyList attributes, HashSet nodes) + { + foreach (var attribute in attributes) + { + AddTypeReference(references, attribute.Type, nodes); + } + } + + private static void AddTypeReference(HashSet references, CSharpType? type, HashSet nodes) + { + if (type == null) + { + return; + } + + var providerTypeName = GetProviderTypeName(type); + if (nodes.Contains(providerTypeName)) + { + references.Add(providerTypeName); + } + + AddTypeReference(references, type.BaseType, nodes); + AddTypeReference(references, type.DeclaringType, nodes); + foreach (var argument in type.Arguments) + { + AddTypeReference(references, argument, nodes); + } + } + + private static void WriteReport( + ProviderReferenceGraph graph, + HashSet customRoots, + HashSet helperRoots, + HashSet internalizeRoots, + HashSet internalizeReachable, + IReadOnlyList internalizeCandidates, + HashSet removeRoots, + HashSet removeReachable, + IReadOnlyList removeCandidates) + { + var directory = GetOutputDirectory(); + + Directory.CreateDirectory(directory); + var path = Path.Combine(directory, $"provider-reference-map-shadow-{DateTime.UtcNow:yyyyMMddHHmmssfff}.txt"); + var builder = new StringBuilder(); + builder.AppendLine("Provider reference map shadow report"); + builder.AppendLine($"Declared providers: {graph.Nodes.Count}"); + builder.AppendLine($"Internalize roots: {internalizeRoots.Count}"); + builder.AppendLine($"Internalize reachable: {internalizeReachable.Count}"); + builder.AppendLine($"Internalize candidates: {internalizeCandidates.Count}"); + builder.AppendLine($"Custom roots: {customRoots.Count}"); + builder.AppendLine($"Helper roots: {helperRoots.Count}"); + builder.AppendLine($"Remove roots: {removeRoots.Count}"); + builder.AppendLine($"Remove reachable: {removeReachable.Count}"); + builder.AppendLine($"Remove candidates: {removeCandidates.Count}"); + builder.AppendLine(); + builder.AppendLine("Custom roots:"); + foreach (var root in customRoots.OrderBy(static name => name, StringComparer.Ordinal)) + { + builder.AppendLine($" {root}"); + } + + builder.AppendLine(); + builder.AppendLine("Helper roots:"); + foreach (var root in helperRoots.OrderBy(static name => name, StringComparer.Ordinal)) + { + builder.AppendLine($" {root}"); + } + + builder.AppendLine(); + builder.AppendLine("Internalize roots:"); + foreach (var root in internalizeRoots.OrderBy(static name => name, StringComparer.Ordinal)) + { + builder.AppendLine($" {root}"); + } + + builder.AppendLine(); + builder.AppendLine("Internalize candidates:"); + foreach (var candidate in internalizeCandidates) + { + builder.AppendLine($" {candidate}"); + } + + builder.AppendLine(); + builder.AppendLine("Remove roots:"); + foreach (var root in removeRoots.OrderBy(static name => name, StringComparer.Ordinal)) + { + builder.AppendLine($" {root}"); + } + + builder.AppendLine(); + builder.AppendLine("Remove candidates:"); + foreach (var candidate in removeCandidates) + { + builder.AppendLine($" {candidate}"); + } + + builder.AppendLine(); + builder.AppendLine("References:"); + foreach (var (type, references) in graph.References.OrderBy(static item => item.Key, StringComparer.Ordinal)) + { + builder.AppendLine($" {type}"); + foreach (var reference in references.OrderBy(static name => name, StringComparer.Ordinal)) + { + builder.AppendLine($" -> {reference}"); + } + } + + File.WriteAllText(path, builder.ToString()); + CodeModelGenerator.Instance.Emitter.Debug($"Provider reference map shadow report written to {path}"); + } + + private static string GetSimpleName(string fullyQualifiedName) + { + var lastDot = fullyQualifiedName.LastIndexOf('.'); + return lastDot < 0 ? fullyQualifiedName : fullyQualifiedName.Substring(lastDot + 1); + } + + private static string GetProviderTypeName(CSharpType type) + { + var name = type.Arguments.Count > 0 && !type.Name.Contains('`', StringComparison.Ordinal) + ? $"{type.Name}`{type.Arguments.Count}" + : type.Name; + return string.IsNullOrEmpty(type.Namespace) ? name : $"{type.Namespace}.{name}"; + } + + private static string StripGenericArity(string name) + { + var tick = name.IndexOf('`'); + return tick < 0 ? name : name.Substring(0, tick); + } + + private sealed record ProviderReferenceGraph( + HashSet Nodes, + Dictionary> References); + + private static string GetOutputDirectory() + { + var directory = Environment.GetEnvironmentVariable(OutputDirectoryEnvironmentVariable); + return string.IsNullOrWhiteSpace(directory) + ? Path.Combine(Path.GetTempPath(), "typespec-provider-reference-map-shadow") + : Path.GetFullPath(directory); + } + } +} diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapShadowResult.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapShadowResult.cs new file mode 100644 index 00000000000..14b68f7d187 --- /dev/null +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapShadowResult.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Collections.Generic; + +namespace Microsoft.TypeSpec.Generator +{ + internal sealed record ProviderReferenceMapShadowResult( + HashSet InternalizeCandidates, + HashSet RemoveCandidates) + { + public static ProviderReferenceMapShadowResult Empty { get; } = new([], []); + } +} From f88a64a9e474d389ffd8aca8c2ef4f1f97bb6f16 Mon Sep 17 00:00:00 2001 From: Wei Hu Date: Fri, 12 Jun 2026 06:31:47 +0000 Subject: [PATCH 03/13] Revert "Profile C# post-processing benchmarks" This reverts commit ff555094d2ee43fb2a9f4d0c2aa7cab0ebcac0d3. --- .../perf/FullGenerationBenchmark.cs | 157 -------------- .../src/CSharpGen.cs | 164 ++++----------- .../PostProcessing/GeneratedCodeWorkspace.cs | 112 ++-------- ...ratedCodeWorkspacePostProcessingProfile.cs | 64 ------ .../src/PostProcessing/PostProcessor.cs | 195 +++++------------- 5 files changed, 113 insertions(+), 579 deletions(-) delete mode 100644 packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/perf/FullGenerationBenchmark.cs delete mode 100644 packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspacePostProcessingProfile.cs diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/perf/FullGenerationBenchmark.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/perf/FullGenerationBenchmark.cs deleted file mode 100644 index 66553e08c38..00000000000 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/perf/FullGenerationBenchmark.cs +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using BenchmarkDotNet.Attributes; - -namespace Microsoft.TypeSpec.Generator.Perf -{ - public class FullGenerationBenchmark - { - private const string ProfileEnvironmentVariable = "POSTPROCESSING_BENCHMARK_PROFILE_STEPS"; - private const string ProfileOutputDirectoryEnvironmentVariable = "POSTPROCESSING_BENCHMARK_PROFILE_DIR"; - - private bool _profileSteps; - - [GlobalSetup] - public void GlobalSetup() - { - _profileSteps = string.Equals( - Environment.GetEnvironmentVariable(ProfileEnvironmentVariable), - "true", - StringComparison.OrdinalIgnoreCase); - } - - [Benchmark] - public async Task GenerateSampleTypeSpecProject() - { - var postProcessingProfile = _profileSteps ? new GeneratedCodeWorkspacePostProcessingProfile() : null; - var generationProfile = _profileSteps ? new GeneratedCodeWorkspacePostProcessingProfile() : null; - GeneratedCodeWorkspace.PostProcessingProfile = postProcessingProfile; - CSharpGen.GenerationProfile = generationProfile; - - var benchmarkDirectory = CreateBenchmarkInputDirectory(); - var stopwatch = Stopwatch.StartNew(); - try - { - CodeModelGenerator.Instance = new BenchmarkCodeModelGenerator(benchmarkDirectory); - CodeModelGenerator.Instance.Configure(); - - var csharpGen = new CSharpGen(); - await csharpGen.ExecuteAsync(); - - return Directory.GetFiles(benchmarkDirectory, "*", SearchOption.AllDirectories) - .Where(static path => !path.EndsWith("tspCodeModel.json", StringComparison.Ordinal) && - !path.EndsWith("Configuration.json", StringComparison.Ordinal)) - .Sum(static path => (int)new FileInfo(path).Length); - } - finally - { - stopwatch.Stop(); - if (generationProfile != null) - { - WriteProfile( - generationProfile, - $"full-generation-profile-{DateTime.UtcNow:yyyyMMddHHmmssfff}.csv", - $"Full generation invocation elapsed ms: {stopwatch.Elapsed.TotalMilliseconds:F3}{Environment.NewLine}" + - $"Input directory: {benchmarkDirectory}{Environment.NewLine}"); - } - - if (postProcessingProfile != null) - { - WriteProfile( - postProcessingProfile, - $"full-generation-post-processing-profile-{DateTime.UtcNow:yyyyMMddHHmmssfff}.csv", - $"Full generation post-processing profile{Environment.NewLine}" + - $"Input directory: {benchmarkDirectory}{Environment.NewLine}"); - } - - CSharpGen.GenerationProfile = null; - GeneratedCodeWorkspace.PostProcessingProfile = null; - TryDeleteDirectory(benchmarkDirectory); - } - } - - private static void WriteProfile(GeneratedCodeWorkspacePostProcessingProfile profile, string fileName, string header) - { - var profileDirectory = GetProfileOutputDirectory(); - Directory.CreateDirectory(profileDirectory); - File.WriteAllText(Path.Combine(profileDirectory, fileName), header + profile.GetSummary()); - } - - private static string CreateBenchmarkInputDirectory() - { - var sourceDirectory = FindFullGenerationInputDirectory(); - var benchmarkDirectory = Path.Combine(Path.GetTempPath(), "typespec-full-generation-benchmark", Guid.NewGuid().ToString("N")); - CopyDirectory(sourceDirectory, benchmarkDirectory); - return benchmarkDirectory; - } - - private static string FindFullGenerationInputDirectory() - { - const string relativePath = "packages/http-client-csharp/generator/TestProjects/Local/Sample-TypeSpec"; - - var directory = new DirectoryInfo(AppContext.BaseDirectory); - while (directory != null) - { - var inputDirectory = Path.Combine(directory.FullName, relativePath); - if (File.Exists(Path.Combine(inputDirectory, "tspCodeModel.json")) && - File.Exists(Path.Combine(inputDirectory, "Configuration.json"))) - { - return inputDirectory; - } - - directory = directory.Parent; - } - - throw new DirectoryNotFoundException($"Could not find '{relativePath}' from '{AppContext.BaseDirectory}'."); - } - - private static void CopyDirectory(string sourceDirectory, string destinationDirectory) - { - Directory.CreateDirectory(destinationDirectory); - foreach (var sourceFile in Directory.GetFiles(sourceDirectory, "*", SearchOption.AllDirectories)) - { - var relativePath = Path.GetRelativePath(sourceDirectory, sourceFile); - var destinationFile = Path.Combine(destinationDirectory, relativePath); - Directory.CreateDirectory(Path.GetDirectoryName(destinationFile)!); - File.Copy(sourceFile, destinationFile, overwrite: true); - } - } - - private static void TryDeleteDirectory(string directory) - { - try - { - if (Directory.Exists(directory)) - { - Directory.Delete(directory, recursive: true); - } - } - catch - { - // Best-effort cleanup for benchmark temp output. - } - } - - private static string GetProfileOutputDirectory() - { - var configuredPath = Environment.GetEnvironmentVariable(ProfileOutputDirectoryEnvironmentVariable); - return string.IsNullOrWhiteSpace(configuredPath) - ? Path.Combine(Path.GetTempPath(), "typespec-post-processing-profiles") - : Path.GetFullPath(configuredPath); - } - - private sealed class BenchmarkCodeModelGenerator : CodeModelGenerator - { - public BenchmarkCodeModelGenerator(string outputPath) - : base(new GeneratorContext(Configuration.Load(outputPath))) - { - } - } - } -} diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/CSharpGen.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/CSharpGen.cs index fdd3ae2191f..8ae87be33d1 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/CSharpGen.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/CSharpGen.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; using System.Threading.Tasks; @@ -21,8 +20,6 @@ internal sealed class CSharpGen private static readonly string[] _filesToKeep = [ConfigurationFileName, CodeModelFileName]; - internal static GeneratedCodeWorkspacePostProcessingProfile? GenerationProfile { get; set; } - /// /// Executes the generator task with the instance. /// @@ -36,21 +33,19 @@ public async Task ExecuteAsync() // Resolve PackageReference items from the .csproj so custom code referencing // external NuGet types (e.g., Azure.Storage.Common) compiles correctly. - await MeasureGenerationStepAsync("Generation.AddPackageReferencesFromProject", GeneratedCodeWorkspace.AddPackageReferencesFromProject); + await GeneratedCodeWorkspace.AddPackageReferencesFromProject(); // Pre-walk the input library and resolve any external types that point at NuGet packages. // This populates ExternalTypeReferenceResolver's cache and registers each resolved assembly // as an additional metadata reference *before* the generated/custom code workspaces are // constructed, so their cached Roslyn projects pick the references up. - await MeasureGenerationStepAsync("Generation.ResolveExternalTypeReferences", ExternalTypeReferenceResolver.ResolveAllAsync); + await ExternalTypeReferenceResolver.ResolveAllAsync(); // Initialize the workspace project AFTER all metadata references have been added so the // eagerly-cached project sees them. GeneratedCodeWorkspace.Initialize(); - GeneratedCodeWorkspace customCodeWorkspace = await MeasureGenerationStepAsync( - "Generation.CreateCustomCodeWorkspace", - () => GeneratedCodeWorkspace.Create(isCustomCodeProject: true)); + GeneratedCodeWorkspace customCodeWorkspace = await GeneratedCodeWorkspace.Create(isCustomCodeProject: true); // The generated attributes need to be added into the workspace before loading the custom code. Otherwise, // Roslyn doesn't load the attributes completely and we are unable to get the attribute arguments. @@ -60,169 +55,90 @@ public async Task ExecuteAsync() generateAttributeTasks.Add(customCodeWorkspace.AddInMemoryFile(attributeProvider)); } - await MeasureGenerationStepAsync("Generation.AddCustomizationAttributeProviders", () => Task.WhenAll(generateAttributeTasks)); + await Task.WhenAll(generateAttributeTasks); - CodeModelGenerator.Instance.SourceInputModel = await MeasureGenerationStepAsync( - "Generation.CreateSourceInputModel", - async () => new SourceInputModel( - await customCodeWorkspace.GetCompilationAsync(), - await GeneratedCodeWorkspace.LoadBaselineContract())); + CodeModelGenerator.Instance.SourceInputModel = new SourceInputModel( + await customCodeWorkspace.GetCompilationAsync(), + await GeneratedCodeWorkspace.LoadBaselineContract()); - GeneratedCodeWorkspace generatedCodeWorkspace = await MeasureGenerationStepAsync( - "Generation.CreateGeneratedCodeWorkspace", - () => GeneratedCodeWorkspace.Create(isCustomCodeProject: false)); + GeneratedCodeWorkspace generatedCodeWorkspace = await GeneratedCodeWorkspace.Create(isCustomCodeProject: false); - var output = MeasureGenerationStep("Generation.GetOutputLibrary", () => CodeModelGenerator.Instance.OutputLibrary); + var output = CodeModelGenerator.Instance.OutputLibrary; Directory.CreateDirectory(Path.Combine(generatedSourceOutputPath, "Models")); List generateFilesTasks = new(); // Build all TypeProviders - MeasureGenerationStep("Generation.BuildTypeProviders", () => + foreach (var type in output.TypeProviders) { - foreach (var type in output.TypeProviders) - { - type.EnsureBuilt(); - } - }); + type.EnsureBuilt(); + } LoggingHelpers.LogElapsedTime("All generated type providers built"); // visit the entire library before generating files - MeasureGenerationStep("Generation.ApplyVisitors", () => + foreach (var visitor in CodeModelGenerator.Instance.Visitors) { - foreach (var visitor in CodeModelGenerator.Instance.Visitors) - { - visitor.VisitLibrary(output); - } - }); + visitor.VisitLibrary(output); + } - MeasureGenerationStep("Generation.FilterCustomizedMembers", () => FilterAllCustomizedMembers(output)); + FilterAllCustomizedMembers(output); LoggingHelpers.LogElapsedTime("All visitors have been applied"); - MeasureGenerationStep("Generation.WriteTypeProviders", () => + foreach (var outputType in output.TypeProviders) { - foreach (var outputType in output.TypeProviders) - { - // Ensure back-compatibility processing is done after all visitors have run - outputType.ProcessTypeForBackCompatibility(); + // Ensure back-compatibility processing is done after all visitors have run + outputType.ProcessTypeForBackCompatibility(); - var writer = CodeModelGenerator.Instance.GetWriter(outputType); - generateFilesTasks.Add(generatedCodeWorkspace.AddGeneratedFile(writer.Write())); + var writer = CodeModelGenerator.Instance.GetWriter(outputType); + generateFilesTasks.Add(generatedCodeWorkspace.AddGeneratedFile(writer.Write())); - foreach (var serialization in outputType.SerializationProviders) - { - writer = CodeModelGenerator.Instance.GetWriter(serialization); - generateFilesTasks.Add(generatedCodeWorkspace.AddGeneratedFile(writer.Write())); - } + foreach (var serialization in outputType.SerializationProviders) + { + writer = CodeModelGenerator.Instance.GetWriter(serialization); + generateFilesTasks.Add(generatedCodeWorkspace.AddGeneratedFile(writer.Write())); } - }); + } // Add all the generated files to the workspace - await MeasureGenerationStepAsync("Generation.AddGeneratedFilesToWorkspace", () => Task.WhenAll(generateFilesTasks)); + await Task.WhenAll(generateFilesTasks); - MeasureGenerationStep("Generation.ProviderReferenceMapShadowAnalysis", () => generatedCodeWorkspace.AnalyzeProviderReferenceMap(output.TypeProviders)); + generatedCodeWorkspace.AnalyzeProviderReferenceMap(output.TypeProviders); LoggingHelpers.LogElapsedTime("All generated types have been written into memory"); // Delete any old generated files - MeasureGenerationStep("Generation.DeleteOldGeneratedFiles", () => DeleteDirectory(generatedSourceOutputPath, _filesToKeep)); + DeleteDirectory(generatedSourceOutputPath, _filesToKeep); LoggingHelpers.LogElapsedTime("All old generated files have been deleted"); - await MeasureGenerationStepAsync("Generation.PostProcessAsync", generatedCodeWorkspace.PostProcessAsync); + await generatedCodeWorkspace.PostProcessAsync(); // Write the generated files to the output directory - await MeasureGenerationStepAsync("Generation.WriteGeneratedFilesToDisk", async () => + await foreach (var file in generatedCodeWorkspace.GetGeneratedFilesAsync()) { - await foreach (var file in generatedCodeWorkspace.GetGeneratedFilesAsync()) + if (string.IsNullOrEmpty(file.Text)) { - if (string.IsNullOrEmpty(file.Text)) - { - continue; - } - var filename = Path.Combine(outputPath, file.Name); - CodeModelGenerator.Instance.Emitter.Info($"Writing {Path.GetFullPath(filename)}"); - Directory.CreateDirectory(Path.GetDirectoryName(filename)!); - await File.WriteAllTextAsync(filename, file.Text); + continue; } - }); + var filename = Path.Combine(outputPath, file.Name); + CodeModelGenerator.Instance.Emitter.Info($"Writing {Path.GetFullPath(filename)}"); + Directory.CreateDirectory(Path.GetDirectoryName(filename)!); + await File.WriteAllTextAsync(filename, file.Text); + } // Write additional output files (e.g. configuration schemas, .targets files) - await MeasureGenerationStepAsync("Generation.WriteAdditionalFiles", () => CodeModelGenerator.Instance.WriteAdditionalFiles(outputPath)); + await CodeModelGenerator.Instance.WriteAdditionalFiles(outputPath); // Write project scaffolding files (after additional files so schema existence can be checked) if (CodeModelGenerator.Instance.IsNewProject) { - await MeasureGenerationStepAsync( - "Generation.WriteProjectScaffolding", - () => CodeModelGenerator.Instance.TypeFactory.CreateNewProjectScaffolding().Execute()); + await CodeModelGenerator.Instance.TypeFactory.CreateNewProjectScaffolding().Execute(); } LoggingHelpers.LogElapsedTime("All files have been written to disk"); } - private static void MeasureGenerationStep(string stepName, Action action) - { - MeasureGenerationStep( - stepName, - () => - { - action(); - return 0; - }); - } - - private static T MeasureGenerationStep(string stepName, Func action) - { - var profile = GenerationProfile; - if (profile == null) - { - return action(); - } - - var allocatedBytes = GC.GetTotalAllocatedBytes(precise: false); - var stopwatch = Stopwatch.StartNew(); - try - { - return action(); - } - finally - { - stopwatch.Stop(); - profile.Add(stepName, stopwatch.Elapsed, GC.GetTotalAllocatedBytes(precise: false) - allocatedBytes); - } - } - - private static Task MeasureGenerationStepAsync(string stepName, Func action) => MeasureGenerationStepAsync( - stepName, - async () => - { - await action(); - return 0; - }); - - private static async Task MeasureGenerationStepAsync(string stepName, Func> action) - { - var profile = GenerationProfile; - if (profile == null) - { - return await action(); - } - - var allocatedBytes = GC.GetTotalAllocatedBytes(precise: false); - var stopwatch = Stopwatch.StartNew(); - try - { - return await action(); - } - finally - { - stopwatch.Stop(); - profile.Add(stepName, stopwatch.Elapsed, GC.GetTotalAllocatedBytes(precise: false) - allocatedBytes); - } - } - internal static void FilterAllCustomizedMembers(OutputLibrary output) { foreach (var typeProvider in output.TypeProviders) diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspace.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspace.cs index 07e8595be08..de03b1ca6e7 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspace.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspace.cs @@ -37,8 +37,6 @@ internal class GeneratedCodeWorkspace private static readonly Lazy _metadataReferenceResolver = new(() => new WorkspaceMetadataReferenceResolver()); private static Task? _cachedProject; - internal static GeneratedCodeWorkspacePostProcessingProfile? PostProcessingProfile { get; set; } - private static readonly string[] _generatedFolders = [GeneratedFolder]; private static readonly string[] _sharedFolders = [SharedFolder]; @@ -109,7 +107,7 @@ internal static SyntaxTree GetTree(TypeProvider provider) public async IAsyncEnumerable<(string Name, string Text)> GetGeneratedFilesAsync() { - List docs = new List(); + List> documents = new List>(); var memberRemover = new MemberRemoverRewriter(); foreach (Document document in _project.Documents) { @@ -118,12 +116,9 @@ internal static SyntaxTree GetTree(TypeProvider provider) continue; } - docs.Add(document); + documents.Add(ProcessDocument(document, memberRemover)); } - - docs = PostProcessingProfile == null - ? [.. await Task.WhenAll(docs.Select(document => ProcessDocument(document, memberRemover)))] - : await ProcessDocumentsSequentiallyAsync(docs, memberRemover); + var docs = await Task.WhenAll(documents); LoggingHelpers.LogElapsedTime("Roslyn post processing complete"); @@ -139,101 +134,36 @@ internal static SyntaxTree GetTree(TypeProvider provider) } } - private async Task> ProcessDocumentsSequentiallyAsync(List documents, MemberRemoverRewriter memberRemover) - { - List processedDocuments = new(documents.Count); - foreach (var document in documents) - { - processedDocuments.Add(await ProcessDocument(document, memberRemover)); - } - - return processedDocuments; - } - private async Task ProcessDocument(Document document, MemberRemoverRewriter memberRemover) { - var totalStopwatch = PostProcessingProfile == null ? null : Stopwatch.StartNew(); - try - { - var root = await MeasurePostProcessingStepAsync("GetSyntaxRootAsync", () => document.GetSyntaxRootAsync()); - var semanticModel = await MeasurePostProcessingStepAsync("GetSemanticModelAsync", () => document.GetSemanticModelAsync()); - - if (semanticModel == null || root == null) - { - return document; - } - - root = MeasurePostProcessingStep("MemberRemoverRewriter", () => memberRemover.Visit(root)); - - foreach (var rewriter in CodeModelGenerator.Instance.Rewriters) - { - rewriter.SemanticModel = semanticModel; - root = MeasurePostProcessingStep($"CustomRewriter.{rewriter.GetType().Name}", () => rewriter.Visit(root)); - } - document = document.WithSyntaxRoot(root); - - if (!CodeModelGenerator.Instance.Configuration.DisableRoslynReduce) - { - document = await MeasurePostProcessingStepAsync("Roslyn.Simplifier.ReduceAsync", () => Simplifier.ReduceAsync(document)); - } + var root = await document.GetSyntaxRootAsync(); + var semanticModel = await document.GetSemanticModelAsync(); - // Reformat if any custom rewriters have been applied - if (CodeModelGenerator.Instance.Rewriters.Count > 0) - { - document = await MeasurePostProcessingStepAsync("Formatter.FormatAsync", () => Formatter.FormatAsync(document)); - } - return document; - } - finally + if (semanticModel == null || root == null) { - if (totalStopwatch != null) - { - totalStopwatch.Stop(); - PostProcessingProfile?.Add("ProcessDocument.Total", totalStopwatch.Elapsed, 0); - } + return document; } - } - private static T MeasurePostProcessingStep(string stepName, Func action) - { - var profile = PostProcessingProfile; - if (profile == null) - { - return action(); - } + root = memberRemover.Visit(root); - var allocatedBytes = GC.GetTotalAllocatedBytes(precise: false); - var stopwatch = Stopwatch.StartNew(); - try + foreach (var rewriter in CodeModelGenerator.Instance.Rewriters) { - return action(); + rewriter.SemanticModel = semanticModel; + root = rewriter.Visit(root); } - finally - { - stopwatch.Stop(); - profile.Add(stepName, stopwatch.Elapsed, GC.GetTotalAllocatedBytes(precise: false) - allocatedBytes); - } - } + document = document.WithSyntaxRoot(root); - private static async Task MeasurePostProcessingStepAsync(string stepName, Func> action) - { - var profile = PostProcessingProfile; - if (profile == null) + if (!CodeModelGenerator.Instance.Configuration.DisableRoslynReduce) { - return await action(); + document = await Simplifier.ReduceAsync(document); } - var allocatedBytes = GC.GetTotalAllocatedBytes(precise: false); - var stopwatch = Stopwatch.StartNew(); - try - { - return await action(); - } - finally + // Reformat if any custom rewriters have been applied + if (CodeModelGenerator.Instance.Rewriters.Count > 0) { - stopwatch.Stop(); - profile.Add(stepName, stopwatch.Elapsed, GC.GetTotalAllocatedBytes(precise: false) - allocatedBytes); + document = await Formatter.FormatAsync(document); } + return document; } public static bool IsGeneratedDocument(Document document) => document.Folders.Contains(GeneratedFolder); @@ -350,11 +280,11 @@ public async Task PostProcessAsync() case Configuration.UnreferencedTypesHandlingOption.KeepAll: break; case Configuration.UnreferencedTypesHandlingOption.Internalize: - _project = await MeasurePostProcessingStepAsync("PostProcess.InternalizeAsync", () => postProcessor.InternalizeAsync(_project)); + _project = await postProcessor.InternalizeAsync(_project); break; case Configuration.UnreferencedTypesHandlingOption.RemoveOrInternalize: - _project = await MeasurePostProcessingStepAsync("PostProcess.InternalizeAsync", () => postProcessor.InternalizeAsync(_project)); - _project = await MeasurePostProcessingStepAsync("PostProcess.RemoveAsync", () => postProcessor.RemoveAsync(_project)); + _project = await postProcessor.InternalizeAsync(_project); + _project = await postProcessor.RemoveAsync(_project); break; } } diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspacePostProcessingProfile.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspacePostProcessingProfile.cs deleted file mode 100644 index 74a9a58168a..00000000000 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspacePostProcessingProfile.cs +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.InteropServices; -using System.Text; - -namespace Microsoft.TypeSpec.Generator -{ - internal sealed class GeneratedCodeWorkspacePostProcessingProfile - { - private readonly object _syncRoot = new(); - private readonly Dictionary _steps = new(StringComparer.Ordinal); - - public void Add(string stepName, TimeSpan elapsed, long allocatedBytes) - { - lock (_syncRoot) - { - ref var summary = ref CollectionsMarshal.GetValueRefOrAddDefault(_steps, stepName, out _); - summary.Count++; - summary.ElapsedTicks += elapsed.Ticks; - summary.AllocatedBytes += allocatedBytes; - } - } - - public string GetSummary() - { - KeyValuePair[] steps; - lock (_syncRoot) - { - steps = _steps.ToArray(); - } - - var totalTicks = steps - .Where(static step => step.Key != "ProcessDocument.Total") - .Sum(static step => step.Value.ElapsedTicks); - var builder = new StringBuilder(); - builder.AppendLine("Post-processing step profile:"); - builder.AppendLine("Step, Count, Total ms, Avg ms, Percent of measured steps, Allocated bytes, Avg allocated bytes"); - - foreach (var step in steps.OrderByDescending(static step => step.Value.ElapsedTicks)) - { - var elapsedMs = TimeSpan.FromTicks(step.Value.ElapsedTicks).TotalMilliseconds; - var averageMs = elapsedMs / step.Value.Count; - var averageAllocatedBytes = step.Value.AllocatedBytes / step.Value.Count; - var percentage = totalTicks == 0 || step.Key == "ProcessDocument.Total" - ? 0 - : step.Value.ElapsedTicks * 100.0 / totalTicks; - builder.AppendLine($"{step.Key}, {step.Value.Count}, {elapsedMs:F3}, {averageMs:F3}, {percentage:F1}%, {step.Value.AllocatedBytes}, {averageAllocatedBytes}"); - } - - return builder.ToString(); - } - - private struct StepSummary - { - public int Count; - public long ElapsedTicks; - public long AllocatedBytes; - } - } -} diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs index 57136754da1..5632a41053a 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs @@ -20,8 +20,6 @@ internal class PostProcessor private readonly HashSet _typesToKeep; private INamedTypeSymbol? _modelFactorySymbol; - private static GeneratedCodeWorkspacePostProcessingProfile? Profile => GeneratedCodeWorkspace.PostProcessingProfile; - public PostProcessor( HashSet typesToKeep, string? modelFactoryFullName = null, @@ -127,30 +125,27 @@ protected virtual bool ShouldIncludeDocument(Document document) => /// The processed . is immutable, therefore this should usually be a new instance public async Task InternalizeAsync(Project project) { - var compilation = await MeasureAsync("PostProcessor.Internalize.GetCompilationAsync", () => project.GetCompilationAsync()); + var compilation = await project.GetCompilationAsync(); if (compilation == null) return project; // first get all the declared symbols - var definitions = await MeasureAsync("PostProcessor.Internalize.GetTypeSymbolsAsync", () => GetTypeSymbolsAsync(compilation, project, true)); + var definitions = await GetTypeSymbolsAsync(compilation, project, true); IEnumerable symbolsToInternalize; if (ProviderReferenceMapShadowAnalyzer.UseShadowMap && ProviderReferenceMapShadowAnalyzer.LatestResult is { } useShadowResult) { - symbolsToInternalize = Measure("PostProcessor.Internalize.UseShadowCandidates", () => - GetSymbolsByName(definitions.DeclaredSymbols, useShadowResult.InternalizeCandidates).ToArray()); + symbolsToInternalize = GetSymbolsByName(definitions.DeclaredSymbols, useShadowResult.InternalizeCandidates).ToArray(); } else { // build the reference map var referenceMap = - await MeasureAsync( - "PostProcessor.Internalize.BuildPublicReferenceMapAsync", - () => new ReferenceMapBuilder(compilation, project).BuildPublicReferenceMapAsync( - definitions.DeclaredSymbols, definitions.DeclaredNodesCache)); + await new ReferenceMapBuilder(compilation, project).BuildPublicReferenceMapAsync( + definitions.DeclaredSymbols, definitions.DeclaredNodesCache); // get the root symbols - var rootSymbols = await MeasureAsync("PostProcessor.Internalize.GetRootSymbolsAsync", () => GetRootSymbolsAsync(project, definitions)); + var rootSymbols = await GetRootSymbolsAsync(project, definitions); // traverse all the root and recursively add all the things we met - var publicSymbols = Measure("PostProcessor.Internalize.VisitSymbolsFromRoot", () => VisitSymbolsFromRootAsync(rootSymbols, referenceMap).ToArray()); + var publicSymbols = VisitSymbolsFromRootAsync(rootSymbols, referenceMap); symbolsToInternalize = definitions.DeclaredSymbols.Except(publicSymbols); } @@ -163,36 +158,23 @@ await MeasureAsync( shadowResult.InternalizeCandidates); } - var nodesToInternalize = Measure("PostProcessor.Internalize.CollectNodes", () => + var nodesToInternalize = new Dictionary(); + foreach (var symbol in symbolsToInternalize) { - var nodes = new Dictionary(); - foreach (var symbol in symbolsToInternalize) + foreach (var node in definitions.DeclaredNodesCache[symbol]) { - foreach (var node in definitions.DeclaredNodesCache[symbol]) - { - nodes[node] = project.GetDocumentId(node.SyntaxTree)!; - } + nodesToInternalize[node] = project.GetDocumentId(node.SyntaxTree)!; } + } - return nodes; - }); - - project = Measure("PostProcessor.Internalize.MarkInternal", () => + foreach (var (model, documentId) in nodesToInternalize) { - var updatedProject = project; - foreach (var (model, documentId) in nodesToInternalize) - { - updatedProject = MarkInternal(updatedProject, model, documentId); - } - - return updatedProject; - }); + project = MarkInternal(project, model, documentId); + } var modelNamesToRemove = nodesToInternalize.Keys.Select(item => item.Identifier.Text); - project = await MeasureAsync( - "PostProcessor.Internalize.RemoveMethodsFromModelFactoryAsync", - () => RemoveMethodsFromModelFactoryAsync(project, definitions, modelNamesToRemove.ToHashSet())); + project = await RemoveMethodsFromModelFactoryAsync(project, definitions, modelNamesToRemove.ToHashSet()); return project; } @@ -266,40 +248,36 @@ private async Task RemoveMethodsFromModelFactoryAsync(Project project, /// The processed . is immutable, therefore this should usually be a new instance public async Task RemoveAsync(Project project) { - var compilation = await MeasureAsync("PostProcessor.Remove.GetCompilationAsync", () => project.GetCompilationAsync()); + var compilation = await project.GetCompilationAsync(); if (compilation == null) return project; // find all the declarations, including non-public declared - var definitions = await MeasureAsync("PostProcessor.Remove.GetTypeSymbolsAsync", () => GetTypeSymbolsAsync(compilation, project, false)); + var definitions = await GetTypeSymbolsAsync(compilation, project, false); IEnumerable symbolsToRemove; HashSet referencedSet; if (ProviderReferenceMapShadowAnalyzer.UseShadowMap && ProviderReferenceMapShadowAnalyzer.LatestResult is { } useShadowResult) { - symbolsToRemove = Measure("PostProcessor.Remove.UseShadowCandidates", () => - GetSymbolsByName(definitions.DeclaredSymbols, useShadowResult.RemoveCandidates).ToArray()); - referencedSet = Measure("PostProcessor.Remove.BuildShadowReferencedSet", () => - new HashSet(definitions.DeclaredSymbols.Except(symbolsToRemove), SymbolEqualityComparer.Default)); + symbolsToRemove = GetSymbolsByName(definitions.DeclaredSymbols, useShadowResult.RemoveCandidates).ToArray(); + referencedSet = new HashSet(definitions.DeclaredSymbols.Except(symbolsToRemove), SymbolEqualityComparer.Default); } else { // build reference map var referenceMap = - await MeasureAsync( - "PostProcessor.Remove.BuildAllReferenceMapAsync", - () => new ReferenceMapBuilder(compilation, project).BuildAllReferenceMapAsync( - definitions.DeclaredSymbols, definitions.DocumentsCache)); + await new ReferenceMapBuilder(compilation, project).BuildAllReferenceMapAsync( + definitions.DeclaredSymbols, definitions.DocumentsCache); // get root symbols - var rootSymbols = await MeasureAsync("PostProcessor.Remove.GetRootSymbolsAsync", () => GetRootSymbolsAsync(project, definitions)); + var rootSymbols = await GetRootSymbolsAsync(project, definitions); // include model factory as a root symbol when doing the remove pass so that we are sure to include any internal // helpers that are required by the model factory. if (_modelFactorySymbol != null) rootSymbols.Add(_modelFactorySymbol); // traverse the map to determine the declarations that we are about to remove, starting from root nodes - var referencedSymbols = Measure("PostProcessor.Remove.VisitSymbolsFromRoot", () => VisitSymbolsFromRootAsync(rootSymbols, referenceMap).ToArray().AsEnumerable()); + var referencedSymbols = VisitSymbolsFromRootAsync(rootSymbols, referenceMap); - referencedSymbols = Measure("PostProcessor.Remove.AddSampleSymbols", () => AddSampleSymbols(referencedSymbols, definitions.DeclaredSymbols)); - referencedSet = Measure("PostProcessor.Remove.BuildReferencedSet", () => new HashSet(referencedSymbols, SymbolEqualityComparer.Default)); + referencedSymbols = AddSampleSymbols(referencedSymbols, definitions.DeclaredSymbols); + referencedSet = new HashSet(referencedSymbols, SymbolEqualityComparer.Default); symbolsToRemove = definitions.DeclaredSymbols.Except(referencedSet); } @@ -312,23 +290,18 @@ await MeasureAsync( shadowResult.RemoveCandidates); } - var nodesToRemove = Measure("PostProcessor.Remove.CollectNodes", () => + var nodesToRemove = new List(); + foreach (var symbol in symbolsToRemove) { - var nodes = new List(); - foreach (var symbol in symbolsToRemove) + if (referencedSet.Contains(GetBase(symbol))) { - if (referencedSet.Contains(GetBase(symbol))) - { - continue; - } - nodes.AddRange(definitions.DeclaredNodesCache[symbol]); + continue; } - - return nodes; - }); + nodesToRemove.AddRange(definitions.DeclaredNodesCache[symbol]); + } // remove them one by one - project = await MeasureAsync("PostProcessor.Remove.RemoveModelsAsync", () => RemoveModelsAsync(project, nodesToRemove)); + project = await RemoveModelsAsync(project, nodesToRemove); return project; } @@ -421,35 +394,25 @@ private async Task RemoveModelsAsync(Project project, IEnumerable unusedModels) { // accumulate the definitions from the same document together - var documents = Measure("PostProcessor.Remove.RemoveModelsAsync.GroupByDocument", () => - { - var groupedDocuments = new Dictionary>(); - foreach (var model in unusedModels) - { - var document = project.GetDocument(model.SyntaxTree); - Debug.Assert(document != null); - if (!groupedDocuments.ContainsKey(document)) - groupedDocuments.Add(document, new HashSet()); + var documents = new Dictionary>(); - groupedDocuments[document].Add(model); - } + foreach (var model in unusedModels) + { + var document = project.GetDocument(model.SyntaxTree); + Debug.Assert(document != null); + if (!documents.ContainsKey(document)) + documents.Add(document, new HashSet()); - return groupedDocuments; - }); + documents[document].Add(model); + } - project = await MeasureAsync("PostProcessor.Remove.RemoveModelsAsync.RemoveModelsFromDocuments", async () => + foreach (var models in documents.Values) { - var updatedProject = project; - foreach (var models in documents.Values) - { - updatedProject = await RemoveModelsFromDocumentAsync(updatedProject, models); - } - - return updatedProject; - }); + project = await RemoveModelsFromDocumentAsync(project, models); + } // remove what are now invalid references due to the models being removed - project = await MeasureAsync("PostProcessor.Remove.RemoveModelsAsync.RemoveInvalidRefs", () => RemoveInvalidRefs(project)); + project = await RemoveInvalidRefs(project); return project; } @@ -500,28 +463,16 @@ private async Task RemoveInvalidRefs(Project project) var solution = project.Solution; // Process each document for invalid usings - solution = await MeasureAsync("PostProcessor.Remove.RemoveInvalidRefs.RemoveInvalidUsings", async () => + foreach (var documentId in project.DocumentIds) { - var updatedSolution = solution; - foreach (var documentId in project.DocumentIds) - { - updatedSolution = await RemoveInvalidUsings(updatedSolution, documentId); - } - - return updatedSolution; - }); + solution = await RemoveInvalidUsings(solution, documentId); + } // Process each document for invalid attributes (with fresh semantic models) - solution = await MeasureAsync("PostProcessor.Remove.RemoveInvalidRefs.RemoveInvalidAttributes", async () => + foreach (var documentId in project.DocumentIds) { - var updatedSolution = solution; - foreach (var documentId in project.DocumentIds) - { - updatedSolution = await RemoveInvalidAttributes(updatedSolution, documentId); - } - - return updatedSolution; - }); + solution = await RemoveInvalidAttributes(solution, documentId); + } return solution.GetProject(project.Id)!; } @@ -627,48 +578,6 @@ arg.Expression is TypeOfExpressionSyntax typeOfExpr && return solution; } - private static T Measure(string stepName, Func action) - { - var profile = Profile; - if (profile == null) - { - return action(); - } - - var allocatedBytes = GC.GetTotalAllocatedBytes(precise: false); - var stopwatch = Stopwatch.StartNew(); - try - { - return action(); - } - finally - { - stopwatch.Stop(); - profile.Add(stepName, stopwatch.Elapsed, GC.GetTotalAllocatedBytes(precise: false) - allocatedBytes); - } - } - - private static async Task MeasureAsync(string stepName, Func> action) - { - var profile = Profile; - if (profile == null) - { - return await action(); - } - - var allocatedBytes = GC.GetTotalAllocatedBytes(precise: false); - var stopwatch = Stopwatch.StartNew(); - try - { - return await action(); - } - finally - { - stopwatch.Stop(); - profile.Add(stepName, stopwatch.Elapsed, GC.GetTotalAllocatedBytes(precise: false) - allocatedBytes); - } - } - private async Task> GetRootSymbolsAsync(Project project, TypeSymbols modelSymbols) { var result = new HashSet(SymbolEqualityComparer.Default); From 2d57265107030f3bc5c10cc83ed8c1cba4b25a05 Mon Sep 17 00:00:00 2001 From: Wei Hu Date: Fri, 12 Jun 2026 06:48:29 +0000 Subject: [PATCH 04/13] Remove hybrid map diagnostics from generator --- .../src/PostProcessing/PostProcessor.cs | 66 +------- .../ProviderReferenceMapShadowAnalyzer.cs | 154 ------------------ 2 files changed, 5 insertions(+), 215 deletions(-) diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs index 5632a41053a..a6cecba593f 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs @@ -131,32 +131,8 @@ public async Task InternalizeAsync(Project project) // first get all the declared symbols var definitions = await GetTypeSymbolsAsync(compilation, project, true); - IEnumerable symbolsToInternalize; - if (ProviderReferenceMapShadowAnalyzer.UseShadowMap && ProviderReferenceMapShadowAnalyzer.LatestResult is { } useShadowResult) - { - symbolsToInternalize = GetSymbolsByName(definitions.DeclaredSymbols, useShadowResult.InternalizeCandidates).ToArray(); - } - else - { - // build the reference map - var referenceMap = - await new ReferenceMapBuilder(compilation, project).BuildPublicReferenceMapAsync( - definitions.DeclaredSymbols, definitions.DeclaredNodesCache); - // get the root symbols - var rootSymbols = await GetRootSymbolsAsync(project, definitions); - // traverse all the root and recursively add all the things we met - var publicSymbols = VisitSymbolsFromRootAsync(rootSymbols, referenceMap); - - symbolsToInternalize = definitions.DeclaredSymbols.Except(publicSymbols); - } - - if (ProviderReferenceMapShadowAnalyzer.LatestResult is { } shadowResult) - { - ProviderReferenceMapShadowAnalyzer.WriteComparisonReport( - "internalize", - symbolsToInternalize.Select(static symbol => symbol.GetFullyQualifiedName()), - shadowResult.InternalizeCandidates); - } + var shadowResult = ProviderReferenceMapShadowAnalyzer.LatestResult ?? ProviderReferenceMapShadowResult.Empty; + var symbolsToInternalize = GetSymbolsByName(definitions.DeclaredSymbols, shadowResult.InternalizeCandidates).ToArray(); var nodesToInternalize = new Dictionary(); foreach (var symbol in symbolsToInternalize) @@ -254,41 +230,9 @@ public async Task RemoveAsync(Project project) // find all the declarations, including non-public declared var definitions = await GetTypeSymbolsAsync(compilation, project, false); - IEnumerable symbolsToRemove; - HashSet referencedSet; - if (ProviderReferenceMapShadowAnalyzer.UseShadowMap && ProviderReferenceMapShadowAnalyzer.LatestResult is { } useShadowResult) - { - symbolsToRemove = GetSymbolsByName(definitions.DeclaredSymbols, useShadowResult.RemoveCandidates).ToArray(); - referencedSet = new HashSet(definitions.DeclaredSymbols.Except(symbolsToRemove), SymbolEqualityComparer.Default); - } - else - { - // build reference map - var referenceMap = - await new ReferenceMapBuilder(compilation, project).BuildAllReferenceMapAsync( - definitions.DeclaredSymbols, definitions.DocumentsCache); - // get root symbols - var rootSymbols = await GetRootSymbolsAsync(project, definitions); - // include model factory as a root symbol when doing the remove pass so that we are sure to include any internal - // helpers that are required by the model factory. - if (_modelFactorySymbol != null) - rootSymbols.Add(_modelFactorySymbol); - // traverse the map to determine the declarations that we are about to remove, starting from root nodes - var referencedSymbols = VisitSymbolsFromRootAsync(rootSymbols, referenceMap); - - referencedSymbols = AddSampleSymbols(referencedSymbols, definitions.DeclaredSymbols); - referencedSet = new HashSet(referencedSymbols, SymbolEqualityComparer.Default); - - symbolsToRemove = definitions.DeclaredSymbols.Except(referencedSet); - } - - if (ProviderReferenceMapShadowAnalyzer.LatestResult is { } shadowResult) - { - ProviderReferenceMapShadowAnalyzer.WriteComparisonReport( - "remove", - symbolsToRemove.Select(static symbol => symbol.GetFullyQualifiedName()), - shadowResult.RemoveCandidates); - } + var shadowResult = ProviderReferenceMapShadowAnalyzer.LatestResult ?? ProviderReferenceMapShadowResult.Empty; + var symbolsToRemove = GetSymbolsByName(definitions.DeclaredSymbols, shadowResult.RemoveCandidates).ToArray(); + var referencedSet = new HashSet(definitions.DeclaredSymbols.Except(symbolsToRemove), SymbolEqualityComparer.Default); var nodesToRemove = new List(); foreach (var symbol in symbolsToRemove) diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapShadowAnalyzer.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapShadowAnalyzer.cs index 7d549afb5a1..df8858156cc 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapShadowAnalyzer.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapShadowAnalyzer.cs @@ -3,9 +3,7 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; -using System.Text; using Microsoft.TypeSpec.Generator.Primitives; using Microsoft.TypeSpec.Generator.Providers; using Microsoft.TypeSpec.Generator.Statements; @@ -16,32 +14,12 @@ namespace Microsoft.TypeSpec.Generator { internal static class ProviderReferenceMapShadowAnalyzer { - private const string EnableEnvironmentVariable = "TYPESPEC_PROVIDER_REFERENCE_MAP_SHADOW"; - private const string UseShadowEnvironmentVariable = "TYPESPEC_PROVIDER_REFERENCE_MAP_USE_SHADOW"; - private const string OutputDirectoryEnvironmentVariable = "TYPESPEC_PROVIDER_REFERENCE_MAP_SHADOW_DIR"; - private static ProviderReferenceMapShadowResult? _latestResult; - public static bool IsEnabled => string.Equals( - Environment.GetEnvironmentVariable(EnableEnvironmentVariable), - "true", - StringComparison.OrdinalIgnoreCase); - public static ProviderReferenceMapShadowResult? LatestResult => _latestResult; - public static bool UseShadowMap => string.Equals( - Environment.GetEnvironmentVariable(UseShadowEnvironmentVariable), - "true", - StringComparison.OrdinalIgnoreCase); - public static void Analyze(IReadOnlyList providers, Project project) { - if (!IsEnabled) - { - _latestResult = null; - return; - } - var graph = BuildGraph(providers); var customRoots = GetCustomCodeGeneratedTypeRoots(project, graph.Nodes); var internalizeRoots = GetRootNames(providers, graph.Nodes, helperRoots: [], includeModelFactory: false); @@ -67,8 +45,6 @@ public static void Analyze(IReadOnlyList providers, Project projec _latestResult = new ProviderReferenceMapShadowResult( internalizeCandidates.ToHashSet(StringComparer.Ordinal), removeCandidates.ToHashSet(StringComparer.Ordinal)); - - WriteReport(graph, customRoots, helperRoots, internalizeRoots, internalizeReachable, internalizeCandidates, removeRoots, removeReachable, removeCandidates); } private static HashSet GetCustomCodeGeneratedTypeRoots(Project project, HashSet generatedTypeNames) @@ -132,45 +108,6 @@ private static void AddSymbolRoot(HashSet roots, ITypeSymbol? symbol, Ha } } - public static void WriteComparisonReport(string passName, IEnumerable roslynCandidates, IEnumerable providerCandidates) - { - if (!IsEnabled) - { - return; - } - - var roslynSet = roslynCandidates.ToHashSet(StringComparer.Ordinal); - var providerSet = providerCandidates.ToHashSet(StringComparer.Ordinal); - var missingFromProvider = roslynSet.Except(providerSet, StringComparer.Ordinal).OrderBy(static name => name, StringComparer.Ordinal).ToArray(); - var extraInProvider = providerSet.Except(roslynSet, StringComparer.Ordinal).OrderBy(static name => name, StringComparer.Ordinal).ToArray(); - - var directory = GetOutputDirectory(); - Directory.CreateDirectory(directory); - var path = Path.Combine(directory, $"provider-reference-map-shadow-comparison-{passName}-{DateTime.UtcNow:yyyyMMddHHmmssfff}.txt"); - var builder = new StringBuilder(); - builder.AppendLine($"Provider reference map shadow comparison: {passName}"); - builder.AppendLine($"Roslyn candidates: {roslynSet.Count}"); - builder.AppendLine($"Provider candidates: {providerSet.Count}"); - builder.AppendLine($"Missing from provider: {missingFromProvider.Length}"); - builder.AppendLine($"Extra in provider: {extraInProvider.Length}"); - builder.AppendLine(); - builder.AppendLine("Missing from provider:"); - foreach (var item in missingFromProvider) - { - builder.AppendLine($" {item}"); - } - - builder.AppendLine(); - builder.AppendLine("Extra in provider:"); - foreach (var item in extraInProvider) - { - builder.AppendLine($" {item}"); - } - - File.WriteAllText(path, builder.ToString()); - CodeModelGenerator.Instance.Emitter.Debug($"Provider reference map shadow comparison written to {path}"); - } - private static ProviderReferenceGraph BuildGraph(IReadOnlyList providers) { var nodes = providers @@ -498,89 +435,6 @@ private static void AddTypeReference(HashSet references, CSharpType? typ } } - private static void WriteReport( - ProviderReferenceGraph graph, - HashSet customRoots, - HashSet helperRoots, - HashSet internalizeRoots, - HashSet internalizeReachable, - IReadOnlyList internalizeCandidates, - HashSet removeRoots, - HashSet removeReachable, - IReadOnlyList removeCandidates) - { - var directory = GetOutputDirectory(); - - Directory.CreateDirectory(directory); - var path = Path.Combine(directory, $"provider-reference-map-shadow-{DateTime.UtcNow:yyyyMMddHHmmssfff}.txt"); - var builder = new StringBuilder(); - builder.AppendLine("Provider reference map shadow report"); - builder.AppendLine($"Declared providers: {graph.Nodes.Count}"); - builder.AppendLine($"Internalize roots: {internalizeRoots.Count}"); - builder.AppendLine($"Internalize reachable: {internalizeReachable.Count}"); - builder.AppendLine($"Internalize candidates: {internalizeCandidates.Count}"); - builder.AppendLine($"Custom roots: {customRoots.Count}"); - builder.AppendLine($"Helper roots: {helperRoots.Count}"); - builder.AppendLine($"Remove roots: {removeRoots.Count}"); - builder.AppendLine($"Remove reachable: {removeReachable.Count}"); - builder.AppendLine($"Remove candidates: {removeCandidates.Count}"); - builder.AppendLine(); - builder.AppendLine("Custom roots:"); - foreach (var root in customRoots.OrderBy(static name => name, StringComparer.Ordinal)) - { - builder.AppendLine($" {root}"); - } - - builder.AppendLine(); - builder.AppendLine("Helper roots:"); - foreach (var root in helperRoots.OrderBy(static name => name, StringComparer.Ordinal)) - { - builder.AppendLine($" {root}"); - } - - builder.AppendLine(); - builder.AppendLine("Internalize roots:"); - foreach (var root in internalizeRoots.OrderBy(static name => name, StringComparer.Ordinal)) - { - builder.AppendLine($" {root}"); - } - - builder.AppendLine(); - builder.AppendLine("Internalize candidates:"); - foreach (var candidate in internalizeCandidates) - { - builder.AppendLine($" {candidate}"); - } - - builder.AppendLine(); - builder.AppendLine("Remove roots:"); - foreach (var root in removeRoots.OrderBy(static name => name, StringComparer.Ordinal)) - { - builder.AppendLine($" {root}"); - } - - builder.AppendLine(); - builder.AppendLine("Remove candidates:"); - foreach (var candidate in removeCandidates) - { - builder.AppendLine($" {candidate}"); - } - - builder.AppendLine(); - builder.AppendLine("References:"); - foreach (var (type, references) in graph.References.OrderBy(static item => item.Key, StringComparer.Ordinal)) - { - builder.AppendLine($" {type}"); - foreach (var reference in references.OrderBy(static name => name, StringComparer.Ordinal)) - { - builder.AppendLine($" -> {reference}"); - } - } - - File.WriteAllText(path, builder.ToString()); - CodeModelGenerator.Instance.Emitter.Debug($"Provider reference map shadow report written to {path}"); - } - private static string GetSimpleName(string fullyQualifiedName) { var lastDot = fullyQualifiedName.LastIndexOf('.'); @@ -604,13 +458,5 @@ private static string StripGenericArity(string name) private sealed record ProviderReferenceGraph( HashSet Nodes, Dictionary> References); - - private static string GetOutputDirectory() - { - var directory = Environment.GetEnvironmentVariable(OutputDirectoryEnvironmentVariable); - return string.IsNullOrWhiteSpace(directory) - ? Path.Combine(Path.GetTempPath(), "typespec-provider-reference-map-shadow") - : Path.GetFullPath(directory); - } } } From dc145f4fc400810850fb908c8fb599cfced151f3 Mon Sep 17 00:00:00 2001 From: Wei Hu Date: Fri, 12 Jun 2026 06:53:36 +0000 Subject: [PATCH 05/13] Rename hybrid reference map components --- .../src/PostProcessing/GeneratedCodeWorkspace.cs | 2 +- .../src/PostProcessing/PostProcessor.cs | 8 ++++---- ...pShadowAnalyzer.cs => ProviderReferenceMapAnalyzer.cs} | 8 ++++---- ...ceMapShadowResult.cs => ProviderReferenceMapResult.cs} | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) rename packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/{ProviderReferenceMapShadowAnalyzer.cs => ProviderReferenceMapAnalyzer.cs} (98%) rename packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/{ProviderReferenceMapShadowResult.cs => ProviderReferenceMapResult.cs} (65%) diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspace.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspace.cs index de03b1ca6e7..b999674e9cf 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspace.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/GeneratedCodeWorkspace.cs @@ -85,7 +85,7 @@ public async Task AddInMemoryFile(TypeProvider type) internal void AnalyzeProviderReferenceMap(IReadOnlyList providers) { - ProviderReferenceMapShadowAnalyzer.Analyze(providers, _project); + ProviderReferenceMapAnalyzer.Analyze(providers, _project); } private async Task UpdateProject(Document document) diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs index a6cecba593f..5097677c401 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs @@ -131,8 +131,8 @@ public async Task InternalizeAsync(Project project) // first get all the declared symbols var definitions = await GetTypeSymbolsAsync(compilation, project, true); - var shadowResult = ProviderReferenceMapShadowAnalyzer.LatestResult ?? ProviderReferenceMapShadowResult.Empty; - var symbolsToInternalize = GetSymbolsByName(definitions.DeclaredSymbols, shadowResult.InternalizeCandidates).ToArray(); + var referenceMapResult = ProviderReferenceMapAnalyzer.LatestResult ?? ProviderReferenceMapResult.Empty; + var symbolsToInternalize = GetSymbolsByName(definitions.DeclaredSymbols, referenceMapResult.InternalizeCandidates).ToArray(); var nodesToInternalize = new Dictionary(); foreach (var symbol in symbolsToInternalize) @@ -230,8 +230,8 @@ public async Task RemoveAsync(Project project) // find all the declarations, including non-public declared var definitions = await GetTypeSymbolsAsync(compilation, project, false); - var shadowResult = ProviderReferenceMapShadowAnalyzer.LatestResult ?? ProviderReferenceMapShadowResult.Empty; - var symbolsToRemove = GetSymbolsByName(definitions.DeclaredSymbols, shadowResult.RemoveCandidates).ToArray(); + var referenceMapResult = ProviderReferenceMapAnalyzer.LatestResult ?? ProviderReferenceMapResult.Empty; + var symbolsToRemove = GetSymbolsByName(definitions.DeclaredSymbols, referenceMapResult.RemoveCandidates).ToArray(); var referencedSet = new HashSet(definitions.DeclaredSymbols.Except(symbolsToRemove), SymbolEqualityComparer.Default); var nodesToRemove = new List(); diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapShadowAnalyzer.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs similarity index 98% rename from packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapShadowAnalyzer.cs rename to packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs index df8858156cc..07eca2c5fd1 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapShadowAnalyzer.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs @@ -12,11 +12,11 @@ namespace Microsoft.TypeSpec.Generator { - internal static class ProviderReferenceMapShadowAnalyzer + internal static class ProviderReferenceMapAnalyzer { - private static ProviderReferenceMapShadowResult? _latestResult; + private static ProviderReferenceMapResult? _latestResult; - public static ProviderReferenceMapShadowResult? LatestResult => _latestResult; + public static ProviderReferenceMapResult? LatestResult => _latestResult; public static void Analyze(IReadOnlyList providers, Project project) { @@ -42,7 +42,7 @@ public static void Analyze(IReadOnlyList providers, Project projec var helperRoots = internalizeHelperRoots.Concat(removeHelperRoots).ToHashSet(StringComparer.Ordinal); - _latestResult = new ProviderReferenceMapShadowResult( + _latestResult = new ProviderReferenceMapResult( internalizeCandidates.ToHashSet(StringComparer.Ordinal), removeCandidates.ToHashSet(StringComparer.Ordinal)); } diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapShadowResult.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapResult.cs similarity index 65% rename from packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapShadowResult.cs rename to packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapResult.cs index 14b68f7d187..bb73e08040c 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapShadowResult.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapResult.cs @@ -5,10 +5,10 @@ namespace Microsoft.TypeSpec.Generator { - internal sealed record ProviderReferenceMapShadowResult( + internal sealed record ProviderReferenceMapResult( HashSet InternalizeCandidates, HashSet RemoveCandidates) { - public static ProviderReferenceMapShadowResult Empty { get; } = new([], []); + public static ProviderReferenceMapResult Empty { get; } = new([], []); } } From 0cc3abf89306204bd66a0ca94e9084e1ab566ded Mon Sep 17 00:00:00 2001 From: Wei Hu Date: Fri, 12 Jun 2026 07:20:00 +0000 Subject: [PATCH 06/13] Document hybrid reference map decisions --- .../src/PostProcessing/PostProcessor.cs | 4 ++++ .../src/PostProcessing/ProviderReferenceMapAnalyzer.cs | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs index 5097677c401..9c069baf2c7 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs @@ -132,6 +132,8 @@ public async Task InternalizeAsync(Project project) // first get all the declared symbols var definitions = await GetTypeSymbolsAsync(compilation, project, true); var referenceMapResult = ProviderReferenceMapAnalyzer.LatestResult ?? ProviderReferenceMapResult.Empty; + // ProviderReferenceMapAnalyzer replaces Roslyn reference-map construction for generated code. + // It still uses Roslyn-discovered roots for custom/shared code before this point. var symbolsToInternalize = GetSymbolsByName(definitions.DeclaredSymbols, referenceMapResult.InternalizeCandidates).ToArray(); var nodesToInternalize = new Dictionary(); @@ -231,6 +233,8 @@ public async Task RemoveAsync(Project project) // find all the declarations, including non-public declared var definitions = await GetTypeSymbolsAsync(compilation, project, false); var referenceMapResult = ProviderReferenceMapAnalyzer.LatestResult ?? ProviderReferenceMapResult.Empty; + // The remove pass uses the same precomputed hybrid map to avoid scanning all generated + // documents with Roslyn while preserving custom-code references as roots. var symbolsToRemove = GetSymbolsByName(definitions.DeclaredSymbols, referenceMapResult.RemoveCandidates).ToArray(); var referencedSet = new HashSet(definitions.DeclaredSymbols.Except(symbolsToRemove), SymbolEqualityComparer.Default); diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs index 07eca2c5fd1..ea7e39fbd74 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs @@ -21,7 +21,13 @@ internal static class ProviderReferenceMapAnalyzer public static void Analyze(IReadOnlyList providers, Project project) { var graph = BuildGraph(providers); + + // Generated-code dependencies come from providers. Custom code still needs Roslyn + // because arbitrary user C# can reference generated types in ways providers cannot see. var customRoots = GetCustomCodeGeneratedTypeRoots(project, graph.Nodes); + + // Helper types are rooted after an initial reachability pass so unused infrastructure + // such as change-tracking dictionaries can still be removed when no reachable type needs them. var internalizeRoots = GetRootNames(providers, graph.Nodes, helperRoots: [], includeModelFactory: false); internalizeRoots.UnionWith(customRoots); var internalizeReachableWithoutHelpers = GetReachableTypes(internalizeRoots, graph.References); @@ -122,6 +128,9 @@ private static ProviderReferenceGraph BuildGraph(IReadOnlyList pro AddTypeReference(references[current], provider.BaseType, nodes); AddTypeReference(references[current], provider.DeclaringTypeProvider?.Type, nodes); + // Model factory signatures mention many models. The existing Roslyn post-processor + // removes factory methods for unreachable models, so model factory should only + // contribute helper dependencies, not model reachability edges. if (IsModelFactoryProvider(provider)) { continue; @@ -251,6 +260,7 @@ private static HashSet GetHelperRootNames(IReadOnlyList pr foreach (var method in provider.Methods) { + // Only factory methods for reachable models can instantiate collection helpers. if (isModelFactory && (method.Signature.ReturnType == null || !reachableTypes.Contains(GetProviderTypeName(method.Signature.ReturnType)))) { From c2abe7952be0f199f35fbf017c23cf10178c22be Mon Sep 17 00:00:00 2001 From: Wei Hu Date: Fri, 12 Jun 2026 08:09:10 +0000 Subject: [PATCH 07/13] Fix hybrid reference map CI failures --- .../src/PostProcessing/PostProcessor.cs | 48 ++++++++-- .../ProviderReferenceMapAnalyzer.cs | 87 ++++++++++++++++++- .../ProviderReferenceMapResult.cs | 3 +- 3 files changed, 127 insertions(+), 11 deletions(-) diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs index 9c069baf2c7..09cc693362b 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/PostProcessor.cs @@ -131,10 +131,21 @@ public async Task InternalizeAsync(Project project) // first get all the declared symbols var definitions = await GetTypeSymbolsAsync(compilation, project, true); - var referenceMapResult = ProviderReferenceMapAnalyzer.LatestResult ?? ProviderReferenceMapResult.Empty; - // ProviderReferenceMapAnalyzer replaces Roslyn reference-map construction for generated code. - // It still uses Roslyn-discovered roots for custom/shared code before this point. - var symbolsToInternalize = GetSymbolsByName(definitions.DeclaredSymbols, referenceMapResult.InternalizeCandidates).ToArray(); + IEnumerable symbolsToInternalize; + if (ProviderReferenceMapAnalyzer.LatestResult is { } referenceMapResult && referenceMapResult.ProjectId == project.Id) + { + // ProviderReferenceMapAnalyzer replaces Roslyn reference-map construction for generated code. + // It still uses Roslyn-discovered roots for custom/shared code before this point. + symbolsToInternalize = GetSymbolsByName(definitions.DeclaredSymbols, referenceMapResult.InternalizeCandidates).ToArray(); + } + else + { + var referenceMap = await new ReferenceMapBuilder(compilation, project).BuildPublicReferenceMapAsync( + definitions.DeclaredSymbols, definitions.DeclaredNodesCache); + var rootSymbols = await GetRootSymbolsAsync(project, definitions); + var publicSymbols = VisitSymbolsFromRootAsync(rootSymbols, referenceMap); + symbolsToInternalize = definitions.DeclaredSymbols.Except(publicSymbols); + } var nodesToInternalize = new Dictionary(); foreach (var symbol in symbolsToInternalize) @@ -232,11 +243,30 @@ public async Task RemoveAsync(Project project) // find all the declarations, including non-public declared var definitions = await GetTypeSymbolsAsync(compilation, project, false); - var referenceMapResult = ProviderReferenceMapAnalyzer.LatestResult ?? ProviderReferenceMapResult.Empty; - // The remove pass uses the same precomputed hybrid map to avoid scanning all generated - // documents with Roslyn while preserving custom-code references as roots. - var symbolsToRemove = GetSymbolsByName(definitions.DeclaredSymbols, referenceMapResult.RemoveCandidates).ToArray(); - var referencedSet = new HashSet(definitions.DeclaredSymbols.Except(symbolsToRemove), SymbolEqualityComparer.Default); + IEnumerable symbolsToRemove; + HashSet referencedSet; + if (ProviderReferenceMapAnalyzer.LatestResult is { } referenceMapResult && referenceMapResult.ProjectId == project.Id) + { + // The remove pass uses the same precomputed hybrid map to avoid scanning all generated + // documents with Roslyn while preserving custom-code references as roots. + symbolsToRemove = GetSymbolsByName(definitions.DeclaredSymbols, referenceMapResult.RemoveCandidates).ToArray(); + referencedSet = new HashSet(definitions.DeclaredSymbols.Except(symbolsToRemove), SymbolEqualityComparer.Default); + } + else + { + var referenceMap = await new ReferenceMapBuilder(compilation, project).BuildAllReferenceMapAsync( + definitions.DeclaredSymbols, definitions.DocumentsCache); + var rootSymbols = await GetRootSymbolsAsync(project, definitions); + if (_modelFactorySymbol != null) + { + rootSymbols.Add(_modelFactorySymbol); + } + + var referencedSymbols = VisitSymbolsFromRootAsync(rootSymbols, referenceMap); + referencedSymbols = AddSampleSymbols(referencedSymbols, definitions.DeclaredSymbols); + referencedSet = new HashSet(referencedSymbols, SymbolEqualityComparer.Default); + symbolsToRemove = definitions.DeclaredSymbols.Except(referencedSet); + } var nodesToRemove = new List(); foreach (var symbol in symbolsToRemove) diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs index ea7e39fbd74..4e9cf06aa39 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs @@ -9,6 +9,7 @@ using Microsoft.TypeSpec.Generator.Statements; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.FindSymbols; namespace Microsoft.TypeSpec.Generator { @@ -37,6 +38,10 @@ public static void Analyze(IReadOnlyList providers, Project projec var internalizeDeclaredNodes = GetPostProcessorDeclaredNodes(providers, graph.Nodes, publicOnly: true); var internalizeCandidates = internalizeDeclaredNodes.Except(internalizeReachable, StringComparer.Ordinal).OrderBy(static name => name, StringComparer.Ordinal).ToArray(); + // Body-only generated dependencies are needed to avoid deleting helper files, but they do + // not contribute to public API reachability for internalization. + AddGeneratedBodyReferences(project, providers, graph); + var removeRoots = GetRootNames(providers, graph.Nodes, helperRoots: [], includeModelFactory: true); removeRoots.UnionWith(customRoots); var removeReachableWithoutHelpers = GetReachableTypes(removeRoots, graph.References); @@ -49,6 +54,7 @@ public static void Analyze(IReadOnlyList providers, Project projec var helperRoots = internalizeHelperRoots.Concat(removeHelperRoots).ToHashSet(StringComparer.Ordinal); _latestResult = new ProviderReferenceMapResult( + project.Id, internalizeCandidates.ToHashSet(StringComparer.Ordinal), removeCandidates.ToHashSet(StringComparer.Ordinal)); } @@ -178,6 +184,82 @@ private static ProviderReferenceGraph BuildGraph(IReadOnlyList pro return new ProviderReferenceGraph(nodes, references); } + private static void AddGeneratedBodyReferences(Project project, IReadOnlyList providers, ProviderReferenceGraph graph) + { + var compilation = project.GetCompilationAsync().GetAwaiter().GetResult(); + if (compilation == null) + { + return; + } + + foreach (var provider in providers) + { + var providerName = GetProviderTypeName(provider.Type); + if (!graph.Nodes.Contains(providerName)) + { + continue; + } + + var symbol = compilation.GetTypeByMetadataName(providerName); + if (symbol == null) + { + continue; + } + + AddGeneratedReferencesToHelper(project, compilation, graph, providerName, symbol); + if (provider.DeclarationModifiers.HasFlag(TypeSignatureModifiers.Static)) + { + foreach (var method in symbol.GetMembers().OfType()) + { + if (method.IsExtensionMethod) + { + AddGeneratedReferencesToHelper(project, compilation, graph, providerName, method); + } + } + } + } + } + + private static void AddGeneratedReferencesToHelper(Project project, Compilation compilation, ProviderReferenceGraph graph, string helperName, ISymbol symbol) + { + foreach (var reference in SymbolFinder.FindReferencesAsync(symbol, project.Solution).GetAwaiter().GetResult()) + { + foreach (var location in reference.Locations) + { + var document = location.Document; + if (!GeneratedCodeWorkspace.IsGeneratedDocument(document)) + { + continue; + } + + var root = document.GetSyntaxRootAsync().GetAwaiter().GetResult(); + if (root == null) + { + continue; + } + + var node = root.FindNode(location.Location.SourceSpan); + var owner = node.AncestorsAndSelf().OfType().FirstOrDefault(); + if (owner == null) + { + continue; + } + + var semanticModel = compilation.GetSemanticModel(owner.SyntaxTree); + if (semanticModel.GetDeclaredSymbol(owner) is not INamedTypeSymbol ownerSymbol) + { + continue; + } + + var ownerName = ownerSymbol.GetFullyQualifiedName(); + if (graph.Nodes.Contains(ownerName)) + { + graph.References[ownerName].Add(helperName); + } + } + } + } + private static HashSet GetRootNames(IReadOnlyList providers, HashSet nodes, HashSet helperRoots, bool includeModelFactory) { var generator = CodeModelGenerator.Instance; @@ -187,7 +269,7 @@ private static HashSet GetRootNames(IReadOnlyList provider foreach (var provider in providers) { var name = GetProviderTypeName(provider.Type); - if (provider.Name.EndsWith("Client", StringComparison.Ordinal) || + if (IsClientProviderRoot(provider) || IsKept(provider.Type, generator.AdditionalRootTypes, nodes) || includeModelFactory && string.Equals(name, modelFactoryName, StringComparison.Ordinal) || includeModelFactory && helperRoots.Contains(name)) @@ -225,6 +307,9 @@ private static HashSet GetPostProcessorDeclaredNodes(IReadOnlyList roots, HashSet nodes) => roots.Contains(type.Name) || roots.Contains(GetProviderTypeName(type)) && nodes.Contains(GetProviderTypeName(type)); + private static bool IsClientProviderRoot(TypeProvider provider) => + provider.RelativeFilePath.EndsWith("Client.cs", StringComparison.Ordinal); + private static bool IsModelFactoryProvider(TypeProvider provider) => provider.GetType().Name == "ModelFactoryProvider"; private static HashSet GetHelperRootNames(IReadOnlyList providers, HashSet nodes, HashSet reachableTypes) diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapResult.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapResult.cs index bb73e08040c..e5623b1e7d2 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapResult.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapResult.cs @@ -2,13 +2,14 @@ // Licensed under the MIT License. using System.Collections.Generic; +using Microsoft.CodeAnalysis; namespace Microsoft.TypeSpec.Generator { internal sealed record ProviderReferenceMapResult( + ProjectId ProjectId, HashSet InternalizeCandidates, HashSet RemoveCandidates) { - public static ProviderReferenceMapResult Empty { get; } = new([], []); } } From c46399569b78acc2ba1597b878761e003ec190ab Mon Sep 17 00:00:00 2001 From: Wei Hu Date: Fri, 12 Jun 2026 09:20:09 +0000 Subject: [PATCH 08/13] Track serialization providers in hybrid reference map --- .../ProviderReferenceMapAnalyzer.cs | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs index 4e9cf06aa39..1806f6efbf4 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs @@ -122,12 +122,13 @@ private static void AddSymbolRoot(HashSet roots, ITypeSymbol? symbol, Ha private static ProviderReferenceGraph BuildGraph(IReadOnlyList providers) { - var nodes = providers + var generatedProviders = GetGeneratedProviders(providers); + var nodes = generatedProviders .Select(static provider => GetProviderTypeName(provider.Type)) .ToHashSet(StringComparer.Ordinal); var references = nodes.ToDictionary(static name => name, _ => new HashSet(StringComparer.Ordinal), StringComparer.Ordinal); - foreach (var provider in providers) + foreach (var provider in generatedProviders) { var current = GetProviderTypeName(provider.Type); AddTypeReference(references[current], provider.Type, nodes); @@ -184,6 +185,18 @@ private static ProviderReferenceGraph BuildGraph(IReadOnlyList pro return new ProviderReferenceGraph(nodes, references); } + private static IReadOnlyList GetGeneratedProviders(IReadOnlyList providers) + { + var generatedProviders = new List(); + foreach (var provider in providers) + { + generatedProviders.Add(provider); + generatedProviders.AddRange(provider.SerializationProviders); + } + + return generatedProviders; + } + private static void AddGeneratedBodyReferences(Project project, IReadOnlyList providers, ProviderReferenceGraph graph) { var compilation = project.GetCompilationAsync().GetAwaiter().GetResult(); @@ -194,6 +207,11 @@ private static void AddGeneratedBodyReferences(Project project, IReadOnlyList Date: Fri, 12 Jun 2026 10:25:28 +0000 Subject: [PATCH 09/13] Handle model factory subclasses in reference map --- .../src/PostProcessing/ProviderReferenceMapAnalyzer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs index 1806f6efbf4..6b5fa559497 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs @@ -340,7 +340,7 @@ private static bool IsKept(CSharpType type, HashSet roots, HashSet provider.RelativeFilePath.EndsWith("Client.cs", StringComparison.Ordinal); - private static bool IsModelFactoryProvider(TypeProvider provider) => provider.GetType().Name == "ModelFactoryProvider"; + private static bool IsModelFactoryProvider(TypeProvider provider) => provider is ModelFactoryProvider; private static HashSet GetHelperRootNames(IReadOnlyList providers, HashSet nodes, HashSet reachableTypes) { From 2917111201920d9e0f5b8f3260fe77051a4bac9a Mon Sep 17 00:00:00 2001 From: Wei Hu Date: Mon, 15 Jun 2026 02:01:02 +0000 Subject: [PATCH 10/13] Track generated body type references --- .../ProviderReferenceMapAnalyzer.cs | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs index 6b5fa559497..9c2c790f30f 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs @@ -8,6 +8,7 @@ using Microsoft.TypeSpec.Generator.Providers; using Microsoft.TypeSpec.Generator.Statements; using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.CodeAnalysis.FindSymbols; @@ -235,6 +236,8 @@ private static void AddGeneratedBodyReferences(Project project, IReadOnlyList()) + { + // Declaration names are the owner itself. The old Roslyn map captures references, + // not a declaration making itself reachable. + if (typeSyntax.Parent is BaseTypeDeclarationSyntax baseTypeDeclaration && baseTypeDeclaration.Identifier.Span == typeSyntax.Span) + { + continue; + } + + AddBodyTypeReference(graph.References[ownerName], semanticModel.GetTypeInfo(typeSyntax).Type, graph.Nodes); + } + } + } + + private static void AddBodyTypeReference(HashSet references, ITypeSymbol? symbol, HashSet nodes) + { + if (symbol is not INamedTypeSymbol namedType || namedType.TypeKind == TypeKind.Error) + { + return; + } + + AddMatchingName(references, namedType.GetFullyQualifiedName(), nodes); + foreach (var typeArgument in namedType.TypeArguments) + { + AddBodyTypeReference(references, typeArgument, nodes); + } + } + private static void AddGeneratedReferencesToHelper(Project project, Compilation compilation, ProviderReferenceGraph graph, string helperName, ISymbol symbol) { foreach (var reference in SymbolFinder.FindReferencesAsync(symbol, project.Solution).GetAwaiter().GetResult()) From 2cda1846d2a99112dd56a39ad4bc3f614a4ade7d Mon Sep 17 00:00:00 2001 From: Wei Hu Date: Mon, 15 Jun 2026 03:51:16 +0000 Subject: [PATCH 11/13] Match Roslyn cleanup for body references --- .../ProviderReferenceMapAnalyzer.cs | 55 ++++++++++++++++--- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs index 9c2c790f30f..212fb4129f0 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs @@ -136,6 +136,11 @@ private static ProviderReferenceGraph BuildGraph(IReadOnlyList pro AddTypeReference(references[current], provider.BaseType, nodes); AddTypeReference(references[current], provider.DeclaringTypeProvider?.Type, nodes); + if (IsKept(provider.Type, CodeModelGenerator.Instance.NonRootTypes, nodes)) + { + continue; + } + // Model factory signatures mention many models. The existing Roslyn post-processor // removes factory methods for unreachable models, so model factory should only // contribute helper dependencies, not model reachability edges. @@ -206,8 +211,13 @@ private static void AddGeneratedBodyReferences(Project project, IReadOnlyList()) + AddGeneratedReferencesToHelper(project, compilation, graph, providerName, symbol); + if (provider.DeclarationModifiers.HasFlag(TypeSignatureModifiers.Static)) { - if (method.IsExtensionMethod) + foreach (var method in symbol.GetMembers().OfType()) { - AddGeneratedReferencesToHelper(project, compilation, graph, providerName, method); + if (method.IsExtensionMethod) + { + AddGeneratedReferencesToHelper(project, compilation, graph, providerName, method); + } } } } @@ -241,6 +254,18 @@ private static void AddGeneratedBodyReferences(Project project, IReadOnlyList GetBodyReferenceProviders(IReadOnlyList providers) + { + var bodyReferenceProviders = new List(); + foreach (var provider in providers) + { + bodyReferenceProviders.Add(provider); + bodyReferenceProviders.AddRange(provider.SerializationProviders); + } + + return bodyReferenceProviders; + } + private static bool IsGeneratedBodyReferenceCandidate(TypeProvider provider) { if (provider.DeclarationModifiers.HasFlag(TypeSignatureModifiers.Static)) @@ -249,7 +274,9 @@ private static bool IsGeneratedBodyReferenceCandidate(TypeProvider provider) } var relativePath = provider.RelativeFilePath.Replace('\\', '/'); - return relativePath.EndsWith("/Internal/ClientUriBuilder.cs", StringComparison.Ordinal) || + return IsSerializationProvider(provider) || + relativePath.EndsWith("Client.cs", StringComparison.Ordinal) || + relativePath.EndsWith("/Internal/ClientUriBuilder.cs", StringComparison.Ordinal) || relativePath.Contains("/CollectionResults/", StringComparison.Ordinal); } @@ -388,7 +415,7 @@ private static bool IsClientProviderRoot(TypeProvider provider) => private static HashSet GetHelperRootNames(IReadOnlyList providers, HashSet nodes, HashSet reachableTypes) { var roots = new HashSet(StringComparer.Ordinal); - foreach (var provider in providers) + foreach (var provider in GetGeneratedProviders(providers)) { var providerName = GetProviderTypeName(provider.Type); var isModelFactory = IsModelFactoryProvider(provider); @@ -397,6 +424,11 @@ private static HashSet GetHelperRootNames(IReadOnlyList pr continue; } + if (IsSerializationProvider(provider)) + { + AddMatchingName(roots, "ChangeTrackingDictionary", nodes); + } + foreach (var property in provider.Properties) { AddInitializationHelperRoot(roots, property.Type, nodes); @@ -447,6 +479,13 @@ private static void AddParameterValidationHelperRoot(HashSet roots, Para } } + private static bool IsSerializationProvider(TypeProvider provider) + { + var relativePath = provider.RelativeFilePath.Replace('\\', '/'); + return relativePath.EndsWith(".Serialization.cs", StringComparison.Ordinal) || + relativePath.EndsWith(".Serialization.Multipart.cs", StringComparison.Ordinal); + } + private static void AddInitializationHelperRoot(HashSet roots, CSharpType? type, HashSet nodes) { if (type == null) From ce16922e1f778a6333b107936643beb137c67ac1 Mon Sep 17 00:00:00 2001 From: Wei Hu Date: Mon, 15 Jun 2026 05:33:53 +0000 Subject: [PATCH 12/13] Remove unused serialization providers --- .../src/PostProcessing/ProviderReferenceMapAnalyzer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs index 212fb4129f0..90408e0391d 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs @@ -395,7 +395,7 @@ private static HashSet GetPostProcessorDeclaredNodes(IReadOnlyList !IsModelFactoryProvider(provider)) .Where(provider => !publicOnly || provider.DeclarationModifiers.HasFlag(TypeSignatureModifiers.Public)) .Select(provider => GetProviderTypeName(provider.Type)) From 5a8248f6acd5c448b9a2057ddbac4494cb3d8b09 Mon Sep 17 00:00:00 2001 From: Wei Hu Date: Mon, 15 Jun 2026 07:24:15 +0000 Subject: [PATCH 13/13] Preserve public discriminator subtypes --- .../ProviderReferenceMapAnalyzer.cs | 59 ++++++++++++++++++- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs index 90408e0391d..4cca7806c36 100644 --- a/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs +++ b/packages/http-client-csharp/generator/Microsoft.TypeSpec.Generator/src/PostProcessing/ProviderReferenceMapAnalyzer.cs @@ -23,6 +23,7 @@ internal static class ProviderReferenceMapAnalyzer public static void Analyze(IReadOnlyList providers, Project project) { var graph = BuildGraph(providers); + var publicGraph = BuildGraph(providers, publicOnly: true); // Generated-code dependencies come from providers. Custom code still needs Roslyn // because arbitrary user C# can reference generated types in ways providers cannot see. @@ -30,12 +31,14 @@ public static void Analyze(IReadOnlyList providers, Project projec // Helper types are rooted after an initial reachability pass so unused infrastructure // such as change-tracking dictionaries can still be removed when no reachable type needs them. + var internalizeReferences = CloneReferences(publicGraph.References); + AddDerivedModelReferences(providers, publicGraph.Nodes, internalizeReferences); var internalizeRoots = GetRootNames(providers, graph.Nodes, helperRoots: [], includeModelFactory: false); internalizeRoots.UnionWith(customRoots); - var internalizeReachableWithoutHelpers = GetReachableTypes(internalizeRoots, graph.References); + var internalizeReachableWithoutHelpers = GetReachableTypes(internalizeRoots, internalizeReferences); var internalizeHelperRoots = GetHelperRootNames(providers, graph.Nodes, internalizeReachableWithoutHelpers); internalizeRoots.UnionWith(internalizeHelperRoots); - var internalizeReachable = GetReachableTypes(internalizeRoots, graph.References); + var internalizeReachable = GetReachableTypes(internalizeRoots, internalizeReferences); var internalizeDeclaredNodes = GetPostProcessorDeclaredNodes(providers, graph.Nodes, publicOnly: true); var internalizeCandidates = internalizeDeclaredNodes.Except(internalizeReachable, StringComparer.Ordinal).OrderBy(static name => name, StringComparer.Ordinal).ToArray(); @@ -121,7 +124,7 @@ private static void AddSymbolRoot(HashSet roots, ITypeSymbol? symbol, Ha } } - private static ProviderReferenceGraph BuildGraph(IReadOnlyList providers) + private static ProviderReferenceGraph BuildGraph(IReadOnlyList providers, bool publicOnly = false) { var generatedProviders = GetGeneratedProviders(providers); var nodes = generatedProviders @@ -166,6 +169,11 @@ private static ProviderReferenceGraph BuildGraph(IReadOnlyList pro foreach (var property in provider.Properties) { + if (publicOnly && !IsPublic(property.Modifiers)) + { + continue; + } + AddTypeReference(references[current], property.Type, nodes); AddTypeReference(references[current], property.ExplicitInterface, nodes); AddAttributes(references[current], property.Attributes, nodes); @@ -173,17 +181,32 @@ private static ProviderReferenceGraph BuildGraph(IReadOnlyList pro foreach (var field in provider.Fields) { + if (publicOnly && !field.Modifiers.HasFlag(FieldModifiers.Public)) + { + continue; + } + AddTypeReference(references[current], field.Type, nodes); AddAttributes(references[current], field.Attributes, nodes); } foreach (var constructor in provider.Constructors) { + if (publicOnly && !IsPublic(constructor.Signature.Modifiers)) + { + continue; + } + AddSignatureReferences(references[current], constructor.Signature, nodes); } foreach (var method in provider.Methods) { + if (publicOnly && !IsPublic(method.Signature.Modifiers)) + { + continue; + } + AddSignatureReferences(references[current], method.Signature, nodes); } } @@ -191,6 +214,36 @@ private static ProviderReferenceGraph BuildGraph(IReadOnlyList pro return new ProviderReferenceGraph(nodes, references); } + private static bool IsPublic(MethodSignatureModifiers modifiers) => modifiers.HasFlag(MethodSignatureModifiers.Public); + + private static Dictionary> CloneReferences(IReadOnlyDictionary> references) + { + return references.ToDictionary( + static item => item.Key, + static item => item.Value.ToHashSet(StringComparer.Ordinal), + StringComparer.Ordinal); + } + + private static void AddDerivedModelReferences( + IReadOnlyList providers, + HashSet nodes, + Dictionary> references) + { + foreach (var provider in providers.OfType()) + { + var providerName = GetProviderTypeName(provider.Type); + if (!nodes.Contains(providerName)) + { + continue; + } + + foreach (var derivedModel in provider.DerivedModels) + { + AddTypeReference(references[providerName], derivedModel.Type, nodes); + } + } + } + private static IReadOnlyList GetGeneratedProviders(IReadOnlyList providers) { var generatedProviders = new List();