Add telemetry collection for MSTest usage analytics

src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj

@@ -51,6 +51,11 @@
     <DefineConstants>$(DefineConstants);WINDOWS_UWP</DefineConstants>
   </PropertyGroup>
 
+  <!-- Properties specific to WinUI -->
+  <PropertyGroup Condition=" '$(TargetFramework)' == '$(WinUiMinimum)' ">
+    <DefineConstants>$(DefineConstants);WIN_UI</DefineConstants>
+  </PropertyGroup>
+
   <ItemGroup>
     <ProjectReference Include="$(RepoRoot)src\Adapter\MSTestAdapter.PlatformServices\MSTestAdapter.PlatformServices.csproj" />
     <ProjectReference Include="$(RepoRoot)src\Platform\Microsoft.Testing.Extensions.VSTestBridge\Microsoft.Testing.Extensions.VSTestBridge.csproj" Condition=" '$(TargetFramework)' != '$(UwpMinimum)' " />

src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs

@@ -9,8 +9,10 @@
 using Microsoft.Testing.Platform.Logging;
 using Microsoft.Testing.Platform.Messages;
 using Microsoft.Testing.Platform.Services;
+using Microsoft.Testing.Platform.Telemetry;
 using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter;
 using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions;
+using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices;
 using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Helpers;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 
@@ -41,7 +43,7 @@ protected override Task SynchronizedDiscoverTestsAsync(VSTestDiscoverTestExecuti
             Debugger.Launch();
         }
 
-        new MSTestDiscoverer().DiscoverTests(request.AssemblyPaths, request.DiscoveryContext, request.MessageLogger, request.DiscoverySink, _configuration, isMTP: true);
+        new MSTestDiscoverer(new TestSourceHandler(), CreateTelemetrySender()).DiscoverTests(request.AssemblyPaths, request.DiscoveryContext, request.MessageLogger, request.DiscoverySink, _configuration, isMTP: true);
         return Task.CompletedTask;
     }
 
@@ -55,7 +57,7 @@ protected override async Task SynchronizedRunTestsAsync(VSTestRunTestExecutionRe
             Debugger.Launch();
         }
 
-        MSTestExecutor testExecutor = new(cancellationToken);
+        MSTestExecutor testExecutor = new(cancellationToken, CreateTelemetrySender());
         await testExecutor.RunTestsAsync(request.AssemblyPaths, request.RunContext, request.FrameworkHandle, _configuration, isMTP: true).ConfigureAwait(false);
     }
 
@@ -103,5 +105,19 @@ private static TestMethodIdentifierProperty GetMethodIdentifierPropertyFromManag
         // Or alternatively, does VSTest object model expose the assembly full name somewhere?
         return new TestMethodIdentifierProperty(assemblyFullName: string.Empty, @namespace, typeName, methodName, arity, parameterTypes, returnTypeFullName: string.Empty);
     }
+
+    [SuppressMessage("ApiDesign", "RS0030:Do not use banned APIs", Justification = "We can use MTP from this folder")]
+    private Func<string, IDictionary<string, object>, Task>? CreateTelemetrySender()
+    {
+        ITelemetryInformation telemetryInformation = ServiceProvider.GetTelemetryInformation();
+        if (!telemetryInformation.IsEnabled)
+        {
+            return null;
+        }
+
+        ITelemetryCollector telemetryCollector = ServiceProvider.GetTelemetryCollector();
+
+        return (eventName, metrics) => telemetryCollector.LogEventAsync(eventName, metrics, CancellationToken.None);
+    }
 }
 #endif

src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs

@@ -20,14 +20,24 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter;
 internal sealed class MSTestDiscoverer : ITestDiscoverer
 {
     private readonly ITestSourceHandler _testSourceHandler;
+#if !WINDOWS_UWP && !WIN_UI
+    private readonly Func<string, IDictionary<string, object>, Task>? _telemetrySender;
+#endif
 
     public MSTestDiscoverer()
         : this(new TestSourceHandler())
     {
     }
 
-    internal /* for testing purposes */ MSTestDiscoverer(ITestSourceHandler testSourceHandler)
-        => _testSourceHandler = testSourceHandler;
+    internal MSTestDiscoverer(ITestSourceHandler testSourceHandler, Func<string, IDictionary<string, object>, Task>? telemetrySender = null)
+    {
+        _testSourceHandler = testSourceHandler;
+#if !WINDOWS_UWP && !WIN_UI
+        _telemetrySender = telemetrySender;
+#else
+        _ = telemetrySender;
+#endif
+    }
 
     /// <summary>
     /// Discovers the tests available from the provided source. Not supported for .xap source.
@@ -58,9 +68,26 @@ internal void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext disco
             throw new ArgumentNullException(nameof(discoverySink));
         }
 
-        if (MSTestDiscovererHelpers.InitializeDiscovery(sources, discoveryContext, logger, configuration, _testSourceHandler))
+        // Initialize telemetry collection if not already set (e.g. first call in the session)
+#if !WINDOWS_UWP && !WIN_UI
+        if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut())
+        {
+            _ = MSTestTelemetryDataCollector.EnsureInitialized();
+        }
+#endif
+
+        try
+        {
+            if (MSTestDiscovererHelpers.InitializeDiscovery(sources, discoveryContext, logger, configuration, _testSourceHandler))
+            {
+                new UnitTestDiscoverer(_testSourceHandler).DiscoverTests(sources, logger, discoverySink, discoveryContext, isMTP);
+            }
+        }
+        finally
         {
-            new UnitTestDiscoverer(_testSourceHandler).DiscoverTests(sources, logger, discoverySink, discoveryContext, isMTP);
+#if !WINDOWS_UWP && !WIN_UI
+            MSTestTelemetryDataCollector.SendTelemetryAndResetAsync(_telemetrySender).GetAwaiter().GetResult();
+#endif
         }
     }
 }

src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs

@@ -20,6 +20,9 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter;
 internal sealed class MSTestExecutor : ITestExecutor
 {
     private readonly CancellationToken _cancellationToken;
+#if !WINDOWS_UWP && !WIN_UI
+    private readonly Func<string, IDictionary<string, object>, Task>? _telemetrySender;
+#endif
 
     /// <summary>
     /// Token for canceling the test run.
@@ -35,10 +38,15 @@ public MSTestExecutor()
         _cancellationToken = CancellationToken.None;
     }
 
-    internal MSTestExecutor(CancellationToken cancellationToken)
+    internal MSTestExecutor(CancellationToken cancellationToken, Func<string, IDictionary<string, object>, Task>? telemetrySender = null)
     {
         TestExecutionManager = new TestExecutionManager();
         _cancellationToken = cancellationToken;
+#if !WINDOWS_UWP && !WIN_UI
+        _telemetrySender = telemetrySender;
+#else
+        _ = telemetrySender;
+#endif
     }
 
     /// <summary>
@@ -115,12 +123,27 @@ internal async Task RunTestsAsync(IEnumerable<TestCase>? tests, IRunContext? run
 
         Ensure.NotEmpty(tests);
 
+        // Initialize telemetry collection if not already set
+#if !WINDOWS_UWP && !WIN_UI
+        if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut())
+        {
+            _ = MSTestTelemetryDataCollector.EnsureInitialized();
+        }
+#endif
+
         if (!MSTestDiscovererHelpers.InitializeDiscovery(from test in tests select test.Source, runContext, frameworkHandle, configuration, new TestSourceHandler()))
         {
             return;
         }
 
-        await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(tests, runContext, frameworkHandle, testRunToken).ConfigureAwait(false)).ConfigureAwait(false);
+        try
+        {
+            await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(tests, runContext, frameworkHandle, testRunToken).ConfigureAwait(false)).ConfigureAwait(false);
+        }
+        finally
+        {
+            await SendTelemetryAsync().ConfigureAwait(false);
+        }
     }
 
     internal async Task RunTestsAsync(IEnumerable<string>? sources, IRunContext? runContext, IFrameworkHandle? frameworkHandle, IConfiguration? configuration, bool isMTP)
@@ -143,14 +166,29 @@ internal async Task RunTestsAsync(IEnumerable<string>? sources, IRunContext? run
 
         Ensure.NotEmpty(sources);
 
+        // Initialize telemetry collection if not already set
+#if !WINDOWS_UWP && !WIN_UI
+        if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut())
+        {
+            _ = MSTestTelemetryDataCollector.EnsureInitialized();
+        }
+#endif
+
         TestSourceHandler testSourceHandler = new();
         if (!MSTestDiscovererHelpers.InitializeDiscovery(sources, runContext, frameworkHandle, configuration, testSourceHandler))
         {
             return;
         }
 
         sources = testSourceHandler.GetTestSources(sources);
-        await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(sources, runContext, frameworkHandle, testSourceHandler, isMTP, testRunToken).ConfigureAwait(false)).ConfigureAwait(false);
+        try
+        {
+            await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(sources, runContext, frameworkHandle, testSourceHandler, isMTP, testRunToken).ConfigureAwait(false)).ConfigureAwait(false);
+        }
+        finally
+        {
+            await SendTelemetryAsync().ConfigureAwait(false);
+        }
     }
 
     /// <summary>
@@ -159,6 +197,14 @@ internal async Task RunTestsAsync(IEnumerable<string>? sources, IRunContext? run
     public void Cancel()
         => _testRunCancellationToken?.Cancel();
 
+#if !WINDOWS_UWP && !WIN_UI
+    private Task SendTelemetryAsync()
+        => MSTestTelemetryDataCollector.SendTelemetryAndResetAsync(_telemetrySender);
+#else
+    private static Task SendTelemetryAsync()
+        => Task.CompletedTask;
+#endif
+
     private async Task RunTestsFromRightContextAsync(IFrameworkHandle frameworkHandle, Func<TestRunCancellationToken, Task> runTestsAction)
     {
         ApartmentState? requestedApartmentState = MSTestSettings.RunConfigurationSettings.ExecutionApartmentState;

src/Adapter/MSTestAdapter.PlatformServices/Discovery/AssemblyEnumerator.cs

@@ -146,7 +146,11 @@ internal virtual TypeEnumerator GetTypeEnumerator(Type type, string assemblyFile
         var typeValidator = new TypeValidator(ReflectHelper, discoverInternals);
         var testMethodValidator = new TestMethodValidator(ReflectHelper, discoverInternals);
 
+#if !WINDOWS_UWP && !WIN_UI
+        return new TypeEnumerator(type, assemblyFileName, ReflectHelper, typeValidator, testMethodValidator, Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.MSTestTelemetryDataCollector.Current);
+#else
         return new TypeEnumerator(type, assemblyFileName, ReflectHelper, typeValidator, testMethodValidator);
+#endif
     }
 
     private List<UnitTestElement> DiscoverTestsInType(

src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs

@@ -19,7 +19,22 @@ internal class TypeEnumerator
     private readonly TypeValidator _typeValidator;
     private readonly TestMethodValidator _testMethodValidator;
     private readonly ReflectHelper _reflectHelper;
+#if !WINDOWS_UWP && !WIN_UI
+    private readonly Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.MSTestTelemetryDataCollector? _telemetryDataCollector;
+#endif
 
+#if !WINDOWS_UWP && !WIN_UI
+    /// <summary>
+    /// Initializes a new instance of the <see cref="TypeEnumerator"/> class.
+    /// </summary>
+    /// <param name="type"> The reflected type. </param>
+    /// <param name="assemblyFilePath"> The name of the assembly being reflected. </param>
+    /// <param name="reflectHelper"> An instance to reflection helper for type information. </param>
+    /// <param name="typeValidator"> The validator for test classes. </param>
+    /// <param name="testMethodValidator"> The validator for test methods. </param>
+    /// <param name="telemetryDataCollector"> Optional telemetry data collector for tracking API usage. </param>
+    internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflectHelper, TypeValidator typeValidator, TestMethodValidator testMethodValidator, Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.MSTestTelemetryDataCollector? telemetryDataCollector = null)
+#else
     /// <summary>
     /// Initializes a new instance of the <see cref="TypeEnumerator"/> class.
     /// </summary>
@@ -29,12 +44,16 @@ internal class TypeEnumerator
     /// <param name="typeValidator"> The validator for test classes. </param>
     /// <param name="testMethodValidator"> The validator for test methods. </param>
     internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflectHelper, TypeValidator typeValidator, TestMethodValidator testMethodValidator)
+#endif
     {
         _type = type;
         _assemblyFilePath = assemblyFilePath;
         _reflectHelper = reflectHelper;
         _typeValidator = typeValidator;
         _testMethodValidator = testMethodValidator;
+#if !WINDOWS_UWP && !WIN_UI
+        _telemetryDataCollector = telemetryDataCollector;
+#endif
     }
 
     /// <summary>
@@ -49,6 +68,15 @@ internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflec
             return null;
         }
 
+        // Track class-level attributes for telemetry
+#if !WINDOWS_UWP && !WIN_UI
+        if (_telemetryDataCollector is not null)
+        {
+            Attribute[] classAttributes = _reflectHelper.GetCustomAttributesCached(_type);
+            _telemetryDataCollector.TrackDiscoveredClass(classAttributes);
+        }
+#endif
+
         // If test class is valid, then get the tests
         return GetTests(warnings);
     }
@@ -143,6 +171,9 @@ internal UnitTestElement GetTestFromMethod(MethodInfo method, bool classDisables
         };
 
         Attribute[] attributes = _reflectHelper.GetCustomAttributesCached(method);
+#if !WINDOWS_UWP && !WIN_UI
+        _telemetryDataCollector?.TrackDiscoveredMethod(attributes);
+#endif
         TestMethodAttribute? testMethodAttribute = null;
 
         // Backward looping for backcompat. This used to be calls to _reflectHelper.GetFirstAttributeOrDefault

src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.cs

@@ -288,6 +288,18 @@ internal static void PopulateSettings(IDiscoveryContext? context, IMessageLogger
 
         CurrentSettings = settings;
         RunConfigurationSettings = runConfigurationSettings;
+
+        // Track configuration source for telemetry
+#if !WINDOWS_UWP && !WIN_UI
+        if (MSTestTelemetryDataCollector.Current is { } telemetry)
+        {
+            telemetry.ConfigurationSource = configuration?["mstest"] is not null
+                ? "testconfig.json"
+                : !StringEx.IsNullOrEmpty(context?.RunSettings?.SettingsXml)
+                    ? "runsettings"
+                    : "none";
+        }
+#endif
     }
 
     /// <summary>