diff --git a/src/TALXIS.CLI.MCP/GuideReasoningEngine.cs b/src/TALXIS.CLI.MCP/GuideReasoningEngine.cs index 49b51b0d..9fc3ccbc 100644 --- a/src/TALXIS.CLI.MCP/GuideReasoningEngine.cs +++ b/src/TALXIS.CLI.MCP/GuideReasoningEngine.cs @@ -23,6 +23,7 @@ public class GuideReasoningEngine ["guide_deployment"] = ["deployment-sequence", "solution-management"], ["guide_data"] = ["data-migration-workflow"], ["guide_config"] = [], + ["guide_testing"] = ["testing-workflow"], }; /// diff --git a/src/TALXIS.CLI.MCP/Program.cs b/src/TALXIS.CLI.MCP/Program.cs index c7be0aa4..c2150664 100644 --- a/src/TALXIS.CLI.MCP/Program.cs +++ b/src/TALXIS.CLI.MCP/Program.cs @@ -30,6 +30,10 @@ var publicSkillLoader = new PublicSkillLoader(); publicSkillLoader.LoadIndex(); +// Load UI testing step bindings catalog via reflection +var testingBindingsCatalog = new TestingBindingsCatalog(); +testingBindingsCatalog.Load(); + // Session-scoped active tool set — starts with always-on tools only var activeToolSet = new ActiveToolSet(); var guideHandler = new GuideHandler(mcpToolRegistry.Catalog, activeToolSet, reasoningEngine); @@ -71,6 +75,7 @@ - guide_deployment: Deployment lifecycle — import/export/pack solutions, manage components, publish. Requires profile. - guide_data: LIVE data operations — SQL/FetchXML/OData queries, record CRUD, bulk ops, CMT migration. Requires profile. - guide_config: CLI setup — auth credentials, connections, profiles, settings. Required before environment operations. +- guide_testing: UI test generation — discover available Reqnroll step bindings for Power Apps BDD tests. WORKFLOW: Call a guide tool → use execute_operation for immediate execution → discovered tools become direct calls on next turn. @@ -214,6 +219,10 @@ async ValueTask HandleGuideToolAsync( }; } } + else if (guideName == "guide_testing") + { + result = await HandleGuideTestingAsync(query, top, server, ct); + } else if (workflowScope is not null) { result = await guideHandler.HandleWorkflowGuideAsync(workflowScope, query, top, server, ct, guideName); @@ -228,6 +237,78 @@ async ValueTask HandleGuideToolAsync( return result; } +// Handles guide_testing calls — uses TestingBindingsCatalog + sampling to recommend step bindings +async ValueTask HandleGuideTestingAsync( + string query, int top, McpServer server, CancellationToken ct) +{ + if (testingBindingsCatalog.Count == 0) + { + return new CallToolResult + { + Content = [new TextContentBlock { Text = "No step bindings loaded. Ensure the TALXIS.TestKit.Bindings assembly is available." }], + IsError = true + }; + } + + if (string.IsNullOrEmpty(query)) + { + // No query — return full catalog listing + return new CallToolResult + { + Content = [new TextContentBlock { Text = testingBindingsCatalog.GetCatalogPrompt() }] + }; + } + + // Use sampling to select relevant bindings based on user's query + var skillsContext = reasoningEngine.GetSkillsContext("guide_testing"); + var catalogPrompt = testingBindingsCatalog.GetCatalogPrompt(); + + var systemPrompt = $@"You are a Power Apps UI test automation assistant. Given the user's testing task and available step bindings, produce a Gherkin feature file or scenario. + +FORMAT YOUR RESPONSE AS: +1. A complete Gherkin scenario (or scenarios) using the available step bindings +2. Include comments explaining any custom steps that would need to be implemented + +RULES: +- Use ONLY the step bindings listed below when possible +- For login, ALWAYS start with: Given I am logged in to the '{{app}}' app as '{{user}}' +- Use realistic placeholder values based on the user's description +- Each scenario should test ONE behavior +- Include test data setup (Given steps) before actions (When steps) +- End with assertions (Then steps) +- If the available bindings don't cover something, note it as a custom step with a comment + +{catalogPrompt}{skillsContext}"; + + var samplingParams = new CreateMessageRequestParams + { + Messages = + [ + new SamplingMessage + { + Role = Role.User, + Content = [new TextContentBlock { Text = $"Generate a Gherkin test for this scenario: {query}" }] + } + ], + SystemPrompt = systemPrompt, + MaxTokens = 2000, + ModelPreferences = new ModelPreferences + { + SpeedPriority = 0.6f, + CostPriority = 0.4f, + IntelligencePriority = 0.8f + }, + }; + + var result = await server.SampleAsync(samplingParams, ct); + var responseText = result.Content.OfType().FirstOrDefault()?.Text ?? ""; + + return new CallToolResult + { + Content = [new TextContentBlock { Text = responseText }] + }; +} + // Bridge: execute_operation dispatches any tool from the internal catalog async ValueTask HandleExecuteOperationAsync( IDictionary? arguments, RequestContext ctx, CancellationToken ct) @@ -473,7 +554,7 @@ async ValueTask ExecuteAsTaskAsync( bool IsGuideTool(string toolName) { return toolName is "guide" or "guide_workspace" or "guide_environment" - or "guide_deployment" or "guide_data" or "guide_config"; + or "guide_deployment" or "guide_data" or "guide_config" or "guide_testing"; } // Helper: checks if a tool is an MCP-specific in-process tool (not a CLI subprocess) @@ -563,6 +644,13 @@ void RegisterAlwaysOnTools(ActiveToolSet toolSet, McpToolRegistry registry, Publ InputSchema = BuildGuideInputSchema() }); + toolSet.AddAlwaysOn(new Tool + { + Name = "guide_testing", + Description = @"Helps generate Power Apps UI tests using Reqnroll (BDD). Discovers available step bindings from TALXIS.TestKit.Bindings and generates Gherkin scenarios. Provide a description of what you want to test and get ready-to-use feature file content with Given/When/Then steps.", + InputSchema = BuildGuideInputSchema() + }); + // execute_operation bridge — for same-turn execution toolSet.AddAlwaysOn(new Tool { diff --git a/src/TALXIS.CLI.MCP/Skills/Internal/testing-workflow.md b/src/TALXIS.CLI.MCP/Skills/Internal/testing-workflow.md new file mode 100644 index 00000000..97b99a24 --- /dev/null +++ b/src/TALXIS.CLI.MCP/Skills/Internal/testing-workflow.md @@ -0,0 +1,69 @@ +# UI Testing Workflow + + + + +## User wants to "write a UI test" / "test a form" / "test navigation" + +-> STRUCTURE: Feature file (.feature) with Scenario(s) using Given/When/Then +-> ALWAYS start with a Given step for login: `Given I am logged in to the '{app}' app as '{user}'` +-> ALWAYS use pre-built step bindings from TALXIS.TestKit.Bindings where available +-> ONLY write custom step bindings for app-specific logic not covered by the library + +## Test Structure Best Practices + +-> Feature files group related scenarios by business capability +-> Each scenario should be independent (no shared state between scenarios) +-> Use Background for shared Given steps across all scenarios in a feature +-> Keep scenarios focused on ONE behavior/assertion +-> Use Scenario Outline for data-driven tests + +## Given/When/Then Conventions + +-> Given: Setup preconditions (login, create test data, navigate to starting point) +-> When: Perform the action being tested (click, enter data, navigate) +-> Then: Assert the expected outcome (field values, visibility, error messages) +-> Avoid multiple When steps — split into separate scenarios instead + +## Test Data Setup + +-> Use `Given I have created '{alias}'` with JSON data files in a /data folder +-> Data files use Web API deep-insert syntax with @logicalName, @alias, @extends +-> Use faker.js templates for dynamic data ({{name.firstName}}, {{finance.amount}}) +-> Set `deleteTestData: true` in appsettings.json for cleanup after scenarios + +## Common Patterns + +### Testing form field entry: +```gherkin +When I enter '{value}' into the '{field label}' field on the form +``` + +### Testing navigation: +```gherkin +When I open the sub area '{subarea}' under the '{area}' area +When I open the '{subarea}' sub area of the '{group}' group +``` + +### Testing command bar: +```gherkin +When I select the '{command}' command +Then I should be able to see the '{command}' command +``` + +### Testing grids/views: +```gherkin +When I open the record at position '{n}' in the grid +Then I can see '{n}' records in the grid +``` + +### Testing lookups: +```gherkin +When I select '{value}' from the '{field}' lookup field +``` + +## Error Recovery + +-> If a step binding fails with a timeout: check if Driver.WaitForTransaction() is needed +-> If login fails: verify user credentials in appsettings.json and OTP token configuration +-> If element not found: the app may need WaitForPageToLoad before interaction diff --git a/src/TALXIS.CLI.MCP/TALXIS.CLI.MCP.csproj b/src/TALXIS.CLI.MCP/TALXIS.CLI.MCP.csproj index 449e276e..db9e29c7 100644 --- a/src/TALXIS.CLI.MCP/TALXIS.CLI.MCP.csproj +++ b/src/TALXIS.CLI.MCP/TALXIS.CLI.MCP.csproj @@ -47,6 +47,7 @@ + diff --git a/src/TALXIS.CLI.MCP/TestingBindingsCatalog.cs b/src/TALXIS.CLI.MCP/TestingBindingsCatalog.cs new file mode 100644 index 00000000..9576df85 --- /dev/null +++ b/src/TALXIS.CLI.MCP/TestingBindingsCatalog.cs @@ -0,0 +1,244 @@ +using System.Reflection; +using System.Text; + +namespace TALXIS.CLI.MCP; + +/// +/// Reflects over the TALXIS.TestKit.Bindings assembly to discover all Reqnroll step bindings +/// and builds a catalog of Gherkin patterns for use by the guide_testing endpoint. +/// +public class TestingBindingsCatalog +{ + private readonly List _entries = new(); + private string? _cachedCatalogPrompt; + + /// + /// Loads step bindings from the TALXIS.TestKit.Bindings assembly using reflection. + /// Scans for classes marked with [Binding] and extracts [Given], [When], [Then] patterns. + /// + public void Load() + { + var assembly = FindTestKitBindingsAssembly(); + if (assembly is null) return; + + foreach (var type in assembly.GetExportedTypes()) + { + if (!HasBindingAttribute(type)) + continue; + + var category = DeriveCategoryFromTypeName(type.Name); + + foreach (var method in type.GetMethods(BindingFlags.Public | BindingFlags.Static | BindingFlags.Instance)) + { + foreach (var attr in method.GetCustomAttributes(inherit: false)) + { + var (stepType, pattern) = ExtractStepPattern(attr); + if (stepType is null || pattern is null) continue; + + var description = GetMethodSummary(method); + + _entries.Add(new StepBindingEntry + { + StepType = stepType, + Pattern = pattern, + Category = category, + Description = description, + SourceClass = type.Name + }); + } + } + } + + _cachedCatalogPrompt = null; + } + + /// + /// Builds a formatted catalog string for inclusion in sampling prompts. + /// Groups step bindings by category and step type. + /// + public string GetCatalogPrompt() + { + if (_cachedCatalogPrompt is not null) + return _cachedCatalogPrompt; + + var sb = new StringBuilder(); + sb.AppendLine("# Available Reqnroll Step Bindings (TALXIS.TestKit.Bindings)"); + sb.AppendLine(); + sb.AppendLine("These are pre-built Gherkin step bindings for Power Apps UI test automation."); + sb.AppendLine("Parameters in patterns are denoted by regex groups like '(.*)' — replace with actual values in quotes."); + sb.AppendLine(); + + var grouped = _entries + .GroupBy(e => e.Category) + .OrderBy(g => g.Key); + + foreach (var group in grouped) + { + sb.AppendLine($"## {group.Key}"); + + foreach (var entry in group.OrderBy(e => e.StepType).ThenBy(e => e.Pattern)) + { + var gherkinPattern = FormatAsGherkin(entry); + sb.AppendLine($"- {gherkinPattern}"); + if (!string.IsNullOrWhiteSpace(entry.Description)) + sb.AppendLine($" *{entry.Description}*"); + } + + sb.AppendLine(); + } + + _cachedCatalogPrompt = sb.ToString(); + return _cachedCatalogPrompt; + } + + /// + /// Number of discovered step bindings. + /// + public int Count => _entries.Count; + + /// + /// Gets all discovered entries. + /// + public IReadOnlyList Entries => _entries; + + /// + /// Formats a step binding entry as a Gherkin step line. + /// Converts regex patterns to user-friendly placeholder syntax. + /// + private static string FormatAsGherkin(StepBindingEntry entry) + { + // Convert regex groups like '(.*)' to '{param}' for readability + var readablePattern = entry.Pattern + .Replace("'(.*)'", "'{value}'") + .Replace("([^']+)", "{value}") + .Replace("(.*)", "{value}") + .Replace(@"(\d+)", "{number}") + .Replace(@"(should|should not)", "{should|should not}"); + + return $"{entry.StepType} {readablePattern}"; + } + + /// + /// Extracts step type and pattern from a Reqnroll attribute instance. + /// Supports Given, When, Then (and their aliases). + /// + private static (string? stepType, string? pattern) ExtractStepPattern(object attribute) + { + var attrType = attribute.GetType(); + var attrName = attrType.Name; + + string? stepType = attrName switch + { + "GivenAttribute" => "Given", + "WhenAttribute" => "When", + "ThenAttribute" => "Then", + "StepDefinitionAttribute" => "Step", + _ => null + }; + + if (stepType is null) return (null, null); + + // The pattern is stored in the Regex property (Reqnroll attribute base class) + var regexProp = attrType.GetProperty("Regex"); + var pattern = regexProp?.GetValue(attribute) as string; + + return (stepType, pattern); + } + + /// + /// Derives a category name from the step binding class name. + /// E.g., "NavigationSteps" -> "Navigation", "EntitySubGridSteps" -> "Entity Sub Grid" + /// + private static string DeriveCategoryFromTypeName(string typeName) + { + // Remove "Steps" suffix + var name = typeName.EndsWith("Steps", StringComparison.Ordinal) + ? typeName[..^5] + : typeName; + + // Insert spaces before uppercase letters for readability + var sb = new StringBuilder(); + for (int i = 0; i < name.Length; i++) + { + if (i > 0 && char.IsUpper(name[i]) && !char.IsUpper(name[i - 1])) + sb.Append(' '); + sb.Append(name[i]); + } + + return sb.ToString(); + } + + /// + /// Attempts to extract XML documentation summary from the method. + /// Falls back to null if not available (XML docs are rarely embedded in NuGet packages). + /// + private static string? GetMethodSummary(MethodInfo method) + { + // XML documentation is typically not available via reflection at runtime. + // We rely on the Gherkin pattern being self-documenting. + return null; + } + + /// + /// Checks if a type has the Reqnroll [Binding] attribute. + /// Uses name-based check to avoid version coupling. + /// + private static bool HasBindingAttribute(Type type) + { + return type.GetCustomAttributes(inherit: false) + .Any(a => a.GetType().Name == "BindingAttribute"); + } + + /// + /// Finds the TALXIS.TestKit.Bindings assembly. + /// + private static Assembly? FindTestKitBindingsAssembly() + { + // Try already loaded assemblies first + var loaded = AppDomain.CurrentDomain.GetAssemblies() + .FirstOrDefault(a => a.GetName().Name == "TALXIS.TestKit.Bindings"); + + if (loaded is not null) return loaded; + + // Try explicit load + try + { + return Assembly.Load("TALXIS.TestKit.Bindings"); + } + catch + { + return null; + } + } +} + +/// +/// Represents a single discovered step binding from TALXIS.TestKit.Bindings. +/// +public class StepBindingEntry +{ + /// + /// The step type: Given, When, or Then. + /// + public required string StepType { get; init; } + + /// + /// The regex pattern from the attribute (e.g., "I am logged in to the '(.*)' app as '(.*)'"). + /// + public required string Pattern { get; init; } + + /// + /// Category derived from the source class (e.g., "Navigation", "Entity"). + /// + public required string Category { get; init; } + + /// + /// Optional description from XML documentation. + /// + public string? Description { get; init; } + + /// + /// The source class name (e.g., "NavigationSteps"). + /// + public required string SourceClass { get; init; } +}