diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index c6f76712..7cebeab1 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -15,6 +15,7 @@ A component definition tells azldev where to find the spec file, how to customiz | Render config | `render` | [RenderConfig](#render-configuration) | No | Options controlling spec rendering behavior | | Source files | `source-files` | array of [SourceFileReference](#source-file-references) | No | Additional source files to download for this component | | Package overrides | `packages` | map of string → [PackageConfig](package-groups.md#package-config) | No | Exact per-package configuration overrides; highest priority in the resolution order | +| Tests | `tests` | [ComponentTests](#component-tests) | No | Test references that apply to this component (parse-only; see [Tests and Test Groups](tests.md)) | ### Bare Components @@ -299,6 +300,28 @@ rpm-channel = "rpm-devel" rpm-channel = "none" ``` +## Component Tests + +The `[components..tests]` subtable lists test or test-group +references that apply to the component. Each entry is a [TestRef](tests.md#test-reference) +with exactly one of `name` or `group`. + +| Field | TOML Key | Type | Required | Description | +|-------|----------|------|----------|-------------| +| Tests | `tests` | array of [TestRef](tests.md#test-reference) | No | References to `[tests.]` entries or `[test-groups.]` entries | + +```toml +[components.kernel.tests] +tests = [ + { group = "kernel-bvt" }, + { name = "kdump-smoke" }, +] +``` + +> **Status:** parse-only. The references are validated for structural +> shape but not yet resolved or executed; see [Tests and Test Groups](tests.md) +> for the full schema and roadmap. + ## Source File References The `[[components..source-files]]` array defines additional source files that azldev should download before building. These are files not available in the dist-git repository or lookaside cache — typically binaries, pre-built artifacts, or files from custom hosting. @@ -460,5 +483,6 @@ lines = ["cp -vf %{shimdirx64}/$(basename %{shimefix64}) %{shimefix64} ||:"] - [Distros](distros.md) — distro definitions and `default-component-config` inheritance - [Component Groups](component-groups.md) — grouping components with shared defaults - [Package Groups](package-groups.md) — project-level package groups and full resolution order +- [Tests and Test Groups](tests.md) — definitions referenced by `[components..tests]` - [Configuration System](../../explanation/config-system.md) — inheritance and merge behavior - [JSON Schema](../../../../schemas/azldev.schema.json) — machine-readable schema diff --git a/docs/user/reference/config/config-file.md b/docs/user/reference/config/config-file.md index e1ae30bb..19abd972 100644 --- a/docs/user/reference/config/config-file.md +++ b/docs/user/reference/config/config-file.md @@ -15,6 +15,8 @@ All config files share the same schema — there is no distinction between a "ro | `component-groups` | map of objects | Named groups of components with shared defaults | [Component Groups](component-groups.md) | | `images` | map of objects | Image definitions (VMs, containers) | [Images](images.md) | | `test-suites` | map of objects | Named test suite definitions referenced by images | [Test Suites](test-suites.md) | +| `tests` | map of objects | Named test definitions (new-shape, parse-only) | [Tests and Test Groups](tests.md) | +| `test-groups` | map of objects | Named bundles of test references (new-shape, parse-only) | [Tests and Test Groups](tests.md) | | `tools` | object | Configuration for external tools used by azldev | [Tools](tools.md) | | `default-package-config` | object | Project-wide default applied to all binary packages | [Package Groups — Resolution Order](package-groups.md#resolution-order) | | `package-groups` | map of objects | Named groups of binary packages with shared config | [Package Groups](package-groups.md) | diff --git a/docs/user/reference/config/images.md b/docs/user/reference/config/images.md index ebbd5e53..34dd9cb9 100644 --- a/docs/user/reference/config/images.md +++ b/docs/user/reference/config/images.md @@ -35,11 +35,12 @@ The `capabilities` subtable describes what the image supports. All fields are op ## Image Tests -The `tests` subtable links an image to one or more test suites defined in the top-level [`[test-suites]`](test-suites.md) section. +The `tests` subtable links an image to one or more test suites defined in the top-level [`[test-suites]`](test-suites.md) section, and/or to entries from the new-shape [`[tests]` / `[test-groups]`](tests.md) sections. | Field | TOML Key | Type | Required | Description | |-------|----------|------|----------|-------------| | Test Suites | `test-suites` | array of inline tables | No | List of test suite references. Each entry must have a `name` field matching a key in `[test-suites]`. | +| Tests | `tests` | array of [TestRef](tests.md#test-reference) | No | References to `[tests.]` entries or `[test-groups.]` entries (parse-only; see [Tests and Test Groups](tests.md)). | ## Image Publish @@ -118,4 +119,5 @@ channels = ["registry-prod", "registry-staging"] - [Config File Structure](config-file.md) — top-level config file layout - [Test Suites](test-suites.md) — test suite definitions +- [Tests and Test Groups](tests.md) — new-shape test/group definitions referenced by `[images..tests]` - [Tools](tools.md) — Image Customizer tool configuration diff --git a/docs/user/reference/config/tests.md b/docs/user/reference/config/tests.md new file mode 100644 index 00000000..00c83dca --- /dev/null +++ b/docs/user/reference/config/tests.md @@ -0,0 +1,106 @@ +# Tests and Test Groups + +> **Status:** parse-only. The schema below is accepted and validated for +> structural correctness, but azldev does not yet resolve references, +> flatten groups, or run anything from these sections. The resolver and +> `azldev test` CLI land in a follow-up change. Use [Test Suites](test-suites.md) +> for executable test configuration today. + +The `[tests]` and `[test-groups]` sections declare framework-agnostic test +metadata that components and images can target by name. Each test entry +binds a single test (a pytest run, a LISA case, a TMT plan, an openQA +job, …) to a named identifier; each group entry bundles tests (and +nested groups) under one name so callers can reference a curated set +without enumerating every member. + +## Test Definition + +Each entry under `[tests.]` describes one configuration of one +runner. Framework-specific options live in a typed subtable +(`pytest`, `lisa`, `tmt`, `openqa`) whose contents are passed through +to the runner; their internal schemas are intentionally not validated +by azldev so frameworks can evolve independently. + +| Field | TOML Key | Type | Required | Description | +|-------|----------|------|----------|-------------| +| Type | `type` | string | Yes | Test framework: `pytest`, `lisa`, `tmt`, `openqa`, or `pending` | +| Description | `description` | string | No | Human-readable description | +| Kind | `kind` | string array | No | Free-form hints (e.g. `functional`, `performance`, `bvt`) | +| Long running | `long-running` | boolean | No | Hints that this test may run for hours | +| Required capabilities | `required-capabilities` | string array | No | Capability tokens the image must declare for this test to be applicable | +| Lisa | `lisa` | table | No | LISA-specific configuration (opaque to azldev) | +| Tmt | `tmt` | table | No | TMT-specific configuration (opaque to azldev) | +| Pytest | `pytest` | table | No | pytest-specific configuration (opaque to azldev) | +| Openqa | `openqa` | table | No | openQA-specific configuration (opaque to azldev) | + +## Test Group + +Each entry under `[test-groups.]` names an ordered list of test or +nested-group references that callers can target as a single unit. + +| Field | TOML Key | Type | Required | Description | +|-------|----------|------|----------|-------------| +| Description | `description` | string | No | Human-readable description | +| Tests | `tests` | array of [TestRef](#test-reference) | No | Ordered members of the group | + +## Test Reference + +`TestRef` is an inline table with exactly one of `name` or `group`: + +| Field | TOML Key | Type | Description | +|-------|----------|------|-------------| +| Name | `name` | string | References a `[tests.]` entry | +| Group | `group` | string | References a `[test-groups.]` entry | + +Semantic validation (name-vs-group exclusivity, reference resolution, +cycle detection, capability matching) is deferred to the resolver in +the follow-up PR. + +## Referencing from Components and Images + +Components and images both expose a `tests` subtable that holds a list +of `TestRef`s: + +```toml +[components.kernel.tests] +tests = [{ group = "kernel-bvt" }, { name = "kdump-smoke" }] + +[images.vm-base.tests] +tests = [{ group = "bvt" }] +``` + +> The image `tests` subtable still also accepts the legacy `test-suites` +> field documented in [Images — Image Tests](images.md#image-tests). + +## Example + +```toml +[tests.bvt-ssh] +type = "pytest" +description = "Basic SSH boot verification" +kind = ["functional", "bvt"] +required-capabilities = ["ssh"] +pytest = { working-dir = "tests/bvt", test-paths = ["test_ssh.py"] } + +[tests.kdump-smoke] +type = "lisa" +description = "Smoke test for kdump" +lisa = { case = "kdump.smoke" } + +[test-groups.bvt] +description = "Build verification tests" +tests = [ + { name = "bvt-ssh" }, + { group = "bvt-extras" }, +] + +[test-groups.bvt-extras] +tests = [{ name = "kdump-smoke" }] +``` + +## Related Resources + +- [Test Suites](test-suites.md) — current executable test configuration +- [Components](components.md#component-tests) — per-component `tests` field +- [Images](images.md#image-tests) — per-image `tests` field +- [Config File Structure](config-file.md) — top-level config layout diff --git a/internal/projectconfig/component.go b/internal/projectconfig/component.go index 9d45a2b4..4abc7f19 100644 --- a/internal/projectconfig/component.go +++ b/internal/projectconfig/component.go @@ -290,6 +290,14 @@ type ComponentConfig struct { // all packages produced by this component. Overridden by package-group and per-package settings // for binary and debuginfo channels. Publish ComponentPublishConfig `toml:"publish,omitempty" json:"publish,omitempty" table:"-" jsonschema:"title=Publish settings,description=Component-level publish channel settings" fingerprint:"-"` + + // Tests holds the new-shape per-component tests block: + // + // tests.tests = [{ name = "..." }, { group = "..." }] + // + // References must resolve to entries in the project-level [tests] or + // [test-groups] maps; resolution is the responsibility of the test layer. + Tests ComponentTestsConfig `toml:"tests,omitempty" json:"tests,omitempty" table:"-" jsonschema:"title=Tests,description=Per-component test or test-group references" fingerprint:"-"` } // AllowedSourceFilesHashTypes defines the set of hash types that are supported @@ -383,6 +391,7 @@ func (c *ComponentConfig) WithAbsolutePaths(referenceDir string) *ComponentConfi SourceFiles: deep.MustCopy(c.SourceFiles), Packages: deep.MustCopy(c.Packages), Publish: deep.MustCopy(c.Publish), + Tests: deep.MustCopy(c.Tests), } // Fix up paths. diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index e16b79b7..e7eddc46 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -66,6 +66,12 @@ type ConfigFile struct { // Definitions of test suites. TestSuites map[string]TestSuiteConfig `toml:"test-suites,omitempty" validate:"dive" jsonschema:"title=Test Suites,description=Definitions of test suites for this project"` + // Definitions of individual tests (new schema, [tests.X]). + Tests map[string]TestDefinition `toml:"tests,omitempty" validate:"dive" jsonschema:"title=Tests,description=Definitions of individual tests"` + + // Definitions of test groups (new schema, [test-groups.X]). + TestGroups map[string]TestGroup `toml:"test-groups,omitempty" validate:"dive" jsonschema:"title=Test Groups,description=Definitions of named bundles of tests"` + // Internal fields used to track the origin of the config file; `dir` is the directory // that the config file's relative paths are based from. sourcePath string `toml:"-"` diff --git a/internal/projectconfig/fingerprint_test.go b/internal/projectconfig/fingerprint_test.go index 3bd14eff..25a58a45 100644 --- a/internal/projectconfig/fingerprint_test.go +++ b/internal/projectconfig/fingerprint_test.go @@ -65,6 +65,9 @@ func TestAllFingerprintedFieldsHaveDecision(t *testing.T) { // ComponentConfig.Publish — post-build routing (where to publish), not a build input. "ComponentConfig.Publish": true, + // ComponentConfig.Tests — test selection metadata (new schema), not a build input. + "ComponentConfig.Tests": true, + // ComponentOverlay.Description — human-readable documentation for the overlay. "ComponentOverlay.Description": true, // ComponentOverlay.Source — absolute path that varies by checkout location. diff --git a/internal/projectconfig/image.go b/internal/projectconfig/image.go index f2000851..1e9919f4 100644 --- a/internal/projectconfig/image.go +++ b/internal/projectconfig/image.go @@ -115,6 +115,11 @@ type ImageTestsConfig struct { // reference identifies a test suite defined in the top-level [test-suites] section // and may carry per-test metadata in the future (e.g., required vs optional). TestSuites []TestSuiteRef `toml:"test-suites,omitempty" json:"testSuites,omitempty" jsonschema:"title=Test Suites,description=List of test suite references that apply to this image"` + + // Tests is the new-shape list of test or test-group references that apply to this + // image. References must resolve to entries in the project-level [tests] or + // [test-groups] maps; resolution is the responsibility of the test layer. + Tests []TestRef `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=List of test or test-group references that apply to this image"` } // TestSuiteRef is a reference to a named test suite. Using a structured type (rather than diff --git a/internal/projectconfig/tests.go b/internal/projectconfig/tests.go new file mode 100644 index 00000000..d2233c63 --- /dev/null +++ b/internal/projectconfig/tests.go @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig + +// TestDefinition is the new-shape [tests.X] declaration: one configuration of one +// runner/harness with framework-specific options. Framework subtables are kept as +// loosely-typed maps so the resolver can evolve their schemas without requiring +// matching struct changes here. +type TestDefinition struct { + // Type identifies the framework/runner. Closed enum at the schema layer + // (pytest|lisa|tmt|openqa|pending), but accepted permissively here. + Type string `toml:"type,omitempty" json:"type,omitempty" jsonschema:"title=Type,description=Test framework type (pytest|lisa|tmt|openqa|pending)"` + + // Human-readable description. + Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Description of this test"` + + // Kind hints at what the test exercises (e.g. functional, performance). + Kind []string `toml:"kind,omitempty" json:"kind,omitempty" jsonschema:"title=Kind,description=Test kind hints (e.g. functional or performance)"` + + // LongRunning hints to schedulers/policy that this test may take a long time. + LongRunning bool `toml:"long-running,omitempty" json:"longRunning,omitempty" jsonschema:"title=Long running,description=Hints that this test may run for hours"` + + // RequiredCapabilities lists capability tokens an image must declare to be a + // valid target for this test. Tokens are matched against [ImageCapabilities]. + RequiredCapabilities []string `toml:"required-capabilities,omitempty" json:"requiredCapabilities,omitempty" jsonschema:"title=Required capabilities,description=Capability tokens the image must declare"` + + // Framework-specific subtables. Kept untyped so framework schema can evolve + // independently of the dev-tools type definitions. + Lisa map[string]any `toml:"lisa,omitempty" json:"lisa,omitempty" jsonschema:"title=LISA config,description=LISA-specific configuration"` + Tmt map[string]any `toml:"tmt,omitempty" json:"tmt,omitempty" jsonschema:"title=TMT config,description=TMT-specific configuration"` + Pytest map[string]any `toml:"pytest,omitempty" json:"pytest,omitempty" jsonschema:"title=Pytest config,description=pytest-specific configuration"` + Openqa map[string]any `toml:"openqa,omitempty" json:"openqa,omitempty" jsonschema:"title=openQA config,description=openQA-specific configuration"` +} + +// TestGroup is a [test-groups.X] declaration: a named bundle of test references that +// images or components can target via a single name. +type TestGroup struct { + // Human-readable description. + Description string `toml:"description,omitempty" json:"description,omitempty" jsonschema:"title=Description,description=Description of this test group"` + + // Tests is the ordered list of test or nested-group references that make up + // the group's membership. + Tests []TestRef `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=Member references (each is either {name=...} or {group=...})"` +} + +// TestRef is a reference to either a test (by name) or another group (by name). +// Exactly one of Name or Group should be set; semantic validation is the resolver's +// responsibility. +type TestRef struct { + // Name references a [tests.X] entry. + Name string `toml:"name,omitempty" json:"name,omitempty" jsonschema:"title=Name,description=Name of a test (mutually exclusive with group)"` + + // Group references a [test-groups.X] entry. + Group string `toml:"group,omitempty" json:"group,omitempty" jsonschema:"title=Group,description=Name of a test group (mutually exclusive with name)"` +} + +// ComponentTestsConfig holds the new-shape per-component tests block: +// +// tests.tests = [{ name = "..." }, { group = "..." }] +type ComponentTestsConfig struct { + // Tests is the list of test or group references that apply to the component. + Tests []TestRef `toml:"tests,omitempty" json:"tests,omitempty" jsonschema:"title=Tests,description=Per-component test or group references"` +} diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 7aa38d7e..71c20d87 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -149,6 +149,11 @@ "$ref": "#/$defs/ComponentPublishConfig", "title": "Publish settings", "description": "Component-level publish channel settings" + }, + "tests": { + "$ref": "#/$defs/ComponentTestsConfig", + "title": "Tests", + "description": "Per-component test or test-group references" } }, "additionalProperties": false, @@ -318,6 +323,20 @@ "additionalProperties": false, "type": "object" }, + "ComponentTestsConfig": { + "properties": { + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "Per-component test or group references" + } + }, + "additionalProperties": false, + "type": "object" + }, "ConfigFile": { "properties": { "$schema": { @@ -403,6 +422,22 @@ "type": "object", "title": "Test Suites", "description": "Definitions of test suites for this project" + }, + "tests": { + "additionalProperties": { + "$ref": "#/$defs/TestDefinition" + }, + "type": "object", + "title": "Tests", + "description": "Definitions of individual tests" + }, + "test-groups": { + "additionalProperties": { + "$ref": "#/$defs/TestGroup" + }, + "type": "object", + "title": "Test Groups", + "description": "Definitions of named bundles of tests" } }, "additionalProperties": false, @@ -681,6 +716,14 @@ "type": "array", "title": "Test Suites", "description": "List of test suite references that apply to this image" + }, + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "List of test or test-group references that apply to this image" } }, "additionalProperties": false, @@ -1212,6 +1255,98 @@ "subpath" ] }, + "TestDefinition": { + "properties": { + "type": { + "type": "string", + "title": "Type", + "description": "Test framework type (pytest|lisa|tmt|openqa|pending)" + }, + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test" + }, + "kind": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Kind", + "description": "Test kind hints (e.g. functional or performance)" + }, + "long-running": { + "type": "boolean", + "title": "Long running", + "description": "Hints that this test may run for hours" + }, + "required-capabilities": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Required capabilities", + "description": "Capability tokens the image must declare" + }, + "lisa": { + "type": "object", + "title": "LISA config", + "description": "LISA-specific configuration" + }, + "tmt": { + "type": "object", + "title": "TMT config", + "description": "TMT-specific configuration" + }, + "pytest": { + "type": "object", + "title": "Pytest config", + "description": "pytest-specific configuration" + }, + "openqa": { + "type": "object", + "title": "openQA config", + "description": "openQA-specific configuration" + } + }, + "additionalProperties": false, + "type": "object" + }, + "TestGroup": { + "properties": { + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test group" + }, + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "Member references (each is either {name=...} or {group=...})" + } + }, + "additionalProperties": false, + "type": "object" + }, + "TestRef": { + "properties": { + "name": { + "type": "string", + "title": "Name", + "description": "Name of a test (mutually exclusive with group)" + }, + "group": { + "type": "string", + "title": "Group", + "description": "Name of a test group (mutually exclusive with name)" + } + }, + "additionalProperties": false, + "type": "object" + }, "TestSuiteConfig": { "properties": { "description": { diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 7aa38d7e..71c20d87 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -149,6 +149,11 @@ "$ref": "#/$defs/ComponentPublishConfig", "title": "Publish settings", "description": "Component-level publish channel settings" + }, + "tests": { + "$ref": "#/$defs/ComponentTestsConfig", + "title": "Tests", + "description": "Per-component test or test-group references" } }, "additionalProperties": false, @@ -318,6 +323,20 @@ "additionalProperties": false, "type": "object" }, + "ComponentTestsConfig": { + "properties": { + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "Per-component test or group references" + } + }, + "additionalProperties": false, + "type": "object" + }, "ConfigFile": { "properties": { "$schema": { @@ -403,6 +422,22 @@ "type": "object", "title": "Test Suites", "description": "Definitions of test suites for this project" + }, + "tests": { + "additionalProperties": { + "$ref": "#/$defs/TestDefinition" + }, + "type": "object", + "title": "Tests", + "description": "Definitions of individual tests" + }, + "test-groups": { + "additionalProperties": { + "$ref": "#/$defs/TestGroup" + }, + "type": "object", + "title": "Test Groups", + "description": "Definitions of named bundles of tests" } }, "additionalProperties": false, @@ -681,6 +716,14 @@ "type": "array", "title": "Test Suites", "description": "List of test suite references that apply to this image" + }, + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "List of test or test-group references that apply to this image" } }, "additionalProperties": false, @@ -1212,6 +1255,98 @@ "subpath" ] }, + "TestDefinition": { + "properties": { + "type": { + "type": "string", + "title": "Type", + "description": "Test framework type (pytest|lisa|tmt|openqa|pending)" + }, + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test" + }, + "kind": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Kind", + "description": "Test kind hints (e.g. functional or performance)" + }, + "long-running": { + "type": "boolean", + "title": "Long running", + "description": "Hints that this test may run for hours" + }, + "required-capabilities": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Required capabilities", + "description": "Capability tokens the image must declare" + }, + "lisa": { + "type": "object", + "title": "LISA config", + "description": "LISA-specific configuration" + }, + "tmt": { + "type": "object", + "title": "TMT config", + "description": "TMT-specific configuration" + }, + "pytest": { + "type": "object", + "title": "Pytest config", + "description": "pytest-specific configuration" + }, + "openqa": { + "type": "object", + "title": "openQA config", + "description": "openQA-specific configuration" + } + }, + "additionalProperties": false, + "type": "object" + }, + "TestGroup": { + "properties": { + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test group" + }, + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "Member references (each is either {name=...} or {group=...})" + } + }, + "additionalProperties": false, + "type": "object" + }, + "TestRef": { + "properties": { + "name": { + "type": "string", + "title": "Name", + "description": "Name of a test (mutually exclusive with group)" + }, + "group": { + "type": "string", + "title": "Group", + "description": "Name of a test group (mutually exclusive with name)" + } + }, + "additionalProperties": false, + "type": "object" + }, "TestSuiteConfig": { "properties": { "description": { diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 7aa38d7e..71c20d87 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -149,6 +149,11 @@ "$ref": "#/$defs/ComponentPublishConfig", "title": "Publish settings", "description": "Component-level publish channel settings" + }, + "tests": { + "$ref": "#/$defs/ComponentTestsConfig", + "title": "Tests", + "description": "Per-component test or test-group references" } }, "additionalProperties": false, @@ -318,6 +323,20 @@ "additionalProperties": false, "type": "object" }, + "ComponentTestsConfig": { + "properties": { + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "Per-component test or group references" + } + }, + "additionalProperties": false, + "type": "object" + }, "ConfigFile": { "properties": { "$schema": { @@ -403,6 +422,22 @@ "type": "object", "title": "Test Suites", "description": "Definitions of test suites for this project" + }, + "tests": { + "additionalProperties": { + "$ref": "#/$defs/TestDefinition" + }, + "type": "object", + "title": "Tests", + "description": "Definitions of individual tests" + }, + "test-groups": { + "additionalProperties": { + "$ref": "#/$defs/TestGroup" + }, + "type": "object", + "title": "Test Groups", + "description": "Definitions of named bundles of tests" } }, "additionalProperties": false, @@ -681,6 +716,14 @@ "type": "array", "title": "Test Suites", "description": "List of test suite references that apply to this image" + }, + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "List of test or test-group references that apply to this image" } }, "additionalProperties": false, @@ -1212,6 +1255,98 @@ "subpath" ] }, + "TestDefinition": { + "properties": { + "type": { + "type": "string", + "title": "Type", + "description": "Test framework type (pytest|lisa|tmt|openqa|pending)" + }, + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test" + }, + "kind": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Kind", + "description": "Test kind hints (e.g. functional or performance)" + }, + "long-running": { + "type": "boolean", + "title": "Long running", + "description": "Hints that this test may run for hours" + }, + "required-capabilities": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Required capabilities", + "description": "Capability tokens the image must declare" + }, + "lisa": { + "type": "object", + "title": "LISA config", + "description": "LISA-specific configuration" + }, + "tmt": { + "type": "object", + "title": "TMT config", + "description": "TMT-specific configuration" + }, + "pytest": { + "type": "object", + "title": "Pytest config", + "description": "pytest-specific configuration" + }, + "openqa": { + "type": "object", + "title": "openQA config", + "description": "openQA-specific configuration" + } + }, + "additionalProperties": false, + "type": "object" + }, + "TestGroup": { + "properties": { + "description": { + "type": "string", + "title": "Description", + "description": "Description of this test group" + }, + "tests": { + "items": { + "$ref": "#/$defs/TestRef" + }, + "type": "array", + "title": "Tests", + "description": "Member references (each is either {name=...} or {group=...})" + } + }, + "additionalProperties": false, + "type": "object" + }, + "TestRef": { + "properties": { + "name": { + "type": "string", + "title": "Name", + "description": "Name of a test (mutually exclusive with group)" + }, + "group": { + "type": "string", + "title": "Group", + "description": "Name of a test group (mutually exclusive with name)" + } + }, + "additionalProperties": false, + "type": "object" + }, "TestSuiteConfig": { "properties": { "description": {