diff --git a/.chronus/changes/fix-https-specs-manifest-management-2026-1-19-23-57-14-2.md b/.chronus/changes/fix-https-specs-manifest-management-2026-1-19-23-57-14-2.md new file mode 100644 index 00000000000..9fb37b8000d --- /dev/null +++ b/.chronus/changes/fix-https-specs-manifest-management-2026-1-19-23-57-14-2.md @@ -0,0 +1,8 @@ +--- +# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking +changeKind: internal +packages: + - "@typespec/http-specs" +--- + +Better way to deal with manifests diff --git a/.chronus/changes/fix-https-specs-manifest-management-2026-1-19-23-57-14.md b/.chronus/changes/fix-https-specs-manifest-management-2026-1-19-23-57-14.md new file mode 100644 index 00000000000..f0f55598931 --- /dev/null +++ b/.chronus/changes/fix-https-specs-manifest-management-2026-1-19-23-57-14.md @@ -0,0 +1,9 @@ +--- +# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking +changeKind: fix +packages: + - "@typespec/spec-coverage-sdk" + - "@typespec/spector" +--- + +Update to how coverage manifest are managed. The manifest upload each individual one as a single file diff --git a/eng/tsp-core/pipelines/jobs/build-for-publish.yml b/eng/tsp-core/pipelines/jobs/build-for-publish.yml index 340b4ccf222..3e64d1f221b 100644 --- a/eng/tsp-core/pipelines/jobs/build-for-publish.yml +++ b/eng/tsp-core/pipelines/jobs/build-for-publish.yml @@ -48,7 +48,6 @@ jobs: - task: AzureCLI@2 displayName: Upload scenario manifest - condition: eq(variables['PUBLISH_PKG_TYPESPEC_HTTP_SPECS'], 'true') inputs: workingDirectory: packages/http-specs azureSubscription: "TypeSpec Storage" diff --git a/packages/http-specs/package.json b/packages/http-specs/package.json index b0d244178d2..a38a8d75fea 100644 --- a/packages/http-specs/package.json +++ b/packages/http-specs/package.json @@ -15,7 +15,7 @@ "validate-scenarios": "tsp-spector validate-scenarios ./specs", "generate-scenarios-summary": "tsp-spector generate-scenarios-summary ./specs", "regen-docs": "pnpm generate-scenarios-summary", - "upload-manifest": "tsp-spector upload-manifest ./specs --setName @typespec/http-specs --containerName manifests-typespec --storageAccountName typespec", + "upload-manifest": "tsp-spector upload-manifest ./specs --containerName coverages --storageAccountName typespec --manifestName http-specs", "upload-coverage": "tsp-spector upload-coverage --generatorName @typespec/http-specs --generatorVersion 0.1.0-alpha.4 --containerName coverages --generatorMode standard --storageAccountName typespec", "validate-mock-apis": "tsp-spector validate-mock-apis ./specs", "check-scenario-coverage": "tsp-spector check-coverage ./specs", diff --git a/packages/spec-coverage-sdk/package.json b/packages/spec-coverage-sdk/package.json index dcf6058b43c..a1925ad8da6 100644 --- a/packages/spec-coverage-sdk/package.json +++ b/packages/spec-coverage-sdk/package.json @@ -25,7 +25,9 @@ "dependencies": { "@azure/identity": "~4.13.0", "@azure/storage-blob": "~12.30.0", - "@types/node": "~25.0.2" + "@types/node": "~25.0.2", + "@types/semver": "^7.5.8", + "semver": "^7.7.1" }, "devDependencies": { "rimraf": "~6.1.2", diff --git a/packages/spec-coverage-sdk/src/client.ts b/packages/spec-coverage-sdk/src/client.ts index 3611e67598b..2fe8957b740 100644 --- a/packages/spec-coverage-sdk/src/client.ts +++ b/packages/spec-coverage-sdk/src/client.ts @@ -6,6 +6,7 @@ import { ContainerClient, StorageSharedKeyCredential, } from "@azure/storage-blob"; +import { eq as semverEq, gt as semverGt, valid as semverValid } from "semver"; import { CoverageReport, GeneratorMetadata, @@ -42,26 +43,82 @@ export class SpecCoverageClient { } export class SpecManifestOperations { - #blob: BlockBlobClient; #container: ContainerClient; constructor(container: ContainerClient) { this.#container = container; - this.#blob = this.#container.getBlockBlobClient("manifest.json"); } - public async upload(manifest: ScenarioManifest | ScenarioManifest[]): Promise { + public async upload(name: string, manifest: ScenarioManifest): Promise { + await this.#upload(name, "latest", manifest); + await this.#upload(name, manifest.version, manifest); + } + + async #upload(name: string, version: string, manifest: ScenarioManifest): Promise { + const blob = this.#container.getBlockBlobClient(this.#blobName(name, version)); const content = JSON.stringify(manifest, null, 2); - await this.#blob.upload(content, content.length, { + await blob.upload(content, content.length, { blobHTTPHeaders: { blobContentType: "application/json; charset=utf-8", }, }); } - public async get(): Promise { - return readJsonBlob(this.#blob); + public async uploadIfVersionNew( + name: string, + manifest: ScenarioManifest, + ): Promise<"uploaded" | "skipped"> { + const existingVersion = await this.tryGet(name, manifest.version); + if (existingVersion) { + return "skipped"; + } + const existingLatest = await this.tryGet(name); + if (existingLatest && !isVersionNewer(manifest.version, existingLatest.version)) { + return "skipped"; + } + + await this.upload(name, manifest); + return "uploaded"; + } + + public async get(name: string, version?: string): Promise { + const blob = this.#container.getBlockBlobClient(this.#blobName(name, version)); + return readJsonBlob(blob); + } + + public async tryGet(name: string, version?: string): Promise { + const blob = this.#container.getBlockBlobClient(this.#blobName(name, version)); + try { + return await readJsonBlob(blob); + } catch (e: any) { + if ("code" in e && e.code === "BlobNotFound") { + return undefined; + } + throw e; + } } + + #blobName(name: string, version?: string) { + return `manifests/${name}/${version ?? "latest"}.json`; + } +} + +function areVersionsEquivalent(left: string, right: string): boolean { + const leftValid = semverValid(left); + const rightValid = semverValid(right); + if (leftValid && rightValid) { + return semverEq(leftValid, rightValid); + } + return left === right; +} + +function isVersionNewer(candidate: string, existing: string): boolean { + const candidateValid = semverValid(candidate); + const existingValid = semverValid(existing); + if (candidateValid && existingValid) { + return semverGt(candidateValid, existingValid); + } + return !areVersionsEquivalent(candidate, existing); } export class SpecCoverageOperations { @@ -168,7 +225,18 @@ function getCoverageContainer( async function readJsonBlob(blobClient: BlockBlobClient): Promise { const blob = await blobClient.download(); - const body = await blob.blobBody; - const content = await body!.text(); - return JSON.parse(content); + if (blob.blobBody) { + const body = await blob.blobBody; + const content = await body!.text(); + return JSON.parse(content); + } else if (blob.readableStreamBody) { + const stream = blob.readableStreamBody; + let content = ""; + for await (const chunk of stream) { + content += chunk; + } + return JSON.parse(content); + } else { + throw new Error("Blob has no body"); + } } diff --git a/packages/spec-dashboard/src/apis.ts b/packages/spec-dashboard/src/apis.ts index 4b6b4473535..e5a3f8c8117 100644 --- a/packages/spec-dashboard/src/apis.ts +++ b/packages/spec-dashboard/src/apis.ts @@ -22,8 +22,8 @@ export interface TableDefinition { export interface CoverageFromAzureStorageOptions { readonly storageAccountName: string; readonly containerName: string; - // TODO: why was this not back in the same place as the other options? - readonly manifestContainerName: string; + /** Name of the manifests(As located under manifests/.json) for this dashboard */ + readonly manifests: string[]; readonly emitterNames: string[]; readonly modes?: string[]; /** Optional table definitions to split scenarios into multiple tables */ @@ -51,16 +51,6 @@ export function getCoverageClient(options: CoverageFromAzureStorageOptions) { return client; } -let manifestClient: SpecCoverageClient | undefined; -export function getManifestClient(options: CoverageFromAzureStorageOptions) { - if (manifestClient === undefined) { - manifestClient = new SpecCoverageClient(options.storageAccountName, { - containerName: options.manifestContainerName, - }); - } - return manifestClient; -} - /** * Checks if a scenario name matches any of the given prefixes */ @@ -161,10 +151,9 @@ export async function getCoverageSummaries( options: CoverageFromAzureStorageOptions, ): Promise { const coverageClient = getCoverageClient(options); - const manifestClient = getManifestClient(options); // First, split manifests to determine which emitters we need - const manifests = await manifestClient.manifest.get(); + const manifests = await Promise.all(options.manifests.map((x) => coverageClient.manifest.get(x))); const allManifests: Array<{ manifest: ScenarioManifest; tableName: string; diff --git a/packages/spector/src/actions/upload-scenario-manifest.ts b/packages/spector/src/actions/upload-scenario-manifest.ts index d2ea0cf9356..e6d2119ee61 100644 --- a/packages/spector/src/actions/upload-scenario-manifest.ts +++ b/packages/spector/src/actions/upload-scenario-manifest.ts @@ -7,35 +7,48 @@ import { computeScenarioManifest } from "../coverage/scenario-manifest.js"; import { logger } from "../logger.js"; export interface UploadScenarioManifestConfig { - scenariosPaths: string[]; + scenariosPath: string; storageAccountName: string; containerName: string; + manifestName: string; + override?: boolean; } export async function uploadScenarioManifest({ - scenariosPaths, + scenariosPath, storageAccountName, containerName, + manifestName, + override = false, }: UploadScenarioManifestConfig) { - const manifests = []; - for (let idx = 0; idx < scenariosPaths.length; idx++) { - const path = resolve(process.cwd(), scenariosPaths[idx]); - logger.info(`Computing scenario manifest for ${path}`); - const [manifest, diagnostics] = await computeScenarioManifest(path); - if (manifest === undefined || diagnostics.length > 0) { - process.exit(-1); - } - manifests.push(manifest); + const path = resolve(process.cwd(), scenariosPath); + logger.info(`Computing scenario manifest for ${path}`); + const [manifest, diagnostics] = await computeScenarioManifest(path); + if (manifest === undefined || diagnostics.length > 0) { + process.exit(-1); } - await writeFile("manifest.json", JSON.stringify(manifests, null, 2)); + await writeFile("manifest.json", JSON.stringify(manifest, null, 2)); const client = new SpecCoverageClient(storageAccountName, { credential: new AzureCliCredential(), containerName, }); await client.createIfNotExists(); - await client.manifest.upload(manifests); + if (override) { + await client.manifest.upload(manifestName, manifest); + logger.info( + `${pc.green("✓")} Scenario manifest uploaded to ${storageAccountName} storage account.`, + ); + } else { + const result = await client.manifest.uploadIfVersionNew(manifestName, manifest); - logger.info( - `${pc.green("✓")} Scenario manifest uploaded to ${storageAccountName} storage account.`, - ); + if (result === "uploaded") { + logger.info( + `${pc.green("✓")} Scenario manifest new version uploaded to ${storageAccountName} storage account.`, + ); + } else { + logger.info( + `${pc.white("-")} Existing scenario manifest in ${storageAccountName} storage account is up to date. No upload needed.`, + ); + } + } } diff --git a/packages/spector/src/cli/cli.ts b/packages/spector/src/cli/cli.ts index 00f170d39fa..f0ecef7be8d 100644 --- a/packages/spector/src/cli/cli.ts +++ b/packages/spector/src/cli/cli.ts @@ -273,20 +273,13 @@ async function main() { }, ) .command( - "upload-manifest ", + "upload-manifest ", "Upload the scenario manifest. DO NOT CALL in generator.", (cmd) => { return cmd - .positional("scenariosPaths", { + .positional("scenariosPath", { description: "Path to the scenarios and mock apis", type: "string", - array: true, - demandOption: true, - }) - .option("setName", { - type: "string", - description: "Set used to generate the manifest.", - array: true, demandOption: true, }) .option("storageAccountName", { @@ -298,13 +291,26 @@ async function main() { description: "Name of the Container", demandOption: true, }) + .option("manifestName", { + type: "string", + description: + "Name of the manifest(will be located at manifests/.json in the container).", + demandOption: true, + }) + .option("override", { + type: "boolean", + description: "Override existing manifest with the same version.", + default: false, + }) .demandOption("storageAccountName"); }, async (args) => { await uploadScenarioManifest({ - scenariosPaths: args.scenariosPaths, + scenariosPath: args.scenariosPath, storageAccountName: args.storageAccountName, containerName: args.containerName, + manifestName: args.manifestName, + override: args.override, }); }, ) diff --git a/packages/spector/tsconfig.build.json b/packages/spector/tsconfig.build.json index 6bc74ef7a9a..0085c31c291 100644 --- a/packages/spector/tsconfig.build.json +++ b/packages/spector/tsconfig.build.json @@ -4,7 +4,10 @@ "outDir": "dist", "tsBuildInfoFile": "temp/.tsbuildinfo" }, - "references": [], + "references": [ + { "path": "../spec-api/tsconfig.build.json" }, + { "path": "../spec-coverage-sdk/tsconfig.build.json" } + ], "include": ["src/**/*.ts", "generated-defs/**/*.ts"], "exclude": ["**/*.test.*"] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 42ac27e3349..777b2cef346 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1951,6 +1951,12 @@ importers: '@types/node': specifier: ~25.0.2 version: 25.0.9 + '@types/semver': + specifier: ^7.5.8 + version: 7.7.1 + semver: + specifier: ^7.7.1 + version: 7.7.3 devDependencies: rimraf: specifier: ~6.1.2 @@ -9086,11 +9092,13 @@ packages: glob@10.5.0: resolution: {integrity: sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me hasBin: true glob@11.1.0: resolution: {integrity: sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw==} engines: {node: 20 || >=22} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me hasBin: true glob@13.0.0: @@ -9099,12 +9107,12 @@ packages: glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} - deprecated: Glob versions prior to v9 are no longer supported + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me glob@8.1.0: resolution: {integrity: sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==} engines: {node: '>=12'} - deprecated: Glob versions prior to v9 are no longer supported + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me global-directory@4.0.1: resolution: {integrity: sha512-wHTUcDUoZ1H5/0iVqEudYW4/kAlN5cZ3j/bXn0Dpbizl9iaUVeWSHqiOjsgk6OW2bkLclbBjzewBz6weQ1zA2Q==} @@ -12353,11 +12361,12 @@ packages: tar@6.2.1: resolution: {integrity: sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==} engines: {node: '>=10'} - deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exhorbitant rates) by contacting i@izs.me + deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me tar@7.5.4: resolution: {integrity: sha512-AN04xbWGrSTDmVwlI4/GTlIIwMFk/XEv7uL8aa57zuvRy6s4hdBed+lVq2fAZ89XDa7Us3ANXcE3Tvqvja1kTA==} engines: {node: '>=18'} + deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me tau-prolog@0.2.81: resolution: {integrity: sha512-cHSdGumv+GfRweqE3Okd81+ZH1Ux6PoJ+WPjzoAFVar0SRoUxW93vPvWTbnTtlz++IpSEQ0yUPWlLBcTMQ8uOg==} diff --git a/tsconfig.ws.json b/tsconfig.ws.json index 57204acadb4..fb10203f549 100644 --- a/tsconfig.ws.json +++ b/tsconfig.ws.json @@ -19,6 +19,7 @@ { "path": "packages/openapi3/tsconfig.build.json" }, { "path": "packages/spec-api/tsconfig.build.json" }, { "path": "packages/spector/tsconfig.build.json" }, + { "path": "packages/spec-coverage-sdk/tsconfig.build.json" }, { "path": "packages/http-specs/tsconfig.build.json" }, { "path": "packages/monarch/tsconfig.build.json" }, { "path": "packages/bundler/tsconfig.build.json" }, diff --git a/website/src/pages/can-i-use/http.astro b/website/src/pages/can-i-use/http.astro index 842e1ba0049..ac53e455c05 100644 --- a/website/src/pages/can-i-use/http.astro +++ b/website/src/pages/can-i-use/http.astro @@ -9,7 +9,7 @@ import { const options: CoverageFromAzureStorageOptions = { storageAccountName: "typespec", containerName: "coverage", - manifestContainerName: "manifests-typespec", + manifests: ["http-specs"], emitterNames: [ "@typespec/http-client-python", "@typespec/http-client-csharp",