Skip to content

Commit 346c588

Browse files
matt-aitkenmatt-aitken
committed
Removed unused fn. Fix for prefix key
1 parent e96fcd6 commit 346c588

File tree

3 files changed

+51
-172
lines changed

3 files changed

+51
-172
lines changed

‎apps/webapp/app/v3/objectStore.server.ts‎

Lines changed: 3 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -196,42 +196,6 @@ export async function downloadPacketFromObjectStore(
196196
});
197197
}
198198

199-
export async function uploadDataToObjectStore(
200-
filename: string,
201-
data: string,
202-
contentType: string,
203-
prefix?: string,
204-
storageProtocol?: string
205-
): Promise<string> {
206-
return await startActiveSpan("uploadDataToObjectStore()", async (span) => {
207-
const protocol = storageProtocol || env.OBJECT_STORE_DEFAULT_PROTOCOL;
208-
const client = getObjectStoreClient(protocol);
209-
210-
if (!client) {
211-
throw new Error(`Object store is not configured for protocol: ${protocol || "default"}`);
212-
}
213-
214-
const config = getObjectStoreConfig(protocol);
215-
216-
span.setAttributes({
217-
prefix,
218-
filename,
219-
protocol: protocol || "default",
220-
});
221-
222-
const key = prefix ? `${prefix}/${filename}` : filename;
223-
224-
logger.debug("Uploading to object store", { key, protocol: protocol || "default" });
225-
226-
await client.putObject(key, data, contentType);
227-
228-
// Return a full URL for the caller (reconstruct from baseUrl + key)
229-
const url = new URL(config!.baseUrl);
230-
url.pathname = `/${key}`;
231-
return url.href;
232-
});
233-
}
234-
235199
export async function generatePresignedRequest(
236200
projectRef: string,
237201
envSlug: string,
@@ -285,7 +249,9 @@ export async function generatePresignedRequest(
285249
} catch (error) {
286250
return {
287251
success: false,
288-
error: `Failed to generate presigned URL: ${error instanceof Error ? error.message : String(error)}`,
252+
error: `Failed to generate presigned URL: ${
253+
error instanceof Error ? error.message : String(error)
254+
}`,
289255
};
290256
}
291257
}

‎apps/webapp/app/v3/objectStoreClient.server.ts‎

Lines changed: 47 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { GetObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3
33
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
44

55
interface IObjectStoreClient {
6-
putObject(key: string, body: ReadableStream | string, contentType: string): Promise<void>;
6+
putObject(key: string, body: ReadableStream | string, contentType: string): Promise<string>;
77
getObject(key: string): Promise<string>;
88
presign(key: string, method: "PUT" | "GET", expiresIn: number): Promise<string>;
99
}
@@ -36,15 +36,17 @@ class Aws4FetchClient implements IObjectStoreClient {
3636
return url.toString();
3737
}
3838

39-
async putObject(key: string, body: ReadableStream | string, contentType: string): Promise<void> {
40-
const response = await this.awsClient.fetch(this.buildUrl(key), {
39+
async putObject(key: string, body: ReadableStream | string, contentType: string): Promise<string> {
40+
const objectUrl = this.buildUrl(key);
41+
const response = await this.awsClient.fetch(objectUrl, {
4142
method: "PUT",
4243
headers: { "Content-Type": contentType },
4344
body,
4445
});
4546
if (!response.ok) {
4647
throw new Error(`Failed to upload to object store: ${response.statusText}`);
4748
}
49+
return objectUrl;
4850
}
4951

5052
async getObject(key: string): Promise<string> {
@@ -69,7 +71,7 @@ class Aws4FetchClient implements IObjectStoreClient {
6971

7072
type AwsSdkConfig = {
7173
bucket: string;
72-
baseUrl?: string;
74+
baseUrl: string;
7375
region?: string;
7476
};
7577

@@ -78,25 +80,47 @@ class AwsSdkClient implements IObjectStoreClient {
7880

7981
constructor(private readonly config: AwsSdkConfig) {
8082
this.s3Client = new S3Client({
81-
...(config.baseUrl ? { endpoint: config.baseUrl, forcePathStyle: true } : {}),
83+
endpoint: config.baseUrl,
84+
forcePathStyle: true,
8285
...(config.region ? { region: config.region } : {}),
8386
});
8487
}
8588

86-
async putObject(key: string, body: ReadableStream | string, contentType: string): Promise<void> {
89+
/**
90+
* Callers use a single logical key (same as aws4fetch path: `bucket/object/...`).
91+
* S3 APIs take Bucket + Key where Key must not repeat the bucket name.
92+
*/
93+
private toS3ObjectKey(logicalKey: string): string {
94+
const prefix = `${this.config.bucket}/`;
95+
if (logicalKey.startsWith(prefix)) {
96+
return logicalKey.slice(prefix.length);
97+
}
98+
return logicalKey;
99+
}
100+
101+
private logicalObjectUrl(logicalKey: string): string {
102+
const url = new URL(this.config.baseUrl);
103+
url.pathname = `/${logicalKey}`;
104+
return url.href;
105+
}
106+
107+
async putObject(key: string, body: ReadableStream | string, contentType: string): Promise<string> {
108+
const s3Key = this.toS3ObjectKey(key);
87109
await this.s3Client.send(
88110
new PutObjectCommand({
89111
Bucket: this.config.bucket,
90-
Key: key,
91-
Body: body as string,
112+
Key: s3Key,
113+
Body: body,
92114
ContentType: contentType,
93115
})
94116
);
117+
return this.logicalObjectUrl(key);
95118
}
96119

97120
async getObject(key: string): Promise<string> {
121+
const s3Key = this.toS3ObjectKey(key);
98122
const response = await this.s3Client.send(
99-
new GetObjectCommand({ Bucket: this.config.bucket, Key: key })
123+
new GetObjectCommand({ Bucket: this.config.bucket, Key: s3Key })
100124
);
101125
if (!response.Body) {
102126
throw new Error(`Empty response body from object store for key: ${key}`);
@@ -105,10 +129,11 @@ class AwsSdkClient implements IObjectStoreClient {
105129
}
106130

107131
async presign(key: string, method: "PUT" | "GET", expiresIn: number): Promise<string> {
132+
const s3Key = this.toS3ObjectKey(key);
108133
const command =
109134
method === "PUT"
110-
? new PutObjectCommand({ Bucket: this.config.bucket, Key: key })
111-
: new GetObjectCommand({ Bucket: this.config.bucket, Key: key });
135+
? new PutObjectCommand({ Bucket: this.config.bucket, Key: s3Key })
136+
: new GetObjectCommand({ Bucket: this.config.bucket, Key: s3Key });
112137

113138
return getSignedUrl(this.s3Client, command, { expiresIn });
114139
}
@@ -123,8 +148,12 @@ export type ObjectStoreClientConfig = {
123148
service?: string;
124149
};
125150

126-
export class ObjectStoreClient {
127-
private constructor(private readonly impl: IObjectStoreClient) {}
151+
export class ObjectStoreClient implements IObjectStoreClient {
152+
private constructor(
153+
private readonly impl: IObjectStoreClient,
154+
/** When set, logical keys may start with `${bucket}/…`; AwsSdkClient strips that prefix for S3 APIs. */
155+
readonly bucket: string | undefined
156+
) {}
128157

129158
static create(config: ObjectStoreClientConfig): ObjectStoreClient {
130159
if (config.accessKeyId && config.secretAccessKey) {
@@ -135,7 +164,8 @@ export class ObjectStoreClient {
135164
secretAccessKey: config.secretAccessKey,
136165
region: config.region,
137166
service: config.service,
138-
})
167+
}),
168+
config.bucket
139169
);
140170
}
141171

@@ -151,11 +181,12 @@ export class ObjectStoreClient {
151181
bucket: config.bucket,
152182
baseUrl: config.baseUrl,
153183
region: config.region,
154-
})
184+
}),
185+
config.bucket
155186
);
156187
}
157188

158-
putObject(key: string, body: ReadableStream | string, contentType: string): Promise<void> {
189+
putObject(key: string, body: ReadableStream | string, contentType: string): Promise<string> {
159190
return this.impl.putObject(key, body, contentType);
160191
}
161192

‎apps/webapp/test/objectStore.test.ts‎

Lines changed: 1 addition & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import { postgresAndMinioTest } from "@internal/testcontainers";
2-
import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3";
32
import { type IOPacket } from "@trigger.dev/core/v3";
4-
import { PrismaClient } from "@trigger.dev/database";
3+
import { type PrismaClient } from "@trigger.dev/database";
54
import { afterAll, describe, expect, it, vi } from "vitest";
65
import { env } from "~/env.server";
76
import {
@@ -11,29 +10,9 @@ import {
1110
generatePresignedUrl,
1211
hasObjectStoreClient,
1312
parseStorageUri,
14-
uploadDataToObjectStore,
1513
uploadPacketToObjectStore,
1614
} from "~/v3/objectStore.server";
1715

18-
async function getObjectContent(
19-
baseUrl: string,
20-
bucket: string,
21-
key: string,
22-
credentials: { accessKeyId: string; secretAccessKey: string; region: string }
23-
): Promise<string> {
24-
const client = new S3Client({
25-
endpoint: baseUrl,
26-
forcePathStyle: true,
27-
region: credentials.region,
28-
credentials: {
29-
accessKeyId: credentials.accessKeyId,
30-
secretAccessKey: credentials.secretAccessKey,
31-
},
32-
});
33-
const response = await client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
34-
return response.Body!.transformToString();
35-
}
36-
3716
// Extend the timeout for container tests
3817
vi.setConfig({ testTimeout: 60_000 });
3918

@@ -359,103 +338,6 @@ describe("Object Storage", () => {
359338
});
360339
});
361340

362-
postgresAndMinioTest(
363-
"uploadDataToObjectStore - static credentials (aws4fetch path)",
364-
async ({ minioConfig }) => {
365-
// Named protocol — controlled via process.env; pass "s3" as storageProtocol explicitly
366-
process.env.OBJECT_STORE_S3_BASE_URL = minioConfig.baseUrl;
367-
process.env.OBJECT_STORE_S3_ACCESS_KEY_ID = minioConfig.accessKeyId;
368-
process.env.OBJECT_STORE_S3_SECRET_ACCESS_KEY = minioConfig.secretAccessKey;
369-
process.env.OBJECT_STORE_S3_REGION = minioConfig.region;
370-
process.env.OBJECT_STORE_S3_SERVICE = "s3";
371-
372-
const data = JSON.stringify({ uploaded: true });
373-
374-
// With prefix — key = packets/data.json → bucket=packets, object=data.json in MinIO
375-
const urlWithPrefix = await uploadDataToObjectStore(
376-
"data.json",
377-
data,
378-
"application/json",
379-
"packets",
380-
"s3"
381-
);
382-
expect(urlWithPrefix).toContain("packets/data.json");
383-
const contentWithPrefix = await getObjectContent(
384-
minioConfig.baseUrl,
385-
"packets",
386-
"data.json",
387-
minioConfig
388-
);
389-
expect(contentWithPrefix).toBe(data);
390-
391-
// Without prefix — key = packets/bare.json → bucket=packets, object=bare.json
392-
const urlWithoutPrefix = await uploadDataToObjectStore(
393-
"packets/bare.json",
394-
data,
395-
"application/json",
396-
undefined,
397-
"s3"
398-
);
399-
expect(urlWithoutPrefix).toContain("packets/bare.json");
400-
const contentWithoutPrefix = await getObjectContent(
401-
minioConfig.baseUrl,
402-
"packets",
403-
"bare.json",
404-
minioConfig
405-
);
406-
expect(contentWithoutPrefix).toBe(data);
407-
408-
delete process.env.OBJECT_STORE_S3_BASE_URL;
409-
delete process.env.OBJECT_STORE_S3_ACCESS_KEY_ID;
410-
delete process.env.OBJECT_STORE_S3_SECRET_ACCESS_KEY;
411-
delete process.env.OBJECT_STORE_S3_REGION;
412-
delete process.env.OBJECT_STORE_S3_SERVICE;
413-
}
414-
);
415-
416-
postgresAndMinioTest(
417-
"uploadDataToObjectStore - IAM credential chain (AWS SDK path)",
418-
async ({ minioConfig }) => {
419-
// Named protocol — controlled via process.env; pass "s3" as storageProtocol explicitly
420-
process.env.OBJECT_STORE_S3_BASE_URL = minioConfig.baseUrl;
421-
process.env.OBJECT_STORE_S3_BUCKET = "packets";
422-
process.env.OBJECT_STORE_S3_REGION = minioConfig.region;
423-
delete process.env.OBJECT_STORE_S3_ACCESS_KEY_ID;
424-
delete process.env.OBJECT_STORE_S3_SECRET_ACCESS_KEY;
425-
// AWS SDK picks up credentials from AWS_* env vars (ECS task role equivalent)
426-
process.env.AWS_ACCESS_KEY_ID = minioConfig.accessKeyId;
427-
process.env.AWS_SECRET_ACCESS_KEY = minioConfig.secretAccessKey;
428-
process.env.AWS_REGION = minioConfig.region;
429-
430-
const data = JSON.stringify({ iam: true });
431-
432-
const url = await uploadDataToObjectStore(
433-
"iam-data.json",
434-
data,
435-
"application/json",
436-
"logs",
437-
"s3"
438-
);
439-
expect(url).toContain("logs/iam-data.json");
440-
441-
// AwsSdkClient stores at fixed bucket + key (prefix/filename)
442-
const content = await getObjectContent(
443-
minioConfig.baseUrl,
444-
"packets",
445-
"logs/iam-data.json",
446-
minioConfig
447-
);
448-
expect(content).toBe(data);
449-
450-
delete process.env.OBJECT_STORE_S3_BASE_URL;
451-
delete process.env.OBJECT_STORE_S3_BUCKET;
452-
delete process.env.OBJECT_STORE_S3_REGION;
453-
delete process.env.AWS_ACCESS_KEY_ID;
454-
delete process.env.AWS_SECRET_ACCESS_KEY;
455-
delete process.env.AWS_REGION;
456-
}
457-
);
458-
459341
postgresAndMinioTest(
460342
"generatePresignedUrl - PUT then GET round-trip (static credentials / aws4fetch path)",
461343
async ({ minioConfig }) => {

0 commit comments

Comments
 (0)