added s3 extended apis

examples/s3extended.ts

@@ -0,0 +1,21 @@
+
+import { S3Client, S3ClientConfig } from '@aws-sdk/client-s3';
+import { ListObjectsV2ExtendedCommand } from '@scality/cloudserverclient/clients/s3Extended';
+
+const config: S3ClientConfig = {
+    endpoint: 'http://localhost:8000',
+    credentials: {
+        accessKeyId: 'accessKey1',
+        secretAccessKey: 'verySecretKey1',
+    },
+    region: 'us-east-1',
+};
+
+const client = new S3Client(config);
+
+const response = await client.send(
+    new ListObjectsV2ExtendedCommand({
+        Bucket: 'aBucketName',
+        Query: 'content-length > 0',
+    }),
+);

package.json

@@ -19,8 +19,7 @@
   },
   "files": [
     "dist",
-    "build/smithy/cloudserverBackbeatRoutes/typescript-codegen",
-    "build/smithy/cloudserverBucketQuota/typescript-codegen"
+    "build/smithy/*/typescript-codegen"
   ],
   "publishConfig": {
     "access": "public",
@@ -34,16 +33,8 @@
     "build:wrapper": "tsc",
     "build": "yarn install && yarn clean:build && yarn build:smithy && yarn build:generated:backbeatRoutes && yarn build:generated:bucketQuota && yarn build:wrapper",
     "test": "jest",
-    "test:indexes": "jest tests/testIndexesApis.test.ts",
-    "test:error-handling": "jest tests/testErrorHandling.test.ts",
-    "test:multiple-backend": "jest tests/testMultipleBackendApis.test.ts",
-    "test:api": "jest tests/testApis.test.ts",
-    "test:lifecycle": "jest tests/testLifecycleApis.test.ts",
-    "test:metadata": "jest tests/testMetadataApis.test.ts",
-    "test:raft": "jest tests/testRaftApis.test.ts",
-    "test:bucketQuotas": "jest tests/testQuotaApis.test.ts",
-    "test:mongo-backend": "yarn test:indexes && yarn test:error-handling && yarn test:multiple-backend && yarn test:bucketQuotas",
-    "test:metadata-backend": "yarn test:api && yarn test:lifecycle && yarn test:metadata && yarn test:raft",
+    "test:mongo-backend": "BACKEND_TYPE=mongo jest",
+    "test:metadata-backend": "BACKEND_TYPE=metadata jest",
     "lint": "eslint src tests",
     "typecheck": "tsc --noEmit"
   },

src/clients/s3Extended.ts

@@ -0,0 +1,64 @@
+import { 
+    ListObjectsCommand,
+    ListObjectsCommandInput,
+    ListObjectsV2Command, 
+    ListObjectsV2CommandInput,
+    ListObjectVersionsCommand,
+    ListObjectVersionsCommandInput
+} from '@aws-sdk/client-s3';
+
+const extendCommandWithExtraParametersMiddleware = (query: string) => 
+    (next: any) => async (args: any) => {
+        const request = args.request as any;
+        if (request.query) {
+            request.query.search = query;
+        } else {
+            request.query = { search: query };
+        }
+        return next(args);
+    };
+
+export interface ListObjectsExtendedInput extends ListObjectsCommandInput {
+    Query: string;
+}
+
+export class ListObjectsExtendedCommand extends ListObjectsCommand {
+    constructor(input: ListObjectsExtendedInput) {
+        super(input);
+
+        this.middlewareStack.add(
+            extendCommandWithExtraParametersMiddleware(input.Query),
+            { step: 'build', name: 'extendCommandWithExtraParameters' }
+        );
+    }
+}
+
+export interface ListObjectsV2ExtendedInput extends ListObjectsV2CommandInput {
+    Query: string;
+}
+
+export class ListObjectsV2ExtendedCommand extends ListObjectsV2Command {
+    constructor(input: ListObjectsV2ExtendedInput) {
+        super(input);
+
+        this.middlewareStack.add(
+            extendCommandWithExtraParametersMiddleware(input.Query),
+            { step: 'build', name: 'extendCommandWithExtraParameters' }
+        );
+    }
+}
+
+export interface ListObjectVersionsExtendedInput extends ListObjectVersionsCommandInput {
+    Query: string;
+}
+
+export class ListObjectVersionsExtendedCommand extends ListObjectVersionsCommand {
+    constructor(input: ListObjectVersionsExtendedInput) {
+        super(input);
+
+        this.middlewareStack.add(
+            extendCommandWithExtraParametersMiddleware(input.Query),
+            { step: 'build', name: 'extendCommandWithExtraParameters' }
+        );
+    }
+}

src/index.ts

@@ -1,4 +1,5 @@
 export * from './clients/backbeatRoutes';
 export * from './clients/bucketQuota';
+export * from './clients/s3Extended';
 export { CloudserverClient, CloudserverClientConfig } from './clients/cloudserver';
 export * from './utils';

tests/testApis.test.ts

@@ -13,9 +13,10 @@ import {
 } from '../src/index';
 import { S3Client, GetObjectCommand as S3getCommand } from '@aws-sdk/client-s3';
 import { createTestClient, testConfig } from './testSetup';
+import { describeForMetadataBackend } from './testHelpers';
 import assert from 'assert';
 
-describe('CloudServer API Tests', () => {
+describeForMetadataBackend('CloudServer Backbeat Routes API Tests', () => {
     let backbeatRoutesClient: BackbeatRoutesClient;
     let s3client: S3Client;
 

tests/testErrorHandling.test.ts

@@ -7,8 +7,9 @@ import {
 } from '../src/index';
 import assert from 'assert';
 import { createTestClient, testConfig } from './testSetup';
+import { describeForMongoBackend } from './testHelpers';
 
-describe('CloudServer test error handling', () => {
+describeForMongoBackend('CloudServer test error handling', () => {
     let backbeatRoutesClient: BackbeatRoutesClient;
 
     beforeAll(() => {

tests/testHelpers.ts

@@ -0,0 +1,20 @@
+enum BackendType {
+    MONGO = 'mongo',
+    METADATA = 'metadata',
+}
+
+export function describeForMongoBackend(name: string, fn: () => void): void {
+    if (process.env.BACKEND_TYPE === BackendType.METADATA) {
+        describe.skip(`${name} (tests skipped: mongo backend only)`, fn);
+    } else {
+        describe(name, fn);
+    }
+}
+
+export function describeForMetadataBackend(name: string, fn: () => void): void {
+    if (process.env.BACKEND_TYPE === BackendType.METADATA) {
+        describe(name, fn);
+    } else {
+        describe.skip(`${name} (tests skipped: metadata backend only)`, fn);
+    }
+}

tests/testIndexesApis.test.ts

@@ -9,8 +9,9 @@ import {
 } from '../src/index';
 import assert from 'assert';
 import { createTestClient, testConfig } from './testSetup';
+import { describeForMongoBackend } from './testHelpers';
 
-describe('CloudServer Indexes API Tests', () => {
+describeForMongoBackend('CloudServer Indexes API Tests', () => {
     let backbeatRoutesClient: BackbeatRoutesClient;
 
     beforeAll(() => {

tests/testLifecycleApis.test.ts

@@ -11,8 +11,9 @@ import {
 } from '../src/index';
 import assert from 'assert';
 import { createTestClient, testConfig } from './testSetup';
+import { describeForMetadataBackend } from './testHelpers';
 
-describe('CloudServer Lifecycle API Tests', () => {
+describeForMetadataBackend('CloudServer Lifecycle API Tests', () => {
     let backbeatRoutesClient: BackbeatRoutesClient;
 
     beforeAll(() => {

tests/testMetadataApis.test.ts

@@ -9,8 +9,9 @@ import {
 } from '../src/index';
 import assert from 'assert';
 import { createTestClient, testConfig } from './testSetup';
+import { describeForMetadataBackend } from './testHelpers';
 
-describe('CloudServer Metadata API Tests', () => {
+describeForMetadataBackend('CloudServer Metadata API Tests', () => {
     let backbeatRoutesClient: BackbeatRoutesClient;
 
     beforeAll(() => {

tests/testMultipleBackendApis.test.ts

@@ -21,10 +21,11 @@ import {
     addContentLengthMiddleware,
 } from '../src/index';
 import { createTestClient, testConfig } from './testSetup';
+import { describeForMongoBackend } from './testHelpers';
 import assert from 'assert';
 import crypto from 'crypto';
 
-describe('CloudServer Multiple Backend API Tests', () => {
+describeForMongoBackend('CloudServer Multiple Backend API Tests', () => {
     let backbeatRoutesClient: BackbeatRoutesClient;
 
     beforeAll(() => {

tests/testQuotaApis.test.ts

@@ -5,9 +5,10 @@ import {
     DeleteBucketQuotaCommand,
 } from '../src/clients/bucketQuota';
 import { createTestClient, testConfig } from './testSetup';
+import { describeForMongoBackend } from './testHelpers';
 import assert from 'assert';
 
-describe('Quota API Tests', () => {
+describeForMongoBackend('Quota API Tests', () => {
     let bucketQuotaClient: BucketQuotaClient;
     const quotaValue = 12321;
 

tests/testRaftApis.test.ts

@@ -11,10 +11,11 @@ import {
 } from '../src/index';
 import assert from 'assert';
 import { createTestClient, testConfig } from './testSetup';
+import { describeForMetadataBackend } from './testHelpers';
 import stream from 'stream';
 import JSONStream from 'JSONStream';
 
-describe('CloudServer Raft API Tests', () => {
+describeForMetadataBackend('CloudServer Raft API Tests', () => {
     let backbeatRoutesClient: BackbeatRoutesClient;
 
     beforeAll(() => {

tests/testS3ExtendedApis.test.ts

@@ -0,0 +1,135 @@
+import { S3Client, PutObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3';
+import { createTestClient, testConfig } from './testSetup';
+import { describeForMongoBackend } from './testHelpers';
+import assert from 'assert';
+import {
+    ListObjectsExtendedCommand,
+    ListObjectsV2ExtendedCommand,
+    ListObjectVersionsExtendedCommand,
+} from '../src/clients/s3Extended';
+
+describeForMongoBackend('S3 Extended API Tests', () => {
+    let s3client: S3Client;
+    const key2ndObject = `${testConfig.objectKey}2nd`;
+    const body2ndObject = `${testConfig.objectData}2nd`;
+
+    beforeAll(async () => {
+        const testClients = createTestClient();
+        s3client = testClients.s3client;
+
+        const putObjectCommand = new PutObjectCommand({
+            Bucket: testConfig.bucketName,
+            Key: key2ndObject,
+            Body: body2ndObject,
+        });
+        await s3client.send(putObjectCommand);
+    });
+
+    it('should test ListObjectsExtended', async () => {
+        const getCommand1= new ListObjectsExtendedCommand({
+            Bucket: testConfig.bucketName,
+            Query: `content-length >= ${testConfig.objectData.length}`,
+            MaxKeys: 5
+        });
+        const getData1 = await s3client.send(getCommand1);
+        assert.strictEqual(getData1.Contents?.length, 2);
+
+        const maxKey = 1;
+        const getCommand2= new ListObjectsExtendedCommand({
+            Bucket: testConfig.bucketName,
+            Query: `content-length >= ${testConfig.objectData.length}`,
+            MaxKeys: maxKey
+        });
+        const getData2 = await s3client.send(getCommand2);
+        assert.strictEqual(getData2.Contents?.length, maxKey);
+        assert.strictEqual(getData2.IsTruncated, true);
+
+        const getCommand3 = new ListObjectsExtendedCommand({
+            Bucket: testConfig.bucketName,
+            Query: `content-length > ${testConfig.objectData.length}`,
+            MaxKeys: 5
+        });
+        const getData3 = await s3client.send(getCommand3);
+        assert.strictEqual(getData3.Contents?.length, 1);
+        assert.strictEqual(getData3.Contents[0].Key, key2ndObject);
+
+        const getCommand4 = new ListObjectsExtendedCommand({
+            Bucket: testConfig.bucketName,
+            Query: `key = ${key2ndObject}`,
+            MaxKeys: 5
+        });
+        const getData4 = await s3client.send(getCommand4);
+        assert.strictEqual(getData4.Contents?.length, 1);
+        assert.strictEqual(getData4.Contents[0].Key, key2ndObject);
+
+        const getCommand5 = new ListObjectsExtendedCommand({
+            Bucket: testConfig.bucketName,
+            Query: `key = iDontExists`,
+            MaxKeys: 5
+        });
+        const getData5 = await s3client.send(getCommand5);
+        assert.strictEqual(getData5.Contents, undefined);
+    });
+
+    it('should test ListObjectsV2Extended', async () => {
+        const getCommand1= new ListObjectsV2ExtendedCommand({
+            Bucket: testConfig.bucketName,
+            Query: `content-length >= ${testConfig.objectData.length}`,
+            MaxKeys: 5,
+            FetchOwner: true,
+        });
+        const getData1 = await s3client.send(getCommand1);
+        assert.strictEqual(getData1.Contents?.length, 2);
+        assert.strictEqual(getData1.KeyCount, 2);
+        assert.strictEqual(getData1.Contents[0].Owner?.DisplayName, 'Bart');
+
+        const getCommand2 = new ListObjectsV2ExtendedCommand({
+            Bucket: testConfig.bucketName,
+            Query: `content-length >= 0`,
+            MaxKeys: 5,
+            StartAfter: testConfig.objectKey, // Skip first object
+        });
+        const getData2 = await s3client.send(getCommand2);
+        assert.strictEqual(getData2.Contents?.length, 1);
+        assert.strictEqual(getData2.Contents[0].Key, key2ndObject);
+    });
+
+    it('should test ListObjectVersionsExtended', async () => {
+        const getCommand1 = new ListObjectVersionsExtendedCommand({
+            Bucket: testConfig.bucketName,
+            Query: `content-length >= 0`,
+            MaxKeys: 100,
+        });
+        const getData1 = await s3client.send(getCommand1);
+        assert.strictEqual(getData1.Versions?.length, 2);
+        assert.strictEqual(getData1.Versions[0].IsLatest, true);
+
+        // Delete one object to create a DeleteMarker
+        const deleteCommand = new DeleteObjectCommand({
+            Bucket: testConfig.bucketName,
+            Key: key2ndObject,
+        });
+        await s3client.send(deleteCommand);
+
+        const getCommand2 = new ListObjectVersionsExtendedCommand({
+            Bucket: testConfig.bucketName,
+            Query: `content-length >= 0`,
+            MaxKeys: 100,
+        });
+        const getData2 = await s3client.send(getCommand2);
+        assert.strictEqual(getData2.DeleteMarkers?.[0].Key, key2ndObject);
+        assert.ok(getData2.DeleteMarkers?.[0].VersionId);
+
+        assert.ok(getData2.Versions);
+        const firstVersion = getData2.Versions[0];
+        const getCommand3 = new ListObjectVersionsExtendedCommand({
+            Bucket: testConfig.bucketName,
+            Query: `content-length >= 0`,
+            KeyMarker: firstVersion.Key,
+            VersionIdMarker: firstVersion.VersionId,
+            MaxKeys: 100,
+        });
+        const getData3 = await s3client.send(getCommand3);
+        assert.notStrictEqual(getData3.Versions?.[0]?.Key, firstVersion.Key);
+    });
+});