Add backbeat route apis

.github/docker-compose.yml

@@ -0,0 +1,182 @@
+services:
+  # ===================
+  # MongoDB
+  # ===================
+  mongodb:
+    image: ${MONGODB_IMAGE}
+    command: mongod --replSet rs0 --port ${MONGODB_PORT} --bind_ip_all
+    ports:
+      - "127.0.0.1:${MONGODB_PORT}:${MONGODB_PORT}"
+    healthcheck:
+      test: echo 'db.runCommand("ping").ok' | mongo --port ${MONGODB_PORT} --quiet
+      interval: 5s
+      timeout: 10s
+      retries: 5
+      start_period: 10s
+    profiles:
+      - mongo
+      - backbeat
+
+  mongodb-init:
+    image: ${MONGODB_IMAGE}
+    depends_on:
+      mongodb:
+        condition: service_healthy
+    command: >
+      mongo --host mongodb:${MONGODB_PORT} --eval
+      'rs.initiate({_id: "rs0", members: [{_id: 0, host: "mongodb:${MONGODB_PORT}"}]})'
+    restart: "no"
+    profiles:
+      - mongo
+      - backbeat
+
+  # ===================
+  # Cloudserver with MongoDB backend
+  # ===================
+  cloudserver:
+    image: ${CLOUDSERVER_IMAGE}
+    ports:
+      - "127.0.0.1:${CLOUDSERVER_PORT}:${CLOUDSERVER_PORT}"
+    environment:
+      - S3VAULT=mem
+      - S3METADATA=mongodb
+      - S3DATA=mem
+      - MONGODB_HOSTS=mongodb:${MONGODB_PORT}
+      - MONGODB_RS=rs0
+      - REMOTE_MANAGEMENT_DISABLE=true
+      - LOG_LEVEL=info
+      - CRR_METRICS_HOST=backbeat-api
+      - CRR_METRICS_PORT=${BACKBEAT_PORT}
+    depends_on:
+      mongodb-init:
+        condition: service_completed_successfully
+    healthcheck:
+      test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:${CLOUDSERVER_PORT}/', r => process.exit(r.statusCode < 500 ? 0 : 1)).on('error', () => process.exit(1))\""]
+      interval: 5s
+      timeout: 5s
+      retries: 12
+    profiles:
+      - mongo
+      - backbeat
+
+  # ===================
+  # Metadata backend services
+  # ===================
+  metadata-standalone:
+    image: ${METADATA_IMAGE}
+    network_mode: 'host'
+    volumes:
+      - ./md-config.json:/mnt/standalone_workdir/config.json:ro
+    profiles:
+      - metadata
+
+  cloudserver-metadata:
+    image: ${CLOUDSERVER_IMAGE}
+    network_mode: 'host'
+    environment:
+      - S3VAULT=mem
+      - S3METADATA=scality
+      - S3DATA=mem
+      - REMOTE_MANAGEMENT_DISABLE=true
+      - LOG_LEVEL=info
+    depends_on:
+      - metadata-standalone
+    healthcheck:
+      test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:${CLOUDSERVER_PORT}/', r => process.exit(r.statusCode < 500 ? 0 : 1)).on('error', () => process.exit(1))\""]
+      interval: 5s
+      timeout: 5s
+      retries: 12
+    profiles:
+      - metadata
+
+  # ===================
+  # Backbeat services
+  # ===================
+  zookeeper:
+    image: ${ZOOKEEPER_IMAGE}
+    ports:
+      - "127.0.0.1:${ZOOKEEPER_PORT}:${ZOOKEEPER_PORT}"
+    healthcheck:
+      test: ["CMD", "zkServer.sh", "status"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    profiles:
+      - backbeat
+
+  zk-init:
+    image: ${ZOOKEEPER_IMAGE}
+    depends_on:
+      zookeeper:
+        condition: service_healthy
+    command: >
+      bash -c "
+        sleep 2 &&
+        zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat '' &&
+        zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication '' &&
+        zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state '' &&
+        zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/ingestion '' &&
+        zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/ingestion/state '' &&
+        zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state/us-east-1 '{\"paused\":false}' &&
+        zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state/us-east-2 '{\"paused\":false}' &&
+        zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state/wontwork-location '{\"paused\":false}' &&
+        zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state/location-dmf-v1 '{\"paused\":false}' &&
+        zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state/zenko '{\"paused\":false}' &&
+        zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/state/aws-location '{\"paused\":false}' &&
+        zkCli.sh -server zookeeper:${ZOOKEEPER_PORT} create /backbeat/replication/replayState '' &&
+        echo 'Zookeeper paths initialized successfully'
+      "
+    restart: "no"
+    profiles:
+      - backbeat
+
+  redis:
+    image: ${REDIS_IMAGE}
+    ports:
+      - "127.0.0.1:${REDIS_PORT}:${REDIS_PORT}"
+    profiles:
+      - backbeat
+
+  kafka:
+    image: ${KAFKA_IMAGE}
+    ports:
+      - "127.0.0.1:${KAFKA_PORT}:${KAFKA_PORT}"
+    environment:
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:${ZOOKEEPER_PORT}
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:${KAFKA_PORT}
+      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:${KAFKA_PORT}
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+    depends_on:
+      zookeeper:
+        condition: service_healthy
+    profiles:
+      - backbeat
+
+  backbeat-api:
+    image: ${BACKBEAT_IMAGE}
+    ports:
+      - "127.0.0.1:${BACKBEAT_PORT}:${BACKBEAT_PORT}"
+    environment:
+      ZOOKEEPER_CONNECTION_STRING: zookeeper:${ZOOKEEPER_PORT}
+      REDIS_HOST: redis
+      KAFKA_HOSTS: kafka:${KAFKA_PORT}
+      MONGODB_HOSTS: mongodb:${MONGODB_PORT}
+      REMOTE_MANAGEMENT_DISABLE: "1"
+      HEALTHCHECKS_ALLOWFROM: "0.0.0.0/0"
+    depends_on:
+      mongodb-init:
+        condition: service_completed_successfully
+      zk-init:
+        condition: service_completed_successfully
+      kafka:
+        condition: service_started
+      redis:
+        condition: service_started
+    healthcheck:
+      test: ["CMD-SHELL", "node -e \"require('http').get('http://localhost:${BACKBEAT_PORT}/_/healthcheck', r => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))\""]
+      interval: 5s
+      timeout: 5s
+      retries: 12
+    command: node bin/backbeat.js
+    profiles:
+      - backbeat

.github/docker.env

@@ -0,0 +1,27 @@
+# MongoDB
+MONGODB_IMAGE=mongo:4.4
+MONGODB_PORT=27018
+
+# Cloudserver
+CLOUDSERVER_IMAGE=ghcr.io/scality/cloudserver:9.3.0-preview.1
+CLOUDSERVER_PORT=8000
+
+# Metadata
+METADATA_IMAGE=ghcr.io/scality/metadata:8.11.0-standalone
+METADATA_PORT=9000
+
+# Zookeeper
+ZOOKEEPER_IMAGE=zookeeper:3.8
+ZOOKEEPER_PORT=2181
+
+# Kafka
+KAFKA_IMAGE=wurstmeister/kafka:latest
+KAFKA_PORT=9092
+
+# Redis
+REDIS_IMAGE=redis:7-alpine
+REDIS_PORT=6379
+
+# Backbeat
+BACKBEAT_IMAGE=ghcr.io/scality/backbeat:9.1.3
+BACKBEAT_PORT=8900

.github/workflows/test.yml

@@ -40,20 +40,11 @@ jobs:
         uses: ./.github/actions/setup-and-build
 
       - name: Start Cloudserver with MongoDB backend
-        run: docker compose -f .github/docker-compose.cloudserver-mongo.yml up -d
-
-      - name: Wait for Cloudserver to be ready
-        run: |
-          set -o pipefail
-          bash .github/scripts/wait_for_local_port.bash 8000 40
+        run: docker compose -f .github/docker-compose.yml --env-file .github/docker.env --profile mongo up -d --wait
 
       - name: Run MongoDB backend tests
         run: yarn test:mongo-backend
 
-      - name: Stop Cloudserver
-        if: always()
-        run: docker compose -f .github/docker-compose.cloudserver-mongo.yml down
-
   test-metadata-backend:
     name: Test with Scality metadata backend
     runs-on: ubuntu-24.04
@@ -74,21 +65,37 @@ jobs:
           password: ${{ github.token }}
 
       - name: Start Cloudserver with Scality metadata backend
-        run: docker compose -f .github/docker-compose.cloudserver-metadata.yml up -d
+        run: docker compose -f .github/docker-compose.yml --env-file .github/docker.env --profile metadata up -d --wait
 
       - name: Wait for metadata to be ready
         run: |
           set -o pipefail
           bash .github/scripts/wait_for_local_port.bash 9000 40
 
-      - name: Wait for Cloudserver to be ready
-        run: |
-          set -o pipefail
-          bash .github/scripts/wait_for_local_port.bash 8000 60
-
       - name: Run metadata backend tests
         run: yarn test:metadata-backend
 
-      - name: Stop Cloudserver
-        if: always()
-        run: docker compose -f .github/docker-compose.cloudserver-metadata.yml down
+  test-backbeat-apis:
+    name: Test backbeat apis
+    runs-on: ubuntu-24.04
+    needs: lint
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Setup and Build
+        uses: ./.github/actions/setup-and-build
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
+
+      - name: Start Backbeat with CloudServer
+        run: docker compose -f .github/docker-compose.yml --env-file .github/docker.env --profile backbeat up -d --wait
+
+      - name: Run backbeat apis tests
+        run: yarn test:backbeat-apis

models/backbeatRoutes/commonStructures.smithy

@@ -44,4 +44,4 @@ structure LocationMDObj {
 
     /// Version ID in the data store for this location
     dataStoreVersionId: String
-}
+}

models/proxyBackbeatApis/checkConnection.smithy

@@ -0,0 +1,13 @@
+$version: "2.0"
+namespace cloudserver.proxyBackbeatApis
+
+@readonly
+@http(method: "GET", uri: "/_/backbeat/api/healthcheck")
+operation CheckConnection {
+    input: CheckConnectionInput,
+    output: CheckConnectionOutput
+}
+
+structure CheckConnectionInput {}
+
+structure CheckConnectionOutput {}

models/proxyBackbeatApis/commonStructures.smithy

@@ -0,0 +1,21 @@
+$version: "2.0"
+namespace cloudserver.proxyBackbeatApis
+
+map LocationStatusMap {
+    key: String,
+    value: Document
+}
+
+structure FailedObjectVersion {
+    Bucket: String,
+
+    Key: String,
+
+    VersionId: String,
+
+    StorageClass: String,
+
+    Size: Integer,
+
+    LastModified: Timestamp
+}