Add helper rewrite in rust

.gitlab-ci.yml

@@ -57,7 +57,7 @@ appsec-trigger:
     strategy: depend
   variables:
     PARENT_PIPELINE_ID: $CI_PIPELINE_ID
-    GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/cpp-base64 appsec/third_party/libddwaf appsec/third_party/msgpack-c
+    GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/cpp-base64 appsec/third_party/libddwaf appsec/third_party/libddwaf-rust appsec/third_party/msgpack-c
 
 profiler-trigger:
   stage: tests
@@ -92,6 +92,6 @@ package-trigger:
     strategy: depend
   variables:
     PARENT_PIPELINE_ID: $CI_PIPELINE_ID
-    GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/cpp-base64 appsec/third_party/libddwaf appsec/third_party/msgpack-c
+    GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/cpp-base64 appsec/third_party/libddwaf appsec/third_party/libddwaf-rust appsec/third_party/msgpack-c
     NIGHTLY_BUILD: $NIGHTLY_BUILD
     RELIABILITY_ENV_BRANCH: $RELIABILITY_ENV_BRANCH

.gitlab/build-appsec-helper-rust.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -e -o pipefail
+
+MAKE_JOBS=${MAKE_JOBS:-$(nproc)}
+
+mkdir -p appsec_$(uname -m)
+
+git config --global --add safe.directory '*'
+
+cd appsec/helper-rust
+
+export CARGO_TARGET_DIR=/tmp/cargo-target
+RUST_TARGET=$(uname -m)-unknown-linux-musl
+
+# Build using nightly toolchain with unstable features
+# -Z build-std: Rebuild std library for musl
+# -Z build-std-features=llvm-libunwind: Use LLVM libunwind instead of libgcc_s
+cargo +nightly-"$RUST_TARGET" build \
+    --release \
+    -Zhost-config \
+    -Ztarget-applies-to-host \
+    --target "$RUST_TARGET"
+
+# Remove musl libc dependency using patchelf (makes binary work on both musl and glibc)
+BINARY_PATH="/tmp/cargo-target/$RUST_TARGET/release/libddappsec_helper_rust.so"
+ARCH=$(uname -m)
+if [ "$ARCH" = "x86_64" ]; then
+    patchelf --remove-needed libc.musl-x86_64.so.1 "$BINARY_PATH" 2>/dev/null || true
+elif [ "$ARCH" = "aarch64" ]; then
+    patchelf --remove-needed libc.musl-aarch64.so.1 "$BINARY_PATH" 2>/dev/null || true
+fi
+
+# Copy to output
+cp -v "$BINARY_PATH" "../../appsec_$(uname -m)/libddappsec-helper-rust.so"
+
+# Run tests
+cargo +nightly-"$RUST_TARGET" test \
+    --release \
+    -Zhost-config \
+    -Ztarget-applies-to-host \
+    --target "$RUST_TARGET"

.gitlab/generate-appsec.php

@@ -94,16 +94,52 @@
       -DENABLE_ASAN=$ASAN_FLAG"
     - ASAN_OPTIONS=malloc_context_size=0 make -j 4 xtest
 
-"appsec integration tests":
+.appsec_integration_tests:
   stage: test
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:24.0.4-gbi-focal # TODO: use a proper docker image with java pre-installed?
   tags: [ "docker-in-docker:amd64" ]
   variables:
     KUBERNETES_CPU_REQUEST: 8
     KUBERNETES_MEMORY_REQUEST: 24Gi
     KUBERNETES_MEMORY_LIMIT: 30Gi
+    DOCKER_LOOPBACK_SIZE: 30G
     ARCH: amd64
+    HELPER_RUST_FLAG: ""
     GRADLE_USER_HOME: "$CI_PROJECT_DIR/.gradle-home"
+  before_script:
+<?php echo $ecrLoginSnippet, "\n"; ?>
+<?php dockerhub_login() ?>
+  script:
+    - apt update && apt install -y openjdk-17-jre
+    - find "$CI_PROJECT_DIR"/appsec/tests/integration/build || true
+    - |
+      cd appsec/tests/integration
+      CACHE_PATH=build/php-appsec-volume-caches-${ARCH}.tar.gz
+      if [ -f "$CACHE_PATH" ]; then
+        echo "Loading cache from $CACHE_PATH"
+        TERM=dumb ./gradlew loadCaches --info
+      fi
+
+      TERM=dumb ./gradlew $targets --info -Pbuildscan --scan $HELPER_RUST_FLAG
+      TERM=dumb ./gradlew saveCaches --info
+  after_script:
+    - mkdir -p "${CI_PROJECT_DIR}/artifacts"
+    - find appsec/tests/integration/build/test-results -name "*.xml" -exec cp --parents '{}' "${CI_PROJECT_DIR}/artifacts/" \;
+    - .gitlab/upload-junit-to-datadog.sh "test.source.file:appsec"
+  artifacts:
+    reports:
+      junit: "artifacts/**/test-results/**/TEST-*.xml"
+    paths:
+      - "artifacts/"
+    when: "always"
+  cache:
+    - key: "appsec int test cache"
+      paths:
+        - appsec/tests/integration/build/*.tar.gz
+        - .gradle-home/wrapper/dists/
+
+"appsec integration tests":
+  extends: .appsec_integration_tests
   parallel:
     matrix:
       - targets:
@@ -129,33 +165,194 @@
           - test8.4-release-zts
           - test8.5-release
           - test8.5-release-zts
+          - test8.5-release-musl
+
+"appsec integration tests (helper-rust)":
+  extends: .appsec_integration_tests
+  variables:
+    HELPER_RUST_FLAG: "-PuseHelperRust"
+  parallel:
+    matrix:
+      - targets:
+          - test7.4-release
+          - test8.1-release
+          - test8.3-debug
+          - test8.4-release-zts
+          - test8.5-release-musl
+
+"helper-rust build and test":
+  stage: test
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:24.0.4-gbi-focal
+  tags: [ "docker-in-docker:amd64" ]
+  interruptible: true
+  rules:
+    - if: $CI_COMMIT_BRANCH == "master"
+      interruptible: false
+    - when: on_success
+  variables:
+    KUBERNETES_CPU_REQUEST: 4
+    KUBERNETES_MEMORY_REQUEST: 8Gi
+    KUBERNETES_MEMORY_LIMIT: 10Gi
+    ARCH: amd64
   before_script:
 <?php echo $ecrLoginSnippet, "\n"; ?>
 <?php dockerhub_login() ?>
   script:
     - apt update && apt install -y openjdk-17-jre
-    - find "$CI_PROJECT_DIR"/appsec/tests/integration/build || true
     - |
       cd appsec/tests/integration
       CACHE_PATH=build/php-appsec-volume-caches-${ARCH}.tar.gz
       if [ -f "$CACHE_PATH" ]; then
         echo "Loading cache from $CACHE_PATH"
         TERM=dumb ./gradlew loadCaches --info
       fi
+      # Build and test helper-rust (includes formatting check and cargo test)
+      TERM=dumb ./gradlew testHelperRust --info -Pbuildscan --scan
+      TERM=dumb ./gradlew saveCaches --info
+  cache:
+    - key: "appsec int test cache"
+      paths:
+        - appsec/tests/integration/build/*.tar.gz
+        - .gradle-home/wrapper/dists/
+
+"helper-rust code coverage":
+  stage: test
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:24.0.4-gbi-focal
+  tags: [ "docker-in-docker:amd64" ]
+  interruptible: true
+  rules:
+    - if: $CI_COMMIT_BRANCH == "master"
+      interruptible: false
+    - when: on_success
+  variables:
+    KUBERNETES_CPU_REQUEST: 4
+    KUBERNETES_MEMORY_REQUEST: 8Gi
+    KUBERNETES_MEMORY_LIMIT: 10Gi
+    ARCH: amd64
+  before_script:
+<?php echo $ecrLoginSnippet, "\n"; ?>
+<?php dockerhub_login() ?>
+  script:
+    - apt update && apt install -y openjdk-17-jre
+    - |
+      echo "Installing codecov CLI"
+      curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --keyring trustedkeys.gpg --import
+      CODECOV_VERSION=0.6.1
+      curl -Os https://uploader.codecov.io/v${CODECOV_VERSION}/linux/codecov
+      curl -Os https://uploader.codecov.io/v${CODECOV_VERSION}/linux/codecov.SHA256SUM
+      curl -Os https://uploader.codecov.io/v${CODECOV_VERSION}/linux/codecov.SHA256SUM.sig
+      gpgv codecov.SHA256SUM.sig codecov.SHA256SUM
+      shasum -a 256 -c codecov.SHA256SUM
+      rm codecov.SHA256SUM.sig codecov.SHA256SUM
+      chmod +x codecov
+      mv codecov /usr/local/bin/codecov
+    - |
+      echo "Installing vault for codecov token"
+      curl -o vault.zip https://releases.hashicorp.com/vault/1.20.0/vault_1.20.0_linux_amd64.zip
+      unzip vault.zip
+      mv vault /usr/local/bin/vault
+      rm vault.zip
+    - |
+      cd appsec/tests/integration
+      CACHE_PATH=build/php-appsec-volume-caches-${ARCH}.tar.gz
+      if [ -f "$CACHE_PATH" ]; then
+        echo "Loading cache from $CACHE_PATH"
+        TERM=dumb ./gradlew loadCaches --info
+      fi
+      # Run unit tests with coverage instrumentation
+      TERM=dumb ./gradlew coverageHelperRust --info -Pbuildscan --scan
+      TERM=dumb ./gradlew saveCaches --info
+    - |
+      echo "Extracting coverage data from Docker volume"
+      mkdir -p "$CI_PROJECT_DIR"/appsec/helper-rust
+      docker run --rm -v php-helper-rust-coverage:/vol alpine cat /vol/coverage-unit.lcov > "$CI_PROJECT_DIR"/appsec/helper-rust/coverage-unit.lcov
+    - |
+      echo "Uploading helper-rust unit test coverage to codecov"
+      cd "$CI_PROJECT_DIR"
+      CODECOV_TOKEN=$(vault kv get --format=json kv/k8s/gitlab-runner/dd-trace-php/codecov | jq -r .data.data.token)
+      codecov -t "$CODECOV_TOKEN" -n helper-rust-unit -F helper-rust-unit -v -f appsec/helper-rust/coverage-unit.lcov
+  artifacts:
+    paths:
+      - appsec/helper-rust/coverage-unit.lcov
+    when: always
+  cache:
+    - key: "appsec int test cache"
+      paths:
+        - appsec/tests/integration/build/*.tar.gz
+        - .gradle-home/wrapper/dists/
 
-      TERM=dumb ./gradlew $targets --info -Pbuildscan --scan
+"helper-rust integration coverage":
+  stage: test
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:24.0.4-gbi-focal
+  tags: [ "docker-in-docker:amd64" ]
+  interruptible: true
+  rules:
+    - if: $CI_COMMIT_BRANCH == "master"
+      interruptible: false
+    - when: on_success
+  variables:
+    KUBERNETES_CPU_REQUEST: 8
+    KUBERNETES_MEMORY_REQUEST: 24Gi
+    KUBERNETES_MEMORY_LIMIT: 30Gi
+    ARCH: amd64
+  before_script:
+<?php echo $ecrLoginSnippet, "\n"; ?>
+<?php dockerhub_login() ?>
+  script:
+    - apt update && apt install -y openjdk-17-jre
+    - |
+      echo "Installing codecov CLI"
+      curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --keyring trustedkeys.gpg --import
+      CODECOV_VERSION=0.6.1
+      curl -Os https://uploader.codecov.io/v${CODECOV_VERSION}/linux/codecov
+      curl -Os https://uploader.codecov.io/v${CODECOV_VERSION}/linux/codecov.SHA256SUM
+      curl -Os https://uploader.codecov.io/v${CODECOV_VERSION}/linux/codecov.SHA256SUM.sig
+      gpgv codecov.SHA256SUM.sig codecov.SHA256SUM
+      shasum -a 256 -c codecov.SHA256SUM
+      rm codecov.SHA256SUM.sig codecov.SHA256SUM
+      chmod +x codecov
+      mv codecov /usr/local/bin/codecov
+    - |
+      echo "Installing vault for codecov token"
+      curl -o vault.zip https://releases.hashicorp.com/vault/1.20.0/vault_1.20.0_linux_amd64.zip
+      unzip vault.zip
+      mv vault /usr/local/bin/vault
+      rm vault.zip
+    - |
+      cd appsec/tests/integration
+      CACHE_PATH=build/php-appsec-volume-caches-${ARCH}.tar.gz
+      if [ -f "$CACHE_PATH" ]; then
+        echo "Loading cache from $CACHE_PATH"
+        TERM=dumb ./gradlew loadCaches --info
+      fi
+      # Build helper-rust with coverage instrumentation
+      TERM=dumb ./gradlew buildHelperRustWithCoverage --info -Pbuildscan --scan
+      # Run integration tests with coverage-instrumented binary
+      TERM=dumb ./gradlew test8.3-debug --info -Pbuildscan --scan -PuseHelperRustCoverage
+      # Generate coverage report from profraw files
+      TERM=dumb ./gradlew generateHelperRustIntegrationCoverage --info -Pbuildscan --scan
       TERM=dumb ./gradlew saveCaches --info
+    - |
+      echo "Extracting coverage data from Docker volume"
+      mkdir -p "$CI_PROJECT_DIR"/appsec/helper-rust
+      docker run --rm -v php-helper-rust-coverage:/vol alpine cat /vol/coverage-integration.lcov > "$CI_PROJECT_DIR"/appsec/helper-rust/coverage-integration.lcov
+    - |
+      echo "Uploading helper-rust integration test coverage to codecov"
+      cd "$CI_PROJECT_DIR"
+      CODECOV_TOKEN=$(vault kv get --format=json kv/k8s/gitlab-runner/dd-trace-php/codecov | jq -r .data.data.token)
+      codecov -t "$CODECOV_TOKEN" -n helper-rust-integration -F helper-rust-integration -v -f appsec/helper-rust/coverage-integration.lcov
   after_script:
     - mkdir -p "${CI_PROJECT_DIR}/artifacts"
-    - find appsec/tests/integration/build/test-results -name "*.xml" -exec cp --parents '{}' "${CI_PROJECT_DIR}/artifacts/" \;
+    - find appsec/tests/integration/build/test-results -name "*.xml" -exec cp --parents '{}' "${CI_PROJECT_DIR}/artifacts/" \; || true
     - cp -r appsec/tests/integration/build/test-logs "${CI_PROJECT_DIR}/artifacts/" 2>/dev/null || true
     - .gitlab/silent-upload-junit-to-datadog.sh "test.source.file:appsec"
   artifacts:
     reports:
       junit: "artifacts/**/test-results/**/TEST-*.xml"
     paths:
       - "artifacts/"
-    when: "always"
+      - appsec/helper-rust/coverage-integration.lcov
+    when: always
   cache:
     - key: "appsec int test cache"
       paths:

.gitlab/generate-package.php

@@ -295,6 +295,24 @@
     paths:
       - "appsec_*"
 
+"compile appsec helper rust":
+  stage: appsec
+  image: "registry.ddbuild.io/images/mirror/datadog/dd-appsec-php-ci:nginx-fpm-php-8.5-release-musl"
+  tags: [ "arch:$ARCH" ]
+  needs: [ "prepare code" ]
+  parallel:
+    matrix:
+      - ARCH: ["amd64", "arm64" ]
+  variables:
+    MAKE_JOBS: 12
+    KUBERNETES_CPU_REQUEST: 12
+    KUBERNETES_MEMORY_REQUEST: 8Gi
+    KUBERNETES_MEMORY_LIMIT: 12Gi
+  script: .gitlab/build-appsec-helper-rust.sh
+  artifacts:
+    paths:
+      - "appsec_*"
+
 "pecl build":
   stage: tracing
   image: "registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:php-7.4_bookworm-6"
@@ -628,13 +646,20 @@
 }
 ?>
 
-    # Compile appsec helper
+    # Compile appsec helper (C++)
     - job: "compile appsec helper"
       parallel:
         matrix:
           - ARCH: "<?= $platform['arch'] ?>"
       artifacts: true
 
+    # Compile appsec helper (Rust)
+    - job: "compile appsec helper rust"
+      parallel:
+        matrix:
+          - ARCH: "<?= $platform['arch'] ?>"
+      artifacts: true
+
 <?php
     foreach ($profiler_minor_major_targets as $major_minor) {
 ?>
@@ -705,6 +730,11 @@
         matrix:
           - ARCH: "<?= $arch ?>"
       artifacts: true
+    - job: "compile appsec helper rust"
+      parallel:
+        matrix:
+          - ARCH: "<?= $arch ?>"
+      artifacts: true
     - job: "compile loader: [linux-gnu, <?= $arch ?>]"
       artifacts: true
     - job: "compile loader: [linux-musl, <?= $arch ?>]"

.gitmodules

@@ -14,3 +14,7 @@
 [submodule "tea/benchmarks/google-benchmark"]
 	path = tea/benchmarks/google-benchmark
 	url = https://github.com/google/benchmark.git
+[submodule "appsec/third_party/libddwaf-rust"]
+	path = appsec/third_party/libddwaf-rust
+	url = https://github.com/DataDog/libddwaf-rust.git
+	branch = glopes/v2

appsec/.dockerignore

@@ -0,0 +1,17 @@
+# Ignore everything by default
+*
+
+# Include only what's needed for helper-rust portable build
+!helper-rust/
+!third_party/libddwaf-rust/
+
+# Exclude build artifacts and caches from helper-rust
+helper-rust/target/
+helper-rust/.cargo/
+helper-rust/build/
+helper-rust/.idea/
+helper-rust/*.log
+
+# Exclude build artifacts from libddwaf-rust
+third_party/libddwaf-rust/target/
+third_party/libddwaf-rust/.cargo/