chore: OTLP final wrap-up

.github/workflows/core-build-checks.yml

@@ -16,37 +16,7 @@ jobs:
 
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
-
-    - name: Install Rust toolchain
-      uses: dtolnay/rust-toolchain@master
-      with:
-        toolchain: nightly
-        components: rust-src
-
-    - name: Install BPF dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y \
-          clang \
-          llvm \
-          libelf-dev \
-          libpcap-dev \
-          build-essential \
-          libbpf-dev \
-          linux-tools-generic \
-          linux-tools-common \
-          protobuf-compiler
-
-    - name: Install bindgen-cli and bpf-linker
-      run: |
-        cargo install bindgen-cli
-        cargo install bpf-linker
-
-    - name: Setup bpftool symlink
-      run: |
-        sudo ln -sf /usr/lib/linux-tools/*/bpftool /usr/local/bin/bpftool || \
-        sudo ln -sf /usr/lib/linux-tools-*/bpftool /usr/local/bin/bpftool
-
+
     - name: Build CortexFlow Agent
       run: |
         cd core
@@ -57,18 +27,18 @@ jobs:
 
     - name: Build CortexFlow Identity
       run: |
-        cd core/src/components/identity
-        chmod +x build-identity.sh
+        cd core
+        chmod +x identity-build.sh
         echo "🚀 Starting CortexFlow Identity build..."
-        ./build-identity.sh || { echo "❌ Identity build failed"; exit 1; }
+        ./identity-build.sh || { echo "❌ Identity build failed"; exit 1; }
         echo "✅ Identity build completed"
-        
+
     - name: Build CortexFlow Metrics
       run: |
-        cd core/src/components/metrics
-        chmod +x build-metrics.sh
+        cd core
+        chmod +x metrics-build.sh
         echo "🚀 Starting CortexFlow Metrics build..."
-        ./build-metrics.sh || { echo "❌ Metrics build failed"; exit 1; }
+        ./metrics-build.sh || { echo "❌ Metrics build failed"; exit 1; }
         echo "✅ Metrics build completed"
 
     - name: Verify Docker images were built
@@ -108,4 +78,4 @@ jobs:
 
     - name: Cleanup build artifacts
       run: |
-        docker system prune -f
+        docker system prune -f

build-all.sh

@@ -1,24 +1,19 @@
 #!/bin/bash
-set -e
+set -euo pipefail
 
-echo "Building CortexFlow Agent"
-pushd ./core
-./agent-api-build.sh
-popd
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+CORE_DIR="$ROOT_DIR/core"
 
-sleep 1
+echo "Building CortexFlow Agent"
+"$CORE_DIR/agent-api-build.sh"
 
+echo ""
 echo "Building CortexFlow Identity"
-pushd ./core/src/components/identity
-./build-identity.sh
-popd
-
-sleep 1
+"$CORE_DIR/src/components/identity/build-identity.sh"
 
+echo ""
 echo "Building CortexFlow Metrics"
-pushd ./core/src/components/metrics
-./build-metrics.sh
-popd
+"$CORE_DIR/src/components/metrics/build-metrics.sh"
 
 sleep 1
 

core/agent-api-build.sh

@@ -1,16 +1,8 @@
 #!/bin/bash
+set -euo pipefail
 
-echo "Building the conntracker files"
-pushd src/components/conntracker
-./build-conntracker.sh
-popd
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
-echo "Copying connection tracker binaries"
-cp -r target/bpfel-unknown-none/release/conntracker conntracker
-
-# Run docker build
+echo "Building cortexflow-agent image from core workspace context"
+cd "$SCRIPT_DIR"
 docker build -f api/Dockerfile -t cortexflow-agent:0.0.1 --provenance=false --sbom=false .
-
-# Cleanup
-echo "Cleaning building files"
-rm -rf conntracker

core/api/Dockerfile

@@ -1,7 +1,39 @@
-# Phase 1: Build image
-FROM rust:1.90 AS builder
+# ==============================================================================
+# PHASE 1: Build the eBPF bytecode (Kernel-space Connection Tracker)
+# ==============================================================================
+FROM rust:1.90 AS ebpf-builder
 
-# Install system dependencies including protoc
+RUN apt-get update && apt-get install -y \
+    clang \
+    libclang-dev \
+    llvm \
+    bpftool \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN cargo install --locked bindgen-cli
+RUN cargo install --locked bpf-linker
+RUN rustup toolchain install nightly --component rust-src
+
+WORKDIR /usr/src/app
+COPY . .
+
+# Generate bindings from local Linux VM host kernel definitions
+RUN bpftool btf dump file /sys/kernel/btf/vmlinux format c > src/components/conntracker/vmlinux.h \
+    && bindgen src/components/conntracker/vmlinux.h \
+        -o src/components/conntracker/src/bindings.rs \
+        --use-core \
+        --allowlist-type 'sk_buff'
+
+# Compile the raw eBPF connection tracker target
+RUN cargo +nightly build -Z build-std=core --target bpfel-unknown-none --release -p conntracker
+
+
+# ==============================================================================
+# PHASE 2: Build the Userspace Binaries (agent-api & cortexflow_identity)
+# ==============================================================================
+FROM rust:1.90 AS app-builder
+
+# Install system dependencies including protoc for gRPC/Protobuf generation
 RUN apt-get update && apt-get install -y \
     build-essential \
     libprotobuf-dev \
@@ -16,19 +48,18 @@ RUN apt-get update && apt-get install -y \
 ENV PROTOC=/usr/bin/protoc
 ENV PROTOC_INCLUDE=/usr/include
 
-# Set working directory
-WORKDIR /usr/src/app/agent
-
-# Copy Cargo manifest and sources
+WORKDIR /usr/src/app
 COPY . .
-COPY common ../common
 
-# Fetch dependencies and build release
-RUN cargo fetch 
+# Fetch dependencies and build release components
+RUN cargo fetch
 RUN cargo build -p cortexflow_agent_api --release
 RUN cargo build -p cortexflow_identity --release
 
-# Phase 2: Final minimal image
+
+# ==============================================================================
+# PHASE 3: Final Minimal Production Runtime Image
+# ==============================================================================
 FROM ubuntu:24.04
 
 # Install runtime dependencies
@@ -43,17 +74,17 @@ ENV PATH="/root/.cargo/bin:/usr/local/bin:${PATH}"
 # Create working directory
 WORKDIR /usr/src/cortexbrain-agent
 
-# Copy the compiled binary
-COPY --from=builder /usr/src/app/agent/target/release/agent-api /usr/local/bin/agent-api
-COPY --from=builder /usr/src/app/agent/target/release/cortexflow_identity /usr/local/bin/cortexflow_identity
+# 1. Copy the compiled binaries from Phase 2 (App Builder)
+COPY --from=app-builder /usr/src/app/target/release/agent-api /usr/local/bin/agent-api
+COPY --from=app-builder /usr/src/app/target/release/cortexflow_identity /usr/local/bin/cortexflow_identity
 
-# Copy configuration files
-COPY conntracker /usr/src/cortexbrain-agent/conntracker
+# 2. Copy ONLY the compiled raw eBPF binary from Phase 1 (eBPF Builder)
+COPY --from=ebpf-builder /usr/src/app/target/bpfel-unknown-none/release/conntracker /usr/src/cortexbrain-agent/conntracker
 
 # Set env vars for your app
 ENV BPF_PATH="/usr/src/cortexbrain-agent/conntracker"
 ENV PIN_MAP_PATH="/sys/fs/bpf/cortexbrain-identity-service/"
 ENV PIN_BLOCKLIST_MAP_PATH="/sys/fs/bpf/cortexbrain-agent/"
 
 # Default command
-CMD ["agent-api"]
+CMD ["agent-api"]

core/common/src/logger.rs

@@ -44,18 +44,67 @@ use opentelemetry_otlp::{LogExporter, WithExportConfig};
 use opentelemetry_sdk::Resource;
 use opentelemetry_sdk::logs::SdkLoggerProvider;
 
+const OTEL_SERVICE_NAME: &str = "OTEL_SERVICE_NAME";
+const OTEL_EXPORTER_OTLP_ENDPOINT: &str = "OTEL_EXPORTER_OTLP_ENDPOINT";
+const OTEL_EXPORTER_OTLP_PROTOCOL: &str = "OTEL_EXPORTER_OTLP_PROTOCOL";
+const DEFAULT_OTLP_GRPC_ENDPOINT: &str = "http://localhost:4317";
+const DEFAULT_OTLP_HTTP_ENDPOINT: &str = "http://localhost:4318";
+
+fn resolved_otlp_endpoint() -> String {
+    if let Ok(endpoint) = std::env::var(OTEL_EXPORTER_OTLP_ENDPOINT)
+        && !endpoint.trim().is_empty()
+    {
+        return endpoint;
+    }
+
+    let default = match std::env::var(OTEL_EXPORTER_OTLP_PROTOCOL)
+        .ok()
+        .map(|value| value.to_ascii_lowercase())
+        .as_deref()
+    {
+        Some("http/protobuf") | Some("http/json") => DEFAULT_OTLP_HTTP_ENDPOINT,
+        _ => DEFAULT_OTLP_GRPC_ENDPOINT,
+    };
+
+    default.to_string()
+}
+
+fn resolved_service_name(default_service_name: String) -> String {
+    match std::env::var(OTEL_SERVICE_NAME) {
+        Ok(service_name) if !service_name.trim().is_empty() => service_name,
+        _ => default_service_name,
+    }
+}
+
+fn resolved_otlp_protocol() -> String {
+    std::env::var(OTEL_EXPORTER_OTLP_PROTOCOL)
+        .ok()
+        .map(|value| value.to_ascii_lowercase())
+        .filter(|value| !value.trim().is_empty())
+        .unwrap_or_else(|| "grpc".to_string())
+}
+
 pub fn otlp_logger_init(service_name: String) -> SdkLoggerProvider {
-    //exporter and provider initialization
-    let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT")
-        .unwrap_or_else(|_| "http://localhost:4317".to_string());
+    // exporter and provider initialization
+    let otlp_endpoint = resolved_otlp_endpoint();
+    let otlp_protocol = resolved_otlp_protocol();
+
+    let exporter = match otlp_protocol.as_str() {
+        "http/protobuf" | "http/json" => LogExporter::builder()
+            .with_http()
+            .with_endpoint(otlp_endpoint)
+            .build()
+            .expect("Failed to create OTLP HTTP exporter"),
+        _ => LogExporter::builder()
+            .with_tonic()
+            .with_endpoint(otlp_endpoint)
+            .build()
+            .expect("Failed to create OTLP gRPC exporter"),
+    };
 
-    let exporter = LogExporter::builder()
-        .with_tonic()
-        .with_endpoint(otlp_endpoint)
-        .build()
-        .expect("Failed to create OTLP exporter");
+    // Resource::builder() automatically reads OTEL_RESOURCE_ATTRIBUTES.
+    let service_name = resolved_service_name(service_name);
 
-    //needs a service name
     let provider = SdkLoggerProvider::builder()
         .with_resource(Resource::builder().with_service_name(service_name).build())
         .with_batch_exporter(exporter)

core/identity-build.sh

@@ -0,0 +1,9 @@
+#! /bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+echo "Building cortexflow-identity image from core workspace context"
+cd "$SCRIPT_DIR"
+
+docker build -f src/components/identity/Dockerfile -t identity:0.0.1 --provenance=false --sbom=false .

core/metrics-build.sh

@@ -0,0 +1,9 @@
+#! /bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+echo "Building cortexflow-metrics image from core workspace context"
+cd "$SCRIPT_DIR"
+
+docker build -f src/components/metrics/Dockerfile -t metrics:0.0.1 --provenance=false --sbom=false .