From fe48ae77bc0da2aa6f81483a221b24ccf0883d58 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Thu, 14 May 2026 22:41:33 +0800 Subject: [PATCH] use managed redis and postgres in coolify compose --- .env.example | 2 +- AGENTS.md | 2 +- DEVELOPMENT.md | 9 +++-- ENVIRONMENT.md | 4 +-- README.md | 31 +++++++++--------- compose.local.yaml | 62 +++++++++++++++++++++++++++++++++++ compose.yaml | 69 +++++++-------------------------------- scripts/dev.sh | 10 +++++- scripts/docker-compose.sh | 10 ++++++ 9 files changed, 118 insertions(+), 81 deletions(-) diff --git a/.env.example b/.env.example index 9e0ce114..89714a1d 100644 --- a/.env.example +++ b/.env.example @@ -21,7 +21,7 @@ REDIS_HOST_BIND=127.0.0.1 POSTGRES_URL=postgresql://postgres:postgres@127.0.0.1:5432/workflows POSTGRES_DB=workflows POSTGRES_USER=postgres -# Change in production +# Change in production. URL-encode credentials if editing POSTGRES_URL directly. POSTGRES_PASSWORD=postgres POSTGRES_HOST_BIND=127.0.0.1 # Optional: expose postgres on a fixed host port for local debugging. diff --git a/AGENTS.md b/AGENTS.md index fc19d40c..54ca1615 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,7 +25,7 @@ This repo contains multiple services: - `apps/discord_bot`: Discord gateway and bot commands/cogs - `apps/api`: webhook ingest API and dashboard auth routes - `apps/worker`: queue consumer and processing jobs -- `compose.yaml`: canonical Coolify/base container stack with the external infra network; `compose.local.yaml` adds local host port publishing for `./scripts/docker-compose.sh`; `docker-compose.yml` is a compatibility wrapper; day-to-day dev should prefer host-run app services with Docker-managed infra +- `compose.yaml`: canonical Coolify/base app stack using managed Redis/Postgres URLs; `compose.local.yaml` adds local Redis/Postgres containers and host port publishing for `./scripts/docker-compose.sh`; `docker-compose.yml` is a compatibility wrapper; day-to-day dev should prefer host-run app services with Docker-managed infra 4. Human audit logging - Human-triggered CRM actions from Discord should write to the worker audit ingest endpoint. diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 91585bbb..2dbdb5d7 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -83,9 +83,9 @@ uv run --package five08 crmctl repl For day-to-day development, prefer `./scripts/dev.sh` plus host-run app services. That entrypoint exports deterministic per-worktree localhost ports and service URLs so the apps work without manual overrides. Use the Compose -wrapper when you need full container parity, including Coolify-style runs. +wrapper when you need a full local container stack. -Start full stack (discord_bot + api + worker + redis + postgres + minio): +Start the full local stack (discord_bot + web + worker + redis + postgres + minio): ```bash ./scripts/docker-compose.sh up --build @@ -108,6 +108,9 @@ Show the deterministic host ports assigned to the current worktree: Set `*_HOST_PORT` or `COMPOSE_PROJECT_NAME` in `.env` or the invoking shell if you need fixed values; otherwise the wrapper computes deterministic per-worktree ones. +The wrapper loads `compose.yaml` plus `compose.local.yaml`. Coolify should deploy +`compose.yaml` by itself and provide managed `REDIS_URL` and `POSTGRES_URL` +runtime variables. ## Testing and Quality @@ -192,7 +195,7 @@ contact.save() Use `.env.example` as source of truth. Key categories: -- Shared queue/runtime: `REDIS_URL`, `REDIS_QUEUE_NAME`, `POSTGRES_URL`, `JOB_MAX_ATTEMPTS`, `JOB_RETRY_BASE_SECONDS`, `JOB_RETRY_MAX_SECONDS`, `LOG_LEVEL`, webhook settings. Local defaults target host-run services; `docker-compose.yml` injects Docker-network URLs for containerized runs. +- Shared queue/runtime: `REDIS_URL`, `REDIS_QUEUE_NAME`, `POSTGRES_URL`, `JOB_MAX_ATTEMPTS`, `JOB_RETRY_BASE_SECONDS`, `JOB_RETRY_MAX_SECONDS`, `LOG_LEVEL`, webhook settings. Local defaults target host-run services; `compose.local.yaml` injects Docker-network Redis/Postgres URLs for local containerized runs. - Bot credentials/integrations: Discord, email, Espo, Kimai - Discord CRM audit writer: `AUDIT_API_BASE_URL`, `AUDIT_API_TIMEOUT_SECONDS` (plus shared `API_SHARED_SECRET`) - Worker controls: `WORKER_NAME`, `WORKER_QUEUE_NAMES`, `WORKER_BURST` diff --git a/ENVIRONMENT.md b/ENVIRONMENT.md index b29da591..da59107c 100644 --- a/ENVIRONMENT.md +++ b/ENVIRONMENT.md @@ -21,7 +21,7 @@ Use `.env.example` as the source of defaults. ## Queue + Job Runtime - `Optional`: `LOG_LEVEL` (default: `INFO`) -- `Optional`: `REDIS_URL` (default: `redis://127.0.0.1:6379/0`; `./scripts/dev.sh` overrides it to a deterministic per-worktree localhost port, Compose injects `redis://redis:6379/0`) +- `Optional`: `REDIS_URL` (default: `redis://127.0.0.1:6379/0`; `./scripts/dev.sh` overrides it to a deterministic per-worktree localhost port, the local Compose wrapper injects `redis://redis:6379/0`, and Coolify should provide the managed Redis URL) - `Optional`: `REDIS_QUEUE_NAME` (default: `jobs.default`) - `Optional`: `REDIS_KEY_PREFIX` (default: `jobs`) - `Optional`: `REDIS_HOST_BIND` (default: `127.0.0.1`) @@ -34,7 +34,7 @@ Use `.env.example` as the source of defaults. ## Postgres + Compose Exposure -- `Optional`: `POSTGRES_URL` (default: `postgresql://postgres:postgres@127.0.0.1:5432/workflows`; `./scripts/dev.sh` overrides it to a deterministic per-worktree localhost port, Compose injects a Docker-network URL) +- `Optional`: `POSTGRES_URL` (default: `postgresql://postgres:postgres@127.0.0.1:5432/workflows`; `./scripts/dev.sh` overrides it to a deterministic per-worktree localhost port, the local Compose wrapper injects a Docker-network URL, and Coolify should provide the managed Postgres URL) - `Optional` (Compose DB container): `POSTGRES_DB` (default: `workflows`) - `Optional` (Compose DB container): `POSTGRES_USER` (default: `postgres`) - `Optional` (Compose DB container): `POSTGRES_PASSWORD` (default: `postgres`) diff --git a/README.md b/README.md index b61e4628..9837b367 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,8 @@ This repository follows a service-oriented monorepo layout: ├── packages/ │ └── shared/ │ └── src/five08/ # Shared settings, queue helpers, shared clients -├── compose.yaml # canonical Coolify/base container stack -├── compose.local.yaml # local infra host port publishing override +├── compose.yaml # canonical Coolify/base app stack +├── compose.local.yaml # local Redis/Postgres and host port overlay ├── docker-compose.yml # compatibility wrapper including compose.yaml ├── tests/ # Unit and integration tests └── pyproject.toml # uv workspace root @@ -30,8 +30,8 @@ This repository follows a service-oriented monorepo layout: - `discord_bot`: Discord gateway process. - `web`: FastAPI dashboard + ingest service that validates and enqueues jobs. - `worker`: Dramatiq worker that executes jobs from Redis queue. -- `redis`: queue transport between API and worker. -- `postgres`: job state persistence, retries, idempotency. +- `redis`: queue transport between API and worker; managed by Coolify in production and provided by the local overlay for development. +- `postgres`: job state persistence, retries, idempotency; managed by Coolify in production and provided by the local overlay for development. - `minio`: internal S3-compatible storage transport. Migrations: @@ -140,7 +140,7 @@ uv run --package five08 crmctl batch-update --where timezone__is_null=true --whe `./scripts/dev.sh` exports deterministic per-worktree localhost ports and service URLs so the apps can run on the host without manual overrides. Use -the lower-level Compose wrapper when you want full containerized parity. +the lower-level Compose wrapper when you want a full local container stack. For local full-container runs, including deterministic localhost ports: @@ -148,14 +148,15 @@ For local full-container runs, including deterministic localhost ports: ./scripts/docker-compose.sh up --build ``` -Coolify should use `/compose.yaml` as the base Compose file. A small -`docker-compose.yml` compatibility wrapper includes it for tools still configured -to read the older filename. The `web` service publishes container port `8090` -to `${WEB_HOST_BIND:-127.0.0.1}:${WEB_HOST_PORT:-8090}` -so a host-side Cloudflare Tunnel can target the dashboard/API at localhost. -The base file does not publish Redis, Postgres, or MinIO host ports. The app -services also attach to the shared infra network named by `INFRA_DOCKER_NETWORK` -so they can reach +Coolify should use `/compose.yaml` as the base Compose file and provide managed +`REDIS_URL` and `POSTGRES_URL` runtime variables. A small `docker-compose.yml` +compatibility wrapper includes it for tools still configured to read the older +filename. Local full-container runs should use `./scripts/docker-compose.sh`, +which loads `compose.yaml` plus `compose.local.yaml` for Redis/Postgres and host +port publishing. +The base file does not define local Redis/Postgres containers or publish MinIO +host ports. The app services also attach to the shared infra network named by +`INFRA_DOCKER_NETWORK` so they can reach Portainer-managed Bifrost and Langfuse by Docker DNS. The network is declared as external, so pre-create it before running Compose if it does not already exist. @@ -206,7 +207,7 @@ Use `.env.example` as the source of truth for defaults. ### Queue + Job Runtime -- `Optional`: `REDIS_URL` (default: `redis://127.0.0.1:6379/0`; `./scripts/dev.sh` overrides it to a deterministic per-worktree localhost port, Compose injects `redis://redis:6379/0`) +- `Optional`: `REDIS_URL` (default: `redis://127.0.0.1:6379/0`; `./scripts/dev.sh` overrides it to a deterministic per-worktree localhost port, the local Compose wrapper injects `redis://redis:6379/0`, and Coolify should provide the managed Redis URL) - `Optional`: `REDIS_QUEUE_NAME` (default: `jobs.default`) - `Optional`: `REDIS_KEY_PREFIX` (default: `jobs`) - `Optional`: `REDIS_HOST_BIND` (default: `127.0.0.1`) @@ -219,7 +220,7 @@ Use `.env.example` as the source of truth for defaults. ### Postgres + Compose Exposure -- `Optional`: `POSTGRES_URL` (default: `postgresql://postgres:postgres@127.0.0.1:5432/workflows`; `./scripts/dev.sh` overrides it to a deterministic per-worktree localhost port, Compose injects a Docker-network URL) +- `Optional`: `POSTGRES_URL` (default: `postgresql://postgres:postgres@127.0.0.1:5432/workflows`; `./scripts/dev.sh` overrides it to a deterministic per-worktree localhost port, the local Compose wrapper injects a Docker-network URL, and Coolify should provide the managed Postgres URL) - `Optional` (Compose DB container): `POSTGRES_DB` (default: `workflows`) - `Optional` (Compose DB container): `POSTGRES_USER` (default: `postgres`) - `Optional` (Compose DB container): `POSTGRES_PASSWORD` (default: `postgres`) diff --git a/compose.local.yaml b/compose.local.yaml index 1b8ce11b..ac5b4768 100644 --- a/compose.local.yaml +++ b/compose.local.yaml @@ -1,13 +1,75 @@ services: redis: + image: redis:7-alpine + command: ["redis-server", "--save", "60", "1", "--loglevel", "warning"] + restart: unless-stopped ports: - "${REDIS_HOST_BIND:-127.0.0.1}:${REDIS_HOST_PORT:-6379}:6379" + volumes: + - redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 5 postgres: + image: postgres:17-alpine + environment: + POSTGRES_DB: ${POSTGRES_DB:-workflows} + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} + restart: unless-stopped ports: - "${POSTGRES_HOST_BIND:-127.0.0.1}:${POSTGRES_HOST_PORT:-5432}:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: + [ + "CMD-SHELL", + "pg_isready -U ${POSTGRES_USER:-postgres} -d ${POSTGRES_DB:-workflows}", + ] + interval: 10s + timeout: 3s + retries: 5 minio: ports: - "${MINIO_HOST_BIND:-127.0.0.1}:${MINIO_API_HOST_PORT:-9000}:9000" - "${MINIO_HOST_BIND:-127.0.0.1}:${MINIO_CONSOLE_HOST_PORT:-9001}:9001" + + discord_bot: + environment: + REDIS_URL: redis://redis:6379/0 + depends_on: + redis: + condition: service_healthy + + web: + environment: + REDIS_URL: redis://redis:6379/0 + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} + POSTGRES_DB: ${POSTGRES_DB:-workflows} + depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy + + worker: + environment: + REDIS_URL: redis://redis:6379/0 + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} + POSTGRES_DB: ${POSTGRES_DB:-workflows} + depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy + +volumes: + redis-data: + postgres-data: diff --git a/compose.yaml b/compose.yaml index 6af170eb..207f98be 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,35 +1,4 @@ services: - redis: - image: redis:7-alpine - command: ["redis-server", "--save", "60", "1", "--loglevel", "warning"] - restart: unless-stopped - volumes: - - redis-data:/data - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 3s - retries: 5 - - postgres: - image: postgres:17-alpine - environment: - POSTGRES_DB: ${POSTGRES_DB:-workflows} - POSTGRES_USER: ${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} - restart: unless-stopped - volumes: - - postgres-data:/var/lib/postgresql/data - healthcheck: - test: - [ - "CMD-SHELL", - "pg_isready -U ${POSTGRES_USER:-postgres} -d ${POSTGRES_DB:-workflows}", - ] - interval: 10s - timeout: 3s - retries: 5 - minio: image: cgr.dev/chainguard/minio command: ["server", "/data", "--console-address", ":9001"] @@ -64,18 +33,16 @@ services: context: . dockerfile: apps/discord_bot/Dockerfile env_file: - - .env + - path: .env + required: false environment: - REDIS_URL: redis://redis:6379/0 + REDIS_URL: ${REDIS_URL:?REDIS_URL is required} REDIS_QUEUE_NAME: ${REDIS_QUEUE_NAME:-jobs.default} BACKEND_API_BASE_URL: http://web:8090 restart: unless-stopped networks: - default - infra - depends_on: - redis: - condition: service_healthy web: build: @@ -83,14 +50,12 @@ services: dockerfile: apps/api/Dockerfile command: ["uv", "run", "--package", "api", "backend-api"] env_file: - - .env + - path: .env + required: false environment: - REDIS_URL: redis://redis:6379/0 + REDIS_URL: ${REDIS_URL:?REDIS_URL is required} REDIS_QUEUE_NAME: ${REDIS_QUEUE_NAME:-jobs.default} - POSTGRES_USER: ${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} - POSTGRES_DB: ${POSTGRES_DB:-workflows} - POSTGRES_URL: postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@postgres:5432/${POSTGRES_DB:-workflows} + POSTGRES_URL: ${POSTGRES_URL:?POSTGRES_URL is required} JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-8} JOB_RETRY_BASE_SECONDS: ${JOB_RETRY_BASE_SECONDS:-5} JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:-300} @@ -109,10 +74,6 @@ services: - default - infra depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy minio-init: condition: service_completed_successfully @@ -127,16 +88,14 @@ services: --threads ${WORKER_THREADS:-8} -Q ${WORKER_QUEUE_NAMES:-jobs.default} env_file: - - .env + - path: .env + required: false environment: - REDIS_URL: redis://redis:6379/0 + REDIS_URL: ${REDIS_URL:?REDIS_URL is required} REDIS_QUEUE_NAME: ${REDIS_QUEUE_NAME:-jobs.default} WORKER_API_BASE_URL: http://web:8090 WORKER_QUEUE_NAMES: ${WORKER_QUEUE_NAMES:-jobs.default} - POSTGRES_USER: ${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} - POSTGRES_DB: ${POSTGRES_DB:-workflows} - POSTGRES_URL: postgresql://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@postgres:5432/${POSTGRES_DB:-workflows} + POSTGRES_URL: ${POSTGRES_URL:?POSTGRES_URL is required} JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-8} JOB_RETRY_BASE_SECONDS: ${JOB_RETRY_BASE_SECONDS:-5} JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:-300} @@ -152,16 +111,10 @@ services: depends_on: web: condition: service_started - redis: - condition: service_healthy - postgres: - condition: service_healthy minio-init: condition: service_completed_successfully volumes: - redis-data: - postgres-data: minio-data: networks: diff --git a/scripts/dev.sh b/scripts/dev.sh index 83a74da0..d9a57110 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -5,10 +5,18 @@ script_dir=$(CDPATH= cd "$(dirname "$0")" && pwd) . "$script_dir/worktree-env.sh" worktree_env_load "$script_dir" +url_quote() { + python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$1" +} + +POSTGRES_USER_ENC=$(url_quote "$POSTGRES_USER") +POSTGRES_PASSWORD_ENC=$(url_quote "$POSTGRES_PASSWORD") +POSTGRES_DB_ENC=$(url_quote "$POSTGRES_DB") + # dev.sh owns host-run service URLs so every launched process shares the same # worktree-local infra and app ports. export REDIS_URL="redis://127.0.0.1:${REDIS_HOST_PORT}/0" -export POSTGRES_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@127.0.0.1:${POSTGRES_HOST_PORT}/${POSTGRES_DB}" +export POSTGRES_URL="postgresql://${POSTGRES_USER_ENC}:${POSTGRES_PASSWORD_ENC}@127.0.0.1:${POSTGRES_HOST_PORT}/${POSTGRES_DB_ENC}" export MINIO_ENDPOINT="http://127.0.0.1:${MINIO_API_HOST_PORT}" export BACKEND_API_BASE_URL="http://127.0.0.1:${WEB_PORT}" export WORKER_API_BASE_URL="http://127.0.0.1:${WEB_PORT}" diff --git a/scripts/docker-compose.sh b/scripts/docker-compose.sh index c2f7befa..568a95a9 100755 --- a/scripts/docker-compose.sh +++ b/scripts/docker-compose.sh @@ -6,6 +6,14 @@ script_dir=$(CDPATH= cd "$(dirname "$0")" && pwd) worktree_env_load "$script_dir" compose repo_root=$WORKTREE_ENV_REPO_ROOT +url_quote() { + python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$1" +} + +POSTGRES_USER_ENC=$(url_quote "$POSTGRES_USER") +POSTGRES_PASSWORD_ENC=$(url_quote "$POSTGRES_PASSWORD") +POSTGRES_DB_ENC=$(url_quote "$POSTGRES_DB") + export COMPOSE_PROJECT_NAME export REDIS_HOST_PORT export POSTGRES_HOST_PORT @@ -13,6 +21,8 @@ export WEB_HOST_PORT export WEBHOOK_INGEST_HOST_PORT export MINIO_API_HOST_PORT export MINIO_CONSOLE_HOST_PORT +export REDIS_URL="redis://redis:6379/0" +export POSTGRES_URL="postgresql://${POSTGRES_USER_ENC}:${POSTGRES_PASSWORD_ENC}@postgres:5432/${POSTGRES_DB_ENC}" # Host-run-only app ports must not leak into Compose interpolation, or the web # container can start on a high worktree port while peers still target :8090.