diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md index a926f57..15948e7 100644 --- a/.github/CODE_OF_CONDUCT.md +++ b/.github/CODE_OF_CONDUCT.md @@ -125,4 +125,4 @@ enforcement ladder](https://github.com/mozilla/diversity). For answers to common questions about this code of conduct, see the FAQ at https://www.contributor-covenant.org/faq. Translations are available at -https://www.contributor-covenant.org/translations. \ No newline at end of file +https://www.contributor-covenant.org/translations. diff --git a/.github/assets/templates/agent.yml b/.github/assets/templates/agent.yml index 3a3cb1c..d72c0c7 100644 --- a/.github/assets/templates/agent.yml +++ b/.github/assets/templates/agent.yml @@ -15,11 +15,7 @@ services: networks: - portabase -{{EXTRA_SERVICES}} - -{{EXTRA_VOLUMES}} - networks: portabase: name: portabase_network - external: true \ No newline at end of file + external: true diff --git a/.github/assets/templates/dashboard.yml b/.github/assets/templates/dashboard.yml index 1b527b5..8286e18 100644 --- a/.github/assets/templates/dashboard.yml +++ b/.github/assets/templates/dashboard.yml @@ -1,35 +1,35 @@ name: ${PROJECT_NAME} services: - portabase: - container_name: ${PROJECT_NAME}-app - image: portabase/portabase:latest - env_file: - - .env - ports: - - "${HOST_PORT}:80" - environment: - - TIME_ZONE=Europe/Paris - volumes: - - portabase-data:/data - depends_on: - db: - condition: service_healthy - db: - container_name: ${PROJECT_NAME}-pg - image: postgres:17-alpine - ports: - - "${PG_PORT}:5432" - volumes: - - postgres-data:/var/lib/postgresql/data - environment: - - POSTGRES_DB=${POSTGRES_DB} - - POSTGRES_USER=${POSTGRES_USER} - - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] - interval: 10s - timeout: 5s - retries: 5 + app: + container_name: ${PROJECT_NAME}-app + image: portabase/portabase:latest + env_file: + - .env + ports: + - "${HOST_PORT}:80" + environment: + - TIME_ZONE=Europe/Paris + volumes: + - portabase-data:/data + depends_on: + db: + condition: service_healthy + db: + container_name: ${PROJECT_NAME}-pg + image: postgres:17-alpine + ports: + - "${PG_PORT}:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + environment: + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 10s + timeout: 5s + retries: 5 volumes: - postgres-data: - portabase-data: + postgres-data: + portabase-data: diff --git a/CITATION.cff b/CITATION.cff index f7e1759..9dceb85 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -22,5 +22,5 @@ keywords: - management - integration license: Apache-2.0 -version: 26.02.17 -date-released: "2026-02-17" \ No newline at end of file +version: 26.03.2 +date-released: "2026-03-25" \ No newline at end of file diff --git a/commands/agent.py b/commands/agent.py index 1b0fd1f..d214016 100644 --- a/commands/agent.py +++ b/commands/agent.py @@ -5,12 +5,17 @@ from typing import Optional import typer +import yaml from rich.panel import Panel from rich.prompt import Confirm, IntPrompt, Prompt from rich.table import Table from core.config import add_db_to_json, load_db_config, write_env_file, write_file -from core.docker import ensure_network, run_compose +from core.docker import ( + create_compose_file, + ensure_network, + run_compose, +) from core.network import fetch_template from core.utils import ( check_system, @@ -25,6 +30,10 @@ AGENT_MONGODB_AUTH_SNIPPET, AGENT_MONGODB_SNIPPET, AGENT_POSTGRES_SNIPPET, + AGENT_REDIS_AUTH_SNIPPET, + AGENT_REDIS_SNIPPET, + AGENT_VALKEY_AUTH_SNIPPET, + AGENT_VALKEY_SNIPPET, ) @@ -33,8 +42,6 @@ def agent( key: Optional[str] = typer.Option(None, "--key", "-k", help="Edge Key"), tz: str = typer.Option("UTC", "--tz", help="Timezone"), polling: int = typer.Option(5, "--polling", help="Polling frequency in seconds"), - env: str = typer.Option("production", "--env", help="Application environment"), - data_path: str = typer.Option("/data", "--data-path", help="Internal data path"), start: bool = typer.Option(False, "--start", "-s", help="Start immediately"), ): print_banner() @@ -65,31 +72,19 @@ def agent( if polling == 5: polling = IntPrompt.ask("Polling frequency (seconds)", default=5) - if env == "production": - env = Prompt.ask( - "Environment", - choices=["production", "staging", "development"], - default="production", - ) - - if data_path == "/data": - data_path = Prompt.ask("Internal Data Path", default="/data") - - raw_template = fetch_template("agent.yml") - env_vars = { "EDGE_KEY": key, "PROJECT_NAME": project_name, "TZ": tz, "POLLING": str(polling), - "APP_ENV": env, - "DATA_PATH": data_path, } - extra_services = "" - extra_volumes = "volumes:\n" + raw_template = fetch_template("agent.yml") + raw_template = raw_template.replace("${PROJECT_NAME}", project_name) + + services_to_add = {} + volumes_to_add = [] app_volumes = ["./databases.json:/config/config.json"] - volumes_list = [] json_path = path / "databases.json" if not json_path.exists(): @@ -107,6 +102,24 @@ def agent( "Configuration Mode", choices=["new", "existing"], default="new" ) + table = Table( + title="Supported Databases", show_header=True, header_style="bold magenta" + ) + table.add_column("Type", style="cyan") + table.add_column("Engine", style="green") + table.add_column("Description", style="dim") + + table.add_row("SQL", "postgresql", "PostgreSQL Database") + table.add_row("SQL", "mysql", "MySQL Database") + table.add_row("SQL", "mariadb", "MariaDB Database") + table.add_row("SQL", "sqlite", "SQLite Database") + table.add_row("", "", "") + table.add_row("NoSQL", "mongodb", "MongoDB NoSQL") + table.add_row("NoSQL", "redis", "Redis Key-Value Store") + table.add_row("NoSQL", "valkey", "Valkey Key-Value Store") + + console.print(table) + if mode == "existing": console.print("[info]External/Existing Database Configuration[/info]") category = Prompt.ask("Category", choices=["SQL", "NoSQL"], default="SQL") @@ -120,7 +133,7 @@ def agent( else: db_type = Prompt.ask( "Type", - choices=["mongodb"], + choices=["mongodb", "redis", "valkey"], default="mongodb", ) @@ -146,12 +159,16 @@ def agent( else: db_name = Prompt.ask("Database Name") host = Prompt.ask("Host", default="localhost") - port = IntPrompt.ask( - "Port", - default=5432 - if db_type == "postgresql" - else (3306 if db_type in ["mysql", "mariadb"] else 27017), - ) + + default_port = 5432 + if db_type in ["mysql", "mariadb"]: + default_port = 3306 + elif db_type == "mongodb": + default_port = 27017 + elif db_type in ["redis", "valkey"]: + default_port = 6379 + + port = IntPrompt.ask("Port", default=default_port) user = Prompt.ask("Username") password = Prompt.ask("Password", password=True) @@ -184,12 +201,14 @@ def agent( else: db_engine = Prompt.ask( "Engine", - choices=["mongodb"], + choices=["mongodb", "redis", "valkey"], default="mongodb", ) db_variant = Prompt.ask( - "Type", choices=["standard", "with-auth"], default="standard" + "Variant", choices=["standard", "with-auth"], default="standard" ) + if db_variant == "with-auth": + db_engine = f"{db_engine}-auth" if db_engine == "sqlite": db_name = Prompt.ask("Database Name", default="local") @@ -231,8 +250,13 @@ def agent( .replace("${PASSWORD}", f"${{{var_prefix}_PASS}}") ) - extra_services += snippet - volumes_list.append(f"{service_name}-data") + snippet_data = yaml.safe_load(snippet) + services_to_add.update( + snippet_data.get("services", snippet_data) + if isinstance(snippet_data, dict) + else snippet_data + ) + volumes_to_add.append(f"{service_name}-data") add_db_to_json( path, @@ -273,8 +297,13 @@ def agent( .replace("${PASSWORD}", f"${{{var_prefix}_PASS}}") ) - extra_services += snippet - volumes_list.append(f"{service_name}-data") + snippet_data = yaml.safe_load(snippet) + services_to_add.update( + snippet_data.get("services", snippet_data) + if isinstance(snippet_data, dict) + else snippet_data + ) + volumes_to_add.append(f"{service_name}-data") add_db_to_json( path, @@ -318,8 +347,13 @@ def agent( .replace("${PASSWORD}", f"${{{var_prefix}_PASS}}") ) - extra_services += snippet - volumes_list.append(f"{service_name}-data") + snippet_data = yaml.safe_load(snippet) + services_to_add.update( + snippet_data.get("services", snippet_data) + if isinstance(snippet_data, dict) + else snippet_data + ) + volumes_to_add.append(f"{service_name}-data") add_db_to_json( path, @@ -352,8 +386,14 @@ def agent( .replace("${VOL_NAME}", f"{service_name}-data") .replace("${DB_NAME}", f"${{{var_prefix}_DB}}") ) - extra_services += snippet - volumes_list.append(f"{service_name}-data") + + snippet_data = yaml.safe_load(snippet) + services_to_add.update( + snippet_data.get("services", snippet_data) + if isinstance(snippet_data, dict) + else snippet_data + ) + volumes_to_add.append(f"{service_name}-data") add_db_to_json( path, @@ -372,18 +412,167 @@ def agent( f"[success]✔ Added MongoDB container (Port {mongo_port})[/success]" ) - if volumes_list: - for vol in volumes_list: - extra_volumes += f" {vol}:\n" + elif db_engine == "redis": + redis_port = get_free_port() + db_name = f"redis_{secrets.token_hex(4)}" + service_name = f"db-redis-{secrets.token_hex(2)}" + + var_prefix = service_name.upper().replace("-", "_") + env_vars[f"{var_prefix}_PORT"] = str(redis_port) - final_compose = raw_template.replace("{{EXTRA_SERVICES}}", extra_services) - final_compose = final_compose.replace("{{EXTRA_VOLUMES}}", extra_volumes) - final_compose = final_compose.replace("${PROJECT_NAME}", project_name) + snippet = ( + AGENT_REDIS_SNIPPET.replace("${SERVICE_NAME}", service_name) + .replace("${PORT}", f"${{{var_prefix}_PORT}}") + .replace("${VOL_NAME}", f"{service_name}-data") + ) - vols_str = "\n".join([f" - {v}" for v in app_volumes]) - final_compose = final_compose.replace( - " - ./databases.json:/config/config.json", vols_str - ) + snippet_data = yaml.safe_load(snippet) + services_to_add.update( + snippet_data.get("services", snippet_data) + if isinstance(snippet_data, dict) + else snippet_data + ) + volumes_to_add.append(f"{service_name}-data") + + add_db_to_json( + path, + { + "name": db_name, + "database": "0", + "type": "redis", + "username": "", + "password": "", + "port": redis_port, + "host": "localhost", + "generated_id": str(uuid.uuid4()), + }, + ) + console.print( + f"[success]✔ Added Redis container (Port {redis_port})[/success]" + ) + + elif db_engine == "redis-auth": + redis_port = get_free_port() + db_name = f"redis_{secrets.token_hex(4)}" + service_name = f"db-redis-auth-{secrets.token_hex(2)}" + db_pass = secrets.token_hex(8) + + var_prefix = service_name.upper().replace("-", "_") + env_vars[f"{var_prefix}_PORT"] = str(redis_port) + env_vars[f"{var_prefix}_PASS"] = db_pass + + snippet = ( + AGENT_REDIS_AUTH_SNIPPET.replace("${SERVICE_NAME}", service_name) + .replace("${PORT}", f"${{{var_prefix}_PORT}}") + .replace("${VOL_NAME}", f"{service_name}-data") + .replace("${PASSWORD}", f"${{{var_prefix}_PASS}}") + ) + + snippet_data = yaml.safe_load(snippet) + services_to_add.update( + snippet_data.get("services", snippet_data) + if isinstance(snippet_data, dict) + else snippet_data + ) + volumes_to_add.append(f"{service_name}-data") + + add_db_to_json( + path, + { + "name": db_name, + "database": "0", + "type": "redis", + "username": "", + "password": db_pass, + "port": redis_port, + "host": "localhost", + "generated_id": str(uuid.uuid4()), + }, + ) + console.print( + f"[success]✔ Added Redis Auth container (Port {redis_port})[/success]" + ) + + elif db_engine == "valkey": + valkey_port = get_free_port() + db_name = f"valkey_{secrets.token_hex(4)}" + service_name = f"db-valkey-{secrets.token_hex(2)}" + + var_prefix = service_name.upper().replace("-", "_") + env_vars[f"{var_prefix}_PORT"] = str(valkey_port) + + snippet = ( + AGENT_VALKEY_SNIPPET.replace("${SERVICE_NAME}", service_name) + .replace("${PORT}", f"${{{var_prefix}_PORT}}") + .replace("${VOL_NAME}", f"{service_name}-data") + ) + + snippet_data = yaml.safe_load(snippet) + services_to_add.update( + snippet_data.get("services", snippet_data) + if isinstance(snippet_data, dict) + else snippet_data + ) + volumes_to_add.append(f"{service_name}-data") + + add_db_to_json( + path, + { + "name": db_name, + "database": "0", + "type": "valkey", + "username": "", + "password": "", + "port": valkey_port, + "host": "localhost", + "generated_id": str(uuid.uuid4()), + }, + ) + console.print( + f"[success]✔ Added Valkey container (Port {valkey_port})[/success]" + ) + + elif db_engine == "valkey-auth": + valkey_port = get_free_port() + db_name = f"valkey_{secrets.token_hex(4)}" + service_name = f"db-valkey-auth-{secrets.token_hex(2)}" + db_pass = secrets.token_hex(8) + + var_prefix = service_name.upper().replace("-", "_") + env_vars[f"{var_prefix}_PORT"] = str(valkey_port) + env_vars[f"{var_prefix}_PASS"] = db_pass + + snippet = ( + AGENT_VALKEY_AUTH_SNIPPET.replace("${SERVICE_NAME}", service_name) + .replace("${PORT}", f"${{{var_prefix}_PORT}}") + .replace("${VOL_NAME}", f"{service_name}-data") + .replace("${PASSWORD}", f"${{{var_prefix}_PASS}}") + ) + + snippet_data = yaml.safe_load(snippet) + services_to_add.update( + snippet_data.get("services", snippet_data) + if isinstance(snippet_data, dict) + else snippet_data + ) + volumes_to_add.append(f"{service_name}-data") + + add_db_to_json( + path, + { + "name": db_name, + "database": "0", + "type": "valkey", + "username": "", + "password": db_pass, + "port": valkey_port, + "host": "localhost", + "generated_id": str(uuid.uuid4()), + }, + ) + console.print( + f"[success]✔ Added Valkey Auth container (Port {valkey_port})[/success]" + ) summary = Table(show_header=False, box=None, padding=(0, 2)) summary.add_column("Property", style="bold cyan") @@ -395,7 +584,6 @@ def agent( summary.add_row("Edge Key", f"{key[:10]}...{key[-10:]}" if len(key) > 20 else key) summary.add_row("Timezone", tz) summary.add_row("Polling", f"{polling}s") - summary.add_row("Environment", env) db_config = load_db_config(path) dbs = db_config.get("databases", []) @@ -419,7 +607,7 @@ def agent( console.print( Panel( summary, - title="[bold white]PROPOSED CONFIGURATION[/bold white]", + title="[bold white]SUMMARY[/bold white]", border_style="bold blue", expand=False, ) @@ -434,7 +622,14 @@ def agent( console.print("[warning]Configuration cancelled.[/warning]") raise typer.Exit() - write_file(path / "docker-compose.yml", final_compose) + create_compose_file( + path=path, + base_template_content=raw_template, + extra_services=services_to_add, + named_volumes=volumes_to_add, + app_volumes=app_volumes, + ) + write_env_file(path, env_vars) console.print( diff --git a/commands/common.py b/commands/common.py index a456844..a92b907 100644 --- a/commands/common.py +++ b/commands/common.py @@ -49,6 +49,7 @@ def uninstall( path: Path = typer.Argument(..., help="Path to component folder"), force: bool = typer.Option(False, "--force", "-f") ): + """Uninstall and delete a Portabase component's folder and containers.""" path = path.resolve() validate_work_dir(path) diff --git a/commands/config.py b/commands/config.py index 7660f52..4e0cd86 100644 --- a/commands/config.py +++ b/commands/config.py @@ -3,7 +3,11 @@ from core.config import get_config_value, set_config_value from core.utils import console -app = typer.Typer(help="Manage global CLI configuration.") +app = typer.Typer( + help="Manage global CLI configuration.", + no_args_is_help=True, + rich_markup_mode="rich", +) @app.command() @@ -23,6 +27,7 @@ def channel( @app.command() def show(): + """Show the current CLI configuration.""" channel = get_config_value("update_channel", "auto (based on current version)") console.print(f"[info]Current Configuration:[/info]") console.print(f" [bold]Update Channel:[/bold] {channel}") diff --git a/commands/dashboard.py b/commands/dashboard.py index 4ddd863..80981ca 100644 --- a/commands/dashboard.py +++ b/commands/dashboard.py @@ -1,13 +1,13 @@ -import re import secrets from pathlib import Path import typer +import yaml from rich.panel import Panel from rich.prompt import Confirm, IntPrompt, Prompt from rich.table import Table -from core.config import write_env_file, write_file +from core.config import write_env_file from core.docker import run_compose from core.network import fetch_template from core.utils import ( @@ -37,6 +37,13 @@ def dashboard( project_name = name.lower().replace(" ", "-") raw_template = fetch_template("dashboard.yml") + raw_template = raw_template.replace("${PROJECT_NAME}", project_name) + + try: + compose_data = yaml.safe_load(raw_template) or {} + except yaml.YAMLError as exc: + console.print(f"[danger]Error parsing dashboard template : {exc}[/danger]") + raise typer.Exit(1) auth_secret = secrets.token_hex(32) base_url = f"http://localhost:{port}" @@ -48,13 +55,52 @@ def dashboard( "PROJECT_NAME": project_name, } + console.print() + console.print( + Panel( + "• [bold cyan]internal[/bold cyan] : Embedded database\n" + "• [bold cyan]compose[/bold cyan] : Dedicated local PostgreSQL container\n" + "• [bold cyan]external[/bold cyan] : Existing external database", + title="[bold white]Database Mode Selection[/bold white]", + border_style="cyan", + expand=False, + padding=(1, 2), + ) + ) + console.print() + mode = Prompt.ask( - "Database Setup", choices=["internal", "external"], default="internal" + "Database System Setup", + choices=["internal", "compose", "external"], + default="internal", ) if mode == "internal": + console.print("[info]Using embedded internal database.[/info]") + + if "services" in compose_data and "db" in compose_data["services"]: + del compose_data["services"]["db"] + + if "services" in compose_data and "app" in compose_data["services"]: + if "depends_on" in compose_data["services"]["app"]: + depends_on_data = compose_data["services"]["app"]["depends_on"] + + if isinstance(depends_on_data, dict) and "db" in depends_on_data: + del compose_data["services"]["app"]["depends_on"]["db"] + elif isinstance(depends_on_data, list) and "db" in depends_on_data: + compose_data["services"]["app"]["depends_on"].remove("db") + + if not compose_data["services"]["app"]["depends_on"]: + del compose_data["services"]["app"]["depends_on"] + + if "volumes" in compose_data and "postgres-data" in compose_data["volumes"]: + del compose_data["volumes"]["postgres-data"] + + elif mode == "compose": + console.print("[info]Creating a dedicated local PostgreSQL container.[/info]") pg_port = get_free_port() pg_pass = secrets.token_hex(16) + env_vars.update( { "POSTGRES_DB": "portabase", @@ -65,7 +111,7 @@ def dashboard( "PG_PORT": str(pg_port), } ) - final_compose = raw_template.replace("${PROJECT_NAME}", project_name) + else: console.print("[info]External Database Configuration[/info]") db_host = Prompt.ask("Host", default="localhost") @@ -85,14 +131,23 @@ def dashboard( } ) - final_compose = re.sub( - r"[ ]{8}depends_on:.*?service_healthy\n", "", raw_template, flags=re.DOTALL - ) - final_compose = re.sub( - r"[ ]{4}db:.*?retries: 5\n", "", final_compose, flags=re.DOTALL - ) - final_compose = re.sub(r"[ ]{4}postgres-data:\n", "", final_compose) - final_compose = final_compose.replace("${PROJECT_NAME}", project_name) + if "services" in compose_data and "db" in compose_data["services"]: + del compose_data["services"]["db"] + + if "services" in compose_data and "app" in compose_data["services"]: + if "depends_on" in compose_data["services"]["app"]: + depends_on_data = compose_data["services"]["app"]["depends_on"] + + if isinstance(depends_on_data, dict) and "db" in depends_on_data: + del compose_data["services"]["app"]["depends_on"]["db"] + elif isinstance(depends_on_data, list) and "db" in depends_on_data: + compose_data["services"]["app"]["depends_on"].remove("db") + + if not compose_data["services"]["app"]["depends_on"]: + del compose_data["services"]["app"]["depends_on"] + + if "volumes" in compose_data and "postgres-data" in compose_data["volumes"]: + del compose_data["volumes"]["postgres-data"] summary = Table(show_header=False, box=None, padding=(0, 2)) summary.add_column("Property", style="bold cyan") @@ -101,18 +156,18 @@ def dashboard( summary.add_row("Dashboard Name", name) summary.add_row("Path", str(path)) summary.add_row("Access URL", f"[bold green]http://localhost:{port}[/bold green]") - summary.add_row( - "Database Setup", - "All-in-one (Internal Docker DB)" - if mode == "internal" - else "Custom (External Database)", - ) if mode == "internal": + summary.add_row("Database Setup", "Embedded database") + elif mode == "compose": + summary.add_row("Database Setup", "Dedicated Local") summary.add_row("Internal Port", env_vars["PG_PORT"]) else: + summary.add_row("Database Setup", "External Database") summary.add_row("DB Host", env_vars["POSTGRES_HOST"]) summary.add_row("DB Name", env_vars["POSTGRES_DB"]) + import re + masked_url = re.sub(r":.*?@", ":****@", env_vars["DATABASE_URL"]) summary.add_row("Connection URL", f"[dim]{masked_url}[/dim]") @@ -122,7 +177,7 @@ def dashboard( console.print( Panel( summary, - title="[bold white]PROPOSED CONFIGURATION[/bold white]", + title="[bold white]SUMMARY[/bold white]", border_style="bold blue", expand=False, ) @@ -137,14 +192,17 @@ def dashboard( console.print("[warning]Configuration cancelled.[/warning]") raise typer.Exit() - write_file(path / "docker-compose.yml", final_compose) + with open(path / "docker-compose.yml", "w") as f: + yaml.safe_dump(compose_data, f, default_flow_style=False, sort_keys=False) + write_env_file(path, env_vars) - db_info = ( - f"\n[dim]DB Port: {env_vars.get('PG_PORT')}[/dim]" - if mode == "internal" - else f"\n[dim]External DB: {env_vars.get('POSTGRES_HOST')}[/dim]" - ) + db_info = "" + if mode == "compose": + db_info = f"\n[dim]DB Port: {env_vars.get('PG_PORT')}[/dim]" + elif mode == "external": + db_info = f"\n[dim]External DB: {env_vars.get('POSTGRES_HOST')}[/dim]" + console.print( Panel( f"[bold white]DASHBOARD CREATED: {name}[/bold white]\n[dim]Path: {path}[/dim]{db_info}", diff --git a/commands/db.py b/commands/db.py index 3ef9b8e..a107af3 100644 --- a/commands/db.py +++ b/commands/db.py @@ -3,20 +3,30 @@ from pathlib import Path import typer +import yaml from rich.panel import Panel from rich.prompt import IntPrompt, Prompt from rich.table import Table from core.config import add_db_to_json, load_db_config, save_db_config, write_env_file +from core.docker import run_compose, update_compose_file from core.utils import console, get_free_port, validate_work_dir from templates.compose import ( AGENT_MARIADB_SNIPPET, AGENT_MONGODB_AUTH_SNIPPET, AGENT_MONGODB_SNIPPET, AGENT_POSTGRES_SNIPPET, + AGENT_REDIS_AUTH_SNIPPET, + AGENT_REDIS_SNIPPET, + AGENT_VALKEY_AUTH_SNIPPET, + AGENT_VALKEY_SNIPPET, ) -app = typer.Typer(help="Manage databases configuration.") +app = typer.Typer( + help="Manage databases configuration.", + no_args_is_help=True, + rich_markup_mode="rich", +) @app.command("list") @@ -64,11 +74,30 @@ def add_db(name: str = typer.Argument(..., help="Name of the agent")): path = Path(name).resolve() validate_work_dir(path) - console.print(Panel("Add Database to Agent", style="bold blue")) + console.print(Panel("Add Database Connection", style="bold blue")) mode = Prompt.ask( "Configuration Mode", choices=["new", "existing"], default="existing" ) + + table = Table( + title="Supported Databases", show_header=True, header_style="bold magenta" + ) + table.add_column("Type", style="cyan") + table.add_column("Engine", style="green") + table.add_column("Description", style="dim") + + table.add_row("SQL", "postgresql", "PostgreSQL Database") + table.add_row("SQL", "mysql", "MySQL Database") + table.add_row("SQL", "mariadb", "MariaDB Database") + table.add_row("SQL", "sqlite", "SQLite Database") + table.add_row("", "", "") + table.add_row("NoSQL", "mongodb", "MongoDB NoSQL") + table.add_row("NoSQL", "redis", "Redis Key-Value Store") + table.add_row("NoSQL", "valkey", "Valkey Key-Value Store") + + console.print(table) + category = Prompt.ask("Category", choices=["SQL", "NoSQL"], default="SQL") if mode == "existing": @@ -79,7 +108,11 @@ def add_db(name: str = typer.Argument(..., help="Name of the agent")): default="postgresql", ) else: - db_type = Prompt.ask("Type", choices=["mongodb"], default="mongodb") + db_type = Prompt.ask( + "Type", + choices=["mongodb", "redis", "valkey"], + default="mongodb", + ) friendly_name = Prompt.ask("Display Name", default="External DB") @@ -94,12 +127,16 @@ def add_db(name: str = typer.Argument(..., help="Name of the agent")): else: db_name = Prompt.ask("Database Name") host = Prompt.ask("Host", default="localhost") - port = IntPrompt.ask( - "Port", - default=5432 - if db_type == "postgresql" - else (3306 if db_type in ["mysql", "mariadb"] else 27017), - ) + + default_port = 5432 + if db_type in ["mysql", "mariadb"]: + default_port = 3306 + elif db_type == "mongodb": + default_port = 27017 + elif db_type in ["redis", "valkey"]: + default_port = 6379 + + port = IntPrompt.ask("Port", default=default_port) user = Prompt.ask("Username") password = Prompt.ask("Password", password=True) @@ -122,11 +159,18 @@ def add_db(name: str = typer.Argument(..., help="Name of the agent")): choices=["postgresql", "mysql", "mariadb", "sqlite"], default="postgresql", ) + db_variant = "standard" else: - db_engine = Prompt.ask("Engine", choices=["mongodb"], default="mongodb") + db_engine = Prompt.ask( + "Engine", + choices=["mongodb", "redis", "valkey"], + default="mongodb", + ) db_variant = Prompt.ask( - "Type", choices=["standard", "with-auth"], default="standard" + "Variant", choices=["standard", "with-auth"], default="standard" ) + if db_variant == "with-auth": + db_engine = f"{db_engine}-auth" env_vars = {} snippet = "" @@ -143,25 +187,32 @@ def add_db(name: str = typer.Argument(..., help="Name of the agent")): compose_path = path / "docker-compose.yml" if compose_path.exists(): - with open(compose_path, "r") as f: - lines = f.readlines() - - new_lines = [] - in_app_service = False - in_volumes = False - for line in lines: - new_lines.append(line) - if "app:" in line: - in_app_service = True - if in_app_service and "volumes:" in line: - in_volumes = True - if in_volumes and "- ./databases.json" in line: - new_lines.append(f" - ./{db_name}:/config/{db_name}\n") - in_volumes = False - in_app_service = False - - with open(compose_path, "w") as f: - f.writelines(new_lines) + try: + with open(compose_path, "r") as f: + data = yaml.safe_load(f) or {} + + if "services" in data and "app" in data["services"]: + app_service = data["services"]["app"] + + if ( + "volumes" not in app_service + or app_service["volumes"] is None + ): + app_service["volumes"] = [] + + new_volume = f"./{db_name}:/config/{db_name}" + if new_volume not in app_service["volumes"]: + app_service["volumes"].append(new_volume) + + with open(compose_path, "w") as f: + yaml.safe_dump( + data, f, default_flow_style=False, sort_keys=False + ) + except Exception as e: + console.print( + f"[danger]Error while updating the SQLite database : {e}[/danger]" + ) + raise typer.Exit(1) add_db_to_json( path, @@ -246,48 +297,72 @@ def add_db(name: str = typer.Argument(..., help="Name of the agent")): .replace("${DB_NAME}", f"${{{var_prefix}_DB}}") ) - compose_path = path / "docker-compose.yml" - if compose_path.exists(): - with open(compose_path, "r") as f: - content = f.read() - - insert_pos = content.find("networks:") - if insert_pos == -1: - insert_pos = len(content) - - new_content = content[:insert_pos] + snippet + "\n" + content[insert_pos:] - - vol_snippet = f" {service_name}-data:\n" - vol_pos = new_content.find("volumes:") - if vol_pos != -1: - end_of_volumes = new_content.find("networks:", vol_pos) - if end_of_volumes == -1: - end_of_volumes = len(new_content) - new_content = ( - new_content[:end_of_volumes] - + vol_snippet - + new_content[end_of_volumes:] + elif "redis" in db_engine: + db_port = get_free_port() + db_name = f"redis_{secrets.token_hex(4)}" + service_name = f"db-redis-{'auth-' if 'auth' in db_engine else ''}{secrets.token_hex(2)}" + var_prefix = service_name.upper().replace("-", "_") + env_vars[f"{var_prefix}_PORT"] = str(db_port) + + if "auth" in db_engine: + db_pass = secrets.token_hex(8) + env_vars[f"{var_prefix}_PASS"] = db_pass + snippet = ( + AGENT_REDIS_AUTH_SNIPPET.replace("${SERVICE_NAME}", service_name) + .replace("${PORT}", f"${{{var_prefix}_PORT}}") + .replace("${VOL_NAME}", f"{service_name}-data") + .replace("${PASSWORD}", f"${{{var_prefix}_PASS}}") ) else: - new_content += f"\nvolumes:\n{vol_snippet}" + snippet = ( + AGENT_REDIS_SNIPPET.replace("${SERVICE_NAME}", service_name) + .replace("${PORT}", f"${{{var_prefix}_PORT}}") + .replace("${VOL_NAME}", f"{service_name}-data") + ) - with open(compose_path, "w") as f: - f.write(new_content) + elif "valkey" in db_engine: + db_port = get_free_port() + db_name = f"valkey_{secrets.token_hex(4)}" + service_name = f"db-valkey-{'auth-' if 'auth' in db_engine else ''}{secrets.token_hex(2)}" + var_prefix = service_name.upper().replace("-", "_") + env_vars[f"{var_prefix}_PORT"] = str(db_port) - write_env_file(path, env_vars) - add_db_to_json( - path, - { - "name": db_name, - "database": db_name, - "type": db_engine, - "username": db_user, - "password": db_pass, - "port": db_port, - "host": "localhost", - "generated_id": str(uuid.uuid4()), - }, - ) + if "auth" in db_engine: + db_pass = secrets.token_hex(8) + env_vars[f"{var_prefix}_PASS"] = db_pass + snippet = ( + AGENT_VALKEY_AUTH_SNIPPET.replace("${SERVICE_NAME}", service_name) + .replace("${PORT}", f"${{{var_prefix}_PORT}}") + .replace("${VOL_NAME}", f"{service_name}-data") + .replace("${PASSWORD}", f"${{{var_prefix}_PASS}}") + ) + else: + snippet = ( + AGENT_VALKEY_SNIPPET.replace("${SERVICE_NAME}", service_name) + .replace("${PORT}", f"${{{var_prefix}_PORT}}") + .replace("${VOL_NAME}", f"{service_name}-data") + ) + + if snippet: + update_compose_file(path, snippet, f"{service_name}-data") + write_env_file(path, env_vars) + + db_type_for_json = db_engine.split("-")[0] + add_db_to_json( + path, + { + "name": db_name, + "database": "0" + if db_type_for_json in ["redis", "valkey"] + else db_name, + "type": db_type_for_json, + "username": db_user, + "password": db_pass, + "port": db_port, + "host": "localhost", + "generated_id": str(uuid.uuid4()), + }, + ) console.print("[success]✔ Database added to configuration.[/success]") console.print( @@ -298,6 +373,7 @@ def add_db(name: str = typer.Argument(..., help="Name of the agent")): @app.command("remove") def remove_db(name: str = typer.Argument(..., help="Name of the agent")): + """Remove a database connection from configuration.""" path = Path(name).resolve() validate_work_dir(path) diff --git a/core/config.py b/core/config.py index 6f47004..81dfcbf 100644 --- a/core/config.py +++ b/core/config.py @@ -3,15 +3,19 @@ import uuid from pathlib import Path -TEMPLATE_BASE_URL = "https://s3.eu-central-3.ionoscloud.com/portabase-software/cli/public/templates" +TEMPLATE_BASE_URL = ( + "https://s3.eu-central-3.ionoscloud.com/portabase-software/cli/public/templates" +) GLOBAL_CONFIG_DIR = Path.home() / ".portabase" GLOBAL_CONFIG_FILE = GLOBAL_CONFIG_DIR / "config.json" + def write_file(path: Path, content: str): path.parent.mkdir(parents=True, exist_ok=True) with open(path, "w") as f: f.write(content) + def write_env_file(work_dir: Path, env_vars: dict): existing = {} env_path = work_dir / ".env" @@ -21,13 +25,14 @@ def write_env_file(work_dir: Path, env_vars: dict): if "=" in line: k, v = line.strip().split("=", 1) existing[k] = v.strip('"') - + existing.update(env_vars) content = "" for k, v in existing.items(): content += f'{k}="{v}"\n' write_file(env_path, content) + def load_global_config() -> dict: if not GLOBAL_CONFIG_FILE.exists(): return {} @@ -37,20 +42,24 @@ def load_global_config() -> dict: except: return {} + def save_global_config(config: dict): GLOBAL_CONFIG_DIR.mkdir(parents=True, exist_ok=True) with open(GLOBAL_CONFIG_FILE, "w") as f: json.dump(config, f, indent=2) + def get_config_value(key: str, default=None): config = load_global_config() return config.get(key, default) + def set_config_value(key: str, value): config = load_global_config() config[key] = value save_global_config(config) + def load_db_config(path: Path) -> dict: json_path = path / "databases.json" if not json_path.exists(): @@ -61,6 +70,7 @@ def load_db_config(path: Path) -> dict: except: return {"databases": []} + def save_db_config(path: Path, config: dict): json_path = path / "databases.json" with open(json_path, "w") as f: @@ -70,13 +80,14 @@ def save_db_config(path: Path, config: dict): except: pass + def add_db_to_json(path: Path, db_entry: dict): config = load_db_config(path) if "databases" not in config: config["databases"] = [] - + if "generated_id" not in db_entry: db_entry["generated_id"] = str(uuid.uuid4()) - + config["databases"].append(db_entry) - save_db_config(path, config) \ No newline at end of file + save_db_config(path, config) diff --git a/core/docker.py b/core/docker.py index 14ff0fb..9119d88 100644 --- a/core/docker.py +++ b/core/docker.py @@ -1,13 +1,26 @@ import subprocess +from pathlib import Path +from typing import Dict, List, Optional + import typer +import yaml + from core.utils import console -from pathlib import Path + def ensure_network(name: str): try: - subprocess.run(["docker", "network", "inspect", name], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True) + subprocess.run( + ["docker", "network", "inspect", name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True, + ) except subprocess.CalledProcessError: - subprocess.run(["docker", "network", "create", name], stdout=subprocess.DEVNULL, check=True) + subprocess.run( + ["docker", "network", "create", name], stdout=subprocess.DEVNULL, check=True + ) + def run_compose(cwd: Path, args: list): try: @@ -16,4 +29,81 @@ def run_compose(cwd: Path, args: list): subprocess.run(cmd, cwd=cwd, check=True) except subprocess.CalledProcessError: console.print("[danger]Command failed.[/danger]") - raise typer.Exit(1) \ No newline at end of file + raise typer.Exit(1) + + +def update_compose_file(path: Path, service_snippet: str, volume_name: str = None): + compose_path = path / "docker-compose.yml" + if not compose_path.exists(): + return + + try: + with open(compose_path, "r") as f: + data = yaml.safe_load(f) or {} + + snippet_data = yaml.safe_load(service_snippet) + + if "services" not in data or data["services"] is None: + data["services"] = {} + + data["services"].update(snippet_data) + + if volume_name: + if "volumes" not in data or data["volumes"] is None: + data["volumes"] = {} + + if volume_name not in data["volumes"]: + data["volumes"][volume_name] = {} + + with open(compose_path, "w") as f: + yaml.safe_dump(data, f, default_flow_style=False, sort_keys=False) + + except yaml.YAMLError as exc: + console.print(f"[danger]Error while loading YAML : {exc}[/danger]") + raise typer.Exit(1) + except Exception as e: + console.print(f"[danger]Unexpected error : {e}[/danger]") + raise typer.Exit(1) + + +def create_compose_file( + path: Path, + base_template_content: str, + extra_services: Optional[Dict] = None, + named_volumes: Optional[List[str]] = None, + app_volumes: Optional[List[str]] = None, +): + compose_path = path / "docker-compose.yml" + + try: + data = yaml.safe_load(base_template_content) or {} + + if extra_services: + if "services" not in data or data["services"] is None: + data["services"] = {} + data["services"].update(extra_services) + + if named_volumes: + if "volumes" not in data or data["volumes"] is None: + data["volumes"] = {} + for vol in named_volumes: + data["volumes"][vol] = {} + + if app_volumes and "services" in data and "app" in data["services"]: + app_service = data["services"]["app"] + if "volumes" not in app_service or app_service["volumes"] is None: + app_service["volumes"] = [] + + for vol in app_volumes: + if vol not in app_service["volumes"]: + app_service["volumes"].append(vol) + + with open(compose_path, "w") as f: + yaml.safe_dump(data, f, default_flow_style=False, sort_keys=False) + + except yaml.YAMLError as exc: + console.print(f"[danger]Error while creating YAML : {exc}[/danger]") + raise typer.Exit(1) + except Exception as e: + console.print(f"[danger]Unexpected error : {e}[/danger]") + raise typer.Exit(1) diff --git a/core/utils.py b/core/utils.py index bde0210..9930015 100644 --- a/core/utils.py +++ b/core/utils.py @@ -29,7 +29,7 @@ HINTS = [ "The Edge Key contains the connection details for dashboard and agent communication.", - "Portabase uses Docker Compose to isolate your databases.", + "Portabase uses Docker Compose or k8s to isolate your databases.", "You can list all configured databases using 'portabase db list '.", "Running 'portabase stop' will gracefully shut down your containers.", "The agent polls the github for configuration updates.", diff --git a/main.py b/main.py index c054b64..165aff6 100644 --- a/main.py +++ b/main.py @@ -6,7 +6,11 @@ from core.updater import check_for_updates, update_cli from core.utils import console, current_version -app = typer.Typer(no_args_is_help=True, add_completion=False) +app = typer.Typer( + help="Portabase CLI - Manage your Portabase components.", + no_args_is_help=True, + add_completion=False, +) def version_callback(value: bool): @@ -19,7 +23,7 @@ def version_callback(value: bool): @app.callback() def main( ctx: typer.Context, - version: Optional[bool] = typer.Option( + _: Optional[bool] = typer.Option( None, "--version", help="Show the version and exit.", @@ -27,25 +31,56 @@ def main( is_eager=True, ), ): + """ + Portabase CLI to manage agents, dashboards and databases. + """ if ctx.invoked_subcommand != "update": check_for_updates() -@app.command() +@app.command(help="Update the CLI to the latest version.", rich_help_panel="System") def update(): update_cli() -app.command()(agent.agent) -app.command()(dashboard.dashboard) -app.command()(common.start) -app.command()(common.stop) -app.command()(common.restart) -app.command()(common.logs) -app.command()(common.uninstall) +app.command( + help="Create a new Portabase Agent instance.", + rich_help_panel="Creation", + no_args_is_help=True, +)(agent.agent) +app.command( + help="Create a new Portabase Dashboard instance.", + rich_help_panel="Creation", + no_args_is_help=True, +)(dashboard.dashboard) +app.command( + help="Start a Portabase component.", + rich_help_panel="Lifecycle", + no_args_is_help=True, +)(common.start) +app.command( + help="Stop a Portabase component.", + rich_help_panel="Lifecycle", + no_args_is_help=True, +)(common.stop) +app.command( + help="Restart a Portabase component.", + rich_help_panel="Lifecycle", + no_args_is_help=True, +)(common.restart) +app.command( + help="View logs of a Portabase component.", + rich_help_panel="Lifecycle", + no_args_is_help=True, +)(common.logs) +app.command( + help="Uninstall and delete a Portabase component.", + rich_help_panel="Lifecycle", + no_args_is_help=True, +)(common.uninstall) -app.add_typer(db.app, name="db") -app.add_typer(config.app, name="config") +app.add_typer(db.app, name="db", rich_help_panel="Configuration") +app.add_typer(config.app, name="config", rich_help_panel="Configuration") if __name__ == "__main__": app() diff --git a/pyproject.toml b/pyproject.toml index 9b8d350..cea1341 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "portabase-cli" -version = "26.02.17" +version = "26.03.2" description = "The official command line interface (CLI) for managing and deploying Portabase instances with ease." readme = "README.md" requires-python = ">=3.12" @@ -8,5 +8,6 @@ dependencies = [ "rich>=14.2.0", "typer>=0.20.0", "pyinstaller>=6.17.0", - "requests>=2.32.5" + "requests>=2.32.5", + "pyyaml>=6.0.3" ] diff --git a/templates/compose.py b/templates/compose.py index 03eaa43..132662c 100644 --- a/templates/compose.py +++ b/templates/compose.py @@ -57,5 +57,61 @@ - ${VOL_NAME}:/data/db """ +AGENT_REDIS_SNIPPET = """ + ${SERVICE_NAME}: + image: redis:latest + container_name: ${PROJECT_NAME}-${SERVICE_NAME} + ports: + - "${PORT}:6379" + volumes: + - ${VOL_NAME}:/data + command: [ "redis-server", "--appendonly", "yes" ] + networks: + - portabase + - default +""" +AGENT_REDIS_AUTH_SNIPPET = """ + ${SERVICE_NAME}: + image: redis:latest + container_name: ${PROJECT_NAME}-${SERVICE_NAME} + ports: + - "${PORT}:6379" + volumes: + - ${VOL_NAME}:/data + environment: + - REDIS_PASSWORD=${PASSWORD} + command: [ "redis-server", "--requirepass", "${PASSWORD}", "--appendonly", "yes" ] + networks: + - portabase + - default +""" +AGENT_VALKEY_SNIPPET = """ + ${SERVICE_NAME}: + image: valkey/valkey:latest + container_name: ${PROJECT_NAME}-${SERVICE_NAME} + environment: + - ALLOW_EMPTY_PASSWORD=yes + ports: + - "${PORT}:6379" + volumes: + - ${VOL_NAME}:/data + networks: + - portabase + - default +""" + +AGENT_VALKEY_AUTH_SNIPPET = """ + ${SERVICE_NAME}: + image: valkey/valkey:latest + container_name: ${PROJECT_NAME}-${SERVICE_NAME} + command: --requirepass "${PASSWORD}" + ports: + - "${PORT}:6379" + volumes: + - ${VOL_NAME}:/data + networks: + - portabase + - default +"""