Skip to content

feat: add endpoint_url support to s3_sync for MinIO compatibility #589

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
koriyoshi2041 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: add endpoint_url support to s3_sync for MinIO compatibility #589

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion src/art/utils/s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ async def s3_sync(
destination: str,
*,
profile: Optional[str] = None,
endpoint_url: Optional[str] = None,
verbose: bool = False,
delete: bool = False,
exclude: list[ExcludableOption] | None = None,
Expand All @@ -84,17 +85,27 @@ async def s3_sync(
source: The *from* path. Can be a local path or an ``s3://`` URI.
destination: The *to* path. Can be a local path or an ``s3://`` URI.
profile: Optional AWS profile name to pass to the CLI.
endpoint_url: Optional custom endpoint URL for S3-compatible services
(e.g. MinIO). Falls back to the ``AWS_ENDPOINT_URL`` environment
variable when not provided.
verbose: When *True*, the output of the AWS CLI is streamed to the
calling process; otherwise it is suppressed.
delete: When *True*, delete files in *destination* that don't exist in
*source*.
exclude: List of directories to exclude from sync.

Raises:
S3SyncError: If the *aws s3 sync* command exits with a non‑zero status.
"""

if endpoint_url is None:
endpoint_url = os.environ.get("AWS_ENDPOINT_URL")

cmd: list[str] = ["aws"]
if profile:
cmd += ["--profile", profile]
if endpoint_url:
cmd += ["--endpoint-url", endpoint_url]

cmd += ["s3"]
# use cp for files, sync for directories
Expand Down Expand Up @@ -125,15 +136,22 @@ async def s3_sync(


async def ensure_bucket_exists(
s3_bucket: str | None = None, profile: str | None = None
s3_bucket: str | None = None,
profile: str | None = None,
endpoint_url: str | None = None,
) -> None:
if s3_bucket is None:
s3_bucket = os.environ["BACKUP_BUCKET"]

if endpoint_url is None:
endpoint_url = os.environ.get("AWS_ENDPOINT_URL")

# Check if bucket exists
cmd = ["aws"]
if profile:
cmd += ["--profile", profile]
if endpoint_url:
cmd += ["--endpoint-url", endpoint_url]
cmd += ["s3api", "head-bucket", "--bucket", s3_bucket]

result = await asyncio.create_subprocess_exec(*cmd, stdout=DEVNULL, stderr=DEVNULL)
Expand All @@ -147,6 +165,8 @@ async def ensure_bucket_exists(
cmd = ["aws"]
if profile:
cmd += ["--profile", profile]
if endpoint_url:
cmd += ["--endpoint-url", endpoint_url]
cmd += ["s3api", "create-bucket", "--bucket", s3_bucket]

result = await asyncio.create_subprocess_exec(*cmd)
Expand Down