diff --git a/.env.example b/.env.example index 707bbf6..eec05c5 100644 --- a/.env.example +++ b/.env.example @@ -1,9 +1,7 @@ -MINIO_ROOT_USER=minioadmin -MINIO_ROOT_PASSWORD=minioadmin -MINIO_ENDPOINT=minio:9000 -MINIO_PUBLIC_ENDPOINT=localhost:9000 -MINIO_USE_SSL=false -MINIO_ACCESS_KEY=minioadmin -MINIO_SECRET_KEY=minioadmin -MINIO_SESSION_TTL_SECONDS=3600 +S3_ENDPOINT=seaweedfs:8333 +S3_PUBLIC_ENDPOINT=localhost:8333 +S3_USE_SSL=false +S3_ACCESS_KEY=minioadmin +S3_SECRET_KEY=minioadmin +S3_SESSION_TTL_SECONDS=3600 CONVERSION_CLEANUP_DELAY_SECONDS=3600 diff --git a/Makefile b/Makefile index 8704a52..4cebf50 100644 --- a/Makefile +++ b/Makefile @@ -27,11 +27,13 @@ run-server: if [ -f .env ]; then . ./.env; fi; \ set +a; \ export PYTHONPATH="$${PYTHONPATH:-gen/python:python/packages/officeconvert/src:python/packages/server/src}"; \ - export MINIO_ENDPOINT="$${MINIO_ENDPOINT:-localhost:9000}"; \ - export MINIO_PUBLIC_ENDPOINT="$${MINIO_PUBLIC_ENDPOINT:-localhost:9000}"; \ - export MINIO_USE_SSL="$${MINIO_USE_SSL:-false}"; \ - export MINIO_ACCESS_KEY="$${MINIO_ACCESS_KEY:-minioadmin}"; \ - export MINIO_SECRET_KEY="$${MINIO_SECRET_KEY:-minioadmin}"; \ - export MINIO_SESSION_TTL_SECONDS="$${MINIO_SESSION_TTL_SECONDS:-3600}"; \ + if [ "$${S3_ENDPOINT:-}" = "seaweedfs:8333" ]; then S3_ENDPOINT=localhost:8333; fi; \ + if [ "$${S3_PUBLIC_ENDPOINT:-}" = "seaweedfs:8333" ]; then S3_PUBLIC_ENDPOINT=localhost:8333; fi; \ + export S3_ENDPOINT="$${S3_ENDPOINT:-localhost:8333}"; \ + export S3_PUBLIC_ENDPOINT="$${S3_PUBLIC_ENDPOINT:-localhost:8333}"; \ + export S3_USE_SSL="$${S3_USE_SSL:-false}"; \ + export S3_ACCESS_KEY="$${S3_ACCESS_KEY:-minioadmin}"; \ + export S3_SECRET_KEY="$${S3_SECRET_KEY:-minioadmin}"; \ + export S3_SESSION_TTL_SECONDS="$${S3_SESSION_TTL_SECONDS:-3600}"; \ export CONVERSION_CLEANUP_DELAY_SECONDS="$${CONVERSION_CLEANUP_DELAY_SECONDS:-3600}"; \ uv run --project python uvicorn officeconvert_server.app:app --host "$${UVICORN_HOST:-0.0.0.0}" --port "$${UVICORN_PORT:-8080}" diff --git a/README.md b/README.md index c3b939b..8489f78 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ and client compatibility. - `proto/` contains protobuf schemas and RPC definitions. - `gen/python` and `gen/go` contain generated protocol and Connect code. - `python/packages/officeconvert` is the core conversion library (PPTX -> PDF -> images + notes). -- `python/packages/server` is the ConnectRPC Python server with MinIO orchestration. +- `python/packages/server` is the ConnectRPC Python server with SeaweedFS (S3-compatible) orchestration. - `clients/go` is the first client library with layered orchestration helpers. - `deploy/` contains production-ish and dev Docker Compose files. @@ -29,14 +29,14 @@ Use the root `Makefile`: - `make buf-generate` to regenerate Go and Python types - `make py-sync` to sync Python workspace dependencies with uv - `make go-test` to run Go client tests -- `make compose-up` to run server + MinIO -- `make compose-up-dev` to run MinIO only +- `make compose-up` to run server + SeaweedFS +- `make compose-up-dev` to run SeaweedFS only - `make run-server` to start host `uvicorn` with `.env` (if present) plus defaults ## Development Server Workflow This is the recommended local workflow for iterating on the Python server and conversion -library while keeping MinIO in Docker. +library while keeping SeaweedFS in Docker. ### 1) Prerequisites @@ -64,7 +64,7 @@ From repo root: make py-sync ``` -### 4) Start MinIO dependency stack (dev compose) +### 4) Start SeaweedFS dependency stack (dev compose) From repo root: @@ -72,11 +72,12 @@ From repo root: make compose-up-dev ``` -MinIO endpoints: +SeaweedFS endpoints: -- API: `http://localhost:9000` -- Console: `http://localhost:9001` -- Default creds: `minioadmin` / `minioadmin` +- S3 API: `http://localhost:8333` +- Master API: `http://localhost:9333` +- Filer API: `http://localhost:8888` +- Default S3 creds: `minioadmin` / `minioadmin` ### 5) Start Connect server (host process) @@ -90,11 +91,10 @@ make run-server - loads `.env` automatically if present - applies reasonable defaults when values are not set -- defaults MinIO endpoint to `localhost:9000` for host-based development +- defaults S3 endpoint to `localhost:8333` for host-based development +- auto-normalizes `seaweedfs:8333` to `localhost:8333` for host runs - supports optional `UVICORN_HOST` and `UVICORN_PORT` overrides -If you copy from `.env.example`, set `MINIO_ENDPOINT=localhost:9000` for host mode. - Server endpoint base URL: - `http://localhost:8080` @@ -120,10 +120,16 @@ Then: ### 7) Full container workflow (optional) -If you want to run both server and MinIO in Docker: +If you want to run both server and SeaweedFS in Docker: ```bash make compose-up ``` Use `.env.example` as your baseline env configuration. + +## Storage Backend Notes + +- This project defaults to **SeaweedFS S3 API** for object transit in development and compose deployments. +- The Python server uses the `minio` Python SDK, which is intentional because SeaweedFS is S3-compatible. +- Runtime configuration uses `S3_*` environment variables. diff --git a/deploy/docker-compose.dev.yml b/deploy/docker-compose.dev.yml index 16c4a7b..9ae6cbc 100644 --- a/deploy/docker-compose.dev.yml +++ b/deploy/docker-compose.dev.yml @@ -1,15 +1,18 @@ services: - minio: - image: minio/minio:RELEASE.2026-02-17T00-53-00Z - command: server /data --console-address ":9001" + seaweedfs: + image: chrislusf/seaweedfs:latest + command: + - server + - -s3 environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin} + AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY:-minioadmin} + AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY:-minioadmin} ports: - - "9000:9000" - - "9001:9001" + - "8333:8333" + - "9333:9333" + - "8888:8888" volumes: - - minio_data:/data + - seaweedfs_data:/data volumes: - minio_data: + seaweedfs_data: diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index d64cf76..519176f 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -1,32 +1,35 @@ services: - minio: - image: minio/minio:RELEASE.2026-02-17T00-53-00Z - command: server /data --console-address ":9001" + seaweedfs: + image: chrislusf/seaweedfs:latest + command: + - server + - -s3 environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin} + AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY:-minioadmin} + AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY:-minioadmin} ports: - - "9000:9000" - - "9001:9001" + - "8333:8333" + - "9333:9333" + - "8888:8888" volumes: - - minio_data:/data + - seaweedfs_data:/data server: build: context: .. dockerfile: Dockerfile.server depends_on: - - minio + - seaweedfs environment: - MINIO_ENDPOINT: ${MINIO_ENDPOINT:-minio:9000} - MINIO_PUBLIC_ENDPOINT: ${MINIO_PUBLIC_ENDPOINT:-localhost:9000} - MINIO_USE_SSL: ${MINIO_USE_SSL:-false} - MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-minioadmin} - MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin} - MINIO_SESSION_TTL_SECONDS: ${MINIO_SESSION_TTL_SECONDS:-3600} + S3_ENDPOINT: ${S3_ENDPOINT:-seaweedfs:8333} + S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-localhost:8333} + S3_USE_SSL: ${S3_USE_SSL:-false} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:-minioadmin} + S3_SECRET_KEY: ${S3_SECRET_KEY:-minioadmin} + S3_SESSION_TTL_SECONDS: ${S3_SESSION_TTL_SECONDS:-3600} CONVERSION_CLEANUP_DELAY_SECONDS: ${CONVERSION_CLEANUP_DELAY_SECONDS:-3600} ports: - "8080:8080" volumes: - minio_data: + seaweedfs_data: diff --git a/python/packages/server/pyproject.toml b/python/packages/server/pyproject.toml index a5c47ee..d54c89f 100644 --- a/python/packages/server/pyproject.toml +++ b/python/packages/server/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "officeconvert-server" version = "0.1.0" -description = "ConnectRPC server orchestrating file conversions with MinIO." +description = "ConnectRPC server orchestrating file conversions with S3-compatible storage." readme = "../../../README.md" requires-python = ">=3.12" dependencies = [ diff --git a/python/packages/server/src/officeconvert_server/app.py b/python/packages/server/src/officeconvert_server/app.py index 51f8b32..c4bfb09 100644 --- a/python/packages/server/src/officeconvert_server/app.py +++ b/python/packages/server/src/officeconvert_server/app.py @@ -6,18 +6,18 @@ from officeconvertapi.v1.conversion_connect import ConversionServiceASGIApplicat from officeconvert_server.config import load_server_config from officeconvert_server.service import ConversionServiceImpl -from officeconvert_server.storage import MinIOStore +from officeconvert_server.storage import S3Store def create_app() -> ConversionServiceASGIApplication: """Construct and return the configured Connect ASGI application.""" config = load_server_config() - store = MinIOStore( - endpoint=config.minio_endpoint, - access_key=config.minio_access_key, - secret_key=config.minio_secret_key, - secure=config.minio_secure, - public_endpoint=config.minio_public_endpoint, + store = S3Store( + endpoint=config.s3_endpoint, + access_key=config.s3_access_key, + secret_key=config.s3_secret_key, + secure=config.s3_secure, + public_endpoint=config.s3_public_endpoint, ) service = ConversionServiceImpl(config=config, store=store) return ConversionServiceASGIApplication(service) diff --git a/python/packages/server/src/officeconvert_server/config.py b/python/packages/server/src/officeconvert_server/config.py index c9d56c5..67f4548 100644 --- a/python/packages/server/src/officeconvert_server/config.py +++ b/python/packages/server/src/officeconvert_server/config.py @@ -10,24 +10,24 @@ import os class ServerConfig: """Defines environment-driven settings for server orchestration.""" - minio_endpoint: str - minio_access_key: str - minio_secret_key: str - minio_secure: bool - minio_public_endpoint: str - minio_session_ttl_seconds: int + s3_endpoint: str + s3_access_key: str + s3_secret_key: str + s3_secure: bool + s3_public_endpoint: str + s3_session_ttl_seconds: int conversion_cleanup_delay_seconds: int def load_server_config() -> ServerConfig: """Load server configuration from environment variables.""" return ServerConfig( - minio_endpoint=os.getenv("MINIO_ENDPOINT", "localhost:9000"), - minio_access_key=os.getenv("MINIO_ACCESS_KEY", "minioadmin"), - minio_secret_key=os.getenv("MINIO_SECRET_KEY", "minioadmin"), - minio_secure=os.getenv("MINIO_USE_SSL", "false").lower() == "true", - minio_public_endpoint=os.getenv("MINIO_PUBLIC_ENDPOINT", "localhost:9000"), - minio_session_ttl_seconds=int(os.getenv("MINIO_SESSION_TTL_SECONDS", "3600")), + s3_endpoint=os.getenv("S3_ENDPOINT", "localhost:8333"), + s3_access_key=os.getenv("S3_ACCESS_KEY", "minioadmin"), + s3_secret_key=os.getenv("S3_SECRET_KEY", "minioadmin"), + s3_secure=os.getenv("S3_USE_SSL", "false").lower() == "true", + s3_public_endpoint=os.getenv("S3_PUBLIC_ENDPOINT", "localhost:8333"), + s3_session_ttl_seconds=int(os.getenv("S3_SESSION_TTL_SECONDS", "3600")), conversion_cleanup_delay_seconds=int( os.getenv("CONVERSION_CLEANUP_DELAY_SECONDS", "3600") ), diff --git a/python/packages/server/src/officeconvert_server/service.py b/python/packages/server/src/officeconvert_server/service.py index daaeb5d..2f24b88 100644 --- a/python/packages/server/src/officeconvert_server/service.py +++ b/python/packages/server/src/officeconvert_server/service.py @@ -18,13 +18,13 @@ from officeconvertapi.v1 import conversion_connect, conversion_pb2 from officeconvert_server.config import ServerConfig from officeconvert_server.models import ConversionSession, utc_now -from officeconvert_server.storage import MinIOStore +from officeconvert_server.storage import S3Store class ConversionServiceImpl(conversion_connect.ConversionService): - """Implements the conversion API with in-memory state and MinIO orchestration.""" + """Implements the conversion API with in-memory state and S3 orchestration.""" - def __init__(self, config: ServerConfig, store: MinIOStore) -> None: + def __init__(self, config: ServerConfig, store: S3Store) -> None: """Initialize service with runtime config and storage adapter.""" self._config = config self._store = store @@ -47,13 +47,13 @@ class ConversionServiceImpl(conversion_connect.ConversionService): conversion_id = str(uuid.uuid4()) bucket_name = f"oc-{conversion_id}" upload_key = "input/source.pptx" - expires_at = utc_now() + timedelta(seconds=self._config.minio_session_ttl_seconds) + expires_at = utc_now() + timedelta(seconds=self._config.s3_session_ttl_seconds) self._store.ensure_bucket(bucket_name) upload_url = self._store.presigned_put_url( bucket_name, upload_key, - ttl_seconds=self._config.minio_session_ttl_seconds, + ttl_seconds=self._config.s3_session_ttl_seconds, ) session = ConversionSession( @@ -143,7 +143,7 @@ class ConversionServiceImpl(conversion_connect.ConversionService): request: conversion_pb2.DeleteConversionRequest, ctx: RequestContext, ) -> conversion_pb2.DeleteConversionResponse: - """Delete a conversion session and associated MinIO/local artifacts.""" + """Delete a conversion session and associated object storage/local artifacts.""" del ctx async with self._lock: session = self._sessions.pop(request.conversion_id, None) @@ -218,7 +218,7 @@ class ConversionServiceImpl(conversion_connect.ConversionService): image_url = self._store.presigned_get_url( session.bucket_name, object_key, - ttl_seconds=self._config.minio_session_ttl_seconds, + ttl_seconds=self._config.s3_session_ttl_seconds, ) response_slides.append( conversion_pb2.Slide( diff --git a/python/packages/server/src/officeconvert_server/storage.py b/python/packages/server/src/officeconvert_server/storage.py index 2f0a940..1bdd215 100644 --- a/python/packages/server/src/officeconvert_server/storage.py +++ b/python/packages/server/src/officeconvert_server/storage.py @@ -1,4 +1,4 @@ -"""MinIO helper abstraction for upload and artifact lifecycle.""" +"""S3-compatible storage helper abstraction for upload and artifact lifecycle.""" from __future__ import annotations @@ -11,8 +11,8 @@ from minio.deleteobjects import DeleteObject from minio.error import S3Error -class MinIOStore: - """Provides typed helper methods around MinIO object storage operations.""" +class S3Store: + """Provides typed helper methods around S3-compatible object storage operations.""" def __init__( self, @@ -23,7 +23,7 @@ class MinIOStore: secure: bool, public_endpoint: str, ) -> None: - """Initialize MinIO clients for internal and public URL generation.""" + """Initialize S3 clients for internal and public URL generation.""" self._client = Minio( endpoint, access_key=access_key, @@ -59,12 +59,12 @@ class MinIOStore: ) def fget_object(self, bucket_name: str, object_key: str, output_path: Path) -> None: - """Download one object from MinIO to a local filesystem path.""" + """Download one object from storage to a local filesystem path.""" output_path.parent.mkdir(parents=True, exist_ok=True) self._client.fget_object(bucket_name, object_key, str(output_path)) def fput_object(self, bucket_name: str, object_key: str, source_path: Path) -> None: - """Upload one local filesystem object to MinIO.""" + """Upload one local filesystem object to storage.""" self._client.fput_object(bucket_name, object_key, str(source_path)) def remove_bucket_tree(self, bucket_name: str) -> None: