switch from minio to seaweedfs

This commit is contained in:
2026-03-26 16:57:48 -07:00
parent 56f4c345cb
commit bb5f8b8494
10 changed files with 97 additions and 85 deletions
+6 -8
View File
@@ -1,9 +1,7 @@
MINIO_ROOT_USER=minioadmin S3_ENDPOINT=seaweedfs:8333
MINIO_ROOT_PASSWORD=minioadmin S3_PUBLIC_ENDPOINT=localhost:8333
MINIO_ENDPOINT=minio:9000 S3_USE_SSL=false
MINIO_PUBLIC_ENDPOINT=localhost:9000 S3_ACCESS_KEY=minioadmin
MINIO_USE_SSL=false S3_SECRET_KEY=minioadmin
MINIO_ACCESS_KEY=minioadmin S3_SESSION_TTL_SECONDS=3600
MINIO_SECRET_KEY=minioadmin
MINIO_SESSION_TTL_SECONDS=3600
CONVERSION_CLEANUP_DELAY_SECONDS=3600 CONVERSION_CLEANUP_DELAY_SECONDS=3600
+8 -6
View File
@@ -27,11 +27,13 @@ run-server:
if [ -f .env ]; then . ./.env; fi; \ if [ -f .env ]; then . ./.env; fi; \
set +a; \ set +a; \
export PYTHONPATH="$${PYTHONPATH:-gen/python:python/packages/officeconvert/src:python/packages/server/src}"; \ export PYTHONPATH="$${PYTHONPATH:-gen/python:python/packages/officeconvert/src:python/packages/server/src}"; \
export MINIO_ENDPOINT="$${MINIO_ENDPOINT:-localhost:9000}"; \ if [ "$${S3_ENDPOINT:-}" = "seaweedfs:8333" ]; then S3_ENDPOINT=localhost:8333; fi; \
export MINIO_PUBLIC_ENDPOINT="$${MINIO_PUBLIC_ENDPOINT:-localhost:9000}"; \ if [ "$${S3_PUBLIC_ENDPOINT:-}" = "seaweedfs:8333" ]; then S3_PUBLIC_ENDPOINT=localhost:8333; fi; \
export MINIO_USE_SSL="$${MINIO_USE_SSL:-false}"; \ export S3_ENDPOINT="$${S3_ENDPOINT:-localhost:8333}"; \
export MINIO_ACCESS_KEY="$${MINIO_ACCESS_KEY:-minioadmin}"; \ export S3_PUBLIC_ENDPOINT="$${S3_PUBLIC_ENDPOINT:-localhost:8333}"; \
export MINIO_SECRET_KEY="$${MINIO_SECRET_KEY:-minioadmin}"; \ export S3_USE_SSL="$${S3_USE_SSL:-false}"; \
export MINIO_SESSION_TTL_SECONDS="$${MINIO_SESSION_TTL_SECONDS:-3600}"; \ export S3_ACCESS_KEY="$${S3_ACCESS_KEY:-minioadmin}"; \
export S3_SECRET_KEY="$${S3_SECRET_KEY:-minioadmin}"; \
export S3_SESSION_TTL_SECONDS="$${S3_SESSION_TTL_SECONDS:-3600}"; \
export CONVERSION_CLEANUP_DELAY_SECONDS="$${CONVERSION_CLEANUP_DELAY_SECONDS:-3600}"; \ export CONVERSION_CLEANUP_DELAY_SECONDS="$${CONVERSION_CLEANUP_DELAY_SECONDS:-3600}"; \
uv run --project python uvicorn officeconvert_server.app:app --host "$${UVICORN_HOST:-0.0.0.0}" --port "$${UVICORN_PORT:-8080}" uv run --project python uvicorn officeconvert_server.app:app --host "$${UVICORN_HOST:-0.0.0.0}" --port "$${UVICORN_PORT:-8080}"
+19 -13
View File
@@ -10,7 +10,7 @@ and client compatibility.
- `proto/` contains protobuf schemas and RPC definitions. - `proto/` contains protobuf schemas and RPC definitions.
- `gen/python` and `gen/go` contain generated protocol and Connect code. - `gen/python` and `gen/go` contain generated protocol and Connect code.
- `python/packages/officeconvert` is the core conversion library (PPTX -> PDF -> images + notes). - `python/packages/officeconvert` is the core conversion library (PPTX -> PDF -> images + notes).
- `python/packages/server` is the ConnectRPC Python server with MinIO orchestration. - `python/packages/server` is the ConnectRPC Python server with SeaweedFS (S3-compatible) orchestration.
- `clients/go` is the first client library with layered orchestration helpers. - `clients/go` is the first client library with layered orchestration helpers.
- `deploy/` contains production-ish and dev Docker Compose files. - `deploy/` contains production-ish and dev Docker Compose files.
@@ -29,14 +29,14 @@ Use the root `Makefile`:
- `make buf-generate` to regenerate Go and Python types - `make buf-generate` to regenerate Go and Python types
- `make py-sync` to sync Python workspace dependencies with uv - `make py-sync` to sync Python workspace dependencies with uv
- `make go-test` to run Go client tests - `make go-test` to run Go client tests
- `make compose-up` to run server + MinIO - `make compose-up` to run server + SeaweedFS
- `make compose-up-dev` to run MinIO only - `make compose-up-dev` to run SeaweedFS only
- `make run-server` to start host `uvicorn` with `.env` (if present) plus defaults - `make run-server` to start host `uvicorn` with `.env` (if present) plus defaults
## Development Server Workflow ## Development Server Workflow
This is the recommended local workflow for iterating on the Python server and conversion This is the recommended local workflow for iterating on the Python server and conversion
library while keeping MinIO in Docker. library while keeping SeaweedFS in Docker.
### 1) Prerequisites ### 1) Prerequisites
@@ -64,7 +64,7 @@ From repo root:
make py-sync make py-sync
``` ```
### 4) Start MinIO dependency stack (dev compose) ### 4) Start SeaweedFS dependency stack (dev compose)
From repo root: From repo root:
@@ -72,11 +72,12 @@ From repo root:
make compose-up-dev make compose-up-dev
``` ```
MinIO endpoints: SeaweedFS endpoints:
- API: `http://localhost:9000` - S3 API: `http://localhost:8333`
- Console: `http://localhost:9001` - Master API: `http://localhost:9333`
- Default creds: `minioadmin` / `minioadmin` - Filer API: `http://localhost:8888`
- Default S3 creds: `minioadmin` / `minioadmin`
### 5) Start Connect server (host process) ### 5) Start Connect server (host process)
@@ -90,11 +91,10 @@ make run-server
- loads `.env` automatically if present - loads `.env` automatically if present
- applies reasonable defaults when values are not set - applies reasonable defaults when values are not set
- defaults MinIO endpoint to `localhost:9000` for host-based development - defaults S3 endpoint to `localhost:8333` for host-based development
- auto-normalizes `seaweedfs:8333` to `localhost:8333` for host runs
- supports optional `UVICORN_HOST` and `UVICORN_PORT` overrides - supports optional `UVICORN_HOST` and `UVICORN_PORT` overrides
If you copy from `.env.example`, set `MINIO_ENDPOINT=localhost:9000` for host mode.
Server endpoint base URL: Server endpoint base URL:
- `http://localhost:8080` - `http://localhost:8080`
@@ -120,10 +120,16 @@ Then:
### 7) Full container workflow (optional) ### 7) Full container workflow (optional)
If you want to run both server and MinIO in Docker: If you want to run both server and SeaweedFS in Docker:
```bash ```bash
make compose-up make compose-up
``` ```
Use `.env.example` as your baseline env configuration. Use `.env.example` as your baseline env configuration.
## Storage Backend Notes
- This project defaults to **SeaweedFS S3 API** for object transit in development and compose deployments.
- The Python server uses the `minio` Python SDK, which is intentional because SeaweedFS is S3-compatible.
- Runtime configuration uses `S3_*` environment variables.
+12 -9
View File
@@ -1,15 +1,18 @@
services: services:
minio: seaweedfs:
image: minio/minio:RELEASE.2026-02-17T00-53-00Z image: chrislusf/seaweedfs:latest
command: server /data --console-address ":9001" command:
- server
- -s3
environment: environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY:-minioadmin}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin} AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY:-minioadmin}
ports: ports:
- "9000:9000" - "8333:8333"
- "9001:9001" - "9333:9333"
- "8888:8888"
volumes: volumes:
- minio_data:/data - seaweedfs_data:/data
volumes: volumes:
minio_data: seaweedfs_data:
+19 -16
View File
@@ -1,32 +1,35 @@
services: services:
minio: seaweedfs:
image: minio/minio:RELEASE.2026-02-17T00-53-00Z image: chrislusf/seaweedfs:latest
command: server /data --console-address ":9001" command:
- server
- -s3
environment: environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-minioadmin} AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY:-minioadmin}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-minioadmin} AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY:-minioadmin}
ports: ports:
- "9000:9000" - "8333:8333"
- "9001:9001" - "9333:9333"
- "8888:8888"
volumes: volumes:
- minio_data:/data - seaweedfs_data:/data
server: server:
build: build:
context: .. context: ..
dockerfile: Dockerfile.server dockerfile: Dockerfile.server
depends_on: depends_on:
- minio - seaweedfs
environment: environment:
MINIO_ENDPOINT: ${MINIO_ENDPOINT:-minio:9000} S3_ENDPOINT: ${S3_ENDPOINT:-seaweedfs:8333}
MINIO_PUBLIC_ENDPOINT: ${MINIO_PUBLIC_ENDPOINT:-localhost:9000} S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-localhost:8333}
MINIO_USE_SSL: ${MINIO_USE_SSL:-false} S3_USE_SSL: ${S3_USE_SSL:-false}
MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-minioadmin} S3_ACCESS_KEY: ${S3_ACCESS_KEY:-minioadmin}
MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-minioadmin} S3_SECRET_KEY: ${S3_SECRET_KEY:-minioadmin}
MINIO_SESSION_TTL_SECONDS: ${MINIO_SESSION_TTL_SECONDS:-3600} S3_SESSION_TTL_SECONDS: ${S3_SESSION_TTL_SECONDS:-3600}
CONVERSION_CLEANUP_DELAY_SECONDS: ${CONVERSION_CLEANUP_DELAY_SECONDS:-3600} CONVERSION_CLEANUP_DELAY_SECONDS: ${CONVERSION_CLEANUP_DELAY_SECONDS:-3600}
ports: ports:
- "8080:8080" - "8080:8080"
volumes: volumes:
minio_data: seaweedfs_data:
+1 -1
View File
@@ -1,7 +1,7 @@
[project] [project]
name = "officeconvert-server" name = "officeconvert-server"
version = "0.1.0" version = "0.1.0"
description = "ConnectRPC server orchestrating file conversions with MinIO." description = "ConnectRPC server orchestrating file conversions with S3-compatible storage."
readme = "../../../README.md" readme = "../../../README.md"
requires-python = ">=3.12" requires-python = ">=3.12"
dependencies = [ dependencies = [
@@ -6,18 +6,18 @@ from officeconvertapi.v1.conversion_connect import ConversionServiceASGIApplicat
from officeconvert_server.config import load_server_config from officeconvert_server.config import load_server_config
from officeconvert_server.service import ConversionServiceImpl from officeconvert_server.service import ConversionServiceImpl
from officeconvert_server.storage import MinIOStore from officeconvert_server.storage import S3Store
def create_app() -> ConversionServiceASGIApplication: def create_app() -> ConversionServiceASGIApplication:
"""Construct and return the configured Connect ASGI application.""" """Construct and return the configured Connect ASGI application."""
config = load_server_config() config = load_server_config()
store = MinIOStore( store = S3Store(
endpoint=config.minio_endpoint, endpoint=config.s3_endpoint,
access_key=config.minio_access_key, access_key=config.s3_access_key,
secret_key=config.minio_secret_key, secret_key=config.s3_secret_key,
secure=config.minio_secure, secure=config.s3_secure,
public_endpoint=config.minio_public_endpoint, public_endpoint=config.s3_public_endpoint,
) )
service = ConversionServiceImpl(config=config, store=store) service = ConversionServiceImpl(config=config, store=store)
return ConversionServiceASGIApplication(service) return ConversionServiceASGIApplication(service)
@@ -10,24 +10,24 @@ import os
class ServerConfig: class ServerConfig:
"""Defines environment-driven settings for server orchestration.""" """Defines environment-driven settings for server orchestration."""
minio_endpoint: str s3_endpoint: str
minio_access_key: str s3_access_key: str
minio_secret_key: str s3_secret_key: str
minio_secure: bool s3_secure: bool
minio_public_endpoint: str s3_public_endpoint: str
minio_session_ttl_seconds: int s3_session_ttl_seconds: int
conversion_cleanup_delay_seconds: int conversion_cleanup_delay_seconds: int
def load_server_config() -> ServerConfig: def load_server_config() -> ServerConfig:
"""Load server configuration from environment variables.""" """Load server configuration from environment variables."""
return ServerConfig( return ServerConfig(
minio_endpoint=os.getenv("MINIO_ENDPOINT", "localhost:9000"), s3_endpoint=os.getenv("S3_ENDPOINT", "localhost:8333"),
minio_access_key=os.getenv("MINIO_ACCESS_KEY", "minioadmin"), s3_access_key=os.getenv("S3_ACCESS_KEY", "minioadmin"),
minio_secret_key=os.getenv("MINIO_SECRET_KEY", "minioadmin"), s3_secret_key=os.getenv("S3_SECRET_KEY", "minioadmin"),
minio_secure=os.getenv("MINIO_USE_SSL", "false").lower() == "true", s3_secure=os.getenv("S3_USE_SSL", "false").lower() == "true",
minio_public_endpoint=os.getenv("MINIO_PUBLIC_ENDPOINT", "localhost:9000"), s3_public_endpoint=os.getenv("S3_PUBLIC_ENDPOINT", "localhost:8333"),
minio_session_ttl_seconds=int(os.getenv("MINIO_SESSION_TTL_SECONDS", "3600")), s3_session_ttl_seconds=int(os.getenv("S3_SESSION_TTL_SECONDS", "3600")),
conversion_cleanup_delay_seconds=int( conversion_cleanup_delay_seconds=int(
os.getenv("CONVERSION_CLEANUP_DELAY_SECONDS", "3600") os.getenv("CONVERSION_CLEANUP_DELAY_SECONDS", "3600")
), ),
@@ -18,13 +18,13 @@ from officeconvertapi.v1 import conversion_connect, conversion_pb2
from officeconvert_server.config import ServerConfig from officeconvert_server.config import ServerConfig
from officeconvert_server.models import ConversionSession, utc_now from officeconvert_server.models import ConversionSession, utc_now
from officeconvert_server.storage import MinIOStore from officeconvert_server.storage import S3Store
class ConversionServiceImpl(conversion_connect.ConversionService): class ConversionServiceImpl(conversion_connect.ConversionService):
"""Implements the conversion API with in-memory state and MinIO orchestration.""" """Implements the conversion API with in-memory state and S3 orchestration."""
def __init__(self, config: ServerConfig, store: MinIOStore) -> None: def __init__(self, config: ServerConfig, store: S3Store) -> None:
"""Initialize service with runtime config and storage adapter.""" """Initialize service with runtime config and storage adapter."""
self._config = config self._config = config
self._store = store self._store = store
@@ -47,13 +47,13 @@ class ConversionServiceImpl(conversion_connect.ConversionService):
conversion_id = str(uuid.uuid4()) conversion_id = str(uuid.uuid4())
bucket_name = f"oc-{conversion_id}" bucket_name = f"oc-{conversion_id}"
upload_key = "input/source.pptx" upload_key = "input/source.pptx"
expires_at = utc_now() + timedelta(seconds=self._config.minio_session_ttl_seconds) expires_at = utc_now() + timedelta(seconds=self._config.s3_session_ttl_seconds)
self._store.ensure_bucket(bucket_name) self._store.ensure_bucket(bucket_name)
upload_url = self._store.presigned_put_url( upload_url = self._store.presigned_put_url(
bucket_name, bucket_name,
upload_key, upload_key,
ttl_seconds=self._config.minio_session_ttl_seconds, ttl_seconds=self._config.s3_session_ttl_seconds,
) )
session = ConversionSession( session = ConversionSession(
@@ -143,7 +143,7 @@ class ConversionServiceImpl(conversion_connect.ConversionService):
request: conversion_pb2.DeleteConversionRequest, request: conversion_pb2.DeleteConversionRequest,
ctx: RequestContext, ctx: RequestContext,
) -> conversion_pb2.DeleteConversionResponse: ) -> conversion_pb2.DeleteConversionResponse:
"""Delete a conversion session and associated MinIO/local artifacts.""" """Delete a conversion session and associated object storage/local artifacts."""
del ctx del ctx
async with self._lock: async with self._lock:
session = self._sessions.pop(request.conversion_id, None) session = self._sessions.pop(request.conversion_id, None)
@@ -218,7 +218,7 @@ class ConversionServiceImpl(conversion_connect.ConversionService):
image_url = self._store.presigned_get_url( image_url = self._store.presigned_get_url(
session.bucket_name, session.bucket_name,
object_key, object_key,
ttl_seconds=self._config.minio_session_ttl_seconds, ttl_seconds=self._config.s3_session_ttl_seconds,
) )
response_slides.append( response_slides.append(
conversion_pb2.Slide( conversion_pb2.Slide(
@@ -1,4 +1,4 @@
"""MinIO helper abstraction for upload and artifact lifecycle.""" """S3-compatible storage helper abstraction for upload and artifact lifecycle."""
from __future__ import annotations from __future__ import annotations
@@ -11,8 +11,8 @@ from minio.deleteobjects import DeleteObject
from minio.error import S3Error from minio.error import S3Error
class MinIOStore: class S3Store:
"""Provides typed helper methods around MinIO object storage operations.""" """Provides typed helper methods around S3-compatible object storage operations."""
def __init__( def __init__(
self, self,
@@ -23,7 +23,7 @@ class MinIOStore:
secure: bool, secure: bool,
public_endpoint: str, public_endpoint: str,
) -> None: ) -> None:
"""Initialize MinIO clients for internal and public URL generation.""" """Initialize S3 clients for internal and public URL generation."""
self._client = Minio( self._client = Minio(
endpoint, endpoint,
access_key=access_key, access_key=access_key,
@@ -59,12 +59,12 @@ class MinIOStore:
) )
def fget_object(self, bucket_name: str, object_key: str, output_path: Path) -> None: def fget_object(self, bucket_name: str, object_key: str, output_path: Path) -> None:
"""Download one object from MinIO to a local filesystem path.""" """Download one object from storage to a local filesystem path."""
output_path.parent.mkdir(parents=True, exist_ok=True) output_path.parent.mkdir(parents=True, exist_ok=True)
self._client.fget_object(bucket_name, object_key, str(output_path)) self._client.fget_object(bucket_name, object_key, str(output_path))
def fput_object(self, bucket_name: str, object_key: str, source_path: Path) -> None: def fput_object(self, bucket_name: str, object_key: str, source_path: Path) -> None:
"""Upload one local filesystem object to MinIO.""" """Upload one local filesystem object to storage."""
self._client.fput_object(bucket_name, object_key, str(source_path)) self._client.fput_object(bucket_name, object_key, str(source_path))
def remove_bucket_tree(self, bucket_name: str) -> None: def remove_bucket_tree(self, bucket_name: str) -> None: