increase logging verbosity around s3 access errors
Docker server image / build-and-push (push) Successful in 3m14s
Docker server image / build-and-push (push) Successful in 3m14s
This commit is contained in:
@@ -5,7 +5,11 @@ S3_USE_SSL=false
|
|||||||
S3_PUBLIC_USE_SSL=false
|
S3_PUBLIC_USE_SSL=false
|
||||||
S3_ACCESS_KEY=minioadmin
|
S3_ACCESS_KEY=minioadmin
|
||||||
S3_SECRET_KEY=minioadmin
|
S3_SECRET_KEY=minioadmin
|
||||||
|
# Optional SigV4 region for strict S3 gateways (e.g. some SeaweedFS / proxy setups).
|
||||||
|
# S3_REGION=us-east-1
|
||||||
S3_SESSION_TTL_SECONDS=3600
|
S3_SESSION_TTL_SECONDS=3600
|
||||||
|
# Set to true to dump raw S3 HTTP on stderr (debugging AccessDenied / proxies).
|
||||||
|
# OFFICECONVERT_S3_TRACE=true
|
||||||
CONVERSION_PPTX_TO_PDF_TIMEOUT_SECONDS=180
|
CONVERSION_PPTX_TO_PDF_TIMEOUT_SECONDS=180
|
||||||
CONVERSION_PDF_TO_IMAGES_TIMEOUT_SECONDS=1800
|
CONVERSION_PDF_TO_IMAGES_TIMEOUT_SECONDS=1800
|
||||||
CONVERSION_PPTX_TO_PDF_BASE_TIMEOUT_SECONDS=45
|
CONVERSION_PPTX_TO_PDF_BASE_TIMEOUT_SECONDS=45
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
from officeconvertapi.v1.conversion_connect import ConversionServiceASGIApplication
|
from officeconvertapi.v1.conversion_connect import ConversionServiceASGIApplication
|
||||||
|
|
||||||
@@ -47,9 +48,13 @@ def create_app() -> ConversionServiceASGIApplication:
|
|||||||
access_key=config.s3_access_key,
|
access_key=config.s3_access_key,
|
||||||
secret_key=config.s3_secret_key,
|
secret_key=config.s3_secret_key,
|
||||||
secure=config.s3_secure,
|
secure=config.s3_secure,
|
||||||
|
region=config.s3_region,
|
||||||
public_endpoint=config.s3_public_endpoint,
|
public_endpoint=config.s3_public_endpoint,
|
||||||
public_secure=config.s3_public_secure,
|
public_secure=config.s3_public_secure,
|
||||||
)
|
)
|
||||||
|
if os.getenv("OFFICECONVERT_S3_TRACE", "").lower() in ("1", "true", "yes"):
|
||||||
|
store.enable_http_trace(sys.stderr)
|
||||||
|
logger.warning("OFFICECONVERT_S3_TRACE enabled: S3 HTTP dumps on stderr")
|
||||||
service = ConversionServiceImpl(config=config, store=store)
|
service = ConversionServiceImpl(config=config, store=store)
|
||||||
return ConversionServiceASGIApplication(service)
|
return ConversionServiceASGIApplication(service)
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ class ServerConfig:
|
|||||||
s3_access_key: str
|
s3_access_key: str
|
||||||
s3_secret_key: str
|
s3_secret_key: str
|
||||||
s3_secure: bool
|
s3_secure: bool
|
||||||
|
s3_region: str | None
|
||||||
s3_public_endpoint: str
|
s3_public_endpoint: str
|
||||||
s3_public_secure: bool
|
s3_public_secure: bool
|
||||||
s3_session_ttl_seconds: int
|
s3_session_ttl_seconds: int
|
||||||
@@ -35,11 +36,13 @@ def load_server_config() -> ServerConfig:
|
|||||||
if public_ssl_env is not None
|
if public_ssl_env is not None
|
||||||
else s3_secure
|
else s3_secure
|
||||||
)
|
)
|
||||||
|
region_env = os.getenv("S3_REGION", "").strip()
|
||||||
return ServerConfig(
|
return ServerConfig(
|
||||||
s3_endpoint=os.getenv("S3_ENDPOINT", "localhost:8333"),
|
s3_endpoint=os.getenv("S3_ENDPOINT", "localhost:8333"),
|
||||||
s3_access_key=os.getenv("S3_ACCESS_KEY", "minioadmin"),
|
s3_access_key=os.getenv("S3_ACCESS_KEY", "minioadmin"),
|
||||||
s3_secret_key=os.getenv("S3_SECRET_KEY", "minioadmin"),
|
s3_secret_key=os.getenv("S3_SECRET_KEY", "minioadmin"),
|
||||||
s3_secure=s3_secure,
|
s3_secure=s3_secure,
|
||||||
|
s3_region=region_env or None,
|
||||||
s3_public_endpoint=os.getenv("S3_PUBLIC_ENDPOINT", "localhost:8333"),
|
s3_public_endpoint=os.getenv("S3_PUBLIC_ENDPOINT", "localhost:8333"),
|
||||||
s3_public_secure=s3_public_secure,
|
s3_public_secure=s3_public_secure,
|
||||||
s3_session_ttl_seconds=int(os.getenv("S3_SESSION_TTL_SECONDS", "3600")),
|
s3_session_ttl_seconds=int(os.getenv("S3_SESSION_TTL_SECONDS", "3600")),
|
||||||
|
|||||||
@@ -30,9 +30,11 @@ from officeconvert.conversion import (
|
|||||||
)
|
)
|
||||||
from officeconvertapi.v1 import conversion_connect, conversion_pb2
|
from officeconvertapi.v1 import conversion_connect, conversion_pb2
|
||||||
|
|
||||||
|
from minio.error import S3Error
|
||||||
|
|
||||||
from officeconvert_server.config import ServerConfig
|
from officeconvert_server.config import ServerConfig
|
||||||
from officeconvert_server.models import ConversionSession, utc_now
|
from officeconvert_server.models import ConversionSession, utc_now
|
||||||
from officeconvert_server.storage import S3Store
|
from officeconvert_server.storage import S3Store, log_s3_error
|
||||||
|
|
||||||
logger = logging.getLogger("uvicorn.error")
|
logger = logging.getLogger("uvicorn.error")
|
||||||
|
|
||||||
@@ -78,7 +80,16 @@ class ConversionServiceImpl(conversion_connect.ConversionService):
|
|||||||
upload_key = "input/source.pptx"
|
upload_key = "input/source.pptx"
|
||||||
expires_at = utc_now() + timedelta(seconds=self._config.s3_session_ttl_seconds)
|
expires_at = utc_now() + timedelta(seconds=self._config.s3_session_ttl_seconds)
|
||||||
|
|
||||||
self._store.ensure_bucket(bucket_name)
|
try:
|
||||||
|
self._store.ensure_bucket(bucket_name)
|
||||||
|
except S3Error as exc:
|
||||||
|
log_s3_error(
|
||||||
|
"ensure_bucket",
|
||||||
|
endpoint=self._config.s3_endpoint,
|
||||||
|
secure=self._config.s3_secure,
|
||||||
|
exc=exc,
|
||||||
|
)
|
||||||
|
raise
|
||||||
upload_url = self._store.presigned_put_url(
|
upload_url = self._store.presigned_put_url(
|
||||||
bucket_name,
|
bucket_name,
|
||||||
upload_key,
|
upload_key,
|
||||||
|
|||||||
@@ -2,14 +2,51 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import TextIO
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from minio import Minio
|
from minio import Minio
|
||||||
from minio.deleteobjects import DeleteObject
|
from minio.deleteobjects import DeleteObject
|
||||||
from minio.error import S3Error
|
from minio.error import S3Error
|
||||||
|
|
||||||
|
_log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def log_s3_error(
|
||||||
|
operation: str,
|
||||||
|
*,
|
||||||
|
endpoint: str,
|
||||||
|
secure: bool,
|
||||||
|
exc: S3Error,
|
||||||
|
) -> None:
|
||||||
|
"""Emit HTTP details for S3 failures (MinIO often maps HTTP 403 to AccessDenied)."""
|
||||||
|
status = getattr(exc.response, "status", None)
|
||||||
|
ctype = (
|
||||||
|
exc.response.headers.get("content-type")
|
||||||
|
if exc.response is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
body = ""
|
||||||
|
if exc.response is not None and exc.response.data:
|
||||||
|
body = exc.response.data.decode(errors="replace")[:4000]
|
||||||
|
_log.error(
|
||||||
|
"S3 %s failed: code=%r message=%r http_status=%s content_type=%r "
|
||||||
|
"endpoint=%s secure=%s resource=%r request_id=%r body=%r",
|
||||||
|
operation,
|
||||||
|
exc.code,
|
||||||
|
exc.message,
|
||||||
|
status,
|
||||||
|
ctype,
|
||||||
|
endpoint,
|
||||||
|
secure,
|
||||||
|
exc.resource,
|
||||||
|
exc.request_id,
|
||||||
|
body,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class S3Store:
|
class S3Store:
|
||||||
"""Provides typed helper methods around S3-compatible object storage operations."""
|
"""Provides typed helper methods around S3-compatible object storage operations."""
|
||||||
@@ -21,6 +58,7 @@ class S3Store:
|
|||||||
access_key: str,
|
access_key: str,
|
||||||
secret_key: str,
|
secret_key: str,
|
||||||
secure: bool,
|
secure: bool,
|
||||||
|
region: str | None,
|
||||||
public_endpoint: str,
|
public_endpoint: str,
|
||||||
public_secure: bool,
|
public_secure: bool,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -30,14 +68,21 @@ class S3Store:
|
|||||||
access_key=access_key,
|
access_key=access_key,
|
||||||
secret_key=secret_key,
|
secret_key=secret_key,
|
||||||
secure=secure,
|
secure=secure,
|
||||||
|
region=region,
|
||||||
)
|
)
|
||||||
self._public_client = Minio(
|
self._public_client = Minio(
|
||||||
public_endpoint,
|
public_endpoint,
|
||||||
access_key=access_key,
|
access_key=access_key,
|
||||||
secret_key=secret_key,
|
secret_key=secret_key,
|
||||||
secure=public_secure,
|
secure=public_secure,
|
||||||
|
region=region,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def enable_http_trace(self, stream: TextIO) -> None:
|
||||||
|
"""Write raw HTTP request/response traces for both clients (debugging)."""
|
||||||
|
self._client.trace_on(stream)
|
||||||
|
self._public_client.trace_on(stream)
|
||||||
|
|
||||||
def ensure_bucket(self, bucket_name: str) -> None:
|
def ensure_bucket(self, bucket_name: str) -> None:
|
||||||
"""Create a bucket if it does not already exist.
|
"""Create a bucket if it does not already exist.
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user