Handle DuckDB lock errors gracefully in UI

- Catch IOException when engine holds exclusive lock
- Show informative message instead of crashing

.gitea/workflows/ci-cd.yaml

@@ -10,7 +10,6 @@ on:
 env:
   REGISTRY: gitea.thefetagroup.com
   IMAGE_NAME: bnair/cryptobot
-  # Gitea URL for git operations
   GITEA_URL: https://git.thefetagroup.com
 
 jobs:
@@ -54,7 +53,7 @@ jobs:
       - name: Run tests
         run: pytest --cov=tradefinder --cov-report=term-missing
 
-  build-and-push:
+  build-engine:
     runs-on: ubuntu-latest
     needs: test
     if: startsWith(github.ref, 'refs/tags/v')
@@ -89,8 +88,31 @@ jobs:
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.version.outputs.VERSION }}
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+
+  build-ui:
+    runs-on: ubuntu-latest
+    needs: test
+    if: startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - name: Checkout code
+        run: |
+          git clone --depth 1 ${{ env.GITEA_URL }}/bnair/CryptoTrading.git .
+          git fetch origin ${{ github.ref_name }} --depth 1
+          git checkout ${{ github.ref_name }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Extract version from tag
+        id: version
+        run: echo "VERSION=${{ github.ref_name }}" >> $GITHUB_OUTPUT
+
+      - name: Login to Gitea Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ secrets.REGISTRY_USERNAME }}
+          password: ${{ secrets.REGISTRY_PASSWORD }}
 
       - name: Build and push UI image
         uses: docker/build-push-action@v5
@@ -102,5 +124,3 @@ jobs:
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ui:${{ steps.version.outputs.VERSION }}
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ui:latest
-          cache-from: type=gha
-          cache-to: type=gha,mode=max

Dockerfile

@@ -11,12 +11,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
     wget \
     curl \
+    dpkg-dev \
     && rm -rf /var/lib/apt/lists/*
 
-# Install TA-Lib C library
+# Install TA-Lib C library (with ARM64 support)
 RUN wget -q http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-src.tar.gz \
     && tar -xzf ta-lib-0.4.0-src.tar.gz \
     && cd ta-lib \
+    && wget -q -O config.guess 'https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD' \
+    && wget -q -O config.sub 'https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD' \
     && ./configure --prefix=/usr \
     && make \
     && make install \

docker-compose.override.yml

@@ -0,0 +1,31 @@
+# Local development override - mounts source code for hot reload
+x-engine-image: &engine-image
+  image: tf-engine:local
+
+x-ui-image: &ui-image
+  image: tf-ui:local
+
+services:
+  engine:
+    <<: *engine-image
+    volumes:
+      - /Users/bnair/Documents/Crypto/engine:/data/engine
+      - /Users/bnair/Documents/Crypto/shared:/data/shared
+      - /Users/bnair/Documents/TradeFinder/src:/app/src:ro
+    user: root
+
+  ui:
+    <<: *ui-image
+    volumes:
+      - /Users/bnair/Documents/Crypto/engine:/data/engine  # DuckDB needs write access for WAL
+      - /Users/bnair/Documents/Crypto/shared:/data/shared
+      - /Users/bnair/Documents/TradeFinder/src:/app/src:ro
+    user: root
+
+  optimizer:
+    <<: *engine-image
+    user: root
+
+  backtester:
+    <<: *engine-image
+    user: root

docker-compose.yml

@@ -1,7 +1,15 @@
-version: "3.9"
-
 # TradeFinder - Automated Crypto Trading System
-# Mount point: /opt/trading/crypto (configurable via DATA_ROOT)
+# Uses pre-built images from Gitea Container Registry
+#
+# Quick Start:
+#   docker compose up -d
+#
+# With optimizer/backtester:
+#   docker compose --profile optimizer up -d
+#   docker compose --profile backtest run --rm backtester
+#
+# Environment:
+#   Copy .env.example to .env and configure API keys
 
 x-common: &common
   restart: unless-stopped
@@ -13,43 +21,38 @@ x-common: &common
       max-size: "10m"
       max-file: "3"
 
+x-engine-image: &engine-image
+  image: gitea.thefetagroup.com/bnair/cryptobot:latest
+
+x-ui-image: &ui-image
+  image: gitea.thefetagroup.com/bnair/cryptobot-ui:latest
+
 services:
-  # ==========================================================================
-  # CORE TRADING ENGINE
-  # ==========================================================================
   engine:
-    <<: *common
-    build:
-      context: .
-      dockerfile: Dockerfile
-      target: engine
+    <<: [*common, *engine-image]
     container_name: tf-engine
     env_file:
       - .env
     environment:
       - PYTHONUNBUFFERED=1
       - DUCKDB_PATH=/data/engine/tradefinder.duckdb
+      - TRADING_MODE=${TRADING_MODE:-paper}
     volumes:
-      - ${DATA_ROOT:-/opt/trading/crypto}/engine:/data/engine
-      - ${DATA_ROOT:-/opt/trading/crypto}/shared:/data/shared
+      - ${DATA_ROOT:-./data}/engine:/data/engine
+      - ${DATA_ROOT:-./data}/shared:/data/shared
     depends_on:
-      - redis
+      redis:
+        condition: service_healthy
     healthcheck:
       test: ["CMD", "python", "-c", "import tradefinder; print('ok')"]
       interval: 30s
       timeout: 10s
       retries: 3
+      start_period: 10s
     command: ["python", "-m", "tradefinder.core.main"]
 
-  # ==========================================================================
-  # STREAMLIT UI
-  # ==========================================================================
   ui:
-    <<: *common
-    build:
-      context: .
-      dockerfile: Dockerfile
-      target: ui
+    <<: [*common, *ui-image]
     container_name: tf-ui
     env_file:
       - .env
@@ -59,8 +62,8 @@ services:
     ports:
       - "${STREAMLIT_PORT:-8501}:8501"
     volumes:
-      - ${DATA_ROOT:-/opt/trading/crypto}/engine:/data/engine:ro
-      - ${DATA_ROOT:-/opt/trading/crypto}/shared:/data/shared
+      - ${DATA_ROOT:-./data}/engine:/data/engine:ro
+      - ${DATA_ROOT:-./data}/shared:/data/shared
     depends_on:
       - engine
     healthcheck:
@@ -68,40 +71,15 @@ services:
       interval: 30s
       timeout: 10s
       retries: 3
+      start_period: 15s
     command: ["streamlit", "run", "src/tradefinder/ui/app.py", "--server.port=8501", "--server.address=0.0.0.0"]
 
-  # ==========================================================================
-  # OPTIMIZER (runs weekly via scheduler or manually)
-  # ==========================================================================
-  optimizer:
-    <<: *common
-    build:
-      context: .
-      dockerfile: Dockerfile
-      target: engine
-    container_name: tf-optimizer
-    env_file:
-      - .env
-    environment:
-      - PYTHONUNBUFFERED=1
-      - DUCKDB_PATH=/data/engine/tradefinder.duckdb
-    volumes:
-      - ${DATA_ROOT:-/opt/trading/crypto}/engine:/data/engine
-      - ${DATA_ROOT:-/opt/trading/crypto}/optimizer:/data/optimizer
-      - ${DATA_ROOT:-/opt/trading/crypto}/shared:/data/shared
-    profiles:
-      - optimizer  # Only runs when explicitly started: docker compose --profile optimizer up optimizer
-    command: ["python", "-m", "tradefinder.core.optimize"]
-
-  # ==========================================================================
-  # REDIS (for real-time state, pub/sub, caching)
-  # ==========================================================================
   redis:
     <<: *common
     image: redis:7-alpine
     container_name: tf-redis
     volumes:
-      - ${DATA_ROOT:-/opt/trading/crypto}/redis:/data
+      - ${DATA_ROOT:-./data}/redis:/data
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
       interval: 10s
@@ -109,15 +87,27 @@ services:
       retries: 3
     command: ["redis-server", "--appendonly", "yes", "--maxmemory", "256mb", "--maxmemory-policy", "allkeys-lru"]
 
-  # ==========================================================================
-  # BACKTESTER (runs on-demand)
-  # ==========================================================================
+  optimizer:
+    <<: [*common, *engine-image]
+    container_name: tf-optimizer
+    env_file:
+      - .env
+    environment:
+      - PYTHONUNBUFFERED=1
+      - DUCKDB_PATH=/data/engine/tradefinder.duckdb
+    volumes:
+      - ${DATA_ROOT:-./data}/engine:/data/engine
+      - ${DATA_ROOT:-./data}/optimizer:/data/optimizer
+      - ${DATA_ROOT:-./data}/shared:/data/shared
+    depends_on:
+      redis:
+        condition: service_healthy
+    profiles:
+      - optimizer
+    command: ["python", "-m", "tradefinder.core.optimize"]
+
   backtester:
-    <<: *common
-    build:
-      context: .
-      dockerfile: Dockerfile
-      target: engine
+    <<: [*common, *engine-image]
     container_name: tf-backtester
     env_file:
       - .env
@@ -125,11 +115,11 @@ services:
       - PYTHONUNBUFFERED=1
       - DUCKDB_PATH=/data/engine/tradefinder.duckdb
     volumes:
-      - ${DATA_ROOT:-/opt/trading/crypto}/engine:/data/engine
-      - ${DATA_ROOT:-/opt/trading/crypto}/backtest:/data/backtest
-      - ${DATA_ROOT:-/opt/trading/crypto}/shared:/data/shared
+      - ${DATA_ROOT:-./data}/engine:/data/engine
+      - ${DATA_ROOT:-./data}/backtest:/data/backtest
+      - ${DATA_ROOT:-./data}/shared:/data/shared
     profiles:
-      - backtest  # Only runs when explicitly started
+      - backtest
     command: ["python", "-m", "tradefinder.core.backtest"]
 
 networks:
@@ -137,10 +127,9 @@ networks:
     driver: bridge
     name: tradefinder-network
 
-# Volume labels for clarity
-# All data persisted under ${DATA_ROOT:-/opt/trading/crypto}/
-# ├── engine/          # DuckDB, order logs, positions
-# ├── optimizer/       # Optuna studies, best params
+# Data Structure (default: ./data/)
+# ├── engine/          # DuckDB database, order logs, positions
+# ├── optimizer/       # Optuna studies, best parameters
 # ├── backtest/        # Backtest results, reports
 # ├── redis/           # Redis AOF persistence
 # └── shared/          # Shared configs, FX rates cache

pyproject.toml

@@ -58,6 +58,9 @@ dependencies = [
     "structlog>=24.1.0",
     "rich>=13.7.0",
     
+    # Redis
+    "redis>=5.0.0",
+
     # HTTP client
     "httpx>=0.26.0",
 ]

src/tradefinder/core/main.py

@@ -396,21 +396,18 @@ async def _determine_equity(settings: Settings, adapter: BinanceUSDMAdapter | No
 
 def _register_signal_handlers(shutdown_event: asyncio.Event) -> None:
     """Register signal handlers that trigger a graceful shutdown."""
-
     loop = asyncio.get_running_loop()
 
     def _handle(sig: signal.Signals) -> None:
         logger.info("Shutdown signal received", signal=sig.name)
         shutdown_event.set()
 
-    def _handle_signal(signum: int, frame: object) -> None:
-        shutdown_event.set()
-
     for sig in (signal.SIGINT, signal.SIGTERM):
         try:
             loop.add_signal_handler(sig, partial(_handle, sig))
-        except (NotImplementedError, RuntimeError):
-            signal.signal(sig, _handle_signal)
+            logger.debug("Registered async signal handler", signal=sig.name)
+        except (NotImplementedError, RuntimeError, OSError) as e:
+            logger.debug("Could not register async signal handler", signal=sig.name, error=str(e))
 
 
 async def _sleep_or_cancel(delay: float, shutdown_event: asyncio.Event) -> None:
@@ -431,3 +428,7 @@ async def _initialize_adapter(adapter: BinanceUSDMAdapter, symbols: list[str]) -
     await adapter.configure_hedge_mode(True)
     for symbol in symbols:
         await adapter.configure_margin_type(symbol, MarginType.ISOLATED)
+
+
+if __name__ == "__main__":
+    main()

src/tradefinder/data/storage.py

@@ -27,22 +27,24 @@ class DataStorage:
         storage.disconnect()
     """
 
-    def __init__(self, db_path: Path) -> None:
+    def __init__(self, db_path: Path, *, read_only: bool = False) -> None:
         """Initialize storage with database path.
 
         Args:
             db_path: Path to DuckDB database file
+            read_only: If True, open database in read-only mode (no locking)
         """
         self.db_path = db_path
+        self._read_only = read_only
         self._conn: duckdb.DuckDBPyConnection | None = None
 
     def connect(self) -> None:
         """Connect to the database."""
-        # Ensure parent directory exists
-        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        if not self._read_only:
+            self.db_path.parent.mkdir(parents=True, exist_ok=True)
 
-        self._conn = duckdb.connect(str(self.db_path))
-        logger.info("Connected to DuckDB", path=str(self.db_path))
+        self._conn = duckdb.connect(str(self.db_path), read_only=self._read_only)
+        logger.info("Connected to DuckDB", path=str(self.db_path), read_only=self._read_only)
 
     def disconnect(self) -> None:
         """Close database connection."""

src/tradefinder/ui/app.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 from datetime import datetime
 from typing import Any
 
+import duckdb
 import streamlit as st
 import structlog
 
@@ -33,9 +34,12 @@ def get_storage() -> DataStorage | None:
     if not db_path.exists():
         return None
 
-    storage = DataStorage(db_path)
-    storage.connect()
-    return storage
+    storage = DataStorage(db_path, read_only=True)
+    try:
+        storage.connect()
+        return storage
+    except duckdb.IOException:
+        return None
 
 
 @st.cache_data(ttl=30)
@@ -103,9 +107,16 @@ def render_database_status() -> None:
     """Render database connection status."""
     st.subheader("Database Status")
 
+    settings = get_settings()
+    db_path = settings.duckdb_path
+
+    if not db_path.exists():
+        st.warning("Database not initialized yet. Start the trading engine first.")
+        return
+
     storage = get_storage()
     if storage is None:
-        st.warning("Database not initialized yet. Start the trading engine first.")
+        st.info(f"Database exists at {db_path} (engine has exclusive lock)")
         return
 
     try: