Add data layer (DuckDB storage, fetcher) and spot adapter with tests
- Add DataStorage class for DuckDB-based market data persistence - Add DataFetcher for historical candle backfill and sync operations - Add BinanceSpotAdapter for spot wallet balance queries - Add binance_spot_base_url to Settings for spot testnet support - Add comprehensive unit tests (50 new tests, 82 total) - Coverage increased from 62% to 86%
This commit is contained in:
bnair123
@@ -5,9 +5,10 @@ connecting to cryptocurrency exchanges.
|
||||
|
||||
Supported Exchanges:
|
||||
- Binance USDⓈ-M Perpetual Futures (primary)
|
||||
- Binance Spot (balance checking only)
|
||||
|
||||
Usage:
|
||||
from tradefinder.adapters import BinanceUSDMAdapter
|
||||
from tradefinder.adapters import BinanceUSDMAdapter, BinanceSpotAdapter
|
||||
from tradefinder.adapters.types import OrderRequest, Side, PositionSide, OrderType
|
||||
|
||||
adapter = BinanceUSDMAdapter(settings)
|
||||
@@ -23,6 +24,7 @@ from tradefinder.adapters.base import (
|
||||
OrderValidationError,
|
||||
RateLimitError,
|
||||
)
|
||||
from tradefinder.adapters.binance_spot import BinanceSpotAdapter, SpotBalance
|
||||
from tradefinder.adapters.binance_usdm import BinanceUSDMAdapter
|
||||
from tradefinder.adapters.types import (
|
||||
AccountBalance,
|
||||
@@ -45,6 +47,7 @@ __all__ = [
|
||||
"ExchangeAdapter",
|
||||
# Implementations
|
||||
"BinanceUSDMAdapter",
|
||||
"BinanceSpotAdapter",
|
||||
# Types
|
||||
"AccountBalance",
|
||||
"Candle",
|
||||
@@ -57,6 +60,7 @@ __all__ = [
|
||||
"Position",
|
||||
"PositionSide",
|
||||
"Side",
|
||||
"SpotBalance",
|
||||
"SymbolInfo",
|
||||
"TimeInForce",
|
||||
# Exceptions
|
||||
|
||||
214
src/tradefinder/adapters/binance_spot.py
Normal file
214
src/tradefinder/adapters/binance_spot.py
Normal file
@@ -0,0 +1,214 @@
|
||||
"""Binance Spot API adapter for balance checking.
|
||||
|
||||
Provides read-only access to spot wallet balances.
|
||||
Uses the same API credentials as the futures adapter.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import httpx
|
||||
import structlog
|
||||
|
||||
from tradefinder.adapters.base import AuthenticationError, ExchangeError
|
||||
from tradefinder.core.config import Settings
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SpotBalance:
|
||||
"""Spot wallet balance for an asset."""
|
||||
|
||||
asset: str
|
||||
free: Decimal
|
||||
locked: Decimal
|
||||
updated_at: datetime
|
||||
|
||||
@property
|
||||
def total(self) -> Decimal:
|
||||
"""Total balance (free + locked)."""
|
||||
return self.free + self.locked
|
||||
|
||||
|
||||
class BinanceSpotAdapter:
|
||||
"""Binance Spot API adapter for balance checking.
|
||||
|
||||
This adapter connects to Binance Spot (either testnet or production)
|
||||
and provides read-only access to spot wallet balances.
|
||||
|
||||
Usage:
|
||||
settings = get_settings()
|
||||
adapter = BinanceSpotAdapter(settings)
|
||||
await adapter.connect()
|
||||
balances = await adapter.get_all_balances()
|
||||
await adapter.disconnect()
|
||||
"""
|
||||
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
"""Initialize adapter with settings.
|
||||
|
||||
Args:
|
||||
settings: Application settings containing API credentials
|
||||
"""
|
||||
self.settings = settings
|
||||
self.base_url = settings.binance_spot_base_url
|
||||
self._client: httpx.AsyncClient | None = None
|
||||
self._recv_window = 5000
|
||||
|
||||
@property
|
||||
def _api_key(self) -> str:
|
||||
"""Get active API key."""
|
||||
key = self.settings.get_active_api_key()
|
||||
if key is None:
|
||||
raise AuthenticationError("No API key configured for current trading mode")
|
||||
return key.get_secret_value()
|
||||
|
||||
@property
|
||||
def _secret(self) -> str:
|
||||
"""Get active secret."""
|
||||
secret = self.settings.get_active_secret()
|
||||
if secret is None:
|
||||
raise AuthenticationError("No secret configured for current trading mode")
|
||||
return secret.get_secret_value()
|
||||
|
||||
def _sign(self, params: dict[str, Any]) -> str:
|
||||
"""Generate HMAC-SHA256 signature for request."""
|
||||
query_string = urlencode(params)
|
||||
signature = hmac.new(
|
||||
self._secret.encode("utf-8"),
|
||||
query_string.encode("utf-8"),
|
||||
hashlib.sha256,
|
||||
).hexdigest()
|
||||
return signature
|
||||
|
||||
def _get_timestamp(self) -> int:
|
||||
"""Get current timestamp in milliseconds."""
|
||||
return int(time.time() * 1000)
|
||||
|
||||
async def _request(
|
||||
self,
|
||||
method: str,
|
||||
endpoint: str,
|
||||
params: dict[str, Any] | None = None,
|
||||
signed: bool = False,
|
||||
) -> Any:
|
||||
"""Make HTTP request to Binance Spot API."""
|
||||
if self._client is None:
|
||||
raise ExchangeError("Not connected. Call connect() first.")
|
||||
|
||||
params = params or {}
|
||||
headers = {"X-MBX-APIKEY": self._api_key}
|
||||
|
||||
if signed:
|
||||
params["timestamp"] = self._get_timestamp()
|
||||
params["recvWindow"] = self._recv_window
|
||||
params["signature"] = self._sign(params)
|
||||
|
||||
url = f"{self.base_url}{endpoint}"
|
||||
|
||||
try:
|
||||
if method == "GET":
|
||||
response = await self._client.get(url, params=params, headers=headers)
|
||||
else:
|
||||
raise ValueError(f"Unsupported HTTP method: {method}")
|
||||
|
||||
data = response.json()
|
||||
|
||||
if response.status_code >= 400:
|
||||
code = data.get("code", 0)
|
||||
msg = data.get("msg", "Unknown error")
|
||||
logger.error("API error", status_code=response.status_code, code=code, msg=msg)
|
||||
if code in (-1021, -1022):
|
||||
raise AuthenticationError(msg)
|
||||
raise ExchangeError(f"[{code}] {msg}")
|
||||
|
||||
return data
|
||||
|
||||
except httpx.RequestError as e:
|
||||
logger.error("Request failed", endpoint=endpoint, error=str(e))
|
||||
raise ExchangeError(f"Request failed: {e}") from e
|
||||
|
||||
async def connect(self) -> None:
|
||||
"""Establish connection and validate credentials."""
|
||||
self._client = httpx.AsyncClient(timeout=30.0)
|
||||
|
||||
try:
|
||||
# Validate credentials by fetching account info
|
||||
await self._request("GET", "/api/v3/account", signed=True)
|
||||
logger.info(
|
||||
"Connected to Binance Spot",
|
||||
mode=self.settings.trading_mode.value,
|
||||
base_url=self.base_url,
|
||||
)
|
||||
except Exception as e:
|
||||
await self.disconnect()
|
||||
raise AuthenticationError(f"Failed to authenticate: {e}") from e
|
||||
|
||||
async def disconnect(self) -> None:
|
||||
"""Close HTTP client."""
|
||||
if self._client:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
logger.info("Disconnected from Binance Spot")
|
||||
|
||||
async def get_balance(self, asset: str) -> SpotBalance:
|
||||
"""Get spot wallet balance for a specific asset.
|
||||
|
||||
Args:
|
||||
asset: Asset symbol (e.g., "BTC", "USDT")
|
||||
|
||||
Returns:
|
||||
SpotBalance with free and locked amounts
|
||||
|
||||
Raises:
|
||||
ExchangeError: If asset not found
|
||||
"""
|
||||
data = await self._request("GET", "/api/v3/account", signed=True)
|
||||
|
||||
for balance in data.get("balances", []):
|
||||
if balance["asset"] == asset:
|
||||
return SpotBalance(
|
||||
asset=asset,
|
||||
free=Decimal(balance["free"]),
|
||||
locked=Decimal(balance["locked"]),
|
||||
updated_at=datetime.now(UTC),
|
||||
)
|
||||
|
||||
raise ExchangeError(f"Asset {asset} not found in spot wallet")
|
||||
|
||||
async def get_all_balances(self, min_balance: Decimal = Decimal("0")) -> list[SpotBalance]:
|
||||
"""Get all spot wallet balances.
|
||||
|
||||
Args:
|
||||
min_balance: Minimum total balance to include (default: 0, includes all)
|
||||
|
||||
Returns:
|
||||
List of SpotBalance objects with non-zero balances
|
||||
"""
|
||||
data = await self._request("GET", "/api/v3/account", signed=True)
|
||||
now = datetime.now(UTC)
|
||||
|
||||
balances = []
|
||||
for balance in data.get("balances", []):
|
||||
free = Decimal(balance["free"])
|
||||
locked = Decimal(balance["locked"])
|
||||
total = free + locked
|
||||
|
||||
if total > min_balance:
|
||||
balances.append(
|
||||
SpotBalance(
|
||||
asset=balance["asset"],
|
||||
free=free,
|
||||
locked=locked,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
|
||||
return balances
|
||||
@@ -271,6 +271,20 @@ class Settings(BaseSettings):
|
||||
return "wss://stream.binancefuture.com"
|
||||
return "wss://fstream.binance.com"
|
||||
|
||||
@property
|
||||
def binance_spot_base_url(self) -> str:
|
||||
"""Get the appropriate Binance Spot API base URL for current mode."""
|
||||
if self.trading_mode == TradingMode.TESTNET:
|
||||
return "https://testnet.binance.vision"
|
||||
return "https://api.binance.com"
|
||||
|
||||
@property
|
||||
def binance_spot_ws_url(self) -> str:
|
||||
"""Get the appropriate Binance Spot WebSocket URL for current mode."""
|
||||
if self.trading_mode == TradingMode.TESTNET:
|
||||
return "wss://testnet.binance.vision"
|
||||
return "wss://stream.binance.com:9443"
|
||||
|
||||
def get_active_api_key(self) -> SecretStr | None:
|
||||
"""Get the API key for the current trading mode."""
|
||||
if self.trading_mode == TradingMode.TESTNET:
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Data ingestion and storage module for TradeFinder.
|
||||
|
||||
This module provides functionality for fetching, storing, and
|
||||
retrieving market data.
|
||||
|
||||
Components:
|
||||
- DataStorage: DuckDB storage manager
|
||||
- DataFetcher: Historical data fetcher
|
||||
- schemas: Database table definitions
|
||||
|
||||
Usage:
|
||||
from tradefinder.data import DataStorage, DataFetcher
|
||||
|
||||
storage = DataStorage(Path("/data/tradefinder.duckdb"))
|
||||
storage.connect()
|
||||
storage.initialize_schema()
|
||||
|
||||
fetcher = DataFetcher(adapter, storage)
|
||||
await fetcher.sync_candles("BTCUSDT", "4h")
|
||||
"""
|
||||
|
||||
from tradefinder.data.fetcher import DataFetcher
|
||||
from tradefinder.data.storage import DataStorage
|
||||
|
||||
__all__ = ["DataStorage", "DataFetcher"]
|
||||
|
||||
207
src/tradefinder/data/fetcher.py
Normal file
207
src/tradefinder/data/fetcher.py
Normal file
@@ -0,0 +1,207 @@
|
||||
"""Historical data fetcher for market data.
|
||||
|
||||
Fetches historical OHLCV data from exchange and stores in database.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import structlog
|
||||
|
||||
from tradefinder.adapters.base import ExchangeAdapter
|
||||
from tradefinder.data.storage import DataStorage
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
# Timeframe to milliseconds mapping
|
||||
TIMEFRAME_MS = {
|
||||
"1m": 60 * 1000,
|
||||
"3m": 3 * 60 * 1000,
|
||||
"5m": 5 * 60 * 1000,
|
||||
"15m": 15 * 60 * 1000,
|
||||
"30m": 30 * 60 * 1000,
|
||||
"1h": 60 * 60 * 1000,
|
||||
"2h": 2 * 60 * 60 * 1000,
|
||||
"4h": 4 * 60 * 60 * 1000,
|
||||
"6h": 6 * 60 * 60 * 1000,
|
||||
"8h": 8 * 60 * 60 * 1000,
|
||||
"12h": 12 * 60 * 60 * 1000,
|
||||
"1d": 24 * 60 * 60 * 1000,
|
||||
"3d": 3 * 24 * 60 * 60 * 1000,
|
||||
"1w": 7 * 24 * 60 * 60 * 1000,
|
||||
}
|
||||
|
||||
|
||||
class DataFetcher:
|
||||
"""Fetches historical market data from exchange.
|
||||
|
||||
Usage:
|
||||
fetcher = DataFetcher(adapter, storage)
|
||||
await fetcher.backfill_candles("BTCUSDT", "4h", start_date, end_date)
|
||||
await fetcher.sync_candles("BTCUSDT", "4h")
|
||||
"""
|
||||
|
||||
def __init__(self, adapter: ExchangeAdapter, storage: DataStorage) -> None:
|
||||
"""Initialize fetcher with adapter and storage.
|
||||
|
||||
Args:
|
||||
adapter: Exchange adapter for fetching data
|
||||
storage: Data storage for persisting data
|
||||
"""
|
||||
self.adapter = adapter
|
||||
self.storage = storage
|
||||
|
||||
async def backfill_candles(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
start_date: datetime,
|
||||
end_date: datetime | None = None,
|
||||
batch_size: int = 1000,
|
||||
) -> int:
|
||||
"""Fetch and store historical candle data.
|
||||
|
||||
Args:
|
||||
symbol: Trading symbol (e.g., "BTCUSDT")
|
||||
timeframe: Candle timeframe (e.g., "1h", "4h")
|
||||
start_date: Start date for backfill
|
||||
end_date: End date for backfill (default: now)
|
||||
batch_size: Number of candles per API request (max 1500)
|
||||
|
||||
Returns:
|
||||
Total number of candles fetched and stored
|
||||
"""
|
||||
if end_date is None:
|
||||
end_date = datetime.now()
|
||||
|
||||
tf_ms = TIMEFRAME_MS.get(timeframe)
|
||||
if tf_ms is None:
|
||||
raise ValueError(f"Unknown timeframe: {timeframe}")
|
||||
|
||||
start_ms = int(start_date.timestamp() * 1000)
|
||||
end_ms = int(end_date.timestamp() * 1000)
|
||||
|
||||
total_fetched = 0
|
||||
current_start = start_ms
|
||||
|
||||
logger.info(
|
||||
"Starting backfill",
|
||||
symbol=symbol,
|
||||
timeframe=timeframe,
|
||||
start=start_date.isoformat(),
|
||||
end=end_date.isoformat(),
|
||||
)
|
||||
|
||||
while current_start < end_ms:
|
||||
# Fetch batch of candles
|
||||
candles = await self.adapter.get_candles(
|
||||
symbol=symbol,
|
||||
timeframe=timeframe,
|
||||
limit=min(batch_size, 1500),
|
||||
start_time=current_start,
|
||||
end_time=end_ms,
|
||||
)
|
||||
|
||||
if not candles:
|
||||
break
|
||||
|
||||
# Store candles
|
||||
inserted = self.storage.insert_candles(candles, symbol, timeframe)
|
||||
total_fetched += inserted
|
||||
|
||||
# Move to next batch
|
||||
last_candle_ts = int(candles[-1].timestamp.timestamp() * 1000)
|
||||
current_start = last_candle_ts + tf_ms
|
||||
|
||||
logger.debug(
|
||||
"Fetched batch",
|
||||
symbol=symbol,
|
||||
timeframe=timeframe,
|
||||
count=len(candles),
|
||||
total=total_fetched,
|
||||
)
|
||||
|
||||
# Break if we got fewer candles than requested (end of data)
|
||||
if len(candles) < batch_size:
|
||||
break
|
||||
|
||||
logger.info(
|
||||
"Backfill complete",
|
||||
symbol=symbol,
|
||||
timeframe=timeframe,
|
||||
total_candles=total_fetched,
|
||||
)
|
||||
|
||||
return total_fetched
|
||||
|
||||
async def sync_candles(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
lookback_days: int = 30,
|
||||
) -> int:
|
||||
"""Sync candles from last stored timestamp to now.
|
||||
|
||||
If no candles exist, fetches from lookback_days ago.
|
||||
|
||||
Args:
|
||||
symbol: Trading symbol
|
||||
timeframe: Candle timeframe
|
||||
lookback_days: Days to look back if no existing data
|
||||
|
||||
Returns:
|
||||
Number of new candles fetched
|
||||
"""
|
||||
# Get latest stored candle timestamp
|
||||
latest_ts = self.storage.get_latest_candle_timestamp(symbol, timeframe)
|
||||
|
||||
if latest_ts:
|
||||
# Start from next candle after latest
|
||||
tf_ms = TIMEFRAME_MS.get(timeframe, 3600000)
|
||||
start_date = latest_ts + timedelta(milliseconds=tf_ms)
|
||||
logger.info(
|
||||
"Syncing from last candle",
|
||||
symbol=symbol,
|
||||
timeframe=timeframe,
|
||||
last_candle=latest_ts.isoformat(),
|
||||
)
|
||||
else:
|
||||
# No existing data, use lookback
|
||||
start_date = datetime.now() - timedelta(days=lookback_days)
|
||||
logger.info(
|
||||
"No existing data, fetching from lookback",
|
||||
symbol=symbol,
|
||||
timeframe=timeframe,
|
||||
lookback_days=lookback_days,
|
||||
)
|
||||
|
||||
return await self.backfill_candles(
|
||||
symbol=symbol,
|
||||
timeframe=timeframe,
|
||||
start_date=start_date,
|
||||
)
|
||||
|
||||
async def fetch_latest_candles(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
limit: int = 100,
|
||||
) -> int:
|
||||
"""Fetch and store the most recent candles.
|
||||
|
||||
Args:
|
||||
symbol: Trading symbol
|
||||
timeframe: Candle timeframe
|
||||
limit: Number of recent candles to fetch
|
||||
|
||||
Returns:
|
||||
Number of candles fetched
|
||||
"""
|
||||
candles = await self.adapter.get_candles(
|
||||
symbol=symbol,
|
||||
timeframe=timeframe,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
if candles:
|
||||
return self.storage.insert_candles(candles, symbol, timeframe)
|
||||
return 0
|
||||
54
src/tradefinder/data/schemas.py
Normal file
54
src/tradefinder/data/schemas.py
Normal file
@@ -0,0 +1,54 @@
|
||||
"""Database schemas for market data storage."""
|
||||
|
||||
CANDLES_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS candles (
|
||||
symbol VARCHAR NOT NULL,
|
||||
timeframe VARCHAR NOT NULL,
|
||||
timestamp TIMESTAMP NOT NULL,
|
||||
open DOUBLE NOT NULL,
|
||||
high DOUBLE NOT NULL,
|
||||
low DOUBLE NOT NULL,
|
||||
close DOUBLE NOT NULL,
|
||||
volume DOUBLE NOT NULL,
|
||||
PRIMARY KEY (symbol, timeframe, timestamp)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_candles_symbol_tf
|
||||
ON candles (symbol, timeframe, timestamp DESC);
|
||||
"""
|
||||
|
||||
TRADES_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS trades (
|
||||
id VARCHAR PRIMARY KEY,
|
||||
symbol VARCHAR NOT NULL,
|
||||
side VARCHAR NOT NULL,
|
||||
position_side VARCHAR NOT NULL,
|
||||
entry_price DOUBLE NOT NULL,
|
||||
exit_price DOUBLE,
|
||||
quantity DOUBLE NOT NULL,
|
||||
pnl_usdt DOUBLE,
|
||||
pnl_pct DOUBLE,
|
||||
strategy VARCHAR NOT NULL,
|
||||
entry_time TIMESTAMP NOT NULL,
|
||||
exit_time TIMESTAMP,
|
||||
status VARCHAR NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_trades_symbol ON trades (symbol);
|
||||
CREATE INDEX IF NOT EXISTS idx_trades_strategy ON trades (strategy);
|
||||
CREATE INDEX IF NOT EXISTS idx_trades_entry_time ON trades (entry_time DESC);
|
||||
"""
|
||||
|
||||
FUNDING_RATES_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS funding_rates (
|
||||
symbol VARCHAR NOT NULL,
|
||||
funding_rate DOUBLE NOT NULL,
|
||||
funding_time TIMESTAMP NOT NULL,
|
||||
mark_price DOUBLE NOT NULL,
|
||||
PRIMARY KEY (symbol, funding_time)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_funding_symbol ON funding_rates (symbol, funding_time DESC);
|
||||
"""
|
||||
|
||||
ALL_SCHEMAS = [CANDLES_SCHEMA, TRADES_SCHEMA, FUNDING_RATES_SCHEMA]
|
||||
242
src/tradefinder/data/storage.py
Normal file
242
src/tradefinder/data/storage.py
Normal file
@@ -0,0 +1,242 @@
|
||||
"""DuckDB storage manager for market data.
|
||||
|
||||
Provides async-compatible interface for storing and retrieving
|
||||
market data using DuckDB.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import duckdb
|
||||
import structlog
|
||||
|
||||
from tradefinder.adapters.types import Candle, FundingRate
|
||||
from tradefinder.data.schemas import ALL_SCHEMAS
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
class DataStorage:
|
||||
"""DuckDB storage manager for market data.
|
||||
|
||||
Usage:
|
||||
storage = DataStorage(Path("/data/tradefinder.duckdb"))
|
||||
storage.connect()
|
||||
storage.initialize_schema()
|
||||
storage.insert_candles(candles)
|
||||
storage.disconnect()
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: Path) -> None:
|
||||
"""Initialize storage with database path.
|
||||
|
||||
Args:
|
||||
db_path: Path to DuckDB database file
|
||||
"""
|
||||
self.db_path = db_path
|
||||
self._conn: duckdb.DuckDBPyConnection | None = None
|
||||
|
||||
def connect(self) -> None:
|
||||
"""Connect to the database."""
|
||||
# Ensure parent directory exists
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self._conn = duckdb.connect(str(self.db_path))
|
||||
logger.info("Connected to DuckDB", path=str(self.db_path))
|
||||
|
||||
def disconnect(self) -> None:
|
||||
"""Close database connection."""
|
||||
if self._conn:
|
||||
self._conn.close()
|
||||
self._conn = None
|
||||
logger.info("Disconnected from DuckDB")
|
||||
|
||||
def __enter__(self) -> "DataStorage":
|
||||
"""Context manager entry."""
|
||||
self.connect()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None:
|
||||
"""Context manager exit."""
|
||||
self.disconnect()
|
||||
|
||||
@property
|
||||
def conn(self) -> duckdb.DuckDBPyConnection:
|
||||
"""Get database connection."""
|
||||
if self._conn is None:
|
||||
raise RuntimeError("Not connected. Call connect() first.")
|
||||
return self._conn
|
||||
|
||||
def initialize_schema(self) -> None:
|
||||
"""Create all database tables and indexes."""
|
||||
for schema in ALL_SCHEMAS:
|
||||
# Execute each statement separately
|
||||
for statement in schema.strip().split(";"):
|
||||
statement = statement.strip()
|
||||
if statement:
|
||||
self.conn.execute(statement)
|
||||
|
||||
logger.info("Database schema initialized")
|
||||
|
||||
def insert_candles(self, candles: list[Candle], symbol: str, timeframe: str) -> int:
|
||||
"""Insert candles into the database.
|
||||
|
||||
Args:
|
||||
candles: List of Candle objects to insert
|
||||
symbol: Trading symbol (e.g., "BTCUSDT")
|
||||
timeframe: Candle timeframe (e.g., "1h", "4h")
|
||||
|
||||
Returns:
|
||||
Number of candles inserted
|
||||
"""
|
||||
if not candles:
|
||||
return 0
|
||||
|
||||
# Prepare data for insertion
|
||||
data = [
|
||||
(
|
||||
symbol,
|
||||
timeframe,
|
||||
c.timestamp,
|
||||
float(c.open),
|
||||
float(c.high),
|
||||
float(c.low),
|
||||
float(c.close),
|
||||
float(c.volume),
|
||||
)
|
||||
for c in candles
|
||||
]
|
||||
|
||||
# Use INSERT OR REPLACE to handle duplicates
|
||||
self.conn.executemany(
|
||||
"""
|
||||
INSERT OR REPLACE INTO candles
|
||||
(symbol, timeframe, timestamp, open, high, low, close, volume)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
data,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"Inserted candles",
|
||||
symbol=symbol,
|
||||
timeframe=timeframe,
|
||||
count=len(candles),
|
||||
)
|
||||
return len(candles)
|
||||
|
||||
def get_candles(
|
||||
self,
|
||||
symbol: str,
|
||||
timeframe: str,
|
||||
start: datetime | None = None,
|
||||
end: datetime | None = None,
|
||||
limit: int | None = None,
|
||||
) -> list[Candle]:
|
||||
"""Retrieve candles from the database.
|
||||
|
||||
Args:
|
||||
symbol: Trading symbol
|
||||
timeframe: Candle timeframe
|
||||
start: Start timestamp (inclusive)
|
||||
end: End timestamp (inclusive)
|
||||
limit: Maximum number of candles to return
|
||||
|
||||
Returns:
|
||||
List of Candle objects, oldest first
|
||||
"""
|
||||
query = """
|
||||
SELECT timestamp, open, high, low, close, volume
|
||||
FROM candles
|
||||
WHERE symbol = ? AND timeframe = ?
|
||||
"""
|
||||
params: list[str | datetime | int] = [symbol, timeframe]
|
||||
|
||||
if start:
|
||||
query += " AND timestamp >= ?"
|
||||
params.append(start)
|
||||
if end:
|
||||
query += " AND timestamp <= ?"
|
||||
params.append(end)
|
||||
|
||||
query += " ORDER BY timestamp ASC"
|
||||
|
||||
if limit:
|
||||
query += " LIMIT ?"
|
||||
params.append(limit)
|
||||
|
||||
result = self.conn.execute(query, params).fetchall()
|
||||
|
||||
from decimal import Decimal
|
||||
|
||||
return [
|
||||
Candle(
|
||||
timestamp=row[0],
|
||||
open=Decimal(str(row[1])),
|
||||
high=Decimal(str(row[2])),
|
||||
low=Decimal(str(row[3])),
|
||||
close=Decimal(str(row[4])),
|
||||
volume=Decimal(str(row[5])),
|
||||
)
|
||||
for row in result
|
||||
]
|
||||
|
||||
def get_latest_candle_timestamp(self, symbol: str, timeframe: str) -> datetime | None:
|
||||
"""Get the timestamp of the most recent candle.
|
||||
|
||||
Args:
|
||||
symbol: Trading symbol
|
||||
timeframe: Candle timeframe
|
||||
|
||||
Returns:
|
||||
Timestamp of latest candle, or None if no candles exist
|
||||
"""
|
||||
result = self.conn.execute(
|
||||
"""
|
||||
SELECT MAX(timestamp) FROM candles
|
||||
WHERE symbol = ? AND timeframe = ?
|
||||
""",
|
||||
[symbol, timeframe],
|
||||
).fetchone()
|
||||
|
||||
return result[0] if result and result[0] else None
|
||||
|
||||
def insert_funding_rate(self, rate: FundingRate) -> None:
|
||||
"""Insert a funding rate record.
|
||||
|
||||
Args:
|
||||
rate: FundingRate object to insert
|
||||
"""
|
||||
self.conn.execute(
|
||||
"""
|
||||
INSERT OR REPLACE INTO funding_rates
|
||||
(symbol, funding_rate, funding_time, mark_price)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""",
|
||||
[
|
||||
rate.symbol,
|
||||
float(rate.funding_rate),
|
||||
rate.funding_time,
|
||||
float(rate.mark_price),
|
||||
],
|
||||
)
|
||||
|
||||
def get_candle_count(self, symbol: str, timeframe: str) -> int:
|
||||
"""Get the number of candles stored for a symbol/timeframe.
|
||||
|
||||
Args:
|
||||
symbol: Trading symbol
|
||||
timeframe: Candle timeframe
|
||||
|
||||
Returns:
|
||||
Number of candles
|
||||
"""
|
||||
result = self.conn.execute(
|
||||
"""
|
||||
SELECT COUNT(*) FROM candles
|
||||
WHERE symbol = ? AND timeframe = ?
|
||||
""",
|
||||
[symbol, timeframe],
|
||||
).fetchone()
|
||||
|
||||
return result[0] if result else 0
|
||||
Reference in New Issue
Block a user