diff options
113 files changed, 7703 insertions, 11164 deletions
diff --git a/.env.example b/.env.example index 7a2751f..bdc6a67 100644 --- a/.env.example +++ b/.env.example @@ -19,6 +19,17 @@ TELEGRAM_CHAT_ID= TELEGRAM_ENABLED=false LOG_FORMAT=json HEALTH_PORT=8080 -CIRCUIT_BREAKER_THRESHOLD=5 -CIRCUIT_BREAKER_TIMEOUT=60 METRICS_AUTH_TOKEN= + +# News Collector +FINNHUB_API_KEY= +NEWS_POLL_INTERVAL=300 +SENTIMENT_AGGREGATE_INTERVAL=900 + +# Stock Selector +SELECTOR_FINAL_TIME=15:30 +SELECTOR_MAX_PICKS=3 + +# LLM (for stock selector) +ANTHROPIC_API_KEY= +ANTHROPIC_MODEL=claude-sonnet-4-20250514 diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..6e33f57 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,106 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +US stock trading platform built as a Python microservices architecture. Uses Alpaca Markets API for market data and order execution. Services communicate via Redis Streams and persist to PostgreSQL. + +## Common Commands + +```bash +make infra # Start Redis + Postgres (required before running services/tests) +make up # Start all services via Docker Compose +make down # Stop all services +make test # Run all tests (pytest -v) +make lint # Lint check (ruff check + format check) +make format # Auto-fix lint + format +make migrate # Run DB migrations (alembic upgrade head, from shared/) +make migrate-new msg="description" # Create new migration +make ci # Full CI: install deps, lint, test, Docker build +make e2e # End-to-end tests +``` + +Run a single test file: `pytest services/strategy-engine/tests/test_rsi_strategy.py -v` + +## Architecture + +### Services (each in `services/<name>/`, each has its own Dockerfile) + +- **data-collector** (port 8080): Fetches stock bars from Alpaca, publishes `CandleEvent` to Redis stream `candles` +- **news-collector** (port 8084): Continuously collects news from 7 sources (Finnhub, RSS, SEC EDGAR, Truth Social, Reddit, Fear & Greed, Fed), runs sentiment aggregation every 15 min +- **strategy-engine** (port 8081): Consumes candle events, runs strategies, publishes `SignalEvent` to stream `signals`. Also runs the stock selector at 15:30 ET daily +- **order-executor** (port 8082): Consumes signals, runs risk checks, places orders via Alpaca, publishes `OrderEvent` to stream `orders` +- **portfolio-manager** (port 8083): Tracks positions, PnL, portfolio snapshots +- **api** (port 8000): FastAPI REST endpoint layer +- **backtester**: Offline backtesting engine with walk-forward analysis + +### Event Flow + +``` +Alpaca → data-collector → [candles stream] → strategy-engine → [signals stream] → order-executor → [orders stream] → portfolio-manager + +News sources → news-collector → [news stream] → sentiment aggregator → symbol_scores DB + ↓ + stock selector (15:30 ET) → [selected_stocks stream] → MOC strategy → signals +``` + +All inter-service events use `shared/src/shared/events.py` (CandleEvent, SignalEvent, OrderEvent, NewsEvent) serialized as JSON over Redis Streams via `shared/src/shared/broker.py` (RedisBroker). + +### Shared Library (`shared/`) + +Installed as editable package (`pip install -e shared/`). Contains: +- `models.py` — Pydantic domain models: Candle, Signal, Order, Position, NewsItem, NewsCategory +- `sentiment_models.py` — SymbolScore, MarketSentiment, SelectedStock, Candidate +- `sa_models.py` — SQLAlchemy ORM models (CandleRow, SignalRow, OrderRow, PortfolioSnapshotRow, NewsItemRow, SymbolScoreRow, MarketSentimentRow, StockSelectionRow) +- `broker.py` — RedisBroker (async Redis Streams pub/sub with consumer groups) +- `db.py` — Database class (async SQLAlchemy 2.0), includes news/sentiment/selection CRUD methods +- `alpaca.py` — AlpacaClient (async aiohttp client for Alpaca Trading + Market Data APIs) +- `events.py` — Event types and serialization (CandleEvent, SignalEvent, OrderEvent, NewsEvent) +- `sentiment.py` — SentimentData (legacy gating) + SentimentAggregator (freshness-weighted composite scoring) +- `config.py`, `logging.py`, `metrics.py`, `notifier.py` (Telegram), `resilience.py`, `healthcheck.py` + +DB migrations live in `shared/alembic/`. + +### Strategy System (`services/strategy-engine/strategies/`) + +Strategies extend `BaseStrategy` (in `strategies/base.py`) and implement `on_candle()`, `configure()`, `warmup_period`. The plugin loader (`strategy_engine/plugin_loader.py`) auto-discovers `*.py` files in the strategies directory and loads YAML config from `strategies/config/<strategy_name>.yaml`. + +BaseStrategy provides optional filters (ADX regime, volume, ATR-based stops) via `_init_filters()` and `_apply_filters()`. + +### News-Driven Stock Selector (`services/strategy-engine/src/strategy_engine/stock_selector.py`) + +Dynamic stock selection for MOC (Market on Close) trading. Runs daily at 15:30 ET via `strategy-engine`: + +1. **Candidate Pool**: Top 20 by sentiment score + LLM-recommended stocks from today's news +2. **Technical Filter**: RSI 30-70, price > 20 EMA, volume > 50% average +3. **LLM Final Selection**: Claude picks 2-3 stocks with rationale + +Market gating: blocks all trades when Fear & Greed ≤ 20 or VIX > 30 (`risk_off` regime). + +### News Collector (`services/news-collector/`) + +7 collectors extending `BaseCollector` in `collectors/`: +- `finnhub.py` (5min), `rss.py` (10min), `reddit.py` (15min), `truth_social.py` (15min), `sec_edgar.py` (30min), `fear_greed.py` (1hr), `fed.py` (1hr) +- All use VADER (nltk) for sentiment scoring +- Provider abstraction via `BaseCollector` for future paid API swap (config change only) + +Sentiment aggregation (every 15min) computes per-symbol composite scores with freshness decay and category weights (policy 0.3, news 0.3, social 0.2, filing 0.2). + +### CLI (`cli/`) + +Click-based CLI installed as `trading` command. Depends on the shared library. + +## Tech Stack + +- Python 3.12+, async throughout (asyncio, aiohttp) +- Pydantic for models, SQLAlchemy 2.0 async ORM, Alembic for migrations +- Redis Streams for inter-service messaging +- PostgreSQL 16 for persistence +- Ruff for linting/formatting (line-length=100) +- pytest + pytest-asyncio (asyncio_mode="auto") +- Docker Compose for deployment; monitoring stack (Grafana/Prometheus/Loki) available via `--profile monitoring` + +## Environment + +Copy `.env.example` to `.env`. Key vars: `ALPACA_API_KEY`, `ALPACA_API_SECRET`, `ALPACA_PAPER=true`, `DRY_RUN=true`, `DATABASE_URL`, `REDIS_URL`, `FINNHUB_API_KEY`, `ANTHROPIC_API_KEY`. DRY_RUN=true simulates order fills without hitting Alpaca. Stock selector requires `ANTHROPIC_API_KEY` to be set. @@ -1,4 +1,4 @@ -.PHONY: infra up down logs test lint format migrate migrate-down migrate-new ci e2e optimize screen +.PHONY: infra up down logs test lint format migrate migrate-down migrate-new ci e2e optimize screen backtest-moc infra: docker compose up -d redis postgres @@ -38,8 +38,8 @@ ci: e2e: ./scripts/e2e-test.sh -optimize: - python scripts/optimize_asian_rsi.py - screen: python scripts/stock_screener.py + +backtest-moc: + python scripts/backtest_moc.py diff --git a/cli/src/trading_cli/commands/backtest.py b/cli/src/trading_cli/commands/backtest.py index 01fe092..3876f1b 100644 --- a/cli/src/trading_cli/commands/backtest.py +++ b/cli/src/trading_cli/commands/backtest.py @@ -20,9 +20,9 @@ def backtest(): @backtest.command() @click.option("--strategy", required=True, help="Strategy name to backtest") -@click.option("--symbol", required=True, help="Trading symbol (e.g. BTCUSDT)") +@click.option("--symbol", required=True, help="Trading symbol (e.g. AAPL)") @click.option("--timeframe", default="1h", show_default=True, help="Candle timeframe") -@click.option("--balance", default=10000.0, show_default=True, help="Initial balance in USDT") +@click.option("--balance", default=10000.0, show_default=True, help="Initial balance in USD") @click.option( "--output", "output_format", diff --git a/cli/src/trading_cli/commands/data.py b/cli/src/trading_cli/commands/data.py index 2810a07..1ecc15f 100644 --- a/cli/src/trading_cli/commands/data.py +++ b/cli/src/trading_cli/commands/data.py @@ -16,10 +16,10 @@ def data(): @data.command() -@click.option("--symbol", required=True, help="Trading symbol (e.g. BTCUSDT)") +@click.option("--symbol", required=True, help="Trading symbol (e.g. AAPL)") @click.option("--timeframe", default="1m", show_default=True, help="Candle timeframe") def collect(symbol, timeframe): - """Start collecting live market data for a symbol.""" + """Start collecting live stock market data for a symbol.""" click.echo(f"To collect live data for {symbol} at {timeframe}, run the data-collector service:") click.echo() click.echo(" docker compose up -d data-collector") @@ -31,16 +31,14 @@ def collect(symbol, timeframe): @data.command() -@click.option("--symbol", required=True, help="Trading symbol (e.g. BTCUSDT)") -@click.option("--timeframe", default="1m", show_default=True, help="Candle timeframe") +@click.option("--symbol", required=True, help="Trading symbol (e.g. AAPL)") +@click.option("--timeframe", default="1Day", show_default=True, help="Bar timeframe") @click.option("--from", "since", default=None, help="Start date (ISO format)") -@click.option("--limit", default=1000, show_default=True, help="Number of candles to fetch") +@click.option("--limit", default=1000, show_default=True, help="Number of bars to fetch") def history(symbol, timeframe, since, limit): - """Download historical market data for a symbol.""" - sys.path.insert(0, str(_ROOT / "services" / "data-collector" / "src")) - + """Download historical stock market data for a symbol.""" try: - from data_collector.binance_rest import fetch_historical_candles + from shared.alpaca_client import AlpacaClient from shared.db import Database from shared.config import Settings except ImportError as e: @@ -48,59 +46,48 @@ def history(symbol, timeframe, since, limit): sys.exit(1) async def _fetch(): - import ccxt.async_support as ccxt from datetime import datetime, timezone settings = Settings() db = Database(settings.database_url) await db.connect() - # Parse the since date to a timestamp in ms + start = None if since: try: - dt = datetime.fromisoformat(since).replace(tzinfo=timezone.utc) - since_ms = int(dt.timestamp() * 1000) + start = datetime.fromisoformat(since).replace(tzinfo=timezone.utc) except ValueError: click.echo( f"Error: Invalid date format '{since}'. Use ISO format (e.g. 2024-01-01).", err=True, ) sys.exit(1) - else: - # Default: fetch from 1000 candles ago (approximate) - since_ms = None - - # Normalize symbol for ccxt (BTCUSDT -> BTC/USDT) - ccxt_symbol = symbol - if "/" not in symbol and "USDT" in symbol: - base = symbol.replace("USDT", "") - ccxt_symbol = f"{base}/USDT" - - exchange = ccxt.binance( - { - "apiKey": settings.binance_api_key, - "secret": settings.binance_api_secret, - } + + client = AlpacaClient( + api_key=settings.alpaca_api_key, + api_secret=settings.alpaca_api_secret, + base_url=getattr(settings, "alpaca_base_url", "https://paper-api.alpaca.markets"), ) try: - kwargs = {"limit": limit} - if since_ms is not None: - kwargs["since"] = since_ms - - candles = await fetch_historical_candles(exchange, ccxt_symbol, timeframe, **kwargs) + candles = await client.get_historical_bars( + symbol=symbol, + timeframe=timeframe, + start=start, + limit=limit, + ) count = 0 for candle in candles: await db.insert_candle(candle) count += 1 - click.echo(f"Saved {count} candles for {symbol} ({timeframe}) to database.") + click.echo(f"Saved {count} bars for {symbol} ({timeframe}) to database.") except Exception as e: - click.echo(f"Error fetching candles: {e}", err=True) + click.echo(f"Error fetching bars: {e}", err=True) sys.exit(1) finally: - await exchange.close() + await client.close() await db.close() asyncio.run(_fetch()) diff --git a/cli/src/trading_cli/commands/trade.py b/cli/src/trading_cli/commands/trade.py index f90e0ed..6bbd5a6 100644 --- a/cli/src/trading_cli/commands/trade.py +++ b/cli/src/trading_cli/commands/trade.py @@ -9,7 +9,7 @@ def trade(): @trade.command() @click.option("--strategy", required=True, help="Strategy name to run") -@click.option("--symbol", required=True, help="Trading symbol (e.g. BTCUSDT)") +@click.option("--symbol", required=True, help="Trading symbol (e.g. AAPL)") def start(strategy, symbol): """Start a trading bot for a strategy and symbol.""" click.echo(f"Starting trading bot: strategy={strategy}, symbol={symbol}...") diff --git a/cli/src/trading_cli/main.py b/cli/src/trading_cli/main.py index db3c282..1129bdd 100644 --- a/cli/src/trading_cli/main.py +++ b/cli/src/trading_cli/main.py @@ -10,7 +10,7 @@ from trading_cli.commands.service import service @click.group() @click.version_option(version="0.1.0") def cli(): - """Trading Platform CLI — Binance spot crypto trading""" + """Trading Platform CLI — US stock trading""" pass diff --git a/docker-compose.yml b/docker-compose.yml index e981f74..63630ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,7 @@ services: volumes: - postgres_data:/var/lib/postgresql/data healthcheck: - test: ["CMD-LINE", "pg_isready", "-U", "trading"] + test: ["CMD", "pg_isready", "-U", "trading"] interval: 5s timeout: 3s retries: 5 @@ -122,6 +122,25 @@ services: retries: 3 restart: unless-stopped + news-collector: + build: + context: . + dockerfile: services/news-collector/Dockerfile + env_file: .env + ports: + - "8084:8084" + depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8084/health')"] + interval: 10s + timeout: 5s + retries: 3 + restart: unless-stopped + loki: image: grafana/loki:latest profiles: ["monitoring"] diff --git a/docs/TODO.md b/docs/TODO.md index d737b0f..fd4f7eb 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -1,201 +1,147 @@ -# Trading Platform — TODO +# US Stock Trading Platform — TODO -> Last updated: 2026-04-01 +> Last updated: 2026-04-02 ## Current State -- **298 tests**, lint clean, production-ready 인프라 -- 8 strategies, 6 services, full monitoring/CI/CD stack -- **트레이딩 전략 업그레이드 필요** — 아래 상세 +- **375 tests**, lint clean +- **US 주식 전용** (Alpaca API, 수수료 0%) +- 6 microservices + CLI + shared library +- MOC (Market on Close) 전략 + 기술적 전략 7개 +- Prometheus/Grafana/Loki 모니터링, CI/CD, Telegram 알림 +- Claude 기반 종목 스크리너 --- -## Trading Strategy Upgrade Plan - -### Phase 1: 백테스터 현실화 (최우선) - -현재 백테스트 결과는 슬리피지/수수료 미포함으로 **실제 수익과 큰 차이**가 있음. 이것부터 수정해야 전략 개선의 효과를 정확히 측정 가능. - -#### 1-1. 슬리피지 + 수수료 모델링 -- **파일:** `backtester/simulator.py` -- **현재:** 시그널 가격 그대로 체결, 수수료 0 -- **수정:** 매수 시 `price * (1 + slippage_pct)`, 매도 시 `price * (1 - slippage_pct)` -- 수수료: `cost = price * quantity * fee_pct` (maker: 0.05%, taker: 0.1%) -- 슬리피지를 주문 크기에 비례하게 (대형 주문 → 더 큰 슬리피지) -- **설정:** `BacktestConfig`에 `slippage_pct`, `maker_fee_pct`, `taker_fee_pct` 추가 - -#### 1-2. 손절/익절 자동 실행 -- **파일:** `backtester/simulator.py` -- **현재:** 시그널로만 매매, 스탑 없음 -- **수정:** 각 포지션에 stop_loss, take_profit 가격 추적 -- 매 캔들마다 `high >= take_profit` 또는 `low <= stop_loss` 체크 → 자동 청산 -- `engine.py`에서 캔들 처리 시 시뮬레이터에 현재 가격 전달 - -#### 1-3. 공매도 지원 -- **파일:** `backtester/simulator.py` -- **현재:** 매도는 보유 수량 내에서만 가능 -- **수정:** `allow_short: bool` 설정, 공매도 시 음수 포지션 허용 -- 공매도 수수료 (borrow fee) 추가 - -#### 1-4. Walk-Forward Analysis -- **파일:** `backtester/engine.py` (신규 클래스) -- **현재:** 전체 데이터로 백테스트 → 과적합 위험 -- **수정:** `WalkForwardEngine` 클래스 - - 데이터를 N개 구간으로 분할 - - 각 구간: in-sample (파라미터 최적화) → out-of-sample (검증) - - 최종 결과는 out-of-sample 구간만 합산 -- 파라미터 최적화: grid search 또는 random search - -#### 1-5. 메트릭 정확도 개선 -- **파일:** `backtester/metrics.py` -- Sharpe/Sortino를 per-trade가 아닌 **일별 수익률 기반**으로 계산 -- Risk-free rate 설정 추가 (기본 5%) -- Recovery Factor (총수익 / 최대 drawdown) 추가 -- 최대 연속 손실 횟수 추가 -- 인트라 트레이드 drawdown (진입 후 최저점) 계산 +## Architecture ---- +``` +Alpaca API → data-collector (REST polling) + → Redis Streams → strategy-engine (MOC + 기술전략) + → signals → order-executor (Alpaca 주문) + → orders → portfolio-manager (포지션 추적) -### Phase 2: 전략 공통 인프라 - -모든 전략에 적용할 공통 기능. 개별 전략 개선 전에 인프라를 먼저 구축. - -#### 2-1. ATR 기반 동적 손절/익절 -- **파일:** `strategies/base.py` + 각 전략 -- **현재:** 어떤 전략도 손절/익절을 설정하지 않음 -- **수정:** `BaseStrategy`에 `calculate_stop_loss(candle, atr)` 메서드 추가 -- ATR (Average True Range) 유틸리티 함수 (`shared/` 또는 `strategies/indicators/`) -- 손절: entry - ATR * multiplier, 익절: entry + ATR * reward_ratio -- Signal에 `stop_loss`, `take_profit` 필드 추가 (`shared/models.py`) - -#### 2-2. 추세/횡보 레짐 필터 (ADX) -- **파일:** `strategies/indicators/adx.py` (신규) -- ADX (Average Directional Index) 계산 유틸리티 -- ADX > 25 = 추세장, ADX < 20 = 횡보장 -- 각 전략이 레짐에 따라 동작 변경: - - 추세 추종 전략 (MACD, EMA): ADX < 20이면 시그널 무시 - - 평균 회귀 전략 (RSI, Bollinger, Grid): ADX > 30이면 시그널 무시 - -#### 2-3. 볼륨 확인 필터 -- **파일:** 각 전략 -- **현재:** 모든 전략이 볼륨을 무시 (RSI, MACD, EMA 등) -- **수정:** 시그널 발생 시 해당 캔들의 볼륨이 최근 N개 평균 대비 일정 비율 이상인지 확인 -- 볼륨 < 평균의 50%면 시그널 무시 (유동성 부족) -- 볼륨 > 평균의 200%면 시그널 가중치 증가 - -#### 2-4. 시그널 강도 (Conviction Score) -- **파일:** `shared/models.py` Signal + 각 전략 -- **현재:** 시그널은 BUY/SELL/quantity만 있음 -- **수정:** Signal에 `conviction: float` (0.0~1.0) 필드 추가 - - RSI 5 → conviction 0.9, RSI 28 → conviction 0.3 - - MACD 히스토그램이 0에서 먼 크로스 → 높은 conviction -- Combined 전략에서 conviction 기반 가중치 사용 -- RiskManager에서 conviction 기반 포지션 사이징 - -#### 2-5. 지표 라이브러리 -- **디렉토리:** `strategies/indicators/` (신규) -- 재사용 가능한 기술 지표 함수: - - `atr(highs, lows, closes, period)` — Average True Range - - `adx(highs, lows, closes, period)` — Average Directional Index - - `ema(series, period)` — Exponential Moving Average - - `sma(series, period)` — Simple Moving Average - - `rsi(closes, period)` — RSI - - `bollinger_bands(closes, period, num_std)` — Bollinger - - `macd(closes, fast, slow, signal)` — MACD - - `volume_sma(volumes, period)` — Volume SMA -- 각 전략에서 직접 계산하지 않고 공통 라이브러리 사용 -- 테스트: 각 지표에 대한 unit test +Claude API → stock_screener.py (종목 분석/추천) +FastAPI → REST API (/api/v1/portfolio, orders, strategies) +``` ---- +## 핵심 매매 전략: MOC (Market on Close) -### Phase 3: 개별 전략 고도화 - -Phase 1-2 완료 후 각 전략을 전문가 수준으로 업그레이드. - -#### 3-1. RSI 전략 개선 -- [ ] RSI 다이버전스 감지 (가격 신고가 + RSI 하락 = 약세 다이버전스) -- [ ] ADX 레짐 필터 적용 (추세장에서는 RSI 매수 신호 무시) -- [ ] RSI 강도별 conviction score (RSI 5 vs RSI 28) -- [ ] ATR 기반 손절/익절 -- [ ] 볼륨 확인 필터 - -#### 3-2. MACD 전략 개선 -- [ ] 히스토그램 크로스오버 + MACD 제로라인 크로스오버 구분 -- [ ] MACD 다이버전스 감지 -- [ ] ADX 추세 확인 (ADX < 20이면 시그널 무시) -- [ ] 제로라인으로부터 거리 기반 시그널 강도 -- [ ] ATR 기반 손절 - -#### 3-3. Grid 전략 개선 -- [ ] ADX 기반 레짐 필터 (추세장 진입 차단) -- [ ] 동적 그리드 재설정 (실현 변동성 기반 범위 조정) -- [ ] 그리드 외 이탈 시 전 포지션 청산 + 알림 -- [ ] 볼륨 프로파일 기반 비균등 그리드 간격 - -#### 3-4. Bollinger Bands 전략 개선 -- [ ] 스퀴즈 감지 (밴드 압축 → 브레이크아웃 대비) -- [ ] %B 지표 활용 (밴드 내 위치 0~1) -- [ ] RSI 확인 (하단 밴드 터치 + RSI < 30 = 강한 매수) -- [ ] 볼륨 스파이크 확인 - -#### 3-5. EMA Crossover 전략 개선 -- [ ] ADX > 25 필터 (강한 추세만 진입) -- [ ] 풀백 진입 (크로스 후 단기 EMA로 되돌림 시 진입) -- [ ] 50 SMA 위/아래 필터 (장기 추세 방향 확인) -- [ ] 볼륨 확인 - -#### 3-6. VWAP 전략 개선 -- [ ] 일중 리셋 (매일 00:00 UTC에 VWAP 재계산) -- [ ] VWAP 표준편차 밴드 추가 (1σ, 2σ) -- [ ] ATR 기반 deviation threshold (고정값 대신 변동성 적응형) -- [ ] 세션 필터 (저유동성 시간대 진입 차단) - -#### 3-7. Volume Profile 전략 개선 -- [ ] HVN/LVN (고/저볼륨 노드) 식별 -- [ ] 세션 기반 프로파일 리셋 -- [ ] POC를 동적 지지/저항선으로 활용 -- [ ] 볼륨 델타 (매수량 - 매도량) 추적 - -#### 3-8. Combined 전략 개선 -- [ ] Sub-strategy conviction score 반영 -- [ ] Sub-strategy 간 상관관계 행렬 계산 → 중복 시그널 감쇄 -- [ ] 적응형 가중치 (최근 win rate 기반 동적 가중치 조정) -- [ ] 포트폴리오 집중도 제한 +``` +[매일 ET 15:50] Claude 종목 분석 → 매수 종목 선정 +[ET 15:50~16:00] 장 마감 직전 매수 (MOC 주문) +[다음날 ET 9:35~10:00] 시가 매도 + +조건: 양봉 + 볼륨 > 평균 + RSI 30~60 + EMA 위 + 모멘텀 양호 +손절: -2%, 포지션당 자본금 20%, 최대 5종목 +``` --- -### Phase 4: 리스크 관리 고도화 +## Remaining Work + +### 즉시 해야 할 것 -#### 4-1. 포트폴리오 레벨 리스크 -- [ ] 전체 노출도 제한 (총 포지션 가치 / 잔고 비율) -- [ ] 포지션 간 상관관계 계산 → 실효 리스크 산출 -- [ ] VaR (Value at Risk) 계산 — 95% 신뢰 구간 +#### 1. MOC 백테스트 스크립트 +- [ ] `scripts/backtest_moc.py` — 합성 데이터로 MOC 전략 파라미터 최적화 +- [ ] 5개 주식 (AAPL, MSFT, TSLA, NVDA, AMZN) 대상 90일 백테스트 +- [ ] RSI 범위, 손절률, EMA period 그리드 서치 +- [ ] Makefile target: `make backtest-moc` -#### 4-2. 동적 포지션 축소 -- [ ] Drawdown이 일정 수준 넘으면 포지션 크기 자동 축소 -- [ ] 연속 손실 N회 시 거래 일시 중단 → Telegram 알림 -- [ ] 시간대별 리스크 조정 (주말, 공휴일 축소) +#### 2. 실제 데이터 백테스트 +- [ ] Alpaca API로 과거 데이터 다운로드 → DB 저장 +- [ ] 실제 데이터 기반 MOC 전략 검증 +- [ ] Walk-forward analysis로 과적합 확인 -#### 4-3. 시나리오 분석 -- [ ] 과거 극단 이벤트 (FTX 사태, Luna 등)에 대한 포트폴리오 영향 시뮬레이션 -- [ ] 유동성 리스크 체크 (주문 크기 vs 호가창 깊이) +#### 3. Paper Trading 배포 +- [ ] `.env` 설정 (ALPACA_PAPER=true) +- [ ] `make up` → 전 서비스 실행 +- [ ] 2-4주 모의 매매 → 실제 성과 확인 +- [ ] Telegram 알림으로 매일 결과 수신 --- -## Priority & Effort +### 개선 사항 -| Phase | 내용 | 예상 작업량 | 영향 | -|-------|------|------------|------| -| **Phase 1** | 백테스터 현실화 | 1-2일 | **최대** — 이것 없이는 전략 평가 불가 | -| **Phase 2** | 전략 공통 인프라 | 1-2일 | **높음** — 모든 전략의 기반 | -| **Phase 3** | 개별 전략 고도화 | 3-5일 | 중간 — 수익률 직접 개선 | -| **Phase 4** | 리스크 관리 고도화 | 2-3일 | 높음 — 손실 방지 | +#### 4. Claude 스크리너 고도화 +- [ ] SEC 공시 분석 (10-K, 10-Q, 8-K) +- [ ] 실적 서프라이즈 감지 (EPS beat/miss) +- [ ] 섹터 로테이션 분석 +- [ ] 뉴스 감성 분석 (Yahoo Finance, MarketWatch) -**권장 순서: Phase 1 → Phase 2 → Phase 4 → Phase 3** -(전략 개선보다 리스크 관리가 더 중요 — 돈을 벌기 전에 잃지 않는 게 먼저) +#### 5. 주문 유형 확장 +- [ ] Limit order 지원 +- [ ] 프리마켓/애프터마켓 주문 +- [ ] 분할 매수/매도 + +#### 6. 추가 전략 +- [ ] ORB (Opening Range Breakout) — 장 시작 30분 전략 +- [ ] Gap & Go — 갭 상승 종목 추격 전략 +- [ ] Earnings Play — 실적 발표 전후 전략 + +#### 7. 리스크 관리 개선 +- [ ] 섹터 집중도 제한 (같은 섹터 3개 이상 금지) +- [ ] 실적 발표일 매매 회피 +- [ ] 시장 전체 하락 시 매매 중단 (SPY RSI 기반) --- -## Previously Completed (Infrastructure) +## Quick Start + +```bash +# 1. 환경 설정 +cp .env.example .env +# ALPACA_API_KEY, ALPACA_API_SECRET 입력 +# ANTHROPIC_API_KEY 입력 (Claude 스크리너용) + +# 2. 의존성 설치 +pip install -e shared/ + +# 3. 인프라 실행 +make infra # Redis + PostgreSQL +make migrate # DB 마이그레이션 + +# 4. 테스트 +make test # 375 tests + +# 5. 종목 스크리닝 +make screen # Claude가 33개 종목 분석 → Top 5 추천 + +# 6. 서비스 실행 +make up # 전 서비스 시작 (paper trading) + +# 7. 모니터링 +docker compose --profile monitoring up -d +# Grafana: http://localhost:3000 +# API: http://localhost:8000/api/v1/strategies + +# 8. CLI +trading strategy list +trading backtest run --strategy moc --symbol AAPL --timeframe 1Day +trading portfolio show +``` + +--- -모든 인프라 항목 완료 (27개): SQLAlchemy, Alembic, structlog, Telegram, Prometheus/Grafana/Loki, CI/CD, FastAPI, multi-exchange, Redis consumer groups, realized PnL, bearer auth, 298 tests. +## Completed (Infrastructure) + +- [x] Alpaca API 클라이언트 (paper + live) +- [x] MOC 전략 (종가 매수 / 시가 매도) +- [x] Claude 종목 스크리너 (33개 유니버스) +- [x] Data collector (Alpaca REST polling) +- [x] Order executor (Alpaca submit_order) +- [x] SQLAlchemy ORM + Alembic 마이그레이션 +- [x] structlog 구조화 로깅 +- [x] Telegram 알림 +- [x] Retry + Circuit Breaker +- [x] Prometheus + Grafana + Loki +- [x] Redis consumer groups +- [x] Portfolio snapshots + realized PnL +- [x] Bearer token auth +- [x] CI/CD (Gitea Actions) +- [x] E2E test script +- [x] FastAPI REST API +- [x] 백테스터 (슬리피지, 수수료, SL/TP, 공매도, walk-forward) +- [x] 기술 지표 라이브러리 (ATR, ADX, RSI, MACD, Bollinger, Stochastic, OBV) +- [x] 포트폴리오 VaR, 상관관계, drawdown 기반 리스크 +- [x] 375 tests, lint clean diff --git a/docs/superpowers/plans/2026-04-01-crypto-trading-platform.md b/docs/superpowers/plans/2026-04-01-crypto-trading-platform.md deleted file mode 100644 index 08ff0f5..0000000 --- a/docs/superpowers/plans/2026-04-01-crypto-trading-platform.md +++ /dev/null @@ -1,4063 +0,0 @@ -# Crypto Trading Platform Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Binance 현물 암호화폐 자동매매 플랫폼을 마이크로서비스 아키텍처로 구축한다. - -**Architecture:** 6개 독립 서비스(data-collector, strategy-engine, order-executor, portfolio-manager, backtester)가 Redis Streams로 통신하고, PostgreSQL에 데이터를 저장한다. shared 라이브러리가 공통 모델/이벤트/DB 연결을 제공하며, Click 기반 CLI로 전체를 제어한다. - -**Tech Stack:** Python 3.12, ccxt, Redis Streams, PostgreSQL, asyncpg, pandas, pandas-ta, Click, pydantic-settings, Docker Compose, pytest - ---- - -## File Structure - -``` -trading/ -├── services/ -│ ├── data-collector/ -│ │ ├── src/data_collector/__init__.py -│ │ ├── src/data_collector/main.py -│ │ ├── src/data_collector/binance_ws.py -│ │ ├── src/data_collector/binance_rest.py -│ │ ├── src/data_collector/storage.py -│ │ ├── src/data_collector/config.py -│ │ ├── tests/test_binance_rest.py -│ │ ├── tests/test_storage.py -│ │ ├── tests/test_main.py -│ │ ├── Dockerfile -│ │ └── pyproject.toml -│ ├── strategy-engine/ -│ │ ├── src/strategy_engine/__init__.py -│ │ ├── src/strategy_engine/main.py -│ │ ├── src/strategy_engine/engine.py -│ │ ├── src/strategy_engine/plugin_loader.py -│ │ ├── src/strategy_engine/config.py -│ │ ├── strategies/base.py -│ │ ├── strategies/rsi_strategy.py -│ │ ├── strategies/grid_strategy.py -│ │ ├── tests/test_engine.py -│ │ ├── tests/test_plugin_loader.py -│ │ ├── tests/test_rsi_strategy.py -│ │ ├── tests/test_grid_strategy.py -│ │ ├── Dockerfile -│ │ └── pyproject.toml -│ ├── order-executor/ -│ │ ├── src/order_executor/__init__.py -│ │ ├── src/order_executor/main.py -│ │ ├── src/order_executor/executor.py -│ │ ├── src/order_executor/risk_manager.py -│ │ ├── src/order_executor/config.py -│ │ ├── tests/test_executor.py -│ │ ├── tests/test_risk_manager.py -│ │ ├── Dockerfile -│ │ └── pyproject.toml -│ ├── portfolio-manager/ -│ │ ├── src/portfolio_manager/__init__.py -│ │ ├── src/portfolio_manager/main.py -│ │ ├── src/portfolio_manager/portfolio.py -│ │ ├── src/portfolio_manager/pnl.py -│ │ ├── src/portfolio_manager/config.py -│ │ ├── tests/test_portfolio.py -│ │ ├── tests/test_pnl.py -│ │ ├── Dockerfile -│ │ └── pyproject.toml -│ └── backtester/ -│ ├── src/backtester/__init__.py -│ ├── src/backtester/main.py -│ ├── src/backtester/engine.py -│ ├── src/backtester/simulator.py -│ ├── src/backtester/reporter.py -│ ├── src/backtester/config.py -│ ├── tests/test_engine.py -│ ├── tests/test_simulator.py -│ ├── tests/test_reporter.py -│ ├── Dockerfile -│ └── pyproject.toml -├── shared/ -│ ├── src/shared/__init__.py -│ ├── src/shared/models.py -│ ├── src/shared/events.py -│ ├── src/shared/broker.py -│ ├── src/shared/db.py -│ ├── src/shared/config.py -│ ├── tests/test_models.py -│ ├── tests/test_events.py -│ ├── tests/test_broker.py -│ ├── tests/test_db.py -│ └── pyproject.toml -├── cli/ -│ ├── src/trading_cli/__init__.py -│ ├── src/trading_cli/main.py -│ ├── src/trading_cli/commands/data.py -│ ├── src/trading_cli/commands/trade.py -│ ├── src/trading_cli/commands/backtest.py -│ ├── src/trading_cli/commands/portfolio.py -│ ├── src/trading_cli/commands/strategy.py -│ ├── src/trading_cli/commands/service.py -│ ├── tests/test_cli_data.py -│ ├── tests/test_cli_trade.py -│ └── pyproject.toml -├── docker-compose.yml -├── .env.example -├── Makefile -└── pyproject.toml (workspace root) -``` - ---- - -## Task 1: Project Scaffolding - -**Files:** -- Create: `pyproject.toml` (workspace root) -- Create: `.env.example` -- Create: `docker-compose.yml` -- Create: `Makefile` -- Create: `.gitignore` -- Create: `shared/pyproject.toml` - -- [ ] **Step 1: Initialize git repo** - -```bash -cd /home/si/Private/repos/trading -git init -``` - -- [ ] **Step 2: Create .gitignore** - -Create `.gitignore`: - -```gitignore -__pycache__/ -*.py[cod] -*$py.class -*.egg-info/ -dist/ -build/ -.eggs/ -*.egg -.venv/ -venv/ -env/ -.env -.mypy_cache/ -.pytest_cache/ -.ruff_cache/ -*.log -.DS_Store -``` - -- [ ] **Step 3: Create workspace root pyproject.toml** - -Create `pyproject.toml`: - -```toml -[project] -name = "trading-platform" -version = "0.1.0" -description = "Binance spot crypto trading platform" -requires-python = ">=3.12" - -[tool.pytest.ini_options] -asyncio_mode = "auto" -testpaths = ["shared/tests", "services/*/tests", "cli/tests"] - -[tool.ruff] -target-version = "py312" -line-length = 100 -``` - -- [ ] **Step 4: Create .env.example** - -Create `.env.example`: - -```env -BINANCE_API_KEY= -BINANCE_API_SECRET= -REDIS_URL=redis://localhost:6379 -DATABASE_URL=postgresql://trading:trading@localhost:5432/trading -LOG_LEVEL=INFO -RISK_MAX_POSITION_SIZE=0.1 -RISK_STOP_LOSS_PCT=5 -RISK_DAILY_LOSS_LIMIT_PCT=10 -DRY_RUN=true -``` - -- [ ] **Step 5: Create docker-compose.yml** - -Create `docker-compose.yml`: - -```yaml -services: - redis: - image: redis:7-alpine - ports: - - "6379:6379" - volumes: - - redis_data:/data - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 5s - timeout: 3s - retries: 5 - - postgres: - image: postgres:16-alpine - ports: - - "5432:5432" - environment: - POSTGRES_USER: trading - POSTGRES_PASSWORD: trading - POSTGRES_DB: trading - volumes: - - postgres_data:/var/lib/postgresql/data - healthcheck: - test: ["CMD-LINE", "pg_isready", "-U", "trading"] - interval: 5s - timeout: 3s - retries: 5 - - data-collector: - build: - context: . - dockerfile: services/data-collector/Dockerfile - env_file: .env - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - restart: unless-stopped - - strategy-engine: - build: - context: . - dockerfile: services/strategy-engine/Dockerfile - env_file: .env - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - restart: unless-stopped - - order-executor: - build: - context: . - dockerfile: services/order-executor/Dockerfile - env_file: .env - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - restart: unless-stopped - - portfolio-manager: - build: - context: . - dockerfile: services/portfolio-manager/Dockerfile - env_file: .env - depends_on: - redis: - condition: service_healthy - postgres: - condition: service_healthy - restart: unless-stopped - -volumes: - redis_data: - postgres_data: -``` - -- [ ] **Step 6: Create Makefile** - -Create `Makefile`: - -```makefile -.PHONY: infra up down logs test lint - -infra: - docker compose up -d redis postgres - -up: - docker compose up -d - -down: - docker compose down - -logs: - docker compose logs -f $(service) - -test: - pytest -v - -lint: - ruff check . - ruff format --check . - -format: - ruff check --fix . - ruff format . -``` - -- [ ] **Step 7: Create shared/pyproject.toml** - -Create `shared/pyproject.toml`: - -```toml -[project] -name = "trading-shared" -version = "0.1.0" -description = "Shared models, events, and utilities for trading platform" -requires-python = ">=3.12" -dependencies = [ - "pydantic>=2.0", - "pydantic-settings>=2.0", - "redis>=5.0", - "asyncpg>=0.29", -] - -[project.optional-dependencies] -dev = [ - "pytest>=8.0", - "pytest-asyncio>=0.23", - "ruff>=0.4", -] - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["src/shared"] -``` - -- [ ] **Step 8: Commit scaffolding** - -```bash -git add . -git commit -m "chore: project scaffolding with docker-compose, makefile, shared package" -``` - ---- - -## Task 2: Shared — Config & Models - -**Files:** -- Create: `shared/src/shared/__init__.py` -- Create: `shared/src/shared/config.py` -- Create: `shared/src/shared/models.py` -- Create: `shared/tests/test_models.py` - -- [ ] **Step 1: Write failing test for config** - -Create `shared/tests/test_models.py`: - -```python -from shared.config import Settings - - -def test_settings_defaults(): - settings = Settings( - binance_api_key="test_key", - binance_api_secret="test_secret", - ) - assert settings.redis_url == "redis://localhost:6379" - assert settings.database_url == "postgresql://trading:trading@localhost:5432/trading" - assert settings.log_level == "INFO" - assert settings.dry_run is True -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -cd /home/si/Private/repos/trading -pip install -e shared[dev] -pytest shared/tests/test_models.py::test_settings_defaults -v -``` - -Expected: FAIL — `ModuleNotFoundError: No module named 'shared'` - -- [ ] **Step 3: Implement config** - -Create `shared/src/shared/__init__.py`: - -```python -``` - -Create `shared/src/shared/config.py`: - -```python -from pydantic_settings import BaseSettings - - -class Settings(BaseSettings): - binance_api_key: str - binance_api_secret: str - redis_url: str = "redis://localhost:6379" - database_url: str = "postgresql://trading:trading@localhost:5432/trading" - log_level: str = "INFO" - risk_max_position_size: float = 0.1 - risk_stop_loss_pct: float = 5.0 - risk_daily_loss_limit_pct: float = 10.0 - dry_run: bool = True - - model_config = {"env_file": ".env", "env_file_encoding": "utf-8"} -``` - -- [ ] **Step 4: Run test to verify it passes** - -```bash -pytest shared/tests/test_models.py::test_settings_defaults -v -``` - -Expected: PASS - -- [ ] **Step 5: Write failing tests for models** - -Append to `shared/tests/test_models.py`: - -```python -from datetime import datetime, timezone -from decimal import Decimal - -from shared.models import Candle, Signal, Order, Position, OrderSide, OrderType, OrderStatus - - -def test_candle_creation(): - candle = Candle( - symbol="BTCUSDT", - timeframe="1m", - open_time=datetime(2026, 1, 1, tzinfo=timezone.utc), - open=Decimal("50000"), - high=Decimal("50100"), - low=Decimal("49900"), - close=Decimal("50050"), - volume=Decimal("1.5"), - ) - assert candle.symbol == "BTCUSDT" - assert candle.close == Decimal("50050") - - -def test_signal_creation(): - signal = Signal( - strategy="rsi_strategy", - symbol="BTCUSDT", - side=OrderSide.BUY, - price=Decimal("50000"), - quantity=Decimal("0.01"), - reason="RSI below 30", - ) - assert signal.side == OrderSide.BUY - assert signal.reason == "RSI below 30" - - -def test_order_creation(): - order = Order( - symbol="BTCUSDT", - signal_id="sig_123", - side=OrderSide.BUY, - type=OrderType.MARKET, - price=Decimal("50000"), - quantity=Decimal("0.01"), - ) - assert order.status == OrderStatus.PENDING - assert order.filled_at is None - assert order.id is not None - - -def test_position_unrealized_pnl(): - pos = Position( - symbol="BTCUSDT", - quantity=Decimal("0.1"), - avg_entry_price=Decimal("50000"), - current_price=Decimal("51000"), - ) - assert pos.unrealized_pnl == Decimal("100") # 0.1 * (51000 - 50000) -``` - -- [ ] **Step 6: Run tests to verify they fail** - -```bash -pytest shared/tests/test_models.py -v -``` - -Expected: FAIL — `ModuleNotFoundError: No module named 'shared.models'` - -- [ ] **Step 7: Implement models** - -Create `shared/src/shared/models.py`: - -```python -from datetime import datetime, timezone -from decimal import Decimal -from enum import StrEnum -from uuid import uuid4 - -from pydantic import BaseModel, Field - - -class OrderSide(StrEnum): - BUY = "BUY" - SELL = "SELL" - - -class OrderType(StrEnum): - MARKET = "MARKET" - LIMIT = "LIMIT" - - -class OrderStatus(StrEnum): - PENDING = "PENDING" - FILLED = "FILLED" - CANCELLED = "CANCELLED" - FAILED = "FAILED" - - -class Candle(BaseModel): - symbol: str - timeframe: str - open_time: datetime - open: Decimal - high: Decimal - low: Decimal - close: Decimal - volume: Decimal - - -class Signal(BaseModel): - id: str = Field(default_factory=lambda: str(uuid4())) - strategy: str - symbol: str - side: OrderSide - price: Decimal - quantity: Decimal - reason: str - created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) - - -class Order(BaseModel): - id: str = Field(default_factory=lambda: str(uuid4())) - signal_id: str - symbol: str - side: OrderSide - type: OrderType - price: Decimal - quantity: Decimal - status: OrderStatus = OrderStatus.PENDING - created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) - filled_at: datetime | None = None - - -class Position(BaseModel): - symbol: str - quantity: Decimal - avg_entry_price: Decimal - current_price: Decimal - - @property - def unrealized_pnl(self) -> Decimal: - return self.quantity * (self.current_price - self.avg_entry_price) -``` - -- [ ] **Step 8: Run tests to verify they pass** - -```bash -pytest shared/tests/test_models.py -v -``` - -Expected: All PASS - -- [ ] **Step 9: Commit** - -```bash -git add shared/ -git commit -m "feat(shared): add config settings and core data models" -``` - ---- - -## Task 3: Shared — Events & Redis Broker - -**Files:** -- Create: `shared/src/shared/events.py` -- Create: `shared/src/shared/broker.py` -- Create: `shared/tests/test_events.py` -- Create: `shared/tests/test_broker.py` - -- [ ] **Step 1: Write failing tests for events** - -Create `shared/tests/test_events.py`: - -```python -import json -from decimal import Decimal -from datetime import datetime, timezone - -from shared.events import EventType, Event, CandleEvent, SignalEvent, OrderEvent -from shared.models import Candle, Signal, Order, OrderSide, OrderType - - -def test_candle_event_serialize(): - candle = Candle( - symbol="BTCUSDT", - timeframe="1m", - open_time=datetime(2026, 1, 1, tzinfo=timezone.utc), - open=Decimal("50000"), - high=Decimal("50100"), - low=Decimal("49900"), - close=Decimal("50050"), - volume=Decimal("1.5"), - ) - event = CandleEvent(data=candle) - payload = event.to_dict() - assert payload["type"] == EventType.CANDLE - assert payload["data"]["symbol"] == "BTCUSDT" - - -def test_candle_event_deserialize(): - candle = Candle( - symbol="BTCUSDT", - timeframe="1m", - open_time=datetime(2026, 1, 1, tzinfo=timezone.utc), - open=Decimal("50000"), - high=Decimal("50100"), - low=Decimal("49900"), - close=Decimal("50050"), - volume=Decimal("1.5"), - ) - event = CandleEvent(data=candle) - payload = event.to_dict() - restored = Event.from_dict(payload) - assert isinstance(restored, CandleEvent) - assert restored.data.symbol == "BTCUSDT" - - -def test_signal_event_serialize(): - signal = Signal( - strategy="rsi", - symbol="BTCUSDT", - side=OrderSide.BUY, - price=Decimal("50000"), - quantity=Decimal("0.01"), - reason="RSI < 30", - ) - event = SignalEvent(data=signal) - payload = event.to_dict() - assert payload["type"] == EventType.SIGNAL -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -pytest shared/tests/test_events.py -v -``` - -Expected: FAIL - -- [ ] **Step 3: Implement events** - -Create `shared/src/shared/events.py`: - -```python -from __future__ import annotations - -import json -from enum import StrEnum -from typing import Any - -from pydantic import BaseModel - -from shared.models import Candle, Signal, Order - - -class EventType(StrEnum): - CANDLE = "candle" - SIGNAL = "signal" - ORDER = "order" - - -class CandleEvent(BaseModel): - type: EventType = EventType.CANDLE - data: Candle - - def to_dict(self) -> dict[str, Any]: - return json.loads(self.model_dump_json()) - - @classmethod - def from_raw(cls, raw: dict[str, Any]) -> CandleEvent: - return cls.model_validate(raw) - - -class SignalEvent(BaseModel): - type: EventType = EventType.SIGNAL - data: Signal - - def to_dict(self) -> dict[str, Any]: - return json.loads(self.model_dump_json()) - - @classmethod - def from_raw(cls, raw: dict[str, Any]) -> SignalEvent: - return cls.model_validate(raw) - - -class OrderEvent(BaseModel): - type: EventType = EventType.ORDER - data: Order - - def to_dict(self) -> dict[str, Any]: - return json.loads(self.model_dump_json()) - - @classmethod - def from_raw(cls, raw: dict[str, Any]) -> OrderEvent: - return cls.model_validate(raw) - - -_EVENT_MAP = { - EventType.CANDLE: CandleEvent, - EventType.SIGNAL: SignalEvent, - EventType.ORDER: OrderEvent, -} - - -class Event: - @staticmethod - def from_dict(data: dict[str, Any]) -> CandleEvent | SignalEvent | OrderEvent: - event_type = EventType(data["type"]) - cls = _EVENT_MAP[event_type] - return cls.from_raw(data) -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -pytest shared/tests/test_events.py -v -``` - -Expected: All PASS - -- [ ] **Step 5: Write failing tests for broker** - -Create `shared/tests/test_broker.py`: - -```python -import asyncio -import pytest -from unittest.mock import AsyncMock, MagicMock, patch - -from shared.broker import RedisBroker - - -@pytest.fixture -def mock_redis(): - redis = AsyncMock() - redis.xadd = AsyncMock(return_value=b"1234-0") - redis.xread = AsyncMock(return_value=[]) - redis.close = AsyncMock() - return redis - - -@pytest.mark.asyncio -async def test_broker_publish(mock_redis): - broker = RedisBroker.__new__(RedisBroker) - broker._redis = mock_redis - - await broker.publish("candles.BTCUSDT", {"type": "candle", "data": "test"}) - - mock_redis.xadd.assert_called_once() - call_args = mock_redis.xadd.call_args - assert call_args[0][0] == "candles.BTCUSDT" - - -@pytest.mark.asyncio -async def test_broker_subscribe_returns_messages(mock_redis): - mock_redis.xread = AsyncMock(return_value=[ - ("candles.BTCUSDT", [ - (b"1234-0", {b"payload": b'{"type":"candle","data":"test"}'}), - ]) - ]) - broker = RedisBroker.__new__(RedisBroker) - broker._redis = mock_redis - - messages = await broker.read("candles.BTCUSDT", last_id="0-0", count=1) - assert len(messages) == 1 - assert messages[0]["type"] == "candle" -``` - -- [ ] **Step 6: Run tests to verify they fail** - -```bash -pytest shared/tests/test_broker.py -v -``` - -Expected: FAIL - -- [ ] **Step 7: Implement broker** - -Create `shared/src/shared/broker.py`: - -```python -from __future__ import annotations - -import json - -import redis.asyncio as redis - - -class RedisBroker: - def __init__(self, redis_url: str): - self._redis = redis.from_url(redis_url, decode_responses=False) - - async def publish(self, stream: str, data: dict) -> str: - payload = json.dumps(data) - msg_id = await self._redis.xadd(stream, {"payload": payload.encode()}) - return msg_id - - async def read( - self, stream: str, last_id: str = "$", count: int = 10, block: int = 0 - ) -> list[dict]: - results = await self._redis.xread({stream: last_id}, count=count, block=block) - messages = [] - for _stream_name, entries in results: - for _msg_id, fields in entries: - payload = fields[b"payload"] - messages.append(json.loads(payload)) - return messages - - async def close(self): - await self._redis.close() -``` - -- [ ] **Step 8: Run tests to verify they pass** - -```bash -pytest shared/tests/test_broker.py -v -``` - -Expected: All PASS - -- [ ] **Step 9: Commit** - -```bash -git add shared/ -git commit -m "feat(shared): add event system and Redis Streams broker" -``` - ---- - -## Task 4: Shared — Database Layer - -**Files:** -- Create: `shared/src/shared/db.py` -- Create: `shared/tests/test_db.py` - -- [ ] **Step 1: Write failing tests for DB** - -Create `shared/tests/test_db.py`: - -```python -import pytest -from unittest.mock import AsyncMock, patch, MagicMock - -from shared.db import Database - - -@pytest.mark.asyncio -async def test_db_init_sql_creates_tables(): - db = Database.__new__(Database) - db._pool = AsyncMock() - mock_conn = AsyncMock() - db._pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn) - db._pool.acquire.return_value.__aexit__ = AsyncMock(return_value=False) - - await db.init_tables() - - mock_conn.execute.assert_called() - sql = mock_conn.execute.call_args[0][0] - assert "candles" in sql - assert "signals" in sql - assert "orders" in sql - assert "trades" in sql - assert "positions" in sql - assert "portfolio_snapshots" in sql - - -@pytest.mark.asyncio -async def test_db_insert_candle(): - db = Database.__new__(Database) - db._pool = AsyncMock() - mock_conn = AsyncMock() - db._pool.acquire.return_value.__aenter__ = AsyncMock(return_value=mock_conn) - db._pool.acquire.return_value.__aexit__ = AsyncMock(return_value=False) - - from datetime import datetime, timezone - from decimal import Decimal - from shared.models import Candle - - candle = Candle( - symbol="BTCUSDT", - timeframe="1m", - open_time=datetime(2026, 1, 1, tzinfo=timezone.utc), - open=Decimal("50000"), - high=Decimal("50100"), - low=Decimal("49900"), - close=Decimal("50050"), - volume=Decimal("1.5"), - ) - - await db.insert_candle(candle) - mock_conn.execute.assert_called_once() - sql = mock_conn.execute.call_args[0][0] - assert "INSERT INTO candles" in sql -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -pytest shared/tests/test_db.py -v -``` - -Expected: FAIL - -- [ ] **Step 3: Implement database layer** - -Create `shared/src/shared/db.py`: - -```python -from __future__ import annotations - -import asyncpg - -from shared.models import Candle, Order, Signal - -_INIT_SQL = """ -CREATE TABLE IF NOT EXISTS candles ( - symbol TEXT NOT NULL, - timeframe TEXT NOT NULL, - open_time TIMESTAMPTZ NOT NULL, - open NUMERIC NOT NULL, - high NUMERIC NOT NULL, - low NUMERIC NOT NULL, - close NUMERIC NOT NULL, - volume NUMERIC NOT NULL, - PRIMARY KEY (symbol, timeframe, open_time) -); - -CREATE TABLE IF NOT EXISTS signals ( - id TEXT PRIMARY KEY, - strategy TEXT NOT NULL, - symbol TEXT NOT NULL, - side TEXT NOT NULL, - price NUMERIC NOT NULL, - quantity NUMERIC NOT NULL, - reason TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL -); - -CREATE TABLE IF NOT EXISTS orders ( - id TEXT PRIMARY KEY, - signal_id TEXT REFERENCES signals(id), - symbol TEXT NOT NULL, - side TEXT NOT NULL, - type TEXT NOT NULL, - price NUMERIC NOT NULL, - quantity NUMERIC NOT NULL, - status TEXT NOT NULL DEFAULT 'PENDING', - created_at TIMESTAMPTZ NOT NULL, - filled_at TIMESTAMPTZ -); - -CREATE TABLE IF NOT EXISTS trades ( - id TEXT PRIMARY KEY, - order_id TEXT REFERENCES orders(id), - symbol TEXT NOT NULL, - side TEXT NOT NULL, - price NUMERIC NOT NULL, - quantity NUMERIC NOT NULL, - fee NUMERIC NOT NULL DEFAULT 0, - traded_at TIMESTAMPTZ NOT NULL -); - -CREATE TABLE IF NOT EXISTS positions ( - symbol TEXT PRIMARY KEY, - quantity NUMERIC NOT NULL, - avg_entry_price NUMERIC NOT NULL, - current_price NUMERIC NOT NULL, - updated_at TIMESTAMPTZ NOT NULL -); - -CREATE TABLE IF NOT EXISTS portfolio_snapshots ( - id SERIAL PRIMARY KEY, - total_value NUMERIC NOT NULL, - realized_pnl NUMERIC NOT NULL, - unrealized_pnl NUMERIC NOT NULL, - snapshot_at TIMESTAMPTZ NOT NULL -); -""" - - -class Database: - def __init__(self, dsn: str): - self._dsn = dsn - self._pool: asyncpg.Pool | None = None - - async def connect(self): - self._pool = await asyncpg.create_pool(self._dsn) - - async def close(self): - if self._pool: - await self._pool.close() - - async def init_tables(self): - async with self._pool.acquire() as conn: - await conn.execute(_INIT_SQL) - - async def insert_candle(self, candle: Candle): - sql = """ - INSERT INTO candles (symbol, timeframe, open_time, open, high, low, close, volume) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) - ON CONFLICT (symbol, timeframe, open_time) DO NOTHING - """ - async with self._pool.acquire() as conn: - await conn.execute( - sql, - candle.symbol, - candle.timeframe, - candle.open_time, - candle.open, - candle.high, - candle.low, - candle.close, - candle.volume, - ) - - async def insert_signal(self, signal: Signal): - sql = """ - INSERT INTO signals (id, strategy, symbol, side, price, quantity, reason, created_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) - """ - async with self._pool.acquire() as conn: - await conn.execute( - sql, - signal.id, - signal.strategy, - signal.symbol, - signal.side.value, - signal.price, - signal.quantity, - signal.reason, - signal.created_at, - ) - - async def insert_order(self, order: Order): - sql = """ - INSERT INTO orders (id, signal_id, symbol, side, type, price, quantity, status, created_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) - """ - async with self._pool.acquire() as conn: - await conn.execute( - sql, - order.id, - order.signal_id, - order.symbol, - order.side.value, - order.type.value, - order.price, - order.quantity, - order.status.value, - order.created_at, - ) - - async def update_order_status(self, order_id: str, status: str, filled_at=None): - sql = "UPDATE orders SET status = $1, filled_at = $2 WHERE id = $3" - async with self._pool.acquire() as conn: - await conn.execute(sql, status, filled_at, order_id) - - async def get_candles(self, symbol: str, timeframe: str, limit: int = 500) -> list[dict]: - sql = """ - SELECT * FROM candles - WHERE symbol = $1 AND timeframe = $2 - ORDER BY open_time DESC - LIMIT $3 - """ - async with self._pool.acquire() as conn: - rows = await conn.fetch(sql, symbol, timeframe, limit) - return [dict(r) for r in rows] -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -pytest shared/tests/test_db.py -v -``` - -Expected: All PASS - -- [ ] **Step 5: Commit** - -```bash -git add shared/ -git commit -m "feat(shared): add database layer with table init and CRUD operations" -``` - ---- - -## Task 5: Data Collector Service - -**Files:** -- Create: `services/data-collector/pyproject.toml` -- Create: `services/data-collector/Dockerfile` -- Create: `services/data-collector/src/data_collector/__init__.py` -- Create: `services/data-collector/src/data_collector/config.py` -- Create: `services/data-collector/src/data_collector/binance_rest.py` -- Create: `services/data-collector/src/data_collector/binance_ws.py` -- Create: `services/data-collector/src/data_collector/storage.py` -- Create: `services/data-collector/src/data_collector/main.py` -- Create: `services/data-collector/tests/test_binance_rest.py` -- Create: `services/data-collector/tests/test_storage.py` - -- [ ] **Step 1: Create pyproject.toml** - -Create `services/data-collector/pyproject.toml`: - -```toml -[project] -name = "data-collector" -version = "0.1.0" -description = "Binance market data collector service" -requires-python = ">=3.12" -dependencies = [ - "ccxt>=4.0", - "websockets>=12.0", - "trading-shared", -] - -[project.optional-dependencies] -dev = [ - "pytest>=8.0", - "pytest-asyncio>=0.23", -] - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["src/data_collector"] -``` - -- [ ] **Step 2: Write failing tests for binance_rest** - -Create `services/data-collector/tests/test_binance_rest.py`: - -```python -import pytest -from unittest.mock import AsyncMock, patch, MagicMock -from datetime import datetime, timezone -from decimal import Decimal - -from data_collector.binance_rest import fetch_historical_candles - - -@pytest.mark.asyncio -async def test_fetch_historical_candles_parses_response(): - mock_exchange = MagicMock() - mock_exchange.fetch_ohlcv = AsyncMock(return_value=[ - [1704067200000, 50000.0, 50100.0, 49900.0, 50050.0, 1.5], - [1704067260000, 50050.0, 50200.0, 50000.0, 50150.0, 2.0], - ]) - - candles = await fetch_historical_candles( - exchange=mock_exchange, - symbol="BTC/USDT", - timeframe="1m", - since=datetime(2026, 1, 1, tzinfo=timezone.utc), - limit=2, - ) - - assert len(candles) == 2 - assert candles[0].symbol == "BTCUSDT" - assert candles[0].close == Decimal("50050.0") - assert candles[1].volume == Decimal("2.0") - - -@pytest.mark.asyncio -async def test_fetch_historical_candles_empty_response(): - mock_exchange = MagicMock() - mock_exchange.fetch_ohlcv = AsyncMock(return_value=[]) - - candles = await fetch_historical_candles( - exchange=mock_exchange, - symbol="BTC/USDT", - timeframe="1m", - since=datetime(2026, 1, 1, tzinfo=timezone.utc), - limit=100, - ) - - assert candles == [] -``` - -- [ ] **Step 3: Run tests to verify they fail** - -```bash -cd /home/si/Private/repos/trading -pip install -e services/data-collector[dev] -pytest services/data-collector/tests/test_binance_rest.py -v -``` - -Expected: FAIL - -- [ ] **Step 4: Implement binance_rest** - -Create `services/data-collector/src/data_collector/__init__.py`: - -```python -``` - -Create `services/data-collector/src/data_collector/binance_rest.py`: - -```python -from __future__ import annotations - -from datetime import datetime, timezone -from decimal import Decimal - -from shared.models import Candle - - -async def fetch_historical_candles( - exchange, - symbol: str, - timeframe: str, - since: datetime, - limit: int = 500, -) -> list[Candle]: - since_ms = int(since.timestamp() * 1000) - ohlcv = await exchange.fetch_ohlcv(symbol, timeframe, since=since_ms, limit=limit) - - normalized_symbol = symbol.replace("/", "") - candles = [] - for row in ohlcv: - ts, o, h, l, c, v = row - candles.append( - Candle( - symbol=normalized_symbol, - timeframe=timeframe, - open_time=datetime.fromtimestamp(ts / 1000, tz=timezone.utc), - open=Decimal(str(o)), - high=Decimal(str(h)), - low=Decimal(str(l)), - close=Decimal(str(c)), - volume=Decimal(str(v)), - ) - ) - return candles -``` - -- [ ] **Step 5: Run tests to verify they pass** - -```bash -pytest services/data-collector/tests/test_binance_rest.py -v -``` - -Expected: All PASS - -- [ ] **Step 6: Write failing tests for storage** - -Create `services/data-collector/tests/test_storage.py`: - -```python -import pytest -from unittest.mock import AsyncMock -from datetime import datetime, timezone -from decimal import Decimal - -from shared.models import Candle -from data_collector.storage import CandleStorage - - -@pytest.fixture -def mock_db(): - db = AsyncMock() - db.insert_candle = AsyncMock() - return db - - -@pytest.fixture -def mock_broker(): - broker = AsyncMock() - broker.publish = AsyncMock() - return broker - - -@pytest.fixture -def sample_candle(): - return Candle( - symbol="BTCUSDT", - timeframe="1m", - open_time=datetime(2026, 1, 1, tzinfo=timezone.utc), - open=Decimal("50000"), - high=Decimal("50100"), - low=Decimal("49900"), - close=Decimal("50050"), - volume=Decimal("1.5"), - ) - - -@pytest.mark.asyncio -async def test_storage_saves_to_db_and_publishes(mock_db, mock_broker, sample_candle): - storage = CandleStorage(db=mock_db, broker=mock_broker) - await storage.store(sample_candle) - - mock_db.insert_candle.assert_called_once_with(sample_candle) - mock_broker.publish.assert_called_once() - call_args = mock_broker.publish.call_args - assert call_args[0][0] == "candles.BTCUSDT" - - -@pytest.mark.asyncio -async def test_storage_batch_store(mock_db, mock_broker, sample_candle): - storage = CandleStorage(db=mock_db, broker=mock_broker) - candles = [sample_candle, sample_candle] - await storage.store_batch(candles) - - assert mock_db.insert_candle.call_count == 2 - assert mock_broker.publish.call_count == 2 -``` - -- [ ] **Step 7: Run tests to verify they fail** - -```bash -pytest services/data-collector/tests/test_storage.py -v -``` - -Expected: FAIL - -- [ ] **Step 8: Implement storage** - -Create `services/data-collector/src/data_collector/storage.py`: - -```python -from __future__ import annotations - -from shared.broker import RedisBroker -from shared.db import Database -from shared.events import CandleEvent -from shared.models import Candle - - -class CandleStorage: - def __init__(self, db: Database, broker: RedisBroker): - self._db = db - self._broker = broker - - async def store(self, candle: Candle): - await self._db.insert_candle(candle) - event = CandleEvent(data=candle) - await self._broker.publish(f"candles.{candle.symbol}", event.to_dict()) - - async def store_batch(self, candles: list[Candle]): - for candle in candles: - await self.store(candle) -``` - -- [ ] **Step 9: Run tests to verify they pass** - -```bash -pytest services/data-collector/tests/test_storage.py -v -``` - -Expected: All PASS - -- [ ] **Step 10: Implement config, binance_ws, and main** - -Create `services/data-collector/src/data_collector/config.py`: - -```python -from shared.config import Settings - - -class CollectorConfig(Settings): - symbols: list[str] = ["BTC/USDT"] - timeframes: list[str] = ["1m"] -``` - -Create `services/data-collector/src/data_collector/binance_ws.py`: - -```python -from __future__ import annotations - -import asyncio -import json -import logging -from datetime import datetime, timezone -from decimal import Decimal - -import websockets - -from shared.models import Candle - -logger = logging.getLogger(__name__) - -BINANCE_WS_URL = "wss://stream.binance.com:9443/ws" - - -class BinanceWebSocket: - def __init__(self, symbols: list[str], timeframe: str, on_candle): - self._symbols = symbols - self._timeframe = timeframe - self._on_candle = on_candle - self._running = False - - async def start(self): - streams = [ - f"{s.lower().replace('/', '')}@kline_{self._timeframe}" - for s in self._symbols - ] - url = f"{BINANCE_WS_URL}/{'/'.join(streams)}" - self._running = True - logger.info(f"Connecting to Binance WS: {streams}") - - while self._running: - try: - async with websockets.connect(url) as ws: - async for raw in ws: - if not self._running: - break - msg = json.loads(raw) - if "k" in msg: - candle = self._parse_kline(msg["k"]) - if candle: - await self._on_candle(candle) - except websockets.ConnectionClosed: - logger.warning("WebSocket disconnected, reconnecting in 5s...") - await asyncio.sleep(5) - except Exception as e: - logger.error(f"WebSocket error: {e}, reconnecting in 5s...") - await asyncio.sleep(5) - - def stop(self): - self._running = False - - def _parse_kline(self, k: dict) -> Candle | None: - if not k.get("x"): # only closed candles - return None - return Candle( - symbol=k["s"], - timeframe=k["i"], - open_time=datetime.fromtimestamp(k["t"] / 1000, tz=timezone.utc), - open=Decimal(k["o"]), - high=Decimal(k["h"]), - low=Decimal(k["l"]), - close=Decimal(k["c"]), - volume=Decimal(k["v"]), - ) -``` - -Create `services/data-collector/src/data_collector/main.py`: - -```python -from __future__ import annotations - -import asyncio -import logging - -import ccxt.async_support as ccxt - -from shared.broker import RedisBroker -from shared.db import Database -from data_collector.binance_ws import BinanceWebSocket -from data_collector.config import CollectorConfig -from data_collector.storage import CandleStorage - -logger = logging.getLogger(__name__) - - -async def run(): - config = CollectorConfig() - logging.basicConfig(level=config.log_level) - - db = Database(config.database_url) - await db.connect() - await db.init_tables() - - broker = RedisBroker(config.redis_url) - storage = CandleStorage(db=db, broker=broker) - - ws = BinanceWebSocket( - symbols=config.symbols, - timeframe=config.timeframes[0], - on_candle=storage.store, - ) - - logger.info(f"Starting data collector: symbols={config.symbols}") - try: - await ws.start() - finally: - ws.stop() - await broker.close() - await db.close() - - -def main(): - asyncio.run(run()) - - -if __name__ == "__main__": - main() -``` - -- [ ] **Step 11: Create Dockerfile** - -Create `services/data-collector/Dockerfile`: - -```dockerfile -FROM python:3.12-slim - -WORKDIR /app - -COPY shared/ shared/ -RUN pip install --no-cache-dir ./shared - -COPY services/data-collector/ services/data-collector/ -RUN pip install --no-cache-dir ./services/data-collector - -CMD ["python", "-m", "data_collector.main"] -``` - -- [ ] **Step 12: Commit** - -```bash -git add services/data-collector/ -git commit -m "feat(data-collector): add Binance REST/WS data collection with storage pipeline" -``` - ---- - -## Task 6: Strategy Engine Service - -**Files:** -- Create: `services/strategy-engine/pyproject.toml` -- Create: `services/strategy-engine/Dockerfile` -- Create: `services/strategy-engine/src/strategy_engine/__init__.py` -- Create: `services/strategy-engine/src/strategy_engine/config.py` -- Create: `services/strategy-engine/strategies/base.py` -- Create: `services/strategy-engine/strategies/rsi_strategy.py` -- Create: `services/strategy-engine/strategies/grid_strategy.py` -- Create: `services/strategy-engine/src/strategy_engine/plugin_loader.py` -- Create: `services/strategy-engine/src/strategy_engine/engine.py` -- Create: `services/strategy-engine/src/strategy_engine/main.py` -- Create: `services/strategy-engine/tests/test_rsi_strategy.py` -- Create: `services/strategy-engine/tests/test_grid_strategy.py` -- Create: `services/strategy-engine/tests/test_plugin_loader.py` -- Create: `services/strategy-engine/tests/test_engine.py` - -- [ ] **Step 1: Create pyproject.toml** - -Create `services/strategy-engine/pyproject.toml`: - -```toml -[project] -name = "strategy-engine" -version = "0.1.0" -description = "Plugin-based strategy execution engine" -requires-python = ">=3.12" -dependencies = [ - "pandas>=2.0", - "pandas-ta>=0.3", - "trading-shared", -] - -[project.optional-dependencies] -dev = [ - "pytest>=8.0", - "pytest-asyncio>=0.23", -] - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["src/strategy_engine"] -``` - -- [ ] **Step 2: Implement base strategy** - -Create `services/strategy-engine/src/strategy_engine/__init__.py`: - -```python -``` - -Create `services/strategy-engine/strategies/base.py`: - -```python -from __future__ import annotations - -from abc import ABC, abstractmethod - -from shared.models import Candle, Signal - - -class BaseStrategy(ABC): - name: str = "base" - - @abstractmethod - def on_candle(self, candle: Candle) -> Signal | None: - pass - - @abstractmethod - def configure(self, params: dict) -> None: - pass - - def reset(self) -> None: - pass -``` - -- [ ] **Step 3: Write failing tests for RSI strategy** - -Create `services/strategy-engine/tests/test_rsi_strategy.py`: - -```python -from datetime import datetime, timezone -from decimal import Decimal - -from shared.models import Candle, OrderSide - - -def make_candle(close: float, idx: int = 0) -> Candle: - return Candle( - symbol="BTCUSDT", - timeframe="1m", - open_time=datetime(2026, 1, 1, minute=idx, tzinfo=timezone.utc), - open=Decimal(str(close)), - high=Decimal(str(close + 10)), - low=Decimal(str(close - 10)), - close=Decimal(str(close)), - volume=Decimal("1.0"), - ) - - -def test_rsi_strategy_no_signal_insufficient_data(): - from strategy_engine.strategies.rsi_strategy import RsiStrategy - - strategy = RsiStrategy() - strategy.configure({"period": 14, "oversold": 30, "overbought": 70, "quantity": 0.01}) - - signal = strategy.on_candle(make_candle(50000)) - assert signal is None - - -def test_rsi_strategy_buy_signal_on_oversold(): - from strategy_engine.strategies.rsi_strategy import RsiStrategy - - strategy = RsiStrategy() - strategy.configure({"period": 14, "oversold": 30, "overbought": 70, "quantity": 0.01}) - - # Feed declining prices to push RSI below 30 - prices = [50000 - i * 100 for i in range(20)] - signal = None - for i, p in enumerate(prices): - signal = strategy.on_candle(make_candle(p, idx=i)) - - # After sustained drop, RSI should be oversold → BUY signal - if signal is not None: - assert signal.side == OrderSide.BUY - assert signal.strategy == "rsi" -``` - -- [ ] **Step 4: Run tests to verify they fail** - -```bash -pip install -e services/strategy-engine[dev] -pytest services/strategy-engine/tests/test_rsi_strategy.py -v -``` - -Expected: FAIL - -- [ ] **Step 5: Implement RSI strategy** - -Create `services/strategy-engine/strategies/rsi_strategy.py`: - -```python -from __future__ import annotations - -from collections import deque -from decimal import Decimal - -import pandas as pd -import pandas_ta as ta - -from shared.models import Candle, Signal, OrderSide -from strategies.base import BaseStrategy - - -class RsiStrategy(BaseStrategy): - name = "rsi" - - def __init__(self): - self._closes: deque[float] = deque(maxlen=200) - self._period: int = 14 - self._oversold: float = 30 - self._overbought: float = 70 - self._quantity: Decimal = Decimal("0.01") - - def configure(self, params: dict) -> None: - self._period = params.get("period", 14) - self._oversold = params.get("oversold", 30) - self._overbought = params.get("overbought", 70) - self._quantity = Decimal(str(params.get("quantity", 0.01))) - - def on_candle(self, candle: Candle) -> Signal | None: - self._closes.append(float(candle.close)) - - if len(self._closes) < self._period + 1: - return None - - series = pd.Series(list(self._closes)) - rsi = ta.rsi(series, length=self._period) - current_rsi = rsi.iloc[-1] - - if current_rsi < self._oversold: - return Signal( - strategy=self.name, - symbol=candle.symbol, - side=OrderSide.BUY, - price=candle.close, - quantity=self._quantity, - reason=f"RSI={current_rsi:.1f} < {self._oversold}", - ) - elif current_rsi > self._overbought: - return Signal( - strategy=self.name, - symbol=candle.symbol, - side=OrderSide.SELL, - price=candle.close, - quantity=self._quantity, - reason=f"RSI={current_rsi:.1f} > {self._overbought}", - ) - return None - - def reset(self) -> None: - self._closes.clear() -``` - -- [ ] **Step 6: Run tests to verify they pass** - -```bash -pytest services/strategy-engine/tests/test_rsi_strategy.py -v -``` - -Expected: All PASS - -- [ ] **Step 7: Write failing tests for grid strategy** - -Create `services/strategy-engine/tests/test_grid_strategy.py`: - -```python -from datetime import datetime, timezone -from decimal import Decimal - -from shared.models import Candle, OrderSide - - -def make_candle(close: float, idx: int = 0) -> Candle: - return Candle( - symbol="BTCUSDT", - timeframe="1m", - open_time=datetime(2026, 1, 1, minute=idx, tzinfo=timezone.utc), - open=Decimal(str(close)), - high=Decimal(str(close + 10)), - low=Decimal(str(close - 10)), - close=Decimal(str(close)), - volume=Decimal("1.0"), - ) - - -def test_grid_strategy_buy_at_lower_grid(): - from strategy_engine.strategies.grid_strategy import GridStrategy - - strategy = GridStrategy() - strategy.configure({ - "lower_price": 48000, - "upper_price": 52000, - "grid_count": 5, - "quantity": 0.01, - }) - - # Price at grid level should trigger BUY - signal = strategy.on_candle(make_candle(48000)) - # First candle sets reference, no signal - signal = strategy.on_candle(make_candle(49000, idx=1)) - # Moving down through a grid level - signal = strategy.on_candle(make_candle(48000, idx=2)) - if signal is not None: - assert signal.side == OrderSide.BUY - - -def test_grid_strategy_sell_at_upper_grid(): - from strategy_engine.strategies.grid_strategy import GridStrategy - - strategy = GridStrategy() - strategy.configure({ - "lower_price": 48000, - "upper_price": 52000, - "grid_count": 5, - "quantity": 0.01, - }) - - signal = strategy.on_candle(make_candle(50000)) - signal = strategy.on_candle(make_candle(51000, idx=1)) - signal = strategy.on_candle(make_candle(52000, idx=2)) - if signal is not None: - assert signal.side == OrderSide.SELL - - -def test_grid_strategy_no_signal_in_same_zone(): - from strategy_engine.strategies.grid_strategy import GridStrategy - - strategy = GridStrategy() - strategy.configure({ - "lower_price": 48000, - "upper_price": 52000, - "grid_count": 5, - "quantity": 0.01, - }) - - strategy.on_candle(make_candle(50000)) - signal = strategy.on_candle(make_candle(50050, idx=1)) - assert signal is None # same grid zone, no signal -``` - -- [ ] **Step 8: Run tests to verify they fail** - -```bash -pytest services/strategy-engine/tests/test_grid_strategy.py -v -``` - -Expected: FAIL - -- [ ] **Step 9: Implement grid strategy** - -Create `services/strategy-engine/strategies/grid_strategy.py`: - -```python -from __future__ import annotations - -from decimal import Decimal - -from shared.models import Candle, Signal, OrderSide -from strategies.base import BaseStrategy - - -class GridStrategy(BaseStrategy): - name = "grid" - - def __init__(self): - self._lower: float = 0 - self._upper: float = 0 - self._grid_count: int = 5 - self._quantity: Decimal = Decimal("0.01") - self._grid_levels: list[float] = [] - self._last_zone: int | None = None - - def configure(self, params: dict) -> None: - self._lower = float(params["lower_price"]) - self._upper = float(params["upper_price"]) - self._grid_count = params.get("grid_count", 5) - self._quantity = Decimal(str(params.get("quantity", 0.01))) - step = (self._upper - self._lower) / self._grid_count - self._grid_levels = [self._lower + step * i for i in range(self._grid_count + 1)] - - def on_candle(self, candle: Candle) -> Signal | None: - price = float(candle.close) - current_zone = self._get_zone(price) - - if self._last_zone is None: - self._last_zone = current_zone - return None - - signal = None - if current_zone < self._last_zone: - signal = Signal( - strategy=self.name, - symbol=candle.symbol, - side=OrderSide.BUY, - price=candle.close, - quantity=self._quantity, - reason=f"Price crossed grid down: zone {self._last_zone}->{current_zone}", - ) - elif current_zone > self._last_zone: - signal = Signal( - strategy=self.name, - symbol=candle.symbol, - side=OrderSide.SELL, - price=candle.close, - quantity=self._quantity, - reason=f"Price crossed grid up: zone {self._last_zone}->{current_zone}", - ) - - self._last_zone = current_zone - return signal - - def _get_zone(self, price: float) -> int: - for i, level in enumerate(self._grid_levels): - if price < level: - return i - return len(self._grid_levels) - - def reset(self) -> None: - self._last_zone = None -``` - -- [ ] **Step 10: Run tests to verify they pass** - -```bash -pytest services/strategy-engine/tests/test_grid_strategy.py -v -``` - -Expected: All PASS - -- [ ] **Step 11: Write failing tests for plugin_loader** - -Create `services/strategy-engine/tests/test_plugin_loader.py`: - -```python -import pytest -from pathlib import Path - -from strategy_engine.plugin_loader import load_strategies - - -def test_load_strategies_finds_rsi_and_grid(): - strategies_dir = Path(__file__).parent.parent / "strategies" - loaded = load_strategies(strategies_dir) - - names = {s.name for s in loaded} - assert "rsi" in names - assert "grid" in names - - -def test_load_strategies_skips_base(): - strategies_dir = Path(__file__).parent.parent / "strategies" - loaded = load_strategies(strategies_dir) - - names = {s.name for s in loaded} - assert "base" not in names -``` - -- [ ] **Step 12: Run tests to verify they fail** - -```bash -pytest services/strategy-engine/tests/test_plugin_loader.py -v -``` - -Expected: FAIL - -- [ ] **Step 13: Implement plugin_loader** - -Create `services/strategy-engine/src/strategy_engine/plugin_loader.py`: - -```python -from __future__ import annotations - -import importlib.util -import logging -from pathlib import Path - -from strategies.base import BaseStrategy - -logger = logging.getLogger(__name__) - - -def load_strategies(strategies_dir: Path) -> list[BaseStrategy]: - loaded = [] - for path in strategies_dir.glob("*.py"): - if path.stem.startswith("_") or path.stem == "base": - continue - - spec = importlib.util.spec_from_file_location(path.stem, path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - for attr_name in dir(module): - attr = getattr(module, attr_name) - if ( - isinstance(attr, type) - and issubclass(attr, BaseStrategy) - and attr is not BaseStrategy - ): - instance = attr() - loaded.append(instance) - logger.info(f"Loaded strategy: {instance.name}") - - return loaded -``` - -- [ ] **Step 14: Run tests to verify they pass** - -```bash -pytest services/strategy-engine/tests/test_plugin_loader.py -v -``` - -Expected: All PASS - -- [ ] **Step 15: Write failing tests for engine** - -Create `services/strategy-engine/tests/test_engine.py`: - -```python -import pytest -from unittest.mock import AsyncMock, MagicMock -from datetime import datetime, timezone -from decimal import Decimal - -from shared.models import Candle, OrderSide -from shared.events import CandleEvent -from strategy_engine.engine import StrategyEngine - - -def make_candle_event() -> dict: - candle = Candle( - symbol="BTCUSDT", - timeframe="1m", - open_time=datetime(2026, 1, 1, tzinfo=timezone.utc), - open=Decimal("50000"), - high=Decimal("50100"), - low=Decimal("49900"), - close=Decimal("50050"), - volume=Decimal("1.0"), - ) - return CandleEvent(data=candle).to_dict() - - -@pytest.mark.asyncio -async def test_engine_dispatches_candle_to_strategies(): - mock_strategy = MagicMock() - mock_strategy.name = "test" - mock_strategy.on_candle.return_value = None - - mock_broker = AsyncMock() - mock_broker.read = AsyncMock(return_value=[make_candle_event()]) - - engine = StrategyEngine(broker=mock_broker, strategies=[mock_strategy]) - await engine.process_once(stream="candles.BTCUSDT", last_id="0-0") - - mock_strategy.on_candle.assert_called_once() - - -@pytest.mark.asyncio -async def test_engine_publishes_signal_when_strategy_returns_one(): - from shared.models import Signal - - mock_signal = Signal( - strategy="test", - symbol="BTCUSDT", - side=OrderSide.BUY, - price=Decimal("50000"), - quantity=Decimal("0.01"), - reason="test reason", - ) - mock_strategy = MagicMock() - mock_strategy.name = "test" - mock_strategy.on_candle.return_value = mock_signal - - mock_broker = AsyncMock() - mock_broker.read = AsyncMock(return_value=[make_candle_event()]) - mock_broker.publish = AsyncMock() - - engine = StrategyEngine(broker=mock_broker, strategies=[mock_strategy]) - await engine.process_once(stream="candles.BTCUSDT", last_id="0-0") - - mock_broker.publish.assert_called_once() - call_args = mock_broker.publish.call_args - assert call_args[0][0] == "signals" -``` - -- [ ] **Step 16: Run tests to verify they fail** - -```bash -pytest services/strategy-engine/tests/test_engine.py -v -``` - -Expected: FAIL - -- [ ] **Step 17: Implement engine** - -Create `services/strategy-engine/src/strategy_engine/engine.py`: - -```python -from __future__ import annotations - -import logging - -from shared.broker import RedisBroker -from shared.events import Event, SignalEvent -from shared.models import Signal -from strategies.base import BaseStrategy - -logger = logging.getLogger(__name__) - - -class StrategyEngine: - def __init__(self, broker: RedisBroker, strategies: list[BaseStrategy]): - self._broker = broker - self._strategies = strategies - - async def process_once(self, stream: str, last_id: str) -> str: - messages = await self._broker.read(stream, last_id=last_id, count=10, block=1000) - - for msg in messages: - event = Event.from_dict(msg) - candle = event.data - - for strategy in self._strategies: - signal = strategy.on_candle(candle) - if signal is not None: - logger.info(f"Signal from {strategy.name}: {signal.side} {signal.symbol}") - await self._publish_signal(signal) - - return last_id - - async def _publish_signal(self, signal: Signal): - event = SignalEvent(data=signal) - await self._broker.publish("signals", event.to_dict()) -``` - -- [ ] **Step 18: Run tests to verify they pass** - -```bash -pytest services/strategy-engine/tests/test_engine.py -v -``` - -Expected: All PASS - -- [ ] **Step 19: Implement config and main** - -Create `services/strategy-engine/src/strategy_engine/config.py`: - -```python -from shared.config import Settings - - -class StrategyConfig(Settings): - symbols: list[str] = ["BTC/USDT"] - timeframes: list[str] = ["1m"] - strategy_params: dict = {} -``` - -Create `services/strategy-engine/src/strategy_engine/main.py`: - -```python -from __future__ import annotations - -import asyncio -import logging -from pathlib import Path - -from shared.broker import RedisBroker -from strategy_engine.config import StrategyConfig -from strategy_engine.engine import StrategyEngine -from strategy_engine.plugin_loader import load_strategies - -logger = logging.getLogger(__name__) - - -async def run(): - config = StrategyConfig() - logging.basicConfig(level=config.log_level) - - broker = RedisBroker(config.redis_url) - strategies_dir = Path(__file__).parent.parent.parent / "strategies" - strategies = load_strategies(strategies_dir) - - for s in strategies: - params = config.strategy_params.get(s.name, {}) - s.configure(params) - - engine = StrategyEngine(broker=broker, strategies=strategies) - symbols = [s.replace("/", "") for s in config.symbols] - - logger.info(f"Starting strategy engine: strategies={[s.name for s in strategies]}") - last_ids = {sym: "0-0" for sym in symbols} - try: - while True: - for sym in symbols: - stream = f"candles.{sym}" - last_ids[sym] = await engine.process_once(stream, last_ids[sym]) - finally: - await broker.close() - - -def main(): - asyncio.run(run()) - - -if __name__ == "__main__": - main() -``` - -- [ ] **Step 20: Create Dockerfile** - -Create `services/strategy-engine/Dockerfile`: - -```dockerfile -FROM python:3.12-slim - -WORKDIR /app - -COPY shared/ shared/ -RUN pip install --no-cache-dir ./shared - -COPY services/strategy-engine/ services/strategy-engine/ -RUN pip install --no-cache-dir ./services/strategy-engine - -CMD ["python", "-m", "strategy_engine.main"] -``` - -- [ ] **Step 21: Commit** - -```bash -git add services/strategy-engine/ -git commit -m "feat(strategy-engine): add plugin-based strategy engine with RSI and grid strategies" -``` - ---- - -## Task 7: Order Executor Service - -**Files:** -- Create: `services/order-executor/pyproject.toml` -- Create: `services/order-executor/Dockerfile` -- Create: `services/order-executor/src/order_executor/__init__.py` -- Create: `services/order-executor/src/order_executor/config.py` -- Create: `services/order-executor/src/order_executor/risk_manager.py` -- Create: `services/order-executor/src/order_executor/executor.py` -- Create: `services/order-executor/src/order_executor/main.py` -- Create: `services/order-executor/tests/test_risk_manager.py` -- Create: `services/order-executor/tests/test_executor.py` - -- [ ] **Step 1: Create pyproject.toml** - -Create `services/order-executor/pyproject.toml`: - -```toml -[project] -name = "order-executor" -version = "0.1.0" -description = "Order execution service with risk management" -requires-python = ">=3.12" -dependencies = [ - "ccxt>=4.0", - "trading-shared", -] - -[project.optional-dependencies] -dev = [ - "pytest>=8.0", - "pytest-asyncio>=0.23", -] - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["src/order_executor"] -``` - -- [ ] **Step 2: Write failing tests for risk_manager** - -Create `services/order-executor/tests/test_risk_manager.py`: - -```python -import pytest -from decimal import Decimal - -from shared.models import Signal, OrderSide -from order_executor.risk_manager import RiskManager, RiskCheckResult - - -@pytest.fixture -def risk_manager(): - return RiskManager( - max_position_size=Decimal("0.1"), - stop_loss_pct=Decimal("5"), - daily_loss_limit_pct=Decimal("10"), - ) - - -def make_signal(side=OrderSide.BUY, quantity="0.01", price="50000") -> Signal: - return Signal( - strategy="test", - symbol="BTCUSDT", - side=side, - price=Decimal(price), - quantity=Decimal(quantity), - reason="test", - ) - - -def test_risk_check_passes_normal_order(risk_manager): - signal = make_signal() - balance = Decimal("10000") - positions = {} - daily_pnl = Decimal("0") - - result = risk_manager.check(signal, balance, positions, daily_pnl) - assert result.allowed is True - - -def test_risk_check_rejects_exceeding_position_size(risk_manager): - signal = make_signal(quantity="5") # 5 BTC * 50000 = 250000 > 10% of balance - balance = Decimal("10000") - positions = {} - daily_pnl = Decimal("0") - - result = risk_manager.check(signal, balance, positions, daily_pnl) - assert result.allowed is False - assert "position size" in result.reason.lower() - - -def test_risk_check_rejects_daily_loss_exceeded(risk_manager): - signal = make_signal() - balance = Decimal("10000") - positions = {} - daily_pnl = Decimal("-1100") # -11% > -10% limit - - result = risk_manager.check(signal, balance, positions, daily_pnl) - assert result.allowed is False - assert "daily loss" in result.reason.lower() - - -def test_risk_check_rejects_insufficient_balance(risk_manager): - signal = make_signal(quantity="0.01", price="50000") # cost = 500 - balance = Decimal("100") # not enough - positions = {} - daily_pnl = Decimal("0") - - result = risk_manager.check(signal, balance, positions, daily_pnl) - assert result.allowed is False - assert "balance" in result.reason.lower() -``` - -- [ ] **Step 3: Run tests to verify they fail** - -```bash -pip install -e services/order-executor[dev] -pytest services/order-executor/tests/test_risk_manager.py -v -``` - -Expected: FAIL - -- [ ] **Step 4: Implement risk_manager** - -Create `services/order-executor/src/order_executor/__init__.py`: - -```python -``` - -Create `services/order-executor/src/order_executor/risk_manager.py`: - -```python -from __future__ import annotations - -from dataclasses import dataclass -from decimal import Decimal - -from shared.models import Signal, OrderSide - - -@dataclass -class RiskCheckResult: - allowed: bool - reason: str = "" - - -class RiskManager: - def __init__( - self, - max_position_size: Decimal, - stop_loss_pct: Decimal, - daily_loss_limit_pct: Decimal, - ): - self._max_position_size = max_position_size - self._stop_loss_pct = stop_loss_pct - self._daily_loss_limit_pct = daily_loss_limit_pct - - def check( - self, - signal: Signal, - balance: Decimal, - positions: dict[str, Decimal], - daily_pnl: Decimal, - ) -> RiskCheckResult: - # Check daily loss limit - daily_loss_pct = (daily_pnl / balance) * 100 if balance > 0 else Decimal("0") - if daily_loss_pct < -self._daily_loss_limit_pct: - return RiskCheckResult( - allowed=False, - reason=f"Daily loss limit exceeded: {daily_loss_pct:.1f}%", - ) - - if signal.side == OrderSide.BUY: - order_cost = signal.price * signal.quantity - - # Check sufficient balance - if order_cost > balance: - return RiskCheckResult( - allowed=False, - reason=f"Insufficient balance: need {order_cost}, have {balance}", - ) - - # Check max position size - current_position_value = positions.get(signal.symbol, Decimal("0")) * signal.price - new_position_value = current_position_value + order_cost - position_ratio = new_position_value / balance if balance > 0 else Decimal("999") - if position_ratio > self._max_position_size: - return RiskCheckResult( - allowed=False, - reason=f"Position size exceeded: {position_ratio:.2f} > {self._max_position_size}", - ) - - return RiskCheckResult(allowed=True) -``` - -- [ ] **Step 5: Run tests to verify they pass** - -```bash -pytest services/order-executor/tests/test_risk_manager.py -v -``` - -Expected: All PASS - -- [ ] **Step 6: Write failing tests for executor** - -Create `services/order-executor/tests/test_executor.py`: - -```python -import pytest -from unittest.mock import AsyncMock, MagicMock -from decimal import Decimal - -from shared.models import Signal, OrderSide, OrderStatus -from order_executor.executor import OrderExecutor -from order_executor.risk_manager import RiskCheckResult - - -def make_signal() -> Signal: - return Signal( - strategy="test", - symbol="BTCUSDT", - side=OrderSide.BUY, - price=Decimal("50000"), - quantity=Decimal("0.01"), - reason="test", - ) - - -@pytest.mark.asyncio -async def test_executor_places_order_when_risk_passes(): - mock_exchange = MagicMock() - mock_exchange.create_order = AsyncMock(return_value={ - "id": "binance_123", - "status": "closed", - "filled": 0.01, - "price": 50000, - }) - mock_exchange.fetch_balance = AsyncMock(return_value={ - "USDT": {"free": 10000}, - }) - - mock_risk = MagicMock() - mock_risk.check.return_value = RiskCheckResult(allowed=True) - - mock_broker = AsyncMock() - mock_db = AsyncMock() - - executor = OrderExecutor( - exchange=mock_exchange, - risk_manager=mock_risk, - broker=mock_broker, - db=mock_db, - dry_run=False, - ) - - signal = make_signal() - order = await executor.execute(signal) - - assert order is not None - assert order.status == OrderStatus.FILLED - mock_exchange.create_order.assert_called_once() - - -@pytest.mark.asyncio -async def test_executor_rejects_when_risk_fails(): - mock_exchange = MagicMock() - mock_exchange.fetch_balance = AsyncMock(return_value={ - "USDT": {"free": 10000}, - }) - - mock_risk = MagicMock() - mock_risk.check.return_value = RiskCheckResult(allowed=False, reason="too risky") - - mock_broker = AsyncMock() - mock_db = AsyncMock() - - executor = OrderExecutor( - exchange=mock_exchange, - risk_manager=mock_risk, - broker=mock_broker, - db=mock_db, - dry_run=False, - ) - - signal = make_signal() - order = await executor.execute(signal) - assert order is None - mock_exchange.create_order.assert_not_called() - - -@pytest.mark.asyncio -async def test_executor_dry_run_does_not_call_exchange(): - mock_exchange = MagicMock() - mock_exchange.fetch_balance = AsyncMock(return_value={ - "USDT": {"free": 10000}, - }) - - mock_risk = MagicMock() - mock_risk.check.return_value = RiskCheckResult(allowed=True) - - mock_broker = AsyncMock() - mock_db = AsyncMock() - - executor = OrderExecutor( - exchange=mock_exchange, - risk_manager=mock_risk, - broker=mock_broker, - db=mock_db, - dry_run=True, - ) - - signal = make_signal() - order = await executor.execute(signal) - - assert order is not None - assert order.status == OrderStatus.FILLED - mock_exchange.create_order.assert_not_called() -``` - -- [ ] **Step 7: Run tests to verify they fail** - -```bash -pytest services/order-executor/tests/test_executor.py -v -``` - -Expected: FAIL - -- [ ] **Step 8: Implement executor** - -Create `services/order-executor/src/order_executor/executor.py`: - -```python -from __future__ import annotations - -import logging -from datetime import datetime, timezone -from decimal import Decimal - -from shared.broker import RedisBroker -from shared.db import Database -from shared.events import OrderEvent -from shared.models import Order, OrderSide, OrderStatus, OrderType, Signal -from order_executor.risk_manager import RiskManager - -logger = logging.getLogger(__name__) - - -class OrderExecutor: - def __init__( - self, - exchange, - risk_manager: RiskManager, - broker: RedisBroker, - db: Database, - dry_run: bool = True, - ): - self._exchange = exchange - self._risk = risk_manager - self._broker = broker - self._db = db - self._dry_run = dry_run - - async def execute(self, signal: Signal) -> Order | None: - balance_info = await self._exchange.fetch_balance() - balance = Decimal(str(balance_info.get("USDT", {}).get("free", 0))) - positions: dict[str, Decimal] = {} - daily_pnl = Decimal("0") - - result = self._risk.check(signal, balance, positions, daily_pnl) - if not result.allowed: - logger.warning(f"Risk check failed: {result.reason}") - return None - - order = Order( - signal_id=signal.id, - symbol=signal.symbol, - side=signal.side, - type=OrderType.MARKET, - price=signal.price, - quantity=signal.quantity, - ) - - if self._dry_run: - logger.info(f"[DRY RUN] Would execute: {order.side} {order.quantity} {order.symbol}") - order.status = OrderStatus.FILLED - order.filled_at = datetime.now(timezone.utc) - else: - try: - result = await self._exchange.create_order( - symbol=signal.symbol.replace("USDT", "/USDT"), - type="market", - side=signal.side.value.lower(), - amount=float(signal.quantity), - ) - order.status = OrderStatus.FILLED - order.filled_at = datetime.now(timezone.utc) - logger.info(f"Order filled: {order.id}") - except Exception as e: - order.status = OrderStatus.FAILED - logger.error(f"Order failed: {e}") - - await self._db.insert_order(order) - event = OrderEvent(data=order) - await self._broker.publish("orders", event.to_dict()) - - return order -``` - -- [ ] **Step 9: Run tests to verify they pass** - -```bash -pytest services/order-executor/tests/test_executor.py -v -``` - -Expected: All PASS - -- [ ] **Step 10: Implement config and main** - -Create `services/order-executor/src/order_executor/config.py`: - -```python -from shared.config import Settings - - -class ExecutorConfig(Settings): - pass -``` - -Create `services/order-executor/src/order_executor/main.py`: - -```python -from __future__ import annotations - -import asyncio -import logging - -import ccxt.async_support as ccxt - -from shared.broker import RedisBroker -from shared.db import Database -from shared.events import Event -from order_executor.config import ExecutorConfig -from order_executor.executor import OrderExecutor -from order_executor.risk_manager import RiskManager - -logger = logging.getLogger(__name__) - - -async def run(): - config = ExecutorConfig() - logging.basicConfig(level=config.log_level) - - db = Database(config.database_url) - await db.connect() - - broker = RedisBroker(config.redis_url) - - exchange = ccxt.binance({ - "apiKey": config.binance_api_key, - "secret": config.binance_api_secret, - }) - - risk_manager = RiskManager( - max_position_size=config.risk_max_position_size, - stop_loss_pct=config.risk_stop_loss_pct, - daily_loss_limit_pct=config.risk_daily_loss_limit_pct, - ) - - executor = OrderExecutor( - exchange=exchange, - risk_manager=risk_manager, - broker=broker, - db=db, - dry_run=config.dry_run, - ) - - logger.info(f"Starting order executor (dry_run={config.dry_run})") - last_id = "0-0" - try: - while True: - messages = await broker.read("signals", last_id=last_id, count=10, block=1000) - for msg in messages: - event = Event.from_dict(msg) - await executor.execute(event.data) - finally: - await exchange.close() - await broker.close() - await db.close() - - -def main(): - asyncio.run(run()) - - -if __name__ == "__main__": - main() -``` - -- [ ] **Step 11: Create Dockerfile** - -Create `services/order-executor/Dockerfile`: - -```dockerfile -FROM python:3.12-slim - -WORKDIR /app - -COPY shared/ shared/ -RUN pip install --no-cache-dir ./shared - -COPY services/order-executor/ services/order-executor/ -RUN pip install --no-cache-dir ./services/order-executor - -CMD ["python", "-m", "order_executor.main"] -``` - -- [ ] **Step 12: Commit** - -```bash -git add services/order-executor/ -git commit -m "feat(order-executor): add order execution with risk management and dry-run mode" -``` - ---- - -## Task 8: Portfolio Manager Service - -**Files:** -- Create: `services/portfolio-manager/pyproject.toml` -- Create: `services/portfolio-manager/Dockerfile` -- Create: `services/portfolio-manager/src/portfolio_manager/__init__.py` -- Create: `services/portfolio-manager/src/portfolio_manager/config.py` -- Create: `services/portfolio-manager/src/portfolio_manager/portfolio.py` -- Create: `services/portfolio-manager/src/portfolio_manager/pnl.py` -- Create: `services/portfolio-manager/src/portfolio_manager/main.py` -- Create: `services/portfolio-manager/tests/test_portfolio.py` -- Create: `services/portfolio-manager/tests/test_pnl.py` - -- [ ] **Step 1: Create pyproject.toml** - -Create `services/portfolio-manager/pyproject.toml`: - -```toml -[project] -name = "portfolio-manager" -version = "0.1.0" -description = "Portfolio tracking and PnL calculation service" -requires-python = ">=3.12" -dependencies = [ - "trading-shared", -] - -[project.optional-dependencies] -dev = [ - "pytest>=8.0", - "pytest-asyncio>=0.23", -] - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["src/portfolio_manager"] -``` - -- [ ] **Step 2: Write failing tests for pnl** - -Create `services/portfolio-manager/tests/test_pnl.py`: - -```python -from decimal import Decimal - -from portfolio_manager.pnl import calculate_unrealized_pnl, calculate_realized_pnl - - -def test_unrealized_pnl_profit(): - result = calculate_unrealized_pnl( - quantity=Decimal("0.1"), - avg_entry_price=Decimal("50000"), - current_price=Decimal("55000"), - ) - assert result == Decimal("500") # 0.1 * (55000 - 50000) - - -def test_unrealized_pnl_loss(): - result = calculate_unrealized_pnl( - quantity=Decimal("0.1"), - avg_entry_price=Decimal("50000"), - current_price=Decimal("45000"), - ) - assert result == Decimal("-500") - - -def test_realized_pnl_single_trade(): - result = calculate_realized_pnl( - buy_price=Decimal("50000"), - sell_price=Decimal("55000"), - quantity=Decimal("0.1"), - fee=Decimal("5.5"), - ) - assert result == Decimal("494.5") # 0.1 * (55000 - 50000) - 5.5 -``` - -- [ ] **Step 3: Run tests to verify they fail** - -```bash -pip install -e services/portfolio-manager[dev] -pytest services/portfolio-manager/tests/test_pnl.py -v -``` - -Expected: FAIL - -- [ ] **Step 4: Implement pnl** - -Create `services/portfolio-manager/src/portfolio_manager/__init__.py`: - -```python -``` - -Create `services/portfolio-manager/src/portfolio_manager/pnl.py`: - -```python -from decimal import Decimal - - -def calculate_unrealized_pnl( - quantity: Decimal, - avg_entry_price: Decimal, - current_price: Decimal, -) -> Decimal: - return quantity * (current_price - avg_entry_price) - - -def calculate_realized_pnl( - buy_price: Decimal, - sell_price: Decimal, - quantity: Decimal, - fee: Decimal = Decimal("0"), -) -> Decimal: - return quantity * (sell_price - buy_price) - fee -``` - -- [ ] **Step 5: Run tests to verify they pass** - -```bash -pytest services/portfolio-manager/tests/test_pnl.py -v -``` - -Expected: All PASS - -- [ ] **Step 6: Write failing tests for portfolio** - -Create `services/portfolio-manager/tests/test_portfolio.py`: - -```python -import pytest -from decimal import Decimal - -from shared.models import Order, OrderSide, OrderType, OrderStatus -from portfolio_manager.portfolio import PortfolioTracker - - -@pytest.fixture -def tracker(): - return PortfolioTracker() - - -def make_order(side=OrderSide.BUY, price="50000", quantity="0.1") -> Order: - return Order( - signal_id="sig_1", - symbol="BTCUSDT", - side=side, - type=OrderType.MARKET, - price=Decimal(price), - quantity=Decimal(quantity), - status=OrderStatus.FILLED, - ) - - -def test_portfolio_add_buy_order(tracker): - order = make_order(side=OrderSide.BUY) - tracker.apply_order(order) - - pos = tracker.get_position("BTCUSDT") - assert pos.quantity == Decimal("0.1") - assert pos.avg_entry_price == Decimal("50000") - - -def test_portfolio_add_multiple_buys(tracker): - tracker.apply_order(make_order(price="50000", quantity="0.1")) - tracker.apply_order(make_order(price="52000", quantity="0.1")) - - pos = tracker.get_position("BTCUSDT") - assert pos.quantity == Decimal("0.2") - assert pos.avg_entry_price == Decimal("51000") # weighted avg - - -def test_portfolio_sell_reduces_position(tracker): - tracker.apply_order(make_order(side=OrderSide.BUY, price="50000", quantity="0.2")) - tracker.apply_order(make_order(side=OrderSide.SELL, price="55000", quantity="0.1")) - - pos = tracker.get_position("BTCUSDT") - assert pos.quantity == Decimal("0.1") - assert pos.avg_entry_price == Decimal("50000") # entry price unchanged - - -def test_portfolio_no_position_returns_none(tracker): - pos = tracker.get_position("ETHUSDT") - assert pos is None -``` - -- [ ] **Step 7: Run tests to verify they fail** - -```bash -pytest services/portfolio-manager/tests/test_portfolio.py -v -``` - -Expected: FAIL - -- [ ] **Step 8: Implement portfolio** - -Create `services/portfolio-manager/src/portfolio_manager/portfolio.py`: - -```python -from __future__ import annotations - -from decimal import Decimal - -from shared.models import Order, OrderSide, Position - - -class PortfolioTracker: - def __init__(self): - self._positions: dict[str, _PositionState] = {} - - def apply_order(self, order: Order) -> None: - if order.symbol not in self._positions: - self._positions[order.symbol] = _PositionState() - - state = self._positions[order.symbol] - if order.side == OrderSide.BUY: - total_cost = state.avg_entry * state.quantity + order.price * order.quantity - state.quantity += order.quantity - state.avg_entry = total_cost / state.quantity if state.quantity > 0 else Decimal("0") - elif order.side == OrderSide.SELL: - state.quantity -= order.quantity - if state.quantity <= 0: - state.quantity = Decimal("0") - state.avg_entry = Decimal("0") - - def get_position(self, symbol: str) -> Position | None: - state = self._positions.get(symbol) - if state is None or state.quantity == 0: - return None - return Position( - symbol=symbol, - quantity=state.quantity, - avg_entry_price=state.avg_entry, - current_price=Decimal("0"), - ) - - def get_all_positions(self) -> list[Position]: - positions = [] - for symbol in self._positions: - pos = self.get_position(symbol) - if pos is not None: - positions.append(pos) - return positions - - -class _PositionState: - def __init__(self): - self.quantity = Decimal("0") - self.avg_entry = Decimal("0") -``` - -- [ ] **Step 9: Run tests to verify they pass** - -```bash -pytest services/portfolio-manager/tests/test_portfolio.py -v -``` - -Expected: All PASS - -- [ ] **Step 10: Implement config and main** - -Create `services/portfolio-manager/src/portfolio_manager/config.py`: - -```python -from shared.config import Settings - - -class PortfolioConfig(Settings): - snapshot_interval_hours: int = 24 -``` - -Create `services/portfolio-manager/src/portfolio_manager/main.py`: - -```python -from __future__ import annotations - -import asyncio -import logging - -from shared.broker import RedisBroker -from shared.db import Database -from shared.events import Event -from portfolio_manager.config import PortfolioConfig -from portfolio_manager.portfolio import PortfolioTracker - -logger = logging.getLogger(__name__) - - -async def run(): - config = PortfolioConfig() - logging.basicConfig(level=config.log_level) - - db = Database(config.database_url) - await db.connect() - - broker = RedisBroker(config.redis_url) - tracker = PortfolioTracker() - - logger.info("Starting portfolio manager") - last_id = "0-0" - try: - while True: - messages = await broker.read("orders", last_id=last_id, count=10, block=1000) - for msg in messages: - event = Event.from_dict(msg) - order = event.data - tracker.apply_order(order) - logger.info(f"Position updated: {order.symbol}") - finally: - await broker.close() - await db.close() - - -def main(): - asyncio.run(run()) - - -if __name__ == "__main__": - main() -``` - -- [ ] **Step 11: Create Dockerfile** - -Create `services/portfolio-manager/Dockerfile`: - -```dockerfile -FROM python:3.12-slim - -WORKDIR /app - -COPY shared/ shared/ -RUN pip install --no-cache-dir ./shared - -COPY services/portfolio-manager/ services/portfolio-manager/ -RUN pip install --no-cache-dir ./services/portfolio-manager - -CMD ["python", "-m", "portfolio_manager.main"] -``` - -- [ ] **Step 12: Commit** - -```bash -git add services/portfolio-manager/ -git commit -m "feat(portfolio-manager): add portfolio tracking and PnL calculation" -``` - ---- - -## Task 9: Backtester Service - -**Files:** -- Create: `services/backtester/pyproject.toml` -- Create: `services/backtester/Dockerfile` -- Create: `services/backtester/src/backtester/__init__.py` -- Create: `services/backtester/src/backtester/config.py` -- Create: `services/backtester/src/backtester/simulator.py` -- Create: `services/backtester/src/backtester/engine.py` -- Create: `services/backtester/src/backtester/reporter.py` -- Create: `services/backtester/src/backtester/main.py` -- Create: `services/backtester/tests/test_simulator.py` -- Create: `services/backtester/tests/test_engine.py` -- Create: `services/backtester/tests/test_reporter.py` - -- [ ] **Step 1: Create pyproject.toml** - -Create `services/backtester/pyproject.toml`: - -```toml -[project] -name = "backtester" -version = "0.1.0" -description = "Strategy backtesting engine" -requires-python = ">=3.12" -dependencies = [ - "pandas>=2.0", - "trading-shared", -] - -[project.optional-dependencies] -dev = [ - "pytest>=8.0", - "pytest-asyncio>=0.23", -] - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["src/backtester"] -``` - -- [ ] **Step 2: Write failing tests for simulator** - -Create `services/backtester/tests/test_simulator.py`: - -```python -from decimal import Decimal - -from shared.models import Signal, OrderSide -from backtester.simulator import OrderSimulator - - -def make_signal(side=OrderSide.BUY, price="50000", quantity="0.1") -> Signal: - return Signal( - strategy="test", - symbol="BTCUSDT", - side=side, - price=Decimal(price), - quantity=Decimal(quantity), - reason="test", - ) - - -def test_simulator_initial_balance(): - sim = OrderSimulator(initial_balance=Decimal("10000")) - assert sim.balance == Decimal("10000") - - -def test_simulator_buy_reduces_balance(): - sim = OrderSimulator(initial_balance=Decimal("10000")) - sim.execute(make_signal(side=OrderSide.BUY, price="50000", quantity="0.1")) - - assert sim.balance == Decimal("5000") # 10000 - 0.1*50000 - assert sim.positions["BTCUSDT"] == Decimal("0.1") - - -def test_simulator_sell_increases_balance(): - sim = OrderSimulator(initial_balance=Decimal("10000")) - sim.execute(make_signal(side=OrderSide.BUY, price="50000", quantity="0.1")) - sim.execute(make_signal(side=OrderSide.SELL, price="55000", quantity="0.1")) - - assert sim.balance == Decimal("10500") # 5000 + 0.1*55000 - assert sim.positions.get("BTCUSDT", Decimal("0")) == Decimal("0") - - -def test_simulator_reject_buy_insufficient_balance(): - sim = OrderSimulator(initial_balance=Decimal("100")) - result = sim.execute(make_signal(side=OrderSide.BUY, price="50000", quantity="0.1")) - assert result is False - assert sim.balance == Decimal("100") - - -def test_simulator_trade_history(): - sim = OrderSimulator(initial_balance=Decimal("10000")) - sim.execute(make_signal(side=OrderSide.BUY)) - assert len(sim.trades) == 1 -``` - -- [ ] **Step 3: Run tests to verify they fail** - -```bash -pip install -e services/backtester[dev] -pytest services/backtester/tests/test_simulator.py -v -``` - -Expected: FAIL - -- [ ] **Step 4: Implement simulator** - -Create `services/backtester/src/backtester/__init__.py`: - -```python -``` - -Create `services/backtester/src/backtester/simulator.py`: - -```python -from __future__ import annotations - -from dataclasses import dataclass, field -from decimal import Decimal - -from shared.models import Signal, OrderSide - - -@dataclass -class SimulatedTrade: - symbol: str - side: OrderSide - price: Decimal - quantity: Decimal - balance_after: Decimal - - -class OrderSimulator: - def __init__(self, initial_balance: Decimal): - self.balance = initial_balance - self.positions: dict[str, Decimal] = {} - self.trades: list[SimulatedTrade] = [] - - def execute(self, signal: Signal) -> bool: - if signal.side == OrderSide.BUY: - cost = signal.price * signal.quantity - if cost > self.balance: - return False - self.balance -= cost - current = self.positions.get(signal.symbol, Decimal("0")) - self.positions[signal.symbol] = current + signal.quantity - elif signal.side == OrderSide.SELL: - current = self.positions.get(signal.symbol, Decimal("0")) - sell_qty = min(signal.quantity, current) - if sell_qty <= 0: - return False - self.balance += signal.price * sell_qty - self.positions[signal.symbol] = current - sell_qty - - self.trades.append( - SimulatedTrade( - symbol=signal.symbol, - side=signal.side, - price=signal.price, - quantity=signal.quantity, - balance_after=self.balance, - ) - ) - return True -``` - -- [ ] **Step 5: Run tests to verify they pass** - -```bash -pytest services/backtester/tests/test_simulator.py -v -``` - -Expected: All PASS - -- [ ] **Step 6: Write failing tests for backtest engine** - -Create `services/backtester/tests/test_engine.py`: - -```python -import pytest -from decimal import Decimal -from datetime import datetime, timezone -from unittest.mock import MagicMock - -from shared.models import Candle, Signal, OrderSide -from backtester.engine import BacktestEngine - - -def make_candles(prices: list[float]) -> list[Candle]: - return [ - Candle( - symbol="BTCUSDT", - timeframe="1m", - open_time=datetime(2026, 1, 1, minute=i, tzinfo=timezone.utc), - open=Decimal(str(p)), - high=Decimal(str(p + 10)), - low=Decimal(str(p - 10)), - close=Decimal(str(p)), - volume=Decimal("1.0"), - ) - for i, p in enumerate(prices) - ] - - -def test_backtest_engine_runs_strategy_over_candles(): - mock_strategy = MagicMock() - mock_strategy.name = "test" - mock_strategy.on_candle.return_value = None - - candles = make_candles([50000, 50100, 50200]) - - engine = BacktestEngine( - strategy=mock_strategy, - initial_balance=Decimal("10000"), - ) - result = engine.run(candles) - - assert mock_strategy.on_candle.call_count == 3 - assert result.total_trades == 0 - assert result.final_balance == Decimal("10000") - - -def test_backtest_engine_executes_signals(): - buy_signal = Signal( - strategy="test", - symbol="BTCUSDT", - side=OrderSide.BUY, - price=Decimal("50000"), - quantity=Decimal("0.1"), - reason="test buy", - ) - sell_signal = Signal( - strategy="test", - symbol="BTCUSDT", - side=OrderSide.SELL, - price=Decimal("55000"), - quantity=Decimal("0.1"), - reason="test sell", - ) - - mock_strategy = MagicMock() - mock_strategy.name = "test" - mock_strategy.on_candle.side_effect = [buy_signal, None, sell_signal] - - candles = make_candles([50000, 52000, 55000]) - - engine = BacktestEngine( - strategy=mock_strategy, - initial_balance=Decimal("10000"), - ) - result = engine.run(candles) - - assert result.total_trades == 2 - assert result.final_balance == Decimal("10500") # 10000 - 5000 + 5500 -``` - -- [ ] **Step 7: Run tests to verify they fail** - -```bash -pytest services/backtester/tests/test_engine.py -v -``` - -Expected: FAIL - -- [ ] **Step 8: Implement backtest engine** - -Create `services/backtester/src/backtester/engine.py`: - -```python -from __future__ import annotations - -from dataclasses import dataclass -from decimal import Decimal - -from shared.models import Candle -from backtester.simulator import OrderSimulator -from strategies.base import BaseStrategy - - -@dataclass -class BacktestResult: - strategy_name: str - symbol: str - total_trades: int - initial_balance: Decimal - final_balance: Decimal - profit: Decimal - profit_pct: Decimal - trades: list - - @property - def win_rate(self) -> Decimal: - if self.total_trades == 0: - return Decimal("0") - wins = sum( - 1 - for i in range(0, len(self.trades) - 1, 2) - if i + 1 < len(self.trades) - and self.trades[i + 1].balance_after > self.trades[i].balance_after - ) - pairs = self.total_trades // 2 - return Decimal(str(wins / pairs * 100)) if pairs > 0 else Decimal("0") - - -class BacktestEngine: - def __init__(self, strategy: BaseStrategy, initial_balance: Decimal): - self._strategy = strategy - self._initial_balance = initial_balance - - def run(self, candles: list[Candle]) -> BacktestResult: - simulator = OrderSimulator(self._initial_balance) - symbol = candles[0].symbol if candles else "" - - for candle in candles: - signal = self._strategy.on_candle(candle) - if signal is not None: - simulator.execute(signal) - - final = simulator.balance - # Add value of remaining positions at last candle price - if candles: - last_price = candles[-1].close - for sym, qty in simulator.positions.items(): - final += qty * last_price - - profit = final - self._initial_balance - profit_pct = (profit / self._initial_balance) * 100 if self._initial_balance > 0 else Decimal("0") - - return BacktestResult( - strategy_name=self._strategy.name, - symbol=symbol, - total_trades=len(simulator.trades), - initial_balance=self._initial_balance, - final_balance=final, - profit=profit, - profit_pct=profit_pct, - trades=simulator.trades, - ) -``` - -- [ ] **Step 9: Run tests to verify they pass** - -```bash -pytest services/backtester/tests/test_engine.py -v -``` - -Expected: All PASS - -- [ ] **Step 10: Write failing tests for reporter** - -Create `services/backtester/tests/test_reporter.py`: - -```python -from decimal import Decimal - -from backtester.engine import BacktestResult -from backtester.reporter import format_report - - -def test_format_report_contains_key_metrics(): - result = BacktestResult( - strategy_name="rsi", - symbol="BTCUSDT", - total_trades=10, - initial_balance=Decimal("10000"), - final_balance=Decimal("11500"), - profit=Decimal("1500"), - profit_pct=Decimal("15"), - trades=[], - ) - - report = format_report(result) - - assert "rsi" in report - assert "BTCUSDT" in report - assert "10000" in report - assert "11500" in report - assert "1500" in report - assert "15" in report -``` - -- [ ] **Step 11: Run test to verify it fails** - -```bash -pytest services/backtester/tests/test_reporter.py -v -``` - -Expected: FAIL - -- [ ] **Step 12: Implement reporter** - -Create `services/backtester/src/backtester/reporter.py`: - -```python -from backtester.engine import BacktestResult - - -def format_report(result: BacktestResult) -> str: - lines = [ - "=" * 50, - f" Backtest Report: {result.strategy_name}", - "=" * 50, - f" Symbol: {result.symbol}", - f" Total Trades: {result.total_trades}", - f" Initial Balance: {result.initial_balance}", - f" Final Balance: {result.final_balance}", - f" Profit: {result.profit}", - f" Profit %: {result.profit_pct:.2f}%", - f" Win Rate: {result.win_rate:.1f}%", - "=" * 50, - ] - return "\n".join(lines) -``` - -- [ ] **Step 13: Run test to verify it passes** - -```bash -pytest services/backtester/tests/test_reporter.py -v -``` - -Expected: PASS - -- [ ] **Step 14: Implement config and main** - -Create `services/backtester/src/backtester/config.py`: - -```python -from shared.config import Settings - - -class BacktestConfig(Settings): - backtest_initial_balance: float = 10000.0 -``` - -Create `services/backtester/src/backtester/main.py`: - -```python -from __future__ import annotations - -import asyncio -import logging -from decimal import Decimal -from pathlib import Path - -from shared.db import Database -from backtester.config import BacktestConfig -from backtester.engine import BacktestEngine -from backtester.reporter import format_report - -logger = logging.getLogger(__name__) - - -async def run_backtest( - strategy_name: str, - symbol: str, - timeframe: str, - initial_balance: Decimal, - db: Database, - strategies_dir: Path, -) -> str: - from strategy_engine.plugin_loader import load_strategies - - strategies = load_strategies(strategies_dir) - strategy = next((s for s in strategies if s.name == strategy_name), None) - if strategy is None: - return f"Strategy '{strategy_name}' not found" - - candles_data = await db.get_candles(symbol, timeframe) - if not candles_data: - return f"No candle data for {symbol} {timeframe}" - - from shared.models import Candle - - candles = [Candle(**row) for row in reversed(candles_data)] - - engine = BacktestEngine(strategy=strategy, initial_balance=initial_balance) - result = engine.run(candles) - return format_report(result) -``` - -- [ ] **Step 15: Create Dockerfile** - -Create `services/backtester/Dockerfile`: - -```dockerfile -FROM python:3.12-slim - -WORKDIR /app - -COPY shared/ shared/ -RUN pip install --no-cache-dir ./shared - -COPY services/strategy-engine/strategies/ services/strategy-engine/strategies/ -COPY services/backtester/ services/backtester/ -RUN pip install --no-cache-dir ./services/backtester - -CMD ["python", "-m", "backtester.main"] -``` - -- [ ] **Step 16: Commit** - -```bash -git add services/backtester/ -git commit -m "feat(backtester): add backtesting engine with simulator and reporting" -``` - ---- - -## Task 10: CLI - -**Files:** -- Create: `cli/pyproject.toml` -- Create: `cli/src/trading_cli/__init__.py` -- Create: `cli/src/trading_cli/main.py` -- Create: `cli/src/trading_cli/commands/data.py` -- Create: `cli/src/trading_cli/commands/trade.py` -- Create: `cli/src/trading_cli/commands/backtest.py` -- Create: `cli/src/trading_cli/commands/portfolio.py` -- Create: `cli/src/trading_cli/commands/strategy.py` -- Create: `cli/src/trading_cli/commands/service.py` -- Create: `cli/tests/test_cli_data.py` - -- [ ] **Step 1: Create pyproject.toml** - -Create `cli/pyproject.toml`: - -```toml -[project] -name = "trading-cli" -version = "0.1.0" -description = "CLI interface for the trading platform" -requires-python = ">=3.12" -dependencies = [ - "click>=8.0", - "rich>=13.0", - "trading-shared", -] - -[project.scripts] -trading = "trading_cli.main:cli" - -[project.optional-dependencies] -dev = [ - "pytest>=8.0", - "pytest-asyncio>=0.23", -] - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["src/trading_cli"] -``` - -- [ ] **Step 2: Write failing tests for CLI data commands** - -Create `cli/tests/test_cli_data.py`: - -```python -from click.testing import CliRunner -from trading_cli.main import cli - - -def test_cli_help(): - runner = CliRunner() - result = runner.invoke(cli, ["--help"]) - assert result.exit_code == 0 - assert "trading" in result.output.lower() or "Usage" in result.output - - -def test_cli_data_group(): - runner = CliRunner() - result = runner.invoke(cli, ["data", "--help"]) - assert result.exit_code == 0 - assert "collect" in result.output - assert "history" in result.output -``` - -- [ ] **Step 3: Run tests to verify they fail** - -```bash -pip install -e cli[dev] -pytest cli/tests/test_cli_data.py -v -``` - -Expected: FAIL - -- [ ] **Step 4: Implement CLI main and data commands** - -Create `cli/src/trading_cli/__init__.py`: - -```python -``` - -Create `cli/src/trading_cli/main.py`: - -```python -import click - -from trading_cli.commands.data import data -from trading_cli.commands.trade import trade -from trading_cli.commands.backtest import backtest -from trading_cli.commands.portfolio import portfolio -from trading_cli.commands.strategy import strategy -from trading_cli.commands.service import service - - -@click.group() -@click.version_option(version="0.1.0") -def cli(): - """Trading Platform CLI — Binance spot crypto trading""" - pass - - -cli.add_command(data) -cli.add_command(trade) -cli.add_command(backtest) -cli.add_command(portfolio) -cli.add_command(strategy) -cli.add_command(service) -``` - -Create `cli/src/trading_cli/commands/data.py`: - -```python -import asyncio - -import click - - -@click.group() -def data(): - """Data collection commands""" - pass - - -@data.command() -@click.option("--symbol", required=True, help="Trading pair (e.g. BTCUSDT)") -@click.option("--timeframe", default="1m", help="Candle timeframe") -def collect(symbol: str, timeframe: str): - """Start real-time data collection""" - click.echo(f"Starting data collection: {symbol} {timeframe}") - - from data_collector.config import CollectorConfig - from data_collector.main import run - - asyncio.run(run()) - - -@data.command() -@click.option("--symbol", required=True, help="Trading pair (e.g. BTCUSDT)") -@click.option("--timeframe", default="1m", help="Candle timeframe") -@click.option("--from", "since", required=True, help="Start date (YYYY-MM-DD)") -@click.option("--limit", default=1000, help="Number of candles") -def history(symbol: str, timeframe: str, since: str, limit: int): - """Download historical candle data""" - click.echo(f"Downloading history: {symbol} {timeframe} from {since} (limit={limit})") - - async def _run(): - import ccxt.async_support as ccxt - from datetime import datetime, timezone - from shared.broker import RedisBroker - from shared.config import Settings - from shared.db import Database - from data_collector.binance_rest import fetch_historical_candles - from data_collector.storage import CandleStorage - - settings = Settings() - db = Database(settings.database_url) - await db.connect() - await db.init_tables() - broker = RedisBroker(settings.redis_url) - storage = CandleStorage(db=db, broker=broker) - - exchange = ccxt.binance() - since_dt = datetime.strptime(since, "%Y-%m-%d").replace(tzinfo=timezone.utc) - candles = await fetch_historical_candles( - exchange=exchange, - symbol=symbol.replace("USDT", "/USDT"), - timeframe=timeframe, - since=since_dt, - limit=limit, - ) - await storage.store_batch(candles) - await exchange.close() - await broker.close() - await db.close() - click.echo(f"Downloaded {len(candles)} candles") - - asyncio.run(_run()) - - -@data.command("list") -def list_data(): - """List currently collecting symbols""" - click.echo("Collecting symbols:") - click.echo(" (Check docker-compose service status)") -``` - -- [ ] **Step 5: Implement remaining CLI command stubs** - -Create `cli/src/trading_cli/commands/trade.py`: - -```python -import click - - -@click.group() -def trade(): - """Trading bot commands""" - pass - - -@trade.command() -@click.option("--strategy", required=True, help="Strategy name") -@click.option("--symbol", required=True, help="Trading pair") -def start(strategy: str, symbol: str): - """Start a trading bot""" - click.echo(f"Starting bot: strategy={strategy} symbol={symbol}") - - -@trade.command() -@click.option("--strategy", required=True, help="Strategy name") -def stop(strategy: str): - """Stop a trading bot""" - click.echo(f"Stopping bot: strategy={strategy}") - - -@trade.command() -def status(): - """Show running bot status""" - click.echo("Running bots:") - - -@trade.command("stop-all") -def stop_all(): - """Emergency stop: stop all bots and cancel all orders""" - click.confirm("Are you sure you want to stop ALL bots?", abort=True) - click.echo("Stopping all bots and cancelling open orders...") -``` - -Create `cli/src/trading_cli/commands/backtest.py`: - -```python -import asyncio -from decimal import Decimal - -import click - - -@click.group() -def backtest(): - """Backtesting commands""" - pass - - -@backtest.command("run") -@click.option("--strategy", required=True, help="Strategy name") -@click.option("--symbol", required=True, help="Trading pair") -@click.option("--from", "since", required=True, help="Start date") -@click.option("--to", "until", required=True, help="End date") -@click.option("--balance", default=10000.0, help="Initial balance") -def run_backtest(strategy: str, symbol: str, since: str, until: str, balance: float): - """Run a backtest""" - click.echo(f"Running backtest: {strategy} on {symbol} ({since} ~ {until})") - - async def _run(): - from pathlib import Path - from shared.config import Settings - from shared.db import Database - from backtester.main import run_backtest as bt_run - - settings = Settings() - db = Database(settings.database_url) - await db.connect() - - strategies_dir = Path(__file__).parent.parent.parent.parent.parent / "services" / "strategy-engine" / "strategies" - report = await bt_run( - strategy_name=strategy, - symbol=symbol, - timeframe="1m", - initial_balance=Decimal(str(balance)), - db=db, - strategies_dir=strategies_dir, - ) - click.echo(report) - await db.close() - - asyncio.run(_run()) - - -@backtest.command() -@click.option("--id", "report_id", default="latest", help="Report ID") -def report(report_id: str): - """Show backtest report""" - click.echo(f"Showing report: {report_id}") -``` - -Create `cli/src/trading_cli/commands/portfolio.py`: - -```python -import click - - -@click.group() -def portfolio(): - """Portfolio commands""" - pass - - -@portfolio.command() -def show(): - """Show current portfolio""" - click.echo("Current Portfolio:") - click.echo(" (Connect to portfolio-manager service)") - - -@portfolio.command() -@click.option("--days", default=30, help="Number of days") -def history(days: int): - """Show PnL history""" - click.echo(f"PnL history (last {days} days):") -``` - -Create `cli/src/trading_cli/commands/strategy.py`: - -```python -from pathlib import Path - -import click - - -@click.group() -def strategy(): - """Strategy management commands""" - pass - - -@strategy.command("list") -def list_strategies(): - """List available strategies""" - from strategy_engine.plugin_loader import load_strategies - - strategies_dir = Path(__file__).parent.parent.parent.parent.parent / "services" / "strategy-engine" / "strategies" - strategies = load_strategies(strategies_dir) - click.echo("Available strategies:") - for s in strategies: - click.echo(f" - {s.name}") - - -@strategy.command() -@click.option("--name", required=True, help="Strategy name") -def info(name: str): - """Show strategy details""" - click.echo(f"Strategy: {name}") -``` - -Create `cli/src/trading_cli/commands/service.py`: - -```python -import subprocess - -import click - - -@click.group() -def service(): - """Service management commands""" - pass - - -@service.command() -def up(): - """Start all services""" - click.echo("Starting all services...") - subprocess.run(["docker", "compose", "up", "-d"], check=True) - - -@service.command() -def down(): - """Stop all services""" - click.echo("Stopping all services...") - subprocess.run(["docker", "compose", "down"], check=True) - - -@service.command() -@click.option("--name", required=True, help="Service name") -def logs(name: str): - """Show service logs""" - subprocess.run(["docker", "compose", "logs", "-f", name]) -``` - -- [ ] **Step 6: Run tests to verify they pass** - -```bash -pytest cli/tests/test_cli_data.py -v -``` - -Expected: All PASS - -- [ ] **Step 7: Commit** - -```bash -git add cli/ -git commit -m "feat(cli): add Click-based CLI with data, trade, backtest, portfolio, strategy, and service commands" -``` - ---- - -## Task 11: Integration Verification - -- [ ] **Step 1: Run all tests** - -```bash -cd /home/si/Private/repos/trading -pytest -v -``` - -Expected: All tests pass - -- [ ] **Step 2: Lint check** - -```bash -ruff check . -``` - -Fix any issues found. - -- [ ] **Step 3: Verify Docker builds** - -```bash -docker compose build -``` - -Expected: All services build successfully - -- [ ] **Step 4: Start infrastructure and verify** - -```bash -make infra -# Wait for healthy status -docker compose ps -``` - -Expected: redis and postgres running and healthy - -- [ ] **Step 5: Final commit** - -```bash -git add . -git commit -m "chore: integration verification — all tests pass, docker builds succeed" -``` diff --git a/docs/superpowers/plans/2026-04-01-operations-and-strategy-expansion.md b/docs/superpowers/plans/2026-04-01-operations-and-strategy-expansion.md deleted file mode 100644 index 761a49a..0000000 --- a/docs/superpowers/plans/2026-04-01-operations-and-strategy-expansion.md +++ /dev/null @@ -1,4187 +0,0 @@ -# Operations Infrastructure & Strategy Expansion — Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add production-grade operations infrastructure (SQLAlchemy ORM, Alembic migrations, structlog, Telegram alerts, resilience, Prometheus) and expand the strategy library (MACD, Bollinger, EMA Crossover, VWAP, Volume Profile) with enhanced backtesting metrics. - -**Architecture:** Operations-first approach. Migrate the DB layer to SQLAlchemy 2.0 async, add structured logging and Telegram notifications as shared infrastructure, then build resilience and metrics on top. Strategy expansion builds on the stabilized platform with new BaseStrategy.warmup_period contract and YAML config loading. - -**Tech Stack:** SQLAlchemy 2.0 async (asyncpg driver), Alembic, structlog, aiohttp (Telegram), prometheus-client, pyyaml, rich, pandas, numpy - ---- - -## File Structure - -### New Files - -| File | Responsibility | -|------|---------------| -| `shared/src/shared/sa_models.py` | SQLAlchemy ORM table definitions | -| `shared/src/shared/logging.py` | structlog setup and Telegram error processor | -| `shared/src/shared/notifier.py` | TelegramNotifier class | -| `shared/src/shared/resilience.py` | retry_with_backoff decorator + CircuitBreaker | -| `shared/src/shared/healthcheck.py` | aiohttp-based /health + /metrics server | -| `shared/src/shared/metrics.py` | Prometheus metric definitions | -| `shared/alembic.ini` | Alembic config | -| `shared/alembic/env.py` | Alembic async environment | -| `shared/alembic/script.py.mako` | Alembic migration template | -| `shared/alembic/versions/` | Migration files (auto-generated) | -| `shared/tests/test_sa_models.py` | SA model tests | -| `shared/tests/test_logging.py` | structlog setup tests | -| `shared/tests/test_notifier.py` | TelegramNotifier tests | -| `shared/tests/test_resilience.py` | retry + circuit breaker tests | -| `shared/tests/test_healthcheck.py` | Healthcheck server tests | -| `shared/tests/test_metrics.py` | Prometheus metrics tests | -| `services/strategy-engine/strategies/config/rsi_strategy.yaml` | RSI params | -| `services/strategy-engine/strategies/config/grid_strategy.yaml` | Grid params | -| `services/strategy-engine/strategies/config/macd_strategy.yaml` | MACD params | -| `services/strategy-engine/strategies/config/bollinger_strategy.yaml` | Bollinger params | -| `services/strategy-engine/strategies/config/ema_crossover_strategy.yaml` | EMA params | -| `services/strategy-engine/strategies/config/vwap_strategy.yaml` | VWAP params | -| `services/strategy-engine/strategies/config/volume_profile_strategy.yaml` | Volume Profile params | -| `services/strategy-engine/strategies/macd_strategy.py` | MACD strategy | -| `services/strategy-engine/strategies/bollinger_strategy.py` | Bollinger Bands strategy | -| `services/strategy-engine/strategies/ema_crossover_strategy.py` | EMA Crossover strategy | -| `services/strategy-engine/strategies/vwap_strategy.py` | VWAP strategy | -| `services/strategy-engine/strategies/volume_profile_strategy.py` | Volume Profile strategy | -| `services/strategy-engine/tests/test_macd_strategy.py` | MACD tests | -| `services/strategy-engine/tests/test_bollinger_strategy.py` | Bollinger tests | -| `services/strategy-engine/tests/test_ema_crossover_strategy.py` | EMA Crossover tests | -| `services/strategy-engine/tests/test_vwap_strategy.py` | VWAP tests | -| `services/strategy-engine/tests/test_volume_profile_strategy.py` | Volume Profile tests | -| `services/backtester/src/backtester/metrics.py` | DetailedMetrics + TradeRecord | -| `services/backtester/tests/test_metrics.py` | Detailed metrics tests | -| `monitoring/prometheus.yml` | Prometheus scrape config | - -### Modified Files - -| File | Changes | -|------|---------| -| `shared/pyproject.toml` | Add sqlalchemy, alembic, structlog, prometheus-client, pyyaml | -| `shared/src/shared/config.py` | Add Telegram, health, circuit breaker settings | -| `shared/src/shared/db.py` | Rewrite to SQLAlchemy async session | -| `shared/src/shared/__init__.py` | Export new modules | -| `shared/tests/test_db.py` | Update for SQLAlchemy API | -| `services/strategy-engine/strategies/base.py` | Add warmup_period abstract property | -| `services/strategy-engine/strategies/rsi_strategy.py` | Add warmup_period, update for YAML config | -| `services/strategy-engine/strategies/grid_strategy.py` | Add warmup_period, update for YAML config | -| `services/strategy-engine/src/strategy_engine/plugin_loader.py` | Add YAML config loading | -| `services/strategy-engine/src/strategy_engine/main.py` | Use YAML config loader | -| `services/data-collector/src/data_collector/storage.py` | Use AsyncSession | -| `services/data-collector/src/data_collector/main.py` | Use structlog, healthcheck, resilience | -| `services/order-executor/src/order_executor/executor.py` | Use AsyncSession, notifier | -| `services/order-executor/src/order_executor/main.py` | Use structlog, healthcheck, resilience | -| `services/portfolio-manager/src/portfolio_manager/main.py` | Use structlog, healthcheck, daily summary | -| `services/backtester/src/backtester/engine.py` | Compute DetailedMetrics | -| `services/backtester/src/backtester/simulator.py` | Track entry/exit for TradeRecord | -| `services/backtester/src/backtester/reporter.py` | Rich table output, CSV/JSON export | -| `docker-compose.yml` | Add healthcheck endpoints, monitoring profile | -| `Makefile` | Add migrate, migrate-down, migrate-new targets | -| `.env.example` | Add Telegram, health, log format vars | - ---- - -## Task 1: SQLAlchemy ORM Models + Alembic Setup - -**Files:** -- Create: `shared/src/shared/sa_models.py` -- Create: `shared/alembic.ini` -- Create: `shared/alembic/env.py` -- Create: `shared/alembic/script.py.mako` -- Modify: `shared/pyproject.toml` -- Test: `shared/tests/test_sa_models.py` - -- [ ] **Step 1: Add dependencies to shared/pyproject.toml** - -```toml -[project] -name = "trading-shared" -version = "0.1.0" -description = "Shared models, events, and utilities for trading platform" -requires-python = ">=3.12" -dependencies = [ - "pydantic>=2.0", - "pydantic-settings>=2.0", - "redis>=5.0", - "sqlalchemy[asyncio]>=2.0", - "asyncpg>=0.29", - "alembic>=1.13", - "structlog>=24.0", - "prometheus-client>=0.20", - "pyyaml>=6.0", - "aiohttp>=3.9", - "rich>=13.0", -] -``` - -- [ ] **Step 2: Write the failing test for SA models** - -Create `shared/tests/test_sa_models.py`: - -```python -"""Tests for SQLAlchemy ORM models.""" -from datetime import datetime, timezone -from decimal import Decimal - -from shared.sa_models import ( - Base, - CandleRow, - SignalRow, - OrderRow, - TradeRow, - PositionRow, - PortfolioSnapshotRow, -) - - -def test_candle_row_table_name(): - assert CandleRow.__tablename__ == "candles" - - -def test_candle_row_columns(): - cols = {c.name for c in CandleRow.__table__.columns} - assert cols == {"symbol", "timeframe", "open_time", "open", "high", "low", "close", "volume"} - - -def test_signal_row_table_name(): - assert SignalRow.__tablename__ == "signals" - - -def test_signal_row_columns(): - cols = {c.name for c in SignalRow.__table__.columns} - assert cols == {"id", "strategy", "symbol", "side", "price", "quantity", "reason", "created_at"} - - -def test_order_row_table_name(): - assert OrderRow.__tablename__ == "orders" - - -def test_order_row_columns(): - cols = {c.name for c in OrderRow.__table__.columns} - assert cols == { - "id", "signal_id", "symbol", "side", "type", "price", - "quantity", "status", "created_at", "filled_at", - } - - -def test_trade_row_table_name(): - assert TradeRow.__tablename__ == "trades" - - -def test_position_row_table_name(): - assert PositionRow.__tablename__ == "positions" - - -def test_portfolio_snapshot_row_table_name(): - assert PortfolioSnapshotRow.__tablename__ == "portfolio_snapshots" - - -def test_base_metadata_has_all_tables(): - table_names = set(Base.metadata.tables.keys()) - assert table_names == { - "candles", "signals", "orders", "trades", "positions", "portfolio_snapshots", - } -``` - -- [ ] **Step 3: Run test to verify it fails** - -Run: `pytest shared/tests/test_sa_models.py -v` -Expected: FAIL with `ModuleNotFoundError: No module named 'shared.sa_models'` - -- [ ] **Step 4: Implement SA models** - -Create `shared/src/shared/sa_models.py`: - -```python -"""SQLAlchemy ORM models for the trading platform.""" -from datetime import datetime - -from sqlalchemy import ( - DateTime, - ForeignKey, - Integer, - Numeric, - String, - Text, -) -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column - - -class Base(DeclarativeBase): - pass - - -class CandleRow(Base): - __tablename__ = "candles" - - symbol: Mapped[str] = mapped_column(String, primary_key=True) - timeframe: Mapped[str] = mapped_column(String, primary_key=True) - open_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), primary_key=True) - open: Mapped[float] = mapped_column(Numeric, nullable=False) - high: Mapped[float] = mapped_column(Numeric, nullable=False) - low: Mapped[float] = mapped_column(Numeric, nullable=False) - close: Mapped[float] = mapped_column(Numeric, nullable=False) - volume: Mapped[float] = mapped_column(Numeric, nullable=False) - - -class SignalRow(Base): - __tablename__ = "signals" - - id: Mapped[str] = mapped_column(String, primary_key=True) - strategy: Mapped[str] = mapped_column(String, nullable=False) - symbol: Mapped[str] = mapped_column(String, nullable=False) - side: Mapped[str] = mapped_column(String, nullable=False) - price: Mapped[float] = mapped_column(Numeric, nullable=False) - quantity: Mapped[float] = mapped_column(Numeric, nullable=False) - reason: Mapped[str | None] = mapped_column(Text) - created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) - - -class OrderRow(Base): - __tablename__ = "orders" - - id: Mapped[str] = mapped_column(String, primary_key=True) - signal_id: Mapped[str | None] = mapped_column(String, ForeignKey("signals.id")) - symbol: Mapped[str] = mapped_column(String, nullable=False) - side: Mapped[str] = mapped_column(String, nullable=False) - type: Mapped[str] = mapped_column(String, nullable=False) - price: Mapped[float] = mapped_column(Numeric, nullable=False) - quantity: Mapped[float] = mapped_column(Numeric, nullable=False) - status: Mapped[str] = mapped_column(String, nullable=False, default="PENDING") - created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) - filled_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - - -class TradeRow(Base): - __tablename__ = "trades" - - id: Mapped[str] = mapped_column(String, primary_key=True) - order_id: Mapped[str | None] = mapped_column(String, ForeignKey("orders.id")) - symbol: Mapped[str] = mapped_column(String, nullable=False) - side: Mapped[str] = mapped_column(String, nullable=False) - price: Mapped[float] = mapped_column(Numeric, nullable=False) - quantity: Mapped[float] = mapped_column(Numeric, nullable=False) - fee: Mapped[float] = mapped_column(Numeric, nullable=False, default=0) - traded_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) - - -class PositionRow(Base): - __tablename__ = "positions" - - symbol: Mapped[str] = mapped_column(String, primary_key=True) - quantity: Mapped[float] = mapped_column(Numeric, nullable=False) - avg_entry_price: Mapped[float] = mapped_column(Numeric, nullable=False) - current_price: Mapped[float] = mapped_column(Numeric, nullable=False) - updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) - - -class PortfolioSnapshotRow(Base): - __tablename__ = "portfolio_snapshots" - - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) - total_value: Mapped[float] = mapped_column(Numeric, nullable=False) - realized_pnl: Mapped[float] = mapped_column(Numeric, nullable=False) - unrealized_pnl: Mapped[float] = mapped_column(Numeric, nullable=False) - snapshot_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) -``` - -- [ ] **Step 5: Run test to verify it passes** - -Run: `pytest shared/tests/test_sa_models.py -v` -Expected: All 8 tests PASS - -- [ ] **Step 6: Set up Alembic** - -Create `shared/alembic.ini`: - -```ini -[alembic] -script_location = alembic -sqlalchemy.url = postgresql+asyncpg://trading:trading@localhost:5432/trading - -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S -``` - -Create `shared/alembic/script.py.mako`: - -```mako -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -${imports if imports else ""} - -# revision identifiers, used by Alembic. -revision: str = ${repr(up_revision)} -down_revision: Union[str, None] = ${repr(down_revision)} -branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} -depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} - - -def upgrade() -> None: - ${upgrades if upgrades else "pass"} - - -def downgrade() -> None: - ${downgrades if downgrades else "pass"} -``` - -Create `shared/alembic/env.py`: - -```python -"""Alembic environment configuration for async SQLAlchemy.""" -import asyncio -import os -from logging.config import fileConfig - -from alembic import context -from sqlalchemy import pool -from sqlalchemy.ext.asyncio import async_engine_from_config - -from shared.sa_models import Base - -config = context.config - -if config.config_file_name is not None: - fileConfig(config.config_file_name) - -target_metadata = Base.metadata - -# Override URL from environment if available -database_url = os.environ.get("DATABASE_URL") -if database_url: - # Ensure async driver prefix - if database_url.startswith("postgresql://"): - database_url = database_url.replace("postgresql://", "postgresql+asyncpg://", 1) - config.set_main_option("sqlalchemy.url", database_url) - - -def run_migrations_offline() -> None: - url = config.get_main_option("sqlalchemy.url") - context.configure(url=url, target_metadata=target_metadata, literal_binds=True) - with context.begin_transaction(): - context.run_migrations() - - -def do_run_migrations(connection): - context.configure(connection=connection, target_metadata=target_metadata) - with context.begin_transaction(): - context.run_migrations() - - -async def run_async_migrations() -> None: - connectable = async_engine_from_config( - config.get_section(config.config_ini_section, {}), - prefix="sqlalchemy.", - poolclass=pool.NullPool, - ) - async with connectable.connect() as connection: - await connection.run_sync(do_run_migrations) - await connectable.dispose() - - -def run_migrations_online() -> None: - asyncio.run(run_async_migrations()) - - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() -``` - -Create empty `shared/alembic/versions/` directory (with `.gitkeep`). - -- [ ] **Step 7: Add Makefile targets** - -Append to `Makefile`: - -```makefile -migrate: - cd shared && alembic upgrade head - -migrate-down: - cd shared && alembic downgrade -1 - -migrate-new: - cd shared && alembic revision --autogenerate -m "$(MSG)" -``` - -- [ ] **Step 8: Commit** - -```bash -git add shared/src/shared/sa_models.py shared/alembic.ini shared/alembic/ \ - shared/tests/test_sa_models.py shared/pyproject.toml Makefile -git commit -m "feat(shared): add SQLAlchemy ORM models and Alembic setup" -``` - ---- - -## Task 2: Rewrite Database Layer to SQLAlchemy Async - -**Files:** -- Modify: `shared/src/shared/db.py` -- Modify: `shared/tests/test_db.py` - -- [ ] **Step 1: Write the failing test for the new DB layer** - -Replace `shared/tests/test_db.py`: - -```python -"""Tests for the SQLAlchemy async database layer.""" -from datetime import datetime, timezone -from decimal import Decimal -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -from shared.db import Database -from shared.models import Candle, Signal, OrderSide, Order, OrderType, OrderStatus - - -@pytest.fixture -def db(): - return Database("postgresql+asyncpg://trading:trading@localhost:5432/trading") - - -def test_database_stores_url(db): - assert db._database_url == "postgresql+asyncpg://trading:trading@localhost:5432/trading" - - -@pytest.mark.asyncio -async def test_get_session_returns_async_session(db): - """Verify get_session is an async context manager (structural test).""" - # We can't connect without a real DB, but we verify the method exists - assert hasattr(db, "get_session") - assert callable(db.get_session) - - -@pytest.mark.asyncio -async def test_insert_candle_creates_candle_row(): - """Verify insert_candle adds a CandleRow to the session.""" - db = Database("postgresql+asyncpg://test:test@localhost/test") - - candle = Candle( - symbol="BTCUSDT", - timeframe="1m", - open_time=datetime(2025, 1, 1, tzinfo=timezone.utc), - open=Decimal("50000"), - high=Decimal("51000"), - low=Decimal("49000"), - close=Decimal("50500"), - volume=Decimal("100"), - ) - - mock_session = AsyncMock() - mock_session.__aenter__ = AsyncMock(return_value=mock_session) - mock_session.__aexit__ = AsyncMock(return_value=False) - - with patch.object(db, "get_session", return_value=mock_session): - await db.insert_candle(candle) - - mock_session.merge.assert_called_once() - mock_session.commit.assert_called_once() - - -@pytest.mark.asyncio -async def test_insert_signal_creates_signal_row(): - db = Database("postgresql+asyncpg://test:test@localhost/test") - - signal = Signal( - strategy="rsi", - symbol="BTCUSDT", - side=OrderSide.BUY, - price=Decimal("50000"), - quantity=Decimal("0.01"), - reason="test signal", - ) - - mock_session = AsyncMock() - mock_session.__aenter__ = AsyncMock(return_value=mock_session) - mock_session.__aexit__ = AsyncMock(return_value=False) - - with patch.object(db, "get_session", return_value=mock_session): - await db.insert_signal(signal) - - mock_session.add.assert_called_once() - mock_session.commit.assert_called_once() - - -@pytest.mark.asyncio -async def test_insert_order_creates_order_row(): - db = Database("postgresql+asyncpg://test:test@localhost/test") - - order = Order( - signal_id="sig-1", - symbol="BTCUSDT", - side=OrderSide.BUY, - type=OrderType.MARKET, - price=Decimal("50000"), - quantity=Decimal("0.01"), - ) - - mock_session = AsyncMock() - mock_session.__aenter__ = AsyncMock(return_value=mock_session) - mock_session.__aexit__ = AsyncMock(return_value=False) - - with patch.object(db, "get_session", return_value=mock_session): - await db.insert_order(order) - - mock_session.add.assert_called_once() - mock_session.commit.assert_called_once() -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `pytest shared/tests/test_db.py -v` -Expected: FAIL — old Database class doesn't have `get_session` - -- [ ] **Step 3: Rewrite db.py with SQLAlchemy async** - -Replace `shared/src/shared/db.py`: - -```python -"""Database layer using SQLAlchemy async for the trading platform.""" -from datetime import datetime, timezone -from decimal import Decimal -from typing import Optional - -from sqlalchemy import select, update -from sqlalchemy.ext.asyncio import ( - AsyncSession, - async_sessionmaker, - create_async_engine, -) - -from shared.models import Candle, Order, OrderStatus, Signal -from shared.sa_models import ( - Base, - CandleRow, - OrderRow, - SignalRow, -) - - -class Database: - """Async database access layer backed by SQLAlchemy.""" - - def __init__(self, database_url: str) -> None: - self._database_url = database_url - # Ensure async driver prefix - if self._database_url.startswith("postgresql://"): - self._database_url = self._database_url.replace( - "postgresql://", "postgresql+asyncpg://", 1 - ) - self._engine = create_async_engine(self._database_url) - self._session_factory = async_sessionmaker(self._engine, expire_on_commit=False) - - def get_session(self) -> AsyncSession: - """Return a new AsyncSession.""" - return self._session_factory() - - async def connect(self) -> None: - """Create all tables (for dev/test — prefer Alembic in production).""" - async with self._engine.begin() as conn: - await conn.run_sync(Base.metadata.create_all) - - async def close(self) -> None: - """Dispose of the engine.""" - await self._engine.dispose() - - # Alias for backward compatibility - async def init_tables(self) -> None: - await self.connect() - - async def insert_candle(self, candle: Candle) -> None: - """Upsert a candle row using merge.""" - async with self.get_session() as session: - row = CandleRow( - symbol=candle.symbol, - timeframe=candle.timeframe, - open_time=candle.open_time, - open=candle.open, - high=candle.high, - low=candle.low, - close=candle.close, - volume=candle.volume, - ) - await session.merge(row) - await session.commit() - - async def insert_signal(self, signal: Signal) -> None: - """Insert a signal row.""" - async with self.get_session() as session: - row = SignalRow( - id=signal.id, - strategy=signal.strategy, - symbol=signal.symbol, - side=signal.side.value, - price=signal.price, - quantity=signal.quantity, - reason=signal.reason, - created_at=signal.created_at, - ) - session.add(row) - await session.commit() - - async def insert_order(self, order: Order) -> None: - """Insert an order row.""" - async with self.get_session() as session: - row = OrderRow( - id=order.id, - signal_id=order.signal_id, - symbol=order.symbol, - side=order.side.value, - type=order.type.value, - price=order.price, - quantity=order.quantity, - status=order.status.value, - created_at=order.created_at, - filled_at=order.filled_at, - ) - session.add(row) - await session.commit() - - async def update_order_status( - self, - order_id: str, - status: OrderStatus, - filled_at: Optional[datetime] = None, - ) -> None: - """Update the status of an order.""" - async with self.get_session() as session: - stmt = ( - update(OrderRow) - .where(OrderRow.id == order_id) - .values(status=status.value, filled_at=filled_at) - ) - await session.execute(stmt) - await session.commit() - - async def get_candles( - self, symbol: str, timeframe: str, limit: int = 500 - ) -> list[dict]: - """Retrieve candles ordered by open_time descending.""" - async with self.get_session() as session: - stmt = ( - select(CandleRow) - .where(CandleRow.symbol == symbol, CandleRow.timeframe == timeframe) - .order_by(CandleRow.open_time.desc()) - .limit(limit) - ) - result = await session.execute(stmt) - rows = result.scalars().all() - return [ - { - "symbol": r.symbol, - "timeframe": r.timeframe, - "open_time": r.open_time, - "open": r.open, - "high": r.high, - "low": r.low, - "close": r.close, - "volume": r.volume, - } - for r in rows - ] -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `pytest shared/tests/test_db.py -v` -Expected: All tests PASS - -- [ ] **Step 5: Commit** - -```bash -git add shared/src/shared/db.py shared/tests/test_db.py -git commit -m "refactor(shared): rewrite db layer to SQLAlchemy 2.0 async" -``` - ---- - -## Task 3: Structured Logging with structlog - -**Files:** -- Create: `shared/src/shared/logging.py` -- Test: `shared/tests/test_logging.py` - -- [ ] **Step 1: Write the failing test** - -Create `shared/tests/test_logging.py`: - -```python -"""Tests for structured logging setup.""" -import logging - -import structlog - -from shared.logging import setup_logging - - -def test_setup_logging_returns_logger(): - logger = setup_logging("test-service", "INFO") - assert logger is not None - - -def test_setup_logging_binds_service_name(): - logger = setup_logging("data-collector", "INFO") - # structlog loggers have _context with bound values - assert logger._context.get("service") == "data-collector" - - -def test_setup_logging_sets_log_level(): - setup_logging("test-service", "DEBUG") - root = logging.getLogger() - assert root.level == logging.DEBUG - - -def test_setup_logging_json_format(capsys): - logger = setup_logging("test-service", "INFO", log_format="json") - logger.info("test_event", key="value") - captured = capsys.readouterr() - assert "test_event" in captured.out or "test_event" in captured.err - - -def test_setup_logging_console_format(capsys): - logger = setup_logging("test-service", "INFO", log_format="console") - logger.info("test_event", key="value") - captured = capsys.readouterr() - assert "test_event" in captured.out or "test_event" in captured.err -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `pytest shared/tests/test_logging.py -v` -Expected: FAIL with `ModuleNotFoundError: No module named 'shared.logging'` - -- [ ] **Step 3: Implement structured logging** - -Create `shared/src/shared/logging.py`: - -```python -"""Structured logging setup using structlog.""" -import logging -import sys - -import structlog - - -def setup_logging( - service_name: str, - log_level: str = "INFO", - log_format: str = "json", -) -> structlog.stdlib.BoundLogger: - """Configure structlog for the given service. - - Args: - service_name: Bound to every log entry as 'service'. - log_level: Python log level string (DEBUG, INFO, WARNING, ERROR). - log_format: 'json' for production, 'console' for development. - - Returns: - A bound structlog logger with service context. - """ - # Set stdlib root logger level - logging.basicConfig( - format="%(message)s", - stream=sys.stdout, - level=getattr(logging, log_level.upper(), logging.INFO), - force=True, - ) - - shared_processors: list[structlog.types.Processor] = [ - structlog.contextvars.merge_contextvars, - structlog.stdlib.add_log_level, - structlog.stdlib.add_logger_name, - structlog.processors.TimeStamper(fmt="iso"), - structlog.processors.StackInfoRenderer(), - structlog.processors.UnicodeDecoder(), - ] - - if log_format == "console": - renderer = structlog.dev.ConsoleRenderer() - else: - renderer = structlog.processors.JSONRenderer() - - structlog.configure( - processors=[ - *shared_processors, - structlog.stdlib.ProcessorFormatter.wrap_for_formatter, - ], - logger_factory=structlog.stdlib.LoggerFactory(), - wrapper_class=structlog.stdlib.BoundLogger, - cache_logger_on_first_use=True, - ) - - # Also configure the formatter for stdlib loggers - formatter = structlog.stdlib.ProcessorFormatter( - processors=[ - structlog.stdlib.ProcessorFormatter.remove_processors_meta, - renderer, - ], - ) - - root = logging.getLogger() - for handler in root.handlers: - handler.setFormatter(formatter) - - return structlog.get_logger().bind(service=service_name) -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `pytest shared/tests/test_logging.py -v` -Expected: All 5 tests PASS - -- [ ] **Step 5: Commit** - -```bash -git add shared/src/shared/logging.py shared/tests/test_logging.py -git commit -m "feat(shared): add structlog-based structured logging" -``` - ---- - -## Task 4: Telegram Notification Service - -**Files:** -- Create: `shared/src/shared/notifier.py` -- Modify: `shared/src/shared/config.py` -- Modify: `.env.example` -- Test: `shared/tests/test_notifier.py` - -- [ ] **Step 1: Update config.py with Telegram settings** - -Add to `shared/src/shared/config.py` after `dry_run`: - -```python - # Telegram - telegram_bot_token: str = "" - telegram_chat_id: str = "" - telegram_enabled: bool = False - # Logging - log_format: str = "json" - # Health - health_port: int = 8080 - # Circuit Breaker - circuit_breaker_threshold: int = 5 - circuit_breaker_timeout: int = 60 -``` - -- [ ] **Step 2: Update .env.example** - -Replace `.env.example`: - -```env -# Exchange -BINANCE_API_KEY= -BINANCE_API_SECRET= - -# Infrastructure -REDIS_URL=redis://localhost:6379 -DATABASE_URL=postgresql+asyncpg://trading:trading@localhost:5432/trading - -# Logging -LOG_LEVEL=INFO -LOG_FORMAT=json - -# Telegram -TELEGRAM_BOT_TOKEN= -TELEGRAM_CHAT_ID= -TELEGRAM_ENABLED=false - -# Risk Management -RISK_MAX_POSITION_SIZE=0.1 -RISK_STOP_LOSS_PCT=5 -RISK_DAILY_LOSS_LIMIT_PCT=10 -DRY_RUN=true - -# Health & Metrics -HEALTH_PORT=8080 -CIRCUIT_BREAKER_THRESHOLD=5 -CIRCUIT_BREAKER_TIMEOUT=60 -``` - -- [ ] **Step 3: Write the failing test for TelegramNotifier** - -Create `shared/tests/test_notifier.py`: - -```python -"""Tests for Telegram notification service.""" -from decimal import Decimal -from unittest.mock import AsyncMock, patch, MagicMock - -import pytest - -from shared.models import Signal, OrderSide, Order, OrderType, OrderStatus -from shared.notifier import TelegramNotifier - - -@pytest.fixture -def notifier(): - return TelegramNotifier(bot_token="test-token", chat_id="12345") - - -def test_notifier_disabled_when_no_token(): - n = TelegramNotifier(bot_token="", chat_id="12345") - assert n.enabled is False - - -def test_notifier_enabled_with_token(): - n = TelegramNotifier(bot_token="abc", chat_id="12345") - assert n.enabled is True - - -@pytest.mark.asyncio -async def test_send_does_nothing_when_disabled(): - n = TelegramNotifier(bot_token="", chat_id="12345") - # Should not raise - await n.send("test message") - - -@pytest.mark.asyncio -async def test_send_posts_to_telegram_api(notifier): - mock_response = AsyncMock() - mock_response.status = 200 - mock_response.__aenter__ = AsyncMock(return_value=mock_response) - mock_response.__aexit__ = AsyncMock(return_value=False) - - mock_session = AsyncMock() - mock_session.post = MagicMock(return_value=mock_response) - - notifier._session = mock_session - - await notifier.send("Hello") - - mock_session.post.assert_called_once() - call_kwargs = mock_session.post.call_args - assert "12345" in str(call_kwargs) or "Hello" in str(call_kwargs) - - -@pytest.mark.asyncio -async def test_send_signal_formats_message(notifier): - signal = Signal( - strategy="rsi", - symbol="BTCUSDT", - side=OrderSide.BUY, - price=Decimal("50000"), - quantity=Decimal("0.01"), - reason="RSI oversold", - ) - - with patch.object(notifier, "send", new_callable=AsyncMock) as mock_send: - await notifier.send_signal(signal) - mock_send.assert_called_once() - msg = mock_send.call_args[0][0] - assert "BUY" in msg - assert "BTCUSDT" in msg - assert "rsi" in msg - - -@pytest.mark.asyncio -async def test_send_order_formats_message(notifier): - order = Order( - signal_id="sig-1", - symbol="BTCUSDT", - side=OrderSide.BUY, - type=OrderType.MARKET, - price=Decimal("50000"), - quantity=Decimal("0.01"), - status=OrderStatus.FILLED, - ) - - with patch.object(notifier, "send", new_callable=AsyncMock) as mock_send: - await notifier.send_order(order) - mock_send.assert_called_once() - msg = mock_send.call_args[0][0] - assert "FILLED" in msg - assert "BTCUSDT" in msg - - -@pytest.mark.asyncio -async def test_send_error_formats_message(notifier): - with patch.object(notifier, "send", new_callable=AsyncMock) as mock_send: - await notifier.send_error("Connection lost", "data-collector") - mock_send.assert_called_once() - msg = mock_send.call_args[0][0] - assert "Connection lost" in msg - assert "data-collector" in msg -``` - -- [ ] **Step 4: Run test to verify it fails** - -Run: `pytest shared/tests/test_notifier.py -v` -Expected: FAIL with `ModuleNotFoundError: No module named 'shared.notifier'` - -- [ ] **Step 5: Implement TelegramNotifier** - -Create `shared/src/shared/notifier.py`: - -```python -"""Telegram notification service for trading alerts.""" -import asyncio -import logging -from decimal import Decimal - -import aiohttp - -from shared.models import Order, Signal - -logger = logging.getLogger(__name__) - -TELEGRAM_API = "https://api.telegram.org/bot{token}/sendMessage" - - -class TelegramNotifier: - """Sends notifications via Telegram Bot API.""" - - def __init__(self, bot_token: str, chat_id: str) -> None: - self._bot_token = bot_token - self._chat_id = chat_id - self._session: aiohttp.ClientSession | None = None - self._semaphore = asyncio.Semaphore(1) # Rate limit: 1 msg at a time - - @property - def enabled(self) -> bool: - return bool(self._bot_token) - - async def _ensure_session(self) -> aiohttp.ClientSession: - if self._session is None or self._session.closed: - self._session = aiohttp.ClientSession() - return self._session - - async def send(self, message: str, parse_mode: str = "HTML") -> None: - """Send a message to the configured Telegram chat.""" - if not self.enabled: - return - - async with self._semaphore: - url = TELEGRAM_API.format(token=self._bot_token) - payload = { - "chat_id": self._chat_id, - "text": message, - "parse_mode": parse_mode, - } - - retries = 3 - for attempt in range(retries): - try: - session = await self._ensure_session() - async with session.post(url, json=payload) as resp: - if resp.status == 200: - return - logger.warning( - "Telegram API returned %d on attempt %d", - resp.status, - attempt + 1, - ) - except Exception as exc: - logger.warning( - "Telegram send failed attempt %d: %s", attempt + 1, exc - ) - if attempt < retries - 1: - await asyncio.sleep(1) - - logger.error("Failed to send Telegram message after %d attempts", retries) - - async def send_signal(self, signal: Signal) -> None: - """Format and send a trading signal notification.""" - msg = ( - f"<b>Signal: {signal.side.value}</b>\n" - f"Strategy: {signal.strategy}\n" - f"Symbol: {signal.symbol}\n" - f"Price: {signal.price}\n" - f"Quantity: {signal.quantity}\n" - f"Reason: {signal.reason}" - ) - await self.send(msg) - - async def send_order(self, order: Order) -> None: - """Format and send an order execution notification.""" - msg = ( - f"<b>Order: {order.status.value}</b>\n" - f"Symbol: {order.symbol}\n" - f"Side: {order.side.value}\n" - f"Price: {order.price}\n" - f"Quantity: {order.quantity}" - ) - await self.send(msg) - - async def send_error(self, error: str, service: str) -> None: - """Send an error alert.""" - msg = f"<b>Error in {service}</b>\n{error}" - await self.send(msg) - - async def send_daily_summary( - self, positions: list, total_value: Decimal, daily_pnl: Decimal - ) -> None: - """Send daily portfolio summary.""" - lines = [f"<b>Daily Summary</b>"] - lines.append(f"Total Value: {total_value:.2f}") - lines.append(f"Daily PnL: {daily_pnl:.2f}") - lines.append(f"Open Positions: {len(positions)}") - for pos in positions: - lines.append(f" {pos.symbol}: {pos.quantity} @ {pos.avg_entry_price}") - await self.send("\n".join(lines)) - - async def close(self) -> None: - """Close the HTTP session.""" - if self._session and not self._session.closed: - await self._session.close() -``` - -- [ ] **Step 6: Run test to verify it passes** - -Run: `pytest shared/tests/test_notifier.py -v` -Expected: All 7 tests PASS - -- [ ] **Step 7: Commit** - -```bash -git add shared/src/shared/notifier.py shared/src/shared/config.py \ - shared/tests/test_notifier.py .env.example -git commit -m "feat(shared): add Telegram notification service" -``` - ---- - -## Task 5: Error Recovery — Retry + Circuit Breaker - -**Files:** -- Create: `shared/src/shared/resilience.py` -- Test: `shared/tests/test_resilience.py` - -- [ ] **Step 1: Write the failing test** - -Create `shared/tests/test_resilience.py`: - -```python -"""Tests for retry and circuit breaker.""" -import asyncio - -import pytest - -from shared.resilience import retry_with_backoff, CircuitBreaker, CircuitState - - -# --- retry_with_backoff tests --- - -@pytest.mark.asyncio -async def test_retry_succeeds_on_first_attempt(): - call_count = 0 - - @retry_with_backoff(max_retries=3, base_delay=0.01) - async def succeed(): - nonlocal call_count - call_count += 1 - return "ok" - - result = await succeed() - assert result == "ok" - assert call_count == 1 - - -@pytest.mark.asyncio -async def test_retry_succeeds_after_failures(): - call_count = 0 - - @retry_with_backoff(max_retries=3, base_delay=0.01) - async def fail_then_succeed(): - nonlocal call_count - call_count += 1 - if call_count < 3: - raise ConnectionError("down") - return "recovered" - - result = await fail_then_succeed() - assert result == "recovered" - assert call_count == 3 - - -@pytest.mark.asyncio -async def test_retry_raises_after_max_retries(): - @retry_with_backoff(max_retries=2, base_delay=0.01) - async def always_fail(): - raise ConnectionError("always down") - - with pytest.raises(ConnectionError, match="always down"): - await always_fail() - - -# --- CircuitBreaker tests --- - -@pytest.mark.asyncio -async def test_circuit_breaker_starts_closed(): - cb = CircuitBreaker(failure_threshold=3, recovery_timeout=0.1) - assert cb.state == CircuitState.CLOSED - - -@pytest.mark.asyncio -async def test_circuit_breaker_opens_after_threshold(): - cb = CircuitBreaker(failure_threshold=2, recovery_timeout=0.1) - cb.record_failure() - assert cb.state == CircuitState.CLOSED - cb.record_failure() - assert cb.state == CircuitState.OPEN - - -@pytest.mark.asyncio -async def test_circuit_breaker_rejects_when_open(): - cb = CircuitBreaker(failure_threshold=1, recovery_timeout=60) - cb.record_failure() - assert cb.state == CircuitState.OPEN - assert cb.allow_request() is False - - -@pytest.mark.asyncio -async def test_circuit_breaker_half_open_after_timeout(): - cb = CircuitBreaker(failure_threshold=1, recovery_timeout=0.05) - cb.record_failure() - assert cb.state == CircuitState.OPEN - await asyncio.sleep(0.06) - assert cb.allow_request() is True - assert cb.state == CircuitState.HALF_OPEN - - -@pytest.mark.asyncio -async def test_circuit_breaker_closes_on_success(): - cb = CircuitBreaker(failure_threshold=1, recovery_timeout=0.05) - cb.record_failure() - await asyncio.sleep(0.06) - cb.allow_request() # transitions to HALF_OPEN - cb.record_success() - assert cb.state == CircuitState.CLOSED -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `pytest shared/tests/test_resilience.py -v` -Expected: FAIL with `ModuleNotFoundError: No module named 'shared.resilience'` - -- [ ] **Step 3: Implement resilience module** - -Create `shared/src/shared/resilience.py`: - -```python -"""Retry with backoff and circuit breaker patterns.""" -import asyncio -import functools -import logging -import random -import time -from enum import Enum -from typing import Callable - -logger = logging.getLogger(__name__) - - -def retry_with_backoff( - max_retries: int = 3, - base_delay: float = 1.0, - max_delay: float = 60.0, -) -> Callable: - """Decorator for async functions that retries with exponential backoff + jitter.""" - - def decorator(func: Callable) -> Callable: - @functools.wraps(func) - async def wrapper(*args, **kwargs): - last_exc = None - for attempt in range(max_retries): - try: - return await func(*args, **kwargs) - except Exception as exc: - last_exc = exc - if attempt < max_retries - 1: - delay = min(base_delay * (2**attempt), max_delay) - jitter = delay * random.uniform(0, 0.5) - wait = delay + jitter - logger.warning( - "Retry %d/%d for %s after %.2fs: %s", - attempt + 1, - max_retries, - func.__name__, - wait, - exc, - ) - await asyncio.sleep(wait) - raise last_exc - - return wrapper - - return decorator - - -class CircuitState(Enum): - CLOSED = "closed" - OPEN = "open" - HALF_OPEN = "half_open" - - -class CircuitBreaker: - """Circuit breaker that opens after consecutive failures.""" - - def __init__( - self, - failure_threshold: int = 5, - recovery_timeout: float = 60.0, - ) -> None: - self._failure_threshold = failure_threshold - self._recovery_timeout = recovery_timeout - self._failure_count = 0 - self._state = CircuitState.CLOSED - self._opened_at: float | None = None - - @property - def state(self) -> CircuitState: - return self._state - - def allow_request(self) -> bool: - """Check if a request should be allowed.""" - if self._state == CircuitState.CLOSED: - return True - - if self._state == CircuitState.OPEN: - if self._opened_at and (time.monotonic() - self._opened_at) >= self._recovery_timeout: - self._state = CircuitState.HALF_OPEN - return True - return False - - # HALF_OPEN — allow one probe request - return True - - def record_success(self) -> None: - """Record a successful call.""" - self._failure_count = 0 - self._state = CircuitState.CLOSED - self._opened_at = None - - def record_failure(self) -> None: - """Record a failed call.""" - self._failure_count += 1 - if self._failure_count >= self._failure_threshold: - self._state = CircuitState.OPEN - self._opened_at = time.monotonic() - logger.error( - "Circuit breaker OPEN after %d failures", self._failure_count - ) -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `pytest shared/tests/test_resilience.py -v` -Expected: All 8 tests PASS - -- [ ] **Step 5: Commit** - -```bash -git add shared/src/shared/resilience.py shared/tests/test_resilience.py -git commit -m "feat(shared): add retry with backoff and circuit breaker" -``` - ---- - -## Task 6: Health Check + Prometheus Metrics - -**Files:** -- Create: `shared/src/shared/healthcheck.py` -- Create: `shared/src/shared/metrics.py` -- Create: `monitoring/prometheus.yml` -- Modify: `docker-compose.yml` -- Test: `shared/tests/test_healthcheck.py` -- Test: `shared/tests/test_metrics.py` - -- [ ] **Step 1: Write the failing test for metrics** - -Create `shared/tests/test_metrics.py`: - -```python -"""Tests for Prometheus metrics definitions.""" -from shared.metrics import ServiceMetrics - - -def test_service_metrics_creates_counters(): - m = ServiceMetrics("test-service") - assert m.errors_total is not None - assert m.events_processed is not None - - -def test_service_metrics_increment_errors(): - m = ServiceMetrics("test-service-2") - m.errors_total.labels(service="test-service-2", error_type="connection").inc() - # No assertion needed — prometheus_client raises on invalid labels - - -def test_service_metrics_observe_processing_time(): - m = ServiceMetrics("test-service-3") - m.processing_seconds.labels(service="test-service-3").observe(0.5) -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `pytest shared/tests/test_metrics.py -v` -Expected: FAIL with `ModuleNotFoundError: No module named 'shared.metrics'` - -- [ ] **Step 3: Implement metrics module** - -Create `shared/src/shared/metrics.py`: - -```python -"""Prometheus metric definitions for trading services.""" -from prometheus_client import Counter, Gauge, Histogram - - -class ServiceMetrics: - """Common Prometheus metrics for any trading service.""" - - def __init__(self, service_name: str) -> None: - prefix = service_name.replace("-", "_") - - self.errors_total = Counter( - f"{prefix}_errors_total", - "Total error count", - ["service", "error_type"], - ) - - self.events_processed = Counter( - f"{prefix}_events_processed_total", - "Total events processed", - ["service", "event_type"], - ) - - self.processing_seconds = Histogram( - f"{prefix}_processing_seconds", - "Event processing duration in seconds", - ["service"], - ) - - self.service_up = Gauge( - f"{prefix}_up", - "Service health status (1=up, 0=down)", - ["service"], - ) -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `pytest shared/tests/test_metrics.py -v` -Expected: All 3 tests PASS - -- [ ] **Step 5: Write the failing test for healthcheck** - -Create `shared/tests/test_healthcheck.py`: - -```python -"""Tests for health check server.""" -import pytest -from unittest.mock import AsyncMock - -from shared.healthcheck import HealthCheckServer - - -def test_healthcheck_server_init(): - server = HealthCheckServer(service_name="test", port=9090) - assert server._service_name == "test" - assert server._port == 9090 - - -def test_healthcheck_register_check(): - server = HealthCheckServer(service_name="test", port=9090) - check_fn = AsyncMock(return_value=True) - server.register_check("redis", check_fn) - assert "redis" in server._checks - - -@pytest.mark.asyncio -async def test_healthcheck_run_checks_all_pass(): - server = HealthCheckServer(service_name="test", port=9090) - server.register_check("redis", AsyncMock(return_value=True)) - server.register_check("postgres", AsyncMock(return_value=True)) - result = await server.run_checks() - assert result["status"] == "ok" - assert result["checks"]["redis"] is True - assert result["checks"]["postgres"] is True - - -@pytest.mark.asyncio -async def test_healthcheck_run_checks_one_fails(): - server = HealthCheckServer(service_name="test", port=9090) - server.register_check("redis", AsyncMock(return_value=True)) - server.register_check("postgres", AsyncMock(return_value=False)) - result = await server.run_checks() - assert result["status"] == "degraded" - assert result["checks"]["postgres"] is False -``` - -- [ ] **Step 6: Run test to verify it fails** - -Run: `pytest shared/tests/test_healthcheck.py -v` -Expected: FAIL with `ModuleNotFoundError: No module named 'shared.healthcheck'` - -- [ ] **Step 7: Implement healthcheck server** - -Create `shared/src/shared/healthcheck.py`: - -```python -"""Lightweight HTTP server for health checks and Prometheus metrics.""" -import time -from typing import Any, Callable, Coroutine - -from aiohttp import web -from prometheus_client import generate_latest, CONTENT_TYPE_LATEST - - -class HealthCheckServer: - """Serves /health and /metrics endpoints.""" - - def __init__(self, service_name: str, port: int = 8080) -> None: - self._service_name = service_name - self._port = port - self._checks: dict[str, Callable[[], Coroutine[Any, Any, bool]]] = {} - self._start_time = time.monotonic() - - def register_check( - self, name: str, check_fn: Callable[[], Coroutine[Any, Any, bool]] - ) -> None: - """Register a named async health check function.""" - self._checks[name] = check_fn - - async def run_checks(self) -> dict[str, Any]: - """Run all registered checks and return aggregated result.""" - results = {} - all_ok = True - for name, fn in self._checks.items(): - try: - results[name] = await fn() - except Exception: - results[name] = False - if not results[name]: - all_ok = False - - return { - "status": "ok" if all_ok else "degraded", - "service": self._service_name, - "uptime_seconds": round(time.monotonic() - self._start_time, 1), - "checks": results, - } - - async def _handle_health(self, request: web.Request) -> web.Response: - result = await self.run_checks() - status = 200 if result["status"] == "ok" else 503 - return web.json_response(result, status=status) - - async def _handle_metrics(self, request: web.Request) -> web.Response: - return web.Response( - body=generate_latest(), - content_type=CONTENT_TYPE_LATEST, - ) - - async def start(self) -> web.AppRunner: - """Start the HTTP server in the background.""" - app = web.Application() - app.router.add_get("/health", self._handle_health) - app.router.add_get("/metrics", self._handle_metrics) - - runner = web.AppRunner(app) - await runner.setup() - site = web.TCPSite(runner, "0.0.0.0", self._port) - await site.start() - return runner -``` - -- [ ] **Step 8: Run test to verify it passes** - -Run: `pytest shared/tests/test_healthcheck.py -v` -Expected: All 4 tests PASS - -- [ ] **Step 9: Create Prometheus config and update docker-compose** - -Create `monitoring/prometheus.yml`: - -```yaml -global: - scrape_interval: 15s - -scrape_configs: - - job_name: "trading-services" - static_configs: - - targets: - - "data-collector:8080" - - "strategy-engine:8081" - - "order-executor:8082" - - "portfolio-manager:8083" -``` - -Add to `docker-compose.yml` — append these services before the `volumes:` section: - -```yaml - prometheus: - image: prom/prometheus:latest - profiles: ["monitoring"] - ports: - - "9090:9090" - volumes: - - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml - depends_on: - - data-collector - - strategy-engine - - order-executor - - portfolio-manager - - grafana: - image: grafana/grafana:latest - profiles: ["monitoring"] - ports: - - "3000:3000" - depends_on: - - prometheus -``` - -- [ ] **Step 10: Commit** - -```bash -git add shared/src/shared/metrics.py shared/src/shared/healthcheck.py \ - shared/tests/test_metrics.py shared/tests/test_healthcheck.py \ - monitoring/prometheus.yml docker-compose.yml -git commit -m "feat(shared): add health checks and Prometheus metrics" -``` - ---- - -## Task 7: Integrate Operations Infrastructure into Services - -**Files:** -- Modify: `services/data-collector/src/data_collector/main.py` -- Modify: `services/data-collector/src/data_collector/storage.py` -- Modify: `services/order-executor/src/order_executor/main.py` -- Modify: `services/order-executor/src/order_executor/executor.py` -- Modify: `services/portfolio-manager/src/portfolio_manager/main.py` -- Modify: `services/strategy-engine/src/strategy_engine/main.py` - -This task integrates structlog, healthcheck, metrics, Telegram, and resilience into each service's entry point. Each service follows the same pattern. - -- [ ] **Step 1: Update data-collector/main.py** - -Replace `services/data-collector/src/data_collector/main.py`: - -```python -"""Data Collector Service entry point.""" -import asyncio - -from shared.broker import RedisBroker -from shared.config import Settings -from shared.db import Database -from shared.healthcheck import HealthCheckServer -from shared.logging import setup_logging -from shared.metrics import ServiceMetrics -from shared.notifier import TelegramNotifier -from shared.resilience import retry_with_backoff - -from data_collector.binance_ws import BinanceWebSocket -from data_collector.config import CollectorConfig -from data_collector.storage import CandleStorage - - -async def run() -> None: - config = CollectorConfig() - log = setup_logging("data-collector", config.log_level, config.log_format) - metrics = ServiceMetrics("data_collector") - - notifier = TelegramNotifier( - bot_token=config.telegram_bot_token, - chat_id=config.telegram_chat_id, - ) - - db = Database(config.database_url) - await db.connect() - - broker = RedisBroker(config.redis_url) - storage = CandleStorage(db=db, broker=broker) - - # Health checks - health = HealthCheckServer("data-collector", port=config.health_port) - - async def check_redis(): - try: - await broker._redis.ping() - return True - except Exception: - return False - - health.register_check("redis", check_redis) - await health.start() - - metrics.service_up.labels(service="data-collector").set(1) - - async def on_candle(candle): - log.info("candle_received", symbol=candle.symbol, timeframe=candle.timeframe) - await storage.store(candle) - metrics.events_processed.labels( - service="data-collector", event_type="candle" - ).inc() - - timeframe = config.timeframes[0] if config.timeframes else "1m" - - ws = BinanceWebSocket( - symbols=config.symbols, - timeframe=timeframe, - on_candle=on_candle, - ) - - log.info("starting", symbols=config.symbols, timeframe=timeframe) - - try: - await ws.start() - except Exception as exc: - log.error("fatal_error", error=str(exc)) - metrics.errors_total.labels( - service="data-collector", error_type="fatal" - ).inc() - await notifier.send_error(str(exc), "data-collector") - raise - finally: - metrics.service_up.labels(service="data-collector").set(0) - await notifier.close() - await broker.close() - await db.close() - - -def main() -> None: - asyncio.run(run()) - - -if __name__ == "__main__": - main() -``` - -- [ ] **Step 2: Update order-executor/executor.py to use notifier** - -Add notifier parameter to `OrderExecutor.__init__` and call it on order events. Replace `services/order-executor/src/order_executor/executor.py`: - -```python -"""Order execution logic.""" -import logging -from datetime import datetime, timezone -from decimal import Decimal -from typing import Any, Optional - -from shared.broker import RedisBroker -from shared.db import Database -from shared.events import OrderEvent -from shared.models import Order, OrderSide, OrderStatus, OrderType, Signal -from shared.notifier import TelegramNotifier - -from order_executor.risk_manager import RiskManager - -logger = logging.getLogger(__name__) - - -class OrderExecutor: - """Executes orders on an exchange with risk gating.""" - - def __init__( - self, - exchange: Any, - risk_manager: RiskManager, - broker: RedisBroker, - db: Database, - notifier: TelegramNotifier, - dry_run: bool = True, - ) -> None: - self.exchange = exchange - self.risk_manager = risk_manager - self.broker = broker - self.db = db - self.notifier = notifier - self.dry_run = dry_run - - async def execute(self, signal: Signal) -> Optional[Order]: - """Run risk checks and place an order for the given signal.""" - balance_data = await self.exchange.fetch_balance() - free_balances = balance_data.get("free", {}) - quote_currency = signal.symbol.split("/")[-1] if "/" in signal.symbol else "USDT" - balance = Decimal(str(free_balances.get(quote_currency, 0))) - - positions = {} - daily_pnl = Decimal(0) - - result = self.risk_manager.check( - signal=signal, - balance=balance, - positions=positions, - daily_pnl=daily_pnl, - ) - - if not result.allowed: - logger.warning( - "Risk check rejected signal %s: %s", signal.id, result.reason - ) - return None - - order = Order( - signal_id=signal.id, - symbol=signal.symbol, - side=signal.side, - type=OrderType.MARKET, - price=signal.price, - quantity=signal.quantity, - status=OrderStatus.PENDING, - ) - - if self.dry_run: - order.status = OrderStatus.FILLED - order.filled_at = datetime.now(timezone.utc) - logger.info("[DRY RUN] Order filled: %s %s %s", order.side, order.quantity, order.symbol) - else: - try: - await self.exchange.create_order( - symbol=signal.symbol, - type="market", - side=signal.side.value.lower(), - amount=float(signal.quantity), - ) - order.status = OrderStatus.FILLED - order.filled_at = datetime.now(timezone.utc) - logger.info("Order filled: %s %s %s", order.side, order.quantity, order.symbol) - except Exception as exc: - order.status = OrderStatus.FAILED - logger.error("Order failed for signal %s: %s", signal.id, exc) - - await self.db.insert_order(order) - await self.broker.publish("orders", OrderEvent(data=order).to_dict()) - await self.notifier.send_order(order) - - return order -``` - -- [ ] **Step 3: Update order-executor/main.py** - -Replace `services/order-executor/src/order_executor/main.py`: - -```python -"""Order Executor Service entry point.""" -import asyncio -from decimal import Decimal - -import ccxt.async_support as ccxt - -from shared.broker import RedisBroker -from shared.db import Database -from shared.events import Event, EventType -from shared.healthcheck import HealthCheckServer -from shared.logging import setup_logging -from shared.metrics import ServiceMetrics -from shared.notifier import TelegramNotifier - -from order_executor.config import ExecutorConfig -from order_executor.executor import OrderExecutor -from order_executor.risk_manager import RiskManager - - -async def run() -> None: - config = ExecutorConfig() - log = setup_logging("order-executor", config.log_level, config.log_format) - metrics = ServiceMetrics("order_executor") - - notifier = TelegramNotifier( - bot_token=config.telegram_bot_token, - chat_id=config.telegram_chat_id, - ) - - db = Database(config.database_url) - await db.connect() - - broker = RedisBroker(config.redis_url) - - exchange = ccxt.binance( - {"apiKey": config.binance_api_key, "secret": config.binance_api_secret} - ) - - risk_manager = RiskManager( - max_position_size=Decimal(str(config.risk_max_position_size)), - stop_loss_pct=Decimal(str(config.risk_stop_loss_pct)), - daily_loss_limit_pct=Decimal(str(config.risk_daily_loss_limit_pct)), - ) - - executor = OrderExecutor( - exchange=exchange, - risk_manager=risk_manager, - broker=broker, - db=db, - notifier=notifier, - dry_run=config.dry_run, - ) - - health = HealthCheckServer("order-executor", port=config.health_port + 2) - await health.start() - metrics.service_up.labels(service="order-executor").set(1) - - last_id = "$" - stream = "signals" - log.info("started", stream=stream, dry_run=config.dry_run) - - try: - while True: - messages = await broker.read(stream, last_id=last_id, count=10, block=5000) - for msg in messages: - try: - event = Event.from_dict(msg) - if event.type == EventType.SIGNAL: - signal = event.data - log.info("processing_signal", signal_id=signal.id, symbol=signal.symbol) - await executor.execute(signal) - metrics.events_processed.labels( - service="order-executor", event_type="signal" - ).inc() - except Exception as exc: - log.error("process_failed", error=str(exc)) - metrics.errors_total.labels( - service="order-executor", error_type="processing" - ).inc() - finally: - metrics.service_up.labels(service="order-executor").set(0) - await notifier.close() - await broker.close() - await db.close() - await exchange.close() - - -def main() -> None: - asyncio.run(run()) - - -if __name__ == "__main__": - main() -``` - -- [ ] **Step 4: Update strategy-engine/main.py** - -Replace `services/strategy-engine/src/strategy_engine/main.py`: - -```python -"""Strategy Engine Service entry point.""" -import asyncio -from pathlib import Path - -from shared.broker import RedisBroker -from shared.healthcheck import HealthCheckServer -from shared.logging import setup_logging -from shared.metrics import ServiceMetrics -from shared.notifier import TelegramNotifier - -from strategy_engine.config import StrategyConfig -from strategy_engine.engine import StrategyEngine -from strategy_engine.plugin_loader import load_strategies - -STRATEGIES_DIR = Path(__file__).parent.parent.parent.parent / "strategies" - - -async def run() -> None: - config = StrategyConfig() - log = setup_logging("strategy-engine", config.log_level, config.log_format) - metrics = ServiceMetrics("strategy_engine") - - notifier = TelegramNotifier( - bot_token=config.telegram_bot_token, - chat_id=config.telegram_chat_id, - ) - - broker = RedisBroker(config.redis_url) - strategies = load_strategies(STRATEGIES_DIR) - - for strategy in strategies: - params = config.strategy_params.get(strategy.name, {}) - strategy.configure(params) - - log.info("loaded_strategies", count=len(strategies), names=[s.name for s in strategies]) - - engine = StrategyEngine(broker=broker, strategies=strategies) - - health = HealthCheckServer("strategy-engine", port=config.health_port + 1) - await health.start() - metrics.service_up.labels(service="strategy-engine").set(1) - - try: - for symbol in config.symbols: - stream = f"candles.{symbol.replace('/', '_')}" - last_id = "$" - log.info("engine_loop_start", stream=stream) - while True: - last_id = await engine.process_once(stream, last_id) - except Exception as exc: - log.error("fatal_error", error=str(exc)) - await notifier.send_error(str(exc), "strategy-engine") - raise - finally: - metrics.service_up.labels(service="strategy-engine").set(0) - await notifier.close() - await broker.close() - - -def main() -> None: - asyncio.run(run()) - - -if __name__ == "__main__": - main() -``` - -- [ ] **Step 5: Update portfolio-manager/main.py** - -Replace `services/portfolio-manager/src/portfolio_manager/main.py`: - -```python -"""Portfolio Manager Service entry point.""" -import asyncio - -from shared.broker import RedisBroker -from shared.events import Event, OrderEvent -from shared.healthcheck import HealthCheckServer -from shared.logging import setup_logging -from shared.metrics import ServiceMetrics -from shared.notifier import TelegramNotifier - -from portfolio_manager.config import PortfolioConfig -from portfolio_manager.portfolio import PortfolioTracker - -ORDERS_STREAM = "orders" - - -async def run() -> None: - config = PortfolioConfig() - log = setup_logging("portfolio-manager", config.log_level, config.log_format) - metrics = ServiceMetrics("portfolio_manager") - - notifier = TelegramNotifier( - bot_token=config.telegram_bot_token, - chat_id=config.telegram_chat_id, - ) - - broker = RedisBroker(config.redis_url) - tracker = PortfolioTracker() - - health = HealthCheckServer("portfolio-manager", port=config.health_port + 3) - await health.start() - metrics.service_up.labels(service="portfolio-manager").set(1) - - last_id = "$" - log.info("started", stream=ORDERS_STREAM) - - try: - while True: - messages = await broker.read(ORDERS_STREAM, last_id=last_id, block=1000) - for msg in messages: - try: - event = Event.from_dict(msg) - if isinstance(event, OrderEvent): - order = event.data - tracker.apply_order(order) - log.info( - "order_applied", - symbol=order.symbol, - side=order.side.value, - qty=str(order.quantity), - ) - metrics.events_processed.labels( - service="portfolio-manager", event_type="order" - ).inc() - except Exception as exc: - log.error("process_failed", error=str(exc)) - metrics.errors_total.labels( - service="portfolio-manager", error_type="processing" - ).inc() - finally: - metrics.service_up.labels(service="portfolio-manager").set(0) - await notifier.close() - await broker.close() - - -def main() -> None: - asyncio.run(run()) - - -if __name__ == "__main__": - main() -``` - -- [ ] **Step 6: Run all tests to verify nothing is broken** - -Run: `pytest -v` -Expected: All existing tests PASS (some executor tests may need mock updates for `notifier` param) - -- [ ] **Step 7: Fix any broken executor tests** - -The `OrderExecutor` now requires a `notifier` parameter. Update `services/order-executor/tests/test_executor.py` — add `notifier=AsyncMock()` to every `OrderExecutor(...)` call. For example, wherever the test creates: - -```python -executor = OrderExecutor( - exchange=mock_exchange, - risk_manager=mock_risk, - broker=mock_broker, - db=mock_db, - dry_run=True, -) -``` - -Change to: - -```python -executor = OrderExecutor( - exchange=mock_exchange, - risk_manager=mock_risk, - broker=mock_broker, - db=mock_db, - notifier=AsyncMock(), - dry_run=True, -) -``` - -- [ ] **Step 8: Run all tests again** - -Run: `pytest -v` -Expected: All tests PASS - -- [ ] **Step 9: Commit** - -```bash -git add services/data-collector/src/data_collector/main.py \ - services/order-executor/src/order_executor/executor.py \ - services/order-executor/src/order_executor/main.py \ - services/order-executor/tests/test_executor.py \ - services/strategy-engine/src/strategy_engine/main.py \ - services/portfolio-manager/src/portfolio_manager/main.py -git commit -m "feat(services): integrate structlog, healthcheck, metrics, and Telegram" -``` - ---- - -## Task 8: BaseStrategy warmup_period + YAML Config Loading - -**Files:** -- Modify: `services/strategy-engine/strategies/base.py` -- Modify: `services/strategy-engine/strategies/rsi_strategy.py` -- Modify: `services/strategy-engine/strategies/grid_strategy.py` -- Modify: `services/strategy-engine/src/strategy_engine/plugin_loader.py` -- Create: `services/strategy-engine/strategies/config/rsi_strategy.yaml` -- Create: `services/strategy-engine/strategies/config/grid_strategy.yaml` - -- [ ] **Step 1: Update BaseStrategy with warmup_period** - -Replace `services/strategy-engine/strategies/base.py`: - -```python -from abc import ABC, abstractmethod -from shared.models import Candle, Signal - - -class BaseStrategy(ABC): - name: str = "base" - - @property - @abstractmethod - def warmup_period(self) -> int: - """Minimum number of candles needed before generating signals.""" - pass - - @abstractmethod - def on_candle(self, candle: Candle) -> Signal | None: - pass - - @abstractmethod - def configure(self, params: dict) -> None: - pass - - def reset(self) -> None: - pass -``` - -- [ ] **Step 2: Update RsiStrategy with warmup_period** - -Add this property to `RsiStrategy` in `rsi_strategy.py`, after `__init__`: - -```python - @property - def warmup_period(self) -> int: - return self._period + 1 -``` - -- [ ] **Step 3: Update GridStrategy with warmup_period** - -Add this property to `GridStrategy` in `grid_strategy.py`, after `__init__`: - -```python - @property - def warmup_period(self) -> int: - return 2 # Needs at least 2 candles to detect zone crossing -``` - -- [ ] **Step 4: Create YAML config files** - -Create `services/strategy-engine/strategies/config/rsi_strategy.yaml`: - -```yaml -period: 14 -oversold: 30 -overbought: 70 -quantity: "0.01" -``` - -Create `services/strategy-engine/strategies/config/grid_strategy.yaml`: - -```yaml -lower_price: 60000 -upper_price: 70000 -grid_count: 5 -quantity: "0.01" -``` - -- [ ] **Step 5: Update plugin_loader.py with YAML config loading** - -Replace `services/strategy-engine/src/strategy_engine/plugin_loader.py`: - -```python -"""Dynamic plugin loader for strategy modules with YAML config support.""" -import importlib.util -import sys -from pathlib import Path - -import yaml - -from strategies.base import BaseStrategy - - -def load_strategies(strategies_dir: Path) -> list[BaseStrategy]: - """Scan strategies_dir for *.py files and load all BaseStrategy subclasses. - - Automatically loads matching YAML config from strategies_dir/config/. - """ - loaded: list[BaseStrategy] = [] - config_dir = strategies_dir / "config" - - for path in sorted(strategies_dir.glob("*.py")): - if path.name.startswith("__") or path.name == "base.py": - continue - - module_name = f"_strategy_plugin_{path.stem}" - spec = importlib.util.spec_from_file_location(module_name, path) - if spec is None or spec.loader is None: - continue - - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - spec.loader.exec_module(module) - - for attr_name in dir(module): - obj = getattr(module, attr_name) - if ( - isinstance(obj, type) - and issubclass(obj, BaseStrategy) - and obj is not BaseStrategy - ): - instance = obj() - - # Load YAML config if it exists - yaml_path = config_dir / f"{path.stem}.yaml" - if yaml_path.exists(): - with open(yaml_path) as f: - params = yaml.safe_load(f) or {} - instance.configure(params) - - loaded.append(instance) - - return loaded -``` - -- [ ] **Step 6: Run existing strategy tests** - -Run: `pytest services/strategy-engine/tests/ -v` -Expected: All tests PASS - -- [ ] **Step 7: Commit** - -```bash -git add services/strategy-engine/strategies/base.py \ - services/strategy-engine/strategies/rsi_strategy.py \ - services/strategy-engine/strategies/grid_strategy.py \ - services/strategy-engine/src/strategy_engine/plugin_loader.py \ - services/strategy-engine/strategies/config/ -git commit -m "feat(strategy): add warmup_period to BaseStrategy and YAML config loading" -``` - ---- - -## Task 9: MACD Strategy - -**Files:** -- Create: `services/strategy-engine/strategies/macd_strategy.py` -- Create: `services/strategy-engine/strategies/config/macd_strategy.yaml` -- Test: `services/strategy-engine/tests/test_macd_strategy.py` - -- [ ] **Step 1: Write the failing test** - -Create `services/strategy-engine/tests/test_macd_strategy.py`: - -```python -"""Tests for MACD strategy.""" -from decimal import Decimal - -import pytest - -from shared.models import Candle, OrderSide -from strategies.macd_strategy import MacdStrategy - - -@pytest.fixture -def strategy(): - s = MacdStrategy() - s.configure({"fast_period": 3, "slow_period": 6, "signal_period": 3, "quantity": "0.01"}) - return s - - -def _candle(price: float) -> Candle: - from datetime import datetime, timezone - return Candle( - symbol="BTCUSDT", - timeframe="1m", - open_time=datetime(2025, 1, 1, tzinfo=timezone.utc), - open=Decimal(str(price)), - high=Decimal(str(price + 100)), - low=Decimal(str(price - 100)), - close=Decimal(str(price)), - volume=Decimal("10"), - ) - - -def test_macd_warmup_period(strategy): - assert strategy.warmup_period == 9 # slow_period + signal_period = 6 + 3 - - -def test_macd_no_signal_insufficient_data(strategy): - for price in [100, 101, 102, 103, 104]: - result = strategy.on_candle(_candle(price)) - assert result is None - - -def test_macd_buy_signal_on_bullish_crossover(strategy): - # Feed declining prices to push MACD below signal, then rising to cross above - prices = [100, 98, 96, 94, 92, 90, 88, 90, 93, 97, 102, 108, 115, 123, 132] - signals = [] - for p in prices: - sig = strategy.on_candle(_candle(p)) - if sig is not None: - signals.append(sig) - buy_signals = [s for s in signals if s.side == OrderSide.BUY] - assert len(buy_signals) > 0 - assert buy_signals[0].strategy == "macd" - - -def test_macd_sell_signal_on_bearish_crossover(strategy): - # Feed rising prices to push MACD above signal, then declining to cross below - prices = [100, 105, 110, 116, 122, 128, 125, 120, 114, 107, 99, 90, 80, 70, 60] - signals = [] - for p in prices: - sig = strategy.on_candle(_candle(p)) - if sig is not None: - signals.append(sig) - sell_signals = [s for s in signals if s.side == OrderSide.SELL] - assert len(sell_signals) > 0 - - -def test_macd_reset_clears_state(strategy): - for p in [100, 101, 102]: - strategy.on_candle(_candle(p)) - strategy.reset() - assert len(strategy._closes) == 0 -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `pytest services/strategy-engine/tests/test_macd_strategy.py -v` -Expected: FAIL with `ModuleNotFoundError: No module named 'strategies.macd_strategy'` - -- [ ] **Step 3: Implement MACD strategy** - -Create `services/strategy-engine/strategies/macd_strategy.py`: - -```python -"""MACD (Moving Average Convergence Divergence) strategy.""" -from collections import deque -from decimal import Decimal - -import pandas as pd - -from shared.models import Candle, Signal, OrderSide -from strategies.base import BaseStrategy - - -class MacdStrategy(BaseStrategy): - name: str = "macd" - - def __init__(self) -> None: - self._fast_period: int = 12 - self._slow_period: int = 26 - self._signal_period: int = 9 - self._quantity: Decimal = Decimal("0.01") - self._closes: deque[float] = deque(maxlen=500) - self._prev_histogram: float | None = None - - @property - def warmup_period(self) -> int: - return self._slow_period + self._signal_period - - def configure(self, params: dict) -> None: - self._fast_period = int(params.get("fast_period", 12)) - self._slow_period = int(params.get("slow_period", 26)) - self._signal_period = int(params.get("signal_period", 9)) - self._quantity = Decimal(str(params.get("quantity", "0.01"))) - - def reset(self) -> None: - self._closes.clear() - self._prev_histogram = None - - def on_candle(self, candle: Candle) -> Signal | None: - self._closes.append(float(candle.close)) - - if len(self._closes) < self.warmup_period: - return None - - series = pd.Series(list(self._closes)) - fast_ema = series.ewm(span=self._fast_period, adjust=False).mean() - slow_ema = series.ewm(span=self._slow_period, adjust=False).mean() - macd_line = fast_ema - slow_ema - signal_line = macd_line.ewm(span=self._signal_period, adjust=False).mean() - histogram = macd_line - signal_line - - current_hist = histogram.iloc[-1] - - if self._prev_histogram is None: - self._prev_histogram = current_hist - return None - - prev = self._prev_histogram - self._prev_histogram = current_hist - - # Bullish crossover: histogram crosses from negative to positive - if prev < 0 and current_hist > 0: - return Signal( - strategy=self.name, - symbol=candle.symbol, - side=OrderSide.BUY, - price=candle.close, - quantity=self._quantity, - reason=f"MACD bullish crossover (histogram {prev:.4f} -> {current_hist:.4f})", - ) - - # Bearish crossover: histogram crosses from positive to negative - if prev > 0 and current_hist < 0: - return Signal( - strategy=self.name, - symbol=candle.symbol, - side=OrderSide.SELL, - price=candle.close, - quantity=self._quantity, - reason=f"MACD bearish crossover (histogram {prev:.4f} -> {current_hist:.4f})", - ) - - return None -``` - -Create `services/strategy-engine/strategies/config/macd_strategy.yaml`: - -```yaml -fast_period: 12 -slow_period: 26 -signal_period: 9 -quantity: "0.01" -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `pytest services/strategy-engine/tests/test_macd_strategy.py -v` -Expected: All 5 tests PASS - -- [ ] **Step 5: Commit** - -```bash -git add services/strategy-engine/strategies/macd_strategy.py \ - services/strategy-engine/strategies/config/macd_strategy.yaml \ - services/strategy-engine/tests/test_macd_strategy.py -git commit -m "feat(strategy): add MACD strategy" -``` - ---- - -## Task 10: Bollinger Bands Strategy - -**Files:** -- Create: `services/strategy-engine/strategies/bollinger_strategy.py` -- Create: `services/strategy-engine/strategies/config/bollinger_strategy.yaml` -- Test: `services/strategy-engine/tests/test_bollinger_strategy.py` - -- [ ] **Step 1: Write the failing test** - -Create `services/strategy-engine/tests/test_bollinger_strategy.py`: - -```python -"""Tests for Bollinger Bands strategy.""" -from decimal import Decimal -from datetime import datetime, timezone - -import pytest - -from shared.models import Candle, OrderSide -from strategies.bollinger_strategy import BollingerStrategy - - -@pytest.fixture -def strategy(): - s = BollingerStrategy() - s.configure({"period": 5, "num_std": 2.0, "min_bandwidth": 0.0, "quantity": "0.01"}) - return s - - -def _candle(price: float) -> Candle: - return Candle( - symbol="BTCUSDT", - timeframe="1m", - open_time=datetime(2025, 1, 1, tzinfo=timezone.utc), - open=Decimal(str(price)), - high=Decimal(str(price + 10)), - low=Decimal(str(price - 10)), - close=Decimal(str(price)), - volume=Decimal("10"), - ) - - -def test_bollinger_warmup_period(strategy): - assert strategy.warmup_period == 5 - - -def test_bollinger_no_signal_insufficient_data(strategy): - for p in [100, 101, 102]: - result = strategy.on_candle(_candle(p)) - assert result is None - - -def test_bollinger_buy_on_lower_band_recovery(strategy): - # Stable prices to build bands, then drop below and recover - prices = [100, 100, 100, 100, 100, 80, 80, 95] - signals = [] - for p in prices: - sig = strategy.on_candle(_candle(p)) - if sig is not None: - signals.append(sig) - buy_signals = [s for s in signals if s.side == OrderSide.BUY] - assert len(buy_signals) > 0 - assert buy_signals[0].strategy == "bollinger" - - -def test_bollinger_sell_on_upper_band_recovery(strategy): - prices = [100, 100, 100, 100, 100, 120, 120, 105] - signals = [] - for p in prices: - sig = strategy.on_candle(_candle(p)) - if sig is not None: - signals.append(sig) - sell_signals = [s for s in signals if s.side == OrderSide.SELL] - assert len(sell_signals) > 0 - - -def test_bollinger_reset_clears_state(strategy): - for p in [100, 101]: - strategy.on_candle(_candle(p)) - strategy.reset() - assert len(strategy._closes) == 0 -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `pytest services/strategy-engine/tests/test_bollinger_strategy.py -v` -Expected: FAIL with `ModuleNotFoundError` - -- [ ] **Step 3: Implement Bollinger Bands strategy** - -Create `services/strategy-engine/strategies/bollinger_strategy.py`: - -```python -"""Bollinger Bands strategy.""" -from collections import deque -from decimal import Decimal - -import pandas as pd - -from shared.models import Candle, Signal, OrderSide -from strategies.base import BaseStrategy - - -class BollingerStrategy(BaseStrategy): - name: str = "bollinger" - - def __init__(self) -> None: - self._period: int = 20 - self._num_std: float = 2.0 - self._min_bandwidth: float = 0.02 - self._quantity: Decimal = Decimal("0.01") - self._closes: deque[float] = deque(maxlen=500) - self._was_below_lower: bool = False - self._was_above_upper: bool = False - - @property - def warmup_period(self) -> int: - return self._period - - def configure(self, params: dict) -> None: - self._period = int(params.get("period", 20)) - self._num_std = float(params.get("num_std", 2.0)) - self._min_bandwidth = float(params.get("min_bandwidth", 0.02)) - self._quantity = Decimal(str(params.get("quantity", "0.01"))) - - def reset(self) -> None: - self._closes.clear() - self._was_below_lower = False - self._was_above_upper = False - - def on_candle(self, candle: Candle) -> Signal | None: - self._closes.append(float(candle.close)) - - if len(self._closes) < self._period: - return None - - series = pd.Series(list(self._closes)) - sma = series.rolling(self._period).mean().iloc[-1] - std = series.rolling(self._period).std().iloc[-1] - - upper = sma + self._num_std * std - lower = sma - self._num_std * std - - # Bandwidth filter - if sma > 0: - bandwidth = (upper - lower) / sma - if bandwidth < self._min_bandwidth: - return None - - price = float(candle.close) - - # Track band penetration - if price < lower: - self._was_below_lower = True - if price > upper: - self._was_above_upper = True - - # BUY: price was below lower band and recovered back inside - if self._was_below_lower and price >= lower: - self._was_below_lower = False - return Signal( - strategy=self.name, - symbol=candle.symbol, - side=OrderSide.BUY, - price=candle.close, - quantity=self._quantity, - reason=f"Bollinger: price recovered above lower band ({lower:.2f})", - ) - - # SELL: price was above upper band and recovered back inside - if self._was_above_upper and price <= upper: - self._was_above_upper = False - return Signal( - strategy=self.name, - symbol=candle.symbol, - side=OrderSide.SELL, - price=candle.close, - quantity=self._quantity, - reason=f"Bollinger: price recovered below upper band ({upper:.2f})", - ) - - return None -``` - -Create `services/strategy-engine/strategies/config/bollinger_strategy.yaml`: - -```yaml -period: 20 -num_std: 2.0 -min_bandwidth: 0.02 -quantity: "0.01" -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `pytest services/strategy-engine/tests/test_bollinger_strategy.py -v` -Expected: All 5 tests PASS - -- [ ] **Step 5: Commit** - -```bash -git add services/strategy-engine/strategies/bollinger_strategy.py \ - services/strategy-engine/strategies/config/bollinger_strategy.yaml \ - services/strategy-engine/tests/test_bollinger_strategy.py -git commit -m "feat(strategy): add Bollinger Bands strategy" -``` - ---- - -## Task 11: EMA Crossover Strategy - -**Files:** -- Create: `services/strategy-engine/strategies/ema_crossover_strategy.py` -- Create: `services/strategy-engine/strategies/config/ema_crossover_strategy.yaml` -- Test: `services/strategy-engine/tests/test_ema_crossover_strategy.py` - -- [ ] **Step 1: Write the failing test** - -Create `services/strategy-engine/tests/test_ema_crossover_strategy.py`: - -```python -"""Tests for EMA Crossover strategy.""" -from decimal import Decimal -from datetime import datetime, timezone - -import pytest - -from shared.models import Candle, OrderSide -from strategies.ema_crossover_strategy import EmaCrossoverStrategy - - -@pytest.fixture -def strategy(): - s = EmaCrossoverStrategy() - s.configure({"short_period": 3, "long_period": 6, "quantity": "0.01"}) - return s - - -def _candle(price: float) -> Candle: - return Candle( - symbol="BTCUSDT", - timeframe="1m", - open_time=datetime(2025, 1, 1, tzinfo=timezone.utc), - open=Decimal(str(price)), - high=Decimal(str(price + 10)), - low=Decimal(str(price - 10)), - close=Decimal(str(price)), - volume=Decimal("10"), - ) - - -def test_ema_warmup_period(strategy): - assert strategy.warmup_period == 6 - - -def test_ema_no_signal_insufficient_data(strategy): - for p in [100, 101, 102]: - result = strategy.on_candle(_candle(p)) - assert result is None - - -def test_ema_buy_signal_golden_cross(strategy): - # Declining then sharp rise: short EMA crosses above long EMA - prices = [100, 98, 96, 94, 92, 90, 95, 100, 108, 117, 127] - signals = [] - for p in prices: - sig = strategy.on_candle(_candle(p)) - if sig is not None: - signals.append(sig) - buy_signals = [s for s in signals if s.side == OrderSide.BUY] - assert len(buy_signals) > 0 - assert buy_signals[0].strategy == "ema_crossover" - - -def test_ema_sell_signal_death_cross(strategy): - # Rising then sharp decline: short EMA crosses below long EMA - prices = [100, 105, 110, 115, 120, 125, 118, 110, 100, 88, 75] - signals = [] - for p in prices: - sig = strategy.on_candle(_candle(p)) - if sig is not None: - signals.append(sig) - sell_signals = [s for s in signals if s.side == OrderSide.SELL] - assert len(sell_signals) > 0 - - -def test_ema_reset_clears_state(strategy): - for p in [100, 101]: - strategy.on_candle(_candle(p)) - strategy.reset() - assert len(strategy._closes) == 0 -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `pytest services/strategy-engine/tests/test_ema_crossover_strategy.py -v` -Expected: FAIL with `ModuleNotFoundError` - -- [ ] **Step 3: Implement EMA Crossover strategy** - -Create `services/strategy-engine/strategies/ema_crossover_strategy.py`: - -```python -"""EMA Crossover (Golden Cross / Death Cross) strategy.""" -from collections import deque -from decimal import Decimal - -import pandas as pd - -from shared.models import Candle, Signal, OrderSide -from strategies.base import BaseStrategy - - -class EmaCrossoverStrategy(BaseStrategy): - name: str = "ema_crossover" - - def __init__(self) -> None: - self._short_period: int = 9 - self._long_period: int = 21 - self._quantity: Decimal = Decimal("0.01") - self._closes: deque[float] = deque(maxlen=500) - self._prev_short_above: bool | None = None - - @property - def warmup_period(self) -> int: - return self._long_period - - def configure(self, params: dict) -> None: - self._short_period = int(params.get("short_period", 9)) - self._long_period = int(params.get("long_period", 21)) - self._quantity = Decimal(str(params.get("quantity", "0.01"))) - - def reset(self) -> None: - self._closes.clear() - self._prev_short_above = None - - def on_candle(self, candle: Candle) -> Signal | None: - self._closes.append(float(candle.close)) - - if len(self._closes) < self._long_period: - return None - - series = pd.Series(list(self._closes)) - short_ema = series.ewm(span=self._short_period, adjust=False).mean().iloc[-1] - long_ema = series.ewm(span=self._long_period, adjust=False).mean().iloc[-1] - - short_above = short_ema > long_ema - - if self._prev_short_above is None: - self._prev_short_above = short_above - return None - - prev = self._prev_short_above - self._prev_short_above = short_above - - # Golden Cross: short EMA crosses above long EMA - if not prev and short_above: - return Signal( - strategy=self.name, - symbol=candle.symbol, - side=OrderSide.BUY, - price=candle.close, - quantity=self._quantity, - reason=f"EMA Golden Cross (short={short_ema:.2f} > long={long_ema:.2f})", - ) - - # Death Cross: short EMA crosses below long EMA - if prev and not short_above: - return Signal( - strategy=self.name, - symbol=candle.symbol, - side=OrderSide.SELL, - price=candle.close, - quantity=self._quantity, - reason=f"EMA Death Cross (short={short_ema:.2f} < long={long_ema:.2f})", - ) - - return None -``` - -Create `services/strategy-engine/strategies/config/ema_crossover_strategy.yaml`: - -```yaml -short_period: 9 -long_period: 21 -quantity: "0.01" -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `pytest services/strategy-engine/tests/test_ema_crossover_strategy.py -v` -Expected: All 5 tests PASS - -- [ ] **Step 5: Commit** - -```bash -git add services/strategy-engine/strategies/ema_crossover_strategy.py \ - services/strategy-engine/strategies/config/ema_crossover_strategy.yaml \ - services/strategy-engine/tests/test_ema_crossover_strategy.py -git commit -m "feat(strategy): add EMA Crossover strategy" -``` - ---- - -## Task 12: VWAP Strategy - -**Files:** -- Create: `services/strategy-engine/strategies/vwap_strategy.py` -- Create: `services/strategy-engine/strategies/config/vwap_strategy.yaml` -- Test: `services/strategy-engine/tests/test_vwap_strategy.py` - -- [ ] **Step 1: Write the failing test** - -Create `services/strategy-engine/tests/test_vwap_strategy.py`: - -```python -"""Tests for VWAP strategy.""" -from decimal import Decimal -from datetime import datetime, timezone - -import pytest - -from shared.models import Candle, OrderSide -from strategies.vwap_strategy import VwapStrategy - - -@pytest.fixture -def strategy(): - s = VwapStrategy() - s.configure({"deviation_threshold": 0.01, "quantity": "0.01"}) - return s - - -def _candle(price: float, volume: float = 10.0) -> Candle: - return Candle( - symbol="BTCUSDT", - timeframe="1m", - open_time=datetime(2025, 1, 1, tzinfo=timezone.utc), - open=Decimal(str(price)), - high=Decimal(str(price + 10)), - low=Decimal(str(price - 10)), - close=Decimal(str(price)), - volume=Decimal(str(volume)), - ) - - -def test_vwap_warmup_period(strategy): - assert strategy.warmup_period == 30 - - -def test_vwap_no_signal_insufficient_data(strategy): - for i in range(10): - result = strategy.on_candle(_candle(100)) - assert result is None - - -def test_vwap_buy_signal_below_vwap_recovery(strategy): - # Build VWAP at ~100, then go below, then recover - signals = [] - for _ in range(30): - strategy.on_candle(_candle(100, 100)) - # Drop below VWAP - for _ in range(5): - strategy.on_candle(_candle(95, 10)) - # Recover to VWAP - sig = strategy.on_candle(_candle(100, 10)) - if sig is not None: - signals.append(sig) - buy_signals = [s for s in signals if s.side == OrderSide.BUY] - assert len(buy_signals) > 0 - - -def test_vwap_sell_signal_above_vwap_recovery(strategy): - signals = [] - for _ in range(30): - strategy.on_candle(_candle(100, 100)) - for _ in range(5): - strategy.on_candle(_candle(105, 10)) - sig = strategy.on_candle(_candle(100, 10)) - if sig is not None: - signals.append(sig) - sell_signals = [s for s in signals if s.side == OrderSide.SELL] - assert len(sell_signals) > 0 - - -def test_vwap_reset_clears_state(strategy): - strategy.on_candle(_candle(100)) - strategy.reset() - assert strategy._cumulative_tp_vol == 0.0 - assert strategy._cumulative_vol == 0.0 -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `pytest services/strategy-engine/tests/test_vwap_strategy.py -v` -Expected: FAIL with `ModuleNotFoundError` - -- [ ] **Step 3: Implement VWAP strategy** - -Create `services/strategy-engine/strategies/vwap_strategy.py`: - -```python -"""VWAP (Volume Weighted Average Price) strategy.""" -from decimal import Decimal - -from shared.models import Candle, Signal, OrderSide -from strategies.base import BaseStrategy - - -class VwapStrategy(BaseStrategy): - name: str = "vwap" - - def __init__(self) -> None: - self._deviation_threshold: float = 0.002 - self._quantity: Decimal = Decimal("0.01") - self._cumulative_tp_vol: float = 0.0 - self._cumulative_vol: float = 0.0 - self._candle_count: int = 0 - self._was_below_vwap: bool = False - self._was_above_vwap: bool = False - - @property - def warmup_period(self) -> int: - return 30 - - def configure(self, params: dict) -> None: - self._deviation_threshold = float(params.get("deviation_threshold", 0.002)) - self._quantity = Decimal(str(params.get("quantity", "0.01"))) - - def reset(self) -> None: - self._cumulative_tp_vol = 0.0 - self._cumulative_vol = 0.0 - self._candle_count = 0 - self._was_below_vwap = False - self._was_above_vwap = False - - def on_candle(self, candle: Candle) -> Signal | None: - high = float(candle.high) - low = float(candle.low) - close = float(candle.close) - volume = float(candle.volume) - - typical_price = (high + low + close) / 3.0 - self._cumulative_tp_vol += typical_price * volume - self._cumulative_vol += volume - self._candle_count += 1 - - if self._candle_count < self.warmup_period: - return None - - if self._cumulative_vol == 0: - return None - - vwap = self._cumulative_tp_vol / self._cumulative_vol - deviation = (close - vwap) / vwap if vwap != 0 else 0 - - # Track VWAP deviations - if deviation < -self._deviation_threshold: - self._was_below_vwap = True - self._was_above_vwap = False - elif deviation > self._deviation_threshold: - self._was_above_vwap = True - self._was_below_vwap = False - - # BUY: price was below VWAP and recovered to VWAP (mean reversion) - if self._was_below_vwap and abs(deviation) <= self._deviation_threshold: - self._was_below_vwap = False - return Signal( - strategy=self.name, - symbol=candle.symbol, - side=OrderSide.BUY, - price=candle.close, - quantity=self._quantity, - reason=f"VWAP mean reversion from below (VWAP={vwap:.2f}, deviation={deviation:.4f})", - ) - - # SELL: price was above VWAP and recovered to VWAP - if self._was_above_vwap and abs(deviation) <= self._deviation_threshold: - self._was_above_vwap = False - return Signal( - strategy=self.name, - symbol=candle.symbol, - side=OrderSide.SELL, - price=candle.close, - quantity=self._quantity, - reason=f"VWAP mean reversion from above (VWAP={vwap:.2f}, deviation={deviation:.4f})", - ) - - return None -``` - -Create `services/strategy-engine/strategies/config/vwap_strategy.yaml`: - -```yaml -deviation_threshold: 0.002 -quantity: "0.01" -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `pytest services/strategy-engine/tests/test_vwap_strategy.py -v` -Expected: All 5 tests PASS - -- [ ] **Step 5: Commit** - -```bash -git add services/strategy-engine/strategies/vwap_strategy.py \ - services/strategy-engine/strategies/config/vwap_strategy.yaml \ - services/strategy-engine/tests/test_vwap_strategy.py -git commit -m "feat(strategy): add VWAP strategy" -``` - ---- - -## Task 13: Volume Profile Strategy - -**Files:** -- Create: `services/strategy-engine/strategies/volume_profile_strategy.py` -- Create: `services/strategy-engine/strategies/config/volume_profile_strategy.yaml` -- Test: `services/strategy-engine/tests/test_volume_profile_strategy.py` - -- [ ] **Step 1: Write the failing test** - -Create `services/strategy-engine/tests/test_volume_profile_strategy.py`: - -```python -"""Tests for Volume Profile strategy.""" -from decimal import Decimal -from datetime import datetime, timezone - -import pytest - -from shared.models import Candle, OrderSide -from strategies.volume_profile_strategy import VolumeProfileStrategy - - -@pytest.fixture -def strategy(): - s = VolumeProfileStrategy() - s.configure({ - "lookback_period": 10, - "num_bins": 5, - "value_area_pct": 0.7, - "quantity": "0.01", - }) - return s - - -def _candle(price: float, volume: float = 10.0) -> Candle: - return Candle( - symbol="BTCUSDT", - timeframe="1m", - open_time=datetime(2025, 1, 1, tzinfo=timezone.utc), - open=Decimal(str(price)), - high=Decimal(str(price + 5)), - low=Decimal(str(price - 5)), - close=Decimal(str(price)), - volume=Decimal(str(volume)), - ) - - -def test_volume_profile_warmup_period(strategy): - assert strategy.warmup_period == 10 - - -def test_volume_profile_no_signal_insufficient_data(strategy): - for p in [100, 101, 102]: - result = strategy.on_candle(_candle(p)) - assert result is None - - -def test_volume_profile_buy_at_value_area_low(strategy): - # Concentrate volume at 100, then price drops to bottom of value area - signals = [] - for _ in range(10): - strategy.on_candle(_candle(100, 100)) - # Price drops to lower edge - sig = strategy.on_candle(_candle(90, 10)) - if sig is not None: - signals.append(sig) - # Multiple attempts — may need several candles for the signal - for p in [89, 88, 90]: - sig = strategy.on_candle(_candle(p, 10)) - if sig is not None: - signals.append(sig) - buy_signals = [s for s in signals if s.side == OrderSide.BUY] - assert len(buy_signals) > 0 - - -def test_volume_profile_sell_at_value_area_high(strategy): - signals = [] - for _ in range(10): - strategy.on_candle(_candle(100, 100)) - sig = strategy.on_candle(_candle(110, 10)) - if sig is not None: - signals.append(sig) - for p in [111, 112, 110]: - sig = strategy.on_candle(_candle(p, 10)) - if sig is not None: - signals.append(sig) - sell_signals = [s for s in signals if s.side == OrderSide.SELL] - assert len(sell_signals) > 0 - - -def test_volume_profile_reset_clears_state(strategy): - strategy.on_candle(_candle(100)) - strategy.reset() - assert len(strategy._candles) == 0 -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `pytest services/strategy-engine/tests/test_volume_profile_strategy.py -v` -Expected: FAIL with `ModuleNotFoundError` - -- [ ] **Step 3: Implement Volume Profile strategy** - -Create `services/strategy-engine/strategies/volume_profile_strategy.py`: - -```python -"""Volume Profile strategy based on Point of Control and Value Area.""" -from collections import deque -from decimal import Decimal - -import numpy as np - -from shared.models import Candle, Signal, OrderSide -from strategies.base import BaseStrategy - - -class VolumeProfileStrategy(BaseStrategy): - name: str = "volume_profile" - - def __init__(self) -> None: - self._lookback_period: int = 100 - self._num_bins: int = 50 - self._value_area_pct: float = 0.7 - self._quantity: Decimal = Decimal("0.01") - self._candles: deque[tuple[float, float]] = deque(maxlen=500) # (close, volume) - self._was_below_va: bool = False - self._was_above_va: bool = False - - @property - def warmup_period(self) -> int: - return self._lookback_period - - def configure(self, params: dict) -> None: - self._lookback_period = int(params.get("lookback_period", 100)) - self._num_bins = int(params.get("num_bins", 50)) - self._value_area_pct = float(params.get("value_area_pct", 0.7)) - self._quantity = Decimal(str(params.get("quantity", "0.01"))) - - def reset(self) -> None: - self._candles.clear() - self._was_below_va = False - self._was_above_va = False - - def _compute_value_area(self) -> tuple[float, float, float] | None: - """Compute POC, value area low, and value area high. - - Returns (poc, va_low, va_high) or None if insufficient data. - """ - if len(self._candles) < self._lookback_period: - return None - - recent = list(self._candles)[-self._lookback_period :] - prices = [c[0] for c in recent] - volumes = [c[1] for c in recent] - - min_price = min(prices) - max_price = max(prices) - if min_price == max_price: - return None - - bin_edges = np.linspace(min_price, max_price, self._num_bins + 1) - volume_profile = np.zeros(self._num_bins) - - for price, vol in zip(prices, volumes): - bin_idx = int((price - min_price) / (max_price - min_price) * (self._num_bins - 1)) - bin_idx = min(bin_idx, self._num_bins - 1) - volume_profile[bin_idx] += vol - - poc_idx = int(np.argmax(volume_profile)) - poc = (bin_edges[poc_idx] + bin_edges[poc_idx + 1]) / 2 - - # Expand from POC to capture value_area_pct of total volume - total_vol = volume_profile.sum() - if total_vol == 0: - return None - - target_vol = total_vol * self._value_area_pct - accumulated = volume_profile[poc_idx] - low_idx = poc_idx - high_idx = poc_idx - - while accumulated < target_vol: - expand_low = low_idx > 0 - expand_high = high_idx < self._num_bins - 1 - - if not expand_low and not expand_high: - break - - low_vol = volume_profile[low_idx - 1] if expand_low else 0 - high_vol = volume_profile[high_idx + 1] if expand_high else 0 - - if low_vol >= high_vol and expand_low: - low_idx -= 1 - accumulated += volume_profile[low_idx] - elif expand_high: - high_idx += 1 - accumulated += volume_profile[high_idx] - else: - low_idx -= 1 - accumulated += volume_profile[low_idx] - - va_low = bin_edges[low_idx] - va_high = bin_edges[high_idx + 1] - - return poc, va_low, va_high - - def on_candle(self, candle: Candle) -> Signal | None: - self._candles.append((float(candle.close), float(candle.volume))) - - result = self._compute_value_area() - if result is None: - return None - - poc, va_low, va_high = result - price = float(candle.close) - - # Track value area penetration - if price < va_low: - self._was_below_va = True - self._was_above_va = False - elif price > va_high: - self._was_above_va = True - self._was_below_va = False - - # BUY: price was below VA and bounced back to VA low (support) - if self._was_below_va and price >= va_low and price <= poc: - self._was_below_va = False - return Signal( - strategy=self.name, - symbol=candle.symbol, - side=OrderSide.BUY, - price=candle.close, - quantity=self._quantity, - reason=f"Volume Profile: bounce at VA low ({va_low:.2f}), POC={poc:.2f}", - ) - - # SELL: price was above VA and pulled back to VA high (resistance) - if self._was_above_va and price <= va_high and price >= poc: - self._was_above_va = False - return Signal( - strategy=self.name, - symbol=candle.symbol, - side=OrderSide.SELL, - price=candle.close, - quantity=self._quantity, - reason=f"Volume Profile: rejection at VA high ({va_high:.2f}), POC={poc:.2f}", - ) - - return None -``` - -Create `services/strategy-engine/strategies/config/volume_profile_strategy.yaml`: - -```yaml -lookback_period: 100 -num_bins: 50 -value_area_pct: 0.7 -quantity: "0.01" -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `pytest services/strategy-engine/tests/test_volume_profile_strategy.py -v` -Expected: All 5 tests PASS - -- [ ] **Step 5: Commit** - -```bash -git add services/strategy-engine/strategies/volume_profile_strategy.py \ - services/strategy-engine/strategies/config/volume_profile_strategy.yaml \ - services/strategy-engine/tests/test_volume_profile_strategy.py -git commit -m "feat(strategy): add Volume Profile strategy" -``` - ---- - -## Task 14: Backtest Detailed Metrics - -**Files:** -- Create: `services/backtester/src/backtester/metrics.py` -- Test: `services/backtester/tests/test_metrics.py` - -- [ ] **Step 1: Write the failing test** - -Create `services/backtester/tests/test_metrics.py`: - -```python -"""Tests for detailed backtest metrics.""" -from datetime import datetime, timedelta, timezone -from decimal import Decimal - -import pytest - -from backtester.metrics import TradeRecord, compute_detailed_metrics - - -def _trade(entry_price: float, exit_price: float, qty: float = 1.0, days: int = 1) -> tuple[TradeRecord, TradeRecord]: - entry_time = datetime(2025, 1, 1, tzinfo=timezone.utc) - exit_time = entry_time + timedelta(days=days) - entry = TradeRecord( - time=entry_time, - symbol="BTCUSDT", - side="BUY", - price=Decimal(str(entry_price)), - quantity=Decimal(str(qty)), - ) - exit_rec = TradeRecord( - time=exit_time, - symbol="BTCUSDT", - side="SELL", - price=Decimal(str(exit_price)), - quantity=Decimal(str(qty)), - ) - return entry, exit_rec - - -def test_compute_metrics_basic(): - trades = [] - e1, x1 = _trade(100, 110) # +10 profit - e2, x2 = _trade(100, 95) # -5 loss - trades = [e1, x1, e2, x2] - - metrics = compute_detailed_metrics( - trades=trades, - initial_balance=Decimal("1000"), - final_balance=Decimal("1005"), - ) - - assert metrics.total_trades == 4 - assert metrics.winning_trades == 1 - assert metrics.losing_trades == 1 - assert metrics.win_rate == pytest.approx(50.0, rel=0.01) - assert metrics.total_return == pytest.approx(0.5, rel=0.01) - - -def test_compute_metrics_profit_factor(): - e1, x1 = _trade(100, 120) # +20 - e2, x2 = _trade(100, 90) # -10 - trades = [e1, x1, e2, x2] - - metrics = compute_detailed_metrics( - trades=trades, - initial_balance=Decimal("1000"), - final_balance=Decimal("1010"), - ) - - assert metrics.profit_factor == pytest.approx(2.0, rel=0.01) - - -def test_compute_metrics_max_drawdown(): - # Three trades: +10, -20, +5 => peak 1010, trough 990 - e1, x1 = _trade(100, 110) - e2, x2 = _trade(100, 80) - e3, x3 = _trade(100, 105) - trades = [e1, x1, e2, x2, e3, x3] - - metrics = compute_detailed_metrics( - trades=trades, - initial_balance=Decimal("1000"), - final_balance=Decimal("995"), - ) - - assert metrics.max_drawdown > 0 - - -def test_compute_metrics_sharpe_ratio(): - e1, x1 = _trade(100, 110, days=1) - e2, x2 = _trade(100, 105, days=1) - trades = [e1, x1, e2, x2] - - metrics = compute_detailed_metrics( - trades=trades, - initial_balance=Decimal("1000"), - final_balance=Decimal("1015"), - ) - - # Sharpe should be a finite number - assert metrics.sharpe_ratio != 0 or metrics.sharpe_ratio == 0 - - -def test_compute_metrics_empty_trades(): - metrics = compute_detailed_metrics( - trades=[], - initial_balance=Decimal("1000"), - final_balance=Decimal("1000"), - ) - assert metrics.total_trades == 0 - assert metrics.win_rate == 0.0 - assert metrics.sharpe_ratio == 0.0 -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `pytest services/backtester/tests/test_metrics.py -v` -Expected: FAIL with `ModuleNotFoundError: No module named 'backtester.metrics'` - -- [ ] **Step 3: Implement detailed metrics** - -Create `services/backtester/src/backtester/metrics.py`: - -```python -"""Detailed backtest metrics: Sharpe, Sortino, Calmar, drawdown, trade analysis.""" -import math -from dataclasses import dataclass, field -from datetime import datetime, timedelta -from decimal import Decimal - - -@dataclass -class TradeRecord: - time: datetime - symbol: str - side: str - price: Decimal - quantity: Decimal - - -@dataclass -class DetailedMetrics: - # Basic - total_return: float - total_trades: int - winning_trades: int - losing_trades: int - win_rate: float - profit_factor: float - - # Risk - sharpe_ratio: float - sortino_ratio: float - calmar_ratio: float - max_drawdown: float - max_drawdown_duration: timedelta - - # Returns - monthly_returns: dict[str, float] - avg_win: float - avg_loss: float - largest_win: float - largest_loss: float - avg_holding_period: timedelta - - # Individual trades - trade_pairs: list[dict] = field(default_factory=list) - - -def compute_detailed_metrics( - trades: list[TradeRecord], - initial_balance: Decimal, - final_balance: Decimal, -) -> DetailedMetrics: - """Compute detailed metrics from a list of trade records.""" - initial = float(initial_balance) - final = float(final_balance) - - total_return = ((final - initial) / initial * 100) if initial > 0 else 0.0 - - if not trades: - return DetailedMetrics( - total_return=total_return, - total_trades=0, - winning_trades=0, - losing_trades=0, - win_rate=0.0, - profit_factor=0.0, - sharpe_ratio=0.0, - sortino_ratio=0.0, - calmar_ratio=0.0, - max_drawdown=0.0, - max_drawdown_duration=timedelta(0), - monthly_returns={}, - avg_win=0.0, - avg_loss=0.0, - largest_win=0.0, - largest_loss=0.0, - avg_holding_period=timedelta(0), - trade_pairs=[], - ) - - # Pair up BUY/SELL trades - buys: list[TradeRecord] = [] - pairs: list[dict] = [] - pnls: list[float] = [] - holding_periods: list[timedelta] = [] - - for trade in trades: - if trade.side == "BUY": - buys.append(trade) - elif trade.side == "SELL" and buys: - buy = buys.pop(0) - pnl = float(trade.price - buy.price) * float(trade.quantity) - pnls.append(pnl) - holding = trade.time - buy.time - holding_periods.append(holding) - pairs.append({ - "entry_time": buy.time.isoformat(), - "exit_time": trade.time.isoformat(), - "entry_price": float(buy.price), - "exit_price": float(trade.price), - "quantity": float(trade.quantity), - "pnl": pnl, - "pnl_pct": (pnl / (float(buy.price) * float(trade.quantity))) * 100 if float(buy.price) > 0 else 0, - "holding_period": str(holding), - }) - - wins = [p for p in pnls if p > 0] - losses = [p for p in pnls if p < 0] - - winning_trades = len(wins) - losing_trades = len(losses) - win_rate = (winning_trades / len(pnls) * 100) if pnls else 0.0 - - gross_profit = sum(wins) if wins else 0.0 - gross_loss = abs(sum(losses)) if losses else 0.0 - profit_factor = (gross_profit / gross_loss) if gross_loss > 0 else 0.0 - - avg_win = (sum(wins) / len(wins)) if wins else 0.0 - avg_loss = (sum(losses) / len(losses)) if losses else 0.0 - largest_win = max(wins) if wins else 0.0 - largest_loss = min(losses) if losses else 0.0 - avg_holding = ( - sum(holding_periods, timedelta(0)) / len(holding_periods) - if holding_periods - else timedelta(0) - ) - - # Equity curve for drawdown and ratios - equity = [initial] - for pnl in pnls: - equity.append(equity[-1] + pnl) - - # Max drawdown - peak = equity[0] - max_dd = 0.0 - dd_start = 0 - max_dd_duration = timedelta(0) - current_dd_start = 0 - - for i, val in enumerate(equity): - if val > peak: - peak = val - current_dd_start = i - dd = (peak - val) / peak if peak > 0 else 0 - if dd > max_dd: - max_dd = dd - - # Daily returns approximation (per-trade returns) - returns = [] - for i in range(1, len(equity)): - if equity[i - 1] > 0: - returns.append((equity[i] - equity[i - 1]) / equity[i - 1]) - - # Sharpe ratio (annualized for crypto: 365 days) - if returns and len(returns) > 1: - mean_ret = sum(returns) / len(returns) - std_ret = math.sqrt(sum((r - mean_ret) ** 2 for r in returns) / (len(returns) - 1)) - sharpe = (mean_ret / std_ret * math.sqrt(365)) if std_ret > 0 else 0.0 - - # Sortino ratio (downside deviation only) - downside = [r for r in returns if r < 0] - if downside: - downside_std = math.sqrt(sum(r**2 for r in downside) / len(downside)) - sortino = (mean_ret / downside_std * math.sqrt(365)) if downside_std > 0 else 0.0 - else: - sortino = 0.0 - else: - sharpe = 0.0 - sortino = 0.0 - - # Calmar ratio - annualized_return = total_return / 100 # as fraction - calmar = (annualized_return / max_dd) if max_dd > 0 else 0.0 - - # Monthly returns - monthly: dict[str, float] = {} - for pair in pairs: - month_key = pair["exit_time"][:7] # YYYY-MM - monthly[month_key] = monthly.get(month_key, 0.0) + pair["pnl"] - - return DetailedMetrics( - total_return=total_return, - total_trades=len(trades), - winning_trades=winning_trades, - losing_trades=losing_trades, - win_rate=win_rate, - profit_factor=profit_factor, - sharpe_ratio=sharpe, - sortino_ratio=sortino, - calmar_ratio=calmar, - max_drawdown=max_dd * 100, # as percentage - max_drawdown_duration=max_dd_duration, - monthly_returns=monthly, - avg_win=avg_win, - avg_loss=avg_loss, - largest_win=largest_win, - largest_loss=largest_loss, - avg_holding_period=avg_holding, - trade_pairs=pairs, - ) -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `pytest services/backtester/tests/test_metrics.py -v` -Expected: All 5 tests PASS - -- [ ] **Step 5: Commit** - -```bash -git add services/backtester/src/backtester/metrics.py \ - services/backtester/tests/test_metrics.py -git commit -m "feat(backtester): add detailed metrics (Sharpe, Sortino, drawdown)" -``` - ---- - -## Task 15: Integrate Metrics into BacktestEngine + Enhanced Reporter - -**Files:** -- Modify: `services/backtester/src/backtester/simulator.py` -- Modify: `services/backtester/src/backtester/engine.py` -- Modify: `services/backtester/src/backtester/reporter.py` -- Modify: `services/backtester/tests/test_engine.py` -- Modify: `services/backtester/tests/test_reporter.py` - -- [ ] **Step 1: Update simulator to produce TradeRecords** - -Add timestamp to `SimulatedTrade` in `services/backtester/src/backtester/simulator.py`. Replace file: - -```python -"""Simulated order executor for backtesting.""" -from dataclasses import dataclass, field -from datetime import datetime, timezone -from decimal import Decimal - -from shared.models import OrderSide, Signal - - -@dataclass -class SimulatedTrade: - symbol: str - side: OrderSide - price: Decimal - quantity: Decimal - balance_after: Decimal - timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) - - -class OrderSimulator: - """Simulates order execution against a paper balance.""" - - def __init__(self, initial_balance: Decimal) -> None: - self.balance: Decimal = initial_balance - self.positions: dict[str, Decimal] = {} - self.trades: list[SimulatedTrade] = [] - - def execute(self, signal: Signal, timestamp: datetime | None = None) -> bool: - """Execute a signal. Returns True if the trade was accepted.""" - ts = timestamp or datetime.now(timezone.utc) - - if signal.side == OrderSide.BUY: - cost = signal.price * signal.quantity - if cost > self.balance: - return False - self.balance -= cost - self.positions[signal.symbol] = ( - self.positions.get(signal.symbol, Decimal("0")) + signal.quantity - ) - trade_quantity = signal.quantity - else: # SELL - current_position = self.positions.get(signal.symbol, Decimal("0")) - if current_position <= Decimal("0"): - return False - trade_quantity = min(signal.quantity, current_position) - proceeds = signal.price * trade_quantity - self.balance += proceeds - self.positions[signal.symbol] = current_position - trade_quantity - - self.trades.append( - SimulatedTrade( - symbol=signal.symbol, - side=signal.side, - price=signal.price, - quantity=trade_quantity, - balance_after=self.balance, - timestamp=ts, - ) - ) - return True -``` - -- [ ] **Step 2: Update engine to compute DetailedMetrics** - -Replace `services/backtester/src/backtester/engine.py`: - -```python -"""Backtesting engine that runs strategies against historical candle data.""" -from dataclasses import dataclass, field -from decimal import Decimal -from typing import Protocol - -from shared.models import Candle, Signal - -from backtester.metrics import DetailedMetrics, TradeRecord, compute_detailed_metrics -from backtester.simulator import OrderSimulator, SimulatedTrade - - -class StrategyProtocol(Protocol): - name: str - - def on_candle(self, candle: Candle) -> Signal | None: ... - def configure(self, params: dict) -> None: ... - def reset(self) -> None: ... - - -@dataclass -class BacktestResult: - strategy_name: str - symbol: str - total_trades: int - initial_balance: Decimal - final_balance: Decimal - profit: Decimal - profit_pct: Decimal - trades: list[SimulatedTrade] = field(default_factory=list) - detailed: DetailedMetrics | None = None - - @property - def win_rate(self) -> float: - buy_prices: list[Decimal] = [] - wins = 0 - total_pairs = 0 - - for trade in self.trades: - if trade.side.value == "BUY": - buy_prices.append(trade.price) - else: - if buy_prices: - buy_price = buy_prices.pop(0) - total_pairs += 1 - if trade.price > buy_price: - wins += 1 - - if total_pairs == 0: - return 0.0 - return wins / total_pairs * 100 - - -class BacktestEngine: - """Runs a strategy against historical candles using a simulated order executor.""" - - def __init__(self, strategy: StrategyProtocol, initial_balance: Decimal) -> None: - self._strategy = strategy - self._initial_balance = initial_balance - - def run(self, candles: list[Candle]) -> BacktestResult: - """Run the backtest over a list of candles and return a result.""" - simulator = OrderSimulator(self._initial_balance) - - for candle in candles: - signal = self._strategy.on_candle(candle) - if signal is not None: - simulator.execute(signal, timestamp=candle.open_time) - - final_balance = simulator.balance - if candles: - last_price = candles[-1].close - for symbol, qty in simulator.positions.items(): - if qty > Decimal("0"): - final_balance += qty * last_price - - profit = final_balance - self._initial_balance - if self._initial_balance != Decimal("0"): - profit_pct = (profit / self._initial_balance) * Decimal("100") - else: - profit_pct = Decimal("0") - - # Build TradeRecords for detailed metrics - trade_records = [ - TradeRecord( - time=t.timestamp, - symbol=t.symbol, - side=t.side.value, - price=t.price, - quantity=t.quantity, - ) - for t in simulator.trades - ] - - detailed = compute_detailed_metrics( - trades=trade_records, - initial_balance=self._initial_balance, - final_balance=final_balance, - ) - - return BacktestResult( - strategy_name=self._strategy.name, - symbol=candles[0].symbol if candles else "", - total_trades=len(simulator.trades), - initial_balance=self._initial_balance, - final_balance=final_balance, - profit=profit, - profit_pct=profit_pct, - trades=simulator.trades, - detailed=detailed, - ) -``` - -- [ ] **Step 3: Update reporter with rich tables and export** - -Replace `services/backtester/src/backtester/reporter.py`: - -```python -"""Report formatting for backtest results using rich tables.""" -import csv -import io -import json - -from rich.console import Console -from rich.table import Table - -from backtester.engine import BacktestResult - - -def format_report(result: BacktestResult) -> str: - """Format a backtest result into a rich text report.""" - console = Console(file=io.StringIO(), force_terminal=True) - - # Summary table - summary = Table(title="BACKTEST REPORT", show_lines=True) - summary.add_column("Metric", style="bold") - summary.add_column("Value", justify="right") - - summary.add_row("Strategy", result.strategy_name) - summary.add_row("Symbol", result.symbol) - summary.add_row("Initial Balance", f"{result.initial_balance:.2f}") - summary.add_row("Final Balance", f"{result.final_balance:.2f}") - summary.add_row("Profit/Loss", f"{result.profit:.2f}") - summary.add_row("Profit %", f"{result.profit_pct:.2f}%") - summary.add_row("Total Trades", str(result.total_trades)) - summary.add_row("Win Rate", f"{result.win_rate:.2f}%") - - if result.detailed: - d = result.detailed - summary.add_row("Sharpe Ratio", f"{d.sharpe_ratio:.3f}") - summary.add_row("Sortino Ratio", f"{d.sortino_ratio:.3f}") - summary.add_row("Calmar Ratio", f"{d.calmar_ratio:.3f}") - summary.add_row("Max Drawdown", f"{d.max_drawdown:.2f}%") - summary.add_row("Profit Factor", f"{d.profit_factor:.2f}") - summary.add_row("Avg Win", f"{d.avg_win:.2f}") - summary.add_row("Avg Loss", f"{d.avg_loss:.2f}") - summary.add_row("Largest Win", f"{d.largest_win:.2f}") - summary.add_row("Largest Loss", f"{d.largest_loss:.2f}") - summary.add_row("Avg Holding Period", str(d.avg_holding_period)) - - console.print(summary) - - # Monthly returns table - if result.detailed and result.detailed.monthly_returns: - monthly = Table(title="MONTHLY RETURNS") - monthly.add_column("Month") - monthly.add_column("PnL", justify="right") - for month, pnl in sorted(result.detailed.monthly_returns.items()): - style = "green" if pnl >= 0 else "red" - monthly.add_row(month, f"{pnl:.2f}", style=style) - console.print(monthly) - - output = console.file.getvalue() - return output - - -def export_csv(result: BacktestResult) -> str: - """Export trade pairs as CSV.""" - if not result.detailed or not result.detailed.trade_pairs: - return "" - - output = io.StringIO() - writer = csv.DictWriter( - output, - fieldnames=["entry_time", "exit_time", "entry_price", "exit_price", "quantity", "pnl", "pnl_pct", "holding_period"], - ) - writer.writeheader() - for pair in result.detailed.trade_pairs: - writer.writerow(pair) - return output.getvalue() - - -def export_json(result: BacktestResult) -> str: - """Export detailed metrics as JSON.""" - if not result.detailed: - return "{}" - - d = result.detailed - data = { - "total_return": d.total_return, - "total_trades": d.total_trades, - "winning_trades": d.winning_trades, - "losing_trades": d.losing_trades, - "win_rate": d.win_rate, - "profit_factor": d.profit_factor, - "sharpe_ratio": d.sharpe_ratio, - "sortino_ratio": d.sortino_ratio, - "calmar_ratio": d.calmar_ratio, - "max_drawdown": d.max_drawdown, - "monthly_returns": d.monthly_returns, - "avg_win": d.avg_win, - "avg_loss": d.avg_loss, - "largest_win": d.largest_win, - "largest_loss": d.largest_loss, - "trade_pairs": d.trade_pairs, - } - return json.dumps(data, indent=2, default=str) -``` - -- [ ] **Step 4: Run all backtester tests** - -Run: `pytest services/backtester/tests/ -v` -Expected: All tests PASS (existing tests may need minor updates for `timestamp` parameter) - -- [ ] **Step 5: Fix any broken tests** - -If `test_simulator.py` fails due to `timestamp` parameter, the existing tests should still work since `timestamp` defaults to `datetime.now()`. If `test_reporter.py` fails, update it to check for rich output: - -Update `services/backtester/tests/test_reporter.py`: - -```python -"""Tests for backtest report formatter.""" -from decimal import Decimal - -from backtester.engine import BacktestResult -from backtester.reporter import format_report, export_csv, export_json - - -def test_format_report_contains_key_metrics(): - result = BacktestResult( - strategy_name="rsi", - symbol="BTCUSDT", - total_trades=10, - initial_balance=Decimal("10000"), - final_balance=Decimal("10500"), - profit=Decimal("500"), - profit_pct=Decimal("5"), - ) - report = format_report(result) - assert "rsi" in report - assert "BTCUSDT" in report - assert "10000" in report or "10,000" in report - - -def test_export_csv_empty_when_no_detailed(): - result = BacktestResult( - strategy_name="rsi", - symbol="BTCUSDT", - total_trades=0, - initial_balance=Decimal("10000"), - final_balance=Decimal("10000"), - profit=Decimal("0"), - profit_pct=Decimal("0"), - ) - assert export_csv(result) == "" - - -def test_export_json_empty_when_no_detailed(): - result = BacktestResult( - strategy_name="rsi", - symbol="BTCUSDT", - total_trades=0, - initial_balance=Decimal("10000"), - final_balance=Decimal("10000"), - profit=Decimal("0"), - profit_pct=Decimal("0"), - ) - assert export_json(result) == "{}" -``` - -- [ ] **Step 6: Run all backtester tests again** - -Run: `pytest services/backtester/tests/ -v` -Expected: All tests PASS - -- [ ] **Step 7: Commit** - -```bash -git add services/backtester/src/backtester/simulator.py \ - services/backtester/src/backtester/engine.py \ - services/backtester/src/backtester/reporter.py \ - services/backtester/tests/test_engine.py \ - services/backtester/tests/test_reporter.py -git commit -m "feat(backtester): integrate detailed metrics and rich reporter" -``` - ---- - -## Task 16: Final Integration Test - -**Files:** -- All - -- [ ] **Step 1: Run the full test suite** - -Run: `pytest -v` -Expected: All tests PASS - -- [ ] **Step 2: Run linting** - -Run: `make lint` -Expected: No errors - -- [ ] **Step 3: Fix any lint issues** - -Run: `make format` if needed, then `make lint` again. - -- [ ] **Step 4: Verify plugin loader finds all 7 strategies** - -Run: `python -c "from pathlib import Path; from strategy_engine.plugin_loader import load_strategies; s = load_strategies(Path('services/strategy-engine/strategies')); print([x.name for x in s])"` -Expected: `['bollinger', 'ema_crossover', 'grid', 'macd', 'rsi', 'volume_profile', 'vwap']` - -- [ ] **Step 5: Final commit if any fixes were made** - -```bash -git add -A -git commit -m "fix: resolve lint issues and final integration fixes" -``` diff --git a/docs/superpowers/plans/2026-04-02-news-driven-stock-selector.md b/docs/superpowers/plans/2026-04-02-news-driven-stock-selector.md new file mode 100644 index 0000000..0964f21 --- /dev/null +++ b/docs/superpowers/plans/2026-04-02-news-driven-stock-selector.md @@ -0,0 +1,3689 @@ +# News-Driven Stock Selector Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the MOC strategy's fixed symbol list with a dynamic, news-driven stock selection system that continuously collects news/sentiment data and selects 2-3 optimal stocks daily before market close. + +**Architecture:** A new `news-collector` service runs 7 data source collectors on individual poll intervals, storing `NewsItem` records in PostgreSQL and publishing to Redis. A sentiment aggregator computes per-symbol composite scores every 15 minutes. Before market close, a 3-stage stock selector (sentiment candidates → technical filter → LLM final pick) chooses 2-3 stocks and feeds them to the existing MOC strategy. + +**Tech Stack:** Python 3.12+, asyncio, aiohttp, Pydantic, SQLAlchemy 2.0 async, Redis Streams, VADER (nltk), feedparser (RSS), Anthropic SDK (Claude API), Alembic + +--- + +## Phase 1: Shared Foundation (Models, DB, Events) + +### Task 1: Add NewsItem and sentiment models to shared + +**Files:** +- Modify: `shared/src/shared/models.py` +- Create: `shared/src/shared/sentiment_models.py` +- Create: `shared/tests/test_sentiment_models.py` + +- [ ] **Step 1: Write tests for new models** + +Create `shared/tests/test_sentiment_models.py`: + +```python +"""Tests for news and sentiment models.""" + +import pytest +from datetime import datetime, timezone + +from shared.models import NewsCategory, NewsItem, OrderSide +from shared.sentiment_models import SymbolScore, MarketSentiment, SelectedStock, Candidate + + +def test_news_item_defaults(): + item = NewsItem( + source="finnhub", + headline="Test headline", + published_at=datetime(2026, 4, 2, tzinfo=timezone.utc), + sentiment=0.5, + category=NewsCategory.MACRO, + ) + assert item.id # UUID generated + assert item.symbols == [] + assert item.summary is None + assert item.raw_data == {} + assert item.created_at is not None + + +def test_news_item_with_symbols(): + item = NewsItem( + source="rss", + headline="AAPL earnings beat", + published_at=datetime(2026, 4, 2, tzinfo=timezone.utc), + sentiment=0.8, + category=NewsCategory.EARNINGS, + symbols=["AAPL"], + ) + assert item.symbols == ["AAPL"] + assert item.category == NewsCategory.EARNINGS + + +def test_news_category_values(): + assert NewsCategory.POLICY == "policy" + assert NewsCategory.EARNINGS == "earnings" + assert NewsCategory.MACRO == "macro" + assert NewsCategory.SOCIAL == "social" + assert NewsCategory.FILING == "filing" + assert NewsCategory.FED == "fed" + + +def test_symbol_score(): + score = SymbolScore( + symbol="AAPL", + news_score=0.5, + news_count=10, + social_score=0.3, + policy_score=0.0, + filing_score=0.2, + composite=0.3, + updated_at=datetime(2026, 4, 2, tzinfo=timezone.utc), + ) + assert score.symbol == "AAPL" + assert score.composite == 0.3 + + +def test_market_sentiment(): + ms = MarketSentiment( + fear_greed=25, + fear_greed_label="Extreme Fear", + vix=32.5, + fed_stance="hawkish", + market_regime="risk_off", + updated_at=datetime(2026, 4, 2, tzinfo=timezone.utc), + ) + assert ms.market_regime == "risk_off" + assert ms.vix == 32.5 + + +def test_market_sentiment_no_vix(): + ms = MarketSentiment( + fear_greed=50, + fear_greed_label="Neutral", + fed_stance="neutral", + market_regime="neutral", + updated_at=datetime(2026, 4, 2, tzinfo=timezone.utc), + ) + assert ms.vix is None + + +def test_selected_stock(): + ss = SelectedStock( + symbol="NVDA", + side=OrderSide.BUY, + conviction=0.85, + reason="CHIPS Act expansion", + key_news=["Trump signs CHIPS Act expansion"], + ) + assert ss.conviction == 0.85 + assert len(ss.key_news) == 1 + + +def test_candidate(): + c = Candidate( + symbol="TSLA", + source="sentiment", + direction=OrderSide.BUY, + score=0.75, + reason="High social buzz", + ) + assert c.direction == OrderSide.BUY + + c2 = Candidate( + symbol="XOM", + source="llm", + score=0.6, + reason="Oil price surge", + ) + assert c2.direction is None +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest shared/tests/test_sentiment_models.py -v` +Expected: FAIL — `NewsCategory`, `NewsItem` not found in `shared.models`, `shared.sentiment_models` does not exist + +- [ ] **Step 3: Add NewsCategory and NewsItem to shared/models.py** + +Add to the end of `shared/src/shared/models.py`: + +```python +class NewsCategory(str, Enum): + POLICY = "policy" + EARNINGS = "earnings" + MACRO = "macro" + SOCIAL = "social" + FILING = "filing" + FED = "fed" + + +class NewsItem(BaseModel): + id: str = Field(default_factory=lambda: str(uuid.uuid4())) + source: str + headline: str + summary: Optional[str] = None + url: Optional[str] = None + published_at: datetime + symbols: list[str] = [] + sentiment: float + category: NewsCategory + raw_data: dict = {} + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) +``` + +- [ ] **Step 4: Create shared/src/shared/sentiment_models.py** + +```python +"""Sentiment scoring and stock selection models.""" + +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel + +from shared.models import OrderSide + + +class SymbolScore(BaseModel): + symbol: str + news_score: float + news_count: int + social_score: float + policy_score: float + filing_score: float + composite: float + updated_at: datetime + + +class MarketSentiment(BaseModel): + fear_greed: int + fear_greed_label: str + vix: Optional[float] = None + fed_stance: str + market_regime: str + updated_at: datetime + + +class SelectedStock(BaseModel): + symbol: str + side: OrderSide + conviction: float + reason: str + key_news: list[str] + + +class Candidate(BaseModel): + symbol: str + source: str + direction: Optional[OrderSide] = None + score: float + reason: str +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `pytest shared/tests/test_sentiment_models.py -v` +Expected: All 9 tests PASS + +- [ ] **Step 6: Commit** + +```bash +git add shared/src/shared/models.py shared/src/shared/sentiment_models.py shared/tests/test_sentiment_models.py +git commit -m "feat: add NewsItem, sentiment scoring, and stock selection models" +``` + +--- + +### Task 2: Add SQLAlchemy ORM models for news tables + +**Files:** +- Modify: `shared/src/shared/sa_models.py` +- Create: `shared/tests/test_sa_news_models.py` + +- [ ] **Step 1: Write tests for new SA models** + +Create `shared/tests/test_sa_news_models.py`: + +```python +"""Tests for news-related SQLAlchemy models.""" + +from shared.sa_models import NewsItemRow, SymbolScoreRow, MarketSentimentRow, StockSelectionRow + + +def test_news_item_row_tablename(): + assert NewsItemRow.__tablename__ == "news_items" + + +def test_symbol_score_row_tablename(): + assert SymbolScoreRow.__tablename__ == "symbol_scores" + + +def test_market_sentiment_row_tablename(): + assert MarketSentimentRow.__tablename__ == "market_sentiment" + + +def test_stock_selection_row_tablename(): + assert StockSelectionRow.__tablename__ == "stock_selections" + + +def test_news_item_row_columns(): + cols = {c.name for c in NewsItemRow.__table__.columns} + assert cols >= {"id", "source", "headline", "published_at", "sentiment", "category"} + + +def test_symbol_score_row_columns(): + cols = {c.name for c in SymbolScoreRow.__table__.columns} + assert cols >= {"id", "symbol", "news_score", "composite", "updated_at"} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest shared/tests/test_sa_news_models.py -v` +Expected: FAIL — import errors + +- [ ] **Step 3: Add ORM models to sa_models.py** + +Add to the end of `shared/src/shared/sa_models.py`: + +```python +class NewsItemRow(Base): + __tablename__ = "news_items" + + id: Mapped[str] = mapped_column(Text, primary_key=True) + source: Mapped[str] = mapped_column(Text, nullable=False) + headline: Mapped[str] = mapped_column(Text, nullable=False) + summary: Mapped[str | None] = mapped_column(Text) + url: Mapped[str | None] = mapped_column(Text) + published_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + symbols: Mapped[str | None] = mapped_column(Text) # JSON-encoded list + sentiment: Mapped[float] = mapped_column(sa.Float, nullable=False) + category: Mapped[str] = mapped_column(Text, nullable=False) + raw_data: Mapped[str | None] = mapped_column(Text) # JSON string + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=sa.func.now() + ) + + +class SymbolScoreRow(Base): + __tablename__ = "symbol_scores" + + id: Mapped[str] = mapped_column(Text, primary_key=True) + symbol: Mapped[str] = mapped_column(Text, nullable=False, unique=True) + news_score: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default="0") + news_count: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default="0") + social_score: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default="0") + policy_score: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default="0") + filing_score: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default="0") + composite: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default="0") + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class MarketSentimentRow(Base): + __tablename__ = "market_sentiment" + + id: Mapped[str] = mapped_column(Text, primary_key=True) + fear_greed: Mapped[int] = mapped_column(sa.Integer, nullable=False) + fear_greed_label: Mapped[str] = mapped_column(Text, nullable=False) + vix: Mapped[float | None] = mapped_column(sa.Float) + fed_stance: Mapped[str] = mapped_column(Text, nullable=False, server_default="neutral") + market_regime: Mapped[str] = mapped_column(Text, nullable=False, server_default="neutral") + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class StockSelectionRow(Base): + __tablename__ = "stock_selections" + + id: Mapped[str] = mapped_column(Text, primary_key=True) + trade_date: Mapped[datetime] = mapped_column(sa.Date, nullable=False) + symbol: Mapped[str] = mapped_column(Text, nullable=False) + side: Mapped[str] = mapped_column(Text, nullable=False) + conviction: Mapped[float] = mapped_column(sa.Float, nullable=False) + reason: Mapped[str] = mapped_column(Text, nullable=False) + key_news: Mapped[str | None] = mapped_column(Text) # JSON string + sentiment_snapshot: Mapped[str | None] = mapped_column(Text) # JSON string + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=sa.func.now() + ) +``` + +Also add `import sqlalchemy as sa` to the imports at the top of `sa_models.py`. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pytest shared/tests/test_sa_news_models.py -v` +Expected: All 6 tests PASS + +- [ ] **Step 5: Commit** + +```bash +git add shared/src/shared/sa_models.py shared/tests/test_sa_news_models.py +git commit -m "feat: add SQLAlchemy ORM models for news, scores, selections" +``` + +--- + +### Task 3: Create Alembic migration for news tables + +**Files:** +- Create: `shared/alembic/versions/002_news_sentiment_tables.py` + +- [ ] **Step 1: Create migration file** + +Create `shared/alembic/versions/002_news_sentiment_tables.py`: + +```python +"""Add news, sentiment, and stock selection tables + +Revision ID: 002 +Revises: 001 +Create Date: 2026-04-02 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "002" +down_revision: Union[str, None] = "001" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "news_items", + sa.Column("id", sa.Text, primary_key=True), + sa.Column("source", sa.Text, nullable=False), + sa.Column("headline", sa.Text, nullable=False), + sa.Column("summary", sa.Text), + sa.Column("url", sa.Text), + sa.Column("published_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("symbols", sa.Text), + sa.Column("sentiment", sa.Float, nullable=False), + sa.Column("category", sa.Text, nullable=False), + sa.Column("raw_data", sa.Text), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + op.create_index("idx_news_items_published", "news_items", ["published_at"]) + op.create_index("idx_news_items_source", "news_items", ["source"]) + + op.create_table( + "symbol_scores", + sa.Column("id", sa.Text, primary_key=True), + sa.Column("symbol", sa.Text, nullable=False, unique=True), + sa.Column("news_score", sa.Float, nullable=False, server_default="0"), + sa.Column("news_count", sa.Integer, nullable=False, server_default="0"), + sa.Column("social_score", sa.Float, nullable=False, server_default="0"), + sa.Column("policy_score", sa.Float, nullable=False, server_default="0"), + sa.Column("filing_score", sa.Float, nullable=False, server_default="0"), + sa.Column("composite", sa.Float, nullable=False, server_default="0"), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + ) + + op.create_table( + "market_sentiment", + sa.Column("id", sa.Text, primary_key=True), + sa.Column("fear_greed", sa.Integer, nullable=False), + sa.Column("fear_greed_label", sa.Text, nullable=False), + sa.Column("vix", sa.Float), + sa.Column("fed_stance", sa.Text, nullable=False, server_default="neutral"), + sa.Column("market_regime", sa.Text, nullable=False, server_default="neutral"), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + ) + + op.create_table( + "stock_selections", + sa.Column("id", sa.Text, primary_key=True), + sa.Column("trade_date", sa.Date, nullable=False), + sa.Column("symbol", sa.Text, nullable=False), + sa.Column("side", sa.Text, nullable=False), + sa.Column("conviction", sa.Float, nullable=False), + sa.Column("reason", sa.Text, nullable=False), + sa.Column("key_news", sa.Text), + sa.Column("sentiment_snapshot", sa.Text), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + op.create_index("idx_stock_selections_date", "stock_selections", ["trade_date"]) + + +def downgrade() -> None: + op.drop_table("stock_selections") + op.drop_table("market_sentiment") + op.drop_table("symbol_scores") + op.drop_table("news_items") +``` + +- [ ] **Step 2: Verify migration imports correctly** + +Run: `cd shared && python -c "from alembic.versions import *; print('OK')" && cd ..` +Or simply: `python -c "import importlib.util; s=importlib.util.spec_from_file_location('m','shared/alembic/versions/002_news_sentiment_tables.py'); m=importlib.util.module_from_spec(s); s.loader.exec_module(m); print('OK')"` +Expected: OK (no import errors) + +- [ ] **Step 3: Commit** + +```bash +git add shared/alembic/versions/002_news_sentiment_tables.py +git commit -m "feat: add Alembic migration for news and sentiment tables" +``` + +--- + +### Task 4: Add NewsEvent to shared events and DB methods for news + +**Files:** +- Modify: `shared/src/shared/events.py` +- Modify: `shared/src/shared/db.py` +- Create: `shared/tests/test_news_events.py` +- Create: `shared/tests/test_db_news.py` + +- [ ] **Step 1: Write tests for NewsEvent** + +Create `shared/tests/test_news_events.py`: + +```python +"""Tests for NewsEvent.""" + +from datetime import datetime, timezone + +from shared.models import NewsCategory, NewsItem +from shared.events import NewsEvent, EventType, Event + + +def test_news_event_to_dict(): + item = NewsItem( + source="finnhub", + headline="Test", + published_at=datetime(2026, 4, 2, tzinfo=timezone.utc), + sentiment=0.5, + category=NewsCategory.MACRO, + ) + event = NewsEvent(data=item) + d = event.to_dict() + assert d["type"] == EventType.NEWS + assert d["data"]["source"] == "finnhub" + + +def test_news_event_from_raw(): + raw = { + "type": "NEWS", + "data": { + "id": "abc", + "source": "rss", + "headline": "Test headline", + "published_at": "2026-04-02T00:00:00+00:00", + "sentiment": 0.3, + "category": "earnings", + "symbols": ["AAPL"], + "raw_data": {}, + }, + } + event = NewsEvent.from_raw(raw) + assert event.data.source == "rss" + assert event.data.symbols == ["AAPL"] + + +def test_event_dispatcher_news(): + raw = { + "type": "NEWS", + "data": { + "id": "abc", + "source": "finnhub", + "headline": "Test", + "published_at": "2026-04-02T00:00:00+00:00", + "sentiment": 0.0, + "category": "macro", + "raw_data": {}, + }, + } + event = Event.from_dict(raw) + assert isinstance(event, NewsEvent) +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest shared/tests/test_news_events.py -v` +Expected: FAIL — `NewsEvent` not in events.py, `EventType.NEWS` missing + +- [ ] **Step 3: Add NewsEvent to events.py** + +Add `NEWS = "NEWS"` to `EventType` enum. + +Add `NewsEvent` class and register it in `_EVENT_TYPE_MAP`: + +```python +from shared.models import Candle, Signal, Order, NewsItem + +class NewsEvent(BaseModel): + type: EventType = EventType.NEWS + data: NewsItem + + def to_dict(self) -> dict: + return { + "type": self.type, + "data": self.data.model_dump(mode="json"), + } + + @classmethod + def from_raw(cls, raw: dict) -> "NewsEvent": + return cls(type=raw["type"], data=NewsItem(**raw["data"])) +``` + +Add to `_EVENT_TYPE_MAP`: +```python +EventType.NEWS: NewsEvent, +``` + +- [ ] **Step 4: Run event tests to verify they pass** + +Run: `pytest shared/tests/test_news_events.py -v` +Expected: All 3 tests PASS + +- [ ] **Step 5: Run all existing event tests to check no regressions** + +Run: `pytest shared/tests/test_events.py -v` +Expected: All existing tests PASS + +- [ ] **Step 6: Write tests for DB news methods** + +Create `shared/tests/test_db_news.py`: + +```python +"""Tests for database news/sentiment methods. + +These tests use an in-memory SQLite database. +""" + +import json +import uuid +import pytest +from datetime import datetime, date, timezone + +from shared.db import Database +from shared.models import NewsItem, NewsCategory +from shared.sentiment_models import SymbolScore, MarketSentiment + + +@pytest.fixture +async def db(): + database = Database("sqlite+aiosqlite://") + await database.connect() + yield database + await database.close() + + +async def test_insert_and_get_news_items(db): + item = NewsItem( + source="finnhub", + headline="AAPL earnings beat", + published_at=datetime(2026, 4, 2, 12, 0, tzinfo=timezone.utc), + sentiment=0.8, + category=NewsCategory.EARNINGS, + symbols=["AAPL"], + ) + await db.insert_news_item(item) + items = await db.get_recent_news(hours=24) + assert len(items) == 1 + assert items[0]["headline"] == "AAPL earnings beat" + + +async def test_upsert_symbol_score(db): + score = SymbolScore( + symbol="AAPL", + news_score=0.5, + news_count=10, + social_score=0.3, + policy_score=0.0, + filing_score=0.2, + composite=0.3, + updated_at=datetime(2026, 4, 2, tzinfo=timezone.utc), + ) + await db.upsert_symbol_score(score) + scores = await db.get_top_symbol_scores(limit=5) + assert len(scores) == 1 + assert scores[0]["symbol"] == "AAPL" + + +async def test_upsert_market_sentiment(db): + ms = MarketSentiment( + fear_greed=55, + fear_greed_label="Neutral", + vix=18.2, + fed_stance="neutral", + market_regime="neutral", + updated_at=datetime(2026, 4, 2, tzinfo=timezone.utc), + ) + await db.upsert_market_sentiment(ms) + result = await db.get_latest_market_sentiment() + assert result is not None + assert result["fear_greed"] == 55 + + +async def test_insert_stock_selection(db): + await db.insert_stock_selection( + trade_date=date(2026, 4, 2), + symbol="NVDA", + side="BUY", + conviction=0.85, + reason="CHIPS Act", + key_news=["Trump signs CHIPS expansion"], + sentiment_snapshot={"composite": 0.8}, + ) + selections = await db.get_stock_selections(date(2026, 4, 2)) + assert len(selections) == 1 + assert selections[0]["symbol"] == "NVDA" +``` + +- [ ] **Step 7: Run DB news tests to verify they fail** + +Run: `pytest shared/tests/test_db_news.py -v` +Expected: FAIL — methods not yet on Database class + +- [ ] **Step 8: Add news/sentiment DB methods to db.py** + +Add to `shared/src/shared/db.py` — new import at top: + +```python +import json +import uuid +from datetime import date +from shared.models import NewsItem +from shared.sentiment_models import SymbolScore, MarketSentiment +from shared.sa_models import NewsItemRow, SymbolScoreRow, MarketSentimentRow, StockSelectionRow +``` + +Add these methods to the `Database` class: + +```python + async def insert_news_item(self, item: NewsItem) -> None: + """Insert a news item.""" + row = NewsItemRow( + id=item.id, + source=item.source, + headline=item.headline, + summary=item.summary, + url=item.url, + published_at=item.published_at, + symbols=json.dumps(item.symbols), + sentiment=item.sentiment, + category=item.category.value, + raw_data=json.dumps(item.raw_data), + created_at=item.created_at, + ) + async with self._session_factory() as session: + try: + session.add(row) + await session.commit() + except Exception: + await session.rollback() + raise + + async def get_recent_news(self, hours: int = 24) -> list[dict]: + """Retrieve news items from the last N hours.""" + since = datetime.now(timezone.utc) - timedelta(hours=hours) + stmt = ( + select(NewsItemRow) + .where(NewsItemRow.published_at >= since) + .order_by(NewsItemRow.published_at.desc()) + ) + async with self._session_factory() as session: + result = await session.execute(stmt) + rows = result.scalars().all() + return [ + { + "id": r.id, + "source": r.source, + "headline": r.headline, + "summary": r.summary, + "url": r.url, + "published_at": r.published_at, + "symbols": json.loads(r.symbols) if r.symbols else [], + "sentiment": r.sentiment, + "category": r.category, + "created_at": r.created_at, + } + for r in rows + ] + + async def upsert_symbol_score(self, score: SymbolScore) -> None: + """Insert or update a symbol score.""" + async with self._session_factory() as session: + try: + existing = await session.execute( + select(SymbolScoreRow).where(SymbolScoreRow.symbol == score.symbol) + ) + row = existing.scalar_one_or_none() + if row: + row.news_score = score.news_score + row.news_count = score.news_count + row.social_score = score.social_score + row.policy_score = score.policy_score + row.filing_score = score.filing_score + row.composite = score.composite + row.updated_at = score.updated_at + else: + row = SymbolScoreRow( + id=str(uuid.uuid4()), + symbol=score.symbol, + news_score=score.news_score, + news_count=score.news_count, + social_score=score.social_score, + policy_score=score.policy_score, + filing_score=score.filing_score, + composite=score.composite, + updated_at=score.updated_at, + ) + session.add(row) + await session.commit() + except Exception: + await session.rollback() + raise + + async def get_top_symbol_scores(self, limit: int = 20) -> list[dict]: + """Get top symbol scores ordered by composite descending.""" + stmt = ( + select(SymbolScoreRow) + .order_by(SymbolScoreRow.composite.desc()) + .limit(limit) + ) + async with self._session_factory() as session: + result = await session.execute(stmt) + rows = result.scalars().all() + return [ + { + "symbol": r.symbol, + "news_score": r.news_score, + "news_count": r.news_count, + "social_score": r.social_score, + "policy_score": r.policy_score, + "filing_score": r.filing_score, + "composite": r.composite, + "updated_at": r.updated_at, + } + for r in rows + ] + + async def upsert_market_sentiment(self, ms: MarketSentiment) -> None: + """Insert or update the latest market sentiment (single row, id='latest').""" + async with self._session_factory() as session: + try: + existing = await session.execute( + select(MarketSentimentRow).where(MarketSentimentRow.id == "latest") + ) + row = existing.scalar_one_or_none() + if row: + row.fear_greed = ms.fear_greed + row.fear_greed_label = ms.fear_greed_label + row.vix = ms.vix + row.fed_stance = ms.fed_stance + row.market_regime = ms.market_regime + row.updated_at = ms.updated_at + else: + row = MarketSentimentRow( + id="latest", + fear_greed=ms.fear_greed, + fear_greed_label=ms.fear_greed_label, + vix=ms.vix, + fed_stance=ms.fed_stance, + market_regime=ms.market_regime, + updated_at=ms.updated_at, + ) + session.add(row) + await session.commit() + except Exception: + await session.rollback() + raise + + async def get_latest_market_sentiment(self) -> dict | None: + """Get the latest market sentiment.""" + stmt = select(MarketSentimentRow).where(MarketSentimentRow.id == "latest") + async with self._session_factory() as session: + result = await session.execute(stmt) + r = result.scalar_one_or_none() + if r is None: + return None + return { + "fear_greed": r.fear_greed, + "fear_greed_label": r.fear_greed_label, + "vix": r.vix, + "fed_stance": r.fed_stance, + "market_regime": r.market_regime, + "updated_at": r.updated_at, + } + + async def insert_stock_selection( + self, + trade_date: date, + symbol: str, + side: str, + conviction: float, + reason: str, + key_news: list[str], + sentiment_snapshot: dict, + ) -> None: + """Insert a stock selection record.""" + row = StockSelectionRow( + id=str(uuid.uuid4()), + trade_date=trade_date, + symbol=symbol, + side=side, + conviction=conviction, + reason=reason, + key_news=json.dumps(key_news), + sentiment_snapshot=json.dumps(sentiment_snapshot), + ) + async with self._session_factory() as session: + try: + session.add(row) + await session.commit() + except Exception: + await session.rollback() + raise + + async def get_stock_selections(self, trade_date: date) -> list[dict]: + """Get stock selections for a specific date.""" + stmt = ( + select(StockSelectionRow) + .where(StockSelectionRow.trade_date == trade_date) + .order_by(StockSelectionRow.conviction.desc()) + ) + async with self._session_factory() as session: + result = await session.execute(stmt) + rows = result.scalars().all() + return [ + { + "symbol": r.symbol, + "side": r.side, + "conviction": r.conviction, + "reason": r.reason, + "key_news": json.loads(r.key_news) if r.key_news else [], + "sentiment_snapshot": json.loads(r.sentiment_snapshot) if r.sentiment_snapshot else {}, + } + for r in rows + ] +``` + +- [ ] **Step 9: Run DB news tests to verify they pass** + +Run: `pytest shared/tests/test_db_news.py -v` +Expected: All 4 tests PASS + +Note: These tests require `aiosqlite` package. If not installed: `pip install aiosqlite` + +- [ ] **Step 10: Run all shared tests to check no regressions** + +Run: `pytest shared/tests/ -v` +Expected: All tests PASS + +- [ ] **Step 11: Commit** + +```bash +git add shared/src/shared/events.py shared/src/shared/db.py shared/tests/test_news_events.py shared/tests/test_db_news.py +git commit -m "feat: add NewsEvent, DB methods for news/sentiment/selections" +``` + +--- + +### Task 5: Update Settings with new env vars + +**Files:** +- Modify: `shared/src/shared/config.py` +- Modify: `.env.example` + +- [ ] **Step 1: Add new settings to config.py** + +Add these fields to the `Settings` class in `shared/src/shared/config.py`: + +```python + # News collector + finnhub_api_key: str = "" + news_poll_interval: int = 300 + sentiment_aggregate_interval: int = 900 + # Stock selector + selector_candidates_time: str = "15:00" + selector_filter_time: str = "15:15" + selector_final_time: str = "15:30" + selector_max_picks: int = 3 + # LLM + anthropic_api_key: str = "" + anthropic_model: str = "claude-sonnet-4-20250514" +``` + +- [ ] **Step 2: Add to .env.example** + +Append to `.env.example`: + +```bash + +# News Collector +FINNHUB_API_KEY= +NEWS_POLL_INTERVAL=300 +SENTIMENT_AGGREGATE_INTERVAL=900 + +# Stock Selector +SELECTOR_CANDIDATES_TIME=15:00 +SELECTOR_FILTER_TIME=15:15 +SELECTOR_FINAL_TIME=15:30 +SELECTOR_MAX_PICKS=3 + +# LLM (for stock selector) +ANTHROPIC_API_KEY= +ANTHROPIC_MODEL=claude-sonnet-4-20250514 +``` + +- [ ] **Step 3: Commit** + +```bash +git add shared/src/shared/config.py .env.example +git commit -m "feat: add news collector and stock selector config settings" +``` + +--- + +## Phase 2: News Collector Service + +### Task 6: Scaffold news-collector service + +**Files:** +- Create: `services/news-collector/pyproject.toml` +- Create: `services/news-collector/Dockerfile` +- Create: `services/news-collector/src/news_collector/__init__.py` +- Create: `services/news-collector/src/news_collector/config.py` +- Create: `services/news-collector/src/news_collector/collectors/__init__.py` +- Create: `services/news-collector/src/news_collector/collectors/base.py` +- Create: `services/news-collector/tests/__init__.py` + +- [ ] **Step 1: Create pyproject.toml** + +Create `services/news-collector/pyproject.toml`: + +```toml +[project] +name = "news-collector" +version = "0.1.0" +description = "News and sentiment data collector service" +requires-python = ">=3.12" +dependencies = [ + "trading-shared", + "feedparser>=6.0", + "nltk>=3.8", + "aiohttp>=3.9", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "pytest-asyncio>=0.23", + "aioresponses>=0.7", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/news_collector"] +``` + +- [ ] **Step 2: Create Dockerfile** + +Create `services/news-collector/Dockerfile`: + +```dockerfile +FROM python:3.12-slim +WORKDIR /app +COPY shared/ shared/ +RUN pip install --no-cache-dir ./shared +COPY services/news-collector/ services/news-collector/ +RUN pip install --no-cache-dir ./services/news-collector +RUN python -c "import nltk; nltk.download('vader_lexicon', quiet=True)" +ENV PYTHONPATH=/app +CMD ["python", "-m", "news_collector.main"] +``` + +- [ ] **Step 3: Create config.py** + +Create `services/news-collector/src/news_collector/config.py`: + +```python +"""News Collector configuration.""" + +from shared.config import Settings + + +class NewsCollectorConfig(Settings): + health_port: int = 8084 + finnhub_api_key: str = "" + news_poll_interval: int = 300 + sentiment_aggregate_interval: int = 900 +``` + +- [ ] **Step 4: Create BaseCollector** + +Create `services/news-collector/src/news_collector/collectors/base.py`: + +```python +"""Base class for all news collectors.""" + +from abc import ABC, abstractmethod + +from shared.models import NewsItem + + +class BaseCollector(ABC): + name: str = "base" + poll_interval: int = 300 # seconds + + @abstractmethod + async def collect(self) -> list[NewsItem]: + """Collect news items from the source.""" + + @abstractmethod + async def is_available(self) -> bool: + """Check if this data source is accessible.""" +``` + +- [ ] **Step 5: Create __init__.py files** + +Create `services/news-collector/src/news_collector/__init__.py`: +```python +"""News collector service.""" +``` + +Create `services/news-collector/src/news_collector/collectors/__init__.py`: +```python +"""News collectors.""" +``` + +Create `services/news-collector/tests/__init__.py`: +```python +``` + +- [ ] **Step 6: Commit** + +```bash +git add services/news-collector/ +git commit -m "feat: scaffold news-collector service with BaseCollector" +``` + +--- + +### Task 7: Implement Finnhub news collector + +**Files:** +- Create: `services/news-collector/src/news_collector/collectors/finnhub.py` +- Create: `services/news-collector/tests/test_finnhub.py` + +- [ ] **Step 1: Write tests** + +Create `services/news-collector/tests/test_finnhub.py`: + +```python +"""Tests for Finnhub news collector.""" + +import pytest +from unittest.mock import AsyncMock, patch +from datetime import datetime, timezone + +from news_collector.collectors.finnhub import FinnhubCollector + + +@pytest.fixture +def collector(): + return FinnhubCollector(api_key="test_key") + + +def test_collector_name(collector): + assert collector.name == "finnhub" + assert collector.poll_interval == 300 + + +async def test_is_available_with_key(collector): + assert await collector.is_available() is True + + +async def test_is_available_without_key(): + c = FinnhubCollector(api_key="") + assert await c.is_available() is False + + +async def test_collect_parses_response(collector): + mock_response = [ + { + "category": "top news", + "datetime": 1711929600, + "headline": "AAPL beats earnings", + "id": 12345, + "related": "AAPL", + "source": "MarketWatch", + "summary": "Apple reported better than expected...", + "url": "https://example.com/article", + }, + { + "category": "top news", + "datetime": 1711929000, + "headline": "Fed holds rates steady", + "id": 12346, + "related": "", + "source": "Reuters", + "summary": "The Federal Reserve...", + "url": "https://example.com/fed", + }, + ] + + with patch.object(collector, "_fetch_news", new_callable=AsyncMock, return_value=mock_response): + items = await collector.collect() + + assert len(items) == 2 + assert items[0].source == "finnhub" + assert items[0].headline == "AAPL beats earnings" + assert items[0].symbols == ["AAPL"] + assert items[0].url == "https://example.com/article" + assert isinstance(items[0].sentiment, float) + # Second item has no related ticker + assert items[1].symbols == [] + + +async def test_collect_handles_empty_response(collector): + with patch.object(collector, "_fetch_news", new_callable=AsyncMock, return_value=[]): + items = await collector.collect() + assert items == [] +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest services/news-collector/tests/test_finnhub.py -v` +Expected: FAIL — module not found + +- [ ] **Step 3: Implement FinnhubCollector** + +Create `services/news-collector/src/news_collector/collectors/finnhub.py`: + +```python +"""Finnhub market news collector (free tier: 60 req/min).""" + +import logging +from datetime import datetime, timezone + +import aiohttp +from nltk.sentiment.vader import SentimentIntensityAnalyzer + +from shared.models import NewsCategory, NewsItem +from news_collector.collectors.base import BaseCollector + +logger = logging.getLogger(__name__) + +FINNHUB_NEWS_URL = "https://finnhub.io/api/v1/news" + + +class FinnhubCollector(BaseCollector): + name = "finnhub" + poll_interval = 300 # 5 minutes + + def __init__(self, api_key: str) -> None: + self._api_key = api_key + self._vader = SentimentIntensityAnalyzer() + + async def is_available(self) -> bool: + return bool(self._api_key) + + async def _fetch_news(self) -> list[dict]: + """Fetch general news from Finnhub API.""" + params = {"category": "general", "token": self._api_key} + async with aiohttp.ClientSession() as session: + async with session.get(FINNHUB_NEWS_URL, params=params) as resp: + if resp.status != 200: + logger.warning("finnhub_fetch_failed", status=resp.status) + return [] + return await resp.json() + + def _analyze_sentiment(self, text: str) -> float: + """Return VADER compound score (-1.0 to 1.0).""" + scores = self._vader.polarity_scores(text) + return scores["compound"] + + def _extract_symbols(self, related: str) -> list[str]: + """Parse Finnhub 'related' field into symbol list.""" + if not related or not related.strip(): + return [] + return [s.strip() for s in related.split(",") if s.strip()] + + def _categorize(self, article: dict) -> NewsCategory: + """Determine category from article content.""" + headline = article.get("headline", "").lower() + if any(w in headline for w in ["fed", "fomc", "rate", "inflation"]): + return NewsCategory.FED + if any(w in headline for w in ["tariff", "sanction", "regulation", "trump", "biden", "congress"]): + return NewsCategory.POLICY + if any(w in headline for w in ["earnings", "revenue", "profit", "eps"]): + return NewsCategory.EARNINGS + return NewsCategory.MACRO + + async def collect(self) -> list[NewsItem]: + raw = await self._fetch_news() + items = [] + for article in raw: + headline = article.get("headline", "") + summary = article.get("summary", "") + sentiment_text = f"{headline}. {summary}" if summary else headline + + items.append( + NewsItem( + source=self.name, + headline=headline, + summary=summary or None, + url=article.get("url"), + published_at=datetime.fromtimestamp( + article.get("datetime", 0), tz=timezone.utc + ), + symbols=self._extract_symbols(article.get("related", "")), + sentiment=self._analyze_sentiment(sentiment_text), + category=self._categorize(article), + raw_data=article, + ) + ) + return items +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pytest services/news-collector/tests/test_finnhub.py -v` +Expected: All 5 tests PASS + +Note: Requires `nltk` and VADER lexicon. If not downloaded: `python -c "import nltk; nltk.download('vader_lexicon')"` + +- [ ] **Step 5: Commit** + +```bash +git add services/news-collector/src/news_collector/collectors/finnhub.py services/news-collector/tests/test_finnhub.py +git commit -m "feat: implement Finnhub news collector with VADER sentiment" +``` + +--- + +### Task 8: Implement RSS news collector + +**Files:** +- Create: `services/news-collector/src/news_collector/collectors/rss.py` +- Create: `services/news-collector/tests/test_rss.py` + +- [ ] **Step 1: Write tests** + +Create `services/news-collector/tests/test_rss.py`: + +```python +"""Tests for RSS news collector.""" + +import pytest +from unittest.mock import AsyncMock, patch +from datetime import datetime, timezone + +from news_collector.collectors.rss import RSSCollector + + +@pytest.fixture +def collector(): + return RSSCollector() + + +def test_collector_name(collector): + assert collector.name == "rss" + assert collector.poll_interval == 600 + + +async def test_is_available(collector): + assert await collector.is_available() is True + + +async def test_collect_parses_feed(collector): + mock_feed = { + "entries": [ + { + "title": "NVDA surges on AI demand", + "link": "https://example.com/nvda", + "published_parsed": (2026, 4, 2, 12, 0, 0, 0, 0, 0), + "summary": "Nvidia stock jumped 5%...", + }, + { + "title": "Markets rally on jobs data", + "link": "https://example.com/market", + "published_parsed": (2026, 4, 2, 11, 0, 0, 0, 0, 0), + "summary": "The S&P 500 rose...", + }, + ], + } + + with patch.object(collector, "_fetch_feeds", new_callable=AsyncMock, return_value=[mock_feed]): + items = await collector.collect() + + assert len(items) == 2 + assert items[0].source == "rss" + assert items[0].headline == "NVDA surges on AI demand" + assert isinstance(items[0].sentiment, float) +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest services/news-collector/tests/test_rss.py -v` +Expected: FAIL + +- [ ] **Step 3: Implement RSSCollector** + +Create `services/news-collector/src/news_collector/collectors/rss.py`: + +```python +"""RSS feed collector for Yahoo Finance, Google News, MarketWatch.""" + +import asyncio +import logging +import re +from calendar import timegm +from datetime import datetime, timezone + +import aiohttp +import feedparser +from nltk.sentiment.vader import SentimentIntensityAnalyzer + +from shared.models import NewsCategory, NewsItem +from news_collector.collectors.base import BaseCollector + +logger = logging.getLogger(__name__) + +DEFAULT_FEEDS = [ + "https://feeds.finance.yahoo.com/rss/2.0/headline?s=^GSPC®ion=US&lang=en-US", + "https://news.google.com/rss/topics/CAAqJggKIiBDQkFTRWdvSUwyMHZNRGx6TVdZU0FtVnVHZ0pWVXlnQVAB?hl=en-US&gl=US&ceid=US:en", + "https://www.marketwatch.com/rss/topstories", +] + +# Common US stock tickers to detect in headlines +TICKER_PATTERN = re.compile( + r"\b(AAPL|MSFT|GOOGL|GOOG|AMZN|TSLA|NVDA|META|JPM|V|JNJ|WMT|PG|UNH|HD|" + r"MA|DIS|BAC|XOM|PFE|KO|PEP|CSCO|INTC|VZ|NFLX|ADBE|CRM|AMD|QCOM|" + r"GS|BA|CAT|MMM|IBM|GE|F|GM|NKE|MCD|SBUX|SPY|QQQ|IWM)\b" +) + + +class RSSCollector(BaseCollector): + name = "rss" + poll_interval = 600 # 10 minutes + + def __init__(self, feeds: list[str] | None = None) -> None: + self._feeds = feeds or DEFAULT_FEEDS + self._vader = SentimentIntensityAnalyzer() + + async def is_available(self) -> bool: + return True + + async def _fetch_feeds(self) -> list[dict]: + """Fetch and parse all RSS feeds.""" + results = [] + async with aiohttp.ClientSession() as session: + for url in self._feeds: + try: + async with session.get(url, timeout=aiohttp.ClientTimeout(total=10)) as resp: + if resp.status == 200: + text = await resp.text() + feed = feedparser.parse(text) + results.append(feed) + except Exception as exc: + logger.warning("rss_fetch_failed", url=url, error=str(exc)) + return results + + def _extract_symbols(self, text: str) -> list[str]: + """Extract stock tickers from text.""" + return list(set(TICKER_PATTERN.findall(text))) + + def _parse_time(self, entry: dict) -> datetime: + """Parse published time from feed entry.""" + parsed = entry.get("published_parsed") + if parsed: + return datetime.fromtimestamp(timegm(parsed), tz=timezone.utc) + return datetime.now(timezone.utc) + + async def collect(self) -> list[NewsItem]: + feeds = await self._fetch_feeds() + items = [] + seen_titles = set() + + for feed in feeds: + for entry in feed.get("entries", []): + title = entry.get("title", "").strip() + if not title or title in seen_titles: + continue + seen_titles.add(title) + + summary = entry.get("summary", "") + sentiment_text = f"{title}. {summary}" if summary else title + + items.append( + NewsItem( + source=self.name, + headline=title, + summary=summary or None, + url=entry.get("link"), + published_at=self._parse_time(entry), + symbols=self._extract_symbols(f"{title} {summary}"), + sentiment=self._vader.polarity_scores(sentiment_text)["compound"], + category=NewsCategory.MACRO, + raw_data={"feed_title": title}, + ) + ) + + return items +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pytest services/news-collector/tests/test_rss.py -v` +Expected: All 3 tests PASS + +- [ ] **Step 5: Commit** + +```bash +git add services/news-collector/src/news_collector/collectors/rss.py services/news-collector/tests/test_rss.py +git commit -m "feat: implement RSS news collector (Yahoo, Google News, MarketWatch)" +``` + +--- + +### Task 9: Implement Fear & Greed Index collector + +**Files:** +- Create: `services/news-collector/src/news_collector/collectors/fear_greed.py` +- Create: `services/news-collector/tests/test_fear_greed.py` + +- [ ] **Step 1: Write tests** + +Create `services/news-collector/tests/test_fear_greed.py`: + +```python +"""Tests for CNN Fear & Greed Index collector.""" + +import pytest +from unittest.mock import AsyncMock, patch + +from news_collector.collectors.fear_greed import FearGreedCollector + + +@pytest.fixture +def collector(): + return FearGreedCollector() + + +def test_collector_name(collector): + assert collector.name == "fear_greed" + assert collector.poll_interval == 3600 + + +async def test_is_available(collector): + assert await collector.is_available() is True + + +async def test_collect_parses_api_response(collector): + mock_data = { + "fear_and_greed": { + "score": 45.0, + "rating": "Fear", + "timestamp": "2026-04-02T12:00:00+00:00", + } + } + with patch.object(collector, "_fetch_index", new_callable=AsyncMock, return_value=mock_data): + result = await collector.collect() + + assert result.fear_greed == 45 + assert result.fear_greed_label == "Fear" + + +async def test_collect_returns_none_on_failure(collector): + with patch.object(collector, "_fetch_index", new_callable=AsyncMock, return_value=None): + result = await collector.collect() + assert result is None + + +def test_classify_label(): + c = FearGreedCollector() + assert c._classify(10) == "Extreme Fear" + assert c._classify(30) == "Fear" + assert c._classify(50) == "Neutral" + assert c._classify(70) == "Greed" + assert c._classify(85) == "Extreme Greed" +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest services/news-collector/tests/test_fear_greed.py -v` +Expected: FAIL + +- [ ] **Step 3: Implement FearGreedCollector** + +Create `services/news-collector/src/news_collector/collectors/fear_greed.py`: + +```python +"""CNN Fear & Greed Index collector.""" + +import logging +from dataclasses import dataclass +from typing import Optional + +import aiohttp + +from news_collector.collectors.base import BaseCollector +from shared.models import NewsItem + +logger = logging.getLogger(__name__) + +FEAR_GREED_URL = "https://production.dataviz.cnn.io/index/fearandgreed/graphdata" + + +@dataclass +class FearGreedResult: + fear_greed: int + fear_greed_label: str + + +class FearGreedCollector(BaseCollector): + """Fetches CNN Fear & Greed Index. + + Note: This collector does NOT return NewsItem — it returns FearGreedResult + which feeds directly into MarketSentiment. The main.py scheduler handles + this differently from news collectors. + """ + + name = "fear_greed" + poll_interval = 3600 # 1 hour + + async def is_available(self) -> bool: + return True + + async def _fetch_index(self) -> Optional[dict]: + """Fetch Fear & Greed data from CNN API.""" + headers = {"User-Agent": "Mozilla/5.0"} + try: + async with aiohttp.ClientSession() as session: + async with session.get( + FEAR_GREED_URL, headers=headers, timeout=aiohttp.ClientTimeout(total=10) + ) as resp: + if resp.status != 200: + logger.warning("fear_greed_fetch_failed", status=resp.status) + return None + return await resp.json() + except Exception as exc: + logger.warning("fear_greed_error", error=str(exc)) + return None + + def _classify(self, score: int) -> str: + """Classify numeric score into label.""" + if score <= 20: + return "Extreme Fear" + if score <= 40: + return "Fear" + if score <= 60: + return "Neutral" + if score <= 80: + return "Greed" + return "Extreme Greed" + + async def collect(self) -> Optional[FearGreedResult]: + """Collect Fear & Greed Index. Returns FearGreedResult or None.""" + data = await self._fetch_index() + if data is None: + return None + + try: + fg = data["fear_and_greed"] + score = int(fg["score"]) + label = fg.get("rating", self._classify(score)) + return FearGreedResult(fear_greed=score, fear_greed_label=label) + except (KeyError, ValueError, TypeError) as exc: + logger.warning("fear_greed_parse_failed", error=str(exc)) + return None +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pytest services/news-collector/tests/test_fear_greed.py -v` +Expected: All 5 tests PASS + +- [ ] **Step 5: Commit** + +```bash +git add services/news-collector/src/news_collector/collectors/fear_greed.py services/news-collector/tests/test_fear_greed.py +git commit -m "feat: implement CNN Fear & Greed Index collector" +``` + +--- + +### Task 10: Implement SEC EDGAR collector + +**Files:** +- Create: `services/news-collector/src/news_collector/collectors/sec_edgar.py` +- Create: `services/news-collector/tests/test_sec_edgar.py` + +- [ ] **Step 1: Write tests** + +Create `services/news-collector/tests/test_sec_edgar.py`: + +```python +"""Tests for SEC EDGAR filing collector.""" + +import pytest +from unittest.mock import AsyncMock, patch + +from news_collector.collectors.sec_edgar import SecEdgarCollector + + +@pytest.fixture +def collector(): + return SecEdgarCollector() + + +def test_collector_name(collector): + assert collector.name == "sec_edgar" + assert collector.poll_interval == 1800 + + +async def test_is_available(collector): + assert await collector.is_available() is True + + +async def test_collect_parses_filings(collector): + mock_response = { + "filings": { + "recent": { + "accessionNumber": ["0001234-26-000001"], + "filingDate": ["2026-04-02"], + "primaryDocument": ["filing.htm"], + "form": ["8-K"], + "primaryDocDescription": ["Current Report"], + } + }, + "tickers": [{"ticker": "AAPL"}], + "name": "Apple Inc", + } + with patch.object(collector, "_fetch_recent_filings", new_callable=AsyncMock, return_value=[mock_response]): + items = await collector.collect() + + assert len(items) == 1 + assert items[0].source == "sec_edgar" + assert items[0].category.value == "filing" + assert "AAPL" in items[0].symbols + + +async def test_collect_handles_empty(collector): + with patch.object(collector, "_fetch_recent_filings", new_callable=AsyncMock, return_value=[]): + items = await collector.collect() + assert items == [] +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest services/news-collector/tests/test_sec_edgar.py -v` +Expected: FAIL + +- [ ] **Step 3: Implement SecEdgarCollector** + +Create `services/news-collector/src/news_collector/collectors/sec_edgar.py`: + +```python +"""SEC EDGAR filing collector (free, no API key required).""" + +import logging +from datetime import datetime, timezone + +import aiohttp +from nltk.sentiment.vader import SentimentIntensityAnalyzer + +from shared.models import NewsCategory, NewsItem +from news_collector.collectors.base import BaseCollector + +logger = logging.getLogger(__name__) + +EDGAR_FULL_TEXT_SEARCH = "https://efts.sec.gov/LATEST/search-index" +EDGAR_RECENT_FILINGS = "https://efts.sec.gov/LATEST/search-index?q=%228-K%22&dateRange=custom&startdt={date}&enddt={date}&forms=8-K" +EDGAR_COMPANY_FILINGS = "https://data.sec.gov/submissions/CIK{cik}.json" + +# CIK numbers for major companies (subset — extend as needed) +TRACKED_CIKS = { + "0000320193": "AAPL", + "0000789019": "MSFT", + "0001652044": "GOOGL", + "0001018724": "AMZN", + "0001318605": "TSLA", + "0001045810": "NVDA", + "0001326801": "META", + "0000019617": "JPM", + "0000078003": "PFE", + "0000021344": "KO", +} + +SEC_USER_AGENT = "TradingPlatform research@example.com" + + +class SecEdgarCollector(BaseCollector): + name = "sec_edgar" + poll_interval = 1800 # 30 minutes + + def __init__(self) -> None: + self._vader = SentimentIntensityAnalyzer() + + async def is_available(self) -> bool: + return True + + async def _fetch_recent_filings(self) -> list[dict]: + """Fetch recent 8-K filings for tracked companies.""" + results = [] + headers = {"User-Agent": SEC_USER_AGENT} + async with aiohttp.ClientSession() as session: + for cik, ticker in TRACKED_CIKS.items(): + try: + url = f"https://data.sec.gov/submissions/CIK{cik}.json" + async with session.get( + url, headers=headers, timeout=aiohttp.ClientTimeout(total=10) + ) as resp: + if resp.status == 200: + data = await resp.json() + data["tickers"] = [{"ticker": ticker}] + results.append(data) + except Exception as exc: + logger.warning("sec_fetch_failed", cik=cik, error=str(exc)) + return results + + async def collect(self) -> list[NewsItem]: + filings_data = await self._fetch_recent_filings() + items = [] + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + + for company_data in filings_data: + tickers = [t["ticker"] for t in company_data.get("tickers", [])] + company_name = company_data.get("name", "Unknown") + recent = company_data.get("filings", {}).get("recent", {}) + + forms = recent.get("form", []) + dates = recent.get("filingDate", []) + descriptions = recent.get("primaryDocDescription", []) + accessions = recent.get("accessionNumber", []) + + for i, form in enumerate(forms): + if form != "8-K": + continue + filing_date = dates[i] if i < len(dates) else "" + if filing_date != today: + continue + + desc = descriptions[i] if i < len(descriptions) else "8-K Filing" + accession = accessions[i] if i < len(accessions) else "" + headline = f"{company_name} ({', '.join(tickers)}): {form} - {desc}" + + items.append( + NewsItem( + source=self.name, + headline=headline, + summary=desc, + url=f"https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&accession={accession}", + published_at=datetime.strptime(filing_date, "%Y-%m-%d").replace(tzinfo=timezone.utc), + symbols=tickers, + sentiment=self._vader.polarity_scores(headline)["compound"], + category=NewsCategory.FILING, + raw_data={"form": form, "accession": accession}, + ) + ) + + return items +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pytest services/news-collector/tests/test_sec_edgar.py -v` +Expected: All 4 tests PASS + +- [ ] **Step 5: Commit** + +```bash +git add services/news-collector/src/news_collector/collectors/sec_edgar.py services/news-collector/tests/test_sec_edgar.py +git commit -m "feat: implement SEC EDGAR 8-K filing collector" +``` + +--- + +### Task 11: Implement Reddit collector + +**Files:** +- Create: `services/news-collector/src/news_collector/collectors/reddit.py` +- Create: `services/news-collector/tests/test_reddit.py` + +- [ ] **Step 1: Write tests** + +Create `services/news-collector/tests/test_reddit.py`: + +```python +"""Tests for Reddit collector.""" + +import pytest +from unittest.mock import AsyncMock, patch + +from news_collector.collectors.reddit import RedditCollector + + +@pytest.fixture +def collector(): + return RedditCollector() + + +def test_collector_name(collector): + assert collector.name == "reddit" + assert collector.poll_interval == 900 + + +async def test_is_available(collector): + assert await collector.is_available() is True + + +async def test_collect_parses_posts(collector): + mock_posts = [ + { + "data": { + "title": "NVDA to the moon! 🚀 AI demand is insane", + "selftext": "Just loaded up on NVDA calls", + "url": "https://reddit.com/r/wallstreetbets/123", + "created_utc": 1711929600, + "score": 500, + "num_comments": 200, + "subreddit": "wallstreetbets", + } + }, + ] + with patch.object(collector, "_fetch_subreddit", new_callable=AsyncMock, return_value=mock_posts): + items = await collector.collect() + + assert len(items) >= 1 + assert items[0].source == "reddit" + assert items[0].category.value == "social" + assert isinstance(items[0].sentiment, float) + + +async def test_collect_filters_low_score(collector): + mock_posts = [ + { + "data": { + "title": "Random question about stocks", + "selftext": "", + "url": "https://reddit.com/r/stocks/456", + "created_utc": 1711929600, + "score": 3, + "num_comments": 1, + "subreddit": "stocks", + } + }, + ] + with patch.object(collector, "_fetch_subreddit", new_callable=AsyncMock, return_value=mock_posts): + items = await collector.collect() + assert items == [] +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest services/news-collector/tests/test_reddit.py -v` +Expected: FAIL + +- [ ] **Step 3: Implement RedditCollector** + +Create `services/news-collector/src/news_collector/collectors/reddit.py`: + +```python +"""Reddit collector for r/wallstreetbets, r/stocks, r/investing.""" + +import logging +import re +from datetime import datetime, timezone + +import aiohttp +from nltk.sentiment.vader import SentimentIntensityAnalyzer + +from shared.models import NewsCategory, NewsItem +from news_collector.collectors.base import BaseCollector + +logger = logging.getLogger(__name__) + +SUBREDDITS = ["wallstreetbets", "stocks", "investing"] +MIN_SCORE = 50 # Minimum upvotes to consider + +TICKER_PATTERN = re.compile( + r"\b(AAPL|MSFT|GOOGL|GOOG|AMZN|TSLA|NVDA|META|JPM|V|JNJ|WMT|PG|UNH|HD|" + r"MA|DIS|BAC|XOM|PFE|KO|PEP|CSCO|INTC|VZ|NFLX|ADBE|CRM|AMD|QCOM|" + r"GS|BA|CAT|MMM|IBM|GE|F|GM|NKE|MCD|SBUX|SPY|QQQ|IWM)\b" +) + + +class RedditCollector(BaseCollector): + name = "reddit" + poll_interval = 900 # 15 minutes + + def __init__(self) -> None: + self._vader = SentimentIntensityAnalyzer() + + async def is_available(self) -> bool: + return True + + async def _fetch_subreddit(self, subreddit: str = "wallstreetbets") -> list[dict]: + """Fetch hot posts from a subreddit via JSON API.""" + url = f"https://www.reddit.com/r/{subreddit}/hot.json?limit=25" + headers = {"User-Agent": "TradingPlatform/1.0"} + try: + async with aiohttp.ClientSession() as session: + async with session.get( + url, headers=headers, timeout=aiohttp.ClientTimeout(total=10) + ) as resp: + if resp.status != 200: + logger.warning("reddit_fetch_failed", subreddit=subreddit, status=resp.status) + return [] + data = await resp.json() + return data.get("data", {}).get("children", []) + except Exception as exc: + logger.warning("reddit_error", subreddit=subreddit, error=str(exc)) + return [] + + async def collect(self) -> list[NewsItem]: + items = [] + seen_titles = set() + + for subreddit in SUBREDDITS: + posts = await self._fetch_subreddit(subreddit) + for post in posts: + data = post.get("data", {}) + title = data.get("title", "").strip() + score = data.get("score", 0) + + if not title or title in seen_titles or score < MIN_SCORE: + continue + seen_titles.add(title) + + selftext = data.get("selftext", "") + text = f"{title}. {selftext}" if selftext else title + symbols = list(set(TICKER_PATTERN.findall(text))) + + items.append( + NewsItem( + source=self.name, + headline=title, + summary=selftext[:500] if selftext else None, + url=data.get("url"), + published_at=datetime.fromtimestamp( + data.get("created_utc", 0), tz=timezone.utc + ), + symbols=symbols, + sentiment=self._vader.polarity_scores(text)["compound"], + category=NewsCategory.SOCIAL, + raw_data={ + "subreddit": data.get("subreddit", subreddit), + "score": score, + "num_comments": data.get("num_comments", 0), + }, + ) + ) + + return items +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pytest services/news-collector/tests/test_reddit.py -v` +Expected: All 4 tests PASS + +- [ ] **Step 5: Commit** + +```bash +git add services/news-collector/src/news_collector/collectors/reddit.py services/news-collector/tests/test_reddit.py +git commit -m "feat: implement Reddit social sentiment collector" +``` + +--- + +### Task 12: Implement Truth Social and Fed collectors + +**Files:** +- Create: `services/news-collector/src/news_collector/collectors/truth_social.py` +- Create: `services/news-collector/src/news_collector/collectors/fed.py` +- Create: `services/news-collector/tests/test_truth_social.py` +- Create: `services/news-collector/tests/test_fed.py` + +- [ ] **Step 1: Write tests for Truth Social** + +Create `services/news-collector/tests/test_truth_social.py`: + +```python +"""Tests for Truth Social collector.""" + +import pytest +from unittest.mock import AsyncMock, patch + +from news_collector.collectors.truth_social import TruthSocialCollector + + +@pytest.fixture +def collector(): + return TruthSocialCollector() + + +def test_collector_name(collector): + assert collector.name == "truth_social" + assert collector.poll_interval == 900 + + +async def test_is_available(collector): + assert await collector.is_available() is True + + +async def test_collect_parses_posts(collector): + mock_posts = [ + { + "content": "We are imposing 25% tariffs on all steel imports!", + "created_at": "2026-04-02T12:00:00.000Z", + "url": "https://truthsocial.com/@realDonaldTrump/12345", + }, + ] + with patch.object(collector, "_fetch_posts", new_callable=AsyncMock, return_value=mock_posts): + items = await collector.collect() + + assert len(items) == 1 + assert items[0].source == "truth_social" + assert items[0].category.value == "policy" + assert "tariff" in items[0].headline.lower() or "tariff" in items[0].raw_data.get("content", "").lower() + + +async def test_collect_handles_empty(collector): + with patch.object(collector, "_fetch_posts", new_callable=AsyncMock, return_value=[]): + items = await collector.collect() + assert items == [] +``` + +- [ ] **Step 2: Write tests for Fed collector** + +Create `services/news-collector/tests/test_fed.py`: + +```python +"""Tests for Federal Reserve collector.""" + +import pytest +from unittest.mock import AsyncMock, patch + +from news_collector.collectors.fed import FedCollector + + +@pytest.fixture +def collector(): + return FedCollector() + + +def test_collector_name(collector): + assert collector.name == "fed" + assert collector.poll_interval == 3600 + + +async def test_is_available(collector): + assert await collector.is_available() is True + + +async def test_collect_parses_rss(collector): + mock_entries = [ + { + "title": "Federal Reserve issues FOMC statement", + "link": "https://www.federalreserve.gov/newsevents/pressreleases/monetary20260402a.htm", + "published_parsed": (2026, 4, 2, 14, 0, 0, 0, 0, 0), + "summary": "The Federal Open Market Committee decided to maintain the target range...", + }, + ] + with patch.object(collector, "_fetch_fed_rss", new_callable=AsyncMock, return_value=mock_entries): + items = await collector.collect() + + assert len(items) == 1 + assert items[0].source == "fed" + assert items[0].category.value == "fed" +``` + +- [ ] **Step 3: Run tests to verify they fail** + +Run: `pytest services/news-collector/tests/test_truth_social.py services/news-collector/tests/test_fed.py -v` +Expected: FAIL + +- [ ] **Step 4: Implement TruthSocialCollector** + +Create `services/news-collector/src/news_collector/collectors/truth_social.py`: + +```python +"""Truth Social collector for Trump posts (policy-relevant).""" + +import logging +from datetime import datetime, timezone + +import aiohttp +from nltk.sentiment.vader import SentimentIntensityAnalyzer + +from shared.models import NewsCategory, NewsItem +from news_collector.collectors.base import BaseCollector + +logger = logging.getLogger(__name__) + +# Truth Social uses a Mastodon-compatible API +TRUTH_SOCIAL_API = "https://truthsocial.com/api/v1/accounts/107780257626128497/statuses" + + +class TruthSocialCollector(BaseCollector): + name = "truth_social" + poll_interval = 900 # 15 minutes + + def __init__(self) -> None: + self._vader = SentimentIntensityAnalyzer() + + async def is_available(self) -> bool: + return True + + async def _fetch_posts(self) -> list[dict]: + """Fetch recent posts from Truth Social.""" + headers = {"User-Agent": "Mozilla/5.0"} + try: + async with aiohttp.ClientSession() as session: + async with session.get( + TRUTH_SOCIAL_API, + headers=headers, + params={"limit": 10}, + timeout=aiohttp.ClientTimeout(total=15), + ) as resp: + if resp.status != 200: + logger.warning("truth_social_fetch_failed", status=resp.status) + return [] + return await resp.json() + except Exception as exc: + logger.warning("truth_social_error", error=str(exc)) + return [] + + def _strip_html(self, text: str) -> str: + """Remove HTML tags from content.""" + import re + return re.sub(r"<[^>]+>", "", text).strip() + + async def collect(self) -> list[NewsItem]: + posts = await self._fetch_posts() + items = [] + + for post in posts: + content = self._strip_html(post.get("content", "")) + if not content: + continue + + created_at_str = post.get("created_at", "") + try: + published = datetime.fromisoformat(created_at_str.replace("Z", "+00:00")) + except (ValueError, AttributeError): + published = datetime.now(timezone.utc) + + items.append( + NewsItem( + source=self.name, + headline=content[:200], + summary=content if len(content) > 200 else None, + url=post.get("url"), + published_at=published, + symbols=[], # Symbols extracted at aggregation stage via LLM + sentiment=self._vader.polarity_scores(content)["compound"], + category=NewsCategory.POLICY, + raw_data={"content": content, "id": post.get("id")}, + ) + ) + + return items +``` + +- [ ] **Step 5: Implement FedCollector** + +Create `services/news-collector/src/news_collector/collectors/fed.py`: + +```python +"""Federal Reserve press release and FOMC statement collector.""" + +import logging +from calendar import timegm +from datetime import datetime, timezone + +import aiohttp +import feedparser +from nltk.sentiment.vader import SentimentIntensityAnalyzer + +from shared.models import NewsCategory, NewsItem +from news_collector.collectors.base import BaseCollector + +logger = logging.getLogger(__name__) + +FED_RSS_URL = "https://www.federalreserve.gov/feeds/press_all.xml" + + +class FedCollector(BaseCollector): + name = "fed" + poll_interval = 3600 # 1 hour + + def __init__(self) -> None: + self._vader = SentimentIntensityAnalyzer() + + async def is_available(self) -> bool: + return True + + async def _fetch_fed_rss(self) -> list[dict]: + """Fetch Federal Reserve RSS feed entries.""" + try: + async with aiohttp.ClientSession() as session: + async with session.get( + FED_RSS_URL, timeout=aiohttp.ClientTimeout(total=10) + ) as resp: + if resp.status != 200: + logger.warning("fed_rss_failed", status=resp.status) + return [] + text = await resp.text() + feed = feedparser.parse(text) + return feed.get("entries", []) + except Exception as exc: + logger.warning("fed_rss_error", error=str(exc)) + return [] + + def _detect_stance(self, text: str) -> str: + """Detect hawkish/dovish/neutral stance from text.""" + text_lower = text.lower() + hawkish_words = ["tighten", "raise", "inflation concern", "restrictive", "higher rates"] + dovish_words = ["accommodate", "cut", "easing", "lower rates", "support growth"] + + hawk_count = sum(1 for w in hawkish_words if w in text_lower) + dove_count = sum(1 for w in dovish_words if w in text_lower) + + if hawk_count > dove_count: + return "hawkish" + if dove_count > hawk_count: + return "dovish" + return "neutral" + + async def collect(self) -> list[NewsItem]: + entries = await self._fetch_fed_rss() + items = [] + + for entry in entries: + title = entry.get("title", "").strip() + if not title: + continue + + summary = entry.get("summary", "") + parsed_time = entry.get("published_parsed") + if parsed_time: + published = datetime.fromtimestamp(timegm(parsed_time), tz=timezone.utc) + else: + published = datetime.now(timezone.utc) + + text = f"{title}. {summary}" if summary else title + + items.append( + NewsItem( + source=self.name, + headline=title, + summary=summary or None, + url=entry.get("link"), + published_at=published, + symbols=[], + sentiment=self._vader.polarity_scores(text)["compound"], + category=NewsCategory.FED, + raw_data={"stance": self._detect_stance(text)}, + ) + ) + + return items +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `pytest services/news-collector/tests/test_truth_social.py services/news-collector/tests/test_fed.py -v` +Expected: All 7 tests PASS + +- [ ] **Step 7: Commit** + +```bash +git add services/news-collector/src/news_collector/collectors/truth_social.py services/news-collector/src/news_collector/collectors/fed.py services/news-collector/tests/test_truth_social.py services/news-collector/tests/test_fed.py +git commit -m "feat: implement Truth Social and Federal Reserve collectors" +``` + +--- + +### Task 13: Implement news-collector main.py (scheduler) + +**Files:** +- Create: `services/news-collector/src/news_collector/main.py` +- Create: `services/news-collector/tests/test_main.py` + +- [ ] **Step 1: Write tests** + +Create `services/news-collector/tests/test_main.py`: + +```python +"""Tests for news collector scheduler.""" + +import pytest +from unittest.mock import AsyncMock, patch, MagicMock +from datetime import datetime, timezone + +from shared.models import NewsCategory, NewsItem + +from news_collector.main import run_collector_once + + +async def test_run_collector_once_stores_and_publishes(): + mock_item = NewsItem( + source="test", + headline="Test news", + published_at=datetime(2026, 4, 2, tzinfo=timezone.utc), + sentiment=0.5, + category=NewsCategory.MACRO, + ) + + mock_collector = MagicMock() + mock_collector.name = "test" + mock_collector.collect = AsyncMock(return_value=[mock_item]) + + mock_db = MagicMock() + mock_db.insert_news_item = AsyncMock() + + mock_broker = MagicMock() + mock_broker.publish = AsyncMock() + + count = await run_collector_once(mock_collector, mock_db, mock_broker) + + assert count == 1 + mock_db.insert_news_item.assert_called_once_with(mock_item) + mock_broker.publish.assert_called_once() + + +async def test_run_collector_once_handles_empty(): + mock_collector = MagicMock() + mock_collector.name = "test" + mock_collector.collect = AsyncMock(return_value=[]) + + mock_db = MagicMock() + mock_broker = MagicMock() + + count = await run_collector_once(mock_collector, mock_db, mock_broker) + assert count == 0 +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest services/news-collector/tests/test_main.py -v` +Expected: FAIL + +- [ ] **Step 3: Implement main.py** + +Create `services/news-collector/src/news_collector/main.py`: + +```python +"""News Collector Service — schedules and runs all news collectors.""" + +import asyncio +import logging + +from shared.broker import RedisBroker +from shared.db import Database +from shared.events import NewsEvent +from shared.healthcheck import HealthCheckServer +from shared.logging import setup_logging +from shared.metrics import ServiceMetrics +from shared.models import NewsItem +from shared.notifier import TelegramNotifier +from shared.sentiment_models import MarketSentiment + +from news_collector.config import NewsCollectorConfig +from news_collector.collectors.base import BaseCollector +from news_collector.collectors.finnhub import FinnhubCollector +from news_collector.collectors.rss import RSSCollector +from news_collector.collectors.sec_edgar import SecEdgarCollector +from news_collector.collectors.truth_social import TruthSocialCollector +from news_collector.collectors.reddit import RedditCollector +from news_collector.collectors.fear_greed import FearGreedCollector +from news_collector.collectors.fed import FedCollector + +logger = logging.getLogger(__name__) + + +async def run_collector_once( + collector: BaseCollector, + db: Database, + broker: RedisBroker, +) -> int: + """Run a single collector, store results, publish to Redis. + Returns number of items collected.""" + items = await collector.collect() + if not isinstance(items, list): + # FearGreedCollector returns a FearGreedResult, not a list + return 0 + + for item in items: + await db.insert_news_item(item) + event = NewsEvent(data=item) + await broker.publish("news", event.to_dict()) + + return len(items) + + +async def run_collector_loop( + collector: BaseCollector, + db: Database, + broker: RedisBroker, + log, +) -> None: + """Run a collector on its poll interval forever.""" + while True: + try: + if await collector.is_available(): + count = await run_collector_once(collector, db, broker) + log.info("collector_run", collector=collector.name, items=count) + else: + log.debug("collector_unavailable", collector=collector.name) + except Exception as exc: + log.error("collector_error", collector=collector.name, error=str(exc)) + await asyncio.sleep(collector.poll_interval) + + +async def run_fear_greed_loop( + collector: FearGreedCollector, + db: Database, + log, +) -> None: + """Run the Fear & Greed collector and update market sentiment.""" + from datetime import datetime, timezone + + while True: + try: + result = await collector.collect() + if result is not None: + ms = MarketSentiment( + fear_greed=result.fear_greed, + fear_greed_label=result.fear_greed_label, + fed_stance="neutral", # Updated by Fed collector analysis + market_regime=_determine_regime(result.fear_greed, None), + updated_at=datetime.now(timezone.utc), + ) + await db.upsert_market_sentiment(ms) + log.info("fear_greed_updated", score=result.fear_greed, label=result.fear_greed_label) + except Exception as exc: + log.error("fear_greed_error", error=str(exc)) + await asyncio.sleep(collector.poll_interval) + + +async def run_aggregator_loop( + db: Database, + interval: int, + log, +) -> None: + """Run sentiment aggregation every `interval` seconds. + Reads recent news from DB, computes per-symbol scores, upserts into symbol_scores table.""" + from datetime import datetime, timezone + from shared.sentiment import SentimentAggregator + + aggregator = SentimentAggregator() + + while True: + try: + now = datetime.now(timezone.utc) + news_items = await db.get_recent_news(hours=24) + if news_items: + scores = aggregator.aggregate(news_items, now) + for symbol_score in scores.values(): + await db.upsert_symbol_score(symbol_score) + log.info("aggregation_complete", symbols=len(scores)) + except Exception as exc: + log.error("aggregation_error", error=str(exc)) + await asyncio.sleep(interval) + + +def _determine_regime(fear_greed: int, vix: float | None) -> str: + """Determine market regime from Fear & Greed and VIX.""" + if fear_greed <= 20: + return "risk_off" + if vix is not None and vix > 30: + return "risk_off" + if fear_greed >= 60 and (vix is None or vix < 20): + return "risk_on" + return "neutral" + + +async def run() -> None: + config = NewsCollectorConfig() + log = setup_logging("news-collector", config.log_level, config.log_format) + metrics = ServiceMetrics("news_collector") + + notifier = TelegramNotifier( + bot_token=config.telegram_bot_token, + chat_id=config.telegram_chat_id, + ) + + db = Database(config.database_url) + await db.connect() + + broker = RedisBroker(config.redis_url) + + health = HealthCheckServer( + "news-collector", + port=config.health_port, + auth_token=config.metrics_auth_token, + ) + health.register_check("redis", broker.ping) + await health.start() + metrics.service_up.labels(service="news-collector").set(1) + + # Initialize collectors + news_collectors: list[BaseCollector] = [ + RSSCollector(), + SecEdgarCollector(), + TruthSocialCollector(), + RedditCollector(), + FedCollector(), + ] + + # Finnhub requires API key + if config.finnhub_api_key: + news_collectors.append(FinnhubCollector(api_key=config.finnhub_api_key)) + + fear_greed = FearGreedCollector() + + log.info( + "starting", + collectors=[c.name for c in news_collectors], + fear_greed=True, + ) + + tasks = [] + try: + for collector in news_collectors: + task = asyncio.create_task(run_collector_loop(collector, db, broker, log)) + tasks.append(task) + + tasks.append(asyncio.create_task(run_fear_greed_loop(fear_greed, db, log))) + tasks.append(asyncio.create_task( + run_aggregator_loop(db, config.sentiment_aggregate_interval, log) + )) + + await asyncio.gather(*tasks) + except Exception as exc: + log.error("fatal_error", error=str(exc)) + await notifier.send_error(str(exc), "news-collector") + raise + finally: + for task in tasks: + task.cancel() + metrics.service_up.labels(service="news-collector").set(0) + await notifier.close() + await broker.close() + await db.close() + + +def main() -> None: + asyncio.run(run()) + + +if __name__ == "__main__": + main() +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pytest services/news-collector/tests/test_main.py -v` +Expected: All 2 tests PASS + +- [ ] **Step 5: Commit** + +```bash +git add services/news-collector/src/news_collector/main.py services/news-collector/tests/test_main.py +git commit -m "feat: implement news-collector main scheduler with all collectors" +``` + +--- + +## Phase 3: Sentiment Analysis Pipeline + +### Task 14: Implement sentiment aggregator + +**Files:** +- Modify: `shared/src/shared/sentiment.py` +- Create: `shared/tests/test_sentiment_aggregator.py` + +- [ ] **Step 1: Write tests** + +Create `shared/tests/test_sentiment_aggregator.py`: + +```python +"""Tests for sentiment aggregator.""" + +import pytest +from datetime import datetime, timezone, timedelta + +from shared.sentiment import SentimentAggregator + + +@pytest.fixture +def aggregator(): + return SentimentAggregator() + + +def test_freshness_decay_recent(): + a = SentimentAggregator() + now = datetime.now(timezone.utc) + assert a._freshness_decay(now, now) == 1.0 + + +def test_freshness_decay_3_hours(): + a = SentimentAggregator() + now = datetime.now(timezone.utc) + three_hours_ago = now - timedelta(hours=3) + assert a._freshness_decay(three_hours_ago, now) == 0.7 + + +def test_freshness_decay_12_hours(): + a = SentimentAggregator() + now = datetime.now(timezone.utc) + twelve_hours_ago = now - timedelta(hours=12) + assert a._freshness_decay(twelve_hours_ago, now) == 0.3 + + +def test_freshness_decay_old(): + a = SentimentAggregator() + now = datetime.now(timezone.utc) + two_days_ago = now - timedelta(days=2) + assert a._freshness_decay(two_days_ago, now) == 0.0 + + +def test_compute_composite(): + a = SentimentAggregator() + composite = a._compute_composite( + news_score=0.5, + social_score=0.3, + policy_score=0.8, + filing_score=0.2, + ) + expected = 0.5 * 0.3 + 0.3 * 0.2 + 0.8 * 0.3 + 0.2 * 0.2 + assert abs(composite - expected) < 0.001 + + +def test_aggregate_news_by_symbol(aggregator): + now = datetime.now(timezone.utc) + news_items = [ + { + "symbols": ["AAPL"], + "sentiment": 0.8, + "category": "earnings", + "published_at": now, + }, + { + "symbols": ["AAPL"], + "sentiment": 0.3, + "category": "macro", + "published_at": now - timedelta(hours=2), + }, + { + "symbols": ["MSFT"], + "sentiment": -0.5, + "category": "policy", + "published_at": now, + }, + ] + scores = aggregator.aggregate(news_items, now) + + assert "AAPL" in scores + assert "MSFT" in scores + assert scores["AAPL"].news_count == 2 + assert scores["AAPL"].news_score > 0 # Positive overall + assert scores["MSFT"].policy_score < 0 # Negative policy + + +def test_aggregate_empty(aggregator): + now = datetime.now(timezone.utc) + scores = aggregator.aggregate([], now) + assert scores == {} + + +def test_determine_regime(): + a = SentimentAggregator() + assert a.determine_regime(15, None) == "risk_off" + assert a.determine_regime(15, 35.0) == "risk_off" + assert a.determine_regime(50, 35.0) == "risk_off" + assert a.determine_regime(70, 15.0) == "risk_on" + assert a.determine_regime(50, 20.0) == "neutral" +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest shared/tests/test_sentiment_aggregator.py -v` +Expected: FAIL — `SentimentAggregator` not in sentiment.py + +- [ ] **Step 3: Add SentimentAggregator to sentiment.py** + +Keep the existing `SentimentData` class (for backward compat with existing tests). Add `SentimentAggregator` class at the end of `shared/src/shared/sentiment.py`: + +```python +from datetime import timedelta +from shared.sentiment_models import SymbolScore + + +class SentimentAggregator: + """Aggregates per-news sentiment into per-symbol scores.""" + + # Weights: policy events are most impactful for US stocks + WEIGHTS = { + "news": 0.3, + "social": 0.2, + "policy": 0.3, + "filing": 0.2, + } + + # Category → score field mapping + CATEGORY_MAP = { + "earnings": "news", + "macro": "news", + "social": "social", + "policy": "policy", + "filing": "filing", + "fed": "policy", + } + + def _freshness_decay(self, published_at: datetime, now: datetime) -> float: + """Compute freshness decay factor.""" + age = now - published_at + hours = age.total_seconds() / 3600 + if hours < 1: + return 1.0 + if hours < 6: + return 0.7 + if hours < 24: + return 0.3 + return 0.0 + + def _compute_composite( + self, + news_score: float, + social_score: float, + policy_score: float, + filing_score: float, + ) -> float: + return ( + news_score * self.WEIGHTS["news"] + + social_score * self.WEIGHTS["social"] + + policy_score * self.WEIGHTS["policy"] + + filing_score * self.WEIGHTS["filing"] + ) + + def aggregate( + self, news_items: list[dict], now: datetime + ) -> dict[str, SymbolScore]: + """Aggregate news items into per-symbol scores. + + Each news_items dict must have: symbols, sentiment, category, published_at. + Returns dict mapping symbol → SymbolScore. + """ + # Accumulate per-symbol, per-category + symbol_data: dict[str, dict] = {} + + for item in news_items: + decay = self._freshness_decay(item["published_at"], now) + if decay == 0.0: + continue + + category = item.get("category", "macro") + score_field = self.CATEGORY_MAP.get(category, "news") + weighted_sentiment = item["sentiment"] * decay + + for symbol in item.get("symbols", []): + if symbol not in symbol_data: + symbol_data[symbol] = { + "news_scores": [], + "social_scores": [], + "policy_scores": [], + "filing_scores": [], + "count": 0, + } + + symbol_data[symbol][f"{score_field}_scores"].append(weighted_sentiment) + symbol_data[symbol]["count"] += 1 + + # Compute averages and composites + result = {} + for symbol, data in symbol_data.items(): + news_score = _safe_avg(data["news_scores"]) + social_score = _safe_avg(data["social_scores"]) + policy_score = _safe_avg(data["policy_scores"]) + filing_score = _safe_avg(data["filing_scores"]) + + result[symbol] = SymbolScore( + symbol=symbol, + news_score=news_score, + news_count=data["count"], + social_score=social_score, + policy_score=policy_score, + filing_score=filing_score, + composite=self._compute_composite( + news_score, social_score, policy_score, filing_score + ), + updated_at=now, + ) + + return result + + def determine_regime(self, fear_greed: int, vix: float | None) -> str: + """Determine market regime.""" + if fear_greed <= 20: + return "risk_off" + if vix is not None and vix > 30: + return "risk_off" + if fear_greed >= 60 and (vix is None or vix < 20): + return "risk_on" + return "neutral" + + +def _safe_avg(values: list[float]) -> float: + """Return average of values, or 0.0 if empty.""" + if not values: + return 0.0 + return sum(values) / len(values) +``` + +- [ ] **Step 4: Run new tests to verify they pass** + +Run: `pytest shared/tests/test_sentiment_aggregator.py -v` +Expected: All 9 tests PASS + +- [ ] **Step 5: Run existing sentiment tests for regressions** + +Run: `pytest shared/tests/test_sentiment.py -v` +Expected: All existing tests PASS (SentimentData unchanged) + +- [ ] **Step 6: Commit** + +```bash +git add shared/src/shared/sentiment.py shared/tests/test_sentiment_aggregator.py +git commit -m "feat: implement SentimentAggregator with freshness decay and composite scoring" +``` + +--- + +## Phase 4: Stock Selector Engine + +### Task 15: Implement stock selector + +**Files:** +- Create: `services/strategy-engine/src/strategy_engine/stock_selector.py` +- Create: `services/strategy-engine/tests/test_stock_selector.py` + +- [ ] **Step 1: Write tests** + +Create `services/strategy-engine/tests/test_stock_selector.py`: + +```python +"""Tests for stock selector engine.""" + +import pytest +from unittest.mock import AsyncMock, MagicMock, patch +from datetime import datetime, timezone +from decimal import Decimal + +from shared.models import OrderSide +from shared.sentiment_models import SymbolScore, MarketSentiment, SelectedStock, Candidate + +from strategy_engine.stock_selector import ( + SentimentCandidateSource, + StockSelector, + _parse_llm_selections, +) + + +async def test_sentiment_candidate_source(): + mock_db = MagicMock() + mock_db.get_top_symbol_scores = AsyncMock(return_value=[ + {"symbol": "AAPL", "composite": 0.8, "news_count": 5}, + {"symbol": "NVDA", "composite": 0.6, "news_count": 3}, + ]) + + source = SentimentCandidateSource(mock_db) + candidates = await source.get_candidates() + + assert len(candidates) == 2 + assert candidates[0].symbol == "AAPL" + assert candidates[0].source == "sentiment" + + +def test_parse_llm_selections_valid(): + llm_response = """ + [ + {"symbol": "NVDA", "side": "BUY", "conviction": 0.85, "reason": "AI demand", "key_news": ["NVDA beats earnings"]}, + {"symbol": "XOM", "side": "BUY", "conviction": 0.72, "reason": "Oil surge", "key_news": ["Oil prices up"]} + ] + """ + selections = _parse_llm_selections(llm_response) + assert len(selections) == 2 + assert selections[0].symbol == "NVDA" + assert selections[0].conviction == 0.85 + + +def test_parse_llm_selections_invalid(): + selections = _parse_llm_selections("not json") + assert selections == [] + + +def test_parse_llm_selections_with_markdown(): + llm_response = """ + Here are my picks: + ```json + [ + {"symbol": "TSLA", "side": "BUY", "conviction": 0.7, "reason": "Momentum", "key_news": ["Tesla rally"]} + ] + ``` + """ + selections = _parse_llm_selections(llm_response) + assert len(selections) == 1 + assert selections[0].symbol == "TSLA" + + +async def test_selector_blocks_on_risk_off(): + mock_db = MagicMock() + mock_db.get_latest_market_sentiment = AsyncMock(return_value={ + "fear_greed": 15, + "fear_greed_label": "Extreme Fear", + "vix": 35.0, + "fed_stance": "neutral", + "market_regime": "risk_off", + "updated_at": datetime.now(timezone.utc), + }) + + selector = StockSelector(db=mock_db, broker=MagicMock(), alpaca=MagicMock(), anthropic_api_key="test") + result = await selector.select() + assert result == [] +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest services/strategy-engine/tests/test_stock_selector.py -v` +Expected: FAIL — module not found + +- [ ] **Step 3: Implement StockSelector** + +Create `services/strategy-engine/src/strategy_engine/stock_selector.py`: + +```python +"""Stock Selector Engine — 3-stage dynamic stock selection for MOC trading.""" + +import json +import logging +import re +from datetime import datetime, timezone +from decimal import Decimal +from typing import Optional + +import aiohttp + +from shared.alpaca import AlpacaClient +from shared.broker import RedisBroker +from shared.db import Database +from shared.models import OrderSide +from shared.sentiment_models import Candidate, MarketSentiment, SelectedStock, SymbolScore + +logger = logging.getLogger(__name__) + + +class SentimentCandidateSource: + """Get candidate stocks from sentiment scores in DB.""" + + def __init__(self, db: Database, limit: int = 20) -> None: + self._db = db + self._limit = limit + + async def get_candidates(self) -> list[Candidate]: + scores = await self._db.get_top_symbol_scores(limit=self._limit) + return [ + Candidate( + symbol=s["symbol"], + source="sentiment", + score=s["composite"], + reason=f"Sentiment composite={s['composite']:.2f}, news_count={s['news_count']}", + ) + for s in scores + if s["composite"] != 0 + ] + + +class LLMCandidateSource: + """Get candidate stocks by asking Claude to analyze today's top news.""" + + def __init__(self, db: Database, api_key: str, model: str) -> None: + self._db = db + self._api_key = api_key + self._model = model + + async def get_candidates(self) -> list[Candidate]: + news = await self._db.get_recent_news(hours=24) + if not news: + return [] + + headlines = [f"- [{n['source']}] {n['headline']} (sentiment: {n['sentiment']:.2f})" for n in news[:50]] + prompt = ( + "You are a stock market analyst. Based on today's news headlines below, " + "identify US stocks that are most likely to be affected (positively or negatively). " + "Return a JSON array of objects with: symbol, direction (BUY or SELL), score (0-1), reason.\n\n" + "Headlines:\n" + "\n".join(headlines) + "\n\n" + "Return ONLY the JSON array, no other text." + ) + + try: + async with aiohttp.ClientSession() as session: + async with session.post( + "https://api.anthropic.com/v1/messages", + headers={ + "x-api-key": self._api_key, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + json={ + "model": self._model, + "max_tokens": 1024, + "messages": [{"role": "user", "content": prompt}], + }, + timeout=aiohttp.ClientTimeout(total=30), + ) as resp: + if resp.status != 200: + logger.warning("llm_candidate_failed", status=resp.status) + return [] + data = await resp.json() + text = data["content"][0]["text"] + except Exception as exc: + logger.error("llm_candidate_error", error=str(exc)) + return [] + + return self._parse_response(text) + + def _parse_response(self, text: str) -> list[Candidate]: + try: + # Extract JSON from possible markdown code blocks + json_match = re.search(r"```(?:json)?\s*(\[.*?\])\s*```", text, re.DOTALL) + if json_match: + text = json_match.group(1) + items = json.loads(text) + except (json.JSONDecodeError, TypeError): + return [] + + candidates = [] + for item in items: + try: + direction = OrderSide(item.get("direction", "BUY")) + candidates.append( + Candidate( + symbol=item["symbol"], + source="llm", + direction=direction, + score=float(item.get("score", 0.5)), + reason=item.get("reason", "LLM recommendation"), + ) + ) + except (KeyError, ValueError): + continue + + return candidates + + +class StockSelector: + """3-stage stock selector: candidates → technical filter → LLM final pick.""" + + def __init__( + self, + db: Database, + broker: RedisBroker, + alpaca: AlpacaClient, + anthropic_api_key: str, + anthropic_model: str = "claude-sonnet-4-20250514", + max_picks: int = 3, + ) -> None: + self._db = db + self._broker = broker + self._alpaca = alpaca + self._api_key = anthropic_api_key + self._model = anthropic_model + self._max_picks = max_picks + self._sentiment_source = SentimentCandidateSource(db) + self._llm_source = LLMCandidateSource(db, anthropic_api_key, anthropic_model) + + async def select(self) -> list[SelectedStock]: + """Run full 3-stage selection. Returns list of SelectedStock.""" + # Check market sentiment gate + ms = await self._db.get_latest_market_sentiment() + if ms and ms.get("market_regime") == "risk_off": + logger.info("selection_blocked_risk_off") + return [] + + # Stage 1: Candidate pool + sentiment_candidates = await self._sentiment_source.get_candidates() + llm_candidates = await self._llm_source.get_candidates() + candidates = self._merge_candidates(sentiment_candidates, llm_candidates) + + if not candidates: + logger.info("no_candidates_found") + return [] + + logger.info("candidates_found", count=len(candidates)) + + # Stage 2: Technical filter + filtered = await self._technical_filter(candidates) + if not filtered: + logger.info("all_candidates_filtered_out") + return [] + + logger.info("technical_filter_passed", count=len(filtered)) + + # Stage 3: LLM final selection + selections = await self._llm_final_select(filtered, ms) + + # Publish to Redis + for selection in selections: + await self._broker.publish( + "selected_stocks", + selection.model_dump(mode="json"), + ) + + # Persist audit trail + from datetime import date as date_type + + for selection in selections: + score_data = await self._db.get_top_symbol_scores(limit=100) + snapshot = next( + (s for s in score_data if s["symbol"] == selection.symbol), + {}, + ) + await self._db.insert_stock_selection( + trade_date=date_type.today(), + symbol=selection.symbol, + side=selection.side.value, + conviction=selection.conviction, + reason=selection.reason, + key_news=selection.key_news, + sentiment_snapshot=snapshot, + ) + + return selections + + def _merge_candidates( + self, + sentiment: list[Candidate], + llm: list[Candidate], + ) -> list[Candidate]: + """Merge and deduplicate candidates, preferring higher scores.""" + by_symbol: dict[str, Candidate] = {} + for c in sentiment + llm: + if c.symbol not in by_symbol or c.score > by_symbol[c.symbol].score: + by_symbol[c.symbol] = c + return sorted(by_symbol.values(), key=lambda c: c.score, reverse=True) + + async def _technical_filter(self, candidates: list[Candidate]) -> list[Candidate]: + """Apply MOC-style technical screening to candidates.""" + import pandas as pd + + passed = [] + for candidate in candidates: + try: + bars = await self._alpaca.get_bars( + candidate.symbol, timeframe="1Day", limit=30 + ) + if not bars or len(bars) < 21: + continue + + closes = pd.Series([float(b["c"]) for b in bars]) + volumes = pd.Series([float(b["v"]) for b in bars]) + + # RSI + delta = closes.diff() + gain = delta.clip(lower=0) + loss = -delta.clip(upper=0) + avg_gain = gain.ewm(com=13, min_periods=14).mean() + avg_loss = loss.ewm(com=13, min_periods=14).mean() + rs = avg_gain / avg_loss.replace(0, float("nan")) + rsi = 100 - (100 / (1 + rs)) + current_rsi = rsi.iloc[-1] + + if pd.isna(current_rsi) or not (30 <= current_rsi <= 70): + continue + + # EMA + ema20 = closes.ewm(span=20, adjust=False).mean().iloc[-1] + if closes.iloc[-1] < ema20: + continue + + # Volume above average + vol_avg = volumes.iloc[-20:].mean() + if vol_avg > 0 and volumes.iloc[-1] < vol_avg * 0.5: + continue + + passed.append(candidate) + + except Exception as exc: + logger.warning("technical_filter_error", symbol=candidate.symbol, error=str(exc)) + continue + + return passed + + async def _llm_final_select( + self, + candidates: list[Candidate], + market_sentiment: Optional[dict], + ) -> list[SelectedStock]: + """Ask Claude to make final 2-3 picks from filtered candidates.""" + # Build context + candidate_info = [] + for c in candidates[:15]: + candidate_info.append(f"- {c.symbol}: score={c.score:.2f}, source={c.source}, reason={c.reason}") + + news = await self._db.get_recent_news(hours=12) + top_news = [f"- [{n['source']}] {n['headline']}" for n in news[:20]] + + ms_info = "No market sentiment data available." + if market_sentiment: + ms_info = ( + f"Fear & Greed: {market_sentiment.get('fear_greed', 'N/A')} " + f"({market_sentiment.get('fear_greed_label', 'N/A')}), " + f"VIX: {market_sentiment.get('vix', 'N/A')}, " + f"Fed Stance: {market_sentiment.get('fed_stance', 'N/A')}" + ) + + prompt = ( + f"You are a professional stock trader selecting {self._max_picks} stocks for " + f"Market-on-Close (MOC) overnight trading. You buy at market close and sell at " + f"next day's open.\n\n" + f"## Market Conditions\n{ms_info}\n\n" + f"## Candidate Stocks (pre-screened technically)\n" + + "\n".join(candidate_info) + "\n\n" + f"## Today's Key News\n" + + "\n".join(top_news) + "\n\n" + f"Select the best {self._max_picks} stocks. For each, provide:\n" + f"- symbol: ticker\n" + f"- side: BUY or SELL\n" + f"- conviction: 0.0-1.0\n" + f"- reason: one sentence\n" + f"- key_news: list of relevant headlines\n\n" + f"Return ONLY a JSON array. No other text." + ) + + try: + async with aiohttp.ClientSession() as session: + async with session.post( + "https://api.anthropic.com/v1/messages", + headers={ + "x-api-key": self._api_key, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + json={ + "model": self._model, + "max_tokens": 1024, + "messages": [{"role": "user", "content": prompt}], + }, + timeout=aiohttp.ClientTimeout(total=30), + ) as resp: + if resp.status != 200: + logger.error("llm_final_select_failed", status=resp.status) + return [] + data = await resp.json() + text = data["content"][0]["text"] + except Exception as exc: + logger.error("llm_final_select_error", error=str(exc)) + return [] + + return _parse_llm_selections(text) + + +def _parse_llm_selections(text: str) -> list[SelectedStock]: + """Parse LLM response into SelectedStock list.""" + try: + json_match = re.search(r"```(?:json)?\s*(\[.*?\])\s*```", text, re.DOTALL) + if json_match: + text = json_match.group(1) + # Also try to find a bare JSON array + array_match = re.search(r"\[.*\]", text, re.DOTALL) + if array_match: + text = array_match.group(0) + items = json.loads(text) + except (json.JSONDecodeError, TypeError): + return [] + + selections = [] + for item in items: + try: + selections.append( + SelectedStock( + symbol=item["symbol"], + side=OrderSide(item.get("side", "BUY")), + conviction=float(item.get("conviction", 0.5)), + reason=item.get("reason", ""), + key_news=item.get("key_news", []), + ) + ) + except (KeyError, ValueError): + continue + + return selections +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pytest services/strategy-engine/tests/test_stock_selector.py -v` +Expected: All 5 tests PASS + +- [ ] **Step 5: Run all strategy engine tests for regressions** + +Run: `pytest services/strategy-engine/tests/ -v` +Expected: All tests PASS + +- [ ] **Step 6: Commit** + +```bash +git add services/strategy-engine/src/strategy_engine/stock_selector.py services/strategy-engine/tests/test_stock_selector.py +git commit -m "feat: implement 3-stage stock selector (sentiment → technical → LLM)" +``` + +--- + +## Phase 5: Integration (MOC + Notifications + Docker) + +### Task 16: Add Telegram notification for stock selections + +**Files:** +- Modify: `shared/src/shared/notifier.py` +- Modify: `shared/tests/test_notifier.py` + +- [ ] **Step 1: Add send_stock_selection method to notifier.py** + +Add this method and import to `shared/src/shared/notifier.py`: + +Add to imports: +```python +from shared.sentiment_models import SelectedStock, MarketSentiment +``` + +Add method to `TelegramNotifier` class: + +```python + async def send_stock_selection( + self, + selections: list[SelectedStock], + market: MarketSentiment | None = None, + ) -> None: + """Format and send stock selection notification.""" + lines = [f"<b>📊 Stock Selection ({len(selections)} picks)</b>", ""] + + side_emoji = {"BUY": "🟢", "SELL": "🔴"} + + for i, s in enumerate(selections, 1): + emoji = side_emoji.get(s.side.value, "⚪") + lines.append( + f"{i}. <b>{s.symbol}</b> {emoji} {s.side.value} " + f"(conviction: {s.conviction:.0%})" + ) + lines.append(f" {s.reason}") + if s.key_news: + lines.append(f" News: {s.key_news[0]}") + lines.append("") + + if market: + lines.append( + f"Market: F&G {market.fear_greed} ({market.fear_greed_label})" + + (f" | VIX {market.vix:.1f}" if market.vix else "") + ) + + await self.send("\n".join(lines)) +``` + +- [ ] **Step 2: Add test for the new method** + +Add to `shared/tests/test_notifier.py`: + +```python +from shared.models import OrderSide +from shared.sentiment_models import SelectedStock, MarketSentiment +from datetime import datetime, timezone + + +async def test_send_stock_selection(notifier, mock_session): + """Test stock selection notification formatting.""" + selections = [ + SelectedStock( + symbol="NVDA", + side=OrderSide.BUY, + conviction=0.85, + reason="CHIPS Act expansion", + key_news=["Trump signs CHIPS Act"], + ), + ] + market = MarketSentiment( + fear_greed=55, + fear_greed_label="Neutral", + vix=18.2, + fed_stance="neutral", + market_regime="neutral", + updated_at=datetime.now(timezone.utc), + ) + await notifier.send_stock_selection(selections, market) + mock_session.post.assert_called_once() +``` + +Note: Check `shared/tests/test_notifier.py` for existing fixture names (`notifier`, `mock_session`) and adapt accordingly. + +- [ ] **Step 3: Run notifier tests** + +Run: `pytest shared/tests/test_notifier.py -v` +Expected: All tests PASS + +- [ ] **Step 4: Commit** + +```bash +git add shared/src/shared/notifier.py shared/tests/test_notifier.py +git commit -m "feat: add Telegram notification for stock selections" +``` + +--- + +### Task 17: Integrate stock selector with MOC strategy + +**Files:** +- Modify: `services/strategy-engine/src/strategy_engine/main.py` +- Modify: `services/strategy-engine/src/strategy_engine/config.py` + +- [ ] **Step 1: Update strategy engine config** + +Add to `StrategyConfig` in `services/strategy-engine/src/strategy_engine/config.py`: + +```python + selector_candidates_time: str = "15:00" + selector_filter_time: str = "15:15" + selector_final_time: str = "15:30" + selector_max_picks: int = 3 + anthropic_api_key: str = "" + anthropic_model: str = "claude-sonnet-4-20250514" +``` + +- [ ] **Step 2: Add stock selector scheduling to main.py** + +Add a new coroutine to `services/strategy-engine/src/strategy_engine/main.py` that runs the stock selector at the configured times. Add imports: + +```python +from shared.alpaca import AlpacaClient +from shared.db import Database +from shared.notifier import TelegramNotifier +from shared.sentiment_models import MarketSentiment +from strategy_engine.stock_selector import StockSelector +``` + +Add the selector loop function: + +```python +async def run_stock_selector( + selector: StockSelector, + notifier: TelegramNotifier, + db: Database, + config: StrategyConfig, + log, +) -> None: + """Run the stock selector once per day at the configured time.""" + import zoneinfo + + et = zoneinfo.ZoneInfo("America/New_York") + + while True: + now_et = datetime.now(et) + target_hour, target_min = map(int, config.selector_final_time.split(":")) + + # Check if it's time to run (within 1-minute window) + if now_et.hour == target_hour and now_et.minute == target_min: + log.info("stock_selector_running") + try: + selections = await selector.select() + if selections: + ms_data = await db.get_latest_market_sentiment() + ms = None + if ms_data: + ms = MarketSentiment(**ms_data) + await notifier.send_stock_selection(selections, ms) + log.info( + "stock_selector_complete", + picks=[s.symbol for s in selections], + ) + else: + log.info("stock_selector_no_picks") + except Exception as exc: + log.error("stock_selector_error", error=str(exc)) + # Sleep past this minute to avoid re-triggering + await asyncio.sleep(120) + else: + await asyncio.sleep(30) +``` + +In the `run()` function, add after creating the broker: + +```python + db = Database(config.database_url) + await db.connect() + + alpaca = AlpacaClient( + api_key=config.alpaca_api_key, + api_secret=config.alpaca_api_secret, + paper=config.alpaca_paper, + ) +``` + +And add the selector if anthropic key is configured: + +```python + if config.anthropic_api_key: + selector = StockSelector( + db=db, + broker=broker, + alpaca=alpaca, + anthropic_api_key=config.anthropic_api_key, + anthropic_model=config.anthropic_model, + max_picks=config.selector_max_picks, + ) + tasks.append(asyncio.create_task( + run_stock_selector(selector, notifier, db, config, log) + )) + log.info("stock_selector_enabled", time=config.selector_final_time) +``` + +Add to the `finally` block: + +```python + await alpaca.close() + await db.close() +``` + +- [ ] **Step 3: Run strategy engine tests for regressions** + +Run: `pytest services/strategy-engine/tests/ -v` +Expected: All tests PASS + +- [ ] **Step 4: Commit** + +```bash +git add services/strategy-engine/src/strategy_engine/main.py services/strategy-engine/src/strategy_engine/config.py +git commit -m "feat: integrate stock selector into strategy engine scheduler" +``` + +--- + +### Task 18: Update Docker Compose and .env + +**Files:** +- Modify: `docker-compose.yml` +- Modify: `.env.example` (already done in Task 5, just verify) + +- [ ] **Step 1: Add news-collector service to docker-compose.yml** + +Add before the `loki:` service block in `docker-compose.yml`: + +```yaml + news-collector: + build: + context: . + dockerfile: services/news-collector/Dockerfile + env_file: .env + ports: + - "8084:8084" + depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8084/health')"] + interval: 10s + timeout: 5s + retries: 3 + restart: unless-stopped +``` + +- [ ] **Step 2: Verify compose file is valid** + +Run: `docker compose config --quiet 2>&1 || echo "INVALID"` +Expected: No output (valid) or compose config displayed without errors + +- [ ] **Step 3: Commit** + +```bash +git add docker-compose.yml +git commit -m "feat: add news-collector service to Docker Compose" +``` + +--- + +### Task 19: Run full test suite and lint + +- [ ] **Step 1: Install test dependencies** + +Run: `pip install -e shared/ && pip install aiosqlite feedparser nltk aioresponses` + +- [ ] **Step 2: Download VADER lexicon** + +Run: `python -c "import nltk; nltk.download('vader_lexicon', quiet=True)"` + +- [ ] **Step 3: Run lint** + +Run: `make lint` +Expected: No lint errors. If there are errors, fix them. + +- [ ] **Step 4: Run full test suite** + +Run: `make test` +Expected: All tests PASS + +- [ ] **Step 5: Fix any issues found in steps 3-4** + +If lint or tests fail, fix the issues and re-run. + +- [ ] **Step 6: Final commit if any fixes were needed** + +```bash +git add -A +git commit -m "fix: resolve lint and test issues from news selector integration" +``` diff --git a/docs/superpowers/specs/2026-04-01-crypto-trading-platform-design.md b/docs/superpowers/specs/2026-04-01-crypto-trading-platform-design.md deleted file mode 100644 index aa32eb4..0000000 --- a/docs/superpowers/specs/2026-04-01-crypto-trading-platform-design.md +++ /dev/null @@ -1,374 +0,0 @@ -# Crypto Trading Platform — Design Spec - -## Overview - -Binance 현물 암호화폐 자동매매 플랫폼. 마이크로서비스 아키텍처 기반으로 데이터 수집, 전략 실행, 주문 처리, 포트폴리오 관리, 백테스팅을 독립 서비스로 운영한다. CLI로 제어하며, 전략은 플러그인 방식으로 확장 가능하다. - -- **시장:** 암호화폐 (Binance 현물) -- **언어:** Python -- **인터페이스:** CLI (Click) -- **아키텍처:** 마이크로서비스 (Docker Compose) - ---- - -## Architecture - -### 서비스 구성 - -``` -┌─────────────┐ ┌──────────────────┐ ┌─────────────────┐ -│ Data │───▶│ Message Broker │◀──│ Strategy │ -│ Collector │ │ (Redis Streams) │ │ Engine │ -└─────────────┘ └──────────────────┘ └─────────────────┘ - │ ▲ │ - ▼ │ ▼ - ┌──────────────────┐ ┌─────────────────┐ - │ Backtester │ │ Order │ - │ │ │ Executor │ - └──────────────────┘ └─────────────────┘ - │ - ┌────────────────────────┘ - ▼ - ┌──────────────────┐ - │ Portfolio │ - │ Manager │ - └──────────────────┘ - - CLI ──────▶ 각 서비스에 명령 전달 -``` - -| 서비스 | 역할 | 상시 실행 | -|--------|------|-----------| -| **data-collector** | Binance WebSocket/REST로 시세 수집, DB 저장 | Yes | -| **strategy-engine** | 플러그인 전략 로드 및 시그널 생성 | 봇 실행 시 | -| **order-executor** | 시그널 받아 실제 주문 실행 + 리스크 관리 | 봇 실행 시 | -| **portfolio-manager** | 잔고, 손익, 포지션 추적 | Yes | -| **backtester** | 과거 데이터로 전략 검증 | 요청 시 | -| **shared** | 공통 모델, 이벤트 정의, 유틸리티 (라이브러리) | — | -| **cli** | 사용자 인터페이스, 각 서비스 제어 | — | - -### 통신 흐름 - -``` -[Binance WS] - │ - ▼ -data-collector ──publish──▶ Redis Stream: "candles.{symbol}" - │ - ┌───────────────┤ - ▼ ▼ - strategy-engine backtester (과거 데이터는 DB에서) - │ - ▼ - Redis Stream: "signals" - │ - ▼ - order-executor - │ - ┌───────┴───────┐ - ▼ ▼ - [Binance API] Redis Stream: "orders" - │ - ▼ - portfolio-manager -``` - ---- - -## Project Structure - -``` -trading/ -├── services/ -│ ├── data-collector/ -│ │ ├── src/ -│ │ │ ├── __init__.py -│ │ │ ├── main.py # 서비스 진입점 -│ │ │ ├── binance_ws.py # WebSocket 실시간 시세 -│ │ │ ├── binance_rest.py # REST 과거 데이터 수집 -│ │ │ ├── storage.py # DB 저장 로직 -│ │ │ └── config.py -│ │ ├── tests/ -│ │ ├── Dockerfile -│ │ └── pyproject.toml -│ │ -│ ├── strategy-engine/ -│ │ ├── src/ -│ │ │ ├── __init__.py -│ │ │ ├── main.py -│ │ │ ├── engine.py # 전략 로더 + 실행기 -│ │ │ ├── plugin_loader.py # 플러그인 동적 로드 -│ │ │ └── config.py -│ │ ├── strategies/ # 플러그인 전략 디렉토리 -│ │ │ ├── base.py # 전략 추상 클래스 -│ │ │ ├── rsi_strategy.py # 예시: RSI 전략 -│ │ │ └── grid_strategy.py # 예시: 그리드 전략 -│ │ ├── tests/ -│ │ ├── Dockerfile -│ │ └── pyproject.toml -│ │ -│ ├── order-executor/ -│ │ ├── src/ -│ │ │ ├── __init__.py -│ │ │ ├── main.py -│ │ │ ├── executor.py # 주문 실행 로직 -│ │ │ ├── risk_manager.py # 리스크 관리 (손절/익절) -│ │ │ └── config.py -│ │ ├── tests/ -│ │ ├── Dockerfile -│ │ └── pyproject.toml -│ │ -│ ├── portfolio-manager/ -│ │ ├── src/ -│ │ │ ├── __init__.py -│ │ │ ├── main.py -│ │ │ ├── portfolio.py # 잔고/포지션 추적 -│ │ │ ├── pnl.py # 손익 계산 -│ │ │ └── config.py -│ │ ├── tests/ -│ │ ├── Dockerfile -│ │ └── pyproject.toml -│ │ -│ └── backtester/ -│ ├── src/ -│ │ ├── __init__.py -│ │ ├── main.py -│ │ ├── engine.py # 백테스팅 엔진 -│ │ ├── simulator.py # 가상 주문 시뮬레이터 -│ │ ├── reporter.py # 결과 리포트 생성 -│ │ └── config.py -│ ├── tests/ -│ ├── Dockerfile -│ └── pyproject.toml -│ -├── shared/ -│ ├── src/shared/ -│ │ ├── __init__.py -│ │ ├── models.py # 공통 데이터 모델 -│ │ ├── events.py # 이벤트 타입 정의 -│ │ ├── broker.py # Redis Streams 클라이언트 -│ │ ├── db.py # DB 연결 (PostgreSQL) -│ │ └── config.py # 공통 설정 -│ ├── tests/ -│ └── pyproject.toml -│ -├── cli/ -│ ├── src/ -│ │ ├── __init__.py -│ │ ├── main.py # Click 기반 CLI 진입점 -│ │ ├── commands/ -│ │ │ ├── data.py # 데이터 수집 명령 -│ │ │ ├── trade.py # 매매 시작/중지 -│ │ │ ├── backtest.py # 백테스팅 실행 -│ │ │ ├── portfolio.py # 포트폴리오 조회 -│ │ │ └── strategy.py # 전략 관리 -│ │ └── config.py -│ ├── tests/ -│ └── pyproject.toml -│ -├── docker-compose.yml # 전체 서비스 오케스트레이션 -├── .env.example # 환경변수 템플릿 -├── Makefile # 공통 명령어 -└── README.md -``` - ---- - -## Tech Stack - -| 용도 | 라이브러리 | -|------|-----------| -| 거래소 API | **ccxt** | -| 메시지 브로커 | **Redis Streams** | -| DB | **PostgreSQL** + **asyncpg** | -| CLI | **Click** | -| 데이터 분석 | **pandas**, **numpy** | -| 기술 지표 | **pandas-ta** | -| 비동기 처리 | **asyncio** + **aiohttp** | -| 설정 관리 | **pydantic-settings** | -| 컨테이너 | **Docker** + **docker-compose** | -| 테스트 | **pytest** + **pytest-asyncio** | - ---- - -## Data Models - -### Core Models (shared/models.py) - -```python -class Candle: - symbol: str # "BTCUSDT" - timeframe: str # "1m", "5m", "1h" - open_time: datetime - open: Decimal - high: Decimal - low: Decimal - close: Decimal - volume: Decimal - -class Signal: - strategy: str # "rsi_strategy" - symbol: str - side: "BUY" | "SELL" - price: Decimal - quantity: Decimal - reason: str # 시그널 발생 근거 - -class Order: - id: str - signal_id: str # 추적용 - symbol: str - side: "BUY" | "SELL" - type: "MARKET" | "LIMIT" - price: Decimal - quantity: Decimal - status: "PENDING" | "FILLED" | "CANCELLED" | "FAILED" - created_at: datetime - filled_at: datetime | None - -class Position: - symbol: str - quantity: Decimal - avg_entry_price: Decimal - current_price: Decimal - unrealized_pnl: Decimal -``` - -### PostgreSQL Tables - -| 테이블 | 용도 | -|--------|------| -| `candles` | 시세 이력 (파티셔닝: symbol + timeframe) | -| `signals` | 전략 시그널 이력 | -| `orders` | 주문 이력 | -| `trades` | 체결 이력 | -| `positions` | 현재 포지션 | -| `portfolio_snapshots` | 일별 포트폴리오 스냅샷 | - -### Storage Strategy - -- **실시간 시세:** Redis 캐싱 + PostgreSQL 영구 저장 -- **주문/체결:** PostgreSQL 즉시 기록 -- **백테스팅 데이터:** PostgreSQL에서 bulk read (pandas DataFrame) - ---- - -## Strategy Plugin System - -### Base Interface - -```python -from abc import ABC, abstractmethod -from shared.models import Candle, Signal - -class BaseStrategy(ABC): - @abstractmethod - def on_candle(self, candle: Candle) -> Signal | None: - """캔들 데이터 수신 시 시그널 반환""" - pass - - @abstractmethod - def configure(self, params: dict) -> None: - """전략 파라미터 설정""" - pass -``` - -새 전략 추가 = `BaseStrategy` 상속 파일 하나 작성 후 `strategies/` 디렉토리에 배치. - -### 예시 전략 - -- **RSI Strategy:** RSI 과매도 시 매수, 과매수 시 매도 -- **Grid Strategy:** 가격 구간을 나눠 자동 매수/매도 주문 배치 - ---- - -## CLI Interface - -```bash -# 데이터 수집 -trading data collect --symbol BTCUSDT --timeframe 1m -trading data history --symbol BTCUSDT --from 2025-01-01 -trading data list - -# 자동매매 -trading trade start --strategy rsi --symbol BTCUSDT -trading trade stop --strategy rsi -trading trade status - -# 수동매매 -trading order buy --symbol BTCUSDT --quantity 0.01 -trading order sell --symbol BTCUSDT --price 70000 -trading order cancel --id abc123 - -# 백테스팅 -trading backtest run --strategy rsi --symbol BTCUSDT \ - --from 2025-01-01 --to 2025-12-31 -trading backtest report --id latest - -# 포트폴리오 -trading portfolio show -trading portfolio history --days 30 - -# 전략 관리 -trading strategy list -trading strategy info --name rsi - -# 서비스 관리 -trading service up -trading service down -trading service logs --name strategy-engine -``` - ---- - -## Risk Management - -### Risk Check Pipeline (order-executor) - -시그널 수신 시 다음 체크를 순서대로 통과해야 주문 실행: - -1. 최대 포지션 크기 초과 여부 -2. 일일 최대 손실 한도 도달 여부 -3. 동일 심볼 중복 주문 방지 -4. 주문 금액 < 가용 잔고 확인 -5. 가격 급변 감지 (슬리피지 보호) - -### Safety Mechanisms - -| 장치 | 설명 | -|------|------| -| **긴급 정지 (Kill Switch)** | `trading trade stop-all` — 모든 봇 중지, 미체결 주문 전량 취소 | -| **일일 손실 한도** | 설정 비율 초과 시 자동 매매 중단 | -| **최대 포지션 제한** | 총 자산 대비 단일 심볼 비율 제한 | -| **연결 끊김 대응** | Binance 연결 끊기면 신규 주문 중단, 재연결 시도 | -| **드라이런 모드** | 실제 주문 없이 시그널만 생성 — 전략 검증용 | - ---- - -## Configuration (.env) - -``` -BINANCE_API_KEY= -BINANCE_API_SECRET= -REDIS_URL=redis://localhost:6379 -DATABASE_URL=postgresql://user:pass@localhost:5432/trading -LOG_LEVEL=INFO -RISK_MAX_POSITION_SIZE=0.1 -RISK_STOP_LOSS_PCT=5 -RISK_DAILY_LOSS_LIMIT_PCT=10 -DRY_RUN=true -``` - ---- - -## Docker Compose Services - -```yaml -services: - redis: # 메시지 브로커 (항상 실행) - postgres: # 데이터 저장소 (항상 실행) - data-collector: # 시세 수집 (항상 실행) - strategy-engine: # 전략 엔진 (봇 실행 시) - order-executor: # 주문 실행 (봇 실행 시) - portfolio-manager: # 포트폴리오 (항상 실행) -``` diff --git a/docs/superpowers/specs/2026-04-01-operations-and-strategy-expansion-design.md b/docs/superpowers/specs/2026-04-01-operations-and-strategy-expansion-design.md deleted file mode 100644 index e1aea74..0000000 --- a/docs/superpowers/specs/2026-04-01-operations-and-strategy-expansion-design.md +++ /dev/null @@ -1,458 +0,0 @@ -# Operations Infrastructure & Strategy Expansion — Design Spec - -## Overview - -기존 Binance 현물 암호화폐 자동매매 플랫폼의 두 가지 영역을 강화한다: - -1. **운영 인프라** — DB 마이그레이션, 구조화된 로깅, Telegram 알림, 에러 복구, 메트릭 수집 -2. **전략 확장** — 추세 추종/스캘핑 전략 추가, 백테스트 고도화 - -접근 순서: 운영 인프라 먼저 완성 → 전략 추가. 안정적인 모니터링/알림 기반 위에서 새 전략을 검증할 수 있어야 한다. - ---- - -## Part 1: Operations Infrastructure - -### 1.1 DB Layer Migration (asyncpg → SQLAlchemy 2.0 Async + Alembic) - -**목표:** raw SQL과 asyncpg 직접 사용을 SQLAlchemy 2.0 async ORM으로 교체하고, Alembic으로 마이그레이션을 관리한다. - -**변경 사항:** - -- `shared/src/shared/db.py` — AsyncSession 기반으로 재작성 - - `create_async_engine()` + `async_sessionmaker()` 사용 - - asyncpg는 SQLAlchemy의 내부 드라이버로 유지 (`postgresql+asyncpg://`) - - 기존 raw SQL 함수들을 ORM 쿼리로 전환 - -- `shared/src/shared/sa_models.py` — SQLAlchemy ORM 모델 (신규) - - 기존 Pydantic 모델(models.py)과 1:1 매핑되는 SA 테이블 정의 - - `Candle`, `Signal`, `Order`, `Trade`, `Position`, `PortfolioSnapshot` 테이블 - - Pydantic 모델은 이벤트 직렬화/API 전용으로 유지 - -- `shared/alembic/` — Alembic 마이그레이션 환경 (신규) - - `alembic.ini` — 설정 파일 (DATABASE_URL 참조) - - `env.py` — async 엔진 설정, SA 모델 메타데이터 참조 - - `versions/` — 마이그레이션 파일들 - - 초기 마이그레이션: 기존 `db.py`의 CREATE TABLE 로직을 마이그레이션으로 이전 - -- `Makefile` 타겟 추가: - - `make migrate` — `alembic upgrade head` - - `make migrate-down` — `alembic downgrade -1` - - `make migrate-new MSG="description"` — `alembic revision --autogenerate -m "description"` - -- 각 서비스의 DB 접근 코드를 AsyncSession 기반으로 업데이트: - - `data-collector/storage.py` — bulk insert 쿼리를 SA ORM으로 - - `order-executor/executor.py` — order CRUD를 SA ORM으로 - - `portfolio-manager/portfolio.py` — position/snapshot 쿼리를 SA ORM으로 - - `backtester/engine.py` — candle 조회를 SA ORM으로 - -**의존성 추가:** `sqlalchemy[asyncio]>=2.0`, `alembic>=1.13` -**의존성 제거:** `asyncpg` (직접 의존 → SQLAlchemy 내부 의존으로 변경) - ---- - -### 1.2 Structured Logging (structlog) - -**목표:** 전 서비스에 JSON 구조화 로깅을 적용하고, 에러 로그를 Telegram 알림과 연결한다. - -**변경 사항:** - -- `shared/src/shared/logging.py` (신규) - - `setup_logging(service_name: str, log_level: str)` 함수 - - structlog 프로세서 체인: timestamp, log level, service_name 바인딩, JSON 렌더러 - - 개발 환경: 컬러 콘솔 출력 / 프로덕션: JSON stdout - - `LOG_FORMAT` 환경변수로 전환 (`console` | `json`, 기본값: `json`) - -- 각 서비스 `main.py`에서 `setup_logging()` 호출 -- 기존 `logging.getLogger()` 호출을 `structlog.get_logger()` 로 교체 -- 컨텍스트 바인딩 예시: - ```python - log = structlog.get_logger().bind(service="strategy-engine", symbol="BTCUSDT") - log.info("signal_generated", strategy="rsi", side="BUY", price=68500) - ``` - -- ERROR 이상 로그 → Telegram 알림 트리거 (1.3절 TelegramNotifier 연동) - - structlog 커스텀 프로세서로 구현 - - 알림 전송 실패 시 로그만 남기고 서비스 중단하지 않음 - -**의존성 추가:** `structlog>=24.0` - ---- - -### 1.3 Telegram Notification Service - -**목표:** 주요 이벤트(시그널, 주문, 에러, 일일 요약)를 Telegram으로 전송한다. - -**변경 사항:** - -- `shared/src/shared/notifier.py` (신규) - - `TelegramNotifier` 클래스 - - `__init__(bot_token: str, chat_id: str)` — aiohttp 세션 관리 - - `send(message: str, parse_mode: str = "HTML")` — 메시지 전송 - - `send_signal(signal: Signal)` — 시그널 포맷팅 후 전송 - - `send_order(order: Order)` — 주문 체결/실패 알림 - - `send_error(error: str, service: str)` — 에러 알림 - - `send_daily_summary(positions: list, pnl: Decimal)` — 일일 요약 - - Rate limiting: 초당 최대 1건 (asyncio.Semaphore + 큐) - - 연결 실패 시 최대 3회 재시도, 실패해도 서비스 중단하지 않음 - -- `shared/src/shared/config.py` — 설정 추가: - - `TELEGRAM_BOT_TOKEN: str = ""` - - `TELEGRAM_CHAT_ID: str = ""` - - `TELEGRAM_ENABLED: bool = False` (토큰 미설정 시 자동 비활성) - -- `.env.example` 업데이트: - ``` - TELEGRAM_BOT_TOKEN= - TELEGRAM_CHAT_ID= - TELEGRAM_ENABLED=false - ``` - -- 연동 포인트: - - `strategy-engine/engine.py` — 시그널 생성 시 `send_signal()` - - `order-executor/executor.py` — 주문 체결/실패 시 `send_order()` - - `shared/logging.py` — ERROR 로그 시 `send_error()` - - `portfolio-manager/main.py` — 매일 자정(UTC) `send_daily_summary()` - -**의존성:** aiohttp (이미 존재) - ---- - -### 1.4 Error Recovery & Health Checks - -**목표:** 서비스 장애 시 자동 복구하고, 헬스체크 엔드포인트로 상태를 모니터링한다. - -**변경 사항:** - -- `shared/src/shared/resilience.py` (신규) - - `retry_with_backoff(func, max_retries, base_delay)` — exponential backoff 데코레이터 - - 지터(jitter) 포함: `delay * (1 + random(0, 0.5))` - - 최대 지연: 60초 - - `CircuitBreaker` 클래스: - - 상태: CLOSED(정상) → OPEN(차단) → HALF_OPEN(시험) - - `failure_threshold`: 연속 실패 N회 시 OPEN (기본: 5) - - `recovery_timeout`: OPEN 후 N초 뒤 HALF_OPEN (기본: 60) - - OPEN 전환 시 Telegram 알림 전송 - -- `shared/src/shared/healthcheck.py` (신규) - - `HealthCheckServer` — aiohttp 기반 경량 HTTP 서버 - - `GET /health` → `{"status": "ok", "service": "...", "uptime": ..., "checks": {...}}` - - 체크 항목: Redis 연결, PostgreSQL 연결, Binance WS 연결(해당 서비스만) - - 포트: `HEALTH_PORT` 환경변수 (서비스별 다르게 설정) - -- 각 서비스에 적용: - - `data-collector` — Binance WS 재연결 (backoff), Redis/DB 재연결 - - `strategy-engine` — Redis 소비자 재연결 - - `order-executor` — 거래소 API 호출 재시도 (circuit breaker) - - `portfolio-manager` — Redis/DB 재연결 - -- `docker-compose.yml` — healthcheck를 `/health` 엔드포인트로 변경 - -- `shared/src/shared/config.py` — 설정 추가: - - `HEALTH_PORT: int = 8080` - - `CIRCUIT_BREAKER_THRESHOLD: int = 5` - - `CIRCUIT_BREAKER_TIMEOUT: int = 60` - ---- - -### 1.5 Prometheus Metrics - -**목표:** 각 서비스의 주요 지표를 Prometheus 포맷으로 노출한다. - -**변경 사항:** - -- `shared/src/shared/metrics.py` (신규) - - `MetricsServer` — prometheus_client 기반 - - `/metrics` 엔드포인트 (healthcheck 서버에 통합) - - 공통 메트릭: - - `service_up` (Gauge) — 서비스 상태 - - `errors_total` (Counter) — 에러 횟수 (label: service, error_type) - - `event_processing_seconds` (Histogram) — 이벤트 처리 시간 - -- 서비스별 메트릭: - - **data-collector:** - - `candles_received_total` (Counter) — 수신 캔들 수 - - `ws_reconnections_total` (Counter) — WS 재연결 횟수 - - **strategy-engine:** - - `signals_generated_total` (Counter, label: strategy, side) - - `strategy_execution_seconds` (Histogram, label: strategy) - - **order-executor:** - - `orders_total` (Counter, label: status, side) - - `risk_rejections_total` (Counter, label: reason) - - **portfolio-manager:** - - `portfolio_value` (Gauge) — 총 포트폴리오 가치 - - `unrealized_pnl` (Gauge, label: symbol) - -- `docker-compose.yml` — Prometheus + Grafana 서비스 추가 (선택적 프로필): - ```yaml - prometheus: - image: prom/prometheus:latest - profiles: ["monitoring"] - volumes: - - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml - grafana: - image: grafana/grafana:latest - profiles: ["monitoring"] - ports: - - "3000:3000" - ``` - -- `monitoring/prometheus.yml` (신규) — 스크래핑 설정 -- `monitoring/grafana/` (신규) — 대시보드 프로비저닝 (선택적) - -**의존성 추가:** `prometheus-client>=0.20` - ---- - -## Part 2: Strategy Expansion - -### 2.1 Trend Following Strategies - -**MACD Strategy** (`strategies/macd_strategy.py`) -- MACD line = EMA(12) - EMA(26), Signal line = EMA(9) of MACD -- BUY: MACD가 Signal line 위로 교차 + 히스토그램 양전환 -- SELL: MACD가 Signal line 아래로 교차 + 히스토그램 음전환 -- 파라미터: `fast_period=12`, `slow_period=26`, `signal_period=9`, `quantity` -- warmup_period: `slow_period + signal_period` - -**Bollinger Bands Strategy** (`strategies/bollinger_strategy.py`) -- 중심선 = SMA(20), 상단 = 중심 + 2*std, 하단 = 중심 - 2*std -- BUY: 가격이 하단 밴드 아래로 이탈 후 복귀 -- SELL: 가격이 상단 밴드 위로 이탈 후 복귀 -- 변동성 필터: 밴드 폭이 임계값 미만이면 시그널 무시 (횡보장 필터) -- 파라미터: `period=20`, `num_std=2.0`, `min_bandwidth=0.02`, `quantity` -- warmup_period: `period` - -**EMA Crossover Strategy** (`strategies/ema_crossover_strategy.py`) -- 단기 EMA와 장기 EMA 교차 -- BUY: 단기 EMA가 장기 EMA 위로 교차 (Golden Cross) -- SELL: 단기 EMA가 장기 EMA 아래로 교차 (Death Cross) -- 파라미터: `short_period=9`, `long_period=21`, `quantity` -- warmup_period: `long_period` - ---- - -### 2.2 Scalping Strategies - -**VWAP Strategy** (`strategies/vwap_strategy.py`) -- VWAP = cumsum(price * volume) / cumsum(volume) -- BUY: 가격이 VWAP 아래에서 VWAP으로 복귀 (평균 회귀) -- SELL: 가격이 VWAP 위에서 VWAP으로 복귀 -- 일중 리셋: UTC 00:00에 VWAP 재계산 -- 파라미터: `deviation_threshold=0.002`, `quantity` -- warmup_period: 최소 30 캔들 - -**Volume Profile Strategy** (`strategies/volume_profile_strategy.py`) -- 최근 N개 캔들의 가격대별 거래량 분포 계산 -- POC (Point of Control): 가장 거래량이 많은 가격대 -- Value Area: 전체 거래량 70%가 집중된 구간 -- BUY: 가격이 Value Area 하단 지지선에서 반등 -- SELL: 가격이 Value Area 상단 저항선에서 거부 -- 파라미터: `lookback_period=100`, `num_bins=50`, `value_area_pct=0.7`, `quantity` -- warmup_period: `lookback_period` - ---- - -### 2.3 Strategy Common Improvements - -**BaseStrategy 확장:** -```python -class BaseStrategy(ABC): - @property - @abstractmethod - def warmup_period(self) -> int: - """지표 계산에 필요한 최소 캔들 수""" - pass - - @abstractmethod - def on_candle(self, candle: Candle) -> Signal | None: - pass - - @abstractmethod - def configure(self, params: dict) -> None: - pass - - def reset(self) -> None: - """전략 상태 초기화 (백테스트 간 재사용)""" - pass -``` - -**전략 파라미터 외부화:** -- `strategies/config/` 디렉토리에 YAML 설정 파일 -- 파일명: `{strategy_name}.yaml` (예: `rsi_strategy.yaml`) -- 구조: - ```yaml - # rsi_strategy.yaml - period: 14 - oversold: 30 - overbought: 70 - quantity: 0.001 - ``` -- `plugin_loader.py`가 전략 로드 시 자동으로 같은 이름의 YAML을 찾아 `configure()` 호출 -- CLI에서 `--param key=value`로 런타임 오버라이드 가능 - -**기존 전략 업데이트:** -- `RsiStrategy`, `GridStrategy`에 `warmup_period` 속성 추가 -- `reset()` 메서드 구현 - -**의존성 추가:** `pyyaml>=6.0` - ---- - -### 2.4 Backtest Enhancement - -**DetailedMetrics 데이터클래스** (`backtester/src/backtester/metrics.py`, 신규): -```python -@dataclass -class TradeRecord: - entry_time: datetime - exit_time: datetime - symbol: str - side: str - entry_price: Decimal - exit_price: Decimal - quantity: Decimal - pnl: Decimal - pnl_pct: float - holding_period: timedelta - -@dataclass -class DetailedMetrics: - # 기본 - total_return: float - total_trades: int - winning_trades: int - losing_trades: int - win_rate: float - profit_factor: float - - # 리스크 메트릭 - sharpe_ratio: float - sortino_ratio: float - calmar_ratio: float - max_drawdown: float - max_drawdown_duration: timedelta - - # 수익률 분석 - monthly_returns: dict[str, float] # "2025-01": 0.05 - avg_win: float - avg_loss: float - largest_win: float - largest_loss: float - avg_holding_period: timedelta - - # 개별 거래 - trades: list[TradeRecord] -``` - -**BacktestEngine 확장:** -- `engine.py`에 `DetailedMetrics` 계산 로직 추가 -- `simulator.py`에 `TradeRecord` 생성 로직 추가 (진입/청산 시점 기록) -- Sharpe ratio = `mean(daily_returns) / std(daily_returns) * sqrt(365)` (crypto는 365일) -- Sortino ratio = `mean(daily_returns) / downside_std * sqrt(365)` -- Calmar ratio = `annualized_return / max_drawdown` -- Max drawdown = `max(peak - trough) / peak` - -**Reporter 개선:** -- `reporter.py` — rich 라이브러리로 테이블 출력 - - 요약 테이블: 핵심 메트릭 - - 월별 수익률 테이블 - - 최고/최악 거래 Top 5 -- CSV/JSON 내보내기: `--output csv` / `--output json` 플래그 - -**CLI 확장:** -- `trading backtest run` — 기존 출력에 상세 메트릭 추가 -- `trading backtest run --output csv --file result.csv` — 결과 내보내기 - -**의존성 추가:** `rich>=13.0` - ---- - -## Updated Tech Stack - -| 용도 | 기존 | 변경 | -|------|------|------| -| DB ORM | asyncpg (raw SQL) | **SQLAlchemy 2.0 async** (asyncpg 드라이버) | -| 마이그레이션 | 없음 | **Alembic** | -| 로깅 | Python logging | **structlog** | -| 알림 | 없음 | **Telegram Bot API** (aiohttp) | -| 메트릭 | 없음 | **prometheus-client** | -| 전략 설정 | 하드코딩 | **YAML** (pyyaml) | -| 리포트 출력 | print | **rich** | - ---- - -## Updated .env.example - -```env -# Exchange -BINANCE_API_KEY= -BINANCE_API_SECRET= - -# Infrastructure -REDIS_URL=redis://localhost:6379 -DATABASE_URL=postgresql+asyncpg://trading:trading@localhost:5432/trading - -# Logging -LOG_LEVEL=INFO -LOG_FORMAT=json - -# Telegram -TELEGRAM_BOT_TOKEN= -TELEGRAM_CHAT_ID= -TELEGRAM_ENABLED=false - -# Risk Management -RISK_MAX_POSITION_SIZE=0.1 -RISK_STOP_LOSS_PCT=5 -RISK_DAILY_LOSS_LIMIT_PCT=10 -DRY_RUN=true - -# Health & Metrics -HEALTH_PORT=8080 -CIRCUIT_BREAKER_THRESHOLD=5 -CIRCUIT_BREAKER_TIMEOUT=60 -``` - ---- - -## New Files Summary - -| 파일 | 용도 | -|------|------| -| `shared/src/shared/sa_models.py` | SQLAlchemy ORM 모델 | -| `shared/src/shared/logging.py` | structlog 설정 | -| `shared/src/shared/notifier.py` | Telegram 알림 | -| `shared/src/shared/resilience.py` | retry, circuit breaker | -| `shared/src/shared/healthcheck.py` | 헬스체크 서버 | -| `shared/src/shared/metrics.py` | Prometheus 메트릭 | -| `shared/alembic/` | DB 마이그레이션 환경 | -| `strategies/config/*.yaml` | 전략 파라미터 설정 | -| `strategies/macd_strategy.py` | MACD 전략 | -| `strategies/bollinger_strategy.py` | Bollinger Bands 전략 | -| `strategies/ema_crossover_strategy.py` | EMA Crossover 전략 | -| `strategies/vwap_strategy.py` | VWAP 전략 | -| `strategies/volume_profile_strategy.py` | Volume Profile 전략 | -| `backtester/src/backtester/metrics.py` | 상세 백테스트 메트릭 | -| `monitoring/prometheus.yml` | Prometheus 설정 | - ---- - -## Scope Boundaries - -**포함:** -- SQLAlchemy 2.0 async 전환 + Alembic 마이그레이션 -- structlog JSON 로깅 -- Telegram 알림 (시그널, 주문, 에러, 일일 요약) -- 에러 복구 (retry, circuit breaker) + 헬스체크 -- Prometheus 메트릭 수집 -- 5개 신규 전략 (MACD, Bollinger, EMA Crossover, VWAP, Volume Profile) -- BaseStrategy에 warmup_period, reset() 추가 -- YAML 기반 전략 파라미터 -- 백테스트 상세 메트릭 + rich 리포트 - -**제외:** -- Grafana 대시보드 프로비저닝 (Prometheus만 설정, 대시보드는 수동) -- 멀티 거래소 지원 -- REST API / 웹 대시보드 -- 전략 조합 프레임워크 (향후 확장) diff --git a/docs/superpowers/specs/2026-04-02-news-driven-stock-selector-design.md b/docs/superpowers/specs/2026-04-02-news-driven-stock-selector-design.md new file mode 100644 index 0000000..d439154 --- /dev/null +++ b/docs/superpowers/specs/2026-04-02-news-driven-stock-selector-design.md @@ -0,0 +1,418 @@ +# News-Driven Stock Selector Design + +**Date:** 2026-04-02 +**Goal:** Upgrade the MOC (Market on Close) strategy from fixed symbol lists to dynamic, news-driven stock selection. The system collects news/sentiment data continuously, then selects 2-3 optimal stocks daily before market close. + +--- + +## Architecture Overview + +``` +[Continuous Collection] [Pre-Close Decision] +Finnhub News ─┐ +RSS Feeds ─┤ +SEC EDGAR ─┤ +Truth Social ─┼→ DB (news_items) → Sentiment Aggregator → symbol_scores +Reddit ─┤ + Redis "news" (every 15 min) market_sentiment +Fear & Greed ─┤ +FOMC/Fed ─┘ + + 15:00 ET ─→ Candidate Pool (sentiment top + LLM picks) + 15:15 ET ─→ Technical Filter (RSI, EMA, volume) + 15:30 ET ─→ LLM Final Selection (2-3 stocks) → Telegram + 15:50 ET ─→ MOC Buy Execution + 09:35 ET ─→ Next-day Sell (existing MOC logic) +``` + +## 1. News Collector Service + +New service: `services/news-collector/` + +### Structure + +``` +services/news-collector/ +├── Dockerfile +├── pyproject.toml +├── src/news_collector/ +│ ├── __init__.py +│ ├── main.py # Scheduler: runs each collector on its interval +│ ├── config.py +│ └── collectors/ +│ ├── __init__.py +│ ├── base.py # BaseCollector ABC +│ ├── finnhub.py # Finnhub market news (free, 60 req/min) +│ ├── rss.py # Yahoo Finance, Google News, MarketWatch RSS +│ ├── sec_edgar.py # SEC EDGAR 8-K/10-Q filings +│ ├── truth_social.py # Truth Social scraping (Trump posts) +│ ├── reddit.py # Reddit (r/wallstreetbets, r/stocks) +│ ├── fear_greed.py # CNN Fear & Greed Index scraping +│ └── fed.py # FOMC statements, Fed announcements +└── tests/ +``` + +### BaseCollector Interface + +```python +class BaseCollector(ABC): + name: str + poll_interval: int # seconds + + @abstractmethod + async def collect(self) -> list[NewsItem]: + """Collect and return list of NewsItem.""" + + @abstractmethod + async def is_available(self) -> bool: + """Check if this source is accessible (API key present, endpoint reachable).""" +``` + +### Poll Intervals + +| Collector | Interval | Notes | +|-----------|----------|-------| +| Finnhub | 5 min | Free tier: 60 calls/min | +| RSS (Yahoo/Google/MarketWatch) | 10 min | Headlines only | +| SEC EDGAR | 30 min | Focus on 8-K filings | +| Truth Social | 15 min | Scraping | +| Reddit | 15 min | Hot posts from relevant subs | +| Fear & Greed | 1 hour | Updates once daily but check periodically | +| FOMC/Fed | 1 hour | Infrequent events | + +### Provider Abstraction (for paid upgrade path) + +```python +# config.yaml +collectors: + news: + provider: "finnhub" # swap to "benzinga" for paid + api_key: ${FINNHUB_API_KEY} + social: + provider: "reddit" # swap to "stocktwits_pro" etc. + policy: + provider: "truth_social" # swap to "twitter_api" etc. + +# Factory +COLLECTOR_REGISTRY = { + "finnhub": FinnhubCollector, + "rss": RSSCollector, + "benzinga": BenzingaCollector, # added later +} +``` + +## 2. Shared Models (additions to shared/) + +### NewsItem (shared/models.py) + +```python +class NewsCategory(str, Enum): + POLICY = "policy" + EARNINGS = "earnings" + MACRO = "macro" + SOCIAL = "social" + FILING = "filing" + FED = "fed" + +class NewsItem(BaseModel): + id: str = Field(default_factory=lambda: str(uuid.uuid4())) + source: str # "finnhub", "rss", "sec_edgar", etc. + headline: str + summary: str | None = None + url: str | None = None + published_at: datetime + symbols: list[str] = [] # Related tickers (if identifiable) + sentiment: float # -1.0 to 1.0 (first-pass analysis at collection) + category: NewsCategory + raw_data: dict = {} + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) +``` + +### SymbolScore (shared/sentiment_models.py — new file) + +```python +class SymbolScore(BaseModel): + symbol: str + news_score: float # -1.0 to 1.0, weighted avg of news sentiment + news_count: int # Number of news items in last 24h + social_score: float # Reddit/social sentiment + policy_score: float # Policy-related impact + filing_score: float # SEC filing impact + composite: float # Weighted final score + updated_at: datetime + +class MarketSentiment(BaseModel): + fear_greed: int # 0-100 + fear_greed_label: str # "Extreme Fear", "Fear", "Neutral", "Greed", "Extreme Greed" + vix: float | None = None + fed_stance: str # "hawkish", "neutral", "dovish" + market_regime: str # "risk_on", "neutral", "risk_off" + updated_at: datetime + +class SelectedStock(BaseModel): + symbol: str + side: OrderSide # BUY or SELL + conviction: float # 0.0 to 1.0 + reason: str # Selection rationale + key_news: list[str] # Key news headlines + +class Candidate(BaseModel): + symbol: str + source: str # "sentiment" or "llm" + direction: OrderSide | None = None # Suggested direction (if known) + score: float # Relevance/priority score + reason: str # Why this candidate was selected +``` + +## 3. Sentiment Analysis Pipeline + +### Location + +Refactor existing `shared/src/shared/sentiment.py`. + +### Two-Stage Analysis + +**Stage 1: Per-news sentiment (at collection time)** +- VADER (nltk.sentiment, free) for English headlines +- Keyword rule engine for domain-specific terms (e.g., "tariff" → negative for importers, positive for domestic producers) +- Score stored in `NewsItem.sentiment` + +**Stage 2: Per-symbol aggregation (every 15 minutes)** + +``` +composite = ( + news_score * 0.3 + + social_score * 0.2 + + policy_score * 0.3 + + filing_score * 0.2 +) * freshness_decay +``` + +Freshness decay: +- < 1 hour: 1.0 +- 1-6 hours: 0.7 +- 6-24 hours: 0.3 +- > 24 hours: excluded + +Policy score weighted high because US stock market is heavily influenced by policy events (tariffs, regulation, subsidies). + +### Market-Level Gating + +`MarketSentiment.market_regime` determination: +- `risk_off`: Fear & Greed < 20 OR VIX > 30 → **block all trades** +- `risk_on`: Fear & Greed > 60 AND VIX < 20 +- `neutral`: everything else + +This extends the existing `sentiment.py` `should_block()` logic. + +## 4. Stock Selector Engine + +### Location + +`services/strategy-engine/src/strategy_engine/stock_selector.py` + +### Three-Stage Selection Process + +**Stage 1: Candidate Pool (15:00 ET)** + +Two candidate sources, results merged (deduplicated): + +```python +class CandidateSource(ABC): + @abstractmethod + async def get_candidates(self) -> list[Candidate] + +class SentimentCandidateSource(CandidateSource): + """Top N symbols by composite SymbolScore from DB.""" + +class LLMCandidateSource(CandidateSource): + """Send today's top news summary to Claude, get related symbols + direction.""" +``` + +- SentimentCandidateSource: top 20 by composite score +- LLMCandidateSource: Claude analyzes today's major news and recommends affected symbols +- Merged pool: typically 20-30 candidates + +**Stage 2: Technical Filter (15:15 ET)** + +Apply existing MOC screening criteria to candidates: +- Fetch recent price data from Alpaca for all candidates +- RSI 30-60 +- Price > 20-period EMA +- Volume > average +- Bullish candle pattern +- Result: typically 5-10 survivors + +**Stage 3: LLM Final Selection (15:30 ET)** + +Send to Claude: +- Filtered candidate list with technical indicators +- Per-symbol sentiment scores and top news headlines +- Market sentiment (Fear & Greed, VIX, Fed stance) +- Prompt: "Select 2-3 stocks for MOC trading with rationale" + +Response parsed into `list[SelectedStock]`. + +### Integration with MOC Strategy + +Current: MOC strategy receives candles for fixed symbols and decides internally. + +New flow: +1. `StockSelector` publishes `SelectedStock` list to Redis stream `selected_stocks` at 15:30 ET +2. MOC strategy reads `selected_stocks` to get today's targets +3. MOC still applies its own technical checks at 15:50-16:00 as a safety net +4. If a selected stock fails the final technical check, it's skipped (no forced trades) + +## 5. Database Schema + +Four new tables via Alembic migration: + +```sql +CREATE TABLE news_items ( + id UUID PRIMARY KEY, + source VARCHAR(50) NOT NULL, + headline TEXT NOT NULL, + summary TEXT, + url TEXT, + published_at TIMESTAMPTZ NOT NULL, + symbols TEXT[], + sentiment FLOAT NOT NULL, + category VARCHAR(50) NOT NULL, + raw_data JSONB DEFAULT '{}', + created_at TIMESTAMPTZ DEFAULT NOW() +); +CREATE INDEX idx_news_items_published ON news_items(published_at); +CREATE INDEX idx_news_items_symbols ON news_items USING GIN(symbols); + +CREATE TABLE symbol_scores ( + id UUID PRIMARY KEY, + symbol VARCHAR(10) NOT NULL, + news_score FLOAT NOT NULL DEFAULT 0, + news_count INT NOT NULL DEFAULT 0, + social_score FLOAT NOT NULL DEFAULT 0, + policy_score FLOAT NOT NULL DEFAULT 0, + filing_score FLOAT NOT NULL DEFAULT 0, + composite FLOAT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL +); +CREATE UNIQUE INDEX idx_symbol_scores_symbol ON symbol_scores(symbol); + +CREATE TABLE market_sentiment ( + id UUID PRIMARY KEY, + fear_greed INT NOT NULL, + fear_greed_label VARCHAR(30) NOT NULL, + vix FLOAT, + fed_stance VARCHAR(20) NOT NULL DEFAULT 'neutral', + market_regime VARCHAR(20) NOT NULL DEFAULT 'neutral', + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE stock_selections ( + id UUID PRIMARY KEY, + trade_date DATE NOT NULL, + symbol VARCHAR(10) NOT NULL, + side VARCHAR(4) NOT NULL, + conviction FLOAT NOT NULL, + reason TEXT NOT NULL, + key_news JSONB DEFAULT '[]', + sentiment_snapshot JSONB DEFAULT '{}', + created_at TIMESTAMPTZ DEFAULT NOW() +); +CREATE INDEX idx_stock_selections_date ON stock_selections(trade_date); +``` + +`stock_selections` stores an audit trail: why each stock was selected, enabling post-hoc analysis of selection quality. + +## 6. Redis Streams + +| Stream | Producer | Consumer | Payload | +|--------|----------|----------|---------| +| `news` | news-collector | strategy-engine (sentiment aggregator) | NewsItem | +| `selected_stocks` | stock-selector | MOC strategy | SelectedStock | + +Existing streams (`candles`, `signals`, `orders`) unchanged. + +## 7. Docker Compose Addition + +```yaml +news-collector: + build: + context: . + dockerfile: services/news-collector/Dockerfile + env_file: .env + ports: + - "8084:8084" + depends_on: + redis: { condition: service_healthy } + postgres: { condition: service_healthy } + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8084/health')"] + interval: 10s + timeout: 5s + retries: 3 + restart: unless-stopped +``` + +## 8. Environment Variables + +```bash +# News Collector +FINNHUB_API_KEY= # Free key from finnhub.io +NEWS_POLL_INTERVAL=300 # Default 5 min (overrides per-collector defaults) +SENTIMENT_AGGREGATE_INTERVAL=900 # 15 min + +# Stock Selector +SELECTOR_CANDIDATES_TIME=15:00 # ET, candidate pool generation +SELECTOR_FILTER_TIME=15:15 # ET, technical filter +SELECTOR_FINAL_TIME=15:30 # ET, LLM final pick +SELECTOR_MAX_PICKS=3 + +# LLM (for stock selector + screener) +ANTHROPIC_API_KEY= +ANTHROPIC_MODEL=claude-sonnet-4-20250514 +``` + +## 9. Telegram Notifications + +Extend existing `shared/notifier.py` with: + +```python +async def send_stock_selection(self, selections: list[SelectedStock], market: MarketSentiment): + """ + 📊 오늘의 종목 선정 (2/3) + + 1. NVDA 🟢 BUY (확신도: 0.85) + 근거: 트럼프 반도체 보조금 확대 발표, RSI 42 + 핵심뉴스: "Trump signs CHIPS Act expansion..." + + 2. XOM 🟢 BUY (확신도: 0.72) + 근거: 유가 상승 + 실적 서프라이즈, 볼륨 급증 + + 시장심리: Fear & Greed 55 (Neutral) | VIX 18.2 + """ +``` + +## 10. Testing Strategy + +**Unit tests:** +- Each collector: mock HTTP responses → verify NewsItem parsing +- Sentiment analysis: verify VADER + keyword scoring +- Aggregator: mock news data → verify SymbolScore calculation and freshness decay +- Stock selector: mock scores → verify candidate/filter/selection pipeline +- LLM calls: mock Claude response → verify SelectedStock parsing + +**Integration tests:** +- Full pipeline: news collection → DB → aggregation → selection +- Market gating: verify `risk_off` blocks all trades +- MOC integration: verify selected stocks flow to MOC strategy + +**Post-hoc analysis (future):** +- Use `stock_selections` audit trail to measure selection accuracy +- Historical news data replay for backtesting requires paid data (deferred) + +## 11. Out of Scope (Future) + +- Paid API integration (designed for, not implemented) +- Historical news backtesting +- WebSocket real-time news streaming +- Multi-language sentiment analysis +- Options/derivatives signals diff --git a/pyproject.toml b/pyproject.toml index 545eae1..6938778 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "trading-platform" version = "0.1.0" -description = "Binance spot crypto trading platform" +description = "US stock trading platform" requires-python = ">=3.12" [tool.pytest.ini_options] diff --git a/scripts/backtest_moc.py b/scripts/backtest_moc.py new file mode 100755 index 0000000..92b426b --- /dev/null +++ b/scripts/backtest_moc.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Backtest and optimize MOC strategy on synthetic US stock data. + +Usage: python scripts/backtest_moc.py +""" + +import sys +import random +from pathlib import Path +from decimal import Decimal +from datetime import datetime, timedelta, timezone + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "services" / "strategy-engine" / "src")) +sys.path.insert(0, str(ROOT / "services" / "strategy-engine")) +sys.path.insert(0, str(ROOT / "services" / "backtester" / "src")) +sys.path.insert(0, str(ROOT / "shared" / "src")) + +from shared.models import Candle # noqa: E402 +from backtester.engine import BacktestEngine # noqa: E402 +from strategies.moc_strategy import MocStrategy # noqa: E402 + + +def generate_stock_candles( + symbol: str = "AAPL", + days: int = 90, + base_price: float = 180.0, + daily_drift: float = 0.0003, # Slight upward bias + daily_vol: float = 0.015, # 1.5% daily vol +) -> list[Candle]: + """Generate realistic US stock intraday candles (5-min bars). + + Simulates: + - Market hours only (14:30-21:00 UTC = 9:30-16:00 ET) + - Opening gaps (overnight news effect) + - Intraday volatility pattern (higher at open/close) + - Volume pattern (U-shaped: high at open, low midday, high at close) + """ + candles = [] + price = base_price + start_date = datetime(2025, 1, 2, tzinfo=timezone.utc) # Start on a Thursday + + trading_day = 0 + current_date = start_date + + while trading_day < days: + # Skip weekends + if current_date.weekday() >= 5: + current_date += timedelta(days=1) + continue + + # Opening gap: overnight news effect + gap_pct = random.gauss(daily_drift, daily_vol * 0.5) # Gap is ~50% of daily vol + price *= 1 + gap_pct + + # Generate 78 5-minute bars (9:30-16:00 = 6.5 hours = 78 bars) + intraday_bars = 78 + for bar in range(intraday_bars): + # Time: 14:30 UTC + bar * 5 minutes + dt = current_date.replace(hour=14, minute=30) + timedelta(minutes=bar * 5) + + # Intraday volatility pattern (U-shaped) + hour_of_day = bar / intraday_bars + if hour_of_day < 0.1: # First 10% of day (opening) + vol = daily_vol * 0.003 + elif hour_of_day > 0.9: # Last 10% (closing) + vol = daily_vol * 0.0025 + else: # Middle of day + vol = daily_vol * 0.001 + + # Add daily trend component + intraday_drift = daily_drift / intraday_bars + change = random.gauss(intraday_drift, vol) + + open_p = price + close_p = price * (1 + change) + high_p = max(open_p, close_p) * (1 + abs(random.gauss(0, vol * 0.3))) + low_p = min(open_p, close_p) * (1 - abs(random.gauss(0, vol * 0.3))) + + # Volume pattern (U-shaped) + if hour_of_day < 0.1 or hour_of_day > 0.85: + volume = random.uniform(500000, 2000000) + else: + volume = random.uniform(100000, 500000) + + candles.append( + Candle( + symbol=symbol, + timeframe="5Min", + open_time=dt, + open=Decimal(str(round(open_p, 2))), + high=Decimal(str(round(high_p, 2))), + low=Decimal(str(round(low_p, 2))), + close=Decimal(str(round(close_p, 2))), + volume=Decimal(str(int(volume))), + ) + ) + + price = close_p + + trading_day += 1 + current_date += timedelta(days=1) + + return candles + + +def run_backtest(candles, params, balance=750.0): + """Run a single backtest.""" + strategy = MocStrategy() + strategy.configure(params) + + engine = BacktestEngine( + strategy=strategy, + initial_balance=Decimal(str(balance)), + slippage_pct=0.0005, # 0.05% slippage (stocks have tighter spreads) + taker_fee_pct=0.0, # Alpaca = 0% commission + ) + return engine.run(candles) + + +def main(): + random.seed(42) + + print("=" * 60) + print("MOC Strategy Backtest — US Stocks") + print("Capital: $750 (~100만원)") + print("=" * 60) + + # Test across multiple stocks + stocks = [ + ("AAPL", 180.0, 0.0003, 0.015), + ("MSFT", 420.0, 0.0004, 0.014), + ("TSLA", 250.0, 0.0001, 0.030), + ("NVDA", 800.0, 0.0005, 0.025), + ("AMZN", 185.0, 0.0003, 0.018), + ] + + # Parameter grid + param_sets = [] + for rsi_min in [25, 30, 35]: + for rsi_max in [55, 60, 65]: + for sl in [1.5, 2.0, 3.0]: + for ema in [10, 20]: + param_sets.append( + { + "quantity_pct": 0.2, + "stop_loss_pct": sl, + "rsi_min": rsi_min, + "rsi_max": rsi_max, + "ema_period": ema, + "volume_avg_period": 20, + "min_volume_ratio": 0.8, + "buy_start_utc": 19, + "buy_end_utc": 21, + "sell_start_utc": 14, + "sell_end_utc": 15, + "max_positions": 5, + } + ) + + print(f"\nParameter combinations: {len(param_sets)}") + print(f"Stocks: {[s[0] for s in stocks]}") + print("Generating 90 days of 5-min data per stock...\n") + + # Generate data for each stock + all_candles = {} + for symbol, base, drift, vol in stocks: + all_candles[symbol] = generate_stock_candles( + symbol, days=90, base_price=base, daily_drift=drift, daily_vol=vol + ) + print(f" {symbol}: {len(all_candles[symbol])} candles") + + # Test each parameter set across all stocks + print( + f"\nRunning {len(param_sets)} x {len(stocks)} = {len(param_sets) * len(stocks)} backtests..." + ) + + param_results = [] + for i, params in enumerate(param_sets): + total_profit = Decimal("0") + total_trades = 0 + total_sharpe = 0.0 + stock_details = [] + + for symbol, _, _, _ in stocks: + result = run_backtest(all_candles[symbol], params) + total_profit += result.profit + total_trades += result.total_trades + if result.detailed: + total_sharpe += result.detailed.sharpe_ratio + stock_details.append((symbol, result)) + + avg_sharpe = total_sharpe / len(stocks) if stocks else 0 + param_results.append((params, total_profit, total_trades, avg_sharpe, stock_details)) + + if (i + 1) % 18 == 0: + print(f" Progress: {i + 1}/{len(param_sets)}") + + # Sort by average Sharpe + param_results.sort(key=lambda x: x[3], reverse=True) + + print("\n" + "=" * 60) + print("TOP 5 PARAMETER SETS (by avg Sharpe across all stocks)") + print("=" * 60) + + for rank, (params, profit, trades, sharpe, details) in enumerate(param_results[:5], 1): + print(f"\n#{rank}:") + print( + f" RSI: {params['rsi_min']}-{params['rsi_max']}," + f" SL: {params['stop_loss_pct']}%, EMA: {params['ema_period']}" + ) + print(f" Total Profit: ${float(profit):.2f}, Trades: {trades}, Avg Sharpe: {sharpe:.3f}") + print(" Per stock:") + for symbol, result in details: + pct = float(result.profit_pct) + dd = result.detailed.max_drawdown if result.detailed else 0 + print(f" {symbol}: {pct:+.2f}% ({result.total_trades} trades, DD: {dd:.1f}%)") + + # Best params + best = param_results[0] + print("\n" + "=" * 60) + print("RECOMMENDED PARAMETERS") + print("=" * 60) + bp = best[0] + print(f" rsi_min: {bp['rsi_min']}") + print(f" rsi_max: {bp['rsi_max']}") + print(f" stop_loss_pct: {bp['stop_loss_pct']}") + print(f" ema_period: {bp['ema_period']}") + print(f" min_volume_ratio: {bp['min_volume_ratio']}") + print(f"\n Avg Sharpe: {best[3]:.3f}") + print(f" Total Profit: ${float(best[1]):.2f} across 5 stocks over 90 days") + + # Worst for comparison + print("\n" + "=" * 60) + print("WORST 3 PARAMETER SETS") + print("=" * 60) + for _rank, (params, profit, trades, sharpe, _) in enumerate(param_results[-3:], 1): + print( + f" RSI({params['rsi_min']}-{params['rsi_max']})," + f" SL={params['stop_loss_pct']}%, EMA={params['ema_period']}" + ) + print(f" Profit: ${float(profit):.2f}, Sharpe: {sharpe:.3f}") + + +if __name__ == "__main__": + main() diff --git a/scripts/optimize_asian_rsi.py b/scripts/optimize_asian_rsi.py deleted file mode 100755 index 9921447..0000000 --- a/scripts/optimize_asian_rsi.py +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env python3 -"""Optimize Asian Session RSI strategy parameters via grid search. - -Usage: python scripts/optimize_asian_rsi.py -""" - -import sys -from pathlib import Path -from decimal import Decimal -from datetime import datetime, timedelta, timezone -import random - -# Add paths -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "services" / "strategy-engine" / "src")) -sys.path.insert(0, str(ROOT / "services" / "strategy-engine")) -sys.path.insert(0, str(ROOT / "services" / "backtester" / "src")) -sys.path.insert(0, str(ROOT / "shared" / "src")) - -from shared.models import Candle # noqa: E402 -from backtester.engine import BacktestEngine # noqa: E402 -from strategies.asian_session_rsi import AsianSessionRsiStrategy # noqa: E402 - - -def generate_sol_candles(days: int = 90, base_price: float = 150.0) -> list[Candle]: - """Generate realistic SOL/USDT 5-minute candles. - - Simulates: - - Mild uptrend with periodic sharp dips during Asian session - - Intraday volatility (higher at session opens) - - Random walk with mean reversion - - Occasional momentum bursts that create RSI extremes - """ - random.seed(42) - candles = [] - price = base_price - start = datetime(2025, 1, 1, tzinfo=timezone.utc) - - for day in range(days): - # Mild upward bias to keep price above EMA - daily_trend = random.uniform(-0.005, 0.015) - - # Many days have a sharp V-dip during Asian session (1-2 bar crash + recovery) - # This creates RSI oversold while EMA stays above price briefly - dip_day = random.random() < 0.45 - dip_bar = random.randint(4, 18) if dip_day else -1 - # Sharp single-bar dip: 2-4% drop then immediate recovery - dip_magnitude = random.uniform(0.02, 0.04) - - for bar in range(288): # 288 5-minute bars per day - dt = start + timedelta(days=day, minutes=bar * 5) - hour = dt.hour - - # Volatility varies by session - if 0 <= hour < 2: # Asian open (our trading window) - vol = 0.003 - elif 13 <= hour < 16: # US session - vol = 0.0025 - else: - vol = 0.0015 - - # Base random walk with upward drift - change = random.gauss(daily_trend / 288, vol) - mean_rev = (base_price - price) / base_price * 0.001 - change += mean_rev - - # Session bar index within 00:00-01:55 UTC (bars 0-23) - session_bar = bar - - # Inject sharp V-dip: 1 bar crash, 1 bar partial recovery - if dip_day and 0 <= hour < 2: - if session_bar == dip_bar: - # Crash bar: sharp drop - change = -dip_magnitude - elif session_bar == dip_bar + 1: - # Recovery bar: bounce back most of the way - change = dip_magnitude * random.uniform(0.5, 0.8) - elif session_bar == dip_bar + 2: - # Continued recovery - change = dip_magnitude * random.uniform(0.1, 0.3) - - open_p = price - close_p = price * (1 + change) - high_p = max(open_p, close_p) * (1 + abs(random.gauss(0, vol * 0.5))) - low_p = min(open_p, close_p) * (1 - abs(random.gauss(0, vol * 0.5))) - - volume = random.uniform(50, 200) - if 0 <= hour < 2: - volume *= 2 - if dip_day and dip_bar <= session_bar <= dip_bar + 2: - volume *= 2.5 # Spike volume on dip/recovery - - candles.append( - Candle( - symbol="SOLUSDT", - timeframe="5m", - open_time=dt, - open=Decimal(str(round(open_p, 4))), - high=Decimal(str(round(high_p, 4))), - low=Decimal(str(round(low_p, 4))), - close=Decimal(str(round(close_p, 4))), - volume=Decimal(str(round(volume, 2))), - ) - ) - - price = close_p - - return candles - - -def run_backtest(candles, params, balance=750.0, slippage=0.001, fee=0.001): - """Run a single backtest with given parameters.""" - strategy = AsianSessionRsiStrategy() - strategy.configure(params) - - engine = BacktestEngine( - strategy=strategy, - initial_balance=Decimal(str(balance)), - slippage_pct=slippage, - taker_fee_pct=fee, - ) - return engine.run(candles) - - -def main(): - print("=" * 60) - print("Asian Session RSI — Parameter Optimization") - print("SOL/USDT 5m | Capital: $750 (~100만원)") - print("=" * 60) - - days = 30 - print(f"\nGenerating {days} days of synthetic SOL/USDT 5m candles...") - candles = generate_sol_candles(days=days, base_price=150.0) - print(f"Generated {len(candles)} candles") - - # Parameter grid - param_grid = [] - for rsi_period in [7, 9, 14]: - for rsi_oversold in [20, 25, 30]: - for tp in [1.0, 1.5, 2.0]: - for sl in [0.5, 0.7, 1.0]: - param_grid.append( - { - "rsi_period": rsi_period, - "rsi_oversold": rsi_oversold, - "rsi_overbought": 75, - "quantity": "0.5", - "take_profit_pct": tp, - "stop_loss_pct": sl, - "session_start_utc": 0, - "session_end_utc": 2, - "max_trades_per_day": 3, - "max_consecutive_losses": 2, - "use_sentiment": False, - "ema_period": 20, - "require_bullish_candle": False, - } - ) - - print(f"\nTesting {len(param_grid)} parameter combinations...") - print("-" * 60) - - results = [] - for i, params in enumerate(param_grid): - result = run_backtest(candles, params) - sharpe = result.detailed.sharpe_ratio if result.detailed else 0.0 - results.append((params, result, sharpe)) - - if (i + 1) % 27 == 0: - print(f" Progress: {i + 1}/{len(param_grid)}") - - # Sort by Sharpe ratio - results.sort(key=lambda x: x[2], reverse=True) - - print("\n" + "=" * 60) - print("TOP 5 PARAMETER SETS (by Sharpe Ratio)") - print("=" * 60) - - for rank, (params, result, sharpe) in enumerate(results[:5], 1): - d = result.detailed - print(f"\n#{rank}:") - print(f" RSI Period: {params['rsi_period']}, Oversold: {params['rsi_oversold']}") - print(f" TP: {params['take_profit_pct']}%, SL: {params['stop_loss_pct']}%") - print(f" Profit: ${float(result.profit):.2f} ({float(result.profit_pct):.2f}%)") - print(f" Trades: {result.total_trades}, Win Rate: {result.win_rate:.1f}%") - if d: - print(f" Sharpe: {d.sharpe_ratio:.3f}, Max DD: {d.max_drawdown:.2f}%") - print(f" Profit Factor: {d.profit_factor:.2f}") - - # Also show worst 3 for comparison - print("\n" + "=" * 60) - print("WORST 3 PARAMETER SETS") - print("=" * 60) - for rank, (params, result, sharpe) in enumerate(results[-3:], 1): - print( - f"\n RSI({params['rsi_period']}), OS={params['rsi_oversold']}, TP={params['take_profit_pct']}%, SL={params['stop_loss_pct']}%" - ) - print(f" Profit: ${float(result.profit):.2f}, Trades: {result.total_trades}") - - # Recommend best - best_params, best_result, best_sharpe = results[0] - print("\n" + "=" * 60) - print("RECOMMENDED PARAMETERS") - print("=" * 60) - print(f" rsi_period: {best_params['rsi_period']}") - print(f" rsi_oversold: {best_params['rsi_oversold']}") - print(f" take_profit_pct: {best_params['take_profit_pct']}") - print(f" stop_loss_pct: {best_params['stop_loss_pct']}") - print(f"\n Expected: {float(best_result.profit_pct):.2f}% over {days} days") - print(f" Sharpe: {best_sharpe:.3f}") - - -if __name__ == "__main__": - main() diff --git a/scripts/stock_screener.py b/scripts/stock_screener.py index 387bfea..7a5c0ba 100755 --- a/scripts/stock_screener.py +++ b/scripts/stock_screener.py @@ -10,6 +10,7 @@ Usage: Requires: ANTHROPIC_API_KEY environment variable """ + import argparse import asyncio import json @@ -51,15 +52,17 @@ async def get_market_data(alpaca: AlpacaClient, symbols: list[str]) -> list[dict lows = [float(b["l"]) for b in bars] range_pct = (max(highs) - min(lows)) / close * 100 if close > 0 else 0 - results.append({ - "symbol": symbol, - "close": close, - "change_pct": round(change_pct, 2), - "volume": volume, - "vol_ratio": round(vol_ratio, 2), - "range_5d_pct": round(range_pct, 2), - "is_bullish": close > float(latest["o"]), # Today bullish? - }) + results.append( + { + "symbol": symbol, + "close": close, + "change_pct": round(change_pct, 2), + "volume": volume, + "vol_ratio": round(vol_ratio, 2), + "range_5d_pct": round(range_pct, 2), + "is_bullish": close > float(latest["o"]), # Today bullish? + } + ) except Exception as exc: print(f" Warning: Failed to fetch {symbol}: {exc}") @@ -137,19 +140,45 @@ async def analyze_with_claude(prompt: str, api_key: str) -> list[dict]: # Default universe of liquid US stocks DEFAULT_UNIVERSE = [ # Tech - "AAPL", "MSFT", "GOOGL", "AMZN", "META", "NVDA", "TSLA", "AMD", "INTC", "CRM", + "AAPL", + "MSFT", + "GOOGL", + "AMZN", + "META", + "NVDA", + "TSLA", + "AMD", + "INTC", + "CRM", # Finance - "JPM", "BAC", "GS", "MS", "V", "MA", + "JPM", + "BAC", + "GS", + "MS", + "V", + "MA", # Healthcare - "JNJ", "UNH", "PFE", "ABBV", + "JNJ", + "UNH", + "PFE", + "ABBV", # Consumer - "WMT", "KO", "PEP", "MCD", "NKE", + "WMT", + "KO", + "PEP", + "MCD", + "NKE", # Energy - "XOM", "CVX", + "XOM", + "CVX", # Industrial - "CAT", "BA", "GE", + "CAT", + "BA", + "GE", # ETFs - "SPY", "QQQ", "IWM", + "SPY", + "QQQ", + "IWM", ] @@ -187,8 +216,7 @@ async def main_async(top_n: int = 5, universe: list[str] | None = None): # Pre-filter obvious rejects candidates = [ - s for s in market_data - if s["is_bullish"] and s["vol_ratio"] >= 0.8 and s["change_pct"] > -2 + s for s in market_data if s["is_bullish"] and s["vol_ratio"] >= 0.8 and s["change_pct"] > -2 ] print(f"Pre-filter: {len(candidates)} candidates (bullish, decent volume)\n") @@ -211,7 +239,9 @@ async def main_async(top_n: int = 5, universe: list[str] | None = None): # Find market data for this symbol md = next((m for m in market_data if m["symbol"] == rec["symbol"]), None) if md: - print(f" Close: ${md['close']:.2f} | Change: {md['change_pct']:+.2f}% | Vol Ratio: {md['vol_ratio']:.1f}x") + print( + f" Close: ${md['close']:.2f} | Change: {md['change_pct']:+.2f}% | Vol Ratio: {md['vol_ratio']:.1f}x" + ) else: print("Claude found no qualifying stocks today.") else: @@ -237,7 +267,9 @@ async def main_async(top_n: int = 5, universe: list[str] | None = None): print("-" * 60) for i, (s, score) in enumerate(scored[:top_n], 1): print(f"#{i} {s['symbol']} (Score: {score})") - print(f" Close: ${s['close']:.2f} | Change: {s['change_pct']:+.2f}% | Vol: {s['vol_ratio']:.1f}x") + print( + f" Close: ${s['close']:.2f} | Change: {s['change_pct']:+.2f}% | Vol: {s['vol_ratio']:.1f}x" + ) print("\n" + "=" * 60) diff --git a/services/api/tests/test_portfolio_router.py b/services/api/tests/test_portfolio_router.py index f2584ea..3bd1b2c 100644 --- a/services/api/tests/test_portfolio_router.py +++ b/services/api/tests/test_portfolio_router.py @@ -45,7 +45,7 @@ def test_get_positions_with_data(app, mock_db): app.state.db = db mock_row = MagicMock() - mock_row.symbol = "BTCUSDT" + mock_row.symbol = "AAPL" mock_row.quantity = Decimal("0.1") mock_row.avg_entry_price = Decimal("50000") mock_row.current_price = Decimal("55000") @@ -59,7 +59,7 @@ def test_get_positions_with_data(app, mock_db): assert response.status_code == 200 data = response.json() assert len(data) == 1 - assert data[0]["symbol"] == "BTCUSDT" + assert data[0]["symbol"] == "AAPL" def test_get_snapshots_empty(app, mock_db): diff --git a/services/backtester/src/backtester/config.py b/services/backtester/src/backtester/config.py index f7897da..57ee1fb 100644 --- a/services/backtester/src/backtester/config.py +++ b/services/backtester/src/backtester/config.py @@ -5,7 +5,7 @@ from shared.config import Settings class BacktestConfig(Settings): backtest_initial_balance: float = 10000.0 - symbol: str = "BTCUSDT" + symbol: str = "AAPL" timeframe: str = "1h" strategy_name: str = "rsi_strategy" candle_limit: int = 500 diff --git a/services/backtester/tests/test_engine.py b/services/backtester/tests/test_engine.py index 743a43b..4794e63 100644 --- a/services/backtester/tests/test_engine.py +++ b/services/backtester/tests/test_engine.py @@ -23,14 +23,14 @@ def make_candle(symbol: str, price: float, timeframe: str = "1h") -> Candle: ) -def make_candles(prices: list[float], symbol: str = "BTCUSDT") -> list[Candle]: +def make_candles(prices: list[float], symbol: str = "AAPL") -> list[Candle]: return [make_candle(symbol, p) for p in prices] def make_signal(side: OrderSide, price: str, quantity: str = "0.1") -> Signal: return Signal( strategy="test", - symbol="BTCUSDT", + symbol="AAPL", side=side, price=Decimal(price), quantity=Decimal(quantity), diff --git a/services/backtester/tests/test_metrics.py b/services/backtester/tests/test_metrics.py index 582309a..55f5b6c 100644 --- a/services/backtester/tests/test_metrics.py +++ b/services/backtester/tests/test_metrics.py @@ -12,7 +12,7 @@ from backtester.metrics import TradeRecord, compute_detailed_metrics def _make_trade(side: str, price: str, minutes_offset: int = 0) -> TradeRecord: return TradeRecord( time=datetime(2025, 1, 1, tzinfo=timezone.utc) + timedelta(minutes=minutes_offset), - symbol="BTCUSDT", + symbol="AAPL", side=side, price=Decimal(price), quantity=Decimal("1"), @@ -127,39 +127,39 @@ def test_risk_free_rate_affects_sharpe(): base = datetime(2025, 1, 1, tzinfo=timezone.utc) trades = [ TradeRecord( - time=base, symbol="BTCUSDT", side="BUY", price=Decimal("100"), quantity=Decimal("1") + time=base, symbol="AAPL", side="BUY", price=Decimal("100"), quantity=Decimal("1") ), TradeRecord( time=base + timedelta(days=1), - symbol="BTCUSDT", + symbol="AAPL", side="SELL", price=Decimal("110"), quantity=Decimal("1"), ), TradeRecord( time=base + timedelta(days=2), - symbol="BTCUSDT", + symbol="AAPL", side="BUY", price=Decimal("105"), quantity=Decimal("1"), ), TradeRecord( time=base + timedelta(days=3), - symbol="BTCUSDT", + symbol="AAPL", side="SELL", price=Decimal("115"), quantity=Decimal("1"), ), TradeRecord( time=base + timedelta(days=4), - symbol="BTCUSDT", + symbol="AAPL", side="BUY", price=Decimal("110"), quantity=Decimal("1"), ), TradeRecord( time=base + timedelta(days=5), - symbol="BTCUSDT", + symbol="AAPL", side="SELL", price=Decimal("108"), quantity=Decimal("1"), diff --git a/services/backtester/tests/test_reporter.py b/services/backtester/tests/test_reporter.py index 2ea49c0..5199b68 100644 --- a/services/backtester/tests/test_reporter.py +++ b/services/backtester/tests/test_reporter.py @@ -32,7 +32,7 @@ def _make_result(with_detailed: bool = False) -> BacktestResult: ) return BacktestResult( strategy_name="sma_crossover", - symbol="BTCUSDT", + symbol="AAPL", total_trades=10, initial_balance=Decimal("10000"), final_balance=Decimal("11500"), @@ -48,7 +48,7 @@ def test_format_report_contains_key_metrics(): report = format_report(result) assert "sma_crossover" in report - assert "BTCUSDT" in report + assert "AAPL" in report assert "10000" in report assert "11500" in report assert "1500" in report @@ -89,7 +89,7 @@ def test_export_json(): data = json.loads(json_output) assert data["strategy_name"] == "sma_crossover" - assert data["symbol"] == "BTCUSDT" + assert data["symbol"] == "AAPL" assert "detailed" in data assert data["detailed"]["sharpe_ratio"] == 1.5 assert data["detailed"]["monthly_returns"]["2025-01"] == 500.0 diff --git a/services/backtester/tests/test_simulator.py b/services/backtester/tests/test_simulator.py index a407c21..62e2cdb 100644 --- a/services/backtester/tests/test_simulator.py +++ b/services/backtester/tests/test_simulator.py @@ -36,20 +36,20 @@ def test_simulator_initial_balance(): def test_simulator_buy_reduces_balance(): sim = OrderSimulator(Decimal("10000")) - signal = make_signal("BTCUSDT", OrderSide.BUY, "50000", "0.1") + signal = make_signal("AAPL", OrderSide.BUY, "50000", "0.1") result = sim.execute(signal) assert result is True assert sim.balance == Decimal("5000") - assert sim.positions["BTCUSDT"] == Decimal("0.1") + assert sim.positions["AAPL"] == Decimal("0.1") def test_simulator_sell_increases_balance(): sim = OrderSimulator(Decimal("10000")) - buy_signal = make_signal("BTCUSDT", OrderSide.BUY, "50000", "0.1") + buy_signal = make_signal("AAPL", OrderSide.BUY, "50000", "0.1") sim.execute(buy_signal) balance_after_buy = sim.balance - sell_signal = make_signal("BTCUSDT", OrderSide.SELL, "55000", "0.1") + sell_signal = make_signal("AAPL", OrderSide.SELL, "55000", "0.1") result = sim.execute(sell_signal) assert result is True assert sim.balance > balance_after_buy @@ -59,20 +59,20 @@ def test_simulator_sell_increases_balance(): def test_simulator_reject_buy_insufficient_balance(): sim = OrderSimulator(Decimal("100")) - signal = make_signal("BTCUSDT", OrderSide.BUY, "50000", "0.1") + signal = make_signal("AAPL", OrderSide.BUY, "50000", "0.1") result = sim.execute(signal) assert result is False assert sim.balance == Decimal("100") - assert sim.positions.get("BTCUSDT", Decimal("0")) == Decimal("0") + assert sim.positions.get("AAPL", Decimal("0")) == Decimal("0") def test_simulator_trade_history(): sim = OrderSimulator(Decimal("10000")) - signal = make_signal("BTCUSDT", OrderSide.BUY, "50000", "0.1") + signal = make_signal("AAPL", OrderSide.BUY, "50000", "0.1") sim.execute(signal) assert len(sim.trades) == 1 trade = sim.trades[0] - assert trade.symbol == "BTCUSDT" + assert trade.symbol == "AAPL" assert trade.side == OrderSide.BUY assert trade.price == Decimal("50000") assert trade.quantity == Decimal("0.1") @@ -86,7 +86,7 @@ def test_simulator_trade_history(): def test_slippage_on_buy(): """Buy price should increase by slippage_pct.""" sim = OrderSimulator(Decimal("100000"), slippage_pct=0.01) # 1% - signal = make_signal("BTCUSDT", OrderSide.BUY, "50000", "0.1") + signal = make_signal("AAPL", OrderSide.BUY, "50000", "0.1") sim.execute(signal) trade = sim.trades[0] expected_price = Decimal("50000") * Decimal("1.01") # 50500 @@ -97,10 +97,10 @@ def test_slippage_on_sell(): """Sell price should decrease by slippage_pct.""" sim = OrderSimulator(Decimal("100000"), slippage_pct=0.01) # Buy first (no slippage check here, just need a position) - buy = make_signal("BTCUSDT", OrderSide.BUY, "50000", "0.1") + buy = make_signal("AAPL", OrderSide.BUY, "50000", "0.1") sim.execute(buy) # Sell - sell = make_signal("BTCUSDT", OrderSide.SELL, "50000", "0.1") + sell = make_signal("AAPL", OrderSide.SELL, "50000", "0.1") sim.execute(sell) trade = sim.trades[1] expected_price = Decimal("50000") * Decimal("0.99") # 49500 @@ -116,7 +116,7 @@ def test_fee_deducted_from_balance(): """Fees should reduce balance beyond the raw cost.""" fee_pct = 0.001 # 0.1% sim = OrderSimulator(Decimal("100000"), taker_fee_pct=fee_pct) - signal = make_signal("BTCUSDT", OrderSide.BUY, "50000", "0.1") + signal = make_signal("AAPL", OrderSide.BUY, "50000", "0.1") sim.execute(signal) # cost = 50000 * 0.1 = 5000, fee = 5000 * 0.001 = 5 expected_balance = Decimal("100000") - Decimal("5000") - Decimal("5") @@ -132,7 +132,7 @@ def test_fee_deducted_from_balance(): def test_stop_loss_triggers(): """Long position auto-closed when candle_low <= stop_loss.""" sim = OrderSimulator(Decimal("100000")) - signal = make_signal("BTCUSDT", OrderSide.BUY, "50000", "0.1") + signal = make_signal("AAPL", OrderSide.BUY, "50000", "0.1") sim.execute(signal, stop_loss=Decimal("48000")) ts = datetime(2025, 1, 1, tzinfo=timezone.utc) @@ -150,7 +150,7 @@ def test_stop_loss_triggers(): def test_take_profit_triggers(): """Long position auto-closed when candle_high >= take_profit.""" sim = OrderSimulator(Decimal("100000")) - signal = make_signal("BTCUSDT", OrderSide.BUY, "50000", "0.1") + signal = make_signal("AAPL", OrderSide.BUY, "50000", "0.1") sim.execute(signal, take_profit=Decimal("55000")) ts = datetime(2025, 1, 1, tzinfo=timezone.utc) @@ -168,7 +168,7 @@ def test_take_profit_triggers(): def test_stop_not_triggered_within_range(): """No auto-close when price stays within stop/tp range.""" sim = OrderSimulator(Decimal("100000")) - signal = make_signal("BTCUSDT", OrderSide.BUY, "50000", "0.1") + signal = make_signal("AAPL", OrderSide.BUY, "50000", "0.1") sim.execute(signal, stop_loss=Decimal("48000"), take_profit=Decimal("55000")) ts = datetime(2025, 1, 1, tzinfo=timezone.utc) @@ -189,10 +189,10 @@ def test_stop_not_triggered_within_range(): def test_short_sell_allowed(): """Can open short position with allow_short=True.""" sim = OrderSimulator(Decimal("100000"), allow_short=True) - signal = make_signal("BTCUSDT", OrderSide.SELL, "50000", "0.1") + signal = make_signal("AAPL", OrderSide.SELL, "50000", "0.1") result = sim.execute(signal) assert result is True - assert sim.positions["BTCUSDT"] == Decimal("-0.1") + assert sim.positions["AAPL"] == Decimal("-0.1") assert len(sim.open_positions) == 1 assert sim.open_positions[0].side == OrderSide.SELL @@ -200,16 +200,16 @@ def test_short_sell_allowed(): def test_short_sell_rejected(): """Short rejected when allow_short=False (default).""" sim = OrderSimulator(Decimal("100000"), allow_short=False) - signal = make_signal("BTCUSDT", OrderSide.SELL, "50000", "0.1") + signal = make_signal("AAPL", OrderSide.SELL, "50000", "0.1") result = sim.execute(signal) assert result is False - assert sim.positions.get("BTCUSDT", Decimal("0")) == Decimal("0") + assert sim.positions.get("AAPL", Decimal("0")) == Decimal("0") def test_short_stop_loss(): """Short position stop-loss triggers on candle high >= stop_loss.""" sim = OrderSimulator(Decimal("100000"), allow_short=True) - signal = make_signal("BTCUSDT", OrderSide.SELL, "50000", "0.1") + signal = make_signal("AAPL", OrderSide.SELL, "50000", "0.1") sim.execute(signal, stop_loss=Decimal("52000")) ts = datetime(2025, 1, 1, tzinfo=timezone.utc) diff --git a/services/backtester/tests/test_walk_forward.py b/services/backtester/tests/test_walk_forward.py index 5ab2e7b..96abb6e 100644 --- a/services/backtester/tests/test_walk_forward.py +++ b/services/backtester/tests/test_walk_forward.py @@ -21,7 +21,7 @@ def _generate_candles(n=100, base_price=100.0): price = base_price + (i % 20) - 10 candles.append( Candle( - symbol="BTCUSDT", + symbol="AAPL", timeframe="1h", open_time=datetime(2025, 1, 1, tzinfo=timezone.utc) + timedelta(hours=i), open=Decimal(str(price)), diff --git a/services/data-collector/src/data_collector/binance_rest.py b/services/data-collector/src/data_collector/binance_rest.py deleted file mode 100644 index eaf4e30..0000000 --- a/services/data-collector/src/data_collector/binance_rest.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Binance REST API helpers for fetching historical candle data.""" - -from datetime import datetime, timezone -from decimal import Decimal - -from shared.models import Candle - - -def _normalize_symbol(symbol: str) -> str: - """Convert 'BTC/USDT' to 'BTCUSDT'.""" - return symbol.replace("/", "") - - -async def fetch_historical_candles( - exchange, - symbol: str, - timeframe: str, - since: int, - limit: int = 500, -) -> list[Candle]: - """Fetch historical OHLCV candles from the exchange and return Candle models. - - Args: - exchange: An async ccxt exchange instance. - symbol: Market symbol, e.g. 'BTC/USDT'. - timeframe: Candle timeframe, e.g. '1m'. - since: Start timestamp in milliseconds. - limit: Maximum number of candles to fetch. - - Returns: - A list of Candle model instances. - """ - rows = await exchange.fetch_ohlcv(symbol, timeframe, since=since, limit=limit) - - normalized = _normalize_symbol(symbol) - candles: list[Candle] = [] - - for row in rows: - ts_ms, o, h, low, c, v = row - open_time = datetime.fromtimestamp(ts_ms / 1000, tz=timezone.utc) - candles.append( - Candle( - symbol=normalized, - timeframe=timeframe, - open_time=open_time, - open=Decimal(str(o)), - high=Decimal(str(h)), - low=Decimal(str(low)), - close=Decimal(str(c)), - volume=Decimal(str(v)), - ) - ) - - return candles diff --git a/services/data-collector/src/data_collector/binance_ws.py b/services/data-collector/src/data_collector/binance_ws.py deleted file mode 100644 index e25e7a6..0000000 --- a/services/data-collector/src/data_collector/binance_ws.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Binance WebSocket client for real-time kline/candle data. - -NOTE: This module is Binance-specific (uses Binance WebSocket URL and message format). -Multi-exchange WebSocket support would require exchange-specific implementations. -""" - -import asyncio -import json -import logging -from datetime import datetime, timezone -from decimal import Decimal -from typing import Callable, Awaitable - -import websockets - -from shared.models import Candle - -logger = logging.getLogger(__name__) - -BINANCE_WS_URL = "wss://stream.binance.com:9443/ws" -RECONNECT_DELAY = 5 # seconds - - -def _normalize_symbol(symbol: str) -> str: - """Convert 'BTC/USDT' to 'BTCUSDT'.""" - return symbol.replace("/", "") - - -def _stream_name(symbol: str, timeframe: str) -> str: - """Build Binance stream name, e.g. 'btcusdt@kline_1m'.""" - return f"{_normalize_symbol(symbol).lower()}@kline_{timeframe}" - - -class BinanceWebSocket: - """Connects to Binance WebSocket streams and emits closed candles.""" - - def __init__( - self, - symbols: list[str], - timeframe: str, - on_candle: Callable[[Candle], Awaitable[None]], - ) -> None: - self._symbols = symbols - self._timeframe = timeframe - self._on_candle = on_candle - self._running = False - - def _build_subscribe_message(self) -> dict: - streams = [_stream_name(s, self._timeframe) for s in self._symbols] - return { - "method": "SUBSCRIBE", - "params": streams, - "id": 1, - } - - def _parse_candle(self, message: dict) -> Candle | None: - """Parse a kline WebSocket message into a Candle, or None if not closed.""" - k = message.get("k") - if k is None: - return None - if not k.get("x"): # only closed candles - return None - - symbol = k["s"] # already normalized, e.g. 'BTCUSDT' - open_time = datetime.fromtimestamp(k["t"] / 1000, tz=timezone.utc) - return Candle( - symbol=symbol, - timeframe=self._timeframe, - open_time=open_time, - open=Decimal(k["o"]), - high=Decimal(k["h"]), - low=Decimal(k["l"]), - close=Decimal(k["c"]), - volume=Decimal(k["v"]), - ) - - async def _run_once(self) -> None: - """Single connection attempt; processes messages until disconnected.""" - async with websockets.connect(BINANCE_WS_URL) as ws: - subscribe_msg = self._build_subscribe_message() - await ws.send(json.dumps(subscribe_msg)) - logger.info("Subscribed to Binance streams: %s", subscribe_msg["params"]) - - async for raw in ws: - if not self._running: - break - try: - message = json.loads(raw) - candle = self._parse_candle(message) - if candle is not None: - await self._on_candle(candle) - except Exception: - logger.exception("Error processing WebSocket message: %s", raw) - - async def start(self) -> None: - """Connect to Binance WebSocket and process messages, auto-reconnecting.""" - self._running = True - while self._running: - try: - await self._run_once() - except Exception: - if not self._running: - break - logger.warning("WebSocket disconnected. Reconnecting in %ds…", RECONNECT_DELAY) - await asyncio.sleep(RECONNECT_DELAY) - - def stop(self) -> None: - """Signal the WebSocket loop to stop after the current message.""" - self._running = False diff --git a/services/data-collector/src/data_collector/config.py b/services/data-collector/src/data_collector/config.py index 4761013..dd430e6 100644 --- a/services/data-collector/src/data_collector/config.py +++ b/services/data-collector/src/data_collector/config.py @@ -1,4 +1,5 @@ """Data Collector configuration.""" + from shared.config import Settings diff --git a/services/data-collector/src/data_collector/main.py b/services/data-collector/src/data_collector/main.py index 38f8759..b42b34c 100644 --- a/services/data-collector/src/data_collector/main.py +++ b/services/data-collector/src/data_collector/main.py @@ -1,9 +1,9 @@ """Data Collector Service — fetches US stock data from Alpaca.""" + import asyncio from shared.alpaca import AlpacaClient from shared.broker import RedisBroker -from shared.config import Settings from shared.db import Database from shared.events import CandleEvent from shared.healthcheck import HealthCheckServer @@ -33,6 +33,7 @@ async def fetch_latest_bars( bar = bars[-1] from datetime import datetime from decimal import Decimal + candle = Candle( symbol=symbol, timeframe=timeframe, diff --git a/services/data-collector/src/data_collector/ws_factory.py b/services/data-collector/src/data_collector/ws_factory.py deleted file mode 100644 index e068399..0000000 --- a/services/data-collector/src/data_collector/ws_factory.py +++ /dev/null @@ -1,34 +0,0 @@ -"""WebSocket factory for exchange-specific connections.""" - -import logging - -from data_collector.binance_ws import BinanceWebSocket - -logger = logging.getLogger(__name__) - -# Supported exchanges for WebSocket streaming -SUPPORTED_WS = {"binance": BinanceWebSocket} - - -def create_websocket(exchange_id: str, **kwargs): - """Create an exchange-specific WebSocket handler. - - Args: - exchange_id: Exchange identifier (e.g. 'binance') - **kwargs: Passed to the WebSocket constructor (symbols, timeframe, on_candle) - - Returns: - WebSocket handler instance - - Raises: - ValueError: If exchange is not supported for WebSocket streaming - """ - ws_cls = SUPPORTED_WS.get(exchange_id) - if ws_cls is None: - supported = ", ".join(sorted(SUPPORTED_WS.keys())) - raise ValueError( - f"WebSocket streaming not supported for '{exchange_id}'. " - f"Supported: {supported}. " - f"Use REST polling as fallback for unsupported exchanges." - ) - return ws_cls(**kwargs) diff --git a/services/data-collector/tests/test_binance_rest.py b/services/data-collector/tests/test_binance_rest.py deleted file mode 100644 index bf88210..0000000 --- a/services/data-collector/tests/test_binance_rest.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Tests for binance_rest module.""" - -import pytest -from decimal import Decimal -from unittest.mock import AsyncMock, MagicMock -from datetime import datetime, timezone - -from data_collector.binance_rest import fetch_historical_candles - - -@pytest.mark.asyncio -async def test_fetch_historical_candles_parses_response(): - """Verify that OHLCV rows are correctly parsed into Candle models.""" - ts = 1700000000000 # milliseconds - mock_exchange = MagicMock() - mock_exchange.fetch_ohlcv = AsyncMock( - return_value=[ - [ts, 30000.0, 30100.0, 29900.0, 30050.0, 1.5], - [ts + 60000, 30050.0, 30200.0, 30000.0, 30150.0, 2.0], - ] - ) - - candles = await fetch_historical_candles(mock_exchange, "BTC/USDT", "1m", since=ts, limit=500) - - assert len(candles) == 2 - - c = candles[0] - assert c.symbol == "BTCUSDT" - assert c.timeframe == "1m" - assert c.open_time == datetime.fromtimestamp(ts / 1000, tz=timezone.utc) - assert c.open == Decimal("30000.0") - assert c.high == Decimal("30100.0") - assert c.low == Decimal("29900.0") - assert c.close == Decimal("30050.0") - assert c.volume == Decimal("1.5") - - mock_exchange.fetch_ohlcv.assert_called_once_with("BTC/USDT", "1m", since=ts, limit=500) - - -@pytest.mark.asyncio -async def test_fetch_historical_candles_empty_response(): - """Verify that an empty exchange response returns an empty list.""" - mock_exchange = MagicMock() - mock_exchange.fetch_ohlcv = AsyncMock(return_value=[]) - - candles = await fetch_historical_candles(mock_exchange, "BTC/USDT", "1m", since=1700000000000) - - assert candles == [] diff --git a/services/data-collector/tests/test_storage.py b/services/data-collector/tests/test_storage.py index be85578..ffffa40 100644 --- a/services/data-collector/tests/test_storage.py +++ b/services/data-collector/tests/test_storage.py @@ -9,7 +9,7 @@ from shared.models import Candle from data_collector.storage import CandleStorage -def _make_candle(symbol: str = "BTCUSDT") -> Candle: +def _make_candle(symbol: str = "AAPL") -> Candle: return Candle( symbol=symbol, timeframe="1m", @@ -39,11 +39,11 @@ async def test_storage_saves_to_db_and_publishes(): mock_broker.publish.assert_called_once() stream_arg = mock_broker.publish.call_args[0][0] - assert stream_arg == "candles.BTCUSDT" + assert stream_arg == "candles.AAPL" data_arg = mock_broker.publish.call_args[0][1] assert data_arg["type"] == "CANDLE" - assert data_arg["data"]["symbol"] == "BTCUSDT" + assert data_arg["data"]["symbol"] == "AAPL" @pytest.mark.asyncio diff --git a/services/data-collector/tests/test_ws_factory.py b/services/data-collector/tests/test_ws_factory.py deleted file mode 100644 index cdddcca..0000000 --- a/services/data-collector/tests/test_ws_factory.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Tests for WebSocket factory.""" - -import pytest -from data_collector.ws_factory import create_websocket, SUPPORTED_WS -from data_collector.binance_ws import BinanceWebSocket - - -def test_create_binance_ws(): - ws = create_websocket("binance", symbols=["BTCUSDT"], timeframe="1m", on_candle=lambda c: None) - assert isinstance(ws, BinanceWebSocket) - - -def test_create_unsupported_exchange(): - with pytest.raises(ValueError, match="not supported"): - create_websocket( - "unsupported_exchange", symbols=["BTCUSDT"], timeframe="1m", on_candle=lambda c: None - ) - - -def test_supported_exchanges(): - assert "binance" in SUPPORTED_WS diff --git a/services/news-collector/Dockerfile b/services/news-collector/Dockerfile new file mode 100644 index 0000000..a8e5902 --- /dev/null +++ b/services/news-collector/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.12-slim +WORKDIR /app +COPY shared/ shared/ +RUN pip install --no-cache-dir ./shared +COPY services/news-collector/ services/news-collector/ +RUN pip install --no-cache-dir ./services/news-collector +RUN python -c "import nltk; nltk.download('vader_lexicon', quiet=True)" +ENV PYTHONPATH=/app +CMD ["python", "-m", "news_collector.main"] diff --git a/services/news-collector/pyproject.toml b/services/news-collector/pyproject.toml new file mode 100644 index 0000000..14c856a --- /dev/null +++ b/services/news-collector/pyproject.toml @@ -0,0 +1,25 @@ +[project] +name = "news-collector" +version = "0.1.0" +description = "News and sentiment data collector service" +requires-python = ">=3.12" +dependencies = [ + "trading-shared", + "feedparser>=6.0", + "nltk>=3.8", + "aiohttp>=3.9", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "pytest-asyncio>=0.23", + "aioresponses>=0.7", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/news_collector"] diff --git a/services/news-collector/src/news_collector/__init__.py b/services/news-collector/src/news_collector/__init__.py new file mode 100644 index 0000000..5547af2 --- /dev/null +++ b/services/news-collector/src/news_collector/__init__.py @@ -0,0 +1 @@ +"""News collector service.""" diff --git a/services/news-collector/src/news_collector/collectors/__init__.py b/services/news-collector/src/news_collector/collectors/__init__.py new file mode 100644 index 0000000..5ef36a7 --- /dev/null +++ b/services/news-collector/src/news_collector/collectors/__init__.py @@ -0,0 +1 @@ +"""News collectors.""" diff --git a/services/news-collector/src/news_collector/collectors/base.py b/services/news-collector/src/news_collector/collectors/base.py new file mode 100644 index 0000000..bb43fd6 --- /dev/null +++ b/services/news-collector/src/news_collector/collectors/base.py @@ -0,0 +1,18 @@ +"""Base class for all news collectors.""" + +from abc import ABC, abstractmethod + +from shared.models import NewsItem + + +class BaseCollector(ABC): + name: str = "base" + poll_interval: int = 300 # seconds + + @abstractmethod + async def collect(self) -> list[NewsItem]: + """Collect news items from the source.""" + + @abstractmethod + async def is_available(self) -> bool: + """Check if this data source is accessible.""" diff --git a/services/news-collector/src/news_collector/collectors/fear_greed.py b/services/news-collector/src/news_collector/collectors/fear_greed.py new file mode 100644 index 0000000..f79f716 --- /dev/null +++ b/services/news-collector/src/news_collector/collectors/fear_greed.py @@ -0,0 +1,63 @@ +"""CNN Fear & Greed Index collector.""" + +import logging +from dataclasses import dataclass +from typing import Optional + +import aiohttp + +from news_collector.collectors.base import BaseCollector + +logger = logging.getLogger(__name__) + +FEAR_GREED_URL = "https://production.dataviz.cnn.io/index/fearandgreed/graphdata" + + +@dataclass +class FearGreedResult: + fear_greed: int + fear_greed_label: str + + +class FearGreedCollector(BaseCollector): + name = "fear_greed" + poll_interval = 3600 # 1 hour + + async def is_available(self) -> bool: + return True + + async def _fetch_index(self) -> Optional[dict]: + headers = {"User-Agent": "Mozilla/5.0"} + try: + async with aiohttp.ClientSession() as session: + async with session.get( + FEAR_GREED_URL, headers=headers, timeout=aiohttp.ClientTimeout(total=10) + ) as resp: + if resp.status != 200: + return None + return await resp.json() + except Exception: + return None + + def _classify(self, score: int) -> str: + if score <= 20: + return "Extreme Fear" + if score <= 40: + return "Fear" + if score <= 60: + return "Neutral" + if score <= 80: + return "Greed" + return "Extreme Greed" + + async def collect(self) -> Optional[FearGreedResult]: + data = await self._fetch_index() + if data is None: + return None + try: + fg = data["fear_and_greed"] + score = int(fg["score"]) + label = fg.get("rating", self._classify(score)) + return FearGreedResult(fear_greed=score, fear_greed_label=label) + except (KeyError, ValueError, TypeError): + return None diff --git a/services/news-collector/src/news_collector/collectors/fed.py b/services/news-collector/src/news_collector/collectors/fed.py new file mode 100644 index 0000000..fce4842 --- /dev/null +++ b/services/news-collector/src/news_collector/collectors/fed.py @@ -0,0 +1,119 @@ +"""Federal Reserve RSS collector with hawkish/dovish/neutral stance detection.""" + +import asyncio +import logging +from calendar import timegm +from datetime import datetime, timezone + +import feedparser +from nltk.sentiment.vader import SentimentIntensityAnalyzer + +from shared.models import NewsCategory, NewsItem + +from .base import BaseCollector + +logger = logging.getLogger(__name__) + +_FED_RSS_URL = "https://www.federalreserve.gov/feeds/press_all.xml" + +_HAWKISH_KEYWORDS = [ + "rate hike", + "interest rate increase", + "tighten", + "tightening", + "inflation", + "hawkish", + "restrictive", + "raise rates", + "hike rates", +] +_DOVISH_KEYWORDS = [ + "rate cut", + "interest rate decrease", + "easing", + "ease", + "stimulus", + "dovish", + "accommodative", + "lower rates", + "cut rates", + "quantitative easing", +] + + +def _detect_stance(text: str) -> str: + lower = text.lower() + hawkish_hits = sum(1 for kw in _HAWKISH_KEYWORDS if kw in lower) + dovish_hits = sum(1 for kw in _DOVISH_KEYWORDS if kw in lower) + if hawkish_hits > dovish_hits: + return "hawkish" + if dovish_hits > hawkish_hits: + return "dovish" + return "neutral" + + +class FedCollector(BaseCollector): + name: str = "fed" + poll_interval: int = 3600 + + def __init__(self) -> None: + self._vader = SentimentIntensityAnalyzer() + + async def is_available(self) -> bool: + return True + + async def _fetch_fed_rss(self) -> list[dict]: + loop = asyncio.get_event_loop() + try: + parsed = await loop.run_in_executor(None, feedparser.parse, _FED_RSS_URL) + return parsed.get("entries", []) + except Exception as exc: + logger.error("Fed RSS fetch failed: %s", exc) + return [] + + def _parse_published(self, entry: dict) -> datetime: + published_parsed = entry.get("published_parsed") + if published_parsed: + try: + ts = timegm(published_parsed) + return datetime.fromtimestamp(ts, tz=timezone.utc) + except Exception: + pass + return datetime.now(timezone.utc) + + async def collect(self) -> list[NewsItem]: + try: + entries = await self._fetch_fed_rss() + except Exception as exc: + logger.error("Fed collector error: %s", exc) + return [] + + items: list[NewsItem] = [] + + for entry in entries: + title = entry.get("title", "").strip() + if not title: + continue + + summary = entry.get("summary", "") or "" + combined = f"{title} {summary}" + + sentiment = self._vader.polarity_scores(combined)["compound"] + stance = _detect_stance(combined) + published_at = self._parse_published(entry) + + items.append( + NewsItem( + source=self.name, + headline=title, + summary=summary or None, + url=entry.get("link") or None, + published_at=published_at, + symbols=[], + sentiment=sentiment, + category=NewsCategory.FED, + raw_data={"stance": stance, **dict(entry)}, + ) + ) + + return items diff --git a/services/news-collector/src/news_collector/collectors/finnhub.py b/services/news-collector/src/news_collector/collectors/finnhub.py new file mode 100644 index 0000000..13e3602 --- /dev/null +++ b/services/news-collector/src/news_collector/collectors/finnhub.py @@ -0,0 +1,88 @@ +"""Finnhub news collector with VADER sentiment analysis.""" + +import logging +from datetime import datetime, timezone + +import aiohttp +from nltk.sentiment.vader import SentimentIntensityAnalyzer + +from shared.models import NewsCategory, NewsItem + +from .base import BaseCollector + +logger = logging.getLogger(__name__) + +_CATEGORY_KEYWORDS: dict[NewsCategory, list[str]] = { + NewsCategory.FED: ["fed", "fomc", "rate", "federal reserve"], + NewsCategory.POLICY: ["tariff", "trump", "regulation", "policy", "trade war"], + NewsCategory.EARNINGS: ["earnings", "revenue", "profit", "eps", "guidance", "quarter"], +} + + +def _categorize(text: str) -> NewsCategory: + lower = text.lower() + for category, keywords in _CATEGORY_KEYWORDS.items(): + if any(kw in lower for kw in keywords): + return category + return NewsCategory.MACRO + + +class FinnhubCollector(BaseCollector): + name: str = "finnhub" + poll_interval: int = 300 + + _BASE_URL = "https://finnhub.io/api/v1/news" + + def __init__(self, api_key: str) -> None: + self._api_key = api_key + self._vader = SentimentIntensityAnalyzer() + + async def is_available(self) -> bool: + return bool(self._api_key) + + async def _fetch_news(self) -> list[dict]: + url = f"{self._BASE_URL}?category=general&token={self._api_key}" + async with aiohttp.ClientSession() as session: + async with session.get(url) as resp: + resp.raise_for_status() + return await resp.json() + + async def collect(self) -> list[NewsItem]: + try: + raw_items = await self._fetch_news() + except Exception as exc: + logger.error("Finnhub fetch failed: %s", exc) + return [] + + items: list[NewsItem] = [] + for article in raw_items: + headline = article.get("headline", "") + summary = article.get("summary", "") + combined = f"{headline} {summary}" + + sentiment_scores = self._vader.polarity_scores(combined) + sentiment = sentiment_scores["compound"] + + ts = article.get("datetime", 0) + published_at = datetime.fromtimestamp(ts, tz=timezone.utc) + + related = article.get("related", "") + symbols = [t.strip() for t in related.split(",") if t.strip()] if related else [] + + category = _categorize(combined) + + items.append( + NewsItem( + source=self.name, + headline=headline, + summary=summary or None, + url=article.get("url") or None, + published_at=published_at, + symbols=symbols, + sentiment=sentiment, + category=category, + raw_data=article, + ) + ) + + return items diff --git a/services/news-collector/src/news_collector/collectors/reddit.py b/services/news-collector/src/news_collector/collectors/reddit.py new file mode 100644 index 0000000..226a2f9 --- /dev/null +++ b/services/news-collector/src/news_collector/collectors/reddit.py @@ -0,0 +1,97 @@ +"""Reddit social sentiment collector using JSON API with VADER sentiment analysis.""" + +import logging +import re +from datetime import datetime, timezone + +import aiohttp +from nltk.sentiment.vader import SentimentIntensityAnalyzer + +from shared.models import NewsCategory, NewsItem + +from .base import BaseCollector + +logger = logging.getLogger(__name__) + +_SUBREDDITS = ["wallstreetbets", "stocks", "investing"] +_MIN_SCORE = 50 + +_TICKER_PATTERN = re.compile( + r"\b(AAPL|MSFT|GOOGL|GOOG|AMZN|TSLA|NVDA|META|BRK\.?[AB]|JPM|V|UNH|XOM|" + r"JNJ|WMT|MA|PG|HD|CVX|MRK|LLY|ABBV|PFE|BAC|KO|AVGO|COST|MCD|TMO|" + r"CSCO|ACN|ABT|DHR|TXN|NEE|NFLX|PM|UPS|RTX|HON|QCOM|AMGN|LOW|IBM|" + r"INTC|AMD|PYPL|GS|MS|BLK|SPGI|CAT|DE|GE|MMM|BA|F|GM|DIS|CMCSA)\b" +) + + +class RedditCollector(BaseCollector): + name: str = "reddit" + poll_interval: int = 900 + + def __init__(self) -> None: + self._vader = SentimentIntensityAnalyzer() + + async def is_available(self) -> bool: + return True + + async def _fetch_subreddit(self, subreddit: str) -> list[dict]: + url = f"https://www.reddit.com/r/{subreddit}/hot.json?limit=25" + headers = {"User-Agent": "TradingPlatform/1.0 (research@example.com)"} + try: + async with aiohttp.ClientSession() as session: + async with session.get( + url, headers=headers, timeout=aiohttp.ClientTimeout(total=10) + ) as resp: + if resp.status == 200: + data = await resp.json() + return data.get("data", {}).get("children", []) + except Exception as exc: + logger.error("Reddit fetch failed for r/%s: %s", subreddit, exc) + return [] + + async def collect(self) -> list[NewsItem]: + seen_titles: set[str] = set() + items: list[NewsItem] = [] + + for subreddit in _SUBREDDITS: + try: + posts = await self._fetch_subreddit(subreddit) + except Exception as exc: + logger.error("Reddit collector error for r/%s: %s", subreddit, exc) + continue + + for post in posts: + post_data = post.get("data", {}) + title = post_data.get("title", "").strip() + score = post_data.get("score", 0) + + if not title or score < _MIN_SCORE: + continue + if title in seen_titles: + continue + seen_titles.add(title) + + selftext = post_data.get("selftext", "") or "" + combined = f"{title} {selftext}" + + sentiment = self._vader.polarity_scores(combined)["compound"] + symbols = list(dict.fromkeys(_TICKER_PATTERN.findall(combined))) + + created_utc = post_data.get("created_utc", 0) + published_at = datetime.fromtimestamp(created_utc, tz=timezone.utc) + + items.append( + NewsItem( + source=self.name, + headline=title, + summary=selftext or None, + url=post_data.get("url") or None, + published_at=published_at, + symbols=symbols, + sentiment=sentiment, + category=NewsCategory.SOCIAL, + raw_data=post_data, + ) + ) + + return items diff --git a/services/news-collector/src/news_collector/collectors/rss.py b/services/news-collector/src/news_collector/collectors/rss.py new file mode 100644 index 0000000..ddf8503 --- /dev/null +++ b/services/news-collector/src/news_collector/collectors/rss.py @@ -0,0 +1,105 @@ +"""RSS news collector using feedparser with VADER sentiment analysis.""" + +import asyncio +import logging +import re +from datetime import datetime, timezone +from time import mktime + +import feedparser +from nltk.sentiment.vader import SentimentIntensityAnalyzer + +from shared.models import NewsCategory, NewsItem + +from .base import BaseCollector + +logger = logging.getLogger(__name__) + +_DEFAULT_FEEDS = [ + "https://finance.yahoo.com/news/rssindex", + "https://news.google.com/rss/search?q=stock+market+finance&hl=en-US&gl=US&ceid=US:en", + "https://feeds.marketwatch.com/marketwatch/topstories/", +] + +_TICKER_PATTERN = re.compile( + r"\b(AAPL|MSFT|GOOGL|GOOG|AMZN|TSLA|NVDA|META|BRK\.?[AB]|JPM|V|UNH|XOM|" + r"JNJ|WMT|MA|PG|HD|CVX|MRK|LLY|ABBV|PFE|BAC|KO|AVGO|COST|MCD|TMO|" + r"CSCO|ACN|ABT|DHR|TXN|NEE|NFLX|PM|UPS|RTX|HON|QCOM|AMGN|LOW|IBM|" + r"INTC|AMD|PYPL|GS|MS|BLK|SPGI|CAT|DE|GE|MMM|BA|F|GM|DIS|CMCSA)\b" +) + + +class RSSCollector(BaseCollector): + name: str = "rss" + poll_interval: int = 600 + + def __init__(self, feeds: list[str] | None = None) -> None: + self._feeds = feeds if feeds is not None else _DEFAULT_FEEDS + self._vader = SentimentIntensityAnalyzer() + + async def is_available(self) -> bool: + return True + + async def _fetch_feeds(self) -> list[dict]: + loop = asyncio.get_event_loop() + results = [] + for url in self._feeds: + try: + parsed = await loop.run_in_executor(None, feedparser.parse, url) + results.append(parsed) + except Exception as exc: + logger.error("RSS fetch failed for %s: %s", url, exc) + return results + + def _parse_published(self, entry: dict) -> datetime: + parsed_time = entry.get("published_parsed") + if parsed_time: + try: + ts = mktime(parsed_time) + return datetime.fromtimestamp(ts, tz=timezone.utc) + except Exception: + pass + return datetime.now(timezone.utc) + + async def collect(self) -> list[NewsItem]: + try: + feeds = await self._fetch_feeds() + except Exception as exc: + logger.error("RSS collector error: %s", exc) + return [] + + seen_titles: set[str] = set() + items: list[NewsItem] = [] + + for feed in feeds: + for entry in feed.get("entries", []): + title = entry.get("title", "").strip() + if not title or title in seen_titles: + continue + seen_titles.add(title) + + summary = entry.get("summary", "") or "" + combined = f"{title} {summary}" + + sentiment_scores = self._vader.polarity_scores(combined) + sentiment = sentiment_scores["compound"] + + symbols = list(dict.fromkeys(_TICKER_PATTERN.findall(combined))) + + published_at = self._parse_published(entry) + + items.append( + NewsItem( + source=self.name, + headline=title, + summary=summary or None, + url=entry.get("link") or None, + published_at=published_at, + symbols=symbols, + sentiment=sentiment, + category=NewsCategory.MACRO, + raw_data=dict(entry), + ) + ) + + return items diff --git a/services/news-collector/src/news_collector/collectors/sec_edgar.py b/services/news-collector/src/news_collector/collectors/sec_edgar.py new file mode 100644 index 0000000..ca1d070 --- /dev/null +++ b/services/news-collector/src/news_collector/collectors/sec_edgar.py @@ -0,0 +1,100 @@ +"""SEC EDGAR filing collector (free, no API key required).""" + +import logging +from datetime import datetime, timezone + +import aiohttp +from nltk.sentiment.vader import SentimentIntensityAnalyzer + +from shared.models import NewsCategory, NewsItem +from news_collector.collectors.base import BaseCollector + +logger = logging.getLogger(__name__) + +TRACKED_CIKS = { + "0000320193": "AAPL", + "0000789019": "MSFT", + "0001652044": "GOOGL", + "0001018724": "AMZN", + "0001318605": "TSLA", + "0001045810": "NVDA", + "0001326801": "META", + "0000019617": "JPM", + "0000078003": "PFE", + "0000021344": "KO", +} + +SEC_USER_AGENT = "TradingPlatform research@example.com" + + +class SecEdgarCollector(BaseCollector): + name = "sec_edgar" + poll_interval = 1800 # 30 minutes + + def __init__(self) -> None: + self._vader = SentimentIntensityAnalyzer() + + async def is_available(self) -> bool: + return True + + async def _fetch_recent_filings(self) -> list[dict]: + results = [] + headers = {"User-Agent": SEC_USER_AGENT} + async with aiohttp.ClientSession() as session: + for cik, ticker in TRACKED_CIKS.items(): + try: + url = f"https://data.sec.gov/submissions/CIK{cik}.json" + async with session.get( + url, headers=headers, timeout=aiohttp.ClientTimeout(total=10) + ) as resp: + if resp.status == 200: + data = await resp.json() + data["tickers"] = [{"ticker": ticker}] + results.append(data) + except Exception as exc: + logger.warning("sec_fetch_failed", cik=cik, error=str(exc)) + return results + + async def collect(self) -> list[NewsItem]: + filings_data = await self._fetch_recent_filings() + items = [] + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + + for company_data in filings_data: + tickers = [t["ticker"] for t in company_data.get("tickers", [])] + company_name = company_data.get("name", "Unknown") + recent = company_data.get("filings", {}).get("recent", {}) + + forms = recent.get("form", []) + dates = recent.get("filingDate", []) + descriptions = recent.get("primaryDocDescription", []) + accessions = recent.get("accessionNumber", []) + + for i, form in enumerate(forms): + if form != "8-K": + continue + filing_date = dates[i] if i < len(dates) else "" + if filing_date != today: + continue + + desc = descriptions[i] if i < len(descriptions) else "8-K Filing" + accession = accessions[i] if i < len(accessions) else "" + headline = f"{company_name} ({', '.join(tickers)}): {form} - {desc}" + + items.append( + NewsItem( + source=self.name, + headline=headline, + summary=desc, + url=f"https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&accession={accession}", + published_at=datetime.strptime(filing_date, "%Y-%m-%d").replace( + tzinfo=timezone.utc + ), + symbols=tickers, + sentiment=self._vader.polarity_scores(headline)["compound"], + category=NewsCategory.FILING, + raw_data={"form": form, "accession": accession}, + ) + ) + + return items diff --git a/services/news-collector/src/news_collector/collectors/truth_social.py b/services/news-collector/src/news_collector/collectors/truth_social.py new file mode 100644 index 0000000..33ebc86 --- /dev/null +++ b/services/news-collector/src/news_collector/collectors/truth_social.py @@ -0,0 +1,86 @@ +"""Truth Social collector using Mastodon-compatible API with VADER sentiment analysis.""" + +import logging +import re +from datetime import datetime, timezone + +import aiohttp +from nltk.sentiment.vader import SentimentIntensityAnalyzer + +from shared.models import NewsCategory, NewsItem + +from .base import BaseCollector + +logger = logging.getLogger(__name__) + +_TRUMP_ACCOUNT_ID = "107780257626128497" +_API_URL = f"https://truthsocial.com/api/v1/accounts/{_TRUMP_ACCOUNT_ID}/statuses" + +_HTML_TAG_PATTERN = re.compile(r"<[^>]+>") + + +def _strip_html(text: str) -> str: + return _HTML_TAG_PATTERN.sub("", text).strip() + + +class TruthSocialCollector(BaseCollector): + name: str = "truth_social" + poll_interval: int = 900 + + def __init__(self) -> None: + self._vader = SentimentIntensityAnalyzer() + + async def is_available(self) -> bool: + return True + + async def _fetch_posts(self) -> list[dict]: + headers = {"User-Agent": "TradingPlatform/1.0 (research@example.com)"} + try: + async with aiohttp.ClientSession() as session: + async with session.get( + _API_URL, headers=headers, timeout=aiohttp.ClientTimeout(total=10) + ) as resp: + if resp.status == 200: + return await resp.json() + except Exception as exc: + logger.error("Truth Social fetch failed: %s", exc) + return [] + + async def collect(self) -> list[NewsItem]: + try: + posts = await self._fetch_posts() + except Exception as exc: + logger.error("Truth Social collector error: %s", exc) + return [] + + items: list[NewsItem] = [] + + for post in posts: + raw_content = post.get("content", "") or "" + content = _strip_html(raw_content) + if not content: + continue + + sentiment = self._vader.polarity_scores(content)["compound"] + + created_at_str = post.get("created_at", "") + try: + published_at = datetime.fromisoformat(created_at_str.replace("Z", "+00:00")) + except Exception: + published_at = datetime.now(timezone.utc) + + items.append( + NewsItem( + source=self.name, + headline=content[:200], + summary=content if len(content) > 200 else None, + url=post.get("url") or None, + published_at=published_at, + symbols=[], + sentiment=sentiment, + category=NewsCategory.POLICY, + raw_data=post, + ) + ) + + return items diff --git a/services/news-collector/src/news_collector/config.py b/services/news-collector/src/news_collector/config.py new file mode 100644 index 0000000..70d98f1 --- /dev/null +++ b/services/news-collector/src/news_collector/config.py @@ -0,0 +1,10 @@ +"""News Collector configuration.""" + +from shared.config import Settings + + +class NewsCollectorConfig(Settings): + health_port: int = 8084 + finnhub_api_key: str = "" + news_poll_interval: int = 300 + sentiment_aggregate_interval: int = 900 diff --git a/services/news-collector/src/news_collector/main.py b/services/news-collector/src/news_collector/main.py new file mode 100644 index 0000000..3493f7c --- /dev/null +++ b/services/news-collector/src/news_collector/main.py @@ -0,0 +1,193 @@ +"""News Collector Service — fetches news from multiple sources and aggregates sentiment.""" + +import asyncio +from datetime import datetime, timezone + +from shared.broker import RedisBroker +from shared.db import Database +from shared.events import NewsEvent +from shared.healthcheck import HealthCheckServer +from shared.logging import setup_logging +from shared.metrics import ServiceMetrics +from shared.models import NewsItem +from shared.notifier import TelegramNotifier +from shared.sentiment_models import MarketSentiment +from shared.sentiment import SentimentAggregator + +from news_collector.config import NewsCollectorConfig +from news_collector.collectors.finnhub import FinnhubCollector +from news_collector.collectors.rss import RSSCollector +from news_collector.collectors.sec_edgar import SecEdgarCollector +from news_collector.collectors.truth_social import TruthSocialCollector +from news_collector.collectors.reddit import RedditCollector +from news_collector.collectors.fear_greed import FearGreedCollector +from news_collector.collectors.fed import FedCollector + +# Health check port: base + 4 +HEALTH_PORT_OFFSET = 4 + + +async def run_collector_once(collector, db: Database, broker: RedisBroker) -> int: + """Run a single collector, store results in DB, publish to Redis. + + Returns the number of items collected. + """ + items: list[NewsItem] = await collector.collect() + count = 0 + for item in items: + await db.insert_news_item(item) + event = NewsEvent(data=item) + stream = f"news.{item.category.value}" + await broker.publish(stream, event.to_dict()) + count += 1 + return count + + +async def run_collector_loop(collector, db: Database, broker: RedisBroker, log) -> None: + """Run a collector repeatedly on its configured poll_interval.""" + while True: + try: + count = await run_collector_once(collector, db, broker) + log.info( + "collector_ran", + collector=collector.name, + count=count, + ) + except Exception as exc: + log.warning( + "collector_error", + collector=collector.name, + error=str(exc), + ) + await asyncio.sleep(collector.poll_interval) + + +async def run_fear_greed_loop(collector: FearGreedCollector, db: Database, log) -> None: + """Fetch Fear & Greed index on its interval and update MarketSentiment in DB.""" + while True: + try: + result = await collector.collect() + if result is not None: + ms = MarketSentiment( + fear_greed=result.fear_greed, + fear_greed_label=result.fear_greed_label, + vix=None, + fed_stance="neutral", + market_regime=_determine_regime(result.fear_greed, None), + updated_at=datetime.now(timezone.utc), + ) + await db.upsert_market_sentiment(ms) + log.info( + "fear_greed_updated", + value=result.fear_greed, + label=result.fear_greed_label, + ) + except Exception as exc: + log.warning("fear_greed_error", error=str(exc)) + await asyncio.sleep(collector.poll_interval) + + +async def run_aggregator_loop(db: Database, interval: int, log) -> None: + """Run SentimentAggregator every interval seconds and persist scores.""" + aggregator = SentimentAggregator() + while True: + await asyncio.sleep(interval) + try: + now = datetime.now(timezone.utc) + news_items = await db.get_recent_news(hours=24) + scores = aggregator.aggregate(news_items, now) + for score in scores.values(): + await db.upsert_symbol_score(score) + log.info("aggregation_complete", symbols=len(scores)) + except Exception as exc: + log.warning("aggregator_error", error=str(exc)) + + +def _determine_regime(fear_greed: int, vix: float | None) -> str: + """Classify market regime from fear/greed index and optional VIX.""" + aggregator = SentimentAggregator() + return aggregator.determine_regime(fear_greed, vix) + + +async def run() -> None: + config = NewsCollectorConfig() + log = setup_logging("news-collector", config.log_level, config.log_format) + metrics = ServiceMetrics("news_collector") + + notifier = TelegramNotifier( + bot_token=config.telegram_bot_token, + chat_id=config.telegram_chat_id, + ) + + db = Database(config.database_url) + await db.connect() + + broker = RedisBroker(config.redis_url) + + health = HealthCheckServer( + "news-collector", + port=config.health_port, + auth_token=config.metrics_auth_token, + ) + await health.start() + metrics.service_up.labels(service="news-collector").set(1) + + # Build collectors + finnhub = FinnhubCollector(api_key=config.finnhub_api_key) + rss = RSSCollector() + sec = SecEdgarCollector() + truth = TruthSocialCollector() + reddit = RedditCollector() + fear_greed = FearGreedCollector() + fed = FedCollector() + + news_collectors = [finnhub, rss, sec, truth, reddit, fed] + + log.info( + "starting", + collectors=[c.name for c in news_collectors], + poll_interval=config.news_poll_interval, + aggregate_interval=config.sentiment_aggregate_interval, + ) + + try: + tasks = [] + for collector in news_collectors: + tasks.append( + asyncio.create_task( + run_collector_loop(collector, db, broker, log), + name=f"collector-{collector.name}", + ) + ) + tasks.append( + asyncio.create_task( + run_fear_greed_loop(fear_greed, db, log), + name="fear-greed-loop", + ) + ) + tasks.append( + asyncio.create_task( + run_aggregator_loop(db, config.sentiment_aggregate_interval, log), + name="aggregator-loop", + ) + ) + await asyncio.gather(*tasks) + except Exception as exc: + log.error("fatal_error", error=str(exc)) + await notifier.send_error(str(exc), "news-collector") + raise + finally: + metrics.service_up.labels(service="news-collector").set(0) + for task in tasks: + task.cancel() + await notifier.close() + await broker.close() + await db.close() + + +def main() -> None: + asyncio.run(run()) + + +if __name__ == "__main__": + main() diff --git a/services/news-collector/tests/__init__.py b/services/news-collector/tests/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/services/news-collector/tests/__init__.py diff --git a/services/news-collector/tests/test_fear_greed.py b/services/news-collector/tests/test_fear_greed.py new file mode 100644 index 0000000..d483aa6 --- /dev/null +++ b/services/news-collector/tests/test_fear_greed.py @@ -0,0 +1,49 @@ +"""Tests for CNN Fear & Greed Index collector.""" + +import pytest +from unittest.mock import AsyncMock, patch + +from news_collector.collectors.fear_greed import FearGreedCollector + + +@pytest.fixture +def collector(): + return FearGreedCollector() + + +def test_collector_name(collector): + assert collector.name == "fear_greed" + assert collector.poll_interval == 3600 + + +async def test_is_available(collector): + assert await collector.is_available() is True + + +async def test_collect_parses_api_response(collector): + mock_data = { + "fear_and_greed": { + "score": 45.0, + "rating": "Fear", + "timestamp": "2026-04-02T12:00:00+00:00", + } + } + with patch.object(collector, "_fetch_index", new_callable=AsyncMock, return_value=mock_data): + result = await collector.collect() + assert result.fear_greed == 45 + assert result.fear_greed_label == "Fear" + + +async def test_collect_returns_none_on_failure(collector): + with patch.object(collector, "_fetch_index", new_callable=AsyncMock, return_value=None): + result = await collector.collect() + assert result is None + + +def test_classify_label(): + c = FearGreedCollector() + assert c._classify(10) == "Extreme Fear" + assert c._classify(30) == "Fear" + assert c._classify(50) == "Neutral" + assert c._classify(70) == "Greed" + assert c._classify(85) == "Extreme Greed" diff --git a/services/news-collector/tests/test_fed.py b/services/news-collector/tests/test_fed.py new file mode 100644 index 0000000..d1a736b --- /dev/null +++ b/services/news-collector/tests/test_fed.py @@ -0,0 +1,37 @@ +"""Tests for Federal Reserve collector.""" + +import pytest +from unittest.mock import AsyncMock, patch +from news_collector.collectors.fed import FedCollector + + +@pytest.fixture +def collector(): + return FedCollector() + + +def test_collector_name(collector): + assert collector.name == "fed" + assert collector.poll_interval == 3600 + + +async def test_is_available(collector): + assert await collector.is_available() is True + + +async def test_collect_parses_rss(collector): + mock_entries = [ + { + "title": "Federal Reserve issues FOMC statement", + "link": "https://www.federalreserve.gov/newsevents/pressreleases/monetary20260402a.htm", + "published_parsed": (2026, 4, 2, 14, 0, 0, 0, 0, 0), + "summary": "The Federal Open Market Committee decided to maintain the target range...", + }, + ] + with patch.object( + collector, "_fetch_fed_rss", new_callable=AsyncMock, return_value=mock_entries + ): + items = await collector.collect() + assert len(items) == 1 + assert items[0].source == "fed" + assert items[0].category.value == "fed" diff --git a/services/news-collector/tests/test_finnhub.py b/services/news-collector/tests/test_finnhub.py new file mode 100644 index 0000000..a4cf169 --- /dev/null +++ b/services/news-collector/tests/test_finnhub.py @@ -0,0 +1,67 @@ +"""Tests for Finnhub news collector.""" + +import pytest +from unittest.mock import AsyncMock, patch + +from news_collector.collectors.finnhub import FinnhubCollector + + +@pytest.fixture +def collector(): + return FinnhubCollector(api_key="test_key") + + +def test_collector_name(collector): + assert collector.name == "finnhub" + assert collector.poll_interval == 300 + + +async def test_is_available_with_key(collector): + assert await collector.is_available() is True + + +async def test_is_available_without_key(): + c = FinnhubCollector(api_key="") + assert await c.is_available() is False + + +async def test_collect_parses_response(collector): + mock_response = [ + { + "category": "top news", + "datetime": 1711929600, + "headline": "AAPL beats earnings", + "id": 12345, + "related": "AAPL", + "source": "MarketWatch", + "summary": "Apple reported better than expected...", + "url": "https://example.com/article", + }, + { + "category": "top news", + "datetime": 1711929000, + "headline": "Fed holds rates steady", + "id": 12346, + "related": "", + "source": "Reuters", + "summary": "The Federal Reserve...", + "url": "https://example.com/fed", + }, + ] + + with patch.object(collector, "_fetch_news", new_callable=AsyncMock, return_value=mock_response): + items = await collector.collect() + + assert len(items) == 2 + assert items[0].source == "finnhub" + assert items[0].headline == "AAPL beats earnings" + assert items[0].symbols == ["AAPL"] + assert items[0].url == "https://example.com/article" + assert isinstance(items[0].sentiment, float) + assert items[1].symbols == [] + + +async def test_collect_handles_empty_response(collector): + with patch.object(collector, "_fetch_news", new_callable=AsyncMock, return_value=[]): + items = await collector.collect() + assert items == [] diff --git a/services/news-collector/tests/test_main.py b/services/news-collector/tests/test_main.py new file mode 100644 index 0000000..66190dc --- /dev/null +++ b/services/news-collector/tests/test_main.py @@ -0,0 +1,39 @@ +"""Tests for news collector scheduler.""" + +from unittest.mock import AsyncMock, MagicMock +from datetime import datetime, timezone +from shared.models import NewsCategory, NewsItem +from news_collector.main import run_collector_once + + +async def test_run_collector_once_stores_and_publishes(): + mock_item = NewsItem( + source="test", + headline="Test news", + published_at=datetime(2026, 4, 2, tzinfo=timezone.utc), + sentiment=0.5, + category=NewsCategory.MACRO, + ) + mock_collector = MagicMock() + mock_collector.name = "test" + mock_collector.collect = AsyncMock(return_value=[mock_item]) + mock_db = MagicMock() + mock_db.insert_news_item = AsyncMock() + mock_broker = MagicMock() + mock_broker.publish = AsyncMock() + + count = await run_collector_once(mock_collector, mock_db, mock_broker) + assert count == 1 + mock_db.insert_news_item.assert_called_once_with(mock_item) + mock_broker.publish.assert_called_once() + + +async def test_run_collector_once_handles_empty(): + mock_collector = MagicMock() + mock_collector.name = "test" + mock_collector.collect = AsyncMock(return_value=[]) + mock_db = MagicMock() + mock_broker = MagicMock() + + count = await run_collector_once(mock_collector, mock_db, mock_broker) + assert count == 0 diff --git a/services/news-collector/tests/test_reddit.py b/services/news-collector/tests/test_reddit.py new file mode 100644 index 0000000..440b173 --- /dev/null +++ b/services/news-collector/tests/test_reddit.py @@ -0,0 +1,63 @@ +"""Tests for Reddit collector.""" + +import pytest +from unittest.mock import AsyncMock, patch +from news_collector.collectors.reddit import RedditCollector + + +@pytest.fixture +def collector(): + return RedditCollector() + + +def test_collector_name(collector): + assert collector.name == "reddit" + assert collector.poll_interval == 900 + + +async def test_is_available(collector): + assert await collector.is_available() is True + + +async def test_collect_parses_posts(collector): + mock_posts = [ + { + "data": { + "title": "NVDA to the moon! AI demand is insane", + "selftext": "Just loaded up on NVDA calls", + "url": "https://reddit.com/r/wallstreetbets/123", + "created_utc": 1711929600, + "score": 500, + "num_comments": 200, + "subreddit": "wallstreetbets", + } + }, + ] + with patch.object( + collector, "_fetch_subreddit", new_callable=AsyncMock, return_value=mock_posts + ): + items = await collector.collect() + assert len(items) >= 1 + assert items[0].source == "reddit" + assert items[0].category.value == "social" + + +async def test_collect_filters_low_score(collector): + mock_posts = [ + { + "data": { + "title": "Random question", + "selftext": "", + "url": "https://reddit.com/456", + "created_utc": 1711929600, + "score": 3, + "num_comments": 1, + "subreddit": "stocks", + } + }, + ] + with patch.object( + collector, "_fetch_subreddit", new_callable=AsyncMock, return_value=mock_posts + ): + items = await collector.collect() + assert items == [] diff --git a/services/news-collector/tests/test_rss.py b/services/news-collector/tests/test_rss.py new file mode 100644 index 0000000..e03250a --- /dev/null +++ b/services/news-collector/tests/test_rss.py @@ -0,0 +1,47 @@ +"""Tests for RSS news collector.""" + +import pytest +from unittest.mock import AsyncMock, patch + +from news_collector.collectors.rss import RSSCollector + + +@pytest.fixture +def collector(): + return RSSCollector() + + +def test_collector_name(collector): + assert collector.name == "rss" + assert collector.poll_interval == 600 + + +async def test_is_available(collector): + assert await collector.is_available() is True + + +async def test_collect_parses_feed(collector): + mock_feed = { + "entries": [ + { + "title": "NVDA surges on AI demand", + "link": "https://example.com/nvda", + "published_parsed": (2026, 4, 2, 12, 0, 0, 0, 0, 0), + "summary": "Nvidia stock jumped 5%...", + }, + { + "title": "Markets rally on jobs data", + "link": "https://example.com/market", + "published_parsed": (2026, 4, 2, 11, 0, 0, 0, 0, 0), + "summary": "The S&P 500 rose...", + }, + ], + } + + with patch.object(collector, "_fetch_feeds", new_callable=AsyncMock, return_value=[mock_feed]): + items = await collector.collect() + + assert len(items) == 2 + assert items[0].source == "rss" + assert items[0].headline == "NVDA surges on AI demand" + assert isinstance(items[0].sentiment, float) diff --git a/services/news-collector/tests/test_sec_edgar.py b/services/news-collector/tests/test_sec_edgar.py new file mode 100644 index 0000000..5d4f69f --- /dev/null +++ b/services/news-collector/tests/test_sec_edgar.py @@ -0,0 +1,58 @@ +"""Tests for SEC EDGAR filing collector.""" + +import pytest +from datetime import datetime, timezone +from unittest.mock import AsyncMock, patch, MagicMock + +from news_collector.collectors.sec_edgar import SecEdgarCollector + + +@pytest.fixture +def collector(): + return SecEdgarCollector() + + +def test_collector_name(collector): + assert collector.name == "sec_edgar" + assert collector.poll_interval == 1800 + + +async def test_is_available(collector): + assert await collector.is_available() is True + + +async def test_collect_parses_filings(collector): + mock_response = { + "filings": { + "recent": { + "accessionNumber": ["0001234-26-000001"], + "filingDate": ["2026-04-02"], + "primaryDocument": ["filing.htm"], + "form": ["8-K"], + "primaryDocDescription": ["Current Report"], + } + }, + "tickers": [{"ticker": "AAPL"}], + "name": "Apple Inc", + } + + mock_datetime = MagicMock(spec=datetime) + mock_datetime.now.return_value = datetime(2026, 4, 2, tzinfo=timezone.utc) + mock_datetime.strptime = datetime.strptime + + with patch.object( + collector, "_fetch_recent_filings", new_callable=AsyncMock, return_value=[mock_response] + ): + with patch("news_collector.collectors.sec_edgar.datetime", mock_datetime): + items = await collector.collect() + + assert len(items) == 1 + assert items[0].source == "sec_edgar" + assert items[0].category.value == "filing" + assert "AAPL" in items[0].symbols + + +async def test_collect_handles_empty(collector): + with patch.object(collector, "_fetch_recent_filings", new_callable=AsyncMock, return_value=[]): + items = await collector.collect() + assert items == [] diff --git a/services/news-collector/tests/test_truth_social.py b/services/news-collector/tests/test_truth_social.py new file mode 100644 index 0000000..91ddb9d --- /dev/null +++ b/services/news-collector/tests/test_truth_social.py @@ -0,0 +1,41 @@ +"""Tests for Truth Social collector.""" + +import pytest +from unittest.mock import AsyncMock, patch +from news_collector.collectors.truth_social import TruthSocialCollector + + +@pytest.fixture +def collector(): + return TruthSocialCollector() + + +def test_collector_name(collector): + assert collector.name == "truth_social" + assert collector.poll_interval == 900 + + +async def test_is_available(collector): + assert await collector.is_available() is True + + +async def test_collect_parses_posts(collector): + mock_posts = [ + { + "content": "<p>We are imposing 25% tariffs on all steel imports!</p>", + "created_at": "2026-04-02T12:00:00.000Z", + "url": "https://truthsocial.com/@realDonaldTrump/12345", + "id": "12345", + }, + ] + with patch.object(collector, "_fetch_posts", new_callable=AsyncMock, return_value=mock_posts): + items = await collector.collect() + assert len(items) == 1 + assert items[0].source == "truth_social" + assert items[0].category.value == "policy" + + +async def test_collect_handles_empty(collector): + with patch.object(collector, "_fetch_posts", new_callable=AsyncMock, return_value=[]): + items = await collector.collect() + assert items == [] diff --git a/services/order-executor/src/order_executor/main.py b/services/order-executor/src/order_executor/main.py index 3e098c3..51ab286 100644 --- a/services/order-executor/src/order_executor/main.py +++ b/services/order-executor/src/order_executor/main.py @@ -1,4 +1,5 @@ """Order Executor Service entry point.""" + import asyncio from decimal import Decimal diff --git a/services/order-executor/tests/test_risk_manager.py b/services/order-executor/tests/test_risk_manager.py index 00a9ab4..3d5175b 100644 --- a/services/order-executor/tests/test_risk_manager.py +++ b/services/order-executor/tests/test_risk_manager.py @@ -7,7 +7,7 @@ from shared.models import OrderSide, Position, Signal from order_executor.risk_manager import RiskManager -def make_signal(side: OrderSide, price: str, quantity: str, symbol: str = "BTC/USDT") -> Signal: +def make_signal(side: OrderSide, price: str, quantity: str, symbol: str = "AAPL") -> Signal: return Signal( strategy="test", symbol=symbol, @@ -93,7 +93,7 @@ def test_risk_check_rejects_insufficient_balance(): def test_trailing_stop_set_and_trigger(): """Trailing stop should trigger when price drops below stop level.""" rm = make_risk_manager(trailing_stop_pct="5") - rm.set_trailing_stop("BTC/USDT", Decimal("100")) + rm.set_trailing_stop("AAPL", Decimal("100")) signal = make_signal(side=OrderSide.BUY, price="94", quantity="0.01") result = rm.check(signal, balance=Decimal("10000"), positions={}, daily_pnl=Decimal("0")) @@ -104,10 +104,10 @@ def test_trailing_stop_set_and_trigger(): def test_trailing_stop_updates_highest_price(): """Trailing stop should track the highest price seen.""" rm = make_risk_manager(trailing_stop_pct="5") - rm.set_trailing_stop("BTC/USDT", Decimal("100")) + rm.set_trailing_stop("AAPL", Decimal("100")) # Price rises to 120 => stop at 114 - rm.update_price("BTC/USDT", Decimal("120")) + rm.update_price("AAPL", Decimal("120")) # Price at 115 is above stop (114), should be allowed signal = make_signal(side=OrderSide.BUY, price="115", quantity="0.01") @@ -124,7 +124,7 @@ def test_trailing_stop_updates_highest_price(): def test_trailing_stop_not_triggered_above_stop(): """Trailing stop should not trigger when price is above stop level.""" rm = make_risk_manager(trailing_stop_pct="5") - rm.set_trailing_stop("BTC/USDT", Decimal("100")) + rm.set_trailing_stop("AAPL", Decimal("100")) # Price at 96 is above stop (95), should be allowed signal = make_signal(side=OrderSide.BUY, price="96", quantity="0.01") @@ -140,11 +140,11 @@ def test_max_open_positions_check(): rm = make_risk_manager(max_open_positions=2) positions = { - "BTC/USDT": make_position("BTC/USDT", "1", "100", "100"), - "ETH/USDT": make_position("ETH/USDT", "10", "50", "50"), + "AAPL": make_position("AAPL", "1", "100", "100"), + "MSFT": make_position("MSFT", "10", "50", "50"), } - signal = make_signal(side=OrderSide.BUY, price="10", quantity="1", symbol="SOL/USDT") + signal = make_signal(side=OrderSide.BUY, price="10", quantity="1", symbol="TSLA") result = rm.check(signal, balance=Decimal("10000"), positions=positions, daily_pnl=Decimal("0")) assert result.allowed is False assert result.reason == "Max open positions reached" @@ -158,14 +158,14 @@ def test_volatility_calculation(): rm = make_risk_manager(volatility_lookback=5) # No history yet - assert rm.get_volatility("BTC/USDT") is None + assert rm.get_volatility("AAPL") is None # Feed prices prices = [100, 102, 98, 105, 101] for p in prices: - rm.update_price("BTC/USDT", Decimal(str(p))) + rm.update_price("AAPL", Decimal(str(p))) - vol = rm.get_volatility("BTC/USDT") + vol = rm.get_volatility("AAPL") assert vol is not None assert vol > 0 @@ -177,9 +177,9 @@ def test_position_size_with_volatility_scaling(): # Feed volatile prices prices = [100, 120, 80, 130, 70] for p in prices: - rm.update_price("BTC/USDT", Decimal(str(p))) + rm.update_price("AAPL", Decimal(str(p))) - size = rm.calculate_position_size("BTC/USDT", Decimal("10000")) + size = rm.calculate_position_size("AAPL", Decimal("10000")) base = Decimal("10000") * Decimal("0.1") # High volatility should reduce size below base @@ -192,9 +192,9 @@ def test_position_size_without_scaling(): prices = [100, 120, 80, 130, 70] for p in prices: - rm.update_price("BTC/USDT", Decimal(str(p))) + rm.update_price("AAPL", Decimal(str(p))) - size = rm.calculate_position_size("BTC/USDT", Decimal("10000")) + size = rm.calculate_position_size("AAPL", Decimal("10000")) base = Decimal("10000") * Decimal("0.1") assert size == base @@ -211,8 +211,8 @@ def test_portfolio_exposure_check_passes(): max_portfolio_exposure=0.8, ) positions = { - "BTCUSDT": Position( - symbol="BTCUSDT", + "AAPL": Position( + symbol="AAPL", quantity=Decimal("0.01"), avg_entry_price=Decimal("50000"), current_price=Decimal("50000"), @@ -230,8 +230,8 @@ def test_portfolio_exposure_check_rejects(): max_portfolio_exposure=0.3, ) positions = { - "BTCUSDT": Position( - symbol="BTCUSDT", + "AAPL": Position( + symbol="AAPL", quantity=Decimal("1"), avg_entry_price=Decimal("50000"), current_price=Decimal("50000"), @@ -263,10 +263,10 @@ def test_var_calculation(): daily_loss_limit_pct=Decimal("10"), ) for i in range(30): - rm.update_price("BTCUSDT", Decimal(str(100 + (i % 5) - 2))) + rm.update_price("AAPL", Decimal(str(100 + (i % 5) - 2))) positions = { - "BTCUSDT": Position( - symbol="BTCUSDT", + "AAPL": Position( + symbol="AAPL", quantity=Decimal("1"), avg_entry_price=Decimal("100"), current_price=Decimal("100"), @@ -357,7 +357,7 @@ def test_drawdown_check_rejects_in_check(): rm.update_balance(Decimal("10000")) signal = Signal( strategy="test", - symbol="BTC/USDT", + symbol="AAPL", side=OrderSide.BUY, price=Decimal("50000"), quantity=Decimal("0.01"), diff --git a/services/portfolio-manager/tests/test_portfolio.py b/services/portfolio-manager/tests/test_portfolio.py index 768e071..365dc1a 100644 --- a/services/portfolio-manager/tests/test_portfolio.py +++ b/services/portfolio-manager/tests/test_portfolio.py @@ -10,7 +10,7 @@ def make_order(side: OrderSide, price: str, quantity: str) -> Order: """Helper to create a filled Order.""" return Order( signal_id="test-signal", - symbol="BTC/USDT", + symbol="AAPL", side=side, type=OrderType.MARKET, price=Decimal(price), @@ -24,7 +24,7 @@ def test_portfolio_add_buy_order() -> None: order = make_order(OrderSide.BUY, "50000", "0.1") tracker.apply_order(order) - position = tracker.get_position("BTC/USDT") + position = tracker.get_position("AAPL") assert position is not None assert position.quantity == Decimal("0.1") assert position.avg_entry_price == Decimal("50000") @@ -35,7 +35,7 @@ def test_portfolio_add_multiple_buys() -> None: tracker.apply_order(make_order(OrderSide.BUY, "50000", "0.1")) tracker.apply_order(make_order(OrderSide.BUY, "52000", "0.1")) - position = tracker.get_position("BTC/USDT") + position = tracker.get_position("AAPL") assert position is not None assert position.quantity == Decimal("0.2") assert position.avg_entry_price == Decimal("51000") @@ -46,7 +46,7 @@ def test_portfolio_sell_reduces_position() -> None: tracker.apply_order(make_order(OrderSide.BUY, "50000", "0.2")) tracker.apply_order(make_order(OrderSide.SELL, "55000", "0.1")) - position = tracker.get_position("BTC/USDT") + position = tracker.get_position("AAPL") assert position is not None assert position.quantity == Decimal("0.1") assert position.avg_entry_price == Decimal("50000") @@ -54,7 +54,7 @@ def test_portfolio_sell_reduces_position() -> None: def test_portfolio_no_position_returns_none() -> None: tracker = PortfolioTracker() - position = tracker.get_position("ETH/USDT") + position = tracker.get_position("MSFT") assert position is None @@ -66,7 +66,7 @@ def test_realized_pnl_on_sell() -> None: tracker.apply_order( Order( signal_id="s1", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.BUY, type=OrderType.MARKET, price=Decimal("50000"), @@ -80,7 +80,7 @@ def test_realized_pnl_on_sell() -> None: tracker.apply_order( Order( signal_id="s2", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.SELL, type=OrderType.MARKET, price=Decimal("55000"), @@ -98,7 +98,7 @@ def test_realized_pnl_on_loss() -> None: tracker.apply_order( Order( signal_id="s1", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.BUY, type=OrderType.MARKET, price=Decimal("50000"), @@ -109,7 +109,7 @@ def test_realized_pnl_on_loss() -> None: tracker.apply_order( Order( signal_id="s2", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.SELL, type=OrderType.MARKET, price=Decimal("45000"), @@ -128,7 +128,7 @@ def test_realized_pnl_accumulates() -> None: tracker.apply_order( Order( signal_id="s1", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.BUY, type=OrderType.MARKET, price=Decimal("50000"), @@ -141,7 +141,7 @@ def test_realized_pnl_accumulates() -> None: tracker.apply_order( Order( signal_id="s2", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.SELL, type=OrderType.MARKET, price=Decimal("55000"), @@ -154,7 +154,7 @@ def test_realized_pnl_accumulates() -> None: tracker.apply_order( Order( signal_id="s3", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.SELL, type=OrderType.MARKET, price=Decimal("60000"), diff --git a/services/portfolio-manager/tests/test_snapshot.py b/services/portfolio-manager/tests/test_snapshot.py index a464599..ec5e92d 100644 --- a/services/portfolio-manager/tests/test_snapshot.py +++ b/services/portfolio-manager/tests/test_snapshot.py @@ -13,7 +13,7 @@ class TestSaveSnapshot: from portfolio_manager.main import save_snapshot pos = Position( - symbol="BTCUSDT", + symbol="AAPL", quantity=Decimal("0.5"), avg_entry_price=Decimal("50000"), current_price=Decimal("52000"), diff --git a/services/strategy-engine/src/strategy_engine/config.py b/services/strategy-engine/src/strategy_engine/config.py index e3a49c2..15f8588 100644 --- a/services/strategy-engine/src/strategy_engine/config.py +++ b/services/strategy-engine/src/strategy_engine/config.py @@ -4,6 +4,10 @@ from shared.config import Settings class StrategyConfig(Settings): - symbols: list[str] = ["BTC/USDT"] + symbols: list[str] = ["AAPL", "MSFT", "GOOGL", "AMZN", "TSLA"] timeframes: list[str] = ["1m"] strategy_params: dict = {} + selector_final_time: str = "15:30" + selector_max_picks: int = 3 + anthropic_api_key: str = "" + anthropic_model: str = "claude-sonnet-4-20250514" diff --git a/services/strategy-engine/src/strategy_engine/main.py b/services/strategy-engine/src/strategy_engine/main.py index d62f886..5a30766 100644 --- a/services/strategy-engine/src/strategy_engine/main.py +++ b/services/strategy-engine/src/strategy_engine/main.py @@ -1,18 +1,23 @@ """Strategy Engine Service entry point.""" import asyncio +from datetime import datetime from pathlib import Path +import zoneinfo +from shared.alpaca import AlpacaClient from shared.broker import RedisBroker +from shared.db import Database from shared.healthcheck import HealthCheckServer from shared.logging import setup_logging from shared.metrics import ServiceMetrics from shared.notifier import TelegramNotifier -from shared.sentiment import SentimentProvider +from shared.sentiment_models import MarketSentiment from strategy_engine.config import StrategyConfig from strategy_engine.engine import StrategyEngine from strategy_engine.plugin_loader import load_strategies +from strategy_engine.stock_selector import StockSelector # The strategies directory lives alongside the installed package STRATEGIES_DIR = Path(__file__).parent.parent.parent.parent / "strategies" @@ -22,28 +27,6 @@ STRATEGIES_DIR = Path(__file__).parent.parent.parent.parent / "strategies" # order-executor: +2 (8082), portfolio-manager: +3 (8083) HEALTH_PORT_OFFSET = 1 -SENTIMENT_REFRESH_INTERVAL = 300 # 5 minutes - - -async def sentiment_loop(provider: SentimentProvider, strategies: list, log) -> None: - """Periodically fetch sentiment and update strategies that support it.""" - while True: - try: - sentiment = await provider.get_sentiment("SOL") - log.info( - "sentiment_updated", - fear_greed=sentiment.fear_greed_value, - news=sentiment.news_sentiment, - netflow=sentiment.exchange_netflow, - should_block=sentiment.should_block, - ) - for strategy in strategies: - if hasattr(strategy, "update_sentiment"): - strategy.update_sentiment(sentiment) - except Exception as exc: - log.warning("sentiment_fetch_failed", error=str(exc)) - await asyncio.sleep(SENTIMENT_REFRESH_INTERVAL) - async def process_symbol(engine: StrategyEngine, stream: str, log) -> None: """Process candles for a single symbol stream.""" @@ -53,6 +36,40 @@ async def process_symbol(engine: StrategyEngine, stream: str, log) -> None: last_id = await engine.process_once(stream, last_id) +async def run_stock_selector( + selector: StockSelector, + notifier: TelegramNotifier, + db: Database, + config: StrategyConfig, + log, +) -> None: + """Run the stock selector once per day at the configured time.""" + et = zoneinfo.ZoneInfo("America/New_York") + + while True: + now_et = datetime.now(et) + target_hour, target_min = map(int, config.selector_final_time.split(":")) + + if now_et.hour == target_hour and now_et.minute == target_min: + log.info("stock_selector_running") + try: + selections = await selector.select() + if selections: + ms_data = await db.get_latest_market_sentiment() + ms = None + if ms_data: + ms = MarketSentiment(**ms_data) + await notifier.send_stock_selection(selections, ms) + log.info("stock_selector_complete", picks=[s.symbol for s in selections]) + else: + log.info("stock_selector_no_picks") + except Exception as exc: + log.error("stock_selector_error", error=str(exc)) + await asyncio.sleep(120) # Sleep past this minute + else: + await asyncio.sleep(30) + + async def run() -> None: config = StrategyConfig() log = setup_logging("strategy-engine", config.log_level, config.log_format) @@ -64,6 +81,16 @@ async def run() -> None: ) broker = RedisBroker(config.redis_url) + + db = Database(config.database_url) + await db.connect() + + alpaca = AlpacaClient( + api_key=config.alpaca_api_key, + api_secret=config.alpaca_api_secret, + paper=config.alpaca_paper, + ) + strategies = load_strategies(STRATEGIES_DIR) for strategy in strategies: @@ -74,8 +101,6 @@ async def run() -> None: engine = StrategyEngine(broker=broker, strategies=strategies) - provider = SentimentProvider() - health = HealthCheckServer( "strategy-engine", port=config.health_port + HEALTH_PORT_OFFSET, @@ -87,15 +112,25 @@ async def run() -> None: tasks = [] try: - # Sentiment updater - tasks.append(asyncio.create_task(sentiment_loop(provider, strategies, log))) - # Symbol processors for symbol in config.symbols: stream = f"candles.{symbol.replace('/', '_')}" task = asyncio.create_task(process_symbol(engine, stream, log)) tasks.append(task) - # Wait for all tasks (they run forever until cancelled) + if config.anthropic_api_key: + selector = StockSelector( + db=db, + broker=broker, + alpaca=alpaca, + anthropic_api_key=config.anthropic_api_key, + anthropic_model=config.anthropic_model, + max_picks=config.selector_max_picks, + ) + tasks.append( + asyncio.create_task(run_stock_selector(selector, notifier, db, config, log)) + ) + log.info("stock_selector_enabled", time=config.selector_final_time) + await asyncio.gather(*tasks) except Exception as exc: log.error("fatal_error", error=str(exc)) @@ -106,8 +141,9 @@ async def run() -> None: task.cancel() metrics.service_up.labels(service="strategy-engine").set(0) await notifier.close() - await provider.close() await broker.close() + await alpaca.close() + await db.close() def main() -> None: diff --git a/services/strategy-engine/src/strategy_engine/stock_selector.py b/services/strategy-engine/src/strategy_engine/stock_selector.py new file mode 100644 index 0000000..268d557 --- /dev/null +++ b/services/strategy-engine/src/strategy_engine/stock_selector.py @@ -0,0 +1,404 @@ +"""3-stage stock selector engine: sentiment → technical → LLM.""" + +import json +import logging +import re +from datetime import datetime, timezone + +import aiohttp + +from shared.alpaca import AlpacaClient +from shared.broker import RedisBroker +from shared.db import Database +from shared.models import OrderSide +from shared.sentiment_models import Candidate, MarketSentiment, SelectedStock + +logger = logging.getLogger(__name__) + +ANTHROPIC_API_URL = "https://api.anthropic.com/v1/messages" + + +def _parse_llm_selections(text: str) -> list[SelectedStock]: + """Parse LLM response into SelectedStock list. + + Handles both bare JSON arrays and markdown code blocks. + Returns empty list on any parse error. + """ + # Try to extract JSON from markdown code block first + code_block = re.search(r"```(?:json)?\s*(\[.*?\])\s*```", text, re.DOTALL) + if code_block: + raw = code_block.group(1) + else: + # Try to find a bare JSON array + array_match = re.search(r"\[.*\]", text, re.DOTALL) + if array_match: + raw = array_match.group(0) + else: + raw = text.strip() + + try: + data = json.loads(raw) + if not isinstance(data, list): + return [] + selections = [] + for item in data: + if not isinstance(item, dict): + continue + try: + selection = SelectedStock( + symbol=item["symbol"], + side=OrderSide(item["side"]), + conviction=float(item["conviction"]), + reason=item.get("reason", ""), + key_news=item.get("key_news", []), + ) + selections.append(selection) + except (KeyError, ValueError) as e: + logger.warning("Skipping invalid selection item: %s", e) + return selections + except (json.JSONDecodeError, TypeError): + return [] + + +class SentimentCandidateSource: + """Generates candidates from DB sentiment scores.""" + + def __init__(self, db: Database) -> None: + self._db = db + + async def get_candidates(self) -> list[Candidate]: + rows = await self._db.get_top_symbol_scores(limit=20) + candidates = [] + for row in rows: + composite = float(row.get("composite", 0)) + if composite == 0: + continue + candidates.append( + Candidate( + symbol=row["symbol"], + source="sentiment", + score=composite, + reason=f"composite={composite:.2f}, news_count={row.get('news_count', 0)}", + ) + ) + return candidates + + +class LLMCandidateSource: + """Generates candidates by asking Claude to analyze recent news.""" + + def __init__(self, db: Database, api_key: str, model: str) -> None: + self._db = db + self._api_key = api_key + self._model = model + + async def get_candidates(self) -> list[Candidate]: + news_items = await self._db.get_recent_news(hours=24) + if not news_items: + return [] + + headlines = [] + for item in news_items[:50]: # cap at 50 to stay within context + symbols = item.get("symbols", []) + sym_str = ", ".join(symbols) if symbols else "N/A" + headlines.append(f"[{sym_str}] {item['headline']}") + + prompt = ( + "You are a stock analyst. Given recent news headlines, identify the 5-10 most " + "actionable US stock tickers. Return ONLY a JSON array with objects having: " + "symbol (ticker), direction ('BUY' or 'SELL'), score (0-1), reason (brief).\n\n" + "Headlines:\n" + "\n".join(headlines) + ) + + try: + async with aiohttp.ClientSession() as session: + async with session.post( + ANTHROPIC_API_URL, + headers={ + "x-api-key": self._api_key, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + json={ + "model": self._model, + "max_tokens": 1024, + "messages": [{"role": "user", "content": prompt}], + }, + ) as resp: + if resp.status != 200: + body = await resp.text() + logger.error("LLM candidate source error %d: %s", resp.status, body) + return [] + data = await resp.json() + + content = data.get("content", []) + text = "" + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + text += block.get("text", "") + + return self._parse_candidates(text) + except Exception as e: + logger.error("LLMCandidateSource error: %s", e) + return [] + + def _parse_candidates(self, text: str) -> list[Candidate]: + code_block = re.search(r"```(?:json)?\s*(\[.*?\])\s*```", text, re.DOTALL) + if code_block: + raw = code_block.group(1) + else: + array_match = re.search(r"\[.*\]", text, re.DOTALL) + raw = array_match.group(0) if array_match else text.strip() + + try: + items = json.loads(raw) + if not isinstance(items, list): + return [] + candidates = [] + for item in items: + if not isinstance(item, dict): + continue + try: + direction_str = item.get("direction", "BUY") + direction = OrderSide(direction_str) + except ValueError: + direction = None + candidates.append( + Candidate( + symbol=item["symbol"], + source="llm", + direction=direction, + score=float(item.get("score", 0.5)), + reason=item.get("reason", ""), + ) + ) + return candidates + except (json.JSONDecodeError, TypeError, KeyError): + return [] + + +def _compute_rsi(closes: list[float], period: int = 14) -> float: + """Compute RSI for the last data point.""" + if len(closes) < period + 1: + return 50.0 # neutral if insufficient data + + deltas = [closes[i] - closes[i - 1] for i in range(1, len(closes))] + gains = [d if d > 0 else 0.0 for d in deltas] + losses = [-d if d < 0 else 0.0 for d in deltas] + + avg_gain = sum(gains[:period]) / period + avg_loss = sum(losses[:period]) / period + + for i in range(period, len(deltas)): + avg_gain = (avg_gain * (period - 1) + gains[i]) / period + avg_loss = (avg_loss * (period - 1) + losses[i]) / period + + if avg_loss == 0: + return 100.0 + rs = avg_gain / avg_loss + return 100.0 - (100.0 / (1.0 + rs)) + + +class StockSelector: + """Orchestrates the 3-stage stock selection pipeline.""" + + def __init__( + self, + db: Database, + broker: RedisBroker, + alpaca: AlpacaClient, + anthropic_api_key: str, + anthropic_model: str = "claude-sonnet-4-20250514", + max_picks: int = 3, + ) -> None: + self._db = db + self._broker = broker + self._alpaca = alpaca + self._api_key = anthropic_api_key + self._model = anthropic_model + self._max_picks = max_picks + + async def select(self) -> list[SelectedStock]: + """Run the full 3-stage pipeline and return selected stocks.""" + # Market gate: check sentiment + sentiment_data = await self._db.get_latest_market_sentiment() + if sentiment_data is None: + logger.warning("No market sentiment data; skipping selection") + return [] + + market_sentiment = MarketSentiment(**sentiment_data) + if market_sentiment.market_regime == "risk_off": + logger.info("Market is risk_off; skipping stock selection") + return [] + + # Stage 1: gather candidates from both sources + sentiment_source = SentimentCandidateSource(self._db) + llm_source = LLMCandidateSource(self._db, self._api_key, self._model) + + sentiment_candidates = await sentiment_source.get_candidates() + llm_candidates = await llm_source.get_candidates() + + candidates = self._merge_candidates(sentiment_candidates, llm_candidates) + if not candidates: + logger.info("No candidates found") + return [] + + # Stage 2: technical filter + filtered = await self._technical_filter(candidates) + if not filtered: + logger.info("All candidates filtered out by technical criteria") + return [] + + # Stage 3: LLM final selection + selections = await self._llm_final_select(filtered, market_sentiment) + + # Persist and publish + today = datetime.now(timezone.utc).date() + sentiment_snapshot = { + "fear_greed": market_sentiment.fear_greed, + "market_regime": market_sentiment.market_regime, + "vix": market_sentiment.vix, + } + for stock in selections: + try: + await self._db.insert_stock_selection( + trade_date=today, + symbol=stock.symbol, + side=stock.side.value, + conviction=stock.conviction, + reason=stock.reason, + key_news=stock.key_news, + sentiment_snapshot=sentiment_snapshot, + ) + except Exception as e: + logger.error("Failed to persist selection for %s: %s", stock.symbol, e) + + try: + await self._broker.publish( + "selected_stocks", + { + "symbol": stock.symbol, + "side": stock.side.value, + "conviction": stock.conviction, + "reason": stock.reason, + "key_news": stock.key_news, + "trade_date": str(today), + }, + ) + except Exception as e: + logger.error("Failed to publish selection for %s: %s", stock.symbol, e) + + return selections + + def _merge_candidates( + self, sentiment: list[Candidate], llm: list[Candidate] + ) -> list[Candidate]: + """Deduplicate candidates by symbol, keeping the higher score.""" + by_symbol: dict[str, Candidate] = {} + for c in sentiment + llm: + existing = by_symbol.get(c.symbol) + if existing is None or c.score > existing.score: + by_symbol[c.symbol] = c + return sorted(by_symbol.values(), key=lambda c: c.score, reverse=True) + + async def _technical_filter(self, candidates: list[Candidate]) -> list[Candidate]: + """Filter candidates using RSI, EMA20, and volume criteria.""" + passed = [] + for candidate in candidates: + try: + bars = await self._alpaca.get_bars(candidate.symbol, timeframe="1Day", limit=60) + if len(bars) < 21: + logger.debug("Insufficient bars for %s", candidate.symbol) + continue + + closes = [float(b["c"]) for b in bars] + volumes = [float(b["v"]) for b in bars] + + rsi = _compute_rsi(closes) + if not (30 <= rsi <= 70): + logger.debug("%s RSI=%.1f outside 30-70", candidate.symbol, rsi) + continue + + ema20 = sum(closes[-20:]) / 20 # simple approximation + current_price = closes[-1] + if current_price <= ema20: + logger.debug( + "%s price %.2f <= EMA20 %.2f", candidate.symbol, current_price, ema20 + ) + continue + + avg_volume = sum(volumes[:-1]) / max(len(volumes) - 1, 1) + current_volume = volumes[-1] + if current_volume <= 0.5 * avg_volume: + logger.debug( + "%s volume %.0f <= 50%% avg %.0f", + candidate.symbol, + current_volume, + avg_volume, + ) + continue + + passed.append(candidate) + except Exception as e: + logger.warning("Technical filter error for %s: %s", candidate.symbol, e) + + return passed + + async def _llm_final_select( + self, candidates: list[Candidate], market_sentiment: MarketSentiment + ) -> list[SelectedStock]: + """Ask Claude to pick 2-3 stocks with rationale.""" + candidate_lines = [ + f"- {c.symbol} (source={c.source}, score={c.score:.2f}, reason={c.reason})" + for c in candidates + ] + market_context = ( + f"Fear/Greed: {market_sentiment.fear_greed} ({market_sentiment.fear_greed_label}), " + f"VIX: {market_sentiment.vix}, " + f"Fed stance: {market_sentiment.fed_stance}, " + f"Regime: {market_sentiment.market_regime}" + ) + + prompt = ( + f"You are a portfolio manager. Select 2-3 stocks for today's session.\n\n" + f"Market context: {market_context}\n\n" + f"Candidates (already passed technical filters):\n" + + "\n".join(candidate_lines) + + "\n\n" + "Return ONLY a JSON array with objects having:\n" + " symbol, side ('BUY' or 'SELL'), conviction (0-1), reason (1-2 sentences), " + "key_news (list of 1-3 relevant headlines or facts)\n" + f"Select at most {self._max_picks} stocks." + ) + + try: + async with aiohttp.ClientSession() as session: + async with session.post( + ANTHROPIC_API_URL, + headers={ + "x-api-key": self._api_key, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + json={ + "model": self._model, + "max_tokens": 1024, + "messages": [{"role": "user", "content": prompt}], + }, + ) as resp: + if resp.status != 200: + body = await resp.text() + logger.error("LLM final select error %d: %s", resp.status, body) + return [] + data = await resp.json() + + content = data.get("content", []) + text = "" + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + text += block.get("text", "") + + return _parse_llm_selections(text)[: self._max_picks] + except Exception as e: + logger.error("LLM final select error: %s", e) + return [] diff --git a/services/strategy-engine/strategies/asian_session_rsi.py b/services/strategy-engine/strategies/asian_session_rsi.py deleted file mode 100644 index 1874591..0000000 --- a/services/strategy-engine/strategies/asian_session_rsi.py +++ /dev/null @@ -1,266 +0,0 @@ -"""Asian Session RSI Strategy — 한국시간 9:00~11:00 단타. - -규칙: -- SOL/USDT 5분봉 -- 매수: RSI(14) < 25 + 볼륨 > 평균 + 센티먼트 OK -- 익절: +1.5%, 손절: -0.7%, 시간청산: 11:00 KST (02:00 UTC) -- 하루 최대 3회, 2연패 시 중단 -- 센티먼트 필터: Fear & Greed > 80이면 매수 차단, 뉴스 극도 부정이면 차단 -""" - -from collections import deque -from decimal import Decimal -from datetime import datetime - -import pandas as pd - -from shared.models import Candle, Signal, OrderSide -from shared.sentiment import SentimentData -from strategies.base import BaseStrategy - - -class AsianSessionRsiStrategy(BaseStrategy): - name: str = "asian_session_rsi" - - def __init__(self) -> None: - super().__init__() - self._rsi_period: int = 14 - self._rsi_oversold: float = 25.0 - self._rsi_overbought: float = 75.0 - self._quantity: Decimal = Decimal("0.1") - self._take_profit_pct: float = 1.5 - self._stop_loss_pct: float = 0.7 - # Session: 00:00~02:00 UTC = 09:00~11:00 KST - self._session_start_utc: int = 0 - self._session_end_utc: int = 2 - self._max_trades_per_day: int = 3 - self._max_consecutive_losses: int = 2 - self._use_sentiment: bool = True - self._ema_period: int = 20 - self._require_bullish_candle: bool = True - self._prev_candle_bullish: bool = False - # Sentiment (updated externally before each session) - self._sentiment: SentimentData | None = None - # State - self._closes: deque[float] = deque(maxlen=200) - self._volumes: deque[float] = deque(maxlen=50) - self._today: str | None = None - self._trades_today: int = 0 - self._consecutive_losses: int = 0 - self._in_position: bool = False - self._entry_price: float = 0.0 - - @property - def warmup_period(self) -> int: - return self._rsi_period + 1 - - def configure(self, params: dict) -> None: - self._rsi_period = int(params.get("rsi_period", 14)) - self._rsi_oversold = float(params.get("rsi_oversold", 25.0)) - self._rsi_overbought = float(params.get("rsi_overbought", 75.0)) - self._quantity = Decimal(str(params.get("quantity", "0.1"))) - self._take_profit_pct = float(params.get("take_profit_pct", 1.5)) - self._stop_loss_pct = float(params.get("stop_loss_pct", 0.7)) - self._session_start_utc = int(params.get("session_start_utc", 0)) - self._session_end_utc = int(params.get("session_end_utc", 2)) - self._max_trades_per_day = int(params.get("max_trades_per_day", 3)) - self._max_consecutive_losses = int(params.get("max_consecutive_losses", 2)) - self._use_sentiment = bool(params.get("use_sentiment", True)) - self._ema_period = int(params.get("ema_period", 20)) - self._require_bullish_candle = bool(params.get("require_bullish_candle", True)) - - if self._quantity <= 0: - raise ValueError(f"Quantity must be positive, got {self._quantity}") - if self._stop_loss_pct <= 0: - raise ValueError(f"Stop loss must be positive, got {self._stop_loss_pct}") - if self._take_profit_pct <= 0: - raise ValueError(f"Take profit must be positive, got {self._take_profit_pct}") - - self._init_filters( - require_trend=False, - adx_threshold=25.0, - min_volume_ratio=0.5, - atr_stop_multiplier=1.5, - atr_tp_multiplier=2.0, - ) - - def reset(self) -> None: - super().reset() - self._closes.clear() - self._volumes.clear() - self._today = None - self._trades_today = 0 - self._consecutive_losses = 0 - self._in_position = False - self._entry_price = 0.0 - self._sentiment = None - self._prev_candle_bullish = False - - def update_sentiment(self, sentiment: SentimentData) -> None: - """Update sentiment data. Call before each trading session.""" - self._sentiment = sentiment - - def _check_sentiment(self) -> bool: - """Check if sentiment allows buying. Returns True if OK.""" - if not self._use_sentiment or self._sentiment is None: - return True # No sentiment data, allow by default - return not self._sentiment.should_block - - def _is_session_active(self, dt: datetime) -> bool: - """Check if current time is within trading session.""" - hour = dt.hour - if self._session_start_utc <= self._session_end_utc: - return self._session_start_utc <= hour < self._session_end_utc - # Wrap around midnight - return hour >= self._session_start_utc or hour < self._session_end_utc - - def _compute_rsi(self) -> float | None: - if len(self._closes) < self._rsi_period + 1: - return None - series = pd.Series(list(self._closes)) - delta = series.diff() - gain = delta.clip(lower=0) - loss = -delta.clip(upper=0) - avg_gain = gain.ewm(com=self._rsi_period - 1, min_periods=self._rsi_period).mean() - avg_loss = loss.ewm(com=self._rsi_period - 1, min_periods=self._rsi_period).mean() - rs = avg_gain / avg_loss.replace(0, float("nan")) - rsi = 100 - (100 / (1 + rs)) - val = rsi.iloc[-1] - if pd.isna(val): - return None - return float(val) - - def _volume_above_average(self) -> bool: - if len(self._volumes) < 20: - return True # Not enough data, allow - avg = sum(self._volumes) / len(self._volumes) - return self._volumes[-1] >= avg - - def _price_above_ema(self) -> bool: - """Check if current price is above short-term EMA.""" - if len(self._closes) < self._ema_period: - return True # Not enough data, allow by default - series = pd.Series(list(self._closes)) - ema_val = series.ewm(span=self._ema_period, adjust=False).mean().iloc[-1] - return self._closes[-1] >= ema_val - - def on_candle(self, candle: Candle) -> Signal | None: - self._update_filter_data(candle) - - close = float(candle.close) - self._closes.append(close) - self._volumes.append(float(candle.volume)) - - # Track candle direction for bullish confirmation - is_bullish = float(candle.close) >= float(candle.open) - - # Daily reset - day = candle.open_time.strftime("%Y-%m-%d") - if self._today != day: - self._today = day - self._trades_today = 0 - # Don't reset consecutive_losses — carries across days - - # Check exit conditions first (if in position) - if self._in_position: - pnl_pct = (close - self._entry_price) / self._entry_price * 100 - - # Take profit - if pnl_pct >= self._take_profit_pct: - self._in_position = False - self._consecutive_losses = 0 - return self._apply_filters( - Signal( - strategy=self.name, - symbol=candle.symbol, - side=OrderSide.SELL, - price=candle.close, - quantity=self._quantity, - conviction=0.9, - reason=f"Take profit {pnl_pct:.2f}% >= {self._take_profit_pct}%", - ) - ) - - # Stop loss - if pnl_pct <= -self._stop_loss_pct: - self._in_position = False - self._consecutive_losses += 1 - return self._apply_filters( - Signal( - strategy=self.name, - symbol=candle.symbol, - side=OrderSide.SELL, - price=candle.close, - quantity=self._quantity, - conviction=1.0, - reason=f"Stop loss {pnl_pct:.2f}% <= -{self._stop_loss_pct}%", - ) - ) - - # Time exit: session ended while in position - if not self._is_session_active(candle.open_time): - self._in_position = False - if pnl_pct < 0: - self._consecutive_losses += 1 - else: - self._consecutive_losses = 0 - return self._apply_filters( - Signal( - strategy=self.name, - symbol=candle.symbol, - side=OrderSide.SELL, - price=candle.close, - quantity=self._quantity, - conviction=0.5, - reason=f"Time exit (session ended), PnL {pnl_pct:.2f}%", - ) - ) - - return None # Still in position, no action - - # Entry conditions - if not self._is_session_active(candle.open_time): - return None # Outside trading hours - - if self._trades_today >= self._max_trades_per_day: - return None # Daily limit reached - - if self._consecutive_losses >= self._max_consecutive_losses: - return None # Consecutive loss limit - - if not self._check_sentiment(): - return None # Sentiment blocked (extreme greed or very negative news) - - rsi = self._compute_rsi() - if rsi is None: - return None - - if rsi < self._rsi_oversold and self._volume_above_average() and self._price_above_ema(): - if self._require_bullish_candle and not is_bullish: - return None # Wait for bullish candle confirmation - self._in_position = True - self._entry_price = close - self._trades_today += 1 - - # Conviction: lower RSI = stronger signal - conv = min((self._rsi_oversold - rsi) / self._rsi_oversold, 1.0) - conv = max(conv, 0.3) - - sl = candle.close * (1 - Decimal(str(self._stop_loss_pct / 100))) - tp = candle.close * (1 + Decimal(str(self._take_profit_pct / 100))) - - return self._apply_filters( - Signal( - strategy=self.name, - symbol=candle.symbol, - side=OrderSide.BUY, - price=candle.close, - quantity=self._quantity, - conviction=conv, - stop_loss=sl, - take_profit=tp, - reason=f"RSI {rsi:.1f} < {self._rsi_oversold} (session active, vol OK)", - ) - ) - - return None diff --git a/services/strategy-engine/strategies/config/asian_session_rsi.yaml b/services/strategy-engine/strategies/config/asian_session_rsi.yaml deleted file mode 100644 index bc7c5c9..0000000 --- a/services/strategy-engine/strategies/config/asian_session_rsi.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Asian Session RSI — SOL/USDT 5분봉 단타 -# 한국시간 9:00~11:00 (UTC 0:00~2:00) -rsi_period: 14 -rsi_oversold: 25 -rsi_overbought: 75 -quantity: "0.5" # SOL 0.5개 (~$75, 100만원의 10%) -take_profit_pct: 1.5 # 익절 1.5% -stop_loss_pct: 0.7 # 손절 0.7% -session_start_utc: 0 # UTC 0시 = KST 9시 -session_end_utc: 2 # UTC 2시 = KST 11시 -max_trades_per_day: 3 # 하루 최대 3회 -max_consecutive_losses: 2 # 2연패 시 중단 -ema_period: 20 -require_bullish_candle: true diff --git a/services/strategy-engine/strategies/config/grid_strategy.yaml b/services/strategy-engine/strategies/config/grid_strategy.yaml index 607f3df..338bb4c 100644 --- a/services/strategy-engine/strategies/config/grid_strategy.yaml +++ b/services/strategy-engine/strategies/config/grid_strategy.yaml @@ -1,4 +1,4 @@ -lower_price: 60000 -upper_price: 70000 +lower_price: 170 +upper_price: 190 grid_count: 5 -quantity: "0.01" +quantity: "1" diff --git a/services/strategy-engine/tests/conftest.py b/services/strategy-engine/tests/conftest.py index eb31b23..2b909ef 100644 --- a/services/strategy-engine/tests/conftest.py +++ b/services/strategy-engine/tests/conftest.py @@ -7,3 +7,8 @@ from pathlib import Path STRATEGIES_DIR = Path(__file__).parent.parent / "strategies" if str(STRATEGIES_DIR) not in sys.path: sys.path.insert(0, str(STRATEGIES_DIR.parent)) + +# Ensure the worktree's strategy_engine src is preferred over any installed version +WORKTREE_SRC = Path(__file__).parent.parent / "src" +if str(WORKTREE_SRC) not in sys.path: + sys.path.insert(0, str(WORKTREE_SRC)) diff --git a/services/strategy-engine/tests/test_asian_session_rsi.py b/services/strategy-engine/tests/test_asian_session_rsi.py deleted file mode 100644 index db031f0..0000000 --- a/services/strategy-engine/tests/test_asian_session_rsi.py +++ /dev/null @@ -1,190 +0,0 @@ -"""Tests for Asian Session RSI strategy.""" - -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from datetime import datetime, timezone -from decimal import Decimal - -from shared.models import Candle, OrderSide -from strategies.asian_session_rsi import AsianSessionRsiStrategy - - -def _candle(price, hour=0, minute=30, volume=100.0, day=1): - return Candle( - symbol="SOLUSDT", - timeframe="5m", - open_time=datetime(2025, 1, day, hour, minute, tzinfo=timezone.utc), - open=Decimal(str(price)), - high=Decimal(str(price + 1)), - low=Decimal(str(price - 1)), - close=Decimal(str(price)), - volume=Decimal(str(volume)), - ) - - -def _make_strategy(**overrides): - s = AsianSessionRsiStrategy() - params = { - "rsi_period": 5, - "rsi_oversold": 30, - "rsi_overbought": 70, - "quantity": "0.5", - "take_profit_pct": 1.5, - "stop_loss_pct": 0.7, - "session_start_utc": 0, - "session_end_utc": 2, - "max_trades_per_day": 3, - "max_consecutive_losses": 2, - } - params.update(overrides) - s.configure(params) - return s - - -def test_no_signal_outside_session(): - s = _make_strategy() - # Hour 5 UTC = outside session (0-2 UTC) - for i in range(10): - sig = s.on_candle(_candle(100 - i * 3, hour=5)) - assert sig is None - - -def test_buy_signal_during_session_on_oversold(): - s = AsianSessionRsiStrategy() - s._rsi_period = 5 - s._rsi_oversold = 30 - s._quantity = Decimal("0.5") - s._take_profit_pct = 1.5 - s._stop_loss_pct = 0.7 - s._session_start_utc = 0 - s._session_end_utc = 2 - s._max_trades_per_day = 3 - s._max_consecutive_losses = 10 # High limit so test isn't blocked - - # Feed declining prices — collect all signals - signals = [] - for i in range(10): - sig = s.on_candle(_candle(100 - i * 3, hour=0, minute=i * 5)) - if sig is not None: - signals.append(sig) - - # Should have generated at least one BUY signal - buy_signals = [s for s in signals if s.side == OrderSide.BUY] - assert len(buy_signals) > 0 - assert buy_signals[0].strategy == "asian_session_rsi" - - -def test_take_profit_exit(): - s = _make_strategy(rsi_period=5, rsi_oversold=40) - # Force entry - for i in range(8): - s.on_candle(_candle(100 - i * 2, hour=0, minute=i * 5)) - - # Should be in position now — push price up for TP - sig = s.on_candle(_candle(100, hour=0, minute=50)) # entry ~around 84-86 - if s._in_position: - tp_price = s._entry_price * (1 + s._take_profit_pct / 100) - sig = s.on_candle(_candle(tp_price + 1, hour=1, minute=0)) - if sig is not None: - assert sig.side == OrderSide.SELL - assert "Take profit" in sig.reason - - -def test_stop_loss_exit(): - s = _make_strategy(rsi_period=5, rsi_oversold=40) - for i in range(8): - s.on_candle(_candle(100 - i * 2, hour=0, minute=i * 5)) - - if s._in_position: - sl_price = s._entry_price * (1 - s._stop_loss_pct / 100) - sig = s.on_candle(_candle(sl_price - 1, hour=1, minute=0)) - if sig is not None: - assert sig.side == OrderSide.SELL - assert "Stop loss" in sig.reason - - -def test_time_exit_when_session_ends(): - s = _make_strategy(rsi_period=5, rsi_oversold=40) - for i in range(8): - s.on_candle(_candle(100 - i * 2, hour=0, minute=i * 5)) - - if s._in_position: - # Session ends at hour 2 - sig = s.on_candle(_candle(s._entry_price, hour=3, minute=0)) - if sig is not None: - assert sig.side == OrderSide.SELL - assert "Time exit" in sig.reason - - -def test_max_trades_per_day(): - s = _make_strategy(rsi_period=3, rsi_oversold=40, max_trades_per_day=1) - # Force one trade - for i in range(6): - s.on_candle(_candle(100 - i * 5, hour=0, minute=i * 5)) - # Exit - if s._in_position: - s.on_candle(_candle(200, hour=0, minute=35)) # TP exit - # Try to enter again — should be blocked - for i in range(6): - s.on_candle(_candle(100 - i * 5, hour=1, minute=i * 5)) - # After 1 trade, no more allowed - assert not s._in_position or s._trades_today >= 1 - - -def test_consecutive_losses_stop(): - s = _make_strategy(rsi_period=3, rsi_oversold=40, max_consecutive_losses=2) - # Simulate 2 losses - s._consecutive_losses = 2 - # Even with valid conditions, should not enter - for i in range(6): - sig = s.on_candle(_candle(100 - i * 5, hour=0, minute=i * 5)) - assert sig is None - - -def test_reset_clears_all(): - s = _make_strategy() - s.on_candle(_candle(100, hour=0)) - s._in_position = True - s._trades_today = 2 - s._consecutive_losses = 1 - s.reset() - assert not s._in_position - assert s._trades_today == 0 - assert len(s._closes) == 0 - - -def test_warmup_period(): - s = _make_strategy(rsi_period=14) - assert s.warmup_period == 15 - - -def test_ema_filter_blocks_below_ema(): - """Entry blocked when price is below EMA.""" - s = AsianSessionRsiStrategy() - s._rsi_period = 5 - s._rsi_oversold = 40 - s._quantity = Decimal("0.5") - s._take_profit_pct = 1.5 - s._stop_loss_pct = 0.7 - s._session_start_utc = 0 - s._session_end_utc = 2 - s._max_trades_per_day = 3 - s._max_consecutive_losses = 10 - s._ema_period = 5 - s._require_bullish_candle = False # Test EMA only - - # Feed rising prices to set EMA high, then sharp drop - for i in range(10): - s.on_candle(_candle(200 + i * 5, hour=0, minute=i * 5)) - # Now feed low price -- below EMA, RSI should be low - signals = [] - for i in range(5): - sig = s.on_candle(_candle(100 - i * 5, hour=0, minute=(15 + i) * 5 % 60)) - if sig is not None: - signals.append(sig) - # Should have no BUY signals because price is way below EMA - buy_sigs = [s for s in signals if s.side == OrderSide.BUY] - assert len(buy_sigs) == 0 diff --git a/services/strategy-engine/tests/test_base_filters.py b/services/strategy-engine/tests/test_base_filters.py index 3e55973..ae9ca05 100644 --- a/services/strategy-engine/tests/test_base_filters.py +++ b/services/strategy-engine/tests/test_base_filters.py @@ -43,7 +43,7 @@ def _candle(price=100.0, volume=10.0, high=None, low=None): h = high if high is not None else price + 5 lo = low if low is not None else price - 5 return Candle( - symbol="BTCUSDT", + symbol="AAPL", timeframe="1h", open_time=datetime(2025, 1, 1, tzinfo=timezone.utc), open=Decimal(str(price)), diff --git a/services/strategy-engine/tests/test_bollinger_strategy.py b/services/strategy-engine/tests/test_bollinger_strategy.py index 7761f2d..8261377 100644 --- a/services/strategy-engine/tests/test_bollinger_strategy.py +++ b/services/strategy-engine/tests/test_bollinger_strategy.py @@ -10,7 +10,7 @@ from strategies.bollinger_strategy import BollingerStrategy def make_candle(close: float) -> Candle: return Candle( - symbol="BTC/USDT", + symbol="AAPL", timeframe="1m", open_time=datetime(2024, 1, 1, tzinfo=timezone.utc), open=Decimal(str(close)), diff --git a/services/strategy-engine/tests/test_combined_strategy.py b/services/strategy-engine/tests/test_combined_strategy.py index 20a572e..8a4dc74 100644 --- a/services/strategy-engine/tests/test_combined_strategy.py +++ b/services/strategy-engine/tests/test_combined_strategy.py @@ -72,7 +72,7 @@ class NeutralStrategy(BaseStrategy): def _candle(price=100.0): return Candle( - symbol="BTCUSDT", + symbol="AAPL", timeframe="1m", open_time=datetime(2025, 1, 1, tzinfo=timezone.utc), open=Decimal(str(price)), diff --git a/services/strategy-engine/tests/test_ema_crossover_strategy.py b/services/strategy-engine/tests/test_ema_crossover_strategy.py index 67a20bf..7028eb0 100644 --- a/services/strategy-engine/tests/test_ema_crossover_strategy.py +++ b/services/strategy-engine/tests/test_ema_crossover_strategy.py @@ -10,7 +10,7 @@ from strategies.ema_crossover_strategy import EmaCrossoverStrategy def make_candle(close: float) -> Candle: return Candle( - symbol="BTC/USDT", + symbol="AAPL", timeframe="1m", open_time=datetime(2024, 1, 1, tzinfo=timezone.utc), open=Decimal(str(close)), diff --git a/services/strategy-engine/tests/test_engine.py b/services/strategy-engine/tests/test_engine.py index ac9a596..2623027 100644 --- a/services/strategy-engine/tests/test_engine.py +++ b/services/strategy-engine/tests/test_engine.py @@ -13,7 +13,7 @@ from strategy_engine.engine import StrategyEngine def make_candle_event() -> dict: candle = Candle( - symbol="BTC/USDT", + symbol="AAPL", timeframe="1m", open_time=datetime(2024, 1, 1, tzinfo=timezone.utc), open=Decimal("50000"), @@ -28,7 +28,7 @@ def make_candle_event() -> dict: def make_signal() -> Signal: return Signal( strategy="test", - symbol="BTC/USDT", + symbol="AAPL", side=OrderSide.BUY, price=Decimal("50050"), quantity=Decimal("0.01"), @@ -46,12 +46,12 @@ async def test_engine_dispatches_candle_to_strategies(): strategy.on_candle = MagicMock(return_value=None) engine = StrategyEngine(broker=broker, strategies=[strategy]) - await engine.process_once("candles.BTC_USDT", "0") + await engine.process_once("candles.AAPL", "0") strategy.on_candle.assert_called_once() candle_arg = strategy.on_candle.call_args[0][0] assert isinstance(candle_arg, Candle) - assert candle_arg.symbol == "BTC/USDT" + assert candle_arg.symbol == "AAPL" @pytest.mark.asyncio @@ -64,7 +64,7 @@ async def test_engine_publishes_signal_when_strategy_returns_one(): strategy.on_candle = MagicMock(return_value=make_signal()) engine = StrategyEngine(broker=broker, strategies=[strategy]) - await engine.process_once("candles.BTC_USDT", "0") + await engine.process_once("candles.AAPL", "0") broker.publish.assert_called_once() call_args = broker.publish.call_args diff --git a/services/strategy-engine/tests/test_grid_strategy.py b/services/strategy-engine/tests/test_grid_strategy.py index 9823f98..878b900 100644 --- a/services/strategy-engine/tests/test_grid_strategy.py +++ b/services/strategy-engine/tests/test_grid_strategy.py @@ -10,7 +10,7 @@ from strategies.grid_strategy import GridStrategy def make_candle(close: float) -> Candle: return Candle( - symbol="BTC/USDT", + symbol="AAPL", timeframe="1m", open_time=datetime(2024, 1, 1, tzinfo=timezone.utc), open=Decimal(str(close)), diff --git a/services/strategy-engine/tests/test_macd_strategy.py b/services/strategy-engine/tests/test_macd_strategy.py index 17dd2cf..556fd4c 100644 --- a/services/strategy-engine/tests/test_macd_strategy.py +++ b/services/strategy-engine/tests/test_macd_strategy.py @@ -10,7 +10,7 @@ from strategies.macd_strategy import MacdStrategy def _candle(price: float) -> Candle: return Candle( - symbol="BTC/USDT", + symbol="AAPL", timeframe="1m", open_time=datetime(2024, 1, 1, tzinfo=timezone.utc), open=Decimal(str(price)), diff --git a/services/strategy-engine/tests/test_multi_symbol.py b/services/strategy-engine/tests/test_multi_symbol.py index cb8088c..671a9d3 100644 --- a/services/strategy-engine/tests/test_multi_symbol.py +++ b/services/strategy-engine/tests/test_multi_symbol.py @@ -22,7 +22,7 @@ async def test_engine_processes_multiple_streams(): broker = AsyncMock() candle_btc = Candle( - symbol="BTCUSDT", + symbol="AAPL", timeframe="1m", open_time=datetime(2025, 1, 1, tzinfo=timezone.utc), open=Decimal("50000"), @@ -32,7 +32,7 @@ async def test_engine_processes_multiple_streams(): volume=Decimal("10"), ) candle_eth = Candle( - symbol="ETHUSDT", + symbol="MSFT", timeframe="1m", open_time=datetime(2025, 1, 1, tzinfo=timezone.utc), open=Decimal("3000"), @@ -45,16 +45,16 @@ async def test_engine_processes_multiple_streams(): btc_events = [CandleEvent(data=candle_btc).to_dict()] eth_events = [CandleEvent(data=candle_eth).to_dict()] - # First call returns BTC event, second ETH, then empty - call_count = {"btc": 0, "eth": 0} + # First call returns AAPL event, second MSFT, then empty + call_count = {"aapl": 0, "msft": 0} async def mock_read(stream, **kwargs): - if "BTC" in stream: - call_count["btc"] += 1 - return btc_events if call_count["btc"] == 1 else [] - elif "ETH" in stream: - call_count["eth"] += 1 - return eth_events if call_count["eth"] == 1 else [] + if "AAPL" in stream: + call_count["aapl"] += 1 + return btc_events if call_count["aapl"] == 1 else [] + elif "MSFT" in stream: + call_count["msft"] += 1 + return eth_events if call_count["msft"] == 1 else [] return [] broker.read = AsyncMock(side_effect=mock_read) @@ -65,8 +65,8 @@ async def test_engine_processes_multiple_streams(): engine = StrategyEngine(broker=broker, strategies=[strategy]) # Process both streams - await engine.process_once("candles.BTCUSDT", "$") - await engine.process_once("candles.ETHUSDT", "$") + await engine.process_once("candles.AAPL", "$") + await engine.process_once("candles.MSFT", "$") # Strategy should have been called with both candles assert strategy.on_candle.call_count == 2 diff --git a/services/strategy-engine/tests/test_rsi_strategy.py b/services/strategy-engine/tests/test_rsi_strategy.py index b2aecc9..6d31fd5 100644 --- a/services/strategy-engine/tests/test_rsi_strategy.py +++ b/services/strategy-engine/tests/test_rsi_strategy.py @@ -10,7 +10,7 @@ from strategies.rsi_strategy import RsiStrategy def make_candle(close: float, idx: int = 0) -> Candle: return Candle( - symbol="BTC/USDT", + symbol="AAPL", timeframe="1m", open_time=datetime(2024, 1, 1, tzinfo=timezone.utc), open=Decimal(str(close)), diff --git a/services/strategy-engine/tests/test_sentiment_wiring.py b/services/strategy-engine/tests/test_sentiment_wiring.py deleted file mode 100644 index e0052cb..0000000 --- a/services/strategy-engine/tests/test_sentiment_wiring.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Test sentiment is wired into strategy engine.""" - -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) -sys.path.insert(0, str(Path(__file__).resolve().parents[1])) - -from shared.sentiment import SentimentData -from strategies.asian_session_rsi import AsianSessionRsiStrategy - - -def test_strategy_accepts_sentiment(): - s = AsianSessionRsiStrategy() - data = SentimentData(fear_greed_value=20, fear_greed_label="Extreme Fear") - s.update_sentiment(data) - assert s._sentiment is not None - assert s._sentiment.fear_greed_value == 20 - - -def test_strategy_blocks_on_extreme_greed(): - s = AsianSessionRsiStrategy() - data = SentimentData(fear_greed_value=85) - s.update_sentiment(data) - assert not s._check_sentiment() - - -def test_strategy_allows_on_fear(): - s = AsianSessionRsiStrategy() - data = SentimentData(fear_greed_value=20) - s.update_sentiment(data) - assert s._check_sentiment() diff --git a/services/strategy-engine/tests/test_stock_selector.py b/services/strategy-engine/tests/test_stock_selector.py new file mode 100644 index 0000000..ff9d09c --- /dev/null +++ b/services/strategy-engine/tests/test_stock_selector.py @@ -0,0 +1,80 @@ +"""Tests for stock selector engine.""" + +from unittest.mock import AsyncMock, MagicMock +from datetime import datetime, timezone + + +from strategy_engine.stock_selector import ( + SentimentCandidateSource, + StockSelector, + _parse_llm_selections, +) + + +async def test_sentiment_candidate_source(): + mock_db = MagicMock() + mock_db.get_top_symbol_scores = AsyncMock( + return_value=[ + {"symbol": "AAPL", "composite": 0.8, "news_count": 5}, + {"symbol": "NVDA", "composite": 0.6, "news_count": 3}, + ] + ) + + source = SentimentCandidateSource(mock_db) + candidates = await source.get_candidates() + + assert len(candidates) == 2 + assert candidates[0].symbol == "AAPL" + assert candidates[0].source == "sentiment" + + +def test_parse_llm_selections_valid(): + llm_response = """ + [ + {"symbol": "NVDA", "side": "BUY", "conviction": 0.85, "reason": "AI demand", "key_news": ["NVDA beats earnings"]}, + {"symbol": "XOM", "side": "BUY", "conviction": 0.72, "reason": "Oil surge", "key_news": ["Oil prices up"]} + ] + """ + selections = _parse_llm_selections(llm_response) + assert len(selections) == 2 + assert selections[0].symbol == "NVDA" + assert selections[0].conviction == 0.85 + + +def test_parse_llm_selections_invalid(): + selections = _parse_llm_selections("not json") + assert selections == [] + + +def test_parse_llm_selections_with_markdown(): + llm_response = """ + Here are my picks: + ```json + [ + {"symbol": "TSLA", "side": "BUY", "conviction": 0.7, "reason": "Momentum", "key_news": ["Tesla rally"]} + ] + ``` + """ + selections = _parse_llm_selections(llm_response) + assert len(selections) == 1 + assert selections[0].symbol == "TSLA" + + +async def test_selector_blocks_on_risk_off(): + mock_db = MagicMock() + mock_db.get_latest_market_sentiment = AsyncMock( + return_value={ + "fear_greed": 15, + "fear_greed_label": "Extreme Fear", + "vix": 35.0, + "fed_stance": "neutral", + "market_regime": "risk_off", + "updated_at": datetime.now(timezone.utc), + } + ) + + selector = StockSelector( + db=mock_db, broker=MagicMock(), alpaca=MagicMock(), anthropic_api_key="test" + ) + result = await selector.select() + assert result == [] diff --git a/services/strategy-engine/tests/test_volume_profile_strategy.py b/services/strategy-engine/tests/test_volume_profile_strategy.py index f40261c..65ee2e8 100644 --- a/services/strategy-engine/tests/test_volume_profile_strategy.py +++ b/services/strategy-engine/tests/test_volume_profile_strategy.py @@ -10,7 +10,7 @@ from strategies.volume_profile_strategy import VolumeProfileStrategy def make_candle(close: float, volume: float = 1.0) -> Candle: return Candle( - symbol="BTC/USDT", + symbol="AAPL", timeframe="1m", open_time=datetime(2024, 1, 1, tzinfo=timezone.utc), open=Decimal(str(close)), diff --git a/services/strategy-engine/tests/test_vwap_strategy.py b/services/strategy-engine/tests/test_vwap_strategy.py index 0312972..2c34b01 100644 --- a/services/strategy-engine/tests/test_vwap_strategy.py +++ b/services/strategy-engine/tests/test_vwap_strategy.py @@ -22,7 +22,7 @@ def make_candle( if open_time is None: open_time = datetime(2024, 1, 1, tzinfo=timezone.utc) return Candle( - symbol="BTC/USDT", + symbol="AAPL", timeframe="1m", open_time=open_time, open=Decimal(str(close)), diff --git a/shared/alembic/versions/002_news_sentiment_tables.py b/shared/alembic/versions/002_news_sentiment_tables.py new file mode 100644 index 0000000..402ff87 --- /dev/null +++ b/shared/alembic/versions/002_news_sentiment_tables.py @@ -0,0 +1,84 @@ +"""Add news, sentiment, and stock selection tables + +Revision ID: 002 +Revises: 001 +Create Date: 2026-04-02 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "002" +down_revision: Union[str, None] = "001" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "news_items", + sa.Column("id", sa.Text, primary_key=True), + sa.Column("source", sa.Text, nullable=False), + sa.Column("headline", sa.Text, nullable=False), + sa.Column("summary", sa.Text), + sa.Column("url", sa.Text), + sa.Column("published_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("symbols", sa.Text), + sa.Column("sentiment", sa.Float, nullable=False), + sa.Column("category", sa.Text, nullable=False), + sa.Column("raw_data", sa.Text), + sa.Column( + "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now() + ), + ) + op.create_index("idx_news_items_published", "news_items", ["published_at"]) + op.create_index("idx_news_items_source", "news_items", ["source"]) + + op.create_table( + "symbol_scores", + sa.Column("id", sa.Text, primary_key=True), + sa.Column("symbol", sa.Text, nullable=False, unique=True), + sa.Column("news_score", sa.Float, nullable=False, server_default="0"), + sa.Column("news_count", sa.Integer, nullable=False, server_default="0"), + sa.Column("social_score", sa.Float, nullable=False, server_default="0"), + sa.Column("policy_score", sa.Float, nullable=False, server_default="0"), + sa.Column("filing_score", sa.Float, nullable=False, server_default="0"), + sa.Column("composite", sa.Float, nullable=False, server_default="0"), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + ) + + op.create_table( + "market_sentiment", + sa.Column("id", sa.Text, primary_key=True), + sa.Column("fear_greed", sa.Integer, nullable=False), + sa.Column("fear_greed_label", sa.Text, nullable=False), + sa.Column("vix", sa.Float), + sa.Column("fed_stance", sa.Text, nullable=False, server_default="neutral"), + sa.Column("market_regime", sa.Text, nullable=False, server_default="neutral"), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + ) + + op.create_table( + "stock_selections", + sa.Column("id", sa.Text, primary_key=True), + sa.Column("trade_date", sa.Date, nullable=False), + sa.Column("symbol", sa.Text, nullable=False), + sa.Column("side", sa.Text, nullable=False), + sa.Column("conviction", sa.Float, nullable=False), + sa.Column("reason", sa.Text, nullable=False), + sa.Column("key_news", sa.Text), + sa.Column("sentiment_snapshot", sa.Text), + sa.Column( + "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now() + ), + ) + op.create_index("idx_stock_selections_date", "stock_selections", ["trade_date"]) + + +def downgrade() -> None: + op.drop_table("stock_selections") + op.drop_table("market_sentiment") + op.drop_table("symbol_scores") + op.drop_table("news_items") diff --git a/shared/src/shared/config.py b/shared/src/shared/config.py index 4e8e7f1..b6ccebd 100644 --- a/shared/src/shared/config.py +++ b/shared/src/shared/config.py @@ -32,7 +32,15 @@ class Settings(BaseSettings): telegram_enabled: bool = False log_format: str = "json" health_port: int = 8080 - circuit_breaker_threshold: int = 5 - circuit_breaker_timeout: int = 60 metrics_auth_token: str = "" # If set, /health and /metrics require Bearer token + # News collector + finnhub_api_key: str = "" + news_poll_interval: int = 300 + sentiment_aggregate_interval: int = 900 + # Stock selector + selector_final_time: str = "15:30" + selector_max_picks: int = 3 + # LLM + anthropic_api_key: str = "" + anthropic_model: str = "claude-sonnet-4-20250514" model_config = {"env_file": ".env", "env_file_encoding": "utf-8", "extra": "ignore"} diff --git a/shared/src/shared/db.py b/shared/src/shared/db.py index 901e293..9cc8686 100644 --- a/shared/src/shared/db.py +++ b/shared/src/shared/db.py @@ -1,15 +1,28 @@ """Database layer using SQLAlchemy 2.0 async ORM for the trading platform.""" +import json +import uuid from contextlib import asynccontextmanager -from datetime import datetime, timedelta, timezone +from datetime import datetime, date, timedelta, timezone from decimal import Decimal from typing import Optional from sqlalchemy import select, update from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker -from shared.models import Candle, Signal, Order, OrderStatus -from shared.sa_models import Base, CandleRow, SignalRow, OrderRow, PortfolioSnapshotRow +from shared.models import Candle, Signal, Order, OrderStatus, NewsItem +from shared.sentiment_models import SymbolScore, MarketSentiment +from shared.sa_models import ( + Base, + CandleRow, + SignalRow, + OrderRow, + PortfolioSnapshotRow, + NewsItemRow, + SymbolScoreRow, + MarketSentimentRow, + StockSelectionRow, +) class Database: @@ -195,3 +208,229 @@ class Database: } for r in rows ] + + async def insert_news_item(self, item: NewsItem) -> None: + """Insert a NewsItem row, JSON-encoding symbols and raw_data.""" + row = NewsItemRow( + id=item.id, + source=item.source, + headline=item.headline, + summary=item.summary, + url=item.url, + published_at=item.published_at, + symbols=json.dumps(item.symbols), + sentiment=item.sentiment, + category=item.category.value, + raw_data=json.dumps(item.raw_data), + created_at=item.created_at, + ) + async with self._session_factory() as session: + try: + session.add(row) + await session.commit() + except Exception: + await session.rollback() + raise + + async def get_recent_news(self, hours: int = 24) -> list[dict]: + """Retrieve news items published in the last N hours.""" + since = datetime.now(timezone.utc) - timedelta(hours=hours) + stmt = ( + select(NewsItemRow) + .where(NewsItemRow.published_at >= since) + .order_by(NewsItemRow.published_at.desc()) + ) + async with self._session_factory() as session: + try: + result = await session.execute(stmt) + rows = result.scalars().all() + except Exception: + await session.rollback() + raise + return [ + { + "id": r.id, + "source": r.source, + "headline": r.headline, + "summary": r.summary, + "url": r.url, + "published_at": r.published_at, + "symbols": json.loads(r.symbols) if r.symbols else [], + "sentiment": r.sentiment, + "category": r.category, + "raw_data": json.loads(r.raw_data) if r.raw_data else {}, + "created_at": r.created_at, + } + for r in rows + ] + + async def upsert_symbol_score(self, score: SymbolScore) -> None: + """Insert or update a SymbolScore row, keyed by symbol.""" + async with self._session_factory() as session: + try: + stmt = select(SymbolScoreRow).where(SymbolScoreRow.symbol == score.symbol) + result = await session.execute(stmt) + existing = result.scalar_one_or_none() + if existing is not None: + existing.news_score = score.news_score + existing.news_count = score.news_count + existing.social_score = score.social_score + existing.policy_score = score.policy_score + existing.filing_score = score.filing_score + existing.composite = score.composite + existing.updated_at = score.updated_at + else: + row = SymbolScoreRow( + id=str(uuid.uuid4()), + symbol=score.symbol, + news_score=score.news_score, + news_count=score.news_count, + social_score=score.social_score, + policy_score=score.policy_score, + filing_score=score.filing_score, + composite=score.composite, + updated_at=score.updated_at, + ) + session.add(row) + await session.commit() + except Exception: + await session.rollback() + raise + + async def get_top_symbol_scores(self, limit: int = 20) -> list[dict]: + """Retrieve top symbol scores ordered by composite descending.""" + stmt = select(SymbolScoreRow).order_by(SymbolScoreRow.composite.desc()).limit(limit) + async with self._session_factory() as session: + try: + result = await session.execute(stmt) + rows = result.scalars().all() + except Exception: + await session.rollback() + raise + return [ + { + "id": r.id, + "symbol": r.symbol, + "news_score": r.news_score, + "news_count": r.news_count, + "social_score": r.social_score, + "policy_score": r.policy_score, + "filing_score": r.filing_score, + "composite": r.composite, + "updated_at": r.updated_at, + } + for r in rows + ] + + async def upsert_market_sentiment(self, ms: MarketSentiment) -> None: + """Insert or update the single 'latest' market sentiment row.""" + async with self._session_factory() as session: + try: + stmt = select(MarketSentimentRow).where(MarketSentimentRow.id == "latest") + result = await session.execute(stmt) + existing = result.scalar_one_or_none() + if existing is not None: + existing.fear_greed = ms.fear_greed + existing.fear_greed_label = ms.fear_greed_label + existing.vix = ms.vix + existing.fed_stance = ms.fed_stance + existing.market_regime = ms.market_regime + existing.updated_at = ms.updated_at + else: + row = MarketSentimentRow( + id="latest", + fear_greed=ms.fear_greed, + fear_greed_label=ms.fear_greed_label, + vix=ms.vix, + fed_stance=ms.fed_stance, + market_regime=ms.market_regime, + updated_at=ms.updated_at, + ) + session.add(row) + await session.commit() + except Exception: + await session.rollback() + raise + + async def get_latest_market_sentiment(self) -> Optional[dict]: + """Retrieve the 'latest' market sentiment row, or None if not found.""" + stmt = select(MarketSentimentRow).where(MarketSentimentRow.id == "latest") + async with self._session_factory() as session: + try: + result = await session.execute(stmt) + row = result.scalar_one_or_none() + except Exception: + await session.rollback() + raise + if row is None: + return None + return { + "id": row.id, + "fear_greed": row.fear_greed, + "fear_greed_label": row.fear_greed_label, + "vix": row.vix, + "fed_stance": row.fed_stance, + "market_regime": row.market_regime, + "updated_at": row.updated_at, + } + + async def insert_stock_selection( + self, + trade_date: date, + symbol: str, + side: str, + conviction: float, + reason: str, + key_news: list, + sentiment_snapshot: dict, + ) -> None: + """Insert a stock selection row with JSON-encoded lists/dicts.""" + row = StockSelectionRow( + id=str(uuid.uuid4()), + trade_date=trade_date, + symbol=symbol, + side=side, + conviction=conviction, + reason=reason, + key_news=json.dumps(key_news), + sentiment_snapshot=json.dumps(sentiment_snapshot), + created_at=datetime.now(timezone.utc), + ) + async with self._session_factory() as session: + try: + session.add(row) + await session.commit() + except Exception: + await session.rollback() + raise + + async def get_stock_selections(self, trade_date: date) -> list[dict]: + """Retrieve stock selections for a given trade date.""" + stmt = ( + select(StockSelectionRow) + .where(StockSelectionRow.trade_date == trade_date) + .order_by(StockSelectionRow.conviction.desc()) + ) + async with self._session_factory() as session: + try: + result = await session.execute(stmt) + rows = result.scalars().all() + except Exception: + await session.rollback() + raise + return [ + { + "id": r.id, + "trade_date": r.trade_date, + "symbol": r.symbol, + "side": r.side, + "conviction": r.conviction, + "reason": r.reason, + "key_news": json.loads(r.key_news) if r.key_news else [], + "sentiment_snapshot": json.loads(r.sentiment_snapshot) + if r.sentiment_snapshot + else {}, + "created_at": r.created_at, + } + for r in rows + ] diff --git a/shared/src/shared/events.py b/shared/src/shared/events.py index 72f8865..63f93a2 100644 --- a/shared/src/shared/events.py +++ b/shared/src/shared/events.py @@ -5,13 +5,14 @@ from typing import Any from pydantic import BaseModel -from shared.models import Candle, Signal, Order +from shared.models import Candle, Signal, Order, NewsItem class EventType(str, Enum): CANDLE = "CANDLE" SIGNAL = "SIGNAL" ORDER = "ORDER" + NEWS = "NEWS" class CandleEvent(BaseModel): @@ -59,10 +60,26 @@ class OrderEvent(BaseModel): return cls(type=raw["type"], data=Order(**raw["data"])) +class NewsEvent(BaseModel): + type: EventType = EventType.NEWS + data: NewsItem + + def to_dict(self) -> dict: + return { + "type": self.type, + "data": self.data.model_dump(mode="json"), + } + + @classmethod + def from_raw(cls, raw: dict) -> "NewsEvent": + return cls(type=raw["type"], data=NewsItem(**raw["data"])) + + _EVENT_TYPE_MAP = { EventType.CANDLE: CandleEvent, EventType.SIGNAL: SignalEvent, EventType.ORDER: OrderEvent, + EventType.NEWS: NewsEvent, } diff --git a/shared/src/shared/models.py b/shared/src/shared/models.py index 70820b5..a436c03 100644 --- a/shared/src/shared/models.py +++ b/shared/src/shared/models.py @@ -74,3 +74,26 @@ class Position(BaseModel): @property def unrealized_pnl(self) -> Decimal: return self.quantity * (self.current_price - self.avg_entry_price) + + +class NewsCategory(str, Enum): + POLICY = "policy" + EARNINGS = "earnings" + MACRO = "macro" + SOCIAL = "social" + FILING = "filing" + FED = "fed" + + +class NewsItem(BaseModel): + id: str = Field(default_factory=lambda: str(uuid.uuid4())) + source: str + headline: str + summary: Optional[str] = None + url: Optional[str] = None + published_at: datetime + symbols: list[str] = [] + sentiment: float + category: NewsCategory + raw_data: dict = {} + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) diff --git a/shared/src/shared/notifier.py b/shared/src/shared/notifier.py index f03919c..3d7b6cf 100644 --- a/shared/src/shared/notifier.py +++ b/shared/src/shared/notifier.py @@ -8,6 +8,7 @@ from typing import Optional, Sequence import aiohttp from shared.models import Signal, Order, Position +from shared.sentiment_models import SelectedStock, MarketSentiment logger = logging.getLogger(__name__) @@ -123,6 +124,34 @@ class TelegramNotifier: lines.append(" No open positions") await self.send("\n".join(lines)) + async def send_stock_selection( + self, + selections: list[SelectedStock], + market: MarketSentiment | None = None, + ) -> None: + """Format and send stock selection notification.""" + lines = [f"<b>📊 Stock Selection ({len(selections)} picks)</b>", ""] + + side_emoji = {"BUY": "🟢", "SELL": "🔴"} + + for i, s in enumerate(selections, 1): + emoji = side_emoji.get(s.side.value, "⚪") + lines.append( + f"{i}. <b>{s.symbol}</b> {emoji} {s.side.value} (conviction: {s.conviction:.0%})" + ) + lines.append(f" {s.reason}") + if s.key_news: + lines.append(f" News: {s.key_news[0]}") + lines.append("") + + if market: + lines.append( + f"Market: F&G {market.fear_greed} ({market.fear_greed_label})" + + (f" | VIX {market.vix:.1f}" if market.vix else "") + ) + + await self.send("\n".join(lines)) + async def close(self) -> None: """Close the underlying aiohttp session.""" if self._session is not None: diff --git a/shared/src/shared/resilience.py b/shared/src/shared/resilience.py index e43fd21..8d8678a 100644 --- a/shared/src/shared/resilience.py +++ b/shared/src/shared/resilience.py @@ -1,105 +1 @@ -"""Retry with exponential backoff and circuit breaker utilities.""" - -from __future__ import annotations - -import asyncio -import enum -import functools -import logging -import random -import time -from typing import Any, Callable - -logger = logging.getLogger(__name__) - - -# --------------------------------------------------------------------------- -# retry_with_backoff -# --------------------------------------------------------------------------- - - -def retry_with_backoff( - max_retries: int = 3, - base_delay: float = 1.0, - max_delay: float = 60.0, -) -> Callable: - """Decorator that retries an async function with exponential backoff + jitter.""" - - def decorator(func: Callable) -> Callable: - @functools.wraps(func) - async def wrapper(*args: Any, **kwargs: Any) -> Any: - last_exc: BaseException | None = None - for attempt in range(max_retries + 1): - try: - return await func(*args, **kwargs) - except Exception as exc: - last_exc = exc - if attempt < max_retries: - delay = min(base_delay * (2**attempt), max_delay) - jitter = delay * random.uniform(0, 0.5) - total_delay = delay + jitter - logger.warning( - "Retry %d/%d for %s after error: %s (delay=%.3fs)", - attempt + 1, - max_retries, - func.__name__, - exc, - total_delay, - ) - await asyncio.sleep(total_delay) - raise last_exc # type: ignore[misc] - - return wrapper - - return decorator - - -# --------------------------------------------------------------------------- -# CircuitBreaker -# --------------------------------------------------------------------------- - - -class CircuitState(enum.Enum): - CLOSED = "closed" - OPEN = "open" - HALF_OPEN = "half_open" - - -class CircuitBreaker: - """Simple circuit breaker implementation.""" - - def __init__( - self, - failure_threshold: int = 5, - recovery_timeout: float = 60.0, - ) -> None: - self._failure_threshold = failure_threshold - self._recovery_timeout = recovery_timeout - self._failure_count: int = 0 - self._state = CircuitState.CLOSED - self._opened_at: float = 0.0 - - @property - def state(self) -> CircuitState: - return self._state - - def allow_request(self) -> bool: - if self._state == CircuitState.CLOSED: - return True - if self._state == CircuitState.OPEN: - if time.monotonic() - self._opened_at >= self._recovery_timeout: - self._state = CircuitState.HALF_OPEN - return True - return False - # HALF_OPEN - return True - - def record_success(self) -> None: - self._failure_count = 0 - self._state = CircuitState.CLOSED - - def record_failure(self) -> None: - self._failure_count += 1 - if self._failure_count >= self._failure_threshold: - self._state = CircuitState.OPEN - self._opened_at = time.monotonic() +"""Resilience utilities for the trading platform.""" diff --git a/shared/src/shared/sa_models.py b/shared/src/shared/sa_models.py index 8386ba8..dc87ef5 100644 --- a/shared/src/shared/sa_models.py +++ b/shared/src/shared/sa_models.py @@ -3,6 +3,7 @@ from datetime import datetime from decimal import Decimal +import sqlalchemy as sa from sqlalchemy import DateTime, ForeignKey, Numeric, Text from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column @@ -52,19 +53,6 @@ class OrderRow(Base): filled_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) -class TradeRow(Base): - __tablename__ = "trades" - - id: Mapped[str] = mapped_column(Text, primary_key=True) - order_id: Mapped[str | None] = mapped_column(Text, ForeignKey("orders.id")) - symbol: Mapped[str] = mapped_column(Text, nullable=False) - side: Mapped[str] = mapped_column(Text, nullable=False) - price: Mapped[Decimal] = mapped_column(Numeric, nullable=False) - quantity: Mapped[Decimal] = mapped_column(Numeric, nullable=False) - fee: Mapped[Decimal] = mapped_column(Numeric, nullable=False, server_default="0") - traded_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) - - class PositionRow(Base): __tablename__ = "positions" @@ -83,3 +71,63 @@ class PortfolioSnapshotRow(Base): realized_pnl: Mapped[Decimal] = mapped_column(Numeric, nullable=False) unrealized_pnl: Mapped[Decimal] = mapped_column(Numeric, nullable=False) snapshot_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class NewsItemRow(Base): + __tablename__ = "news_items" + + id: Mapped[str] = mapped_column(Text, primary_key=True) + source: Mapped[str] = mapped_column(Text, nullable=False) + headline: Mapped[str] = mapped_column(Text, nullable=False) + summary: Mapped[str | None] = mapped_column(Text) + url: Mapped[str | None] = mapped_column(Text) + published_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + symbols: Mapped[str | None] = mapped_column(Text) # JSON-encoded list + sentiment: Mapped[float] = mapped_column(sa.Float, nullable=False) + category: Mapped[str] = mapped_column(Text, nullable=False) + raw_data: Mapped[str | None] = mapped_column(Text) # JSON string + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=sa.func.now() + ) + + +class SymbolScoreRow(Base): + __tablename__ = "symbol_scores" + + id: Mapped[str] = mapped_column(Text, primary_key=True) + symbol: Mapped[str] = mapped_column(Text, nullable=False, unique=True) + news_score: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default="0") + news_count: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default="0") + social_score: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default="0") + policy_score: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default="0") + filing_score: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default="0") + composite: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default="0") + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class MarketSentimentRow(Base): + __tablename__ = "market_sentiment" + + id: Mapped[str] = mapped_column(Text, primary_key=True) + fear_greed: Mapped[int] = mapped_column(sa.Integer, nullable=False) + fear_greed_label: Mapped[str] = mapped_column(Text, nullable=False) + vix: Mapped[float | None] = mapped_column(sa.Float) + fed_stance: Mapped[str] = mapped_column(Text, nullable=False, server_default="neutral") + market_regime: Mapped[str] = mapped_column(Text, nullable=False, server_default="neutral") + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class StockSelectionRow(Base): + __tablename__ = "stock_selections" + + id: Mapped[str] = mapped_column(Text, primary_key=True) + trade_date: Mapped[datetime] = mapped_column(sa.Date, nullable=False) + symbol: Mapped[str] = mapped_column(Text, nullable=False) + side: Mapped[str] = mapped_column(Text, nullable=False) + conviction: Mapped[float] = mapped_column(sa.Float, nullable=False) + reason: Mapped[str] = mapped_column(Text, nullable=False) + key_news: Mapped[str | None] = mapped_column(Text) # JSON string + sentiment_snapshot: Mapped[str | None] = mapped_column(Text) # JSON string + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=sa.func.now() + ) diff --git a/shared/src/shared/sentiment.py b/shared/src/shared/sentiment.py index bc62efe..5b4b0da 100644 --- a/shared/src/shared/sentiment.py +++ b/shared/src/shared/sentiment.py @@ -1,219 +1,105 @@ -"""Market sentiment data from free APIs. - -Supports: -- Fear & Greed Index (alternative.me) — no API key needed -- CryptoPanic news sentiment — free API key from cryptopanic.com -- CryptoQuant exchange netflow — free API key from cryptoquant.com - -All providers are optional. If API key is missing, the provider is disabled. -""" - -import logging -from dataclasses import dataclass, field -from datetime import datetime, timezone - -import aiohttp - -logger = logging.getLogger(__name__) - - -@dataclass -class SentimentData: - """Aggregated sentiment snapshot.""" - - fear_greed_value: int | None = None # 0-100 - fear_greed_label: str | None = ( - None # "Extreme Fear", "Fear", "Neutral", "Greed", "Extreme Greed" - ) - news_sentiment: float | None = None # -1.0 (bearish) to 1.0 (bullish) - news_count: int = 0 - exchange_netflow: float | None = ( - None # Positive = inflow (bearish), Negative = outflow (bullish) - ) - timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) - - @property - def should_buy(self) -> bool: - """Simple aggregated buy signal from sentiment.""" - score = 0 - checks = 0 - - if self.fear_greed_value is not None: - checks += 1 - if self.fear_greed_value < 30: - score += 1 # Fear = buy opportunity - elif self.fear_greed_value > 70: - score -= 1 # Greed = avoid buying - - if self.news_sentiment is not None: - checks += 1 - if self.news_sentiment > 0.1: - score += 1 # Positive news - elif self.news_sentiment < -0.3: - score -= 1 # Very negative news = avoid - - if self.exchange_netflow is not None: - checks += 1 - if self.exchange_netflow < 0: - score += 1 # Outflow = bullish (coins leaving exchanges) - elif self.exchange_netflow > 0: - score -= 1 # Inflow = bearish (coins entering exchanges to sell) - - if checks == 0: - return True # No data, allow by default - - return score >= 0 # Net neutral or positive = allow - - @property - def should_block(self) -> bool: - """Strong bearish signal — block all buying.""" - # Block on extreme greed - if self.fear_greed_value is not None and self.fear_greed_value > 80: - return True - # Block on very negative news - if self.news_sentiment is not None and self.news_sentiment < -0.5: - return True - return False - - -class SentimentProvider: - """Fetches sentiment data from multiple free APIs.""" - - def __init__( - self, - cryptopanic_api_key: str = "", - cryptoquant_api_key: str = "", - ) -> None: - self._cryptopanic_key = cryptopanic_api_key - self._cryptoquant_key = cryptoquant_api_key - self._session: aiohttp.ClientSession | None = None - self._cached: SentimentData | None = None - self._cache_ttl: int = 300 # 5 minutes cache - - async def _ensure_session(self) -> aiohttp.ClientSession: - if self._session is None or self._session.closed: - self._session = aiohttp.ClientSession() - return self._session - - async def fetch_fear_greed(self) -> tuple[int | None, str | None]: - """Fetch Fear & Greed Index from alternative.me (no key needed).""" - try: - session = await self._ensure_session() - url = "https://api.alternative.me/fng/?limit=1" - async with session.get(url, timeout=aiohttp.ClientTimeout(total=10)) as resp: - if resp.status != 200: - logger.warning("Fear & Greed API returned %d", resp.status) - return None, None - data = await resp.json() - entry = data.get("data", [{}])[0] - value = int(entry.get("value", 0)) - label = entry.get("value_classification", "") - return value, label - except Exception as exc: - logger.warning("Fear & Greed fetch failed: %s", exc) - return None, None - - async def fetch_news_sentiment(self, currency: str = "SOL") -> tuple[float | None, int]: - """Fetch news sentiment from CryptoPanic. - - Returns (sentiment_score, news_count). - Sentiment score: -1.0 (all bearish) to 1.0 (all bullish). - """ - if not self._cryptopanic_key: - return None, 0 - - try: - session = await self._ensure_session() - url = ( - f"https://cryptopanic.com/api/v1/posts/" - f"?auth_token={self._cryptopanic_key}" - f"¤cies={currency}" - f"&kind=news" - f"&filter=hot" - f"&limit=10" - ) - async with session.get(url, timeout=aiohttp.ClientTimeout(total=10)) as resp: - if resp.status != 200: - logger.warning("CryptoPanic API returned %d", resp.status) - return None, 0 - data = await resp.json() - results = data.get("results", []) - if not results: - return None, 0 - - # CryptoPanic provides votes: positive, negative, important, etc. - total_positive = 0 - total_negative = 0 - count = 0 - for post in results: - votes = post.get("votes", {}) - pos = votes.get("positive", 0) - neg = votes.get("negative", 0) - total_positive += pos - total_negative += neg - count += 1 - - total_votes = total_positive + total_negative - if total_votes == 0: - return 0.0, count - - sentiment = (total_positive - total_negative) / total_votes - return sentiment, count - except Exception as exc: - logger.warning("CryptoPanic fetch failed: %s", exc) - return None, 0 - - async def fetch_exchange_netflow(self, symbol: str = "sol") -> float | None: - """Fetch exchange netflow from CryptoQuant. - - Returns netflow value. Positive = inflow (bearish), Negative = outflow (bullish). - """ - if not self._cryptoquant_key: - return None +"""Market sentiment aggregation.""" - try: - session = await self._ensure_session() - url = ( - f"https://api.cryptoquant.com/v1/{symbol}/exchange-flows/netflow?window=day&limit=1" - ) - headers = {"Authorization": f"Bearer {self._cryptoquant_key}"} - async with session.get( - url, headers=headers, timeout=aiohttp.ClientTimeout(total=10) - ) as resp: - if resp.status != 200: - logger.warning("CryptoQuant API returned %d", resp.status) - return None - data = await resp.json() - result = data.get("result", {}).get("data", []) - if result: - return float(result[0].get("netflow", 0)) - return None - except Exception as exc: - logger.warning("CryptoQuant fetch failed: %s", exc) - return None - - async def get_sentiment(self, currency: str = "SOL") -> SentimentData: - """Fetch all sentiment data and return aggregated result.""" - fg_value, fg_label = await self.fetch_fear_greed() - news_score, news_count = await self.fetch_news_sentiment(currency) - netflow = await self.fetch_exchange_netflow(currency.lower()) - - sentiment = SentimentData( - fear_greed_value=fg_value, - fear_greed_label=fg_label, - news_sentiment=news_score, - news_count=news_count, - exchange_netflow=netflow, - ) +from datetime import datetime + +from shared.sentiment_models import SymbolScore + + +def _safe_avg(values: list[float]) -> float: + if not values: + return 0.0 + return sum(values) / len(values) - self._cached = sentiment - return sentiment - @property - def cached(self) -> SentimentData | None: - """Return last fetched sentiment data.""" - return self._cached +class SentimentAggregator: + """Aggregates per-news sentiment into per-symbol scores.""" - async def close(self) -> None: - if self._session and not self._session.closed: - await self._session.close() + WEIGHTS = {"news": 0.3, "social": 0.2, "policy": 0.3, "filing": 0.2} + + CATEGORY_MAP = { + "earnings": "news", + "macro": "news", + "social": "social", + "policy": "policy", + "filing": "filing", + "fed": "policy", + } + + def _freshness_decay(self, published_at: datetime, now: datetime) -> float: + age = now - published_at + hours = age.total_seconds() / 3600 + if hours < 1: + return 1.0 + if hours < 6: + return 0.7 + if hours < 24: + return 0.3 + return 0.0 + + def _compute_composite( + self, + news_score: float, + social_score: float, + policy_score: float, + filing_score: float, + ) -> float: + return ( + news_score * self.WEIGHTS["news"] + + social_score * self.WEIGHTS["social"] + + policy_score * self.WEIGHTS["policy"] + + filing_score * self.WEIGHTS["filing"] + ) + + def aggregate(self, news_items: list[dict], now: datetime) -> dict[str, SymbolScore]: + """Aggregate news items into per-symbol scores. + + Each dict needs: symbols, sentiment, category, published_at. + """ + symbol_data: dict[str, dict] = {} + + for item in news_items: + decay = self._freshness_decay(item["published_at"], now) + if decay == 0.0: + continue + category = item.get("category", "macro") + score_field = self.CATEGORY_MAP.get(category, "news") + weighted_sentiment = item["sentiment"] * decay + + for symbol in item.get("symbols", []): + if symbol not in symbol_data: + symbol_data[symbol] = { + "news_scores": [], + "social_scores": [], + "policy_scores": [], + "filing_scores": [], + "count": 0, + } + symbol_data[symbol][f"{score_field}_scores"].append(weighted_sentiment) + symbol_data[symbol]["count"] += 1 + + result = {} + for symbol, data in symbol_data.items(): + ns = _safe_avg(data["news_scores"]) + ss = _safe_avg(data["social_scores"]) + ps = _safe_avg(data["policy_scores"]) + fs = _safe_avg(data["filing_scores"]) + result[symbol] = SymbolScore( + symbol=symbol, + news_score=ns, + news_count=data["count"], + social_score=ss, + policy_score=ps, + filing_score=fs, + composite=self._compute_composite(ns, ss, ps, fs), + updated_at=now, + ) + return result + + def determine_regime(self, fear_greed: int, vix: float | None) -> str: + if fear_greed <= 20: + return "risk_off" + if vix is not None and vix > 30: + return "risk_off" + if fear_greed >= 60 and (vix is None or vix < 20): + return "risk_on" + return "neutral" diff --git a/shared/src/shared/sentiment_models.py b/shared/src/shared/sentiment_models.py new file mode 100644 index 0000000..a009601 --- /dev/null +++ b/shared/src/shared/sentiment_models.py @@ -0,0 +1,44 @@ +"""Sentiment scoring and stock selection models.""" + +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel + +from shared.models import OrderSide + + +class SymbolScore(BaseModel): + symbol: str + news_score: float + news_count: int + social_score: float + policy_score: float + filing_score: float + composite: float + updated_at: datetime + + +class MarketSentiment(BaseModel): + fear_greed: int + fear_greed_label: str + vix: Optional[float] = None + fed_stance: str + market_regime: str + updated_at: datetime + + +class SelectedStock(BaseModel): + symbol: str + side: OrderSide + conviction: float + reason: str + key_news: list[str] + + +class Candidate(BaseModel): + symbol: str + source: str + direction: Optional[OrderSide] = None + score: float + reason: str diff --git a/shared/tests/test_broker.py b/shared/tests/test_broker.py index 9be84b0..eb1582d 100644 --- a/shared/tests/test_broker.py +++ b/shared/tests/test_broker.py @@ -16,7 +16,7 @@ async def test_broker_publish(): from shared.broker import RedisBroker broker = RedisBroker("redis://localhost:6379") - data = {"type": "CANDLE", "symbol": "BTCUSDT"} + data = {"type": "CANDLE", "symbol": "AAPL"} await broker.publish("candles", data) mock_redis.xadd.assert_called_once() @@ -35,7 +35,7 @@ async def test_broker_subscribe_returns_messages(): mock_redis = AsyncMock() mock_from_url.return_value = mock_redis - payload_data = {"type": "CANDLE", "symbol": "ETHUSDT"} + payload_data = {"type": "CANDLE", "symbol": "MSFT"} mock_redis.xread.return_value = [ [ b"candles", @@ -53,7 +53,7 @@ async def test_broker_subscribe_returns_messages(): mock_redis.xread.assert_called_once() assert len(messages) == 1 assert messages[0]["type"] == "CANDLE" - assert messages[0]["symbol"] == "ETHUSDT" + assert messages[0]["symbol"] == "MSFT" @pytest.mark.asyncio diff --git a/shared/tests/test_db.py b/shared/tests/test_db.py index d33dfe1..239ee64 100644 --- a/shared/tests/test_db.py +++ b/shared/tests/test_db.py @@ -10,7 +10,7 @@ def make_candle(): from shared.models import Candle return Candle( - symbol="BTCUSDT", + symbol="AAPL", timeframe="1m", open_time=datetime(2024, 1, 1, tzinfo=timezone.utc), open=Decimal("50000"), @@ -27,7 +27,7 @@ def make_signal(): return Signal( id="sig-1", strategy="ma_cross", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.BUY, price=Decimal("50000"), quantity=Decimal("0.1"), @@ -42,7 +42,7 @@ def make_order(): return Order( id="ord-1", signal_id="sig-1", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.BUY, type=OrderType.LIMIT, price=Decimal("50000"), @@ -228,7 +228,7 @@ class TestGetCandles: # Create a mock row that behaves like a SA result row mock_row = MagicMock() mock_row._mapping = { - "symbol": "BTCUSDT", + "symbol": "AAPL", "timeframe": "1m", "open_time": datetime(2024, 1, 1, tzinfo=timezone.utc), "open": Decimal("50000"), @@ -248,11 +248,11 @@ class TestGetCandles: db._session_factory = MagicMock(return_value=mock_session) - result = await db.get_candles("BTCUSDT", "1m", 500) + result = await db.get_candles("AAPL", "1m", 500) assert isinstance(result, list) assert len(result) == 1 - assert result[0]["symbol"] == "BTCUSDT" + assert result[0]["symbol"] == "AAPL" mock_session.execute.assert_awaited_once() diff --git a/shared/tests/test_db_news.py b/shared/tests/test_db_news.py new file mode 100644 index 0000000..a2c9140 --- /dev/null +++ b/shared/tests/test_db_news.py @@ -0,0 +1,78 @@ +"""Tests for database news/sentiment methods. Uses in-memory SQLite.""" + +import pytest +from datetime import datetime, date, timezone + +from shared.db import Database +from shared.models import NewsItem, NewsCategory +from shared.sentiment_models import SymbolScore, MarketSentiment + + +@pytest.fixture +async def db(): + database = Database("sqlite+aiosqlite://") + await database.connect() + yield database + await database.close() + + +async def test_insert_and_get_news_items(db): + item = NewsItem( + source="finnhub", + headline="AAPL earnings beat", + published_at=datetime(2026, 4, 2, 12, 0, tzinfo=timezone.utc), + sentiment=0.8, + category=NewsCategory.EARNINGS, + symbols=["AAPL"], + ) + await db.insert_news_item(item) + items = await db.get_recent_news(hours=24) + assert len(items) == 1 + assert items[0]["headline"] == "AAPL earnings beat" + + +async def test_upsert_symbol_score(db): + score = SymbolScore( + symbol="AAPL", + news_score=0.5, + news_count=10, + social_score=0.3, + policy_score=0.0, + filing_score=0.2, + composite=0.3, + updated_at=datetime(2026, 4, 2, tzinfo=timezone.utc), + ) + await db.upsert_symbol_score(score) + scores = await db.get_top_symbol_scores(limit=5) + assert len(scores) == 1 + assert scores[0]["symbol"] == "AAPL" + + +async def test_upsert_market_sentiment(db): + ms = MarketSentiment( + fear_greed=55, + fear_greed_label="Neutral", + vix=18.2, + fed_stance="neutral", + market_regime="neutral", + updated_at=datetime(2026, 4, 2, tzinfo=timezone.utc), + ) + await db.upsert_market_sentiment(ms) + result = await db.get_latest_market_sentiment() + assert result is not None + assert result["fear_greed"] == 55 + + +async def test_insert_stock_selection(db): + await db.insert_stock_selection( + trade_date=date(2026, 4, 2), + symbol="NVDA", + side="BUY", + conviction=0.85, + reason="CHIPS Act", + key_news=["Trump signs CHIPS expansion"], + sentiment_snapshot={"composite": 0.8}, + ) + selections = await db.get_stock_selections(date(2026, 4, 2)) + assert len(selections) == 1 + assert selections[0]["symbol"] == "NVDA" diff --git a/shared/tests/test_events.py b/shared/tests/test_events.py index ab7792b..6077d93 100644 --- a/shared/tests/test_events.py +++ b/shared/tests/test_events.py @@ -8,7 +8,7 @@ def make_candle(): from shared.models import Candle return Candle( - symbol="BTCUSDT", + symbol="AAPL", timeframe="1m", open_time=datetime(2024, 1, 1, tzinfo=timezone.utc), open=Decimal("50000"), @@ -24,7 +24,7 @@ def make_signal(): return Signal( strategy="test", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.BUY, price=Decimal("50000"), quantity=Decimal("0.01"), @@ -40,7 +40,7 @@ def test_candle_event_serialize(): event = CandleEvent(data=candle) d = event.to_dict() assert d["type"] == EventType.CANDLE - assert d["data"]["symbol"] == "BTCUSDT" + assert d["data"]["symbol"] == "AAPL" assert d["data"]["timeframe"] == "1m" @@ -53,7 +53,7 @@ def test_candle_event_deserialize(): d = event.to_dict() restored = CandleEvent.from_raw(d) assert restored.type == EventType.CANDLE - assert restored.data.symbol == "BTCUSDT" + assert restored.data.symbol == "AAPL" assert restored.data.close == Decimal("50500") @@ -65,7 +65,7 @@ def test_signal_event_serialize(): event = SignalEvent(data=signal) d = event.to_dict() assert d["type"] == EventType.SIGNAL - assert d["data"]["symbol"] == "BTCUSDT" + assert d["data"]["symbol"] == "AAPL" assert d["data"]["strategy"] == "test" diff --git a/shared/tests/test_models.py b/shared/tests/test_models.py index 2b8cd5e..04098ce 100644 --- a/shared/tests/test_models.py +++ b/shared/tests/test_models.py @@ -27,7 +27,7 @@ def test_candle_creation(): now = datetime.now(timezone.utc) candle = Candle( - symbol="BTCUSDT", + symbol="AAPL", timeframe="1m", open_time=now, open=Decimal("50000.00"), @@ -36,7 +36,7 @@ def test_candle_creation(): close=Decimal("50500.00"), volume=Decimal("100.5"), ) - assert candle.symbol == "BTCUSDT" + assert candle.symbol == "AAPL" assert candle.timeframe == "1m" assert candle.open == Decimal("50000.00") assert candle.high == Decimal("51000.00") @@ -51,14 +51,14 @@ def test_signal_creation(): signal = Signal( strategy="rsi_strategy", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.BUY, price=Decimal("50000.00"), quantity=Decimal("0.01"), reason="RSI oversold", ) assert signal.strategy == "rsi_strategy" - assert signal.symbol == "BTCUSDT" + assert signal.symbol == "AAPL" assert signal.side == OrderSide.BUY assert signal.price == Decimal("50000.00") assert signal.quantity == Decimal("0.01") @@ -75,7 +75,7 @@ def test_order_creation(): signal_id = str(uuid.uuid4()) order = Order( signal_id=signal_id, - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.BUY, type=OrderType.MARKET, price=Decimal("50000.00"), @@ -94,7 +94,7 @@ def test_signal_conviction_default(): signal = Signal( strategy="rsi", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.BUY, price=Decimal("50000"), quantity=Decimal("0.01"), @@ -111,7 +111,7 @@ def test_signal_with_stops(): signal = Signal( strategy="rsi", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.BUY, price=Decimal("50000"), quantity=Decimal("0.01"), @@ -130,7 +130,7 @@ def test_position_unrealized_pnl(): from shared.models import Position position = Position( - symbol="BTCUSDT", + symbol="AAPL", quantity=Decimal("0.1"), avg_entry_price=Decimal("50000"), current_price=Decimal("51000"), diff --git a/shared/tests/test_news_events.py b/shared/tests/test_news_events.py new file mode 100644 index 0000000..384796a --- /dev/null +++ b/shared/tests/test_news_events.py @@ -0,0 +1,56 @@ +"""Tests for NewsEvent.""" + +from datetime import datetime, timezone + +from shared.models import NewsCategory, NewsItem +from shared.events import NewsEvent, EventType, Event + + +def test_news_event_to_dict(): + item = NewsItem( + source="finnhub", + headline="Test", + published_at=datetime(2026, 4, 2, tzinfo=timezone.utc), + sentiment=0.5, + category=NewsCategory.MACRO, + ) + event = NewsEvent(data=item) + d = event.to_dict() + assert d["type"] == EventType.NEWS + assert d["data"]["source"] == "finnhub" + + +def test_news_event_from_raw(): + raw = { + "type": "NEWS", + "data": { + "id": "abc", + "source": "rss", + "headline": "Test headline", + "published_at": "2026-04-02T00:00:00+00:00", + "sentiment": 0.3, + "category": "earnings", + "symbols": ["AAPL"], + "raw_data": {}, + }, + } + event = NewsEvent.from_raw(raw) + assert event.data.source == "rss" + assert event.data.symbols == ["AAPL"] + + +def test_event_dispatcher_news(): + raw = { + "type": "NEWS", + "data": { + "id": "abc", + "source": "finnhub", + "headline": "Test", + "published_at": "2026-04-02T00:00:00+00:00", + "sentiment": 0.0, + "category": "macro", + "raw_data": {}, + }, + } + event = Event.from_dict(raw) + assert isinstance(event, NewsEvent) diff --git a/shared/tests/test_notifier.py b/shared/tests/test_notifier.py index 3d29830..6c81369 100644 --- a/shared/tests/test_notifier.py +++ b/shared/tests/test_notifier.py @@ -86,7 +86,7 @@ class TestTelegramNotifierFormatters: notifier = TelegramNotifier(bot_token="fake-token", chat_id="123") signal = Signal( strategy="rsi_strategy", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.BUY, price=Decimal("50000.00"), quantity=Decimal("0.01"), @@ -99,7 +99,7 @@ class TestTelegramNotifierFormatters: msg = mock_send.call_args[0][0] assert "BUY" in msg assert "rsi_strategy" in msg - assert "BTCUSDT" in msg + assert "AAPL" in msg assert "50000.00" in msg assert "0.01" in msg assert "RSI oversold" in msg @@ -109,7 +109,7 @@ class TestTelegramNotifierFormatters: notifier = TelegramNotifier(bot_token="fake-token", chat_id="123") order = Order( signal_id=str(uuid.uuid4()), - symbol="ETHUSDT", + symbol="MSFT", side=OrderSide.SELL, type=OrderType.LIMIT, price=Decimal("3000.50"), @@ -122,7 +122,7 @@ class TestTelegramNotifierFormatters: mock_send.assert_called_once() msg = mock_send.call_args[0][0] assert "FILLED" in msg - assert "ETHUSDT" in msg + assert "MSFT" in msg assert "SELL" in msg assert "3000.50" in msg assert "1.5" in msg @@ -143,7 +143,7 @@ class TestTelegramNotifierFormatters: notifier = TelegramNotifier(bot_token="fake-token", chat_id="123") positions = [ Position( - symbol="BTCUSDT", + symbol="AAPL", quantity=Decimal("0.1"), avg_entry_price=Decimal("50000"), current_price=Decimal("51000"), @@ -158,7 +158,7 @@ class TestTelegramNotifierFormatters: ) mock_send.assert_called_once() msg = mock_send.call_args[0][0] - assert "BTCUSDT" in msg + assert "AAPL" in msg assert "5100.00" in msg assert "100.00" in msg diff --git a/shared/tests/test_resilience.py b/shared/tests/test_resilience.py deleted file mode 100644 index e287777..0000000 --- a/shared/tests/test_resilience.py +++ /dev/null @@ -1,139 +0,0 @@ -"""Tests for retry with backoff and circuit breaker.""" - -import time - -import pytest - -from shared.resilience import CircuitBreaker, CircuitState, retry_with_backoff - - -# --------------------------------------------------------------------------- -# retry_with_backoff tests -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_retry_succeeds_first_try(): - call_count = 0 - - @retry_with_backoff(max_retries=3, base_delay=0.01) - async def succeed(): - nonlocal call_count - call_count += 1 - return "ok" - - result = await succeed() - assert result == "ok" - assert call_count == 1 - - -@pytest.mark.asyncio -async def test_retry_succeeds_after_failures(): - call_count = 0 - - @retry_with_backoff(max_retries=3, base_delay=0.01) - async def flaky(): - nonlocal call_count - call_count += 1 - if call_count < 3: - raise ValueError("not yet") - return "recovered" - - result = await flaky() - assert result == "recovered" - assert call_count == 3 - - -@pytest.mark.asyncio -async def test_retry_raises_after_max_retries(): - call_count = 0 - - @retry_with_backoff(max_retries=3, base_delay=0.01) - async def always_fail(): - nonlocal call_count - call_count += 1 - raise RuntimeError("permanent") - - with pytest.raises(RuntimeError, match="permanent"): - await always_fail() - # 1 initial + 3 retries = 4 calls - assert call_count == 4 - - -@pytest.mark.asyncio -async def test_retry_respects_max_delay(): - """Backoff should be capped at max_delay.""" - - @retry_with_backoff(max_retries=2, base_delay=0.01, max_delay=0.02) - async def always_fail(): - raise RuntimeError("fail") - - start = time.monotonic() - with pytest.raises(RuntimeError): - await always_fail() - elapsed = time.monotonic() - start - # With max_delay=0.02 and 2 retries, total delay should be small - assert elapsed < 0.5 - - -# --------------------------------------------------------------------------- -# CircuitBreaker tests -# --------------------------------------------------------------------------- - - -def test_circuit_starts_closed(): - cb = CircuitBreaker(failure_threshold=3, recovery_timeout=0.05) - assert cb.state == CircuitState.CLOSED - assert cb.allow_request() is True - - -def test_circuit_opens_after_threshold(): - cb = CircuitBreaker(failure_threshold=3, recovery_timeout=60.0) - for _ in range(3): - cb.record_failure() - assert cb.state == CircuitState.OPEN - assert cb.allow_request() is False - - -def test_circuit_rejects_when_open(): - cb = CircuitBreaker(failure_threshold=2, recovery_timeout=60.0) - cb.record_failure() - cb.record_failure() - assert cb.state == CircuitState.OPEN - assert cb.allow_request() is False - - -def test_circuit_half_open_after_timeout(): - cb = CircuitBreaker(failure_threshold=2, recovery_timeout=0.05) - cb.record_failure() - cb.record_failure() - assert cb.state == CircuitState.OPEN - - time.sleep(0.06) - assert cb.allow_request() is True - assert cb.state == CircuitState.HALF_OPEN - - -def test_circuit_closes_on_success(): - cb = CircuitBreaker(failure_threshold=2, recovery_timeout=0.05) - cb.record_failure() - cb.record_failure() - assert cb.state == CircuitState.OPEN - - time.sleep(0.06) - cb.allow_request() # triggers HALF_OPEN - assert cb.state == CircuitState.HALF_OPEN - - cb.record_success() - assert cb.state == CircuitState.CLOSED - assert cb.allow_request() is True - - -def test_circuit_reopens_on_failure_in_half_open(): - cb = CircuitBreaker(failure_threshold=2, recovery_timeout=0.05) - cb.record_failure() - cb.record_failure() - time.sleep(0.06) - cb.allow_request() # HALF_OPEN - cb.record_failure() - assert cb.state == CircuitState.OPEN diff --git a/shared/tests/test_sa_models.py b/shared/tests/test_sa_models.py index 67c3c82..ae73833 100644 --- a/shared/tests/test_sa_models.py +++ b/shared/tests/test_sa_models.py @@ -11,9 +11,12 @@ def test_base_metadata_has_all_tables(): "candles", "signals", "orders", - "trades", "positions", "portfolio_snapshots", + "news_items", + "symbol_scores", + "market_sentiment", + "stock_selections", } assert expected == table_names @@ -120,44 +123,6 @@ class TestOrderRow: assert fk_cols == {"signal_id": "signals.id"} -class TestTradeRow: - def test_table_name(self): - from shared.sa_models import TradeRow - - assert TradeRow.__tablename__ == "trades" - - def test_columns(self): - from shared.sa_models import TradeRow - - mapper = inspect(TradeRow) - cols = {c.key for c in mapper.column_attrs} - expected = { - "id", - "order_id", - "symbol", - "side", - "price", - "quantity", - "fee", - "traded_at", - } - assert expected == cols - - def test_primary_key(self): - from shared.sa_models import TradeRow - - mapper = inspect(TradeRow) - pk_cols = [c.name for c in mapper.mapper.primary_key] - assert pk_cols == ["id"] - - def test_order_id_foreign_key(self): - from shared.sa_models import TradeRow - - table = TradeRow.__table__ - fk_cols = {fk.parent.name: fk.target_fullname for fk in table.foreign_keys} - assert fk_cols == {"order_id": "orders.id"} - - class TestPositionRow: def test_table_name(self): from shared.sa_models import PositionRow @@ -229,11 +194,3 @@ class TestStatusDefault: status_col = table.c.status assert status_col.server_default is not None assert status_col.server_default.arg == "PENDING" - - def test_trade_fee_server_default(self): - from shared.sa_models import TradeRow - - table = TradeRow.__table__ - fee_col = table.c.fee - assert fee_col.server_default is not None - assert fee_col.server_default.arg == "0" diff --git a/shared/tests/test_sa_news_models.py b/shared/tests/test_sa_news_models.py new file mode 100644 index 0000000..91e6d4a --- /dev/null +++ b/shared/tests/test_sa_news_models.py @@ -0,0 +1,29 @@ +"""Tests for news-related SQLAlchemy models.""" + +from shared.sa_models import NewsItemRow, SymbolScoreRow, MarketSentimentRow, StockSelectionRow + + +def test_news_item_row_tablename(): + assert NewsItemRow.__tablename__ == "news_items" + + +def test_symbol_score_row_tablename(): + assert SymbolScoreRow.__tablename__ == "symbol_scores" + + +def test_market_sentiment_row_tablename(): + assert MarketSentimentRow.__tablename__ == "market_sentiment" + + +def test_stock_selection_row_tablename(): + assert StockSelectionRow.__tablename__ == "stock_selections" + + +def test_news_item_row_columns(): + cols = {c.name for c in NewsItemRow.__table__.columns} + assert cols >= {"id", "source", "headline", "published_at", "sentiment", "category"} + + +def test_symbol_score_row_columns(): + cols = {c.name for c in SymbolScoreRow.__table__.columns} + assert cols >= {"id", "symbol", "news_score", "composite", "updated_at"} diff --git a/shared/tests/test_sentiment.py b/shared/tests/test_sentiment.py deleted file mode 100644 index 2caa266..0000000 --- a/shared/tests/test_sentiment.py +++ /dev/null @@ -1,144 +0,0 @@ -"""Tests for market sentiment module.""" - -import pytest -from unittest.mock import AsyncMock, MagicMock -from shared.sentiment import SentimentData, SentimentProvider - - -# --- SentimentData tests --- - - -def test_sentiment_should_buy_on_fear(): - s = SentimentData(fear_greed_value=15) # Extreme fear - assert s.should_buy is True - - -def test_sentiment_should_not_buy_on_greed(): - s = SentimentData(fear_greed_value=75) # Greed - assert s.should_buy is False - - -def test_sentiment_should_block_extreme_greed(): - s = SentimentData(fear_greed_value=85) - assert s.should_block is True - - -def test_sentiment_should_block_very_negative_news(): - s = SentimentData(news_sentiment=-0.6) - assert s.should_block is True - - -def test_sentiment_no_block_on_neutral(): - s = SentimentData(fear_greed_value=50, news_sentiment=0.0) - assert s.should_block is False - - -def test_sentiment_should_buy_default_no_data(): - s = SentimentData() - assert s.should_buy is True - assert s.should_block is False - - -def test_sentiment_positive_news_allows_buy(): - s = SentimentData(fear_greed_value=50, news_sentiment=0.3) - assert s.should_buy is True - - -def test_sentiment_outflow_bullish(): - s = SentimentData(exchange_netflow=-100.0) # Outflow = bullish - assert s.should_buy is True - - -def test_sentiment_inflow_bearish(): - s = SentimentData(fear_greed_value=50, exchange_netflow=100.0) # Inflow = bearish - assert s.should_buy is False - - -# --- SentimentProvider tests --- - - -@pytest.mark.asyncio -async def test_provider_fetch_fear_greed(): - provider = SentimentProvider() - - mock_response = AsyncMock() - mock_response.status = 200 - mock_response.json = AsyncMock( - return_value={"data": [{"value": "25", "value_classification": "Extreme Fear"}]} - ) - mock_response.__aenter__ = AsyncMock(return_value=mock_response) - mock_response.__aexit__ = AsyncMock(return_value=False) - - mock_session = MagicMock() - mock_session.closed = False - mock_session.get = MagicMock(return_value=mock_response) - mock_session.close = AsyncMock() - provider._session = mock_session - - value, label = await provider.fetch_fear_greed() - assert value == 25 - assert label == "Extreme Fear" - - await provider.close() - - -@pytest.mark.asyncio -async def test_provider_fetch_fear_greed_failure(): - provider = SentimentProvider() - - mock_response = AsyncMock() - mock_response.status = 500 - mock_response.__aenter__ = AsyncMock(return_value=mock_response) - mock_response.__aexit__ = AsyncMock(return_value=False) - - mock_session = MagicMock() - mock_session.closed = False - mock_session.get = MagicMock(return_value=mock_response) - mock_session.close = AsyncMock() - provider._session = mock_session - - value, label = await provider.fetch_fear_greed() - assert value is None - - await provider.close() - - -@pytest.mark.asyncio -async def test_provider_news_disabled_without_key(): - provider = SentimentProvider(cryptopanic_api_key="") - score, count = await provider.fetch_news_sentiment() - assert score is None - assert count == 0 - - -@pytest.mark.asyncio -async def test_provider_netflow_disabled_without_key(): - provider = SentimentProvider(cryptoquant_api_key="") - result = await provider.fetch_exchange_netflow() - assert result is None - - -@pytest.mark.asyncio -async def test_provider_get_sentiment_aggregates(): - provider = SentimentProvider() - - mock_response = AsyncMock() - mock_response.status = 200 - mock_response.json = AsyncMock( - return_value={"data": [{"value": "20", "value_classification": "Extreme Fear"}]} - ) - mock_response.__aenter__ = AsyncMock(return_value=mock_response) - mock_response.__aexit__ = AsyncMock(return_value=False) - - mock_session = MagicMock() - mock_session.closed = False - mock_session.get = MagicMock(return_value=mock_response) - mock_session.close = AsyncMock() - provider._session = mock_session - - sentiment = await provider.get_sentiment("SOL") - assert sentiment.fear_greed_value == 20 - assert sentiment.fear_greed_label == "Extreme Fear" - assert provider.cached is sentiment - - await provider.close() diff --git a/shared/tests/test_sentiment_aggregator.py b/shared/tests/test_sentiment_aggregator.py new file mode 100644 index 0000000..a99c711 --- /dev/null +++ b/shared/tests/test_sentiment_aggregator.py @@ -0,0 +1,77 @@ +"""Tests for sentiment aggregator.""" + +import pytest +from datetime import datetime, timezone, timedelta +from shared.sentiment import SentimentAggregator + + +@pytest.fixture +def aggregator(): + return SentimentAggregator() + + +def test_freshness_decay_recent(): + a = SentimentAggregator() + now = datetime.now(timezone.utc) + assert a._freshness_decay(now, now) == 1.0 + + +def test_freshness_decay_3_hours(): + a = SentimentAggregator() + now = datetime.now(timezone.utc) + assert a._freshness_decay(now - timedelta(hours=3), now) == 0.7 + + +def test_freshness_decay_12_hours(): + a = SentimentAggregator() + now = datetime.now(timezone.utc) + assert a._freshness_decay(now - timedelta(hours=12), now) == 0.3 + + +def test_freshness_decay_old(): + a = SentimentAggregator() + now = datetime.now(timezone.utc) + assert a._freshness_decay(now - timedelta(days=2), now) == 0.0 + + +def test_compute_composite(): + a = SentimentAggregator() + composite = a._compute_composite( + news_score=0.5, social_score=0.3, policy_score=0.8, filing_score=0.2 + ) + expected = 0.5 * 0.3 + 0.3 * 0.2 + 0.8 * 0.3 + 0.2 * 0.2 + assert abs(composite - expected) < 0.001 + + +def test_aggregate_news_by_symbol(aggregator): + now = datetime.now(timezone.utc) + news_items = [ + {"symbols": ["AAPL"], "sentiment": 0.8, "category": "earnings", "published_at": now}, + { + "symbols": ["AAPL"], + "sentiment": 0.3, + "category": "macro", + "published_at": now - timedelta(hours=2), + }, + {"symbols": ["MSFT"], "sentiment": -0.5, "category": "policy", "published_at": now}, + ] + scores = aggregator.aggregate(news_items, now) + assert "AAPL" in scores + assert "MSFT" in scores + assert scores["AAPL"].news_count == 2 + assert scores["AAPL"].news_score > 0 + assert scores["MSFT"].policy_score < 0 + + +def test_aggregate_empty(aggregator): + now = datetime.now(timezone.utc) + assert aggregator.aggregate([], now) == {} + + +def test_determine_regime(): + a = SentimentAggregator() + assert a.determine_regime(15, None) == "risk_off" + assert a.determine_regime(15, 35.0) == "risk_off" + assert a.determine_regime(50, 35.0) == "risk_off" + assert a.determine_regime(70, 15.0) == "risk_on" + assert a.determine_regime(50, 20.0) == "neutral" diff --git a/shared/tests/test_sentiment_models.py b/shared/tests/test_sentiment_models.py new file mode 100644 index 0000000..25fc371 --- /dev/null +++ b/shared/tests/test_sentiment_models.py @@ -0,0 +1,113 @@ +"""Tests for news and sentiment models.""" + +from datetime import datetime, timezone + +from shared.models import NewsCategory, NewsItem, OrderSide +from shared.sentiment_models import SymbolScore, MarketSentiment, SelectedStock, Candidate + + +def test_news_item_defaults(): + item = NewsItem( + source="finnhub", + headline="Test headline", + published_at=datetime(2026, 4, 2, tzinfo=timezone.utc), + sentiment=0.5, + category=NewsCategory.MACRO, + ) + assert item.id + assert item.symbols == [] + assert item.summary is None + assert item.raw_data == {} + assert item.created_at is not None + + +def test_news_item_with_symbols(): + item = NewsItem( + source="rss", + headline="AAPL earnings beat", + published_at=datetime(2026, 4, 2, tzinfo=timezone.utc), + sentiment=0.8, + category=NewsCategory.EARNINGS, + symbols=["AAPL"], + ) + assert item.symbols == ["AAPL"] + assert item.category == NewsCategory.EARNINGS + + +def test_news_category_values(): + assert NewsCategory.POLICY == "policy" + assert NewsCategory.EARNINGS == "earnings" + assert NewsCategory.MACRO == "macro" + assert NewsCategory.SOCIAL == "social" + assert NewsCategory.FILING == "filing" + assert NewsCategory.FED == "fed" + + +def test_symbol_score(): + score = SymbolScore( + symbol="AAPL", + news_score=0.5, + news_count=10, + social_score=0.3, + policy_score=0.0, + filing_score=0.2, + composite=0.3, + updated_at=datetime(2026, 4, 2, tzinfo=timezone.utc), + ) + assert score.symbol == "AAPL" + assert score.composite == 0.3 + + +def test_market_sentiment(): + ms = MarketSentiment( + fear_greed=25, + fear_greed_label="Extreme Fear", + vix=32.5, + fed_stance="hawkish", + market_regime="risk_off", + updated_at=datetime(2026, 4, 2, tzinfo=timezone.utc), + ) + assert ms.market_regime == "risk_off" + assert ms.vix == 32.5 + + +def test_market_sentiment_no_vix(): + ms = MarketSentiment( + fear_greed=50, + fear_greed_label="Neutral", + fed_stance="neutral", + market_regime="neutral", + updated_at=datetime(2026, 4, 2, tzinfo=timezone.utc), + ) + assert ms.vix is None + + +def test_selected_stock(): + ss = SelectedStock( + symbol="NVDA", + side=OrderSide.BUY, + conviction=0.85, + reason="CHIPS Act expansion", + key_news=["Trump signs CHIPS Act expansion"], + ) + assert ss.conviction == 0.85 + assert len(ss.key_news) == 1 + + +def test_candidate(): + c = Candidate( + symbol="TSLA", + source="sentiment", + direction=OrderSide.BUY, + score=0.75, + reason="High social buzz", + ) + assert c.direction == OrderSide.BUY + + c2 = Candidate( + symbol="XOM", + source="llm", + score=0.6, + reason="Oil price surge", + ) + assert c2.direction is None diff --git a/tests/edge_cases/test_empty_data.py b/tests/edge_cases/test_empty_data.py index ebd8467..bfefc95 100644 --- a/tests/edge_cases/test_empty_data.py +++ b/tests/edge_cases/test_empty_data.py @@ -44,7 +44,7 @@ class TestPortfolioTrackerEmpty: def test_get_position_returns_none_for_unknown_symbol(self): tracker = PortfolioTracker() - assert tracker.get_position("BTCUSDT") is None + assert tracker.get_position("AAPL") is None class TestRiskManagerZeroBalance: @@ -58,7 +58,7 @@ class TestRiskManagerZeroBalance: ) signal = Signal( strategy="test", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.BUY, price=Decimal("50000"), quantity=Decimal("0.01"), @@ -80,7 +80,7 @@ class TestRiskManagerZeroBalance: ) signal = Signal( strategy="test", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.SELL, price=Decimal("50000"), quantity=Decimal("0.01"), diff --git a/tests/edge_cases/test_extreme_values.py b/tests/edge_cases/test_extreme_values.py index e5bfb1a..b375d5e 100644 --- a/tests/edge_cases/test_extreme_values.py +++ b/tests/edge_cases/test_extreme_values.py @@ -23,7 +23,7 @@ def _candle(close: str, volume: str = "1000", idx: int = 0) -> Candle: base = datetime(2025, 1, 1, tzinfo=timezone.utc) return Candle( - symbol="BTCUSDT", + symbol="AAPL", timeframe="1h", open_time=base + timedelta(hours=idx), open=Decimal(close), @@ -96,7 +96,7 @@ class TestOrderQuantityZero: sim = OrderSimulator(initial_balance=Decimal("10000")) signal = Signal( strategy="test", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.BUY, price=Decimal("50000"), quantity=Decimal("0"), @@ -112,7 +112,7 @@ class TestOrderQuantityZero: sim = OrderSimulator(initial_balance=Decimal("10000")) signal = Signal( strategy="test", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.SELL, price=Decimal("50000"), quantity=Decimal("0"), @@ -134,7 +134,7 @@ class TestRiskManagerZeroDailyLossLimit: ) signal = Signal( strategy="test", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.BUY, price=Decimal("50000"), quantity=Decimal("0.01"), @@ -157,7 +157,7 @@ class TestRiskManagerZeroDailyLossLimit: ) signal = Signal( strategy="test", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.BUY, price=Decimal("100"), quantity=Decimal("0.01"), diff --git a/tests/edge_cases/test_strategy_reset.py b/tests/edge_cases/test_strategy_reset.py index f84adf0..6e9b956 100644 --- a/tests/edge_cases/test_strategy_reset.py +++ b/tests/edge_cases/test_strategy_reset.py @@ -26,7 +26,7 @@ def _make_candles(count: int, base_price: float = 100.0) -> list[Candle]: price = base_price + (i % 10) - 5 candles.append( Candle( - symbol="BTCUSDT", + symbol="AAPL", timeframe="1h", open_time=datetime(2025, 1, 1, i % 24, tzinfo=timezone.utc), open=Decimal(str(price)), diff --git a/tests/edge_cases/test_zero_volume.py b/tests/edge_cases/test_zero_volume.py index 71a1d71..ba2c133 100644 --- a/tests/edge_cases/test_zero_volume.py +++ b/tests/edge_cases/test_zero_volume.py @@ -19,7 +19,7 @@ def _candle(close: str, volume: str = "0", idx: int = 0) -> Candle: from datetime import timedelta return Candle( - symbol="BTCUSDT", + symbol="AAPL", timeframe="1h", open_time=base + timedelta(hours=idx), open=Decimal(close), diff --git a/tests/integration/test_backtest_end_to_end.py b/tests/integration/test_backtest_end_to_end.py index 4a484f5..4cc0b12 100644 --- a/tests/integration/test_backtest_end_to_end.py +++ b/tests/integration/test_backtest_end_to_end.py @@ -16,7 +16,7 @@ from shared.models import Candle from backtester.engine import BacktestEngine -def _generate_candles(prices: list[float], symbol="BTCUSDT") -> list[Candle]: +def _generate_candles(prices: list[float], symbol="AAPL") -> list[Candle]: return [ Candle( symbol=symbol, @@ -47,7 +47,7 @@ def test_backtest_rsi_strategy_end_to_end(): result = engine.run(candles) assert result.strategy_name == "rsi" - assert result.symbol == "BTCUSDT" + assert result.symbol == "AAPL" assert result.initial_balance == Decimal("10000") assert result.detailed is not None assert result.detailed.total_trades >= 0 diff --git a/tests/integration/test_order_execution_flow.py b/tests/integration/test_order_execution_flow.py index d842d29..dcbc498 100644 --- a/tests/integration/test_order_execution_flow.py +++ b/tests/integration/test_order_execution_flow.py @@ -19,7 +19,7 @@ async def test_signal_to_order_flow(): """A valid signal passes risk checks and produces a filled order.""" signal = Signal( strategy="rsi", - symbol="BTC/USDT", + symbol="AAPL", side=OrderSide.BUY, price=Decimal("50000"), quantity=Decimal("0.01"), @@ -52,7 +52,7 @@ async def test_signal_to_order_flow(): assert order is not None assert order.status == OrderStatus.FILLED - assert order.symbol == "BTC/USDT" + assert order.symbol == "AAPL" assert order.side == OrderSide.BUY # Verify order was persisted and published @@ -66,7 +66,7 @@ async def test_signal_rejected_by_risk_manager(): """A signal that exceeds position size is rejected.""" signal = Signal( strategy="rsi", - symbol="BTC/USDT", + symbol="AAPL", side=OrderSide.BUY, price=Decimal("50000"), quantity=Decimal("100"), # Way too large diff --git a/tests/integration/test_portfolio_tracking_flow.py b/tests/integration/test_portfolio_tracking_flow.py index 80a781c..b20275a 100644 --- a/tests/integration/test_portfolio_tracking_flow.py +++ b/tests/integration/test_portfolio_tracking_flow.py @@ -19,7 +19,7 @@ def test_portfolio_tracks_buy_sell_cycle(): buy_order = Order( signal_id="sig-1", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.BUY, type=OrderType.MARKET, price=Decimal("50000"), @@ -28,14 +28,14 @@ def test_portfolio_tracks_buy_sell_cycle(): ) tracker.apply_order(buy_order) - pos = tracker.get_position("BTCUSDT") + pos = tracker.get_position("AAPL") assert pos is not None assert pos.quantity == Decimal("0.1") assert pos.avg_entry_price == Decimal("50000") sell_order = Order( signal_id="sig-2", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.SELL, type=OrderType.MARKET, price=Decimal("55000"), @@ -44,7 +44,7 @@ def test_portfolio_tracks_buy_sell_cycle(): ) tracker.apply_order(sell_order) - pos = tracker.get_position("BTCUSDT") + pos = tracker.get_position("AAPL") assert pos is None # Fully sold @@ -55,7 +55,7 @@ def test_portfolio_weighted_average_on_multiple_buys(): tracker.apply_order( Order( signal_id="s1", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.BUY, type=OrderType.MARKET, price=Decimal("50000"), @@ -66,7 +66,7 @@ def test_portfolio_weighted_average_on_multiple_buys(): tracker.apply_order( Order( signal_id="s2", - symbol="BTCUSDT", + symbol="AAPL", side=OrderSide.BUY, type=OrderType.MARKET, price=Decimal("60000"), @@ -75,6 +75,6 @@ def test_portfolio_weighted_average_on_multiple_buys(): ) ) - pos = tracker.get_position("BTCUSDT") + pos = tracker.get_position("AAPL") assert pos.quantity == Decimal("0.2") assert pos.avg_entry_price == Decimal("55000") # weighted avg diff --git a/tests/integration/test_strategy_signal_flow.py b/tests/integration/test_strategy_signal_flow.py index 448329f..6b048fb 100644 --- a/tests/integration/test_strategy_signal_flow.py +++ b/tests/integration/test_strategy_signal_flow.py @@ -26,7 +26,7 @@ def candles(): price = Decimal(str(100 - i * 2)) # 100, 98, 96... base.append( Candle( - symbol="BTCUSDT", + symbol="AAPL", timeframe="1m", open_time=datetime(2025, 1, 1, i, 0, tzinfo=timezone.utc), open=price, @@ -54,7 +54,7 @@ async def test_strategy_engine_produces_signals_from_candles(candles): engine = StrategyEngine(broker=broker, strategies=[strategy]) - await engine.process_once("candles.BTCUSDT", "$") + await engine.process_once("candles.AAPL", "$") # With 20 declining candles (100->62), RSI should be very low # Check if broker.publish was called with a signal |