mirror of
https://github.com/Xe138/AI-Trader.git
synced 2026-04-15 14:27:25 -04:00
Compare commits
24 Commits
1c7a9621d0
...
v0.2.0-alp
| Author | SHA1 | Date | |
|---|---|---|---|
| 9e308ee243 | |||
| 90ea10c7ef | |||
| 77ce1b2b11 | |||
| 2585045be1 | |||
| 9d5f449b1c | |||
| e4b7e197d3 | |||
| 928f5fb53f | |||
| 46582d38bb | |||
| 147fab11d5 | |||
| 85fc925b4e | |||
| 156d0cab21 | |||
| 4e7c4ccf99 | |||
| 9dfc8f1ea6 | |||
| 6af1808c2f | |||
| 733b6a4b1c | |||
| 592b105b19 | |||
| 4af00d3709 | |||
| e9baa818a4 | |||
| f25bd87c26 | |||
| 794028e910 | |||
| efc2d754d6 | |||
| 5225950703 | |||
| 6794a65ef6 | |||
| c15e25f826 |
60
.dockerignore
Normal file
60
.dockerignore
Normal file
@@ -0,0 +1,60 @@
|
||||
# Version control
|
||||
.git/
|
||||
.gitignore
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
venv/
|
||||
env/
|
||||
ENV/
|
||||
.venv/
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# Environment and secrets
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# Data files (fetched at runtime)
|
||||
data/*.json
|
||||
data/agent_data/
|
||||
data/merged.jsonl
|
||||
data/merged_daily.jsonl
|
||||
data/merged_hour.jsonl
|
||||
|
||||
# Logs
|
||||
logs/
|
||||
*.log
|
||||
|
||||
# Runtime state
|
||||
runtime_env.json
|
||||
.runtime_env.json
|
||||
|
||||
# Documentation (not needed in image)
|
||||
docs/
|
||||
!README.md
|
||||
|
||||
# CI/CD
|
||||
.github/
|
||||
|
||||
# Git worktrees
|
||||
.worktrees/
|
||||
|
||||
# Test files
|
||||
test.py
|
||||
delete.py
|
||||
refresh_data.sh
|
||||
|
||||
# Build artifacts
|
||||
build/
|
||||
dist/
|
||||
*.egg-info/
|
||||
24
.env.example
24
.env.example
@@ -1,13 +1,25 @@
|
||||
OPENAI_API_BASE=""
|
||||
OPENAI_API_KEY=""
|
||||
ALPHAADVANTAGE_API_KEY =""
|
||||
JINA_API_KEY=""
|
||||
# =============================================================================
|
||||
# AI-Trader Environment Configuration
|
||||
# =============================================================================
|
||||
# Copy this file to .env and fill in your actual values
|
||||
# Docker Compose automatically reads .env from project root
|
||||
|
||||
# AI Model API Configuration
|
||||
OPENAI_API_BASE=https://your-openai-proxy.com/v1
|
||||
OPENAI_API_KEY=your_openai_key_here
|
||||
|
||||
# Data Source Configuration
|
||||
ALPHAADVANTAGE_API_KEY=your_alphavantage_key_here
|
||||
JINA_API_KEY=your_jina_key_here
|
||||
|
||||
# System Configuration (Docker default paths)
|
||||
RUNTIME_ENV_PATH=/app/data/runtime_env.json
|
||||
|
||||
# MCP Service Ports (defaults shown)
|
||||
MATH_HTTP_PORT=8000
|
||||
SEARCH_HTTP_PORT=8001
|
||||
TRADE_HTTP_PORT=8002
|
||||
GETPRICE_HTTP_PORT=8003
|
||||
|
||||
# Agent Configuration
|
||||
AGENT_MAX_STEP=30
|
||||
|
||||
RUNTIME_ENV_PATH = ""
|
||||
139
.github/workflows/docker-release.yml
vendored
Normal file
139
.github/workflows/docker-release.yml
vendored
Normal file
@@ -0,0 +1,139 @@
|
||||
name: Build and Push Docker Image
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*' # Triggers on v1.0.0, v2.1.3, etc.
|
||||
workflow_dispatch: # Manual trigger option
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract version from tag
|
||||
id: meta
|
||||
run: |
|
||||
# Ensure we're building from a tag
|
||||
if [[ "$GITHUB_REF" != refs/tags/* ]]; then
|
||||
echo "Error: This workflow should only run on tag pushes"
|
||||
echo "GITHUB_REF: $GITHUB_REF"
|
||||
exit 1
|
||||
fi
|
||||
VERSION=${GITHUB_REF#refs/tags/v}
|
||||
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
||||
echo "Building version: $VERSION"
|
||||
|
||||
# Docker tags must be lowercase
|
||||
REPO_OWNER_LOWER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
|
||||
echo "repo_owner_lower=$REPO_OWNER_LOWER" >> $GITHUB_OUTPUT
|
||||
echo "Repository owner (lowercase): $REPO_OWNER_LOWER"
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
tags: |
|
||||
ghcr.io/${{ steps.meta.outputs.repo_owner_lower }}/ai-trader:${{ steps.meta.outputs.version }}
|
||||
ghcr.io/${{ steps.meta.outputs.repo_owner_lower }}/ai-trader:latest
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Image published
|
||||
run: |
|
||||
echo "✅ Docker image published successfully!"
|
||||
echo "📦 Pull with: docker pull ghcr.io/${{ steps.meta.outputs.repo_owner_lower }}/ai-trader:${{ steps.meta.outputs.version }}"
|
||||
echo "📦 Or latest: docker pull ghcr.io/${{ steps.meta.outputs.repo_owner_lower }}/ai-trader:latest"
|
||||
|
||||
- name: Generate release notes
|
||||
id: release_notes
|
||||
run: |
|
||||
VERSION="${{ steps.meta.outputs.version }}"
|
||||
REPO_OWNER_LOWER="${{ steps.meta.outputs.repo_owner_lower }}"
|
||||
|
||||
# Check if this is an alpha/beta/rc release
|
||||
if [[ "$VERSION" == *"-alpha"* ]] || [[ "$VERSION" == *"-beta"* ]] || [[ "$VERSION" == *"-rc"* ]]; then
|
||||
PRERELEASE="true"
|
||||
RELEASE_TYPE="Pre-release"
|
||||
else
|
||||
PRERELEASE="false"
|
||||
RELEASE_TYPE="Release"
|
||||
fi
|
||||
echo "prerelease=$PRERELEASE" >> $GITHUB_OUTPUT
|
||||
|
||||
# Generate generic release notes template
|
||||
cat > /tmp/release_notes.md << 'EOF'
|
||||
## Release Notes
|
||||
|
||||
<!-- TODO: Add release summary and highlights here -->
|
||||
|
||||
### Quick Start
|
||||
|
||||
**Using Docker Compose:**
|
||||
```bash
|
||||
git clone https://github.com/Xe138/AI-Trader.git
|
||||
cd AI-Trader
|
||||
cp .env.example .env
|
||||
# Edit .env with your API keys
|
||||
docker-compose up
|
||||
```
|
||||
|
||||
**Using pre-built image:**
|
||||
```bash
|
||||
docker pull ghcr.io/REPO_OWNER/ai-trader:VERSION
|
||||
docker run --env-file .env \
|
||||
-v $(pwd)/data:/app/data \
|
||||
-v $(pwd)/logs:/app/logs \
|
||||
ghcr.io/REPO_OWNER/ai-trader:VERSION
|
||||
```
|
||||
|
||||
### Documentation
|
||||
|
||||
- [Docker Deployment Guide](docs/DOCKER.md)
|
||||
- [Release Process](docs/RELEASING.md)
|
||||
- [Full Changelog](CHANGELOG.md)
|
||||
|
||||
### Changes
|
||||
|
||||
<!-- TODO: Add changes from CHANGELOG.md here -->
|
||||
|
||||
See [CHANGELOG.md](CHANGELOG.md) for detailed changes.
|
||||
|
||||
---
|
||||
|
||||
**Container Registry:** `ghcr.io/REPO_OWNER/ai-trader:VERSION`
|
||||
**Docker Image:** `ghcr.io/REPO_OWNER/ai-trader:latest`
|
||||
EOF
|
||||
|
||||
# Replace placeholders
|
||||
sed -i "s/REPO_OWNER/$REPO_OWNER_LOWER/g" /tmp/release_notes.md
|
||||
sed -i "s/VERSION/$VERSION/g" /tmp/release_notes.md
|
||||
|
||||
cat /tmp/release_notes.md
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
tag_name: v${{ steps.meta.outputs.version }}
|
||||
name: v${{ steps.meta.outputs.version }}
|
||||
body_path: /tmp/release_notes.md
|
||||
draft: true
|
||||
prerelease: ${{ steps.release_notes.outputs.prerelease }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -81,3 +81,6 @@ htmlcov/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Git worktrees
|
||||
.worktrees/
|
||||
|
||||
|
||||
98
CHANGELOG.md
Normal file
98
CHANGELOG.md
Normal file
@@ -0,0 +1,98 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to the AI-Trader project will be documented in this file.
|
||||
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.2.0] - 2025-10-30
|
||||
|
||||
### Added
|
||||
- Complete Docker deployment support with containerization
|
||||
- Docker Compose orchestration for easy local deployment
|
||||
- Multi-stage Dockerfile with Python 3.10-slim base image
|
||||
- Automated CI/CD pipeline via GitHub Actions for release builds
|
||||
- Docker images published to GitHub Container Registry (ghcr.io)
|
||||
- Comprehensive Docker documentation (docs/DOCKER.md)
|
||||
- Release process documentation (docs/RELEASING.md)
|
||||
- CLAUDE.md repository guidance for development
|
||||
- Docker deployment section in main README
|
||||
- Environment variable configuration via docker-compose
|
||||
- Sequential startup script (entrypoint.sh) for data fetch, MCP services, and trading agent
|
||||
- Volume mounts for data and logs persistence
|
||||
- Pre-built image support from ghcr.io/hkuds/ai-trader
|
||||
|
||||
### Changed
|
||||
- Updated .env.example with Docker-specific configuration and paths
|
||||
- Updated .gitignore to exclude git worktrees directory
|
||||
- Removed deprecated version tag from docker-compose.yml
|
||||
|
||||
### Fixed
|
||||
- Docker Compose configuration now follows modern best practices (version-less)
|
||||
|
||||
## [0.1.0] - Initial Release
|
||||
|
||||
### Added
|
||||
- AI trading competition platform for NASDAQ 100 stocks
|
||||
- Support for multiple AI models (GPT, Claude, Qwen, DeepSeek, Gemini)
|
||||
- MCP (Model Context Protocol) toolchain integration
|
||||
- Mathematical calculation tools
|
||||
- Market intelligence search via Jina AI
|
||||
- Trading execution tools
|
||||
- Price query tools
|
||||
- Historical replay architecture with anti-look-ahead controls
|
||||
- Alpha Vantage API integration for price data
|
||||
- Autonomous AI decision-making with zero human intervention
|
||||
- Real-time performance analytics and leaderboard
|
||||
- Position tracking and trading logs
|
||||
- Web-based performance dashboard
|
||||
- Complete NASDAQ 100 stock universe support
|
||||
- Initial capital: $10,000 per AI model
|
||||
- Configurable date range for backtesting
|
||||
- Multi-model concurrent trading support
|
||||
- Automatic data fetching and merging
|
||||
- Comprehensive README with quick start guide
|
||||
|
||||
### Technical Details
|
||||
- Python 3.10+ support
|
||||
- LangChain framework integration
|
||||
- FastMCP for MCP service implementation
|
||||
- JSONL format for position and log storage
|
||||
- Weekday-only trading simulation
|
||||
- Configurable agent parameters (max_steps, max_retries, initial_cash)
|
||||
|
||||
---
|
||||
|
||||
## Release Notes Template
|
||||
|
||||
For future releases, use this template:
|
||||
|
||||
```markdown
|
||||
## [X.Y.Z] - YYYY-MM-DD
|
||||
|
||||
### Added
|
||||
- New features
|
||||
|
||||
### Changed
|
||||
- Changes to existing functionality
|
||||
|
||||
### Deprecated
|
||||
- Soon-to-be removed features
|
||||
|
||||
### Removed
|
||||
- Removed features
|
||||
|
||||
### Fixed
|
||||
- Bug fixes
|
||||
|
||||
### Security
|
||||
- Security improvements
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
[Unreleased]: https://github.com/Xe138/AI-Trader/compare/v0.2.0...HEAD
|
||||
[0.2.0]: https://github.com/Xe138/AI-Trader/compare/v0.1.0...v0.2.0
|
||||
[0.1.0]: https://github.com/Xe138/AI-Trader/releases/tag/v0.1.0
|
||||
329
CLAUDE.md
Normal file
329
CLAUDE.md
Normal file
@@ -0,0 +1,329 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Project Overview
|
||||
|
||||
AI-Trader is an autonomous AI trading competition platform where multiple AI models compete in NASDAQ 100 trading with zero human intervention. Each AI starts with $10,000 and uses standardized MCP (Model Context Protocol) tools to make fully autonomous trading decisions.
|
||||
|
||||
**Key Innovation:** Historical replay architecture with anti-look-ahead controls ensures AI agents can only access data from the current simulation date and earlier.
|
||||
|
||||
## Development Commands
|
||||
|
||||
### Environment Setup
|
||||
```bash
|
||||
# Install dependencies
|
||||
pip install -r requirements.txt
|
||||
|
||||
# Configure environment variables
|
||||
cp .env.example .env
|
||||
# Edit .env and set:
|
||||
# - OPENAI_API_BASE, OPENAI_API_KEY
|
||||
# - ALPHAADVANTAGE_API_KEY, JINA_API_KEY
|
||||
# - RUNTIME_ENV_PATH (recommended: absolute path to runtime_env.json)
|
||||
# - MCP service ports (default: 8000-8003)
|
||||
# - AGENT_MAX_STEP (default: 30)
|
||||
```
|
||||
|
||||
### Data Preparation
|
||||
```bash
|
||||
# Download/update NASDAQ 100 stock data
|
||||
cd data
|
||||
python get_daily_price.py # Fetch daily prices from Alpha Vantage
|
||||
python merge_jsonl.py # Merge into unified format (merged.jsonl)
|
||||
cd ..
|
||||
```
|
||||
|
||||
### Starting Services
|
||||
```bash
|
||||
# Start all MCP services (Math, Search, Trade, LocalPrices)
|
||||
cd agent_tools
|
||||
python start_mcp_services.py
|
||||
cd ..
|
||||
|
||||
# Services run on ports defined in .env:
|
||||
# - MATH_HTTP_PORT (default: 8000)
|
||||
# - SEARCH_HTTP_PORT (default: 8001)
|
||||
# - TRADE_HTTP_PORT (default: 8002)
|
||||
# - GETPRICE_HTTP_PORT (default: 8003)
|
||||
```
|
||||
|
||||
### Docker Deployment
|
||||
|
||||
```bash
|
||||
# Build Docker image
|
||||
docker-compose build
|
||||
|
||||
# Run with Docker Compose
|
||||
docker-compose up
|
||||
|
||||
# Run in background
|
||||
docker-compose up -d
|
||||
|
||||
# Run with custom config
|
||||
docker-compose run ai-trader configs/my_config.json
|
||||
|
||||
# View logs
|
||||
docker-compose logs -f
|
||||
|
||||
# Stop and remove containers
|
||||
docker-compose down
|
||||
|
||||
# Pull pre-built image
|
||||
docker pull ghcr.io/hkuds/ai-trader:latest
|
||||
|
||||
# Test local Docker build
|
||||
docker build -t ai-trader-test .
|
||||
docker run --env-file .env -v $(pwd)/data:/app/data ai-trader-test
|
||||
```
|
||||
|
||||
### Releasing Docker Images
|
||||
|
||||
```bash
|
||||
# Create and push release tag
|
||||
git tag v1.0.0
|
||||
git push origin v1.0.0
|
||||
|
||||
# GitHub Actions automatically:
|
||||
# 1. Builds Docker image
|
||||
# 2. Tags with version and latest
|
||||
# 3. Pushes to ghcr.io/hkuds/ai-trader
|
||||
|
||||
# Verify build in Actions tab
|
||||
# https://github.com/HKUDS/AI-Trader/actions
|
||||
```
|
||||
|
||||
### Running Trading Simulations
|
||||
```bash
|
||||
# Run with default config
|
||||
python main.py
|
||||
|
||||
# Run with custom config
|
||||
python main.py configs/my_config.json
|
||||
|
||||
# Environment variables can override config dates:
|
||||
INIT_DATE=2025-01-01 END_DATE=2025-01-31 python main.py
|
||||
```
|
||||
|
||||
### Complete Workflow
|
||||
```bash
|
||||
# All-in-one startup script (data + services + trading + web)
|
||||
bash main.sh
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
### Core Components
|
||||
|
||||
**1. Agent System** (`agent/base_agent/base_agent.py`)
|
||||
- `BaseAgent`: Base class for all trading agents
|
||||
- Manages MCP tool connections, AI model initialization, trading execution loops
|
||||
- Handles position management and logging
|
||||
- Supports retry logic with exponential backoff (`max_retries`, `base_delay`)
|
||||
|
||||
**2. Main Entry Point** (`main.py`)
|
||||
- Dynamic agent class loading via `AGENT_REGISTRY`
|
||||
- Multi-model concurrent trading support
|
||||
- Date range validation and weekday filtering
|
||||
- Configuration management (JSON + environment variables)
|
||||
|
||||
**3. MCP Toolchain** (`agent_tools/`)
|
||||
- `tool_math.py`: Mathematical calculations (port 8000)
|
||||
- `tool_jina_search.py`: Market intelligence search (port 8001)
|
||||
- `tool_trade.py`: Buy/sell execution (port 8002)
|
||||
- `tool_get_price_local.py`: Price queries (port 8003)
|
||||
- `start_mcp_services.py`: Service orchestration with health checks
|
||||
|
||||
**4. Data Management** (`data/`)
|
||||
- `daily_prices_*.json`: Individual stock OHLCV data
|
||||
- `merged.jsonl`: Unified price data format
|
||||
- `agent_data/[signature]/position/position.jsonl`: Position records
|
||||
- `agent_data/[signature]/log/[date]/log.jsonl`: Trading logs
|
||||
|
||||
**5. Utilities** (`tools/`)
|
||||
- `general_tools.py`: Config management, message extraction
|
||||
- `price_tools.py`: Price queries, position updates
|
||||
- `result_tools.py`: Performance calculations
|
||||
|
||||
### Data Flow
|
||||
|
||||
1. **Initialization**: Agent loads config, connects to MCP services, initializes AI model
|
||||
2. **Trading Loop**: For each date:
|
||||
- Get system prompt with current positions, yesterday's prices, today's buy prices
|
||||
- AI agent analyzes market, calls search/math/price tools
|
||||
- Makes buy/sell decisions via trade tool
|
||||
- Logs all decisions and updates position.jsonl
|
||||
3. **Position Tracking**: Each trade appends to `position.jsonl` with date, action, and updated holdings
|
||||
|
||||
### Configuration System
|
||||
|
||||
**Multi-layered config priority:**
|
||||
1. Environment variables (highest)
|
||||
2. Model-specific config (`openai_base_url`, `openai_api_key` in model config)
|
||||
3. JSON config file
|
||||
4. Default values (lowest)
|
||||
|
||||
**Runtime configuration** (`runtime_env.json` at `RUNTIME_ENV_PATH`):
|
||||
- Dynamic state: `TODAY_DATE`, `SIGNATURE`, `IF_TRADE`
|
||||
- Written by `write_config_value()`, read by `get_config_value()`
|
||||
|
||||
### Agent System
|
||||
|
||||
**BaseAgent Key Methods:**
|
||||
- `initialize()`: Connect to MCP services, create AI model
|
||||
- `run_trading_session(date)`: Execute single day's trading with retry logic
|
||||
- `run_date_range(init_date, end_date)`: Process all weekdays in range
|
||||
- `get_trading_dates()`: Resume from last date in position.jsonl
|
||||
- `register_agent()`: Create initial position file with $10,000 cash
|
||||
|
||||
**Adding Custom Agents:**
|
||||
1. Create new class inheriting from `BaseAgent`
|
||||
2. Add to `AGENT_REGISTRY` in `main.py`:
|
||||
```python
|
||||
"CustomAgent": {
|
||||
"module": "agent.custom.custom_agent",
|
||||
"class": "CustomAgent"
|
||||
}
|
||||
```
|
||||
3. Set `"agent_type": "CustomAgent"` in config JSON
|
||||
|
||||
### System Prompt Construction
|
||||
|
||||
**Dynamic prompt generation** (`prompts/agent_prompt.py`):
|
||||
- `get_agent_system_prompt()` builds prompt with:
|
||||
- Current date
|
||||
- Yesterday's closing positions
|
||||
- Yesterday's closing prices
|
||||
- Today's buy prices
|
||||
- Yesterday's profit/loss
|
||||
- AI agent must output `<FINISH_SIGNAL>` to end trading session
|
||||
|
||||
### Anti-Look-Ahead Controls
|
||||
|
||||
**Data access restrictions:**
|
||||
- Price data: Only returns data for `date <= TODAY_DATE`
|
||||
- Search results: News filtered by publication date
|
||||
- All tools enforce temporal boundaries via `TODAY_DATE` from `runtime_env.json`
|
||||
|
||||
## Configuration File Format
|
||||
|
||||
```json
|
||||
{
|
||||
"agent_type": "BaseAgent",
|
||||
"date_range": {
|
||||
"init_date": "2025-01-01",
|
||||
"end_date": "2025-01-31"
|
||||
},
|
||||
"models": [
|
||||
{
|
||||
"name": "model-display-name",
|
||||
"basemodel": "provider/model-id",
|
||||
"signature": "unique-identifier",
|
||||
"enabled": true,
|
||||
"openai_base_url": "optional-override",
|
||||
"openai_api_key": "optional-override"
|
||||
}
|
||||
],
|
||||
"agent_config": {
|
||||
"max_steps": 30, // Max reasoning iterations per day
|
||||
"max_retries": 3, // Retry attempts on failure
|
||||
"base_delay": 1.0, // Base retry delay (seconds)
|
||||
"initial_cash": 10000.0
|
||||
},
|
||||
"log_config": {
|
||||
"log_path": "./data/agent_data"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Data Formats
|
||||
|
||||
**Position Record** (`position.jsonl`):
|
||||
```json
|
||||
{
|
||||
"date": "2025-01-20",
|
||||
"id": 1,
|
||||
"this_action": {
|
||||
"action": "buy",
|
||||
"symbol": "AAPL",
|
||||
"amount": 10
|
||||
},
|
||||
"positions": {
|
||||
"AAPL": 10,
|
||||
"MSFT": 0,
|
||||
"CASH": 9737.6
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Price Data** (`merged.jsonl`):
|
||||
```json
|
||||
{
|
||||
"Meta Data": {
|
||||
"2. Symbol": "AAPL",
|
||||
"3. Last Refreshed": "2025-01-20"
|
||||
},
|
||||
"Time Series (Daily)": {
|
||||
"2025-01-20": {
|
||||
"1. buy price": "255.8850",
|
||||
"2. high": "264.3750",
|
||||
"3. low": "255.6300",
|
||||
"4. sell price": "262.2400",
|
||||
"5. volume": "90483029"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Important Implementation Details
|
||||
|
||||
**Trading Day Logic:**
|
||||
- Only weekdays (Monday-Friday) are processed
|
||||
- `get_trading_dates()` automatically resumes from last date in `position.jsonl`
|
||||
- Skips days already processed (idempotent)
|
||||
|
||||
**Error Handling:**
|
||||
- All async operations use `_ainvoke_with_retry()` with exponential backoff
|
||||
- MCP service failures raise detailed error messages with troubleshooting hints
|
||||
- Missing API keys halt startup with clear error messages
|
||||
|
||||
**Tool Message Extraction:**
|
||||
- `extract_conversation(response, "final")`: Get AI's final answer
|
||||
- `extract_tool_messages(response)`: Get all tool results
|
||||
- Handles both dict and object-based message formats
|
||||
|
||||
**Logging:**
|
||||
- Each trading day creates `log/[date]/log.jsonl`
|
||||
- Logs include timestamps, signature, and all message exchanges
|
||||
- Position updates append to single `position/position.jsonl`
|
||||
|
||||
## Testing Changes
|
||||
|
||||
When modifying agent behavior or adding tools:
|
||||
1. Create test config with short date range (2-3 days)
|
||||
2. Set `max_steps` low (e.g., 10) to iterate faster
|
||||
3. Check logs in `data/agent_data/[signature]/log/[date]/`
|
||||
4. Verify position updates in `position/position.jsonl`
|
||||
5. Use `main.sh` only for full end-to-end testing
|
||||
|
||||
## Common Issues
|
||||
|
||||
**MCP Services Not Running:**
|
||||
- Error: "Failed to initialize MCP client"
|
||||
- Fix: `cd agent_tools && python start_mcp_services.py`
|
||||
- Verify ports not already in use: `lsof -i :8000-8003`
|
||||
|
||||
**Missing Price Data:**
|
||||
- Ensure `data/merged.jsonl` exists
|
||||
- Run `cd data && python get_daily_price.py && python merge_jsonl.py`
|
||||
- Check Alpha Vantage API key is valid
|
||||
|
||||
**Runtime Config Issues:**
|
||||
- Set `RUNTIME_ENV_PATH` to absolute path in `.env`
|
||||
- Ensure directory is writable
|
||||
- File gets created automatically on first run
|
||||
|
||||
**Agent Doesn't Stop Trading:**
|
||||
- Agent must output `<FINISH_SIGNAL>` within `max_steps`
|
||||
- Increase `max_steps` if agent needs more reasoning time
|
||||
- Check `log.jsonl` for errors preventing completion
|
||||
32
Dockerfile
Normal file
32
Dockerfile
Normal file
@@ -0,0 +1,32 @@
|
||||
# Base stage - dependency installation
|
||||
FROM python:3.10-slim AS base
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install dependencies
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Application stage
|
||||
FROM base
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy application code
|
||||
COPY . .
|
||||
|
||||
# Create necessary directories
|
||||
RUN mkdir -p data logs data/agent_data
|
||||
|
||||
# Make entrypoint executable
|
||||
RUN chmod +x entrypoint.sh
|
||||
|
||||
# Expose MCP service ports and web dashboard
|
||||
EXPOSE 8000 8001 8002 8003 8888
|
||||
|
||||
# Set Python to run unbuffered for real-time logs
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
# Use entrypoint script
|
||||
ENTRYPOINT ["./entrypoint.sh"]
|
||||
CMD ["configs/default_config.json"]
|
||||
46
README.md
46
README.md
@@ -323,6 +323,52 @@ python3 -m http.server 8000
|
||||
# Visit http://localhost:8000
|
||||
```
|
||||
|
||||
## 🐳 Docker Deployment
|
||||
|
||||
### Using Docker Compose (Recommended)
|
||||
|
||||
The easiest way to run AI-Trader is with Docker Compose:
|
||||
|
||||
```bash
|
||||
# 1. Clone and setup
|
||||
git clone https://github.com/HKUDS/AI-Trader.git
|
||||
cd AI-Trader
|
||||
|
||||
# 2. Configure environment
|
||||
cp .env.example .env
|
||||
# Edit .env with your API keys:
|
||||
# - OPENAI_API_KEY
|
||||
# - ALPHAADVANTAGE_API_KEY
|
||||
# - JINA_API_KEY
|
||||
|
||||
# 3. Run with Docker Compose
|
||||
docker-compose up
|
||||
```
|
||||
|
||||
The container automatically:
|
||||
- Fetches latest NASDAQ 100 price data
|
||||
- Starts all MCP services
|
||||
- Runs AI trading agents
|
||||
|
||||
### Using Pre-built Images
|
||||
|
||||
Pull and run pre-built images from GitHub Container Registry:
|
||||
|
||||
```bash
|
||||
# Pull latest version
|
||||
docker pull ghcr.io/hkuds/ai-trader:latest
|
||||
|
||||
# Run container
|
||||
docker run --env-file .env \
|
||||
-v $(pwd)/data:/app/data \
|
||||
-v $(pwd)/logs:/app/logs \
|
||||
ghcr.io/hkuds/ai-trader:latest
|
||||
```
|
||||
|
||||
**📖 See [docs/DOCKER.md](docs/DOCKER.md) for detailed Docker usage, troubleshooting, and advanced configuration.**
|
||||
|
||||
---
|
||||
|
||||
## 📈 Performance Analysis
|
||||
|
||||
### 🏆 Competition Rules
|
||||
|
||||
34
docker-compose.yml
Normal file
34
docker-compose.yml
Normal file
@@ -0,0 +1,34 @@
|
||||
services:
|
||||
ai-trader:
|
||||
build: .
|
||||
container_name: ai-trader-app
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
- ./logs:/app/logs
|
||||
environment:
|
||||
# AI Model API Configuration
|
||||
- OPENAI_API_BASE=${OPENAI_API_BASE}
|
||||
- OPENAI_API_KEY=${OPENAI_API_KEY}
|
||||
|
||||
# Data Source Configuration
|
||||
- ALPHAADVANTAGE_API_KEY=${ALPHAADVANTAGE_API_KEY}
|
||||
- JINA_API_KEY=${JINA_API_KEY}
|
||||
|
||||
# System Configuration
|
||||
- RUNTIME_ENV_PATH=/app/data/runtime_env.json
|
||||
|
||||
# MCP Service Ports
|
||||
- MATH_HTTP_PORT=${MATH_HTTP_PORT:-8000}
|
||||
- SEARCH_HTTP_PORT=${SEARCH_HTTP_PORT:-8001}
|
||||
- TRADE_HTTP_PORT=${TRADE_HTTP_PORT:-8002}
|
||||
- GETPRICE_HTTP_PORT=${GETPRICE_HTTP_PORT:-8003}
|
||||
|
||||
# Agent Configuration
|
||||
- AGENT_MAX_STEP=${AGENT_MAX_STEP:-30}
|
||||
ports:
|
||||
- "8000:8000"
|
||||
- "8001:8001"
|
||||
- "8002:8002"
|
||||
- "8003:8003"
|
||||
- "8888:8888"
|
||||
restart: unless-stopped
|
||||
250
docs/DOCKER.md
Normal file
250
docs/DOCKER.md
Normal file
@@ -0,0 +1,250 @@
|
||||
# Docker Deployment Guide
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Prerequisites
|
||||
- Docker Engine 20.10+
|
||||
- Docker Compose 2.0+
|
||||
- API keys for OpenAI, Alpha Vantage, and Jina AI
|
||||
|
||||
### First-Time Setup
|
||||
|
||||
1. **Clone repository:**
|
||||
```bash
|
||||
git clone https://github.com/Xe138/AI-Trader.git
|
||||
cd AI-Trader
|
||||
```
|
||||
|
||||
2. **Configure environment:**
|
||||
```bash
|
||||
cp .env.example .env
|
||||
# Edit .env and add your API keys
|
||||
```
|
||||
|
||||
3. **Run with Docker Compose:**
|
||||
```bash
|
||||
docker-compose up
|
||||
```
|
||||
|
||||
That's it! The container will:
|
||||
- Fetch latest price data from Alpha Vantage
|
||||
- Start all MCP services
|
||||
- Run the trading agent with default configuration
|
||||
|
||||
## Configuration
|
||||
|
||||
### Environment Variables
|
||||
|
||||
Edit `.env` file with your credentials:
|
||||
|
||||
```bash
|
||||
# Required
|
||||
OPENAI_API_KEY=sk-...
|
||||
ALPHAADVANTAGE_API_KEY=...
|
||||
JINA_API_KEY=...
|
||||
|
||||
# Optional (defaults shown)
|
||||
MATH_HTTP_PORT=8000
|
||||
SEARCH_HTTP_PORT=8001
|
||||
TRADE_HTTP_PORT=8002
|
||||
GETPRICE_HTTP_PORT=8003
|
||||
AGENT_MAX_STEP=30
|
||||
```
|
||||
|
||||
### Custom Trading Configuration
|
||||
|
||||
Pass a custom config file:
|
||||
|
||||
```bash
|
||||
docker-compose run ai-trader configs/my_config.json
|
||||
```
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Run in foreground with logs
|
||||
```bash
|
||||
docker-compose up
|
||||
```
|
||||
|
||||
### Run in background (detached)
|
||||
```bash
|
||||
docker-compose up -d
|
||||
docker-compose logs -f # Follow logs
|
||||
```
|
||||
|
||||
### Run with custom config
|
||||
```bash
|
||||
docker-compose run ai-trader configs/custom_config.json
|
||||
```
|
||||
|
||||
### Stop containers
|
||||
```bash
|
||||
docker-compose down
|
||||
```
|
||||
|
||||
### Rebuild after code changes
|
||||
```bash
|
||||
docker-compose build
|
||||
docker-compose up
|
||||
```
|
||||
|
||||
## Data Persistence
|
||||
|
||||
### Volume Mounts
|
||||
|
||||
Docker Compose mounts two volumes:
|
||||
|
||||
- `./data:/app/data` - Price data and trading records
|
||||
- `./logs:/app/logs` - MCP service logs
|
||||
|
||||
Data persists across container restarts. To reset:
|
||||
|
||||
```bash
|
||||
docker-compose down
|
||||
rm -rf data/agent_data/* logs/*
|
||||
docker-compose up
|
||||
```
|
||||
|
||||
### Backup Trading Data
|
||||
|
||||
```bash
|
||||
# Backup
|
||||
tar -czf ai-trader-backup-$(date +%Y%m%d).tar.gz data/agent_data/
|
||||
|
||||
# Restore
|
||||
tar -xzf ai-trader-backup-YYYYMMDD.tar.gz
|
||||
```
|
||||
|
||||
## Using Pre-built Images
|
||||
|
||||
### Pull from GitHub Container Registry
|
||||
|
||||
```bash
|
||||
docker pull ghcr.io/hkuds/ai-trader:latest
|
||||
```
|
||||
|
||||
### Run without Docker Compose
|
||||
|
||||
```bash
|
||||
docker run --env-file .env \
|
||||
-v $(pwd)/data:/app/data \
|
||||
-v $(pwd)/logs:/app/logs \
|
||||
-p 8000-8003:8000-8003 \
|
||||
ghcr.io/hkuds/ai-trader:latest
|
||||
```
|
||||
|
||||
### Specific version
|
||||
```bash
|
||||
docker pull ghcr.io/hkuds/ai-trader:v1.0.0
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### MCP Services Not Starting
|
||||
|
||||
**Symptom:** Container exits immediately or errors about ports
|
||||
|
||||
**Solutions:**
|
||||
- Check ports 8000-8003 not already in use: `lsof -i :8000-8003`
|
||||
- View container logs: `docker-compose logs`
|
||||
- Check MCP service logs: `cat logs/math.log`
|
||||
|
||||
### Missing API Keys
|
||||
|
||||
**Symptom:** Errors about missing environment variables
|
||||
|
||||
**Solutions:**
|
||||
- Verify `.env` file exists: `ls -la .env`
|
||||
- Check required variables set: `grep OPENAI_API_KEY .env`
|
||||
- Ensure `.env` in same directory as docker-compose.yml
|
||||
|
||||
### Data Fetch Failures
|
||||
|
||||
**Symptom:** Container exits during data preparation step
|
||||
|
||||
**Solutions:**
|
||||
- Verify Alpha Vantage API key valid
|
||||
- Check API rate limits (5 requests/minute for free tier)
|
||||
- View logs: `docker-compose logs | grep "Fetching and merging"`
|
||||
|
||||
### Permission Issues
|
||||
|
||||
**Symptom:** Cannot write to data or logs directories
|
||||
|
||||
**Solutions:**
|
||||
- Ensure directories writable: `chmod -R 755 data logs`
|
||||
- Check volume mount permissions
|
||||
- May need to create directories first: `mkdir -p data logs`
|
||||
|
||||
### Container Keeps Restarting
|
||||
|
||||
**Symptom:** Container restarts repeatedly
|
||||
|
||||
**Solutions:**
|
||||
- View logs to identify error: `docker-compose logs --tail=50`
|
||||
- Disable auto-restart: Comment out `restart: unless-stopped` in docker-compose.yml
|
||||
- Check if main.py exits with error
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Override Entrypoint
|
||||
|
||||
Run bash inside container for debugging:
|
||||
|
||||
```bash
|
||||
docker-compose run --entrypoint /bin/bash ai-trader
|
||||
```
|
||||
|
||||
### Build Multi-platform Images
|
||||
|
||||
For ARM64 (Apple Silicon) and AMD64:
|
||||
|
||||
```bash
|
||||
docker buildx build --platform linux/amd64,linux/arm64 -t ai-trader .
|
||||
```
|
||||
|
||||
### View Container Resource Usage
|
||||
|
||||
```bash
|
||||
docker stats ai-trader-app
|
||||
```
|
||||
|
||||
### Access MCP Services Directly
|
||||
|
||||
Services exposed on host:
|
||||
- Math: http://localhost:8000
|
||||
- Search: http://localhost:8001
|
||||
- Trade: http://localhost:8002
|
||||
- Price: http://localhost:8003
|
||||
|
||||
## Development Workflow
|
||||
|
||||
### Local Code Changes
|
||||
|
||||
1. Edit code in project root
|
||||
2. Rebuild image: `docker-compose build`
|
||||
3. Run updated container: `docker-compose up`
|
||||
|
||||
### Test Different Configurations
|
||||
|
||||
```bash
|
||||
# Create test config
|
||||
cp configs/default_config.json configs/test_config.json
|
||||
# Edit test_config.json
|
||||
|
||||
# Run with test config
|
||||
docker-compose run ai-trader configs/test_config.json
|
||||
```
|
||||
|
||||
## Production Deployment
|
||||
|
||||
For production use, consider:
|
||||
|
||||
1. **Use specific version tags** instead of `latest`
|
||||
2. **External secrets management** (AWS Secrets Manager, etc.)
|
||||
3. **Health checks** in docker-compose.yml
|
||||
4. **Resource limits** (CPU/memory)
|
||||
5. **Log aggregation** (ELK stack, CloudWatch)
|
||||
6. **Orchestration** (Kubernetes, Docker Swarm)
|
||||
|
||||
See design document in `docs/plans/2025-10-30-docker-deployment-design.md` for architecture details.
|
||||
149
docs/RELEASING.md
Normal file
149
docs/RELEASING.md
Normal file
@@ -0,0 +1,149 @@
|
||||
# Release Process
|
||||
|
||||
## Creating a New Release
|
||||
|
||||
### 1. Prepare Release
|
||||
|
||||
1. Ensure `main` branch is stable and tests pass
|
||||
2. Update version numbers if needed
|
||||
3. Update CHANGELOG.md with release notes
|
||||
|
||||
### 2. Create Release Tag
|
||||
|
||||
```bash
|
||||
# Ensure on main branch
|
||||
git checkout main
|
||||
git pull origin main
|
||||
|
||||
# Create annotated tag
|
||||
git tag -a v1.0.0 -m "Release v1.0.0: Docker deployment support"
|
||||
|
||||
# Push tag to trigger CI/CD
|
||||
git push origin v1.0.0
|
||||
```
|
||||
|
||||
### 3. GitHub Actions Automation
|
||||
|
||||
Tag push automatically triggers `.github/workflows/docker-release.yml`:
|
||||
|
||||
1. ✅ Checks out code
|
||||
2. ✅ Sets up Docker Buildx
|
||||
3. ✅ Logs into GitHub Container Registry
|
||||
4. ✅ Extracts version from tag
|
||||
5. ✅ Builds Docker image with caching
|
||||
6. ✅ Pushes to `ghcr.io/hkuds/ai-trader:VERSION`
|
||||
7. ✅ Pushes to `ghcr.io/hkuds/ai-trader:latest`
|
||||
|
||||
### 4. Verify Build
|
||||
|
||||
1. Check GitHub Actions: https://github.com/Xe138/AI-Trader/actions
|
||||
2. Verify workflow completed successfully (green checkmark)
|
||||
3. Check packages: https://github.com/Xe138/AI-Trader/pkgs/container/ai-trader
|
||||
|
||||
### 5. Test Release
|
||||
|
||||
```bash
|
||||
# Pull released image
|
||||
docker pull ghcr.io/hkuds/ai-trader:v1.0.0
|
||||
|
||||
# Test run
|
||||
docker run --env-file .env \
|
||||
-v $(pwd)/data:/app/data \
|
||||
ghcr.io/hkuds/ai-trader:v1.0.0
|
||||
```
|
||||
|
||||
### 6. Create GitHub Release (Optional)
|
||||
|
||||
1. Go to https://github.com/Xe138/AI-Trader/releases/new
|
||||
2. Select tag: `v1.0.0`
|
||||
3. Release title: `v1.0.0 - Docker Deployment Support`
|
||||
4. Add release notes:
|
||||
|
||||
```markdown
|
||||
## 🐳 Docker Deployment
|
||||
|
||||
This release adds full Docker support for easy deployment.
|
||||
|
||||
### Pull and Run
|
||||
|
||||
```bash
|
||||
docker pull ghcr.io/hkuds/ai-trader:v1.0.0
|
||||
docker run --env-file .env -v $(pwd)/data:/app/data ghcr.io/hkuds/ai-trader:v1.0.0
|
||||
```
|
||||
|
||||
Or use Docker Compose:
|
||||
|
||||
```bash
|
||||
docker-compose up
|
||||
```
|
||||
|
||||
See [docs/DOCKER.md](docs/DOCKER.md) for details.
|
||||
|
||||
### What's New
|
||||
- Docker containerization with single-container architecture
|
||||
- docker-compose.yml for easy orchestration
|
||||
- Automated CI/CD builds on release tags
|
||||
- Pre-built images on GitHub Container Registry
|
||||
```
|
||||
|
||||
5. Publish release
|
||||
|
||||
## Version Numbering
|
||||
|
||||
Use Semantic Versioning (SEMVER):
|
||||
|
||||
- `v1.0.0` - Major release (breaking changes)
|
||||
- `v1.1.0` - Minor release (new features, backward compatible)
|
||||
- `v1.1.1` - Patch release (bug fixes)
|
||||
|
||||
## Troubleshooting Releases
|
||||
|
||||
### Build Fails in GitHub Actions
|
||||
|
||||
1. Check Actions logs for error details
|
||||
2. Test local build: `docker build .`
|
||||
3. Fix issues and delete/recreate tag:
|
||||
|
||||
```bash
|
||||
# Delete tag
|
||||
git tag -d v1.0.0
|
||||
git push origin :refs/tags/v1.0.0
|
||||
|
||||
# Recreate after fixes
|
||||
git tag v1.0.0
|
||||
git push origin v1.0.0
|
||||
```
|
||||
|
||||
### Image Not Appearing in Registry
|
||||
|
||||
1. Check Actions permissions (Settings → Actions → General)
|
||||
2. Verify `packages: write` permission in workflow
|
||||
3. Ensure `GITHUB_TOKEN` has registry access
|
||||
|
||||
### Wrong Version Tagged
|
||||
|
||||
Delete and recreate:
|
||||
|
||||
```bash
|
||||
git tag -d v1.0.0
|
||||
git push origin :refs/tags/v1.0.0
|
||||
git tag v1.0.1
|
||||
git push origin v1.0.1
|
||||
```
|
||||
|
||||
## Manual Build and Push
|
||||
|
||||
If automated build fails, manual push:
|
||||
|
||||
```bash
|
||||
# Build locally
|
||||
docker build -t ghcr.io/hkuds/ai-trader:v1.0.0 .
|
||||
|
||||
# Login to GHCR
|
||||
echo $GITHUB_TOKEN | docker login ghcr.io -u USERNAME --password-stdin
|
||||
|
||||
# Push
|
||||
docker push ghcr.io/hkuds/ai-trader:v1.0.0
|
||||
docker tag ghcr.io/hkuds/ai-trader:v1.0.0 ghcr.io/hkuds/ai-trader:latest
|
||||
docker push ghcr.io/hkuds/ai-trader:latest
|
||||
```
|
||||
491
docs/plans/2025-10-30-docker-deployment-design.md
Normal file
491
docs/plans/2025-10-30-docker-deployment-design.md
Normal file
@@ -0,0 +1,491 @@
|
||||
# Docker Deployment and CI/CD Design
|
||||
|
||||
**Date:** 2025-10-30
|
||||
**Status:** Approved
|
||||
**Target:** Development/local testing environment
|
||||
|
||||
## Overview
|
||||
|
||||
Package AI-Trader as a Docker container with docker-compose orchestration and automated image builds via GitHub Actions on release tags. Focus on simplicity and ease of use for researchers and developers.
|
||||
|
||||
## Requirements
|
||||
|
||||
- **Primary Use Case:** Development and local testing
|
||||
- **Deployment Target:** Single monolithic container (all MCP services + trading agent)
|
||||
- **Secrets Management:** Environment variables (no mounted .env file)
|
||||
- **Data Strategy:** Fetch price data on container startup
|
||||
- **Container Registry:** GitHub Container Registry (ghcr.io)
|
||||
- **Trigger:** Build images automatically on release tag push (`v*` pattern)
|
||||
|
||||
## Architecture
|
||||
|
||||
### Components
|
||||
|
||||
1. **Dockerfile** - Builds Python 3.10 image with all dependencies
|
||||
2. **docker-compose.yml** - Orchestrates container with volume mounts and environment config
|
||||
3. **entrypoint.sh** - Sequential startup script (data fetch → MCP services → trading agent)
|
||||
4. **GitHub Actions Workflow** - Automated image build and push on release tags
|
||||
5. **.dockerignore** - Excludes unnecessary files from image
|
||||
6. **Documentation** - Docker usage guide and examples
|
||||
|
||||
### Execution Flow
|
||||
|
||||
```
|
||||
Container Start
|
||||
↓
|
||||
entrypoint.sh
|
||||
↓
|
||||
1. Fetch/merge price data (get_daily_price.py → merge_jsonl.py)
|
||||
↓
|
||||
2. Start MCP services in background (start_mcp_services.py)
|
||||
↓
|
||||
3. Wait 3 seconds for service stabilization
|
||||
↓
|
||||
4. Run trading agent (main.py with config)
|
||||
↓
|
||||
Container Exit → Cleanup MCP services
|
||||
```
|
||||
|
||||
## Detailed Design
|
||||
|
||||
### 1. Dockerfile
|
||||
|
||||
**Multi-stage build:**
|
||||
|
||||
```dockerfile
|
||||
# Base stage
|
||||
FROM python:3.10-slim as base
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install dependencies
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Application stage
|
||||
FROM base
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy application code
|
||||
COPY . .
|
||||
|
||||
# Create necessary directories
|
||||
RUN mkdir -p data logs data/agent_data
|
||||
|
||||
# Make entrypoint executable
|
||||
RUN chmod +x entrypoint.sh
|
||||
|
||||
# Expose MCP service ports
|
||||
EXPOSE 8000 8001 8002 8003
|
||||
|
||||
# Set Python to run unbuffered
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
# Use entrypoint script
|
||||
ENTRYPOINT ["./entrypoint.sh"]
|
||||
CMD ["configs/default_config.json"]
|
||||
```
|
||||
|
||||
**Key Features:**
|
||||
- `python:3.10-slim` base for smaller image size
|
||||
- Multi-stage for dependency caching
|
||||
- Non-root user NOT included (dev/testing focus, can add later)
|
||||
- Unbuffered Python output for real-time logs
|
||||
- Default config path with override support
|
||||
|
||||
### 2. docker-compose.yml
|
||||
|
||||
```yaml
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
ai-trader:
|
||||
build: .
|
||||
container_name: ai-trader-app
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
- ./logs:/app/logs
|
||||
environment:
|
||||
- OPENAI_API_BASE=${OPENAI_API_BASE}
|
||||
- OPENAI_API_KEY=${OPENAI_API_KEY}
|
||||
- ALPHAADVANTAGE_API_KEY=${ALPHAADVANTAGE_API_KEY}
|
||||
- JINA_API_KEY=${JINA_API_KEY}
|
||||
- RUNTIME_ENV_PATH=/app/data/runtime_env.json
|
||||
- MATH_HTTP_PORT=${MATH_HTTP_PORT:-8000}
|
||||
- SEARCH_HTTP_PORT=${SEARCH_HTTP_PORT:-8001}
|
||||
- TRADE_HTTP_PORT=${TRADE_HTTP_PORT:-8002}
|
||||
- GETPRICE_HTTP_PORT=${GETPRICE_HTTP_PORT:-8003}
|
||||
- AGENT_MAX_STEP=${AGENT_MAX_STEP:-30}
|
||||
ports:
|
||||
- "8000:8000"
|
||||
- "8001:8001"
|
||||
- "8002:8002"
|
||||
- "8003:8003"
|
||||
- "8888:8888" # Optional: web dashboard
|
||||
restart: unless-stopped
|
||||
```
|
||||
|
||||
**Key Features:**
|
||||
- Volume mounts for data/logs persistence
|
||||
- Environment variables interpolated from `.env` file (Docker Compose reads automatically)
|
||||
- No `.env` file mounted into container (cleaner separation)
|
||||
- Default port values with override support
|
||||
- Restart policy for recovery
|
||||
|
||||
### 3. entrypoint.sh
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
set -e # Exit on any error
|
||||
|
||||
echo "🚀 Starting AI-Trader..."
|
||||
|
||||
# Step 1: Data preparation
|
||||
echo "📊 Fetching and merging price data..."
|
||||
cd /app/data
|
||||
python get_daily_price.py
|
||||
python merge_jsonl.py
|
||||
cd /app
|
||||
|
||||
# Step 2: Start MCP services in background
|
||||
echo "🔧 Starting MCP services..."
|
||||
cd /app/agent_tools
|
||||
python start_mcp_services.py &
|
||||
MCP_PID=$!
|
||||
cd /app
|
||||
|
||||
# Step 3: Wait for services to initialize
|
||||
echo "⏳ Waiting for MCP services to start..."
|
||||
sleep 3
|
||||
|
||||
# Step 4: Run trading agent with config file
|
||||
echo "🤖 Starting trading agent..."
|
||||
CONFIG_FILE="${1:-configs/default_config.json}"
|
||||
python main.py "$CONFIG_FILE"
|
||||
|
||||
# Cleanup on exit
|
||||
trap "echo '🛑 Stopping MCP services...'; kill $MCP_PID 2>/dev/null" EXIT
|
||||
```
|
||||
|
||||
**Key Features:**
|
||||
- Sequential execution with clear logging
|
||||
- MCP services run in background with PID capture
|
||||
- Trap ensures cleanup on container exit
|
||||
- Config file path as argument (defaults to `configs/default_config.json`)
|
||||
- Fail-fast with `set -e`
|
||||
|
||||
### 4. GitHub Actions Workflow
|
||||
|
||||
**File:** `.github/workflows/docker-release.yml`
|
||||
|
||||
```yaml
|
||||
name: Build and Push Docker Image
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*' # Triggers on v1.0.0, v2.1.3, etc.
|
||||
workflow_dispatch: # Manual trigger option
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract version from tag
|
||||
id: meta
|
||||
run: |
|
||||
VERSION=${GITHUB_REF#refs/tags/v}
|
||||
echo "version=$VERSION" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
tags: |
|
||||
ghcr.io/${{ github.repository_owner }}/ai-trader:${{ steps.meta.outputs.version }}
|
||||
ghcr.io/${{ github.repository_owner }}/ai-trader:latest
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
```
|
||||
|
||||
**Key Features:**
|
||||
- Triggers on `v*` tags (e.g., `git tag v1.0.0 && git push origin v1.0.0`)
|
||||
- Manual dispatch option for testing
|
||||
- Uses `GITHUB_TOKEN` (automatically provided, no secrets needed)
|
||||
- Builds with caching for faster builds
|
||||
- Tags both version and `latest`
|
||||
- Multi-platform support possible by adding `platforms: linux/amd64,linux/arm64`
|
||||
|
||||
### 5. .dockerignore
|
||||
|
||||
```
|
||||
# Version control
|
||||
.git/
|
||||
.gitignore
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
venv/
|
||||
env/
|
||||
ENV/
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# Environment and secrets
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# Data files (fetched at runtime)
|
||||
data/*.json
|
||||
data/agent_data/
|
||||
data/merged.jsonl
|
||||
|
||||
# Logs
|
||||
logs/
|
||||
*.log
|
||||
|
||||
# Runtime state
|
||||
runtime_env.json
|
||||
|
||||
# Documentation (not needed in image)
|
||||
*.md
|
||||
docs/
|
||||
!README.md
|
||||
|
||||
# CI/CD
|
||||
.github/
|
||||
```
|
||||
|
||||
**Purpose:**
|
||||
- Reduces image size
|
||||
- Keeps secrets out of image
|
||||
- Excludes generated files
|
||||
- Keeps only necessary source code and scripts
|
||||
|
||||
## Documentation Updates
|
||||
|
||||
### New File: docs/DOCKER.md
|
||||
|
||||
Create comprehensive Docker usage guide including:
|
||||
|
||||
1. **Quick Start**
|
||||
```bash
|
||||
cp .env.example .env
|
||||
# Edit .env with your API keys
|
||||
docker-compose up
|
||||
```
|
||||
|
||||
2. **Configuration**
|
||||
- Required environment variables
|
||||
- Optional configuration overrides
|
||||
- Custom config file usage
|
||||
|
||||
3. **Usage Examples**
|
||||
```bash
|
||||
# Run with default config
|
||||
docker-compose up
|
||||
|
||||
# Run with custom config
|
||||
docker-compose run ai-trader configs/my_config.json
|
||||
|
||||
# View logs
|
||||
docker-compose logs -f
|
||||
|
||||
# Stop and clean up
|
||||
docker-compose down
|
||||
```
|
||||
|
||||
4. **Data Persistence**
|
||||
- How volume mounts work
|
||||
- Where data is stored
|
||||
- How to backup/restore
|
||||
|
||||
5. **Troubleshooting**
|
||||
- MCP services not starting → Check logs, verify ports available
|
||||
- Missing API keys → Check .env file
|
||||
- Data fetch failures → API rate limits or invalid keys
|
||||
- Permission issues → Volume mount permissions
|
||||
|
||||
6. **Using Pre-built Images**
|
||||
```bash
|
||||
docker pull ghcr.io/hkuds/ai-trader:latest
|
||||
docker run --env-file .env -v $(pwd)/data:/app/data ghcr.io/hkuds/ai-trader:latest
|
||||
```
|
||||
|
||||
### Update .env.example
|
||||
|
||||
Add/clarify Docker-specific variables:
|
||||
|
||||
```bash
|
||||
# AI Model API Configuration
|
||||
OPENAI_API_BASE=https://your-openai-proxy.com/v1
|
||||
OPENAI_API_KEY=your_openai_key
|
||||
|
||||
# Data Source Configuration
|
||||
ALPHAADVANTAGE_API_KEY=your_alpha_vantage_key
|
||||
JINA_API_KEY=your_jina_api_key
|
||||
|
||||
# System Configuration (Docker defaults)
|
||||
RUNTIME_ENV_PATH=/app/data/runtime_env.json
|
||||
|
||||
# MCP Service Ports
|
||||
MATH_HTTP_PORT=8000
|
||||
SEARCH_HTTP_PORT=8001
|
||||
TRADE_HTTP_PORT=8002
|
||||
GETPRICE_HTTP_PORT=8003
|
||||
|
||||
# Agent Configuration
|
||||
AGENT_MAX_STEP=30
|
||||
```
|
||||
|
||||
### Update Main README.md
|
||||
|
||||
Add Docker section after "Quick Start":
|
||||
|
||||
```markdown
|
||||
## Docker Deployment
|
||||
|
||||
### Using Docker Compose (Recommended)
|
||||
|
||||
```bash
|
||||
# Setup environment
|
||||
cp .env.example .env
|
||||
# Edit .env with your API keys
|
||||
|
||||
# Run with docker-compose
|
||||
docker-compose up
|
||||
```
|
||||
|
||||
### Using Pre-built Images
|
||||
|
||||
```bash
|
||||
# Pull latest image
|
||||
docker pull ghcr.io/hkuds/ai-trader:latest
|
||||
|
||||
# Run container
|
||||
docker run --env-file .env \
|
||||
-v $(pwd)/data:/app/data \
|
||||
-v $(pwd)/logs:/app/logs \
|
||||
ghcr.io/hkuds/ai-trader:latest
|
||||
```
|
||||
|
||||
See [docs/DOCKER.md](docs/DOCKER.md) for detailed Docker usage guide.
|
||||
```
|
||||
|
||||
## Release Process
|
||||
|
||||
### For Maintainers
|
||||
|
||||
1. **Prepare release:**
|
||||
```bash
|
||||
# Ensure main branch is ready
|
||||
git checkout main
|
||||
git pull origin main
|
||||
```
|
||||
|
||||
2. **Create and push tag:**
|
||||
```bash
|
||||
git tag v1.0.0
|
||||
git push origin v1.0.0
|
||||
```
|
||||
|
||||
3. **GitHub Actions automatically:**
|
||||
- Builds Docker image
|
||||
- Tags with version and `latest`
|
||||
- Pushes to `ghcr.io/hkuds/ai-trader`
|
||||
|
||||
4. **Verify build:**
|
||||
- Check Actions tab for build status
|
||||
- Test pull: `docker pull ghcr.io/hkuds/ai-trader:v1.0.0`
|
||||
|
||||
5. **Optional: Create GitHub Release**
|
||||
- Add release notes
|
||||
- Include Docker pull command
|
||||
|
||||
### For Users
|
||||
|
||||
```bash
|
||||
# Pull specific version
|
||||
docker pull ghcr.io/hkuds/ai-trader:v1.0.0
|
||||
|
||||
# Or always get latest
|
||||
docker pull ghcr.io/hkuds/ai-trader:latest
|
||||
```
|
||||
|
||||
## Implementation Checklist
|
||||
|
||||
- [ ] Create Dockerfile with multi-stage build
|
||||
- [ ] Create docker-compose.yml with volume mounts and environment config
|
||||
- [ ] Create entrypoint.sh with sequential startup logic
|
||||
- [ ] Create .dockerignore to exclude unnecessary files
|
||||
- [ ] Create .github/workflows/docker-release.yml for CI/CD
|
||||
- [ ] Create docs/DOCKER.md with comprehensive usage guide
|
||||
- [ ] Update .env.example with Docker-specific variables
|
||||
- [ ] Update main README.md with Docker deployment section
|
||||
- [ ] Test local build: `docker-compose build`
|
||||
- [ ] Test local run: `docker-compose up`
|
||||
- [ ] Test with custom config
|
||||
- [ ] Verify data persistence across container restarts
|
||||
- [ ] Test GitHub Actions workflow (create test tag)
|
||||
- [ ] Verify image pushed to ghcr.io
|
||||
- [ ] Test pulling and running pre-built image
|
||||
- [ ] Update CLAUDE.md with Docker commands
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
Possible improvements for production use:
|
||||
|
||||
1. **Multi-container Architecture**
|
||||
- Separate containers for each MCP service
|
||||
- Better isolation and independent scaling
|
||||
- More complex orchestration
|
||||
|
||||
2. **Security Hardening**
|
||||
- Non-root user in container
|
||||
- Docker secrets for production
|
||||
- Read-only filesystem where possible
|
||||
|
||||
3. **Monitoring**
|
||||
- Health checks for MCP services
|
||||
- Prometheus metrics export
|
||||
- Logging aggregation
|
||||
|
||||
4. **Optimization**
|
||||
- Multi-platform builds (ARM64 support)
|
||||
- Smaller base image (alpine)
|
||||
- Layer caching optimization
|
||||
|
||||
5. **Development Tools**
|
||||
- docker-compose.dev.yml with hot reload
|
||||
- Debug container with additional tools
|
||||
- Integration test container
|
||||
|
||||
These are deferred to keep initial implementation simple and focused on development/testing use cases.
|
||||
1183
docs/plans/2025-10-30-docker-deployment.md
Normal file
1183
docs/plans/2025-10-30-docker-deployment.md
Normal file
File diff suppressed because it is too large
Load Diff
102
docs/plans/docker-test-results.txt
Normal file
102
docs/plans/docker-test-results.txt
Normal file
@@ -0,0 +1,102 @@
|
||||
Docker Build Test Results
|
||||
==========================
|
||||
Date: 2025-10-30
|
||||
Branch: docker-deployment
|
||||
Working Directory: /home/bballou/AI-Trader/.worktrees/docker-deployment
|
||||
|
||||
Test 1: Docker Image Build
|
||||
---------------------------
|
||||
Command: docker-compose build
|
||||
Status: SUCCESS
|
||||
Result: Successfully built image 7b36b8f4c0e9
|
||||
|
||||
Build Output Summary:
|
||||
- Base image: python:3.10-slim
|
||||
- Build stages: Multi-stage build (base + application)
|
||||
- Dependencies installed successfully from requirements.txt
|
||||
- Application code copied
|
||||
- Directories created: data, logs, data/agent_data
|
||||
- Entrypoint script made executable
|
||||
- Ports exposed: 8000, 8001, 8002, 8003, 8888
|
||||
- Environment: PYTHONUNBUFFERED=1 set
|
||||
- Image size: 266MB
|
||||
- Build time: ~2 minutes (including dependency installation)
|
||||
|
||||
Key packages installed:
|
||||
- langchain==1.0.2
|
||||
- langchain-openai==1.0.1
|
||||
- langchain-mcp-adapters>=0.1.0
|
||||
- fastmcp==2.12.5
|
||||
- langgraph<1.1.0,>=1.0.0
|
||||
- pydantic<3.0.0,>=2.7.4
|
||||
- openai<3.0.0,>=1.109.1
|
||||
- All dependencies resolved without conflicts
|
||||
|
||||
Test 2: Image Verification
|
||||
---------------------------
|
||||
Command: docker images | grep ai-trader
|
||||
Status: SUCCESS
|
||||
Result: docker-deployment_ai-trader latest 7b36b8f4c0e9 9 seconds ago 266MB
|
||||
|
||||
Image Details:
|
||||
- Repository: docker-deployment_ai-trader
|
||||
- Tag: latest
|
||||
- Image ID: 7b36b8f4c0e9
|
||||
- Created: Just now
|
||||
- Size: 266MB (reasonable for Python 3.10 + ML dependencies)
|
||||
|
||||
Test 3: Configuration Parsing (Dry-Run)
|
||||
----------------------------------------
|
||||
Command: docker-compose --env-file .env.test config
|
||||
Status: SUCCESS
|
||||
Result: Configuration parsed correctly without errors
|
||||
|
||||
Test .env.test contents:
|
||||
OPENAI_API_KEY=test
|
||||
ALPHAADVANTAGE_API_KEY=test
|
||||
JINA_API_KEY=test
|
||||
RUNTIME_ENV_PATH=/app/data/runtime_env.json
|
||||
|
||||
Parsed Configuration:
|
||||
- Service name: ai-trader
|
||||
- Container name: ai-trader-app
|
||||
- Build context: /home/bballou/AI-Trader/.worktrees/docker-deployment
|
||||
- Environment variables correctly injected:
|
||||
* AGENT_MAX_STEP: '30' (default)
|
||||
* ALPHAADVANTAGE_API_KEY: test
|
||||
* GETPRICE_HTTP_PORT: '8003' (default)
|
||||
* JINA_API_KEY: test
|
||||
* MATH_HTTP_PORT: '8000' (default)
|
||||
* OPENAI_API_BASE: '' (not set, defaulted to blank)
|
||||
* OPENAI_API_KEY: test
|
||||
* RUNTIME_ENV_PATH: /app/data/runtime_env.json
|
||||
* SEARCH_HTTP_PORT: '8001' (default)
|
||||
* TRADE_HTTP_PORT: '8002' (default)
|
||||
- Ports correctly mapped: 8000, 8001, 8002, 8003, 8888
|
||||
- Volumes correctly configured:
|
||||
* ./data:/app/data:rw
|
||||
* ./logs:/app/logs:rw
|
||||
- Restart policy: unless-stopped
|
||||
- Docker Compose version: 3.8
|
||||
|
||||
Summary
|
||||
-------
|
||||
All Docker build tests PASSED successfully:
|
||||
✓ Docker image builds without errors
|
||||
✓ Image created with reasonable size (266MB)
|
||||
✓ Multi-stage build optimizes layer caching
|
||||
✓ All Python dependencies install correctly
|
||||
✓ Configuration parsing works with test environment
|
||||
✓ Environment variables properly injected
|
||||
✓ Volume mounts configured correctly
|
||||
✓ Port mappings set up correctly
|
||||
✓ Restart policy configured
|
||||
|
||||
No issues encountered during local Docker build testing.
|
||||
The Docker deployment is ready for use.
|
||||
|
||||
Next Steps:
|
||||
1. Test actual container startup with valid API keys
|
||||
2. Verify MCP services start correctly in container
|
||||
3. Test trading agent execution
|
||||
4. Consider creating test tag for GitHub Actions CI/CD verification
|
||||
30
entrypoint.sh
Executable file
30
entrypoint.sh
Executable file
@@ -0,0 +1,30 @@
|
||||
#!/bin/bash
|
||||
set -e # Exit on any error
|
||||
|
||||
echo "🚀 Starting AI-Trader..."
|
||||
|
||||
# Step 1: Data preparation
|
||||
echo "📊 Fetching and merging price data..."
|
||||
cd /app/data
|
||||
python get_daily_price.py
|
||||
python merge_jsonl.py
|
||||
cd /app
|
||||
|
||||
# Step 2: Start MCP services in background
|
||||
echo "🔧 Starting MCP services..."
|
||||
cd /app/agent_tools
|
||||
python start_mcp_services.py &
|
||||
MCP_PID=$!
|
||||
cd /app
|
||||
|
||||
# Step 3: Wait for services to initialize
|
||||
echo "⏳ Waiting for MCP services to start..."
|
||||
sleep 3
|
||||
|
||||
# Step 4: Run trading agent with config file
|
||||
echo "🤖 Starting trading agent..."
|
||||
CONFIG_FILE="${1:-configs/default_config.json}"
|
||||
python main.py "$CONFIG_FILE"
|
||||
|
||||
# Cleanup on exit
|
||||
trap "echo '🛑 Stopping MCP services...'; kill $MCP_PID 2>/dev/null; exit 0" EXIT SIGTERM SIGINT
|
||||
Reference in New Issue
Block a user