mirror of
https://github.com/Xe138/AI-Trader.git
synced 2026-04-01 17:17:24 -04:00
Bill
f104164187
Complete implementation of reasoning logs retrieval system that replaces JSONL file-based logging with database-only storage. Database Changes: - Add trading_sessions table (one record per model-day) - Add reasoning_logs table (conversation history with summaries) - Add session_id column to positions table - Add indexes for query performance Agent Changes: - Add conversation history tracking to BaseAgent - Add AI-powered summary generation using same model - Remove JSONL logging code (_log_message, _setup_logging) - Preserve in-memory conversation tracking ModelDayExecutor Changes: - Create trading session at start of execution - Store reasoning logs with AI-generated summaries - Update session summary after completion - Link positions to sessions via session_id API Changes: - Add GET /reasoning endpoint with filters (job_id, date, model) - Support include_full_conversation parameter - Return both summaries and full conversation on demand - Include deployment mode info in responses Documentation: - Add complete API reference for GET /reasoning - Add design document with architecture details - Add implementation guide with step-by-step tasks - Update Python and TypeScript client examples Testing: - Add 6 tests for conversation history tracking - Add 4 tests for summary generation - Add 5 tests for model_day_executor integration - Add 8 tests for GET /reasoning endpoint - Add 9 integration tests for E2E flow - Update existing tests for schema changes All 32 new feature tests passing. Total: 285 tests passing.
528 lines
19 KiB
Python
528 lines
19 KiB
Python
"""
|
|
End-to-end integration tests for reasoning logs API feature.
|
|
|
|
Tests the complete flow from simulation trigger to reasoning retrieval.
|
|
|
|
These tests verify:
|
|
- Trading sessions are created with session_id
|
|
- Reasoning logs are stored in database
|
|
- Full conversation history is captured
|
|
- Message summaries are generated
|
|
- GET /reasoning endpoint returns correct data
|
|
- Query filters work (job_id, date, model)
|
|
- include_full_conversation parameter works correctly
|
|
- Positions are linked to sessions
|
|
"""
|
|
|
|
import pytest
|
|
import time
|
|
import os
|
|
import json
|
|
from fastapi.testclient import TestClient
|
|
from pathlib import Path
|
|
|
|
|
|
@pytest.fixture
|
|
def dev_client(tmp_path):
|
|
"""Create test client with DEV mode and clean database."""
|
|
# Set DEV mode environment
|
|
os.environ["DEPLOYMENT_MODE"] = "DEV"
|
|
os.environ["PRESERVE_DEV_DATA"] = "false"
|
|
# Disable auto-download - we'll pre-populate test data
|
|
os.environ["AUTO_DOWNLOAD_PRICE_DATA"] = "false"
|
|
|
|
# Import after setting environment
|
|
from api.main import create_app
|
|
from api.database import initialize_dev_database, get_db_path, get_db_connection
|
|
|
|
# Create dev database
|
|
db_path = str(tmp_path / "test_trading.db")
|
|
dev_db_path = get_db_path(db_path)
|
|
initialize_dev_database(dev_db_path)
|
|
|
|
# Pre-populate price data for test dates to avoid needing API key
|
|
_populate_test_price_data(dev_db_path)
|
|
|
|
# Create test config with mock model
|
|
test_config = tmp_path / "test_config.json"
|
|
test_config.write_text(json.dumps({
|
|
"agent_type": "BaseAgent",
|
|
"date_range": {"init_date": "2025-01-16", "end_date": "2025-01-17"},
|
|
"models": [
|
|
{
|
|
"name": "Test Mock Model",
|
|
"basemodel": "mock/test-trader",
|
|
"signature": "test-mock",
|
|
"enabled": True
|
|
}
|
|
],
|
|
"agent_config": {
|
|
"max_steps": 10,
|
|
"initial_cash": 10000.0,
|
|
"max_retries": 1,
|
|
"base_delay": 0.1
|
|
},
|
|
"log_config": {
|
|
"log_path": str(tmp_path / "dev_agent_data")
|
|
}
|
|
}))
|
|
|
|
# Create app with test config
|
|
app = create_app(db_path=dev_db_path, config_path=str(test_config))
|
|
|
|
# IMPORTANT: Do NOT set test_mode=True to allow worker to actually run
|
|
# This is an integration test - we want the full flow
|
|
|
|
client = TestClient(app)
|
|
client.db_path = dev_db_path
|
|
client.config_path = str(test_config)
|
|
|
|
yield client
|
|
|
|
# Cleanup
|
|
os.environ.pop("DEPLOYMENT_MODE", None)
|
|
os.environ.pop("PRESERVE_DEV_DATA", None)
|
|
os.environ.pop("AUTO_DOWNLOAD_PRICE_DATA", None)
|
|
|
|
|
|
def _populate_test_price_data(db_path: str):
|
|
"""
|
|
Pre-populate test price data in database.
|
|
|
|
This avoids needing Alpha Vantage API key for integration tests.
|
|
Adds mock price data for all NASDAQ 100 stocks on test dates.
|
|
"""
|
|
from api.database import get_db_connection
|
|
from datetime import datetime
|
|
|
|
# All NASDAQ 100 symbols (must match configs/nasdaq100_symbols.json)
|
|
symbols = [
|
|
"NVDA", "MSFT", "AAPL", "GOOG", "GOOGL", "AMZN", "META", "AVGO", "TSLA",
|
|
"NFLX", "PLTR", "COST", "ASML", "AMD", "CSCO", "AZN", "TMUS", "MU", "LIN",
|
|
"PEP", "SHOP", "APP", "INTU", "AMAT", "LRCX", "PDD", "QCOM", "ARM", "INTC",
|
|
"BKNG", "AMGN", "TXN", "ISRG", "GILD", "KLAC", "PANW", "ADBE", "HON",
|
|
"CRWD", "CEG", "ADI", "ADP", "DASH", "CMCSA", "VRTX", "MELI", "SBUX",
|
|
"CDNS", "ORLY", "SNPS", "MSTR", "MDLZ", "ABNB", "MRVL", "CTAS", "TRI",
|
|
"MAR", "MNST", "CSX", "ADSK", "PYPL", "FTNT", "AEP", "WDAY", "REGN", "ROP",
|
|
"NXPI", "DDOG", "AXON", "ROST", "IDXX", "EA", "PCAR", "FAST", "EXC", "TTWO",
|
|
"XEL", "ZS", "PAYX", "WBD", "BKR", "CPRT", "CCEP", "FANG", "TEAM", "CHTR",
|
|
"KDP", "MCHP", "GEHC", "VRSK", "CTSH", "CSGP", "KHC", "ODFL", "DXCM", "TTD",
|
|
"ON", "BIIB", "LULU", "CDW", "GFS", "QQQ"
|
|
]
|
|
|
|
# Test dates
|
|
test_dates = ["2025-01-16", "2025-01-17"]
|
|
|
|
conn = get_db_connection(db_path)
|
|
cursor = conn.cursor()
|
|
|
|
for symbol in symbols:
|
|
for date in test_dates:
|
|
# Insert mock price data
|
|
cursor.execute("""
|
|
INSERT OR IGNORE INTO price_data
|
|
(symbol, date, open, high, low, close, volume, created_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
""", (
|
|
symbol,
|
|
date,
|
|
100.0, # open
|
|
105.0, # high
|
|
98.0, # low
|
|
102.0, # close
|
|
1000000, # volume
|
|
datetime.utcnow().isoformat() + "Z"
|
|
))
|
|
|
|
# Add coverage record
|
|
cursor.execute("""
|
|
INSERT OR IGNORE INTO price_data_coverage
|
|
(symbol, start_date, end_date, downloaded_at, source)
|
|
VALUES (?, ?, ?, ?, ?)
|
|
""", (
|
|
symbol,
|
|
"2025-01-16",
|
|
"2025-01-17",
|
|
datetime.utcnow().isoformat() + "Z",
|
|
"test_fixture"
|
|
))
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
|
|
@pytest.mark.integration
|
|
@pytest.mark.skipif(
|
|
os.getenv("SKIP_INTEGRATION_TESTS") == "true",
|
|
reason="Skipping integration tests that require full environment"
|
|
)
|
|
class TestReasoningLogsE2E:
|
|
"""End-to-end tests for reasoning logs feature."""
|
|
|
|
def test_simulation_stores_reasoning_logs(self, dev_client):
|
|
"""
|
|
Test that running a simulation creates reasoning logs in database.
|
|
|
|
This is the main E2E test that verifies:
|
|
1. Simulation can be triggered
|
|
2. Worker processes the job
|
|
3. Trading sessions are created
|
|
4. Reasoning logs are stored
|
|
5. GET /reasoning returns the data
|
|
|
|
NOTE: This test requires MCP services to be running. It will skip if services are unavailable.
|
|
"""
|
|
# Skip if MCP services not available
|
|
try:
|
|
from agent.base_agent.base_agent import BaseAgent
|
|
except ImportError as e:
|
|
pytest.skip(f"Cannot import BaseAgent: {e}")
|
|
|
|
# Skip test - requires MCP services running
|
|
# This is a known limitation for integration tests
|
|
pytest.skip(
|
|
"Test requires MCP services running. "
|
|
"Use test_reasoning_api_with_mocked_data() instead for automated testing."
|
|
)
|
|
|
|
def test_reasoning_api_with_mocked_data(self, dev_client):
|
|
"""
|
|
Test GET /reasoning API with pre-populated database data.
|
|
|
|
This test verifies the API layer works correctly without requiring
|
|
a full simulation run or MCP services.
|
|
"""
|
|
from api.database import get_db_connection
|
|
from datetime import datetime
|
|
|
|
# Populate test data directly in database
|
|
conn = get_db_connection(dev_client.db_path)
|
|
cursor = conn.cursor()
|
|
|
|
# Create a job
|
|
job_id = "test-job-123"
|
|
cursor.execute("""
|
|
INSERT INTO jobs (job_id, config_path, status, date_range, models, created_at)
|
|
VALUES (?, ?, ?, ?, ?, ?)
|
|
""", (job_id, "test_config.json", "completed", "2025-01-16", '["test-mock"]',
|
|
datetime.utcnow().isoformat() + "Z"))
|
|
|
|
# Create a trading session
|
|
cursor.execute("""
|
|
INSERT INTO trading_sessions
|
|
(job_id, date, model, session_summary, started_at, completed_at, total_messages)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
""", (
|
|
job_id,
|
|
"2025-01-16",
|
|
"test-mock",
|
|
"Analyzed market conditions and executed buy order for AAPL",
|
|
datetime.utcnow().isoformat() + "Z",
|
|
datetime.utcnow().isoformat() + "Z",
|
|
5
|
|
))
|
|
|
|
session_id = cursor.lastrowid
|
|
|
|
# Create reasoning logs
|
|
messages = [
|
|
{
|
|
"session_id": session_id,
|
|
"message_index": 0,
|
|
"role": "user",
|
|
"content": "You are a trading agent. Analyze the market...",
|
|
"summary": None,
|
|
"tool_name": None,
|
|
"tool_input": None,
|
|
"timestamp": datetime.utcnow().isoformat() + "Z"
|
|
},
|
|
{
|
|
"session_id": session_id,
|
|
"message_index": 1,
|
|
"role": "assistant",
|
|
"content": "I will analyze the market and make trading decisions...",
|
|
"summary": "Agent analyzed market conditions",
|
|
"tool_name": None,
|
|
"tool_input": None,
|
|
"timestamp": datetime.utcnow().isoformat() + "Z"
|
|
},
|
|
{
|
|
"session_id": session_id,
|
|
"message_index": 2,
|
|
"role": "tool",
|
|
"content": "Price of AAPL: $150.00",
|
|
"summary": None,
|
|
"tool_name": "get_price",
|
|
"tool_input": json.dumps({"symbol": "AAPL"}),
|
|
"timestamp": datetime.utcnow().isoformat() + "Z"
|
|
},
|
|
{
|
|
"session_id": session_id,
|
|
"message_index": 3,
|
|
"role": "assistant",
|
|
"content": "Based on analysis, I will buy AAPL...",
|
|
"summary": "Agent decided to buy AAPL",
|
|
"tool_name": None,
|
|
"tool_input": None,
|
|
"timestamp": datetime.utcnow().isoformat() + "Z"
|
|
},
|
|
{
|
|
"session_id": session_id,
|
|
"message_index": 4,
|
|
"role": "tool",
|
|
"content": "Successfully bought 10 shares of AAPL",
|
|
"summary": None,
|
|
"tool_name": "buy",
|
|
"tool_input": json.dumps({"symbol": "AAPL", "amount": 10}),
|
|
"timestamp": datetime.utcnow().isoformat() + "Z"
|
|
}
|
|
]
|
|
|
|
for msg in messages:
|
|
cursor.execute("""
|
|
INSERT INTO reasoning_logs
|
|
(session_id, message_index, role, content, summary, tool_name, tool_input, timestamp)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
""", (
|
|
msg["session_id"], msg["message_index"], msg["role"],
|
|
msg["content"], msg["summary"], msg["tool_name"],
|
|
msg["tool_input"], msg["timestamp"]
|
|
))
|
|
|
|
# Create positions linked to session
|
|
cursor.execute("""
|
|
INSERT INTO positions
|
|
(job_id, date, model, action_id, action_type, symbol, amount, price, cash, portfolio_value,
|
|
daily_profit, daily_return_pct, created_at, session_id)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""", (
|
|
job_id, "2025-01-16", "test-mock", 1, "buy", "AAPL", 10, 150.0,
|
|
8500.0, 10000.0, 0.0, 0.0, datetime.utcnow().isoformat() + "Z", session_id
|
|
))
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
# Query reasoning endpoint (summary mode)
|
|
reasoning_response = dev_client.get(f"/reasoning?job_id={job_id}")
|
|
|
|
assert reasoning_response.status_code == 200
|
|
reasoning_data = reasoning_response.json()
|
|
|
|
# Verify response structure
|
|
assert "sessions" in reasoning_data
|
|
assert "count" in reasoning_data
|
|
assert reasoning_data["count"] == 1
|
|
assert reasoning_data["is_dev_mode"] is True
|
|
|
|
# Verify trading session structure
|
|
session = reasoning_data["sessions"][0]
|
|
assert session["session_id"] == session_id
|
|
assert session["job_id"] == job_id
|
|
assert session["date"] == "2025-01-16"
|
|
assert session["model"] == "test-mock"
|
|
assert session["session_summary"] == "Analyzed market conditions and executed buy order for AAPL"
|
|
assert session["total_messages"] == 5
|
|
|
|
# Verify positions are linked to session
|
|
assert "positions" in session
|
|
assert len(session["positions"]) == 1
|
|
position = session["positions"][0]
|
|
assert position["action_id"] == 1
|
|
assert position["action_type"] == "buy"
|
|
assert position["symbol"] == "AAPL"
|
|
assert position["amount"] == 10
|
|
assert position["price"] == 150.0
|
|
assert position["cash_after"] == 8500.0
|
|
assert position["portfolio_value"] == 10000.0
|
|
|
|
# Verify conversation is NOT included in summary mode
|
|
assert session["conversation"] is None
|
|
|
|
# Query again with full conversation
|
|
full_response = dev_client.get(
|
|
f"/reasoning?job_id={job_id}&include_full_conversation=true"
|
|
)
|
|
assert full_response.status_code == 200
|
|
full_data = full_response.json()
|
|
session_full = full_data["sessions"][0]
|
|
|
|
# Verify full conversation is included
|
|
assert session_full["conversation"] is not None
|
|
assert len(session_full["conversation"]) == 5
|
|
|
|
# Verify conversation messages
|
|
conv = session_full["conversation"]
|
|
assert conv[0]["role"] == "user"
|
|
assert conv[0]["message_index"] == 0
|
|
assert conv[0]["summary"] is None # User messages don't have summaries
|
|
|
|
assert conv[1]["role"] == "assistant"
|
|
assert conv[1]["message_index"] == 1
|
|
assert conv[1]["summary"] == "Agent analyzed market conditions"
|
|
|
|
assert conv[2]["role"] == "tool"
|
|
assert conv[2]["message_index"] == 2
|
|
assert conv[2]["tool_name"] == "get_price"
|
|
assert conv[2]["tool_input"] == json.dumps({"symbol": "AAPL"})
|
|
|
|
assert conv[3]["role"] == "assistant"
|
|
assert conv[3]["message_index"] == 3
|
|
assert conv[3]["summary"] == "Agent decided to buy AAPL"
|
|
|
|
assert conv[4]["role"] == "tool"
|
|
assert conv[4]["message_index"] == 4
|
|
assert conv[4]["tool_name"] == "buy"
|
|
|
|
def test_reasoning_endpoint_date_filter(self, dev_client):
|
|
"""Test GET /reasoning date filter works correctly."""
|
|
# This test requires actual data - skip if no data available
|
|
response = dev_client.get("/reasoning?date=2025-01-16")
|
|
|
|
# Should either return 404 (no data) or 200 with filtered data
|
|
assert response.status_code in [200, 404]
|
|
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
for session in data["sessions"]:
|
|
assert session["date"] == "2025-01-16"
|
|
|
|
def test_reasoning_endpoint_model_filter(self, dev_client):
|
|
"""Test GET /reasoning model filter works correctly."""
|
|
response = dev_client.get("/reasoning?model=test-mock")
|
|
|
|
# Should either return 404 (no data) or 200 with filtered data
|
|
assert response.status_code in [200, 404]
|
|
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
for session in data["sessions"]:
|
|
assert session["model"] == "test-mock"
|
|
|
|
def test_reasoning_endpoint_combined_filters(self, dev_client):
|
|
"""Test GET /reasoning with multiple filters."""
|
|
response = dev_client.get(
|
|
"/reasoning?date=2025-01-16&model=test-mock"
|
|
)
|
|
|
|
# Should either return 404 (no data) or 200 with filtered data
|
|
assert response.status_code in [200, 404]
|
|
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
for session in data["sessions"]:
|
|
assert session["date"] == "2025-01-16"
|
|
assert session["model"] == "test-mock"
|
|
|
|
def test_reasoning_endpoint_invalid_date_format(self, dev_client):
|
|
"""Test GET /reasoning rejects invalid date format."""
|
|
response = dev_client.get("/reasoning?date=invalid-date")
|
|
|
|
assert response.status_code == 400
|
|
assert "Invalid date format" in response.json()["detail"]
|
|
|
|
def test_reasoning_endpoint_no_sessions_found(self, dev_client):
|
|
"""Test GET /reasoning returns 404 when no sessions match filters."""
|
|
response = dev_client.get("/reasoning?job_id=nonexistent-job-id")
|
|
|
|
assert response.status_code == 404
|
|
assert "No trading sessions found" in response.json()["detail"]
|
|
|
|
def test_reasoning_summaries_vs_full_conversation(self, dev_client):
|
|
"""
|
|
Test difference between summary mode and full conversation mode.
|
|
|
|
Verifies:
|
|
- Default mode does not include conversation
|
|
- include_full_conversation=true includes full conversation
|
|
- Full conversation has more data than summary
|
|
"""
|
|
# This test needs actual data - skip if none available
|
|
response_summary = dev_client.get("/reasoning")
|
|
|
|
if response_summary.status_code == 404:
|
|
pytest.skip("No reasoning data available for testing")
|
|
|
|
assert response_summary.status_code == 200
|
|
summary_data = response_summary.json()
|
|
|
|
if summary_data["count"] == 0:
|
|
pytest.skip("No reasoning data available for testing")
|
|
|
|
# Get full conversation
|
|
response_full = dev_client.get("/reasoning?include_full_conversation=true")
|
|
assert response_full.status_code == 200
|
|
full_data = response_full.json()
|
|
|
|
# Compare first session
|
|
session_summary = summary_data["sessions"][0]
|
|
session_full = full_data["sessions"][0]
|
|
|
|
# Summary mode should not have conversation
|
|
assert session_summary["conversation"] is None
|
|
|
|
# Full mode should have conversation
|
|
assert session_full["conversation"] is not None
|
|
assert len(session_full["conversation"]) > 0
|
|
|
|
# Session metadata should be the same
|
|
assert session_summary["session_id"] == session_full["session_id"]
|
|
assert session_summary["job_id"] == session_full["job_id"]
|
|
assert session_summary["date"] == session_full["date"]
|
|
assert session_summary["model"] == session_full["model"]
|
|
|
|
|
|
@pytest.mark.integration
|
|
class TestReasoningAPIValidation:
|
|
"""Test GET /reasoning endpoint validation and error handling."""
|
|
|
|
def test_reasoning_endpoint_deployment_mode_flag(self, dev_client):
|
|
"""Test that reasoning endpoint includes deployment mode info."""
|
|
response = dev_client.get("/reasoning")
|
|
|
|
# Even 404 should not be returned - endpoint should work
|
|
# Only 404 if no data matches filters
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
assert "deployment_mode" in data
|
|
assert "is_dev_mode" in data
|
|
assert data["is_dev_mode"] is True
|
|
|
|
def test_reasoning_endpoint_returns_pydantic_models(self, dev_client):
|
|
"""Test that endpoint returns properly validated response models."""
|
|
# This is implicitly tested by FastAPI/TestClient
|
|
# If response doesn't match ReasoningResponse model, will raise error
|
|
|
|
response = dev_client.get("/reasoning")
|
|
|
|
# Should either return 404 or valid response
|
|
assert response.status_code in [200, 404]
|
|
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
|
|
# Verify top-level structure
|
|
assert "sessions" in data
|
|
assert "count" in data
|
|
assert isinstance(data["sessions"], list)
|
|
assert isinstance(data["count"], int)
|
|
|
|
# If sessions exist, verify structure
|
|
if data["count"] > 0:
|
|
session = data["sessions"][0]
|
|
|
|
# Required fields
|
|
assert "session_id" in session
|
|
assert "job_id" in session
|
|
assert "date" in session
|
|
assert "model" in session
|
|
assert "started_at" in session
|
|
assert "positions" in session
|
|
|
|
# Positions structure
|
|
if len(session["positions"]) > 0:
|
|
position = session["positions"][0]
|
|
assert "action_id" in position
|
|
assert "cash_after" in position
|
|
assert "portfolio_value" in position
|