Bill/AI-Trader
1
0
Fork 0
mirror of https://github.com/Xe138/AI-Trader.git synced 2026-04-01 17:17:24 -04:00
Code Issues Packages Projects Releases Wiki Activity

test: remove old-schema tests and update for new schema

Browse Source 
- Removed test files for old schema (reasoning_e2e, position_tracking_bugs)
- Updated test_database.py to reference new tables (trading_days, holdings, actions)
- Updated conftest.py to clean new schema tables
- Fixed index name assertions to match new schema
- Updated table count expectations (9 tables in new schema)

Known issues:
- Some cascade delete tests fail (trading_days FK doesn't have ON DELETE CASCADE)
- Database locking issues in some test scenarios
- These will be addressed in future cleanup
This commit is contained in:
Bill Ballou
2025-11-04 10:36:36 -05:00
parent 45cd1e12b6
commit 0f728549f1
5 changed files with 115 additions and 993 deletions
Download Patch File Download Diff File Expand all files Collapse all files

527
tests/integration/test_reasoning_e2e.py
View File

@@ -1,527 +0,0 @@
"""
End-to-end integration tests for reasoning logs API feature.
Tests the complete flow from simulation trigger to reasoning retrieval.
These tests verify:
- Trading sessions are created with session_id
- Reasoning logs are stored in database
- Full conversation history is captured
- Message summaries are generated
- GET /reasoning endpoint returns correct data
- Query filters work (job_id, date, model)
- include_full_conversation parameter works correctly
- Positions are linked to sessions
"""
import pytest
import time
import os
import json
from fastapi.testclient import TestClient
from pathlib import Path
@pytest.fixture
def dev_client(tmp_path):
"""Create test client with DEV mode and clean database."""
# Set DEV mode environment
os.environ["DEPLOYMENT_MODE"] = "DEV"
os.environ["PRESERVE_DEV_DATA"] = "false"
# Disable auto-download - we'll pre-populate test data
os.environ["AUTO_DOWNLOAD_PRICE_DATA"] = "false"
# Import after setting environment
from api.main import create_app
from api.database import initialize_dev_database, get_db_path, get_db_connection
# Create dev database
db_path = str(tmp_path / "test_trading.db")
dev_db_path = get_db_path(db_path)
initialize_dev_database(dev_db_path)
# Pre-populate price data for test dates to avoid needing API key
_populate_test_price_data(dev_db_path)
# Create test config with mock model
test_config = tmp_path / "test_config.json"
test_config.write_text(json.dumps({
"agent_type": "BaseAgent",
"date_range": {"init_date": "2025-01-16", "end_date": "2025-01-17"},
"models": [
{
"name": "Test Mock Model",
"basemodel": "mock/test-trader",
"signature": "test-mock",
"enabled": True
}
],
"agent_config": {
"max_steps": 10,
"initial_cash": 10000.0,
"max_retries": 1,
"base_delay": 0.1
},
"log_config": {
"log_path": str(tmp_path / "dev_agent_data")
}
}))
# Create app with test config
app = create_app(db_path=dev_db_path, config_path=str(test_config))
# IMPORTANT: Do NOT set test_mode=True to allow worker to actually run
# This is an integration test - we want the full flow
client = TestClient(app)
client.db_path = dev_db_path
client.config_path = str(test_config)
yield client
# Cleanup
os.environ.pop("DEPLOYMENT_MODE", None)
os.environ.pop("PRESERVE_DEV_DATA", None)
os.environ.pop("AUTO_DOWNLOAD_PRICE_DATA", None)
def _populate_test_price_data(db_path: str):
"""
Pre-populate test price data in database.
This avoids needing Alpha Vantage API key for integration tests.
Adds mock price data for all NASDAQ 100 stocks on test dates.
"""
from api.database import get_db_connection
from datetime import datetime
# All NASDAQ 100 symbols (must match configs/nasdaq100_symbols.json)
symbols = [
"NVDA", "MSFT", "AAPL", "GOOG", "GOOGL", "AMZN", "META", "AVGO", "TSLA",
"NFLX", "PLTR", "COST", "ASML", "AMD", "CSCO", "AZN", "TMUS", "MU", "LIN",
"PEP", "SHOP", "APP", "INTU", "AMAT", "LRCX", "PDD", "QCOM", "ARM", "INTC",
"BKNG", "AMGN", "TXN", "ISRG", "GILD", "KLAC", "PANW", "ADBE", "HON",
"CRWD", "CEG", "ADI", "ADP", "DASH", "CMCSA", "VRTX", "MELI", "SBUX",
"CDNS", "ORLY", "SNPS", "MSTR", "MDLZ", "ABNB", "MRVL", "CTAS", "TRI",
"MAR", "MNST", "CSX", "ADSK", "PYPL", "FTNT", "AEP", "WDAY", "REGN", "ROP",
"NXPI", "DDOG", "AXON", "ROST", "IDXX", "EA", "PCAR", "FAST", "EXC", "TTWO",
"XEL", "ZS", "PAYX", "WBD", "BKR", "CPRT", "CCEP", "FANG", "TEAM", "CHTR",
"KDP", "MCHP", "GEHC", "VRSK", "CTSH", "CSGP", "KHC", "ODFL", "DXCM", "TTD",
"ON", "BIIB", "LULU", "CDW", "GFS", "QQQ"
]
# Test dates
test_dates = ["2025-01-16", "2025-01-17"]
conn = get_db_connection(db_path)
cursor = conn.cursor()
for symbol in symbols:
for date in test_dates:
# Insert mock price data
cursor.execute("""
INSERT OR IGNORE INTO price_data
(symbol, date, open, high, low, close, volume, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""", (
symbol,
date,
100.0, # open
105.0, # high
98.0, # low
102.0, # close
1000000, # volume
datetime.utcnow().isoformat() + "Z"
))
# Add coverage record
cursor.execute("""
INSERT OR IGNORE INTO price_data_coverage
(symbol, start_date, end_date, downloaded_at, source)
VALUES (?, ?, ?, ?, ?)
""", (
symbol,
"2025-01-16",
"2025-01-17",
datetime.utcnow().isoformat() + "Z",
"test_fixture"
))
conn.commit()
conn.close()
@pytest.mark.integration
@pytest.mark.skipif(
os.getenv("SKIP_INTEGRATION_TESTS") == "true",
reason="Skipping integration tests that require full environment"
)
class TestReasoningLogsE2E:
"""End-to-end tests for reasoning logs feature."""
def test_simulation_stores_reasoning_logs(self, dev_client):
"""
Test that running a simulation creates reasoning logs in database.
This is the main E2E test that verifies:
1. Simulation can be triggered
2. Worker processes the job
3. Trading sessions are created
4. Reasoning logs are stored
5. GET /reasoning returns the data
NOTE: This test requires MCP services to be running. It will skip if services are unavailable.
"""
# Skip if MCP services not available
try:
from agent.base_agent.base_agent import BaseAgent
except ImportError as e:
pytest.skip(f"Cannot import BaseAgent: {e}")
# Skip test - requires MCP services running
# This is a known limitation for integration tests
pytest.skip(
"Test requires MCP services running. "
"Use test_reasoning_api_with_mocked_data() instead for automated testing."
)
def test_reasoning_api_with_mocked_data(self, dev_client):
"""
Test GET /reasoning API with pre-populated database data.
This test verifies the API layer works correctly without requiring
a full simulation run or MCP services.
"""
from api.database import get_db_connection
from datetime import datetime
# Populate test data directly in database
conn = get_db_connection(dev_client.db_path)
cursor = conn.cursor()
# Create a job
job_id = "test-job-123"
cursor.execute("""
INSERT INTO jobs (job_id, config_path, status, date_range, models, created_at)
VALUES (?, ?, ?, ?, ?, ?)
""", (job_id, "test_config.json", "completed", "2025-01-16", '["test-mock"]',
datetime.utcnow().isoformat() + "Z"))
# Create a trading session
cursor.execute("""
INSERT INTO trading_sessions
(job_id, date, model, session_summary, started_at, completed_at, total_messages)
VALUES (?, ?, ?, ?, ?, ?, ?)
""", (
job_id,
"2025-01-16",
"test-mock",
"Analyzed market conditions and executed buy order for AAPL",
datetime.utcnow().isoformat() + "Z",
datetime.utcnow().isoformat() + "Z",
5
))
session_id = cursor.lastrowid
# Create reasoning logs
messages = [
{
"session_id": session_id,
"message_index": 0,
"role": "user",
"content": "You are a trading agent. Analyze the market...",
"summary": None,
"tool_name": None,
"tool_input": None,
"timestamp": datetime.utcnow().isoformat() + "Z"
},
{
"session_id": session_id,
"message_index": 1,
"role": "assistant",
"content": "I will analyze the market and make trading decisions...",
"summary": "Agent analyzed market conditions",
"tool_name": None,
"tool_input": None,
"timestamp": datetime.utcnow().isoformat() + "Z"
},
{
"session_id": session_id,
"message_index": 2,
"role": "tool",
"content": "Price of AAPL: $150.00",
"summary": None,
"tool_name": "get_price",
"tool_input": json.dumps({"symbol": "AAPL"}),
"timestamp": datetime.utcnow().isoformat() + "Z"
},
{
"session_id": session_id,
"message_index": 3,
"role": "assistant",
"content": "Based on analysis, I will buy AAPL...",
"summary": "Agent decided to buy AAPL",
"tool_name": None,
"tool_input": None,
"timestamp": datetime.utcnow().isoformat() + "Z"
},
{
"session_id": session_id,
"message_index": 4,
"role": "tool",
"content": "Successfully bought 10 shares of AAPL",
"summary": None,
"tool_name": "buy",
"tool_input": json.dumps({"symbol": "AAPL", "amount": 10}),
"timestamp": datetime.utcnow().isoformat() + "Z"
}
]
for msg in messages:
cursor.execute("""
INSERT INTO reasoning_logs
(session_id, message_index, role, content, summary, tool_name, tool_input, timestamp)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""", (
msg["session_id"], msg["message_index"], msg["role"],
msg["content"], msg["summary"], msg["tool_name"],
msg["tool_input"], msg["timestamp"]
))
# Create positions linked to session
cursor.execute("""
INSERT INTO positions
(job_id, date, model, action_id, action_type, symbol, amount, price, cash, portfolio_value,
daily_profit, daily_return_pct, created_at, session_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (
job_id, "2025-01-16", "test-mock", 1, "buy", "AAPL", 10, 150.0,
8500.0, 10000.0, 0.0, 0.0, datetime.utcnow().isoformat() + "Z", session_id
))
conn.commit()
conn.close()
# Query reasoning endpoint (summary mode)
reasoning_response = dev_client.get(f"/reasoning?job_id={job_id}")
assert reasoning_response.status_code == 200
reasoning_data = reasoning_response.json()
# Verify response structure
assert "sessions" in reasoning_data
assert "count" in reasoning_data
assert reasoning_data["count"] == 1
assert reasoning_data["is_dev_mode"] is True
# Verify trading session structure
session = reasoning_data["sessions"][0]
assert session["session_id"] == session_id
assert session["job_id"] == job_id
assert session["date"] == "2025-01-16"
assert session["model"] == "test-mock"
assert session["session_summary"] == "Analyzed market conditions and executed buy order for AAPL"
assert session["total_messages"] == 5
# Verify positions are linked to session
assert "positions" in session
assert len(session["positions"]) == 1
position = session["positions"][0]
assert position["action_id"] == 1
assert position["action_type"] == "buy"
assert position["symbol"] == "AAPL"
assert position["amount"] == 10
assert position["price"] == 150.0
assert position["cash_after"] == 8500.0
assert position["portfolio_value"] == 10000.0
# Verify conversation is NOT included in summary mode
assert session["conversation"] is None
# Query again with full conversation
full_response = dev_client.get(
f"/reasoning?job_id={job_id}&include_full_conversation=true"
)
assert full_response.status_code == 200
full_data = full_response.json()
session_full = full_data["sessions"][0]
# Verify full conversation is included
assert session_full["conversation"] is not None
assert len(session_full["conversation"]) == 5
# Verify conversation messages
conv = session_full["conversation"]
assert conv[0]["role"] == "user"
assert conv[0]["message_index"] == 0
assert conv[0]["summary"] is None # User messages don't have summaries
assert conv[1]["role"] == "assistant"
assert conv[1]["message_index"] == 1
assert conv[1]["summary"] == "Agent analyzed market conditions"
assert conv[2]["role"] == "tool"
assert conv[2]["message_index"] == 2
assert conv[2]["tool_name"] == "get_price"
assert conv[2]["tool_input"] == json.dumps({"symbol": "AAPL"})
assert conv[3]["role"] == "assistant"
assert conv[3]["message_index"] == 3
assert conv[3]["summary"] == "Agent decided to buy AAPL"
assert conv[4]["role"] == "tool"
assert conv[4]["message_index"] == 4
assert conv[4]["tool_name"] == "buy"
def test_reasoning_endpoint_date_filter(self, dev_client):
"""Test GET /reasoning date filter works correctly."""
# This test requires actual data - skip if no data available
response = dev_client.get("/reasoning?date=2025-01-16")
# Should either return 404 (no data) or 200 with filtered data
assert response.status_code in [200, 404]
if response.status_code == 200:
data = response.json()
for session in data["sessions"]:
assert session["date"] == "2025-01-16"
def test_reasoning_endpoint_model_filter(self, dev_client):
"""Test GET /reasoning model filter works correctly."""
response = dev_client.get("/reasoning?model=test-mock")
# Should either return 404 (no data) or 200 with filtered data
assert response.status_code in [200, 404]
if response.status_code == 200:
data = response.json()
for session in data["sessions"]:
assert session["model"] == "test-mock"
def test_reasoning_endpoint_combined_filters(self, dev_client):
"""Test GET /reasoning with multiple filters."""
response = dev_client.get(
"/reasoning?date=2025-01-16&model=test-mock"
)
# Should either return 404 (no data) or 200 with filtered data
assert response.status_code in [200, 404]
if response.status_code == 200:
data = response.json()
for session in data["sessions"]:
assert session["date"] == "2025-01-16"
assert session["model"] == "test-mock"
def test_reasoning_endpoint_invalid_date_format(self, dev_client):
"""Test GET /reasoning rejects invalid date format."""
response = dev_client.get("/reasoning?date=invalid-date")
assert response.status_code == 400
assert "Invalid date format" in response.json()["detail"]
def test_reasoning_endpoint_no_sessions_found(self, dev_client):
"""Test GET /reasoning returns 404 when no sessions match filters."""
response = dev_client.get("/reasoning?job_id=nonexistent-job-id")
assert response.status_code == 404
assert "No trading sessions found" in response.json()["detail"]
def test_reasoning_summaries_vs_full_conversation(self, dev_client):
"""
Test difference between summary mode and full conversation mode.
Verifies:
- Default mode does not include conversation
- include_full_conversation=true includes full conversation
- Full conversation has more data than summary
"""
# This test needs actual data - skip if none available
response_summary = dev_client.get("/reasoning")
if response_summary.status_code == 404:
pytest.skip("No reasoning data available for testing")
assert response_summary.status_code == 200
summary_data = response_summary.json()
if summary_data["count"] == 0:
pytest.skip("No reasoning data available for testing")
# Get full conversation
response_full = dev_client.get("/reasoning?include_full_conversation=true")
assert response_full.status_code == 200
full_data = response_full.json()
# Compare first session
session_summary = summary_data["sessions"][0]
session_full = full_data["sessions"][0]
# Summary mode should not have conversation
assert session_summary["conversation"] is None
# Full mode should have conversation
assert session_full["conversation"] is not None
assert len(session_full["conversation"]) > 0
# Session metadata should be the same
assert session_summary["session_id"] == session_full["session_id"]
assert session_summary["job_id"] == session_full["job_id"]
assert session_summary["date"] == session_full["date"]
assert session_summary["model"] == session_full["model"]
@pytest.mark.integration
class TestReasoningAPIValidation:
"""Test GET /reasoning endpoint validation and error handling."""
def test_reasoning_endpoint_deployment_mode_flag(self, dev_client):
"""Test that reasoning endpoint includes deployment mode info."""
response = dev_client.get("/reasoning")
# Even 404 should not be returned - endpoint should work
# Only 404 if no data matches filters
if response.status_code == 200:
data = response.json()
assert "deployment_mode" in data
assert "is_dev_mode" in data
assert data["is_dev_mode"] is True
def test_reasoning_endpoint_returns_pydantic_models(self, dev_client):
"""Test that endpoint returns properly validated response models."""
# This is implicitly tested by FastAPI/TestClient
# If response doesn't match ReasoningResponse model, will raise error
response = dev_client.get("/reasoning")
# Should either return 404 or valid response
assert response.status_code in [200, 404]
if response.status_code == 200:
data = response.json()
# Verify top-level structure
assert "sessions" in data
assert "count" in data
assert isinstance(data["sessions"], list)
assert isinstance(data["count"], int)
# If sessions exist, verify structure
if data["count"] > 0:
session = data["sessions"][0]
# Required fields
assert "session_id" in session
assert "job_id" in session
assert "date" in session
assert "model" in session
assert "started_at" in session
assert "positions" in session
# Positions structure
if len(session["positions"]) > 0:
position = session["positions"][0]
assert "action_id" in position
assert "cash_after" in position
assert "portfolio_value" in position