From 1785f9b06fc89351692f19b0418c96d068ce34e0 Mon Sep 17 00:00:00 2001
From: Bill <bill@bballou.com>
Date: Thu, 30 Oct 2025 23:48:12 -0400
Subject: [PATCH] feat: automate merged.jsonl creation during price fetching

Streamline the data preparation workflow by having get_daily_price.py
automatically invoke merge_jsonl.py after fetching all stock prices.

Changes:
- Modified get_daily_price.py to call merge_jsonl.py automatically
- Updated entrypoint.sh to remove redundant merge_jsonl.py call
- Updated main.sh to remove redundant merge_jsonl.py call
- Fixed import order for linting compliance

Benefits:
- Single command now handles both fetching and merging
- Ensures merged.jsonl is always created after price updates
- Simplifies Docker container startup process
- Prevents missing merged.jsonl errors in production
---
 data/get_daily_price.py | 25 ++++++++++++++++++++-----
 entrypoint.sh           |  4 ++--
 main.sh                 |  2 +-
 3 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/data/get_daily_price.py b/data/get_daily_price.py
index 909a921..58a37ea 100644
--- a/data/get_daily_price.py
+++ b/data/get_daily_price.py
@@ -1,8 +1,12 @@
-import requests
-import os
-from dotenv import load_dotenv
-load_dotenv()
 import json
+import os
+import subprocess
+import sys
+
+import requests
+from dotenv import load_dotenv
+
+load_dotenv()
 
 
 all_nasdaq_100_symbols = [
@@ -42,4 +46,15 @@ if __name__ == "__main__":
     for symbol in all_nasdaq_100_symbols:
         get_daily_price(symbol)
 
-    get_daily_price("QQQ")
\ No newline at end of file
+    get_daily_price("QQQ")
+
+    # Automatically run merge after fetching
+    print("\n📦 Merging price data...")
+    try:
+        script_dir = os.path.dirname(os.path.abspath(__file__))
+        merge_script = os.path.join(script_dir, "merge_jsonl.py")
+        subprocess.run([sys.executable, merge_script], check=True)
+        print("✅ Price data merged successfully")
+    except Exception as e:
+        print(f"⚠️  Failed to merge data: {e}")
+        print("   Please run 'python merge_jsonl.py' manually")
\ No newline at end of file
diff --git a/entrypoint.sh b/entrypoint.sh
index c1a81b3..7015932 100755
--- a/entrypoint.sh
+++ b/entrypoint.sh
@@ -44,10 +44,10 @@ if [ -f "/app/data/merged.jsonl" ] && [ -s "/app/data/merged.jsonl" ]; then
     echo "   To refresh data, delete /app/data/merged.jsonl and restart"
 else
     echo "📊 Fetching and merging price data..."
-    # Run scripts from /app/scripts but output to /app/data
+    # Run script from /app/scripts but output to /app/data
+    # Note: get_daily_price.py now automatically calls merge_jsonl.py after fetching
     cd /app/data
     python /app/scripts/get_daily_price.py
-    python /app/scripts/merge_jsonl.py
     cd /app
 fi
 
diff --git a/main.sh b/main.sh
index 6b2dc0b..6a85f2b 100644
--- a/main.sh
+++ b/main.sh
@@ -10,8 +10,8 @@ echo "🚀 Launching AI Trader Environment..."
 
 echo "📊 Now getting and merging price data..."
 cd ./data
+# Note: get_daily_price.py now automatically calls merge_jsonl.py after fetching
 python get_daily_price.py
-python merge_jsonl.py
 cd ../
 
 echo "🔧 Now starting MCP services..."