Closes #019 - Live Python Machine Learning Pipeline Integration

backend/core/pipeline.py

@@ -7,6 +7,7 @@ leakage guards, trains 5 models (RF, XGB/GB, ElasticNet LR, SVM, MLP), and expor
 
 import os
 import json
+import urllib.request
 import numpy as np
 import pandas as pd
 
@@ -208,8 +209,78 @@ def get_mock_predictions():
     }
 
 
+def fetch_real_data():
+    """
+    Queries real daily candles from Yahoo Finance and real-time funding rates from
+    the Binance USDS-M Futures REST APIs. Saves the daily candles to backend/data/BTC-USD.csv.
+    """
+    # 1. Fetch candles from Yahoo Finance
+    print("Fetching real daily candles from Yahoo Finance...")
+    yahoo_url = "https://query1.finance.yahoo.com/v8/finance/chart/BTC-USD?range=2y&interval=1d"
+    req_yahoo = urllib.request.Request(
+        yahoo_url, 
+        headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}
+    )
+    try:
+        with urllib.request.urlopen(req_yahoo) as response:
+            data = json.loads(response.read().decode())
+            result = data['chart']['result'][0]
+            timestamps = result['timestamp']
+            quote = result['indicators']['quote'][0]
+            
+            opens = quote['open']
+            highs = quote['high']
+            lows = quote['low']
+            closes = quote['close']
+            volumes = quote['volume']
+            
+            cleaned_rows = []
+            for i in range(len(timestamps)):
+                if (opens[i] is not None and highs[i] is not None and 
+                    lows[i] is not None and closes[i] is not None):
+                    date_str = pd.to_datetime(timestamps[i], unit='s').strftime('%Y-%m-%d')
+                    cleaned_rows.append({
+                        'Date': date_str,
+                        'Open': opens[i],
+                        'High': highs[i],
+                        'Low': lows[i],
+                        'Close': closes[i],
+                        'Volume': volumes[i] if volumes[i] is not None else 0
+                    })
+                    
+            df_new = pd.DataFrame(cleaned_rows).set_index('Date')
+            
+            os.makedirs(os.path.join('backend', 'data'), exist_ok=True)
+            csv_path = os.path.join('backend', 'data', 'BTC-USD.csv')
+            df_new.to_csv(csv_path)
+            print(f"Successfully downloaded {len(df_new)} BTC-USD daily candles and saved to {csv_path}")
+    except Exception as e:
+        print(f"Failed to query daily candles from Yahoo Finance: {e}")
+        
+    # 2. Fetch funding rate from Binance USDS-M Futures API
+    print("Fetching real-time funding rates from Binance USDS-M Futures REST APIs...")
+    binance_url = "https://fapi.binance.com/fapi/v1/fundingRate?symbol=BTCUSDT&limit=1"
+    req_binance = urllib.request.Request(
+        binance_url,
+        headers={'User-Agent': 'Mozilla/5.0'}
+    )
+    try:
+        with urllib.request.urlopen(req_binance) as response:
+            data = json.loads(response.read().decode())
+            latest = data[0]
+            rate = float(latest['fundingRate'])
+            time_ms = latest['fundingTime']
+            print(f"Binance BTCUSDT latest funding rate: {rate} at timestamp {time_ms}")
+    except Exception as e:
+        print(f"Failed to query funding rate from Binance USDS-M Futures REST APIs: {e}")
+
+
 def main():
     print(f"[{datetime_now_str()}] Initializing Multi-Model rolling validation...")
+    
+    # Ingest live data first
+    fetch_real_data()
+    
     preds = train_and_forecast()
     
     # Save the predictions to public/data/ensemble_predictions.json