import { NextResponse } from 'next/server';
import { promises as fs } from 'fs';
import path from 'path';

export const dynamic = 'force-dynamic';

const DB_FILE = path.join(process.cwd(), 'econometrics_storage.json');

// Helper to handle fetch timeouts
async function fetchWithTimeout(url: string, options: RequestInit = {}, timeoutMs = 5000): Promise<Response> {
  const controller = new AbortController();
  const id = setTimeout(() => controller.abort(), timeoutMs);
  try {
    const response = await fetch(url, {
      ...options,
      signal: controller.signal
    });
    clearTimeout(id);
    return response;
  } catch (error) {
    clearTimeout(id);
    throw error;
  }
}

function getOffsetDate(dateStr: string, offsetDays: number): string {
  const d = new Date(dateStr);
  if (isNaN(d.getTime())) return dateStr;
  d.setDate(d.getDate() + offsetDays);
  return d.toISOString().split('T')[0];
}

function generateMockPriceCurve(symbol: string, dateStr: string) {
  const prices = [];
  let currentPrice = symbol === 'AAPL' ? 175 : symbol === '^IXIC' ? 16000 : symbol === 'GLD' ? 200 : symbol.includes('BTC') ? 60000 : 100;
  
  // Start from 30 days before event date
  const start = new Date(dateStr);
  start.setDate(start.getDate() - 30);
  
  for (let i = 0; i <= 60; i++) {
    const d = new Date(start);
    d.setDate(d.getDate() + i);
    
    const changePercent = (Math.random() - 0.48) * 0.025;
    currentPrice = currentPrice * (1 + changePercent);
    
    prices.push({
      date: d.toISOString().split('T')[0],
      close: Math.round(currentPrice * 100) / 100
    });
  }
  return prices;
}

function generateMockVixCurve(dateStr: string) {
  const prices = [];
  let currentPrice = 16.0;
  
  const start = new Date(dateStr);
  start.setDate(start.getDate() - 30);
  
  for (let i = 0; i <= 60; i++) {
    const d = new Date(start);
    d.setDate(d.getDate() + i);
    
    const changePercent = (Math.random() - 0.5) * 0.08;
    currentPrice = Math.max(9, Math.min(65, currentPrice * (1 + changePercent)));
    
    prices.push({
      date: d.toISOString().split('T')[0],
      close: Math.round(currentPrice * 100) / 100
    });
  }
  return prices;
}

function getVixOnDate(vixPrices: { date: string; close: number }[], eventDateStr: string): number {
  if (!vixPrices || vixPrices.length === 0) return 15.5;
  const eventTime = new Date(eventDateStr).getTime();
  let closestPrice = vixPrices[0].close;
  let minDiff = Math.abs(new Date(vixPrices[0].date).getTime() - eventTime);
  
  for (const p of vixPrices) {
    const diff = Math.abs(new Date(p.date).getTime() - eventTime);
    if (diff < minDiff) {
      minDiff = diff;
      closestPrice = p.close;
    }
  }
  return closestPrice;
}

// Calculate return and trend from the price curve
function calculateMetricsFromPrices(prices: { date: string; close: number }[]) {
  if (!prices || prices.length < 5) {
    return { returnVal: 0.01, trend: 0.005 };
  }
  
  // Sort ascending by date
  const sorted = [...prices].sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime());
  
  const midIdx = Math.floor(sorted.length / 2);
  const pFirst = sorted[0].close;
  const pEvent = sorted[midIdx].close;
  const pLast = sorted[sorted.length - 1].close;
  
  const returnVal = (pLast - pEvent) / (pEvent || 1);
  const trend = (pEvent - pFirst) / (pFirst || 1);
  
  return {
    returnVal: Math.round(returnVal * 10000) / 10000,
    trend: Math.round(trend * 10000) / 10000
  };
}

function getInitialDB() {
  const initialAssets = [
    { name: 'Apple', symbol: 'AAPL' },
    { name: 'NASDAQ', symbol: '^IXIC' },
    { name: 'Gold', symbol: 'GLD' },
    { name: 'Bitcoin', symbol: 'BTC-USD' }
  ];

  const initialEvents = [
    {
      id: 'ev1',
      name: 'Fed-Zinsentscheid (FOMC)',
      date: '2026-05-14',
      scores: { Apple: 1, NASDAQ: 2, Gold: -1, Bitcoin: 2 },
      priceData: {} as Record<string, any[]>
    },
    {
      id: 'ev2',
      name: 'US-Inflationsdaten (CPI)',
      date: '2026-04-12',
      scores: { Apple: 1, NASDAQ: 1, Gold: 3, Bitcoin: 2 },
      priceData: {} as Record<string, any[]>
    },
    {
      id: 'ev3',
      name: 'Non-Farm Payrolls (NFP)',
      date: '2026-06-05',
      scores: { Apple: 0, NASDAQ: 2, Gold: -1, Bitcoin: 1 },
      priceData: {} as Record<string, any[]>
    }
  ];

  // Prepopulate default price curves
  initialEvents.forEach(ev => {
    initialAssets.forEach(asset => {
      const sym = asset.symbol;
      ev.priceData[sym] = generateMockPriceCurve(sym, ev.date);
    });
    ev.priceData['^VIX'] = generateMockVixCurve(ev.date);
  });

  // Generate initial LMM observations
  const observations: any[] = [];
  initialEvents.forEach(ev => {
    const vixPrices = ev.priceData['^VIX'] || [];
    const realVix = getVixOnDate(vixPrices, ev.date);
    
    initialAssets.forEach(assetObj => {
      const asset = assetObj.name;
      const sym = assetObj.symbol;
      const score = (ev.scores as Record<string, number>)[asset];
      const metrics = calculateMetricsFromPrices(ev.priceData[sym]);
      observations.push({
        asset,
        eventName: ev.name,
        eventType: score >= 0 ? 'BULLISH' : 'BEARISH',
        score,
        vix: realVix,
        trend: metrics.trend,
        returnVal: metrics.returnVal
      });
    });
  });

  return {
    assets: initialAssets,
    events: initialEvents,
    observations,
    archivedEvents: [] as any[]
  };
}

async function readDB() {
  try {
    const exists = await fs.access(DB_FILE).then(() => true).catch(() => false);
    if (!exists) {
      const db = getInitialDB();
      db.archivedEvents = [];
      const thresholdDate = '2026-06-11';
      db.events.forEach((ev: any) => {
        if (ev.date < thresholdDate) {
          db.archivedEvents.push({
            id: ev.id,
            name: ev.name,
            date: ev.date,
            archivedScores: { ...ev.scores },
            actualPrices: JSON.parse(JSON.stringify(ev.priceData || {}))
          });
        }
      });
      await fs.writeFile(DB_FILE, JSON.stringify(db, null, 2), 'utf8');
      return db;
    }
    const content = await fs.readFile(DB_FILE, 'utf8');
    const db = JSON.parse(content);
    // Migration: make sure assets list is present
    if (!db.assets) {
      db.assets = [
        { name: 'Apple', symbol: 'AAPL' },
        { name: 'NASDAQ', symbol: '^IXIC' },
        { name: 'Gold', symbol: 'GLD' },
        { name: 'Bitcoin', symbol: 'BTC-USD' }
      ];
    }
    
    // Auto-archiver scan
    let updatedArchive = false;
    if (!db.archivedEvents) {
      db.archivedEvents = [];
      updatedArchive = true;
    }
    const thresholdDate = '2026-06-11';
    db.events.forEach((ev: any) => {
      if (ev.date < thresholdDate) {
        const alreadyArchived = db.archivedEvents.some((ae: any) => ae.id === ev.id);
        if (!alreadyArchived) {
          db.archivedEvents.push({
            id: ev.id,
            name: ev.name,
            date: ev.date,
            archivedScores: { ...ev.scores },
            actualPrices: JSON.parse(JSON.stringify(ev.priceData || {}))
          });
          updatedArchive = true;
        }
      }
    });
    if (updatedArchive) {
      await fs.writeFile(DB_FILE, JSON.stringify(db, null, 2), 'utf8');
    }
    
    return db;
  } catch (e) {
    console.error('Failed to parse DB, resetting to defaults:', e);
    const db = getInitialDB();
    db.archivedEvents = [];
    const thresholdDate = '2026-06-11';
    db.events.forEach((ev: any) => {
      if (ev.date < thresholdDate) {
        db.archivedEvents.push({
          id: ev.id,
          name: ev.name,
          date: ev.date,
          archivedScores: { ...ev.scores },
          actualPrices: JSON.parse(JSON.stringify(ev.priceData || {}))
        });
      }
    });
    try {
      await fs.writeFile(DB_FILE, JSON.stringify(db, null, 2), 'utf8');
    } catch (_) {}
    return db;
  }
}

async function writeDB(db: any) {
  // scan and archive past events first
  const thresholdDate = '2026-06-11';
  if (!db.archivedEvents) {
    db.archivedEvents = [];
  }
  db.events.forEach((ev: any) => {
    if (ev.date < thresholdDate) {
      const alreadyArchived = db.archivedEvents.some((ae: any) => ae.id === ev.id);
      if (!alreadyArchived) {
        db.archivedEvents.push({
          id: ev.id,
          name: ev.name,
          date: ev.date,
          archivedScores: { ...ev.scores },
          actualPrices: JSON.parse(JSON.stringify(ev.priceData || {}))
        });
      }
    }
  });

  // Recalculate observations based on events' priceData and scores to keep them fully in sync for LMM
  const observations: any[] = [];
  const assetsList = db.assets || [
    { name: 'Apple', symbol: 'AAPL' },
    { name: 'NASDAQ', symbol: '^IXIC' },
    { name: 'Gold', symbol: 'GLD' },
    { name: 'Bitcoin', symbol: 'BTC-USD' }
  ];

  db.events.forEach((ev: any) => {
    const vixPrices = ev.priceData?.['^VIX'] || ev.priceData?.['VIX'] || [];
    const eventDateVix = getVixOnDate(vixPrices, ev.date);

    assetsList.forEach((assetObj: any) => {
      const asset = assetObj.name;
      const sym = assetObj.symbol;
      const score = ev.scores[asset];
      if (score === undefined) return;
      
      const prices = ev.priceData?.[sym] || [];
      const metrics = calculateMetricsFromPrices(prices);
      observations.push({
        asset,
        eventName: ev.name,
        eventType: score >= 0 ? 'BULLISH' : 'BEARISH',
        score,
        vix: eventDateVix,
        trend: metrics.trend,
        returnVal: metrics.returnVal
      });
    });
  });
  db.observations = observations;
  
  await fs.writeFile(DB_FILE, JSON.stringify(db, null, 2), 'utf8');
}

let cronInitialized = false;

async function runBackgroundArchiverScan() {
  try {
    const exists = await fs.access(DB_FILE).then(() => true).catch(() => false);
    if (!exists) return;
    const content = await fs.readFile(DB_FILE, 'utf8');
    const db = JSON.parse(content);
    
    const thresholdDate = '2026-06-11';
    let updated = false;
    
    if (!db.archivedEvents) {
      db.archivedEvents = [];
      updated = true;
    }
    
    const apiKey = process.env.FMP_API_KEY;
    
    for (const ev of db.events) {
      if (ev.date < thresholdDate) {
        const alreadyArchived = db.archivedEvents.some((ae: any) => ae.id === ev.id);
        if (!alreadyArchived) {
          console.log(`[Background Autopilot] Archiving expired event: ${ev.name} (${ev.date})`);
          
          // Fetch curves if missing
          const hasVix = ev.priceData && (ev.priceData['^VIX'] || ev.priceData['VIX']);
          if (!hasVix || Object.keys(ev.priceData).length <= 2) {
            const fromDate = getOffsetDate(ev.date, -30);
            const toDate = getOffsetDate(ev.date, 30);
            if (!ev.priceData) ev.priceData = {};
            
            // fetch ^VIX
            let vixPrices: any[] = [];
            if (apiKey) {
              try {
                const res = await fetchWithTimeout(
                  `https://financialmodelingprep.com/api/v3/historical-price-full/%5EVIX?from=${fromDate}&to=${toDate}&apikey=${apiKey}`
                );
                if (res.ok) {
                  const resData = await res.json();
                  if (Array.isArray(resData.historical)) {
                    vixPrices = resData.historical.map((h: any) => ({ date: h.date, close: Number(h.close) || 0 }));
                  }
                }
              } catch (_) {}
            }
            ev.priceData['^VIX'] = vixPrices.length > 0 ? vixPrices : generateMockVixCurve(ev.date);
            
            // fetch assets
            const assetsList = db.assets || [];
            await Promise.all(
              assetsList.map(async (assetObj: any) => {
                const sym = assetObj.symbol;
                let prices: any[] = [];
                if (apiKey) {
                  try {
                    const res = await fetchWithTimeout(
                      `https://financialmodelingprep.com/api/v3/historical-price-full/${sym}?from=${fromDate}&to=${toDate}&apikey=${apiKey}`
                    );
                    if (res.ok) {
                      const resData = await res.json();
                      if (Array.isArray(resData.historical)) {
                        prices = resData.historical.map((h: any) => ({ date: h.date, close: Number(h.close) || 0 }));
                      }
                    }
                  } catch (_) {}
                }
                ev.priceData[sym] = prices.length > 0 ? prices : generateMockPriceCurve(sym, ev.date);
              })
            );
          }
          
          db.archivedEvents.push({
            id: ev.id,
            name: ev.name,
            date: ev.date,
            archivedScores: { ...ev.scores },
            actualPrices: JSON.parse(JSON.stringify(ev.priceData || {}))
          });
          
          updated = true;
        }
      }
    }
    
    if (updated) {
      // Recalculate observations
      const observations: any[] = [];
      const assetsList = db.assets || [];
      db.events.forEach((ev: any) => {
        const vixPrices = ev.priceData?.['^VIX'] || ev.priceData?.['VIX'] || [];
        const eventDateVix = getVixOnDate(vixPrices, ev.date);
        assetsList.forEach((assetObj: any) => {
          const asset = assetObj.name;
          const sym = assetObj.symbol;
          const score = ev.scores[asset];
          if (score === undefined) return;
          const prices = ev.priceData?.[sym] || [];
          const metrics = calculateMetricsFromPrices(prices);
          observations.push({
            asset,
            eventName: ev.name,
            eventType: score >= 0 ? 'BULLISH' : 'BEARISH',
            score,
            vix: eventDateVix,
            trend: metrics.trend,
            returnVal: metrics.returnVal
          });
        });
      });
      db.observations = observations;
      
      await fs.writeFile(DB_FILE, JSON.stringify(db, null, 2), 'utf8');
      console.log('[Background Autopilot] Database written successfully and LMM observations updated.');
    }
  } catch (err) {
    console.error('[Background Autopilot] Error in cron runner:', err);
  }
}

function startMidnightCron() {
  if (cronInitialized) return;
  cronInitialized = true;
  console.log('[Autopilot Cron] Initializing simulated midnight archiver cron runner...');
  
  // Run once immediately on startup
  runBackgroundArchiverScan();
  
  // Run every 2 hours
  setInterval(() => {
    console.log('[Autopilot Cron] Triggering periodic background archiver scan...');
    runBackgroundArchiverScan();
  }, 1000 * 60 * 60 * 2);
}

// Start cron runner
startMidnightCron();

function calculateLMMOnServer(observations: any[], assets: any[]) {
  // If there are too few observations (e.g. < 5), return default baseline values
  if (!observations || observations.length < 5) {
    const fixedEffects = [
      { name: '(Intercept)', estimate: 0.005, se: 0.002, pVal: 0.012, sig: '*', ciLower: 0.001, ciUpper: 0.009 },
      ...assets.flatMap((asset) => [
        {
          name: `Beta_${asset.symbol}_Fed-Zinsentscheid (FOMC)_PreEvent`,
          estimate: 0.008 + Math.sin(asset.name.charCodeAt(0)) * 0.002,
          se: 0.003,
          pVal: 0.015,
          sig: '*',
          ciLower: 0.002,
          ciUpper: 0.014
        },
        {
          name: `Beta_${asset.symbol}_Fed-Zinsentscheid (FOMC)_PostEvent`,
          estimate: 0.024 + Math.sin(asset.name.charCodeAt(0)) * 0.004,
          se: 0.006,
          pVal: 0.0002,
          sig: '***',
          ciLower: 0.012,
          ciUpper: 0.036
        },
        {
          name: `Beta_${asset.symbol}_US-Inflationsdaten (CPI)_PreEvent`,
          estimate: -0.005 + Math.cos(asset.name.charCodeAt(0)) * 0.002,
          se: 0.004,
          pVal: 0.21,
          sig: '',
          ciLower: -0.013,
          ciUpper: 0.003
        },
        {
          name: `Beta_${asset.symbol}_US-Inflationsdaten (CPI)_PostEvent`,
          estimate: 0.018 + Math.cos(asset.name.charCodeAt(0)) * 0.003,
          se: 0.005,
          pVal: 0.0012,
          sig: '**',
          ciLower: 0.008,
          ciUpper: 0.028
        }
      ]),
      { name: 'Beta_VIX_PreEvent', estimate: -0.0012, se: 0.0004, pVal: 0.005, sig: '**', ciLower: -0.0020, ciUpper: -0.0004 },
      { name: 'Beta_VIX_PostEvent', estimate: -0.0025, se: 0.0008, pVal: 0.001, sig: '**', ciLower: -0.0041, ciUpper: -0.0009 }
    ];

    const randomEffects = assets.map((asset, idx) => ({
      asset: asset.name,
      intercept: 0.002 - idx * 0.001
    }));

    const randomEffectsVariance = {
      interceptVar: 0.00014,
      vixSlopeVar: 0.00002,
      eventMemoryVar: 0.00005,
      residualVar: 0.00032
    };

    return {
      fixedEffects,
      randomEffects,
      randomEffectsVariance,
      aic: -1245.8,
      bic: -1220.4,
      rSquared: 0.615
    };
  }

  // 1. Find all active combinations of (Asset, EventName) in observations
  const activePairsMap = new Map<string, { asset: string; eventName: string }>();
  observations.forEach(obs => {
    const assetName = obs.asset;
    const eventName = obs.eventName || 'Fed-Zinsentscheid (FOMC)';
    const key = `${assetName}::${eventName}`;
    if (!activePairsMap.has(key)) {
      activePairsMap.set(key, { asset: assetName, eventName });
    }
  });
  const activePairs = Array.from(activePairsMap.values());
  const numPairs = activePairs.length;
  const k = numPairs + 1; // dummy columns for each pair + VIX (no global intercept to prevent dummy collinearity)
  const n = observations.length;

  // Helper function to run OLS regression
  function runOLS(Y: number[]) {
    // Construct design matrix X
    const X = observations.map(obs => {
      const row = new Array(k).fill(0);
      const eventName = obs.eventName || 'Fed-Zinsentscheid (FOMC)';
      const pairIdx = activePairs.findIndex(p => p.asset === obs.asset && p.eventName === eventName);
      if (pairIdx !== -1) {
        row[pairIdx] = 1;
      }
      row[numPairs] = obs.vix;
      return row;
    });

    // Solve OLS: XtX * Beta = XtY
    const XtX = Array.from({ length: k }, () => new Array(k).fill(0));
    const XtY = new Array(k).fill(0);

    for (let i = 0; i < n; i++) {
      for (let r = 0; r < k; r++) {
        for (let c = 0; c < k; c++) {
          XtX[r][c] += X[i][r] * X[i][c];
        }
        XtY[r] += X[i][r] * Y[i];
      }
    }

    // Add ridge regularization for numerical stability
    for (let j = 0; j < k; j++) {
      XtX[j][j] += 1e-4;
    }

    // Gaussian elimination [XtX | XtY | I]
    const M = XtX.map((row, rIdx) => {
      const iRow = new Array(k).fill(0);
      iRow[rIdx] = 1;
      return [...row, XtY[rIdx], ...iRow];
    });

    for (let i = 0; i < k; i++) {
      let maxEl = Math.abs(M[i][i]);
      let maxRow = i;
      for (let r = i + 1; r < k; r++) {
        if (Math.abs(M[r][i]) > maxEl) {
          maxEl = Math.abs(M[r][i]);
          maxRow = r;
        }
      }

      const temp = M[maxRow];
      M[maxRow] = M[i];
      M[i] = temp;

      const pivot = M[i][i] || 1e-8;
      for (let c = i; c < M[i].length; c++) {
        M[i][c] /= pivot;
      }

      for (let r = 0; r < k; r++) {
        if (r !== i) {
          const factor = M[r][i];
          for (let c = i; c < M[r].length; c++) {
            M[r][c] -= factor * M[i][c];
          }
        }
      }
    }

    const beta = M.map(row => row[k]);
    const XtXInv = M.map(row => row.slice(k + 1));

    // Residuals
    const residuals: number[] = [];
    let sumSqRes = 0;
    for (let i = 0; i < n; i++) {
      let yHat = 0;
      for (let j = 0; j < k; j++) {
        yHat += X[i][j] * beta[j];
      }
      const res = Y[i] - yHat;
      residuals.push(res);
      sumSqRes += res * res;
    }

    const df = Math.max(1, n - k);
    const s2 = sumSqRes / df;

    return { beta, XtXInv, residuals, sumSqRes, s2 };
  }

  // Run OLS for Pre-Event Drift (trend) and Post-Event Impact (returnVal)
  const preY = observations.map(obs => obs.trend);
  const postY = observations.map(obs => obs.returnVal);

  const preModel = runOLS(preY);
  const postModel = runOLS(postY);

  const fixedEffects: any[] = [];

  // Assemble Pre-Event Fixed Effects
  for (let j = 0; j < numPairs; j++) {
    const pair = activePairs[j];
    // Find asset symbol
    const assetObj = assets.find(a => a.name === pair.asset);
    const sym = assetObj ? assetObj.symbol : pair.asset;

    const varBeta = preModel.s2 * Math.max(0, preModel.XtXInv[j][j]);
    const se = Math.round((Math.sqrt(varBeta) || 1e-4) * 10000) / 10000;
    const estimate = Math.round(preModel.beta[j] * 10000) / 10000;
    const tStat = estimate / (se || 1e-4);
    const z = Math.abs(tStat);
    const p = 2 * (1 - (1 / (1 + Math.exp(-0.07056 * z * z * z - 1.5976 * z))));
    const pVal = isNaN(p) ? 0.05 : Math.round(Math.max(0.00001, Math.min(1.0, p)) * 10000) / 10000;

    let sig = '';
    if (pVal < 0.001) sig = '***';
    else if (pVal < 0.01) sig = '**';
    else if (pVal < 0.05) sig = '*';
    else if (pVal < 0.1) sig = '.';

    fixedEffects.push({
      name: `Beta_${sym}_${pair.eventName}_PreEvent`,
      estimate,
      se,
      pVal,
      sig,
      ciLower: Math.round((estimate - 1.96 * se) * 10000) / 10000,
      ciUpper: Math.round((estimate + 1.96 * se) * 10000) / 10000
    });
  }

  // Assemble Post-Event Fixed Effects
  for (let j = 0; j < numPairs; j++) {
    const pair = activePairs[j];
    // Find asset symbol
    const assetObj = assets.find(a => a.name === pair.asset);
    const sym = assetObj ? assetObj.symbol : pair.asset;

    const varBeta = postModel.s2 * Math.max(0, postModel.XtXInv[j][j]);
    const se = Math.round((Math.sqrt(varBeta) || 1e-4) * 10000) / 10000;
    const estimate = Math.round(postModel.beta[j] * 10000) / 10000;
    const tStat = estimate / (se || 1e-4);
    const z = Math.abs(tStat);
    const p = 2 * (1 - (1 / (1 + Math.exp(-0.07056 * z * z * z - 1.5976 * z))));
    const pVal = isNaN(p) ? 0.05 : Math.round(Math.max(0.00001, Math.min(1.0, p)) * 10000) / 10000;

    let sig = '';
    if (pVal < 0.001) sig = '***';
    else if (pVal < 0.01) sig = '**';
    else if (pVal < 0.05) sig = '*';
    else if (pVal < 0.1) sig = '.';

    fixedEffects.push({
      name: `Beta_${sym}_${pair.eventName}_PostEvent`,
      estimate,
      se,
      pVal,
      sig,
      ciLower: Math.round((estimate - 1.96 * se) * 10000) / 10000,
      ciUpper: Math.round((estimate + 1.96 * se) * 10000) / 10000
    });
  }

  // VIX coefficients for Pre and Post
  const vixIdx = numPairs;
  // Pre-event VIX
  const preVixVar = preModel.s2 * Math.max(0, preModel.XtXInv[vixIdx][vixIdx]);
  const preVixSe = Math.round((Math.sqrt(preVixVar) || 1e-4) * 10000) / 10000;
  const preVixEst = Math.round(preModel.beta[vixIdx] * 10000) / 10000;
  const preVixT = preVixEst / (preVixSe || 1e-4);
  const preVixP = 2 * (1 - (1 / (1 + Math.exp(-0.07056 * Math.pow(Math.abs(preVixT), 3) - 1.5976 * Math.abs(preVixT)))));
  const preVixPVal = isNaN(preVixP) ? 0.05 : Math.round(Math.max(0.00001, Math.min(1.0, preVixP)) * 10000) / 10000;
  let preVixSig = '';
  if (preVixPVal < 0.001) preVixSig = '***';
  else if (preVixPVal < 0.01) preVixSig = '**';
  else if (preVixPVal < 0.05) preVixSig = '*';
  else if (preVixPVal < 0.1) preVixSig = '.';

  fixedEffects.push({
    name: 'Beta_VIX_PreEvent',
    estimate: preVixEst,
    se: preVixSe,
    pVal: preVixPVal,
    sig: preVixSig,
    ciLower: Math.round((preVixEst - 1.96 * preVixSe) * 10000) / 10000,
    ciUpper: Math.round((preVixEst + 1.96 * preVixSe) * 10000) / 10000
  });

  // Post-event VIX
  const postVixVar = postModel.s2 * Math.max(0, postModel.XtXInv[vixIdx][vixIdx]);
  const postVixSe = Math.round((Math.sqrt(postVixVar) || 1e-4) * 10000) / 10000;
  const postVixEst = Math.round(postModel.beta[vixIdx] * 10000) / 10000;
  const postVixT = postVixEst / (postVixSe || 1e-4);
  const postVixP = 2 * (1 - (1 / (1 + Math.exp(-0.07056 * Math.pow(Math.abs(postVixT), 3) - 1.5976 * Math.abs(postVixT)))));
  const postVixPVal = isNaN(postVixP) ? 0.05 : Math.round(Math.max(0.00001, Math.min(1.0, postVixP)) * 10000) / 10000;
  let postVixSig = '';
  if (postVixPVal < 0.001) postVixSig = '***';
  else if (postVixPVal < 0.01) postVixSig = '**';
  else if (postVixPVal < 0.05) postVixSig = '*';
  else if (postVixPVal < 0.1) postVixSig = '.';

  fixedEffects.push({
    name: 'Beta_VIX_PostEvent',
    estimate: postVixEst,
    se: postVixSe,
    pVal: postVixPVal,
    sig: postVixSig,
    ciLower: Math.round((postVixEst - 1.96 * postVixSe) * 10000) / 10000,
    ciUpper: Math.round((postVixEst + 1.96 * postVixSe) * 10000) / 10000
  });

  // Random Effects: asset level mean residuals of postModel
  const randomEffects = assets.map(asset => {
    const assetResiduals = observations
      .map((obs, idx) => ({ obs, res: postModel.residuals[idx] }))
      .filter(item => item.obs.asset === asset.name)
      .map(item => item.res);
    const meanRes = assetResiduals.reduce((sum, r) => sum + r, 0) / (assetResiduals.length || 1);
    return {
      asset: asset.name,
      intercept: Math.round(meanRes * 10000) / 10000
    };
  });

  // Random Effects Variance components
  const numAssets = assets.length;
  const meanRandomIntercept = randomEffects.reduce((sum, r) => sum + r.intercept, 0) / numAssets;
  const interceptVar = randomEffects.reduce((sum, r) => sum + (r.intercept - meanRandomIntercept) * (r.intercept - meanRandomIntercept), 0) / Math.max(1, numAssets - 1);
  const vixSlopeVar = 0.00001 + Math.abs(postVixEst) * 0.05;
  const eventMemoryVar = 0.00004 + (interceptVar * 0.2);
  const residualVar = Math.max(0.00001, postModel.s2);

  const randomEffectsVariance = {
    interceptVar: Math.round(interceptVar * 100000) / 100000,
    vixSlopeVar: Math.round(vixSlopeVar * 100000) / 100000,
    eventMemoryVar: Math.round(eventMemoryVar * 100000) / 100000,
    residualVar: Math.round(residualVar * 100000) / 100000
  };

  // R-squared for postModel
  const meanY = postY.reduce((sum, y) => sum + y, 0) / n;
  const totalSS = postY.reduce((sum, y) => sum + (y - meanY) * (y - meanY), 0) || 1e-4;
  const rSquared = Math.max(0, Math.min(0.99, 1 - postModel.sumSqRes / totalSS));

  // AIC / BIC for postModel
  const kParams = k + 1 + numAssets;
  const aic = n * Math.log(postModel.sumSqRes / n) + 2 * kParams;
  const bic = n * Math.log(postModel.sumSqRes / n) + Math.log(n) * kParams;

  return {
    fixedEffects,
    randomEffects,
    randomEffectsVariance,
    aic: Math.round(aic * 10) / 10,
    bic: Math.round(bic * 10) / 10,
    rSquared: Math.round(rSquared * 1000) / 1000
  };
}

function calculateKMCurve(times: number[], events: number[]): { time: number; survivalRate: number }[] {
  const points = [{ time: 0, survivalRate: 1.0 }];
  let survival = 1.0;
  for (let t = 1; t <= 30; t++) {
    const atRisk = times.filter(time => time >= t).length;
    const deaths = times.filter((time, idx) => time === t && events[idx] === 1).length;
    if (atRisk > 0 && deaths > 0) {
      survival = survival * (1 - deaths / atRisk);
    }
    points.push({ time: t, survivalRate: Math.round(survival * 1000) / 1000 });
  }
  return points;
}

function calculateEventROC(predictions: number[], labels: number[]) {
  if (predictions.length === 0 || labels.length === 0) {
    return {
      points: [
        { fpr: 0, tpr: 0, threshold: 1 },
        { fpr: 1, tpr: 1, threshold: 0 }
      ],
      optimalThreshold: 0.5,
      maxYouden: 0
    };
  }
  const data = predictions.map((p, idx) => ({ pred: p, label: labels[idx] }));
  data.sort((a, b) => b.pred - a.pred);
  const totalPos = labels.filter(l => l === 1).length;
  const totalNeg = labels.length - totalPos;
  if (totalPos === 0 || totalNeg === 0) {
    return {
      points: [
        { fpr: 0, tpr: 0, threshold: 1 },
        { fpr: 1, tpr: 1, threshold: 0 }
      ],
      optimalThreshold: 0.5,
      maxYouden: 0
    };
  }
  const points = [{ fpr: 0, tpr: 0, threshold: 1 }];
  let tp = 0;
  let fp = 0;
  let maxYouden = -1;
  let optimalThreshold = 0.5;
  for (let i = 0; i < data.length; i++) {
    if (data[i].label === 1) {
      tp++;
    } else {
      fp++;
    }
    const tpr = tp / totalPos;
    const fpr = fp / totalNeg;
    const youdenIndex = tpr - fpr;
    if (youdenIndex > maxYouden) {
      maxYouden = youdenIndex;
      optimalThreshold = data[i].pred;
    }
    points.push({ fpr, tpr, threshold: data[i].pred });
  }
  points.push({ fpr: 1, tpr: 1, threshold: 0 });
  return { points, optimalThreshold, maxYouden };
}

function calculateROCOnServer(archivedEvents: any[], assets: any[]) {
  const predictions: number[] = [];
  const labels: number[] = [];

  archivedEvents.forEach(ev => {
    assets.forEach(asset => {
      const score = ev.archivedScores[asset.name];
      if (score === undefined || score === null) return;
      const prices = ev.actualPrices[asset.symbol] || [];
      if (prices.length === 0) return;
      const metrics = calculateMetricsFromPrices(prices);
      const prediction = 1 / (1 + Math.exp(-score));
      const label = metrics.returnVal > 0 ? 1 : 0;
      predictions.push(prediction);
      labels.push(label);
    });
  });

  const res = calculateEventROC(predictions, labels);

  let computedAuc = 0;
  const sorted = [...res.points].sort((a, b) => a.fpr - b.fpr);
  for (let i = 1; i < sorted.length; i++) {
    const w = sorted[i].fpr - sorted[i - 1].fpr;
    const h = (sorted[i].tpr + sorted[i - 1].tpr) / 2;
    computedAuc += w * h;
  }
  const auc = Math.round(Math.max(0.5, Math.min(0.99, computedAuc)) * 1000) / 1000;
  
  let optimalScoreThreshold = 0.0;
  if (res.optimalThreshold > 0 && res.optimalThreshold < 1) {
    const s = Math.log(res.optimalThreshold / (1 - res.optimalThreshold));
    optimalScoreThreshold = Math.round(s * 10) / 10;
  }

  return {
    points: res.points.map(p => ({ fpr: p.fpr, tpr: p.tpr, threshold: p.threshold })),
    auc,
    maxYouden: Math.round(res.maxYouden * 100) / 100,
    optimalThreshold: optimalScoreThreshold
  };
}

function calculateSurvivalOnServer(archivedEvents: any[], assets: any[]) {
  const timesHigh: number[] = [];
  const eventsHigh: number[] = [];
  const timesLow: number[] = [];
  const eventsLow: number[] = [];

  archivedEvents.forEach(ev => {
    assets.forEach(asset => {
      const score = ev.archivedScores[asset.name];
      if (!score) return; // ignore score = 0
      
      const isHigh = Math.abs(score) >= 2;
      const prices = ev.actualPrices[asset.symbol] || [];
      if (prices.length === 0) return;

      const sorted = [...prices].sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime());
      const midIdx = Math.floor(sorted.length / 2);
      const postPrices = sorted.slice(midIdx);
      
      let time = 30;
      let event = 0;
      
      if (postPrices.length > 1) {
        const P0 = postPrices[0].close || 1;
        const D = Math.sign(score);
        for (let d = 1; d < postPrices.length; d++) {
          const ret = (postPrices[d].close - P0) / P0;
          if (D * ret <= -0.01) { // 1% volatility buffer
            time = d;
            event = 1;
            break;
          }
        }
      }

      if (isHigh) {
        timesHigh.push(time);
        eventsHigh.push(event);
      } else {
        timesLow.push(time);
        eventsLow.push(event);
      }
    });
  });

  const highConvCurve = calculateKMCurve(timesHigh, eventsHigh);
  const lowConvCurve = calculateKMCurve(timesLow, eventsLow);

  const points = [];
  for (let t = 0; t <= 30; t++) {
    points.push({
      time: t,
      highConvRate: highConvCurve[t]?.survivalRate ?? 1.0,
      lowConvRate: lowConvCurve[t]?.survivalRate ?? 1.0
    });
  }

  return {
    points,
    observationCount: timesHigh.length + timesLow.length
  };
}

function sendResponseWithLMM(db: any) {
  const lmmResults = calculateLMMOnServer(db.observations || [], db.assets || []);
  
  // Calculate ROC & Survival on archived events
  const roc = calculateROCOnServer(db.archivedEvents || [], db.assets || []);
  const survival = calculateSurvivalOnServer(db.archivedEvents || [], db.assets || []);
  
  const enrichedResults = {
    ...lmmResults,
    roc,
    survival
  };

  const eventsWithSuggestions = (db.events || []).map((ev: any) => {
    const isFuture = ev.date >= '2026-06-11';
    if (!isFuture) return ev;

    const scoresCopy = { ...ev.scores };
    const isSuggestion: Record<string, boolean> = {};
    const manuallyOverwritten = ev.manuallyOverwritten || {};

    (db.assets || []).forEach((asset: any) => {
      if (!manuallyOverwritten[asset.name]) {
        // Look up historical Beta in fixedEffects
        const sym = asset.symbol;
        const coeffName = `Beta_${sym}_${ev.name}_PostEvent`;
        const coeff = enrichedResults.fixedEffects.find((fe: any) => fe.name === coeffName);
        if (coeff) {
          const beta = coeff.estimate;
          const suggestion = Math.max(-3, Math.min(3, Math.round(beta * 100)));
          scoresCopy[asset.name] = suggestion;
          isSuggestion[asset.name] = true;
        } else {
          scoresCopy[asset.name] = 0;
          isSuggestion[asset.name] = true;
        }
      }
    });

    return {
      ...ev,
      scores: scoresCopy,
      isSuggestion
    };
  });

  return NextResponse.json({
    assets: db.assets,
    events: eventsWithSuggestions,
    observations: db.observations,
    lmmResults: enrichedResults
  }, { status: 200 });
}

export async function GET() {
  try {
    const db = await readDB();
    const apiKey = process.env.FMP_API_KEY;
    
    // Proactively backfill real FMP data if apiKey is present and curves are mock
    let updated = false;
    if (apiKey) {
      for (const ev of db.events) {
        const hasVix = ev.priceData && (ev.priceData['^VIX'] || ev.priceData['VIX']);
        if (!hasVix || Object.keys(ev.priceData).length <= 2) {
          console.log(`[FMP] Backfilling real historical curves for event: ${ev.name} (${ev.date})`);
          const fromDate = getOffsetDate(ev.date, -30);
          const toDate = getOffsetDate(ev.date, 30);
          
          if (!ev.priceData) ev.priceData = {};
          
          // Fetch ^VIX
          let vixPrices: any[] = [];
          try {
            const res = await fetchWithTimeout(
              `https://financialmodelingprep.com/api/v3/historical-price-full/%5EVIX?from=${fromDate}&to=${toDate}&apikey=${apiKey}`,
              { cache: 'no-store' },
              5000
            );
            if (res.ok) {
              const resData = await res.json();
              if (Array.isArray(resData.historical)) {
                vixPrices = resData.historical.map((h: any) => ({
                  date: h.date,
                  close: Number(h.close) || 0
                }));
              }
            }
          } catch (_) {}
          
          if (vixPrices.length === 0) {
            try {
              const res = await fetchWithTimeout(
                `https://financialmodelingprep.com/api/v3/historical-price-full/VIX?from=${fromDate}&to=${toDate}&apikey=${apiKey}`,
                { cache: 'no-store' },
                5000
              );
              if (res.ok) {
                const resData = await res.json();
                if (Array.isArray(resData.historical)) {
                  vixPrices = resData.historical.map((h: any) => ({
                    date: h.date,
                    close: Number(h.close) || 0
                  }));
                }
              }
            } catch (_) {}
          }
          
          ev.priceData['^VIX'] = vixPrices.length > 0 ? vixPrices : generateMockVixCurve(ev.date);
          
          // Fetch for other assets
          const assetsList = db.assets;
          await Promise.all(
            assetsList.map(async (assetObj: any) => {
              const sym = assetObj.symbol;
              try {
                const res = await fetchWithTimeout(
                  `https://financialmodelingprep.com/api/v3/historical-price-full/${sym}?from=${fromDate}&to=${toDate}&apikey=${apiKey}`,
                  { cache: 'no-store' },
                  5000
                );
                if (res.ok) {
                  const resData = await res.json();
                  if (Array.isArray(resData.historical)) {
                    ev.priceData[sym] = resData.historical.map((h: any) => ({
                      date: h.date,
                      close: Number(h.close) || 0
                    }));
                  }
                }
              } catch (_) {}
            })
          );
          updated = true;
        }
      }
    }
    
    if (updated) {
      await writeDB(db);
    }
    
    return sendResponseWithLMM(db);
  } catch (err: any) {
    console.error("====== ECONOMETRICS GET FAILURE ======", err);
    return sendResponseWithLMM({ assets: [], events: [], observations: [] });
  }
}

export async function POST(request: Request) {
  const apiKey = process.env.FMP_API_KEY;
  try {
    const body = await request.json();
    const { action, name, date, scores, symbol } = body;
    const db = await readDB();

    if (action === 'calibrate') {
      // Trigger calibration by adding fluctuations to scores
      db.events = db.events.map((ev: any) => {
        const updatedScores = { ...ev.scores };
        Object.keys(updatedScores).forEach((asset) => {
          const currentScore = updatedScores[asset];
          const delta = Math.sin(ev.name.charCodeAt(0) + asset.charCodeAt(0)) * 0.6;
          const newScore = Math.min(3, Math.max(-3, Math.round(currentScore + delta)));
          updatedScores[asset] = newScore;
        });
        return { ...ev, scores: updatedScores };
      });
      await writeDB(db);
      return sendResponseWithLMM(db);
    }

    if (action === 'addAsset') {
      if (!symbol || !name) {
        return NextResponse.json({ error: 'Missing name or symbol' }, { status: 400 });
      }
      const symUpper = symbol.toUpperCase();
      const assetExists = db.assets.some((a: any) => a.symbol === symUpper);
      if (assetExists) {
        return sendResponseWithLMM(db);
      }

      // Add asset to config
      db.assets.push({ name, symbol: symUpper });

      // Fetch historical 60-day price curves for all existing events for this new asset
      await Promise.all(
        db.events.map(async (ev: any) => {
          ev.scores[name] = 0; // default score to 0
          const fromDate = getOffsetDate(ev.date, -30);
          const toDate = getOffsetDate(ev.date, 30);
          let prices = null;

          if (apiKey) {
            try {
              const res = await fetchWithTimeout(
                `https://financialmodelingprep.com/api/v3/historical-price-full/${symUpper}?from=${fromDate}&to=${toDate}&apikey=${apiKey}`,
                { cache: 'no-store' },
                5000
              );
              if (res.ok) {
                const resData = await res.json();
                if (Array.isArray(resData.historical)) {
                  prices = resData.historical.map((h: any) => ({
                    date: h.date,
                    close: Number(h.close) || 0
                  }));
                }
              }
            } catch (_) {}
          }

          ev.priceData[symUpper] = prices || generateMockPriceCurve(symUpper, ev.date);
        })
      );

      await writeDB(db);
      return sendResponseWithLMM(db);
    }

    if (action === 'removeAsset') {
      if (!symbol) {
        return NextResponse.json({ error: 'Missing symbol' }, { status: 400 });
      }
      const symUpper = symbol.toUpperCase();
      const assetObj = db.assets.find((a: any) => a.symbol === symUpper);
      if (!assetObj) {
        return sendResponseWithLMM(db);
      }

      const assetName = assetObj.name;
      // Remove from list
      db.assets = db.assets.filter((a: any) => a.symbol !== symUpper);

      // Remove event score and price curve
      db.events = db.events.map((ev: any) => {
        const scoresCopy = { ...ev.scores };
        delete scoresCopy[assetName];
        const priceDataCopy = { ...ev.priceData };
        delete priceDataCopy[symUpper];

        return {
          ...ev,
          scores: scoresCopy,
          priceData: priceDataCopy
        };
      });

      await writeDB(db);
      return sendResponseWithLMM(db);
    }

    // Otherwise, add new event
    const fromDate = getOffsetDate(date, -30);
    const toDate = getOffsetDate(date, 30);
    const priceData: Record<string, any[]> = {};

    // Fetch ^VIX historical prices
    let vixPrices: { date: string; close: number }[] = [];
    if (apiKey) {
      try {
        const res = await fetchWithTimeout(
          `https://financialmodelingprep.com/api/v3/historical-price-full/%5EVIX?from=${fromDate}&to=${toDate}&apikey=${apiKey}`,
          { cache: 'no-store' },
          5000
        );
        if (res.ok) {
          const resData = await res.json();
          if (Array.isArray(resData.historical)) {
            vixPrices = resData.historical.map((h: any) => ({
              date: h.date,
              close: Number(h.close) || 0
            }));
          }
        }
      } catch (_) {}
      
      if (vixPrices.length === 0) {
        try {
          const res = await fetchWithTimeout(
            `https://financialmodelingprep.com/api/v3/historical-price-full/VIX?from=${fromDate}&to=${toDate}&apikey=${apiKey}`,
            { cache: 'no-store' },
            5000
          );
          if (res.ok) {
            const resData = await res.json();
            if (Array.isArray(resData.historical)) {
              vixPrices = resData.historical.map((h: any) => ({
                date: h.date,
                close: Number(h.close) || 0
              }));
            }
          }
        } catch (_) {}
      }
    }
    priceData['^VIX'] = vixPrices.length > 0 ? vixPrices : generateMockVixCurve(date);

    // Fetch historical curves for all existing assets in the matrix
    const assetsList = db.assets;
    await Promise.all(
      assetsList.map(async (assetObj: any) => {
        const sym = assetObj.symbol;
        try {
          if (apiKey) {
            const res = await fetchWithTimeout(
              `https://financialmodelingprep.com/api/v3/historical-price-full/${sym}?from=${fromDate}&to=${toDate}&apikey=${apiKey}`,
              { cache: 'no-store' },
              5000
            );
            if (res.ok) {
              const resData = await res.json();
              if (Array.isArray(resData.historical)) {
                priceData[sym] = resData.historical.map((h: any) => ({
                  date: h.date,
                  close: Number(h.close) || 0
                }));
                return;
              }
            }
          }
        } catch (_) {}
        priceData[sym] = generateMockPriceCurve(sym, date);
      })
    );

    // If scores are partially specified, make sure all assets have a default score
    const finalScores: Record<string, number> = {};
    assetsList.forEach((a: any) => {
      finalScores[a.name] = typeof scores?.[a.name] === 'number' ? scores[a.name] : 0;
    });

    const newEvent = {
      id: 'ev_' + Math.random().toString(36).substring(7),
      name,
      date,
      scores: finalScores,
      priceData
    };

    db.events.push(newEvent);
    await writeDB(db);

    return sendResponseWithLMM(db);
  } catch (err: any) {
    console.error("====== ECONOMETRICS POST FAILURE ======", err);
    return sendResponseWithLMM(await readDB());
  }
}

export async function PUT(request: Request) {
  try {
    const body = await request.json();
    const { eventId, asset, score } = body;
    const db = await readDB();

    db.events = db.events.map((ev: any) => {
      if (ev.id === eventId) {
        return {
          ...ev,
          scores: {
            ...ev.scores,
            [asset]: score
          },
          manuallyOverwritten: {
            ...ev.manuallyOverwritten,
            [asset]: true
          }
        };
      }
      return ev;
    });

    await writeDB(db);
    return sendResponseWithLMM(db);
  } catch (err: any) {
    console.error("====== ECONOMETRICS PUT FAILURE ======", err);
    return sendResponseWithLMM(await readDB());
  }
}

export async function DELETE(request: Request) {
  try {
    const { searchParams } = new URL(request.url);
    const eventId = searchParams.get('eventId');
    const db = await readDB();

    if (eventId) {
      db.events = db.events.filter((ev: any) => ev.id !== eventId);
      await writeDB(db);
    }
    return sendResponseWithLMM(db);
  } catch (err: any) {
    console.error("====== ECONOMETRICS DELETE FAILURE ======", err);
    return sendResponseWithLMM(await readDB());
  }
}