import { ChatOpenAI } from '@langchain/openai';
import { ChatPromptTemplate } from '@langchain/core/prompts';
import { StringOutputParser } from '@langchain/core/output_parsers';

// --- Configuration ---
const llm = new ChatOpenAI({
  model: 'gpt-4o-mini',
  temperature: 0.7,
});

// --- Prompt Optimization System ---

class PromptOptimizer {
  constructor() {
    this.promptHistory = [];
    this.performanceScores = [];
  }

  // Analyze feedback and generate improved prompt
  async optimizePrompt(originalPrompt, feedback) {
    const optimizationPrompt = ChatPromptTemplate.fromMessages([
      [
        'system',
        `You are a Prompt Optimization Expert. Your task is to improve prompts based on feedback.
Analyze the original prompt and feedback, then create an improved version.

Original Prompt: {original}
Feedback: {feedback}

Provide:
1. Issues identified
2. Improved prompt
3. Expected improvement`,
      ],
      ['user', 'Optimize this prompt.'],
    ]);

    const chain = optimizationPrompt.pipe(llm).pipe(new StringOutputParser());
    const result = await chain.invoke({
      original: originalPrompt,
      feedback: feedback,
    });

    return result;
  }

  // Track prompt performance
  recordPerformance(prompt, score) {
    this.promptHistory.push({ prompt, score, timestamp: Date.now() });
    this.performanceScores.push(score);
  }

  // Get best performing prompt
  getBestPrompt() {
    if (this.promptHistory.length === 0) return null;
    const best = this.promptHistory.reduce((a, b) => (a.score > b.score ? a : b));
    return best.prompt;
  }

  // Get average performance
  getAverageScore() {
    if (this.performanceScores.length === 0) return 0;
    return this.performanceScores.reduce((a, b) => a + b, 0) / this.performanceScores.length;
  }
}

// --- Example Usage ---

async function runPromptOptimization() {
  const optimizer = new PromptOptimizer();

  // Initial prompt
  let currentPrompt = 'Explain quantum computing to a 5-year-old.';

  console.log('=== Prompt Optimization Demo ===\n');

  // Simulate feedback and optimization rounds
  const feedbackExamples = [
    'The explanation was too technical and used too many jargon words.',
    'Good, but could be more engaging with analogies.',
    'Perfect! The child understood the basic concept.',
  ];

  for (let i = 0; i < feedbackExamples.length; i++) {
    console.log(`--- Round ${i + 1} ---`);
    console.log(`Original Prompt: ${currentPrompt}`);
    console.log(`Feedback: ${feedbackExamples[i]}\n`);

    const improvement = await optimizer.optimizePrompt(currentPrompt, feedbackExamples[i]);
    console.log('Optimization Result:', improvement.substring(0, 300) + '...\n');

    // Simulate performance score (would come from actual evaluation)
    const score = Math.random() * 0.4 + 0.6; // Random score between 0.6-1.0
    optimizer.recordPerformance(currentPrompt, score);
    console.log(`Performance Score: ${score.toFixed(2)}\n`);

    // Update prompt for next round (in real system, extract the optimized prompt)
    currentPrompt = improvement.split('\n')[1] || currentPrompt;
  }

  console.log('=== Summary ===');
  console.log('Best Prompt:', optimizer.getBestPrompt());
  console.log('Average Score:', optimizer.getAverageScore().toFixed(2));
}

runPromptOptimization();

import { ChatOpenAI } from '@langchain/openai';
import { ChatPromptTemplate } from '@langchain/core/prompts';
import { StringOutputParser } from '@langchain/core/output_parsers';

// --- Configuration ---
const llm = new ChatOpenAI({
  model: 'gpt-4o-mini',
  temperature: 0.7,
});

// --- Tool Definitions ---
const tools = {
  search: {
    name: 'web_search',
    description: 'Best for: current events, latest news, real-time information',
    strengths: ['fresh information', 'broad coverage'],
    bestFor: ['news', 'current events', 'trends'],
  },
  database: {
    name: 'database_query',
    description: 'Best for: factual data, statistics, historical records',
    strengths: ['accurate data', 'structured information'],
    bestFor: ['facts', 'numbers', 'statistics', 'historical'],
  },
  calculator: {
    name: 'math_tool',
    description: 'Best for: calculations, computations, numerical analysis',
    strengths: ['precision', 'complex calculations'],
    bestFor: ['math', 'calculations', 'numbers', 'computation'],
  },
  wikipedia: {
    name: 'wikipedia',
    description: 'Best for: general knowledge, definitions, overviews',
    strengths: ['broad knowledge', 'definitions'],
    bestFor: ['definitions', 'general knowledge', 'concepts'],
  },
};

// --- Tool Selection Learning System ---

class ToolSelectionLearner {
  constructor() {
    this.taskHistory = [];
    this.toolPerformance = {};
  }

  // Analyze task and recommend best tool
  async selectTool(task) {
    const selectionPrompt = ChatPromptTemplate.fromMessages([
      [
        'system',
        `You are a Tool Selection Expert. Analyze the task and select the best tool.
Available tools:
- search: For current events and latest information
- database: For factual data and statistics
- calculator: For mathematical calculations
- wikipedia: For general knowledge and definitions

Task: {task}

Analyze the task keywords and select the most appropriate tool.
Provide your reasoning.`,
      ],
      ['user', 'Select the best tool.'],
    ]);

    const chain = selectionPrompt.pipe(llm).pipe(new StringOutputParser());
    const result = await chain.invoke({ task: task });

    // Extract tool name from response
    const selectedTool =
      Object.keys(tools).find((t) => result.toLowerCase().includes(t)) || 'search';

    return { tool: selectedTool, reasoning: result };
  }

  // Record task outcome and update performance
  recordOutcome(task, selectedTool, success) {
    this.taskHistory.push({ task, selectedTool, success, timestamp: Date.now() });

    if (!this.toolPerformance[selectedTool]) {
      this.toolPerformance[selectedTool] = { successes: 0, total: 0 };
    }

    this.toolPerformance[selectedTool].total++;
    if (success) {
      this.toolPerformance[selectedTool].successes++;
    }
  }

  // Get recommended tool based on history
  getRecommendedTool(taskKeywords) {
    let bestTool = 'search';
    let bestScore = 0;

    for (const [toolName, performance] of Object.entries(this.toolPerformance)) {
      const score = performance.successes / performance.total;
      // Check if tool is suitable for task keywords
      const isSuitable = tools[toolName].bestFor.some((kw) =>
        taskKeywords.toLowerCase().includes(kw),
      );

      if (isSuitable && score > bestScore) {
        bestScore = score;
        bestTool = toolName;
      }
    }

    return bestTool;
  }

  // Get performance statistics
  getPerformanceStats() {
    return Object.entries(this.toolPerformance).map(([tool, perf]) => ({
      tool,
      successRate: ((perf.successes / perf.total) * 100).toFixed(1) + '%',
      totalUses: perf.total,
    }));
  }
}

// --- Example Usage ---

async function runToolSelectionLearning() {
  const learner = new ToolSelectionLearner();

  console.log('=== Tool Selection Learning Demo ===\n');

  // Simulate task execution with feedback
  const tasks = [
    { task: 'What is the population of Tokyo?', expectedTool: 'database', success: true },
    { task: 'Calculate 15% of 2500', expectedTool: 'calculator', success: true },
    { task: 'What happened in the news today?', expectedTool: 'search', success: true },
    { task: 'Who was Albert Einstein?', expectedTool: 'wikipedia', success: true },
    { task: 'What is machine learning?', expectedTool: 'wikipedia', success: true },
  ];

  for (const { task, success } of tasks) {
    const { tool, reasoning } = await learner.selectTool(task);
    console.log(`Task: ${task}`);
    console.log(`Selected Tool: ${tool}`);
    console.log(`Reasoning: ${reasoning.substring(0, 100)}...`);
    console.log(`Success: ${success}\n`);

    learner.recordOutcome(task, tool, success);
  }

  console.log('=== Performance Statistics ===');
  console.log(learner.getPerformanceStats());

  // Test learned recommendation
  console.log('\n=== Learned Recommendations ===');
  console.log("For 'math problem':", learner.getRecommendedTool('math problem'));
  console.log("For 'latest news':", learner.getRecommendedTool('latest news'));
}

runToolSelectionLearning();

import { ChatOpenAI } from '@langchain/openai';
import { ChatPromptTemplate } from '@langchain/core/prompts';
import { StringOutputParser } from '@langchain/core/output_parsers';

// --- Configuration ---
const llm = new ChatOpenAI({
  model: 'gpt-4o-mini',
  temperature: 0.7,
});

// --- Strategy Adjustment System ---

const strategies = {
  direct: {
    name: 'Direct Answer',
    description: 'Provide answer immediately without elaboration',
    bestFor: ['simple facts', 'clear questions'],
  },
  detailed: {
    name: 'Detailed Explanation',
    description: 'Provide comprehensive answer with examples',
    bestFor: ['complex topics', 'educational'],
  },
  conversational: {
    name: 'Conversational',
    description: 'Friendly, engaging tone with follow-up',
    bestFor: ['opinions', 'discussions'],
  },
  analytical: {
    name: 'Analytical',
    description: 'Step-by-step analysis with data',
    bestFor: ['analysis', 'comparisons'],
  },
};

class StrategyAdjuster {
  constructor() {
    this.strategyHistory = [];
    this.strategyPerformance = {};
  }

  // Analyze task and select initial strategy
  async selectStrategy(task) {
    const analysisPrompt = ChatPromptTemplate.fromMessages([
      [
        'system',
        `Analyze this task and select the best response strategy.
Strategies:
- direct: For simple factual questions
- detailed: For complex explanations
- conversational: For discussions and opinions
- analytical: For analysis and comparisons

Task: {task}

Select the most appropriate strategy and explain why.`,
      ],
      ['user', 'Analyze and select strategy.'],
    ]);

    const chain = analysisPrompt.pipe(llm).pipe(new StringOutputParser());
    const result = await chain.invoke({ task });

    const selectedStrategy =
      Object.keys(strategies).find((s) => result.toLowerCase().includes(s)) || 'direct';

    return { strategy: selectedStrategy, analysis: result };
  }

  // Execute task with selected strategy
  async executeWithStrategy(task, strategyKey) {
    const strategy = strategies[strategyKey];

    const executionPrompt = ChatPromptTemplate.fromMessages([
      [
        'system',
        `{strategy}

Task: {task}

Respond using the {strategy} strategy.`,
      ],
      ['user', '{task}'],
    ]);

    const chain = executionPrompt.pipe(llm).pipe(new StringOutputParser());
    return chain.invoke({ task, strategy: strategy.description });
  }

  // Adjust strategy based on outcome
  async adjustStrategy(task, currentStrategy, wasSuccessful, feedback) {
    if (wasSuccessful) {
      console.log(`Strategy ${currentStrategy} worked well! Maintaining...`);
      return currentStrategy;
    }

    console.log(`Strategy ${currentStrategy} failed. Analyzing adjustment...`);

    const adjustmentPrompt = ChatPromptTemplate.fromMessages([
      [
        'system',
        `Analyze why the previous strategy failed and recommend an adjustment.
Previous Strategy: {currentStrategy}
Task: {task}
Feedback: {feedback}

Available strategies:
- direct: For simple factual questions
- detailed: For complex explanations
- conversational: For discussions and opinions
- analytical: For analysis and comparisons

Recommend a new strategy and explain the change.`,
      ],
      ['user', 'Analyze and recommend adjustment.'],
    ]);

    const chain = adjustmentPrompt.pipe(llm).pipe(new StringOutputParser());
    const result = await chain.invoke({
      currentStrategy: strategies[currentStrategy].name,
      task,
      feedback,
    });

    const newStrategy =
      Object.keys(strategies).find((s) => result.toLowerCase().includes(s)) || 'detailed';

    // Record performance
    this.recordPerformance(currentStrategy, false);
    this.recordPerformance(newStrategy, true);

    return newStrategy;
  }

  recordPerformance(strategy, success) {
    if (!this.strategyPerformance[strategy]) {
      this.strategyPerformance[strategy] = { successes: 0, failures: 0 };
    }

    if (success) {
      this.strategyPerformance[strategy].successes++;
    } else {
      this.strategyPerformance[strategy].failures++;
    }
  }

  getBestStrategy() {
    let best = null;
    let bestScore = 0;

    for (const [strategy, perf] of Object.entries(this.strategyPerformance)) {
      const total = perf.successes + perf.failures;
      const score = perf.successes / total;
      if (score > bestScore) {
        bestScore = score;
        best = strategy;
      }
    }

    return best;
  }
}

// --- Example Usage ---

async function runStrategyAdjustment() {
  const adjuster = new StrategyAdjuster();

  console.log('=== Strategy Adjustment Demo ===\n');

  // Simulate task execution with strategy adjustment
  const taskExamples = [
    { task: 'What is 2+2?', feedback: 'Too detailed', success: false },
    { task: 'Explain quantum physics', feedback: 'Too brief', success: false },
    { task: 'What do you think about AI?', feedback: 'Good engagement', success: true },
    { task: 'Compare Python and JavaScript', feedback: 'Good analysis', success: true },
  ];

  let currentStrategy = 'direct';

  for (const { task, feedback, success } of taskExamples) {
    console.log(`--- Task: ${task} ---`);
    console.log(`Current Strategy: ${currentStrategy}`);

    // Execute with current strategy
    const response = await adjuster.executeWithStrategy(task, currentStrategy);
    console.log(`Response: ${response.substring(0, 100)}...`);

    // Adjust if needed
    currentStrategy = await adjuster.adjustStrategy(task, currentStrategy, success, feedback);

    console.log(`New Strategy: ${currentStrategy}\n`);
  }

  console.log('=== Best Performing Strategy ===');
  console.log(adjuster.getBestStrategy());
}

runStrategyAdjustment();

import { ChatOpenAI } from '@langchain/openai';
import { ChatPromptTemplate } from '@langchain/core/prompts';
import { StringOutputParser } from '@langchain/core/output_parsers';

// --- Configuration ---
const llm = new ChatOpenAI({
  model: 'gpt-4o-mini',
  temperature: 0.7,
});

// --- User Preference Learning System ---

class PersonalizedAgent {
  constructor(userId) {
    this.userId = userId;
    this.preferences = {
      detailLevel: 'medium', // brief, medium, detailed
      tone: 'professional', // casual, professional, friendly
      format: 'paragraph', // bullet, paragraph, numbered
      topics: [],
    };
    this.interactionHistory = [];
  }

  // Learn from user feedback
  learnFromFeedback(feedback) {
    const feedback_lower = feedback.toLowerCase();

    // Learn detail level preferences
    if (feedback_lower.includes('too brief') || feedback_lower.includes('more detail')) {
      this.preferences.detailLevel = 'detailed';
    } else if (feedback_lower.includes('too long') || feedback_lower.includes('too much')) {
      this.preferences.detailLevel = 'brief';
    }

    // Learn tone preferences
    if (feedback_lower.includes('too casual') || feedback_lower.includes('more formal')) {
      this.preferences.tone = 'professional';
    } else if (feedback_lower.includes('too formal') || feedback_lower.includes('friendlier')) {
      this.preferences.tone = 'friendly';
    }

    // Learn format preferences
    if (feedback_lower.includes('bullet') || feedback_lower.includes('list')) {
      this.preferences.format = 'bullet';
    } else if (feedback_lower.includes('numbered') || feedback_lower.includes('steps')) {
      this.preferences.format = 'numbered';
    }

    // Learn topic interests
    const topicKeywords = ['AI', 'machine learning', 'technology', 'science', 'business'];
    for (const topic of topicKeywords) {
      if (feedback_lower.includes(topic.toLowerCase())) {
        if (!this.preferences.topics.includes(topic)) {
          this.preferences.topics.push(topic);
        }
      }
    }

    this.interactionHistory.push({ feedback, timestamp: Date.now() });
  }

  // Generate personalized response
  async generateResponse(query) {
    // Build personalization instructions
    const detailInstructions = {
      brief: 'Keep your response concise and to the point.',
      medium: 'Provide a balanced response with moderate detail.',
      detailed: 'Provide comprehensive details with examples and explanations.',
    };

    const toneInstructions = {
      casual: 'Use a casual, relaxed tone.',
      professional: 'Use a professional, formal tone.',
      friendly: 'Use a friendly, warm tone.',
    };

    const formatInstructions = {
      bullet: 'Use bullet points for key information.',
      paragraph: 'Use flowing paragraphs.',
      numbered: 'Use numbered lists for steps or items.',
    };

    const personalization = `
${detailInstructions[this.preferences.detailLevel]}
${toneInstructions[this.preferences.tone]}
${formatInstructions[this.preferences.format]}
${this.preferences.topics.length > 0 ? `Consider mentioning related topics: ${this.preferences.topics.join(', ')}` : ''}
`.trim();

    const prompt = ChatPromptTemplate.fromMessages([
      [
        'system',
        `You are a personalized assistant. Adapt your responses based on user preferences.
User Preferences:
- Detail Level: ${this.preferences.detailLevel}
- Tone: ${this.preferences.tone}
- Format: ${this.preferences.format}
- Interested Topics: ${this.preferences.topics.join(', ') || 'None specified'}

Guidelines:
${personalization}`,
      ],
      ['user', '{query}'],
    ]);

    const chain = prompt.pipe(llm).pipe(new StringOutputParser());
    return chain.invoke({ query });
  }

  // Get current preferences
  getPreferences() {
    return { ...this.preferences };
  }
}

// --- Example Usage ---

async function runPersonalizedAgent() {
  const agent = new PersonalizedAgent('user_123');

  console.log('=== Personalized Agent Demo ===\n');

  // Initial query
  console.log('--- Initial Query ---');
  let response = await agent.generateResponse('What is machine learning?');
  console.log('Response:', response.substring(0, 200) + '...\n');

  // User provides feedback
  console.log("--- User Feedback: 'Too brief, I want more details and examples' ---");
  agent.learnFromFeedback('Too brief, I want more details and examples');
  console.log('Updated Preferences:', agent.getPreferences());

  // Query again with learned preferences
  console.log('\n--- Second Query ---');
  response = await agent.generateResponse('What is machine learning?');
  console.log('Response:', response.substring(0, 200) + '...\n');

  // More feedback
  console.log("--- User Feedback: 'Too formal, be friendlier and use bullet points' ---");
  agent.learnFromFeedback('Too formal, be friendlier and use bullet points');
  console.log('Updated Preferences:', agent.getPreferences());

  // Query with updated preferences
  console.log('\n--- Third Query ---');
  response = await agent.generateResponse('What is machine learning?');
  console.log('Response:', response.substring(0, 200) + '...\n');

  console.log('=== Interaction History ===');
  console.log('Total feedback sessions:', agent.interactionHistory.length);
}

runPersonalizedAgent();

import { ChatOpenAI } from '@langchain/openai';
import { ChatPromptTemplate } from '@langchain/core/prompts';
import { StringOutputParser } from '@langchain/core/output_parsers';

const llm = new ChatOpenAI({ temperature: 0.7 });

// 投资智能体 - 基于内存的经验学习系统
class TradingMemoryLearner {
  constructor(llm) {
    this.llm = llm;
    this.episodicMemory = []; // 情节记忆库
  }

  // 记录交易经验
  async recordTradeExperience(trade) {
    const experience = {
      id: `trade_${Date.now()}`,
      timestamp: new Date().toISOString(),
      marketState: trade.marketState, // 市场状态：牛市/熊市/震荡
      technicalIndicators: trade.technicalIndicators, // 技术指标
      action: trade.action, // 买入/卖出/持有
      reasoning: trade.reasoning, // 交易理由
      result: trade.result, // profit/loss
      pnl: trade.pnl || null, // 盈亏金额
      lesson: null, // 经验教训（后续生成）
    };

    // 如果有结果，生成经验教训
    if (trade.result) {
      experience.lesson = await this.generateLesson(experience);
    }

    this.episodicMemory.push(experience);
    console.log(
      `已记录交易经验: ${experience.action} ${experience.marketState} -> ${experience.result}`,
    );
    return experience;
  }

  // 生成交易经验教训
  async generateLesson(experience) {
    const lessonPrompt = `基于以下交易记录，总结关键经验教训:

市场状态: ${experience.marketState}
技术指标: ${experience.technicalIndicators}
交易操作: ${experience.action}
交易理由: ${experience.reasoning}
结果: ${experience.result}
盈亏: ${experience.pnl}

请用一句话总结最重要的经验教训`;

    const chain = ChatPromptTemplate.fromTemplate(lessonPrompt)
      .pipe(this.llm)
      .pipe(new StringOutputParser());

    return chain.invoke({});
  }

  // RAG检索相似经验
  async retrieveSimilarExperience(currentMarketState, currentIndicators) {
    // 简单关键词匹配（生产环境应使用向量相似度）
    const relevant = this.episodicMemory.filter((e) => {
      const stateMatch = e.marketState === currentMarketState;
      const hasLesson = e.lesson !== null;
      return stateMatch && hasLesson;
    });

    // 按相关性排序
    relevant.sort((a, b) => {
      const aWin = a.result === 'profit' ? 1 : 0;
      const bWin = b.result === 'profit' ? 1 : 0;
      return bWin - aWin;
    });

    return relevant.slice(0, 3);
  }

  // 基于历史经验做决策
  async makeDecisionWithMemory(currentMarketState, currentIndicators, proposedAction) {
    const similarExp = await this.retrieveSimilarExperience(currentMarketState, currentIndicators);

    if (similarExp.length === 0) {
      return { decision: proposedAction, warning: null, fromHistory: false };
    }

    // 分析历史经验
    const profitableCount = similarExp.filter((e) => e.result === 'profit').length;
    const totalCount = similarExp.length;
    const successRate = profitableCount / totalCount;

    let warning = null;
    let adjustedAction = proposedAction;

    if (successRate < 0.4) {
      warning = `警告: 在类似市场状态(${currentMarketState})下，过去${totalCount}次相同操作中有${totalCount - profitableCount}次亏损，成功率仅${(successRate * 100).toFixed(0)}%`;
      // 建议改变策略
      adjustedAction = proposedAction === 'buy' ? 'hold' : proposedAction;
    }

    console.log(`\n--- 历史经验分析 ---`);
    console.log(`相似经验数量: ${totalCount}`);
    console.log(`成功率: ${(successRate * 100).toFixed(0)}%`);
    similarExp.forEach((e) => {
      console.log(`  - ${e.action} ${e.marketState}: ${e.result} (${e.lesson})`);
    });

    return {
      decision: adjustedAction,
      warning,
      fromHistory: true,
      successRate,
      similarExp,
    };
  }

  // 获取所有经验教训
  getAllLessons() {
    return this.episodicMemory
      .filter((e) => e.lesson)
      .map((e) => ({
        marketState: e.marketState,
        action: e.action,
        result: e.result,
        lesson: e.lesson,
      }));
  }
}

// 使用示例
const memoryLearner = new TradingMemoryLearner(llm);

// 模拟历史交易记录
await memoryLearner.recordTradeExperience({
  marketState: '牛市',
  technicalIndicators: 'RSI超买',
  action: 'buy',
  reasoning: '基本面良好，追高买入',
  result: 'loss',
  pnl: -1500,
});

await memoryLearner.recordTradeExperience({
  marketState: '牛市',
  technicalIndicators: 'RSI超买',
  action: 'buy',
  reasoning: '技术面超买但基本面强劲',
  result: 'loss',
  pnl: -800,
});

await memoryLearner.recordTradeExperience({
  marketState: '牛市',
  technicalIndicators: 'RSI超买',
  action: 'hold',
  reasoning: '等待回调',
  result: 'profit',
  pnl: 500,
});

// 新交易决策
console.log('\n=== 新交易决策 ===');
const decision = await memoryLearner.makeDecisionWithMemory('牛市', 'RSI超买', 'buy');
console.log(`建议操作: ${decision.decision}`);
if (decision.warning) console.log(`警告: ${decision.warning}`);

import { ChatOpenAI } from '@langchain/openai';
import { ChatPromptTemplate } from '@langchain/core/prompts';
import { StringOutputParser } from '@langchain/core/output_parsers';

const llm = new ChatOpenAI({ temperature: 0.7 });

// 投资智能体 - 强化学习与偏好对齐系统
class RLTradingAgent {
  constructor(userId) {
    this.userId = userId;
    this.qTable = {};           // Q学习表
    this.strategies = ['conservative', 'moderate', 'aggressive'];
    this.userPreferences = {
      riskTolerance: 'moderate',
      preferredStrategy: null,
    };
    this.tradeHistory = [];
    this.initializeQTable();
  }

  // 初始化Q表
  initializeQTable() {
    const states = ['bull', 'bear', 'volatile', 'stable'];
    const actions = ['buy', 'sell', 'hold'];

    for (const state of states) {
      this.qTable[state] = {};
      for (const action of actions) {
        // 初始Q值为0
        this.qTable[state][action] = 0;
      }
    }
  }

  // 选择动作（epsilon-greedy策略）
  chooseAction(state, epsilon = 0.1) {
    if (Math.random() < epsilon) {
      // 探索：随机选择
      const actions = Object.keys(this.qTable[state] || {});
      return actions[Math.floor(Math.random() * actions.length)];
    } else {
      // 利用：选择Q值最高的动作
      const qValues = this.qTable[state];
      return Object.entries(qValues).sort((a, b) => b[1] - a[1])[0][0];
    }
  }

  // 执行交易并获取奖励
  executeTrade(state, action, actualResult) {
    // 奖励函数设计
    let reward = 0;

    if (actualResult === 'profit') {
      reward = 100; // 盈利奖励
      // 根据用户风险偏好调整
      if (this.userPreferences.riskTolerance === 'conservative' && action === 'aggressive') {
        reward -= 20; // 保守用户做激进操作惩罚
      }
    } else if (actualResult === 'loss') {
      reward = -100; // 亏损惩罚
    } else {
      reward = 10; // 持有（中性）
    }

    // 更新Q值
    this.updateQValue(state, action, reward);
    this.tradeHistory.push({ state, action, result: actualResult, reward });

    return reward;
  }

  // Q值更新（Q-learning公式）
  updateQValue(state, action, reward, alpha = 0.1, gamma = 0.9) {
    const currentQ = this.qTable[state][action];
    const maxNextQ = Math.max(...Object.values(this.qTable[state] || { buy: 0, sell: 0, hold: 0 }));

    // Q(s,a) = Q(s,a) + alpha * (reward + gamma * maxQ(s') - Q(s,a))
    this.qTable[state][action] = currentQ + alpha * (reward + gamma * maxNextQ - currentQ);
  }

  // 获取当前最优策略
  getBestPolicy() {
    const policy = {};
    for (const state of Object.keys(this.qTable)) {
      const actions = this.qTable[state];
      const best = Object.entries(actions).sort((a, b) => b[1] - a0];
[1])[      policy[state] = { action: best[0], qValue: best[1] };
    }
    return policy;
  }

  // DPO: 根据用户反馈调整偏好
  alignWithPreference(userFeedback) {
    const feedback = userFeedback.toLowerCase();

    // 分析用户偏好
    if (feedback.includes('稳健') || feedback.includes('保守') || feedback.includes('低风险')) {
      this.userPreferences.riskTolerance = 'conservative';
      this.userPreferences.preferredStrategy = 'conservative';
    } else if (feedback.includes('激进') || feedback.includes('高风险') || feedback.includes('高回报')) {
      this.userPreferences.riskTolerance = 'aggressive';
      this.userPreferences.preferredStrategy = 'aggressive';
    } else {
      this.userPreferences.riskTolerance = 'moderate';
    }

    // 调整Q表权重（模拟DPO）
    this.adjustQTableForPreference();
    console.log(`用户偏好已更新: ${this.userPreferences.riskTolerance}`);
  }

  // 根据偏好调整Q表
  adjustQTableForPreference() {
    const preferenceMultiplier = {
      conservative: { buy: 0.7, hold: 1.3, sell: 1.0 },
      moderate: { buy: 1.0, hold: 1.0, sell: 1.0 },
      aggressive: { buy: 1.5, hold: 0.7, sell: 0.8 },
    };

    const mult = preferenceMultiplier[this.userPreferences.riskTolerance];
    for (const state of Object.keys(this.qTable)) {
      for (const action of Object.keys(this.qTable[state])) {
        this.qTable[state][action] *= mult[action];
      }
    }
  }

  // 获取学习统计
  getStats() {
    const wins = this.tradeHistory.filter(t => t.result === 'profit').length;
    const total = this.tradeHistory.length;
    return {
      totalTrades: total,
      wins,
      winRate: total > 0 ? ((wins / total) * 100).toFixed(1) + '%') : 'N/A',
      currentPolicy: this.getBestPolicy(),
    };
  }
}

// 使用示例
const rlAgent = new RLTradingAgent('user001');

// 模拟多轮交易学习
const marketStates = ['bull', 'bull', 'volatile', 'bear', 'volatile', 'bull'];
const actions = ['buy', 'buy', 'buy', 'sell', 'hold', 'buy'];
const results = ['profit', 'loss', 'loss', 'profit', 'profit', 'profit'];

console.log('=== 强化学习训练 ===');
for (let i = 0; i < marketStates.length; i++) {
  const state = marketStates[i];
  const action = actions[i];
  const result = results[i];

  // 选择动作
  const chosenAction = rlAgent.chooseAction(state);
  // 执行并获取奖励
  const reward = rlAgent.executeTrade(state, chosenAction, result);
  console.log(`状态: ${state}, 选择: ${chosenAction}, 结果: ${result}, 奖励: ${reward}`);
}

// DPO偏好对齐
console.log('\n=== DPO偏好对齐 ===');
rlAgent.alignWithPreference('我比较保守，希望稳健收益');

// 查看学习结果
console.log('\n=== 学习结果 ===');
console.log(rlAgent.getStats());
console.log('\n当前策略:');
console.log(rlAgent.getBestPolicy());

import { ChatOpenAI } from '@langchain/openai';

// 投资智能体 - 实时在线学习系统
class OnlineLearningTrader {
  constructor() {
    this.dataStream = []; // 数据流缓冲区
    this.indicatorWeights = {
      // 技术指标权重（可学习）
      rsi: 0.25,
      macd: 0.25,
      movingAvg: 0.25,
      volume: 0.25,
    };
    this.marketSentiment = 'neutral'; // 市场情绪
    this.isLearning = false;
  }

  // 接收实时数据流
  async ingestDataStream(data) {
    this.dataStream.push({
      ...data,
      timestamp: Date.now(),
    });

    // 保持最近100条数据
    if (this.dataStream.length > 100) {
      this.dataStream.shift();
    }

    // 实时更新模型
    await this.updateModelRealtime();
  }

  // 实时模型更新
  async updateModelRealtime() {
    if (this.dataStream.length < 10) return; // 需要足够数据

    // 分析最近数据模式
    const recentData = this.dataStream.slice(-10);

    // 检测市场情绪变化
    const priceChanges = recentData.map((d) => d.priceChange || 0);
    const avgChange = priceChanges.reduce((a, b) => a + b, 0) / priceChanges.length;

    if (avgChange > 2) {
      this.marketSentiment = 'bullish';
    } else if (avgChange < -2) {
      this.marketSentiment = 'bearish';
    } else {
      this.marketSentiment = 'neutral';
    }

    // 动态调整指标权重
    await this.adjustIndicatorWeights(recentData);

    console.log(
      `实时学习: 市场情绪=${this.marketSentiment}, 指标权重=${JSON.stringify(this.indicatorWeights)}`,
    );
  }

  // 调整指标权重
  async adjustIndicatorWeights(recentData) {
    // 简单自适应：检测哪些指标在最近数据中更有效
    const correlations = {};

    for (const indicator of Object.keys(this.indicatorWeights)) {
      // 模拟计算指标与价格的相关性
      const values = recentData.map((d) => d[indicator]).filter((v) => v !== undefined);
      if (values.length < 2) continue;

      // 计算简单相关性
      const priceChanges = recentData.map((d) => d.priceChange || 0).slice(-values.length);
      const correlation = this.calculateCorrelation(values, priceChanges);
      correlations[indicator] = Math.abs(correlation);
    }

    // 根据相关性调整权重
    const totalCorrelation = Object.values(correlations).reduce((a, b) => a + b, 0);
    if (totalCorrelation > 0) {
      for (const indicator of Object.keys(correlations)) {
        this.indicatorWeights[indicator] = correlations[indicator] / totalCorrelation;
      }
    }
  }

  // 简单相关性计算
  calculateCorrelation(x, y) {
    const n = Math.min(x.length, y.length);
    if (n === 0) return 0;

    const sumX = x.slice(0, n).reduce((a, b) => a + b, 0);
    const sumY = y.slice(0, n).reduce((a, b) => a + b, 0);
    const sumXY = x.slice(0, n).reduce((acc, xi, i) => acc + xi * y[i], 0);
    const sumX2 = x.slice(0, n).reduce((acc, xi) => acc + xi * xi, 0);
    const sumY2 = y.slice(0, n).reduce((acc, yi) => acc + yi * yi, 0);

    const numerator = n * sumXY - sumX * sumY;
    const denominator = Math.sqrt((n * sumX2 - sumX * sumX) * (n * sumY2 - sumY * sumY));

    return denominator === 0 ? 0 : numerator / denominator;
  }

  // 基于最新数据做决策
  async makeRealtimeDecision() {
    if (this.dataStream.length === 0) {
      return { action: 'hold', confidence: 0, reason: '无足够数据' };
    }

    const latest = this.dataStream[this.dataStream.length - 1];

    // 计算加权信号
    let buySignal = 0;
    let sellSignal = 0;

    // RSI信号
    if (latest.rsi < 30) buySignal += this.indicatorWeights.rsi;
    if (latest.rsi > 70) sellSignal += this.indicatorWeights.rsi;

    // MACD信号
    if (latest.macd > 0) buySignal += this.indicatorWeights.macd;
    if (latest.macd < 0) sellSignal += this.indicatorWeights.macd;

    // 市场情绪调整
    if (this.marketSentiment === 'bullish') buySignal *= 1.2;
    if (this.marketSentiment === 'bearish') sellSignal *= 1.2;

    // 决策
    let action = 'hold';
    let confidence = 0;

    if (buySignal > sellSignal && buySignal > 0.3) {
      action = 'buy';
      confidence = buySignal;
    } else if (sellSignal > buySignal && sellSignal > 0.3) {
      action = 'sell';
      confidence = sellSignal;
    }

    return {
      action,
      confidence: confidence.toFixed(2),
      signals: { buy: buySignal.toFixed(2), sell: sellSignal.toFixed(2) },
      marketSentiment: this.marketSentiment,
      weights: this.indicatorWeights,
    };
  }

  // 获取模型状态
  getModelState() {
    return {
      dataPoints: this.dataStream.length,
      marketSentiment: this.marketSentiment,
      indicatorWeights: this.indicatorWeights,
      lastUpdate: this.dataStream[this.dataStream.length - 1]?.timestamp,
    };
  }
}

// 使用示例
const onlineTrader = new OnlineLearningTrader();

// 模拟实时数据流
console.log('=== 实时在线学习演示 ===\n');

const simulatedData = [
  { rsi: 25, macd: 0.5, priceChange: 1.5 },
  { rsi: 28, macd: 0.6, priceChange: 2.0 },
  { rsi: 35, macd: 0.4, priceChange: 1.8 },
  { rsi: 45, macd: 0.3, priceChange: 1.2 },
  { rsi: 55, macd: 0.2, priceChange: 0.8 },
  { rsi: 65, macd: -0.1, priceChange: -0.5 },
  { rsi: 72, macd: -0.3, priceChange: -1.5 },
  { rsi: 78, macd: -0.5, priceChange: -2.5 },
  { rsi: 82, macd: -0.6, priceChange: -3.0 },
  { rsi: 28, macd: 0.8, priceChange: 3.5 },
];

for (const data of simulatedData) {
  await onlineTrader.ingestDataStream(data);
}

console.log('\n=== 实时决策 ===');
const decision = await onlineTrader.makeRealtimeDecision();
console.log(`决策: ${decision.action}`);
console.log(`置信度: ${decision.confidence}`);
console.log(`信号: ${JSON.stringify(decision.signals)}`);
console.log(`市场情绪: ${decision.marketSentiment}`);

console.log('\n=== 模型状态 ===');
console.log(onlineTrader.getModelState());