Differences

This shows you the differences between two versions of the page.

--- wiki:ai:advanced_rag_implementation_and_example_code [2025/09/03 16:18] – created bgourley
+++ wiki:ai:advanced_rag_implementation_and_example_code [2025/09/03 16:34] (current) – [Embed Your Dataset in Azure AI Search] bgourley
@@ Line 15: / Line 15: @@
 ===== Embed Your Dataset in Azure AI Search =====
-== 1) Configure environment ==
+=== 1) Configure environment ===
 Create/update **.env** (local) for ingest. Use the **Admin** search key here:
@@ Line 67: / Line 67: @@
 MAX_TOKENS_ANSWER=800
-== 2) Install deps ==
+=== 2) Install deps ===
 npm install
-== 3) Create/Update index schema (done by ingest.js) ==
+=== 3) Create/Update index schema (done by ingest.js) ===
   * Our ingest.js creates docs with fields like:
@@ Line 89: / Line 89: @@
   * **Invalid keys** – we sanitized IDs (allowUnsafeKeys not needed).
-== 4) Put your documents in the ingest folder ==
+=== 4) Put your documents in the ingest folder ===
 ingest/
@@ Line 101: / Line 101: @@
 Etc.
-== 5) Run the ingest ==
+=== 5) Run the ingest ===
 node ingest/ingest.js
@@ Line 293: / Line 293: @@
 ==== Example Code for Azure Web App ====
-bench.js
+=== bench.js ===
+<code>
 require('dotenv').config();
 const os = require('os');
 const { answerQuestion } = require('../openaiService');
 function arg(name, def) {
+  const prefix = `--${name}=`;
-const prefix = `--${name}=`;
+  const a = process.argv.find(x => x.startsWith(prefix));
+  return a ? a.slice(prefix.length) : def;
-const a = process.argv.find(x => x.startsWith(prefix));
-return a ? a.slice(prefix.length) : def;
 }
 const q = arg('q', 'What is our onboarding process?');
 const n = Number(arg('n', 20));
 const conc = Number(arg('concurrency', 4));
 function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
 async function worker(id, jobs, results) {
+  while (true) {
-while (true) {
+    const i = jobs.next();
+    if (i.done) break;
-const i = jobs.next();
+    const t0 = Date.now();
+    try {
-if (i.done) break;
+      const r = await answerQuestion(q);
+      const t = Date.now() - t0;
-const t0 = Date.now();
+      results.push({ ok: true, latencyMs: t, usage: r.metrics.usage });
+      process.stdout.write('.');
-try {
+    } catch (e) {
+      const t = Date.now() - t0;
-const r = await answerQuestion(q);
+      results.push({ ok: false, latencyMs: t, error: String(e) });
+      process.stdout.write('x');
-const t = Date.now() - t0;
+    }
+    await sleep(50);
-results.push({ ok: true, latencyMs: t, usage: r.metrics.usage });
+  }
-process.stdout.write('.');
-} catch (e) {
-const t = Date.now() - t0;
-results.push({ ok: false, latencyMs: t, error: String(e) });
-process.stdout.write('x');
 }
-await sleep(50);
-}
-}
 async function main() {
+  const jobs = (function*(){ for (let i=0;i<n;i++) yield i; })();
-const jobs = (function*(){ for (let i=0;i<n;i++) yield i; })();
+  const results = [];
+  const ps = [];
-const results = [];
+  for (let i=0;i<conc;i++) ps.push(worker(i, jobs, results));
+  await Promise.all(ps);
-const ps = [];
+  console.log('\nDone.');
-for (let i=0;i<conc;i++) ps.push(worker(i, jobs, results));
+  const lat = results.map(r => r.latencyMs).sort((a,b)=>a-b);
+  const p = (x) => lat[Math.floor((lat.length-1)*x)];
-await Promise.all(ps);
+  const summary = {
+    q, n, concurrency: conc,
-console.log('\nDone.');
+    p50: p(0.50),
+    p90: p(0.90),
+    p95: p(0.95),
+    p99: p(0.99),
-const lat = results.map(r => r.latencyMs).sort((a,b)=>a-b);
+    errors: results.filter(r => !r.ok).length,
+    node: process.version,
-const p = (x) => lat[Math.floor((lat.length-1)*x)];
+    cpu: os.cpus()[0]?.model || 'unknown'
+  };
-const summary = {
+  console.log(JSON.stringify(summary, null, 2));
-q, n, concurrency: conc,
-p50: p(0.50),
-p90: p(0.90),
-p95: p(0.95),
-p99: p(0.99),
-errors: results.filter(r => !r.ok).length,
-node: process.version,
-cpu: os.cpus()[0]?.model || 'unknown'
-};
-console.log(JSON.stringify(summary, null, 2));
 }
 if (require.main === module) main().catch(console.error);
+</code>
-bot.js
+=== bot.js ===
+<code>
 const { ActivityHandler, MessageFactory } = require('botbuilder');
 const { answerQuestion } = require('./openaiService');
 function renderSources(citations) {
+  if (!citations || !citations.length) return '';
-if (!citations || !citations.length) return '';
+  const lines = citations.map(c => {
+    const title = c.title || c.id || 'Source';
-const lines = citations.map(c => {
+    const url = c.url ? ` (${c.url})` : '';
+    return `- ${c.key} **${title}**${url}`;
-const title = c.title || c.id || 'Source';
+  });
+  return `\n\n**Sources**\n${lines.join('\n')}`;
-const url = c.url ? ` (${c.url})` : '';
-return `- ${c.key} %%**%%${title}%%**%%${url}`;
-});
-return `\n\n%%**%%Sources%%**%%\n${lines.join('\n')}`;
 }
 class EchoBot extends ActivityHandler {
+  constructor() {
-constructor() {
+    super();
-super();
+    this.onMessage(async (context, next) => {
+      const userInput = (context.activity.text || '').trim();
+      if (!userInput) {
+        await context.sendActivity('Please type a question.');
-this.onMessage(async (context, next) => {
+        return;
+      }
-const userInput = (context.activity.text || '').trim();
+      try {
-if (!userInput) {
+        const result = await answerQuestion(userInput);
+        const text = `${result.answer}${renderSources(result.citations)}`;
-await context.sendActivity('Please type a question.');
+        await context.sendActivity(MessageFactory.text(text, text));
+      } catch (err) {
-return;
+        console.error('Error processing message:', err);
+        await context.sendActivity('Oops, something went wrong when talking to AI.');
+      }
+      await next();
+    });
+    this.onMembersAdded(async (context, next) => {
+      const welcomeText = 'Hello and welcome! Ask me any question about our documents.';
+      for (const member of context.activity.membersAdded) {
+        if (member.id !== context.activity.recipient.id) {
+          await context.sendActivity(MessageFactory.text(welcomeText, welcomeText));
+        }
+      }
+      await next();
+    });
+  }
 }
-try {
-const result = await answerQuestion(userInput);
-const text = `${result.answer}${renderSources(result.citations)}`;
-await context.sendActivity(MessageFactory.text(text, text));
-} catch (err) {
-console.error('Error processing message:', err);
-await context.sendActivity('Oops, something went wrong when talking to AI.');
-}
-await next();
-});
-this.onMembersAdded(async (context, next) => {
-const welcomeText = 'Hello and welcome! Ask me any question about our documents.';
-for (const member of context.activity.membersAdded) {
-if (member.id !== context.activity.recipient.id) {
-await context.sendActivity(MessageFactory.text(welcomeText, welcomeText));
-}
-}
-await next();
-});
-}
-}
 module.exports.EchoBot = EchoBot;
+</code>
-evaluate.js
+=== evaluate.js ===
+<code>
 require('dotenv').config();
 const fs = require('fs');
 const path = require('path');
 const { answerQuestion, _internals } = require('../openaiService');
+// ---------------- CLI args ----------------
-%%//%% ---------------- CLI args ----------------
 function arg(name, def) {
+  const p = `--${name}=`;
-const p = `--${name}=`;
+  const a = process.argv.find(x => x.startsWith(p));
+  if (!a) return def;
-const a = process.argv.find(x => x.startsWith(p));
+  const v = a.slice(p.length);
+  if (v === 'true') return true;
-if (!a) return def;
+  if (v === 'false') return false;
+  const n = Number(v);
-const v = a.slice(p.length);
+  return Number.isFinite(n) ? n : v;
-if (v === 'true') return true;
-if (v === 'false') return false;
-const n = Number(v);
-return Number.isFinite(n) ? n : v;
 }
 const K = arg('k', 5);
+const LIMIT = arg('limit', 0);         // 0 = all
-const LIMIT = arg('limit', 0); %%//%% 0 = all
 const USE_JUDGE = arg('judge', false) || String(process.env.EVAL_USE_JUDGE||'false').toLowerCase()==='true';
+// ---------------- Helpers ----------------
-%%//%% ---------------- Helpers ----------------
 function loadJsonl(p) {
+  const lines = fs.readFileSync(p, 'utf8').split(/\r?\n/).filter(Boolean);
-const lines = fs.readFileSync(p, 'utf8').split(/\r?\n/).filter(Boolean);
+  return lines.map((l, i) => {
+    try { return JSON.parse(l); }
-return lines.map((l, i) => {
+    catch {
+      return { id: `row${i+1}`, question: l };
-try { return JSON.parse(l); }
+    }
+  });
-catch {
-return { id: `row${i+1}`, question: l };
 }
-});
-}
 function recallAtK(retrieved, refs, k = K) {
+  if (!Array.isArray(refs) || !refs.length) return null;
-if (!Array.isArray(refs) || !refs.length) return null;
+  const ids = new Set(retrieved.slice(0, k).map(r => r.id));
+  const hits = refs.filter(r => ids.has(r)).length;
-const ids = new Set(retrieved.slice(0, k).map(r => r.id));
+  return { hits, total: refs.length, recall: hits / refs.length };
-const hits = refs.filter(r => ids.has(r)).length;
-return { hits, total: refs.length, recall: hits / refs.length };
 }
 function parentRecallAtK(retrieved, parentRefs, k = K) {
+  if (!Array.isArray(parentRefs) || !parentRefs.length) return null;
-if (!Array.isArray(parentRefs) || !parentRefs.length) return null;
+  const pids = new Set(retrieved.slice(0, k).map(r => r.parentId || r.id));
+  const hits = parentRefs.filter(r => pids.has(r)).length;
-const pids = new Set(retrieved.slice(0, k).map(r => r.parentId || r.id));
+  return { hits, total: parentRefs.length, recall: hits / parentRefs.length };
-const hits = parentRefs.filter(r => pids.has(r)).length;
-return { hits, total: parentRefs.length, recall: hits / parentRefs.length };
 }
 function mustMentionCoverage(answer, must) {
+  if (!Array.isArray(must) || !must.length) return null;
-if (!Array.isArray(must) || !must.length) return null;
+  const a = (answer || '').toLowerCase();
+  const checks = must.map(m => ({ term: m, present: a.includes(String(m).toLowerCase()) }));
-const a = (answer || '').toLowerCase();
+  const pct = checks.reduce((s,c)=> s + (c.present?1:0), 0) / checks.length;
+  return { coverage: pct, checks };
-const checks = must.map(m => ({ term: m, present: a.includes(String(m).toLowerCase()) }));
-const pct = checks.reduce((s,c)=> s + (c.present?1:0), 0) / checks.length;
-return { coverage: pct, checks };
 }
 async function llmJudge(row, result) {
+  // Optional LLM-as-a-judge: groundedness, relevance, completeness (0..5)
-%%//%% Optional LLM-as-a-judge: groundedness, relevance, completeness (0..5)
+  // Uses your chat deployment via openaiService._internals.chat
+  const { chat } = _internals;
-%%//%% Uses your chat deployment via openaiService._internals.chat
+  const sys = { role: 'system', content: [
+    'You are scoring an answer for a Retrieval-Augmented Generation (RAG) system.',
-const { chat } = _internals;
+    'Return a JSON object with integer scores 0..5: {groundedness, relevance, completeness} and a one-sentence "notes".',
+    'Definitions:',
-const sys = { role: 'system', content: [
+    '- groundedness: The answer is explicitly supported by the provided citations/passages. Penalize hallucinations.',
+    '- relevance: The answer addresses the user question (on-topic).',
-'You are scoring an answer for a Retrieval-Augmented Generation (RAG) system.',
+    '- completeness: The answer covers the key points needed for a useful response.'
+  ].join('\n') };
-'Return a JSON object with integer scores 0..5: {groundedness, relevance, completeness} and a one-sentence "notes".',
+  const payload = {
-'Definitions:',
+    role: 'user',
+    content: [
-'- groundedness: The answer is explicitly supported by the provided citations/passages. Penalize hallucinations.',
+      `Question: ${row.question}`,
+      `Answer: ${result.answer}`,
-'- relevance: The answer addresses the user question (on-topic).',
+      `Citations: ${JSON.stringify(result.citations, null, 2)}`,
+      `Top Passages: ${JSON.stringify(result.retrieved.slice(0, K).map(p=>({id:p.id,parentId:p.parentId,title:p.title,content:(p.content||'').slice(0,1000)})), null, 2)}`,
-'- completeness: The answer covers the key points needed for a useful response.'
+      'Return ONLY valid JSON.'
+    ].join('\n\n')
-].join('\n') };
+  };
+  try {
+    const data = await chat([sys, payload], { temperature: 0.0, max_tokens: 200 });
-const payload = {
+    const txt = data.choices?.[0]?.message?.content?.trim() || '{}';
+    const obj = JSON.parse(txt);
-role: 'user',
+    const g = Math.max(0, Math.min(5, Number(obj.groundedness)||0));
+    const r = Math.max(0, Math.min(5, Number(obj.relevance)||0));
-content: [
+    const c = Math.max(0, Math.min(5, Number(obj.completeness)||0));
+    return { groundedness: g, relevance: r, completeness: c, notes: obj.notes || '' };
-`Question: ${row.question}`,
+  } catch (e) {
+    return { groundedness: null, relevance: null, completeness: null, error: String(e) };
-`Answer: ${result.answer}`,
+  }
-`Citations: ${JSON.stringify(result.citations, null, 2)}`,
-`Top Passages: ${JSON.stringify(result.retrieved.slice(0, K).map(p=>({id:p.id,parentId:p.parentId,title:p.title,content:(p.content||'').slice(0,1000)})), null, 2)}`,
-'Return ONLY valid JSON.'
-].join('\n\n')
-};
-try {
-const data = await chat([sys, payload], { temperature: 0.0, max_tokens: 200 });
-const txt = data.choices?.[0]?.message?.content?.trim() || '{}';
-const obj = JSON.parse(txt);
-const g = Math.max(0, Math.min(5, Number(obj.groundedness)||0));
-const r = Math.max(0, Math.min(5, Number(obj.relevance)||0));
-const c = Math.max(0, Math.min(5, Number(obj.completeness)||0));
-return { groundedness: g, relevance: r, completeness: c, notes: obj.notes || '' };
-} catch (e) {
-return { groundedness: null, relevance: null, completeness: null, error: String(e) };
 }
-}
 function mean(arr) {
+  const xs = arr.filter(x => typeof x === 'number' && Number.isFinite(x));
-const xs = arr.filter(x => typeof x === 'number' && Number.isFinite(x));
+  if (!xs.length) return null;
+  return xs.reduce((a,b)=>a+b,0)/xs.length;
-if (!xs.length) return null;
-return xs.reduce((a,b)=>a+b,0)/xs.length;
 }
+// ---------------- Main ----------------
-%%//%% ---------------- Main ----------------
 async function main() {
+  const datasetPath = path.join(__dirname, 'dataset.jsonl');
-const datasetPath = path.join(%%__%%dirname, 'dataset.jsonl');
+  if (!fs.existsSync(datasetPath)) {
+    console.error('No dataset.jsonl found in ./eval. Create one (JSONL) with at least {"id","question"}.');
-if (!fs.existsSync(datasetPath)) {
+    process.exit(1);
+  }
-console.error('No dataset.jsonl found in ./eval. Create one (JSONL) with at least {"id","question"}.');
+  const rowsAll = loadJsonl(datasetPath);
-process.exit(1);
+  const rows = LIMIT ? rowsAll.slice(0, LIMIT) : rowsAll;
+  const results = [];
+  for (const row of rows) {
+    const options = {};
+    if (row.filter) options.filter = row.filter; // OData filter passthrough
+    const res = await answerQuestion(row.question, options);
+    const rDoc = recallAtK(res.retrieved, row.references, K);
+    const rPar = parentRecallAtK(res.retrieved, row.parent_refs, K);
+    const mention = mustMentionCoverage(res.answer, row.must_mention);
+    let judge = null;
+    if (USE_JUDGE) judge = await llmJudge(row, res);
+    results.push({
+      id: row.id,
+      question: row.question,
+      references: row.references || null,
+      parent_refs: row.parent_refs || null,
+      must_mention: row.must_mention || null,
+      filter: row.filter || null,
+      answer: res.answer,
+      citations: res.citations,
+      metrics: {
+        retrievalLatencyMs: res.metrics.retrievalLatencyMs,
+        generationLatencyMs: res.metrics.generationLatencyMs,
+        usage: res.metrics.usage || null,
+        expansions: res.metrics.expansions || null,
+        recallAtK: rDoc,
+        parentRecallAtK: rPar,
+        mustMention: mention,
+        judge
+      }
+    });
+    const recStr = rDoc ? rDoc.recall.toFixed(2) : 'n/a';
+    const lat = res.metrics.generationLatencyMs;
+    console.log(`✓ ${row.id}  recall@${K}=${recStr}  gen=${lat}ms`);
+  }
+  // Summary
+  const p = path.join(__dirname, `results-${Date.now()}.json`);
+  fs.writeFileSync(p, JSON.stringify(results, null, 2));
+  console.log('Saved', p);
+  const rVals = results.map(r => r.metrics.recallAtK?.recall).filter(v => typeof v === 'number');
+  const prVals = results.map(r => r.metrics.parentRecallAtK?.recall).filter(v => typeof v === 'number');
+  const retLat = results.map(r => r.metrics.retrievalLatencyMs).filter(Number.isFinite);
+  const genLat = results.map(r => r.metrics.generationLatencyMs).filter(Number.isFinite);
+  const gScores = results.map(r => r.metrics.judge?.groundedness).filter(Number.isFinite);
+  const relScores = results.map(r => r.metrics.judge?.relevance).filter(Number.isFinite);
+  const compScores = results.map(r => r.metrics.judge?.completeness).filter(Number.isFinite);
+  const summary = {
+    count: results.length,
+    k: K,
+    avgRecallAtK: mean(rVals),
+    avgParentRecallAtK: mean(prVals),
+    avgRetrievalLatencyMs: mean(retLat),
+    avgGenerationLatencyMs: mean(genLat),
+    judgeAverages: USE_JUDGE ? {
+      groundedness: mean(gScores),
+      relevance: mean(relScores),
+      completeness: mean(compScores)
+    } : null
+  };
+  console.log('\nSummary:\n', JSON.stringify(summary, null, 2));
 }
-const rowsAll = loadJsonl(datasetPath);
-const rows = LIMIT ? rowsAll.slice(0, LIMIT) : rowsAll;
-const results = [];
-for (const row of rows) {
-const options = {};
-if (row.filter) options.filter = row.filter; %%//%% OData filter passthrough
-const res = await answerQuestion(row.question, options);
-const rDoc = recallAtK(res.retrieved, row.references, K);
-const rPar = parentRecallAtK(res.retrieved, row.parent_refs, K);
-const mention = mustMentionCoverage(res.answer, row.must_mention);
-let judge = null;
-if (USE_JUDGE) judge = await llmJudge(row, res);
-results.push({
-id: row.id,
-question: row.question,
-references: row.references || null,
-parent_refs: row.parent_refs || null,
-must_mention: row.must_mention || null,
-filter: row.filter || null,
-answer: res.answer,
-citations: res.citations,
-metrics: {
-retrievalLatencyMs: res.metrics.retrievalLatencyMs,
-generationLatencyMs: res.metrics.generationLatencyMs,
-usage: res.metrics.usage || null,
-expansions: res.metrics.expansions || null,
-recallAtK: rDoc,
-parentRecallAtK: rPar,
-mustMention: mention,
-judge
-}
-});
-const recStr = rDoc ? rDoc.recall.toFixed(2) : 'n/a';
-const lat = res.metrics.generationLatencyMs;
-console.log(`✓ ${row.id} recall@${K}=${recStr} gen=${lat}ms`);
-}
-%%//%% Summary
-const p = path.join(%%__%%dirname, `results-${Date.now()}.json`);
-fs.writeFileSync(p, JSON.stringify(results, null, 2));
-console.log('Saved', p);
-const rVals = results.map(r => r.metrics.recallAtK?.recall).filter(v => typeof v === 'number');
-const prVals = results.map(r => r.metrics.parentRecallAtK?.recall).filter(v => typeof v === 'number');
-const retLat = results.map(r => r.metrics.retrievalLatencyMs).filter(Number.isFinite);
-const genLat = results.map(r => r.metrics.generationLatencyMs).filter(Number.isFinite);
-const gScores = results.map(r => r.metrics.judge?.groundedness).filter(Number.isFinite);
-const relScores = results.map(r => r.metrics.judge?.relevance).filter(Number.isFinite);
-const compScores = results.map(r => r.metrics.judge?.completeness).filter(Number.isFinite);
-const summary = {
-count: results.length,
-k: K,
-avgRecallAtK: mean(rVals),
-avgParentRecallAtK: mean(prVals),
-avgRetrievalLatencyMs: mean(retLat),
-avgGenerationLatencyMs: mean(genLat),
-judgeAverages: USE_JUDGE ? {
-groundedness: mean(gScores),
-relevance: mean(relScores),
-completeness: mean(compScores)
-} : null
-};
-console.log('\nSummary:\n', JSON.stringify(summary, null, 2));
-}
 if (require.main === module) {
+  main().catch(err => { console.error(err); process.exit(1); });
-main().catch(err => { console.error(err); process.exit(1); });
 }
+</code>
-index.js
+=== index.js ===
+<code>
 const path = require('path');
+require('dotenv').config({ path: path.join(__dirname, '.env') });
-require('dotenv').config({ path: path.join(%%__%%dirname, '.env') });
 const restify = require('restify');
 const {
+  CloudAdapter,
-CloudAdapter,
+  ConfigurationServiceClientCredentialFactory,
+  createBotFrameworkAuthenticationFromConfiguration
-ConfigurationServiceClientCredentialFactory,
-createBotFrameworkAuthenticationFromConfiguration
 } = require('botbuilder');
+// (Optional) Application Insights for runtime telemetry
-%%//%% (Optional) Application Insights for runtime telemetry
 try {
+  if (process.env.APPINSIGHTS_CONNECTION_STRING) {
-if (process.env.APPINSIGHTS_CONNECTION_STRING) {
+    require('applicationinsights').setup().start();
+    console.log('[init] Application Insights enabled');
-require('applicationinsights').setup().start();
+  }
-console.log('[init] Application Insights enabled');
-}
 } catch (e) {
+  console.warn('[init] App Insights init skipped:', e?.message || e);
-console.warn('[init] App Insights init skipped:', e?.message || e);
 }
+// Import your bot implementation
-%%//%% Import your bot implementation
 const { EchoBot } = require('./bot');
+// --- Server setup ---
-%%//%% --- Server setup ---
 const server = restify.createServer();
 server.use(restify.plugins.bodyParser());
 server.use(restify.plugins.queryParser());
+// Prefer Azure's injected PORT. Fall back to 8080 locally, then 3978 (bot default).
-%%//%% Prefer Azure's injected PORT. Fall back to 8080 locally, then 3978 (bot default).
 const rawPort = process.env.PORT || process.env.port || 8080;
 const port = Number(rawPort);
 console.log('[startup] PORT from env =', rawPort, '→ binding to', port);
 server.listen(port, () => {
+  console.log(`${server.name} listening on ${server.url}`);
-console.log(`${server.name} listening on ${server.url}`);
+  console.log(`Node: ${process.version}  |  ENV: ${process.env.NODE_ENV || 'development'}`);
+  console.log('POST /api/messages  (Bot endpoint)');
-console.log(`Node: ${process.version} | ENV: ${process.env.NODE_ENV || 'development'}`);
+  console.log('GET  /api/health    (Health check)');
-console.log('POST /api/messages (Bot endpoint)');
-console.log('GET /api/health (Health check)');
 });
+// Health endpoint for App Service
-%%//%% Health endpoint for App Service
 server.get('/api/health', (_req, res, next) => {
+  res.send(200, {
-res.send(200, {
+    ok: true,
+    uptimeSec: Math.round(process.uptime()),
-ok: true,
+    node: process.version
+  });
-uptimeSec: Math.round(process.uptime()),
+  return next();
-node: process.version
 });
+// 1) What is Search returning right now?
-return next();
-});
-%%//%% 1) What is Search returning right now?
 const { _internals } = require('./openaiService');
 server.get('/api/search-diag', async (req, res) => {
+  try {
-try {
+    const { _internals } = require('./openaiService');
+    const q = req.query.q || '';
-const { _internals } = require('./openaiService');
+    const items = await _internals.searchOnce({ query: q });
+    res.send(200, {
-const q = req.query.q || '';
+      ok: true,
+      q,
-const items = await _internals.searchOnce({ query: q });
+      returned: items.length,
+      items: items.slice(0, 5).map(x => ({
-res.send(200, {
+        id: x.id,
+        title: x.title,
-ok: true,
+        contentPreview: (x.content || '').slice(0, 200)
+      }))
-q,
+    });
+  } catch (e) {
-returned: items.length,
+    res.send(503, { ok: false, error: e?.message || String(e) });
+  }
-items: items.slice(0, 5).map(x => ({
-id: x.id,
-title: x.title,
-contentPreview: (x.content || '').slice(0, 200)
-}))
 });
-} catch (e) {
+// 2) End-to-end answer (to see retrieved count + citations)
-res.send(503, { ok: false, error: e?.message || String(e) });
-}
-});
-%%//%% 2) End-to-end answer (to see retrieved count + citations)
 const { answerQuestion } = require('./openaiService');
 server.get('/api/answer', async (req, res) => {
+  try {
-try {
+    const { answerQuestion } = require('./openaiService');
+    const q = req.query.q || '';
-const { answerQuestion } = require('./openaiService');
+    const out = await answerQuestion(q);
+    res.send(200, {
-const q = req.query.q || '';
+      ok: true,
+      query: q,
-const out = await answerQuestion(q);
+      retrievedCount: out.retrieved?.length || 0,
+      citationsCount: out.citations?.length || 0,
-res.send(200, {
+      answer: out.answer,
+      sampleRetrieved: out.retrieved?.slice(0,3)?.map(r => ({
-ok: true,
+        id: r.id, title: r.title, preview: (r.content||'').slice(0,160)
+      })) || []
-query: q,
+    });
+  } catch (e) {
-retrievedCount: out.retrieved?.length || 0,
+    res.send(503, { ok: false, error: e?.message || String(e) });
+  }
-citationsCount: out.citations?.length || 0,
-answer: out.answer,
-sampleRetrieved: out.retrieved?.slice(0,3)?.map(r => ({
-id: r.id, title: r.title, preview: (r.content||'').slice(0,160)
-})) || []
 });
+// How many docs are in the index?
-} catch (e) {
-res.send(503, { ok: false, error: e?.message || String(e) });
-}
-});
-%%//%% How many docs are in the index?
 server.get('/api/search-stats', async (_req, res) => {
+  try {
-try {
+    const { SearchClient, AzureKeyCredential } = require('@azure/search-documents');
+    const endpoint = process.env.AZURE_SEARCH_ENDPOINT.trim();
-const { SearchClient, AzureKeyCredential } = require('@azure/search-documents');
+    const index = process.env.AZURE_SEARCH_INDEX_NAME.trim();
+    const key = (process.env.AZURE_SEARCH_QUERY_KEY || process.env.AZURE_SEARCH_API_KEY).trim();
-const endpoint = process.env.AZURE_SEARCH_ENDPOINT.trim();
+    const client = new SearchClient(endpoint, index, new AzureKeyCredential(key));
+    const count = await client.getDocumentsCount();
-const index = process.env.AZURE_SEARCH_INDEX_NAME.trim();
+    res.send(200, { ok: true, index, count });
+  } catch (e) {
-const key = (process.env.AZURE_SEARCH_QUERY_KEY || process.env.AZURE_SEARCH_API_KEY).trim();
+    res.send(503, { ok: false, error: e?.message || String(e) });
+  }
-const client = new SearchClient(endpoint, index, new AzureKeyCredential(key));
-const count = await client.getDocumentsCount();
-res.send(200, { ok: true, index, count });
-} catch (e) {
-res.send(503, { ok: false, error: e?.message || String(e) });
-}
 });
+// Vector diag: embed the query and run a pure vector search
-%%//%% Vector diag: embed the query and run a pure vector search
 server.get('/api/search-diag-vector', async (req, res) => {
+  try {
-try {
+    const q = (req.query && req.query.q) ? String(req.query.q) : '';
+    if (!q) return res.send(400, { ok: false, error: 'q required' });
-const q = (req.query && req.query.q) ? String(req.query.q) : '';
+    const { _internals } = require('./openaiService');
-if (!q) return res.send(400, { ok: false, error: 'q required' });
+    const vec = await _internals.embedMemo(q);
+    const items = await _internals.searchOnce({ query: '', vector: vec });
+    res.send(200, {
-const { _internals } = require('./openaiService');
+      ok: true,
+      q,
-const vec = await _internals.embedMemo(q);
+      returned: items.length,
+      items: items.slice(0, 5).map(x => ({
-const items = await _internals.searchOnce({ query: '', vector: vec });
+        id: x.id,
+        title: x.title,
+        contentPreview: (x.content || '').slice(0, 200)
+      }))
-res.send(200, {
+    });
+  } catch (e) {
-ok: true,
+    res.send(503, { ok: false, error: e?.message || String(e) });
+  }
-q,
-returned: items.length,
-items: items.slice(0, 5).map(x => ({
-id: x.id,
-title: x.title,
-contentPreview: (x.content || '').slice(0, 200)
-}))
 });
-} catch (e) {
-res.send(503, { ok: false, error: e?.message || String(e) });
-}
-});
 const axios = require('axios');
+// Ultra-raw: call Azure Search REST API directly (bypasses SDK iterator)
-%%//%% Ultra-raw: call Azure Search REST API directly (bypasses SDK iterator)
 server.get('/api/search-diag-raw', async (req, res) => {
+  try {
-try {
+    const endpoint = String(process.env.AZURE_SEARCH_ENDPOINT || '').trim().replace(/\/+$/, '');
+    const index = String(process.env.AZURE_SEARCH_INDEX_NAME || '').trim();
-const endpoint = String(process.env.AZURE_SEARCH_ENDPOINT || '').trim().replace(/\/+$/, '');
+    const apiKey = String(process.env.AZURE_SEARCH_QUERY_KEY || process.env.AZURE_SEARCH_API_KEY || '').trim();
-const index = String(process.env.AZURE_SEARCH_INDEX_NAME || '').trim();
+    if (!endpoint || !index || !apiKey) {
+      return res.send(400, { ok: false, error: 'Missing endpoint/index/apiKey env vars' });
-const apiKey = String(process.env.AZURE_SEARCH_QUERY_KEY || process.env.AZURE_SEARCH_API_KEY || '').trim();
+    }
+    const q = (req.query && typeof req.query.q === 'string') ? req.query.q : '';
+    const url = `${endpoint}/indexes('${encodeURIComponent(index)}')/docs/search.post.search?api-version=2024-07-01`;
-if (!endpoint || !index || !apiKey) {
+    // IMPORTANT: set select conservatively; match your schema
-return res.send(400, { ok: false, error: 'Missing endpoint/index/apiKey env vars' });
+    // If your text field is `contents`, keep "contents" below.
+    const body = {
-}
+      search: q,              // '' (match-all) or your query string
+      top: 5,
+      select: 'id,title,contents,url'  // adjust if your fields are named differently
+    };
-const q = (req.query && typeof req.query.q === 'string') ? req.query.q : '';
+    const { data } = await axios.post(url, body, {
-const url = `${endpoint}/indexes('${encodeURIComponent(index)}')/docs/search.post.search?api-version=2024-07-01`;
+      headers: {
+        'api-key': apiKey,
+        'Content-Type': 'application/json',
+        'Accept': 'application/json;odata.metadata=none'
-%%//%% IMPORTANT: set select conservatively; match your schema
+      },
+      timeout: 10000
-%%//%% If your text field is `contents`, keep "contents" below.
+    });
-const body = {
+    const items = Array.isArray(data.value) ? data.value.map(d => ({
+      id: d.id,
-search: q, %%//%% '' (match-all) or your query string
+      title: d.title || null,
+      contentPreview: String(d.contents ?? d.content ?? '').slice(0, 200),
-top: 5,
+      url: d.url || null
+    })) : [];
-select: 'id,title,contents,url' %%//%% adjust if your fields are named differently
+    res.send(200, { ok: true, q, returned: items.length, items });
-};
+  } catch (e) {
+    res.send(503, { ok: false, error: e?.response?.data?.error?.message || e?.message || String(e) });
+  }
-const { data } = await axios.post(url, body, {
-headers: {
-'api-key': apiKey,
-'Content-Type': 'application/json',
-'Accept': 'application/json;odata.metadata=none'
-},
-timeout: 10000
 });
+// Simple root info (optional)
-const items = Array.isArray(data.value) ? data.value.map(d => ({
-id: d.id,
-title: d.title || null,
-contentPreview: String(d.contents ?? d.content ?? '').slice(0, 200),
-url: d.url || null
-})) : [];
-res.send(200, { ok: true, q, returned: items.length, items });
-} catch (e) {
-res.send(503, { ok: false, error: e?.response?.data?.error?.message || e?.message || String(e) });
-}
-});
-%%//%% Simple root info (optional)
 server.get('/', (_req, res, next) => {
+  res.send(200, { ok: true, message: 'RAG bot is running', time: new Date().toISOString() });
-res.send(200, { ok: true, message: 'RAG bot is running', time: new Date().toISOString() });
+  return next();
-return next();
 });
+// --- Bot Framework adapter (credentials from env) ---
-%%//%% --- Bot Framework adapter (credentials from env) ---
 const credentialsFactory = new ConfigurationServiceClientCredentialFactory({
+  MicrosoftAppId: process.env.MicrosoftAppId,
-MicrosoftAppId: process.env.MicrosoftAppId,
+  MicrosoftAppPassword: process.env.MicrosoftAppPassword,       // aka Client Secret
+  MicrosoftAppType: process.env.MicrosoftAppType || 'MultiTenant',
-MicrosoftAppPassword: process.env.MicrosoftAppPassword, %%//%% aka Client Secret
+  MicrosoftAppTenantId: process.env.MicrosoftAppTenantId
-MicrosoftAppType: process.env.MicrosoftAppType || 'MultiTenant',
-MicrosoftAppTenantId: process.env.MicrosoftAppTenantId
 });
 const botFrameworkAuthentication =
+  createBotFrameworkAuthenticationFromConfiguration(null, credentialsFactory);
-createBotFrameworkAuthenticationFromConfiguration(null, credentialsFactory);
 const adapter = new CloudAdapter(botFrameworkAuthentication);
+// Catch-all for errors
-%%//%% Catch-all for errors
 adapter.onTurnError = async (context, error) => {
+  console.error('[onTurnError]', error);
-console.error('[onTurnError]', error);
+  try {
+    await context.sendTraceActivity(
-try {
+      'OnTurnError Trace',
+      `${error}`,
-await context.sendTraceActivity(
+      'https://www.botframework.com/schemas/error',
+      'TurnError'
-'OnTurnError Trace',
+    );
+    await context.sendActivity('Oops—something went wrong processing your message.');
-`${error}`,
+  } catch (_) {
+    // ignore secondary failures
-'https:%%//%%www.botframework.com/schemas/error',
+  }
-'TurnError'
-);
-await context.sendActivity('Oops—something went wrong processing your message.');
-} catch (_) {
-%%//%% ignore secondary failures
-}
 };
+// Create the bot instance
-%%//%% Create the bot instance
 const bot = new EchoBot();
+// Bot messages endpoint (required by Azure Bot Service)
-%%//%% Bot messages endpoint (required by Azure Bot Service)
 server.post('/api/messages', async (req, res) => {
+  await adapter.process(req, res, (context) => bot.run(context));
-await adapter.process(req, res, (context) => bot.run(context));
 });
+// Streaming (Teams / Skills)
-%%//%% Streaming (Teams / Skills)
 server.on('upgrade', async (req, socket, head) => {
+  const streamingAdapter = new CloudAdapter(botFrameworkAuthentication);
-const streamingAdapter = new CloudAdapter(botFrameworkAuthentication);
+  streamingAdapter.onTurnError = adapter.onTurnError;
+  await streamingAdapter.process(req, socket, head, (context) => bot.run(context));
-streamingAdapter.onTurnError = adapter.onTurnError;
-await streamingAdapter.process(req, socket, head, (context) => bot.run(context));
 });
+</code>
-ingest.js
+=== ingest.js ===
+<code>
 require('dotenv').config();
 const mammoth = require('mammoth');
 const fs = require('fs');
 const path = require('path');
 const axios = require('axios');
 const { SearchClient, SearchIndexClient, AzureKeyCredential } = require('@azure/search-documents');
 const SEARCH_ENDPOINT = process.env.AZURE_SEARCH_ENDPOINT;
+const SEARCH_INDEX = process.env.AZURE_SEARCH_AZURE_SEARCH_INDEX_NAME || process.env.AZURE_SEARCH_INDEX_NAME; // support both if user mis-typed
-const SEARCH_INDEX = process.env.AZURE_SEARCH_AZURE_SEARCH_INDEX_NAME || process.env.AZURE_SEARCH_INDEX_NAME; %%//%% support both if user mis-typed
 const SEARCH_KEY = process.env.AZURE_SEARCH_API_KEY;
 const OPENAI_ENDPOINT = process.env.OPENAI_ENDPOINT;
 const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
 const OPENAI_API_VERSION = process.env.OPENAI_API_VERSION || '2024-06-01';
 const OPENAI_EMBEDDING_DEPLOYMENT = process.env.OPENAI_EMBEDDING_DEPLOYMENT || 'text-embedding-3-large';
 const VECTOR_FIELD = process.env.VECTOR_FIELD || 'contentVector';
 const CONTENT_FIELD = process.env.CONTENT_FIELD || 'content';
 const TITLE_FIELD = process.env.TITLE_FIELD || 'title';
 const URL_FIELD = process.env.URL_FIELD || 'url';
 const METADATA_FIELD = process.env.METADATA_FIELD || 'metadata';
 const PARENT_ID_FIELD = process.env.PARENT_ID_FIELD || 'parentId';
 function arg(name, def){ const p=`--${name}=`; const a=process.argv.find(x=>x.startsWith(p)); return a ? a.slice(p.length) : def; }
+const dataDir = require('path').resolve(arg('dir', require('path').join(__dirname, 'data')));
-const dataDir = require('path').resolve(arg('dir', require('path').join(%%__%%dirname, 'data')));
 const indexClient = new SearchIndexClient(SEARCH_ENDPOINT, new AzureKeyCredential(SEARCH_KEY));
 const searchClient = new SearchClient(SEARCH_ENDPOINT, SEARCH_INDEX, new AzureKeyCredential(SEARCH_KEY));
 function approxTokenLen(s) { return Math.ceil((s || '').length / 4); }
 function chunkText(text, { chunkTokens = 700, overlapTokens = 100 } = {}) {
+  const tokens = approxTokenLen(text);
-const tokens = approxTokenLen(text);
+  const estCharsPerTok = (text.length || 1) / Math.max(tokens, 1);
+  const chunkChars = Math.floor(chunkTokens * estCharsPerTok);
-const estCharsPerTok = (text.length || 1) / Math.max(tokens, 1);
+  const overlapChars = Math.floor(overlapTokens * estCharsPerTok);
+  const chunks = [];
-const chunkChars = Math.floor(chunkTokens * estCharsPerTok);
+  let start = 0;
+  while (start < text.length) {
-const overlapChars = Math.floor(overlapTokens * estCharsPerTok);
+    const end = Math.min(start + chunkChars, text.length);
+    chunks.push(text.slice(start, end));
-const chunks = [];
+    start = end - overlapChars;
+    if (start < 0) start = 0;
-let start = 0;
+    if (end === text.length) break;
+  }
-while (start < text.length) {
+  return chunks;
-const end = Math.min(start + chunkChars, text.length);
-chunks.push(text.slice(start, end));
-start = end - overlapChars;
-if (start < 0) start = 0;
-if (end === text.length) break;
 }
-return chunks;
-}
 async function embed(text) {
+  const url = `${OPENAI_ENDPOINT}/openai/deployments/${OPENAI_EMBEDDING_DEPLOYMENT}/embeddings?api-version=${OPENAI_API_VERSION}`;
-const url = `${OPENAI_ENDPOINT}/openai/deployments/${OPENAI_EMBEDDING_DEPLOYMENT}/embeddings?api-version=${OPENAI_API_VERSION}`;
+  const { data } = await axios.post(url, { input: text }, {
+    headers: { 'api-key': OPENAI_API_KEY, 'Content-Type': 'application/json' }
-const { data } = await axios.post(url, { input: text }, {
+  });
+  return data.data[0].embedding;
-headers: { 'api-key': OPENAI_API_KEY, 'Content-Type': 'application/json' }
-});
-return data.data[0].embedding;
 }
 async function ensureIndex() {
+  const definition = {
-const definition = {
+    name: SEARCH_INDEX,
+    fields: [
-name: SEARCH_INDEX,
+      { name: 'id', type: 'Edm.String', key: true, filterable: false, sortable: false, facetable: false },
+      { name: TITLE_FIELD, type: 'Edm.String', searchable: true },
-fields: [
+      { name: URL_FIELD, type: 'Edm.String', searchable: false, filterable: true },
+      { name: CONTENT_FIELD, type: 'Edm.String', searchable: true },
-{ name: 'id', type: 'Edm.String', key: true, filterable: false, sortable: false, facetable: false },
+      { name: METADATA_FIELD, type: 'Edm.ComplexType', fields: [
+        { name: 'source', type: 'Edm.String', filterable: true, facetable: true },
-{ name: TITLE_FIELD, type: 'Edm.String', searchable: true },
+        { name: 'category', type: 'Edm.String', filterable: true, facetable: true },
+        { name: 'tags', type: 'Collection(Edm.String)', filterable: true, facetable: true }
-{ name: URL_FIELD, type: 'Edm.String', searchable: false, filterable: true },
+      ]},
+      { name: PARENT_ID_FIELD, type: 'Edm.String', searchable: false, filterable: true },
-{ name: CONTENT_FIELD, type: 'Edm.String', searchable: true },
+      {
+        name: VECTOR_FIELD, type: 'Collection(Edm.Single)',
-{ name: METADATA_FIELD, type: 'Edm.ComplexType', fields: [
+        searchable: true, filterable: false, sortable: false, facetable: false,
+        vectorSearchDimensions: 3072, // text-embedding-3-large
-{ name: 'source', type: 'Edm.String', filterable: true, facetable: true },
+        vectorSearchProfileName: 'vdb'
+      }
-{ name: 'category', type: 'Edm.String', filterable: true, facetable: true },
+    ],
+    semanticSearch: {
-{ name: 'tags', type: 'Collection(Edm.String)', filterable: true, facetable: true }
+      defaultConfigurationName: (process.env.SEMANTIC_CONFIGURATION || 'default').trim(),
+      configurations: [
-]},
+        {
+          name: (process.env.SEMANTIC_CONFIGURATION || 'default').trim(),
-{ name: PARENT_ID_FIELD, type: 'Edm.String', searchable: false, filterable: true },
+          prioritizedFields: {
+            titleField: { name: TITLE_FIELD },
-{
+            prioritizedContentFields: [{ name: CONTENT_FIELD }]
+          }
-name: VECTOR_FIELD, type: 'Collection(Edm.Single)',
+        }
+      ]
-searchable: true, filterable: false, sortable: false, facetable: false,
+    },
+    vectorSearch: {
-vectorSearchDimensions: 3072, %%//%% text-embedding-3-large
+      algorithms: [{ name: 'hnsw', kind: 'hnsw' }],
+      profiles: [{ name: 'vdb', algorithmConfigurationName: 'hnsw' }]
-vectorSearchProfileName: 'vdb'
+    }
+  };
+  try {
+    await indexClient.getIndex(SEARCH_INDEX);
+    console.log('Index exists. Updating (if schema changed)...');
+    await indexClient.createOrUpdateIndex(definition);
+  } catch {
+    console.log('Creating index...');
+    await indexClient.createIndex(definition);
+  }
 }
-],
-semanticSearch: {
-defaultConfigurationName: (process.env.SEMANTIC_CONFIGURATION || 'default').trim(),
-configurations: [
-{
-name: (process.env.SEMANTIC_CONFIGURATION || 'default').trim(),
-prioritizedFields: {
-titleField: { name: TITLE_FIELD },
-prioritizedContentFields: [{ name: CONTENT_FIELD }]
-}
-}
-]
-},
-vectorSearch: {
-algorithms: [{ name: 'hnsw', kind: 'hnsw' }],
-profiles: [{ name: 'vdb', algorithmConfigurationName: 'hnsw' }]
-}
-};
-try {
-await indexClient.getIndex(SEARCH_INDEX);
-console.log('Index exists. Updating (if schema changed)...');
-await indexClient.createOrUpdateIndex(definition);
-} catch {
-console.log('Creating index...');
-await indexClient.createIndex(definition);
-}
-}
 async function readDocs(dir) {
+  if (!fs.existsSync(dir)) return [];
-if (!fs.existsSync(dir)) return [];
+  const files = fs.readdirSync(dir);
+  const docs = [];
-const files = fs.readdirSync(dir);
+  for (const f of files) {
-const docs = [];
+    const full = path.join(dir, f);
+    const ext = path.extname(f).toLowerCase();
+    let text = null;
-for (const f of files) {
+    if (ext === '.docx') {
-const full = path.join(dir, f);
+      // Convert Word (DOCX) → plain text
+      const buffer = fs.readFileSync(full);
-const ext = path.extname(f).toLowerCase();
+      const { value } = await mammoth.extractRawText({ buffer });
+      text = (value || '').trim();
+    } else if (/\.(txt|md|markdown)$/i.test(f)) {
+      text = fs.readFileSync(full, 'utf8');
-let text = null;
+    } else if (ext === '.json') {
+      // Accept JSON too—either stringify or use a specific field if you prefer
+      const raw = fs.readFileSync(full, 'utf8');
+      try {
-if (ext === '.docx') {
+        const obj = JSON.parse(raw);
+        text = typeof obj === 'string' ? obj : JSON.stringify(obj, null, 2);
-%%//%% Convert Word (DOCX) → plain text
+      } catch {
+        text = raw;
-const buffer = fs.readFileSync(full);
+      }
+    } else {
-const { value } = await mammoth.extractRawText({ buffer });
+      // Unsupported type — skip
+      continue;
-text = (value || '').trim();
+    }
-} else if (/\.(txt|md|markdown)$/i.test(f)) {
+    if (!text) continue;
-text = fs.readFileSync(full, 'utf8');
+    const title = path.parse(f).name;
+    docs.push({
-} else if (ext === '.json') {
+      id: title,
+      title,
-%%//%% Accept JSON too—either stringify or use a specific field if you prefer
+      url: null,
+      content: text,
-const raw = fs.readFileSync(full, 'utf8');
+      metadata: { source: f }
+    });
-try {
+  }
-const obj = JSON.parse(raw);
+  return docs;
-text = typeof obj === 'string' ? obj : JSON.stringify(obj, null, 2);
-} catch {
-text = raw;
 }
-} else {
-%%//%% Unsupported type — skip
-continue;
-}
-if (!text) continue;
-const title = path.parse(f).name;
-docs.push({
-id: title,
-title,
-url: null,
-content: text,
-metadata: { source: f }
-});
-}
-return docs;
-}
 async function run() {
+  await ensureIndex();
-await ensureIndex();
+  const docs = await readDocs(dataDir);
+  console.log(`Found ${docs.length} docs`);
-const docs = await readDocs(dataDir);
+  const actions = [];
-console.log(`Found ${docs.length} docs`);
+  for (const doc of docs) {
+    const chunks = chunkText(doc.content);
+    // Make the parent id safe for Azure Search keys: letters/digits/_/-/=
-const actions = [];
+    const parentKey = String(doc.id || '').replace(/[^A-Za-z0-9_\-=]/g, '_');
-for (const doc of docs) {
+    for (let i = 0; i < chunks.length; i++) {
+      const chunk = chunks[i];
-const chunks = chunkText(doc.content);
+      // Chunk key is parentKey + '-' + index (no spaces or '#')
+      const id = `${parentKey}-${i}`;
-%%//%% Make the parent id safe for Azure Search keys: letters/digits/_/-/=
+      const vec = await embed(chunk);
+      actions.push({
-const parentKey = String(doc.id || '').replace(/[^A-Za-z0-9_\-=]/g, '_');
+        '@search.action': 'mergeOrUpload',
+        id,
+        [TITLE_FIELD]: doc.title,
+        [URL_FIELD]: doc.url,
-for (let i = 0; i < chunks.length; i++) {
+        [CONTENT_FIELD]: chunk,
+        [METADATA_FIELD]: doc.metadata,  // keep only declared subfields (e.g., source/category/tags)
-const chunk = chunks[i];
+        [PARENT_ID_FIELD]: parentKey,
+        [VECTOR_FIELD]: vec
+      });
+    }
-%%//%% Chunk key is parentKey + '-' + index (no spaces or '#')
+  }
-const id = `${parentKey}-${i}`;
+  console.log(`Uploading ${actions.length} chunks...`);
+  for (let i = 0; i < actions.length; i += 1000) {
+    const batch = actions.slice(i, i + 1000);
+    await searchClient.mergeOrUploadDocuments(batch);
-const vec = await embed(chunk);
+    console.log(`Uploaded ${Math.min(i + batch.length, actions.length)}/${actions.length}`);
+  }
-actions.push({
+  console.log('Ingest complete.');
-'@search.action': 'mergeOrUpload',
-id,
-[TITLE_FIELD]: doc.title,
-[URL_FIELD]: doc.url,
-[CONTENT_FIELD]: chunk,
-[METADATA_FIELD]: doc.metadata, %%//%% keep only declared subfields (e.g., source/category/tags)
-[PARENT_ID_FIELD]: parentKey,
-[VECTOR_FIELD]: vec
-});
 }
-}
-console.log(`Uploading ${actions.length} chunks...`);
-for (let i = 0; i < actions.length; i += 1000) {
-const batch = actions.slice(i, i + 1000);
-await searchClient.mergeOrUploadDocuments(batch);
-console.log(`Uploaded ${Math.min(i + batch.length, actions.length)}/${actions.length}`);
-}
-console.log('Ingest complete.');
-}
 if (require.main === module) {
+  run().catch(err => { console.error(err); process.exit(1); });
-run().catch(err => { console.error(err); process.exit(1); });
 }
+</code>
-openaiService.js
+=== openaiService.js ===
+<code>
 require('dotenv').config();
 const axios = require('axios');
 const { SearchClient, AzureKeyCredential } = require('@azure/search-documents');
+// ---------- Helpers / Config ----------
-%%//%% ---------- Helpers / Config ----------
 const cleanName = (v, def) => (v ?? def ?? '').split('#')[0].trim();
 const cleanBool = (v, def = 'false') => String(v ?? def).trim().toLowerCase() === 'true';
 const cleanNum = (v, def) => {
+  const n = Number(String(v ?? '').trim());
-const n = Number(String(v ?? '').trim());
+  return Number.isFinite(n) ? n : def;
-return Number.isFinite(n) ? n : def;
 };
 const sanitizeUrl = s => String(s || '').trim().replace(/\/+$/, '');
 const SEARCH_ENDPOINT = sanitizeUrl(process.env.AZURE_SEARCH_ENDPOINT);
 const SEARCH_INDEX = cleanName(process.env.AZURE_SEARCH_INDEX_NAME);
 const SEARCH_KEY = cleanName(process.env.AZURE_SEARCH_API_KEY);
 const OPENAI_ENDPOINT = sanitizeUrl(process.env.OPENAI_ENDPOINT);
 const OPENAI_API_KEY = cleanName(process.env.OPENAI_API_KEY);
 const OPENAI_API_VERSION = cleanName(process.env.OPENAI_API_VERSION, '2024-06-01');
 const OPENAI_DEPLOYMENT = cleanName(process.env.OPENAI_DEPLOYMENT, 'gpt-4o-mini');
 const OPENAI_EMBEDDING_DEPLOYMENT = cleanName(process.env.OPENAI_EMBEDDING_DEPLOYMENT, 'text-embedding-3-large');
 const VECTOR_FIELD = cleanName(process.env.VECTOR_FIELD, 'contentVector');
+const CONTENT_FIELD = cleanName(process.env.CONTENT_FIELD, 'content'); // env-preferred
-const CONTENT_FIELD = cleanName(process.env.CONTENT_FIELD, 'content'); %%//%% env-preferred
 const TITLE_FIELD = cleanName(process.env.TITLE_FIELD, 'title');
 const URL_FIELD = cleanName(process.env.URL_FIELD, 'url');
 const METADATA_FIELD = cleanName(process.env.METADATA_FIELD, 'metadata');
 const PARENT_ID_FIELD = cleanName(process.env.PARENT_ID_FIELD, null);
 const SEMANTIC_CONFIGURATION = cleanName(process.env.SEMANTIC_CONFIGURATION, 'default');
 const USE_SEMANTIC_RANKER = cleanBool(process.env.USE_SEMANTIC_RANKER, 'true');
 const SEMANTIC_USE_CAPTIONS = cleanBool(process.env.SEMANTIC_USE_CAPTIONS, 'false');
+const SEMANTIC_USE_ANSWERS  = cleanBool(process.env.SEMANTIC_USE_ANSWERS,  'false');
-const SEMANTIC_USE_ANSWERS = cleanBool(process.env.SEMANTIC_USE_ANSWERS, 'false');
 const LLM_RERANK = cleanBool(process.env.LLM_RERANK, 'false');
 const TOP_K = cleanNum(process.env.TOP_K, 8);
 const TEMPERATURE = cleanNum(process.env.TEMPERATURE, 0);
 const MAX_TOKENS_ANSWER = cleanNum(process.env.MAX_TOKENS_ANSWER, 800);
+// Retrieval feature toggles
-%%//%% Retrieval feature toggles
 const RETRIEVER_MULTIQUERY_N = cleanNum(process.env.RETRIEVER_MULTIQUERY_N, 1);
 const RETRIEVER_USE_HYDE = cleanBool(process.env.RETRIEVER_USE_HYDE, 'false');
+// Clients
-%%//%% Clients
 const searchClient = new SearchClient(SEARCH_ENDPOINT, SEARCH_INDEX, new AzureKeyCredential(SEARCH_KEY));
+// Timing helpers
-%%//%% Timing helpers
 function nowMs() { return Date.now(); }
 async function searchOnceRest({ query, top = TOP_K }) {
+  const endpoint = String(process.env.AZURE_SEARCH_ENDPOINT || '').trim().replace(/\/+$/, '');
-const endpoint = String(process.env.AZURE_SEARCH_ENDPOINT || '').trim().replace(/\/+$/, '');
+  const index = String(process.env.AZURE_SEARCH_INDEX_NAME || '').trim();
+  const apiKey = String(process.env.AZURE_SEARCH_QUERY_KEY || process.env.AZURE_SEARCH_API_KEY || '').trim();
-const index = String(process.env.AZURE_SEARCH_INDEX_NAME || '').trim();
+  const url = `${endpoint}/indexes('${encodeURIComponent(index)}')/docs/search.post.search?api-version=2024-07-01`;
-const apiKey = String(process.env.AZURE_SEARCH_QUERY_KEY || process.env.AZURE_SEARCH_API_KEY || '').trim();
+  const body = { search: query ?? '', top };
+  const { data } = await axios.post(url, body, {
-const url = `${endpoint}/indexes('${encodeURIComponent(index)}')/docs/search.post.search?api-version=2024-07-01`;
+    headers: {
+      'api-key': apiKey,
+      'Content-Type': 'application/json',
+      'Accept': 'application/json;odata.metadata=none'
-const body = { search: query ?? '', top };
+    },
+    timeout: 10000
-const { data } = await axios.post(url, body, {
+  });
-headers: {
+  const arr = Array.isArray(data.value) ? data.value : [];
+  return arr.map(d => ({
-'api-key': apiKey,
+    id: d.id,
+    content: String(d[CONTENT_FIELD] ?? d.contents ?? d.content ?? ''),
-'Content-Type': 'application/json',
+    title: d[TITLE_FIELD] ?? d.title ?? null,
+    url: d[URL_FIELD] ?? d.url ?? null,
-'Accept': 'application/json;odata.metadata=none'
+    metadata: d[METADATA_FIELD] ?? d.metadata ?? {},
+    parentId: PARENT_ID_FIELD ? (d[PARENT_ID_FIELD] ?? null) : null,
-},
+    score: d['@search.score'] ?? null,
+    rerankScore: null
-timeout: 10000
+  }));
-});
-const arr = Array.isArray(data.value) ? data.value : [];
-return arr.map(d => ({
-id: d.id,
-content: String(d[CONTENT_FIELD] ?? d.contents ?? d.content ?? ''),
-title: d[TITLE_FIELD] ?? d.title ?? null,
-url: d[URL_FIELD] ?? d.url ?? null,
-metadata: d[METADATA_FIELD] ?? d.metadata ?? {},
-parentId: PARENT_ID_FIELD ? (d[PARENT_ID_FIELD] ?? null) : null,
-score: d['@search.score'] ?? null,
-rerankScore: null
-}));
 }
+// ---------- Reliability: retry + memo ----------
-%%//%% ---------- Reliability: retry + memo ----------
 const sleep = (ms) => new Promise(r => setTimeout(r, ms));
 async function withRetry(fn, desc = 'request', { tries = 4, base = 300 } = {}) {
+  let lastErr;
-let lastErr;
+  for (let i = 0; i < tries; i++) {
+    try { return await fn(); } catch (e) {
-for (let i = 0; i < tries; i++) {
+      const status = e?.response?.status;
+      const retryable = status === 206 || status === 408 || status === 429 || (status >= 500 && status <= 599);
-try { return await fn(); } catch (e) {
+      if (!retryable || i === tries - 1) { lastErr = e; break; }
+      const wait = base * Math.pow(2, i) + Math.floor(Math.random() * 100);
-const status = e?.response?.status;
+      console.warn(`${desc} failed (status=${status}), retrying in ${wait}ms…`);
+      await sleep(wait);
-const retryable = status === 206 || status === 408 || status === 429 || (status >= 500 && status <= 599);
+    }
+  }
-if (!retryable || i === tries - 1) { lastErr = e; break; }
+  throw lastErr;
-const wait = base * Math.pow(2, i) + Math.floor(Math.random() * 100);
-console.warn(`${desc} failed (status=${status}), retrying in ${wait}ms…`);
-await sleep(wait);
 }
-}
-throw lastErr;
-}
 const _memo = new Map();
 const _get = (k) => _memo.get(k);
 const _set = (k, v) => { if (_memo.size > 500) _memo.clear(); _memo.set(k, v); };
+// ---------- Azure OpenAI calls ----------
-%%//%% ---------- Azure OpenAI calls ----------
 async function embed(text) {
+  const url = `${OPENAI_ENDPOINT}/openai/deployments/${OPENAI_EMBEDDING_DEPLOYMENT}/embeddings?api-version=${OPENAI_API_VERSION}`;
-const url = `${OPENAI_ENDPOINT}/openai/deployments/${OPENAI_EMBEDDING_DEPLOYMENT}/embeddings?api-version=${OPENAI_API_VERSION}`;
+  const { data } = await withRetry(
+    () => axios.post(url, { input: text }, { headers: { 'api-key': OPENAI_API_KEY, 'Content-Type': 'application/json' } }),
-const { data } = await withRetry(
+    'embed'
+  );
-() => axios.post(url, { input: text }, { headers: { 'api-key': OPENAI_API_KEY, 'Content-Type': 'application/json' } }),
+  return data.data[0].embedding;
-'embed'
-);
-return data.data[0].embedding;
 }
 async function embedMemo(text) {
+  const k = `emb:${text}`; const hit = _get(k); if (hit) return hit;
-const k = `emb:${text}`; const hit = _get(k); if (hit) return hit;
+  const v = await embed(text); _set(k, v); return v;
-const v = await embed(text); _set(k, v); return v;
 }
 async function chat(messages, opts = {}) {
+  const url = `${OPENAI_ENDPOINT}/openai/deployments/${OPENAI_DEPLOYMENT}/chat/completions?api-version=${OPENAI_API_VERSION}`;
-const url = `${OPENAI_ENDPOINT}/openai/deployments/${OPENAI_DEPLOYMENT}/chat/completions?api-version=${OPENAI_API_VERSION}`;
+  const payload = { messages, temperature: TEMPERATURE, max_tokens: MAX_TOKENS_ANSWER, ...opts };
+  const { data } = await withRetry(
-const payload = { messages, temperature: TEMPERATURE, max_tokens: MAX_TOKENS_ANSWER, ...opts };
+    () => axios.post(url, payload, { headers: { 'api-key': OPENAI_API_KEY, 'Content-Type': 'application/json' } }),
+    'chat'
-const { data } = await withRetry(
+  );
+  return data;
-() => axios.post(url, payload, { headers: { 'api-key': OPENAI_API_KEY, 'Content-Type': 'application/json' } }),
-'chat'
-);
-return data;
 }
+// ---------- Query Expansion ----------
-%%//%% ---------- Query Expansion ----------
 async function multiQueryExpand(userQuery, n = 3) {
+  const k = `mq:${n}:${userQuery}`; const hit = _get(k); if (hit) return hit;
-const k = `mq:${n}:${userQuery}`; const hit = _get(k); if (hit) return hit;
+  const sys = { role: 'system', content: 'You expand search queries. Return only diverse paraphrases as JSON array of strings.' };
+  const usr = { role: 'user', content: `Create ${n} diverse rewrites of:\n"${userQuery}"\nReturn JSON array, no prose.` };
-const sys = { role: 'system', content: 'You expand search queries. Return only diverse paraphrases as JSON array of strings.' };
+  const res = await chat([sys, usr], { temperature: 0.2, max_tokens: 200 });
+  let arr = [];
-const usr = { role: 'user', content: `Create ${n} diverse rewrites of:\n"${userQuery}"\nReturn JSON array, no prose.` };
+  try { arr = JSON.parse(res.choices[0].message.content.trim()); } catch { arr = []; }
+  const out = Array.isArray(arr) ? arr.filter(s => typeof s === 'string') : [];
-const res = await chat([sys, usr], { temperature: 0.2, max_tokens: 200 });
+  _set(k, out);
+  return out;
-let arr = [];
-try { arr = JSON.parse(res.choices[0].message.content.trim()); } catch { arr = []; }
-const out = Array.isArray(arr) ? arr.filter(s => typeof s === 'string') : [];
-_set(k, out);
-return out;
 }
 async function hydeDoc(userQuery) {
+  const k = `hyde:${userQuery}`; const hit = _get(k); if (hit) return hit;
-const k = `hyde:${userQuery}`; const hit = _get(k); if (hit) return hit;
+  const sys = { role: 'system', content: 'You draft a concise answer-like passage that could hypothetically match a knowledge base. Keep it < 1600 chars.' };
+  const usr = { role: 'user', content: userQuery };
-const sys = { role: 'system', content: 'You draft a concise answer-like passage that could hypothetically match a knowledge base. Keep it < 1600 chars.' };
+  const res = await chat([sys, usr], { temperature: 0.2, max_tokens: 400 });
+  const out = (res.choices[0].message.content || '').slice(0, 1600);
-const usr = { role: 'user', content: userQuery };
+  _set(k, out);
+  return out;
-const res = await chat([sys, usr], { temperature: 0.2, max_tokens: 400 });
-const out = (res.choices[0].message.content || '').slice(0, 1600);
-_set(k, out);
-return out;
 }
+// ---------- Retrieval Strategies ----------
-%%//%% ---------- Retrieval Strategies ----------
 function buildCommonOptions({ filter, select } = {}) {
+  // Only select what we know exists from env; no alternates here.
-%%//%% Only select what we know exists from env; no alternates here.
+  const want = select || [
+    CONTENT_FIELD,            // e.g., "contents"
-const want = select || [
+    TITLE_FIELD,              // e.g., "title"
+    URL_FIELD,                // e.g., "url"
-CONTENT_FIELD, %%//%% e.g., "contents"
+    METADATA_FIELD,           // e.g., "metadata"
+    'id',
-TITLE_FIELD, %%//%% e.g., "title"
+    PARENT_ID_FIELD           // may be null
+  ].filter(Boolean);
-URL_FIELD, %%//%% e.g., "url"
+  const base = {
-METADATA_FIELD, %%//%% e.g., "metadata"
+    top: TOP_K,
+    includeTotalCount: false
-'id',
+    // select: want
+  };
-PARENT_ID_FIELD %%//%% may be null
+  if (filter) base.filter = filter;
-].filter(Boolean);
+  if (USE_SEMANTIC_RANKER) {
+    base.queryType = 'semantic';
+    const sc = (SEMANTIC_CONFIGURATION || '').trim();
+    if (sc) base.semanticConfiguration = sc;
-const base = {
+    base.queryLanguage = 'en-us';
+    if (SEMANTIC_USE_CAPTIONS) base.captions = 'extractive|highlight-false';
-top: TOP_K,
+    if (SEMANTIC_USE_ANSWERS)  base.answers  = 'extractive|count-1';
+  }
-includeTotalCount: false
+  return base;
-%%//%% select: want
-};
-if (filter) base.filter = filter;
-if (USE_SEMANTIC_RANKER) {
-base.queryType = 'semantic';
-const sc = (SEMANTIC_CONFIGURATION || '').trim();
-if (sc) base.semanticConfiguration = sc;
-base.queryLanguage = 'en-us';
-if (SEMANTIC_USE_CAPTIONS) base.captions = 'extractive|highlight-false';
-if (SEMANTIC_USE_ANSWERS) base.answers = 'extractive|count-1';
 }
-return base;
-}
 function mapSearchResult(r) {
+  const d = r.document || r;
-const d = r.document || r;
+  // HARDENED: never return empty content if either field is present
+  const content =
+    d[CONTENT_FIELD] ??
-%%//%% HARDENED: never return empty content if either field is present
+    d.contents ??
+    d.content ??
-const content =
+    '';
-d[CONTENT_FIELD] ??
+  return {
+    id: d.id ?? d['@search.action'],
-d.contents ??
+    content,
+    title: d[TITLE_FIELD] ?? d.title ?? null,
-d.content ??
+    url: d[URL_FIELD] ?? d.url ?? null,
+    metadata: d[METADATA_FIELD] ?? d.metadata ?? {},
-'';
+    parentId: PARENT_ID_FIELD ? (d[PARENT_ID_FIELD] ?? null) : null,
+    score: r['@search.score'] ?? r.score,
+    rerankScore: null
+  };
-return {
-id: d.id ?? d['@search.action'],
-content,
-title: d[TITLE_FIELD] ?? d.title ?? null,
-url: d[URL_FIELD] ?? d.url ?? null,
-metadata: d[METADATA_FIELD] ?? d.metadata ?? {},
-parentId: PARENT_ID_FIELD ? (d[PARENT_ID_FIELD] ?? null) : null,
-score: r['@search.score'] ?? r.score,
-rerankScore: null
-};
 }
+// RRF helper
-%%//%% RRF helper
 function rrfFuse(lists, k = 60) {
+  const scores = new Map();
-const scores = new Map();
+  for (const list of lists) {
+    list.forEach((item, idx) => {
-for (const list of lists) {
+      const prev = scores.get(item.id) || 0;
+      scores.set(item.id, prev + 1 / (k + idx + 1));
-list.forEach((item, idx) => {
+    });
+  }
-const prev = scores.get(item.id) || 0;
+  const itemById = new Map();
+  for (const list of lists) for (const it of list) if (!itemById.has(it.id)) itemById.set(it.id, it);
-scores.set(item.id, prev + 1 / (k + idx + 1));
+  return Array.from(scores.entries()).sort((a,b)=>b[1]-a[1]).map(([id]) => itemById.get(id));
-});
 }
-const itemById = new Map();
-for (const list of lists) for (const it of list) if (!itemById.has(it.id)) itemById.set(it.id, it);
-return Array.from(scores.entries()).sort((a,b)=>b[1]-a[1]).map(([id]) => itemById.get(id));
-}
 async function searchOnce({ query, vector, fields, filter }) {
+  const opts = buildCommonOptions({ filter });
-const opts = buildCommonOptions({ filter });
+  if (vector) {
+    const fieldList = Array.isArray(fields)
+      ? fields
-if (vector) {
+      : [ (fields || VECTOR_FIELD) ].map(s => String(s || '').trim()).filter(Boolean);
+    opts.vectorSearchOptions = {
-const fieldList = Array.isArray(fields)
+      queries: [{
+        kind: 'vector',
-? fields
+        vector,
+        kNearestNeighborsCount: TOP_K,
-: [ (fields || VECTOR_FIELD) ].map(s => String(s || '').trim()).filter(Boolean);
+        fields: fieldList,
+      }]
-opts.vectorSearchOptions = {
+    };
+  } else {
-queries: [{
+    // BM25 path: explicitly tell Search which fields to match on
+    opts.searchFields = [CONTENT_FIELD, TITLE_FIELD].filter(Boolean);
-kind: 'vector',
+  }
-vector,
+  const results = [];
+  const isAsyncIterable = (x) => x && typeof x[Symbol.asyncIterator] === 'function';
-kNearestNeighborsCount: TOP_K,
+  const run = async (o) => {
-fields: fieldList,
+    const iter = searchClient.search(query || '', o);
+    if (isAsyncIterable(iter)) {
-}]
+      for await (const r of iter) results.push(mapSearchResult(r));
+    } else if (iter?.results && isAsyncIterable(iter.results)) {
-};
+      for await (const r of iter.results) results.push(mapSearchResult(r));
+    } else if (Array.isArray(iter?.results)) {
-} else {
+      for (const r of iter.results) results.push(mapSearchResult(r));
+    }
-%%//%% BM25 path: explicitly tell Search which fields to match on
+  };
-opts.searchFields = [CONTENT_FIELD, TITLE_FIELD].filter(Boolean);
+  try {
+    await run(opts);
+  } catch (e) {
+    const msg = String(e?.message || '');
+    const status = e?.statusCode || e?.response?.status;
+    const selectProblem =
+      (status === 400 && /Parameter name:\s*\$select/i.test(msg)) ||
+      /Could not find a property named .* on type 'search\.document'/i.test(msg);
+    const overload =
+      /semanticPartialResponse|CapacityOverloaded/i.test(msg) ||
+      status === 206 || status === 503;
+    if (selectProblem) {
+      console.warn('Invalid $select detected → retrying without select');
+      const fallback = { ...opts };
+      delete fallback.select;
+      await run(fallback);
+    } else if (USE_SEMANTIC_RANKER && overload) {
+      console.warn('Semantic ranker overloaded → falling back to BM25 for this query.');
+      const fallback = { ...opts };
+      delete fallback.queryType;
+      delete fallback.semanticConfiguration;
+      delete fallback.queryLanguage;
+      delete fallback.captions;
+      delete fallback.answers;
+      await run(fallback);
+    } else {
+      throw e;
+    }
+  }
+  // 👉 If BM25 (non-vector) still returned 0, fallback to REST (we know it works)
+  if (!vector && results.length === 0) {
+    try {
+      const restHits = await searchOnceRest({ query, top: TOP_K });
+      if (restHits.length) return restHits;
+    } catch (e) {
+      console.warn('REST fallback failed:', e?.message || e);
+    }
+  }
+  return results;
 }
-const results = [];
-const isAsyncIterable = (x) => x && typeof x[Symbol.asyncIterator] === 'function';
-const run = async (o) => {
-const iter = searchClient.search(query || '', o);
-if (isAsyncIterable(iter)) {
-for await (const r of iter) results.push(mapSearchResult(r));
-} else if (iter?.results && isAsyncIterable(iter.results)) {
-for await (const r of iter.results) results.push(mapSearchResult(r));
-} else if (Array.isArray(iter?.results)) {
-for (const r of iter.results) results.push(mapSearchResult(r));
-}
-};
-try {
-await run(opts);
-} catch (e) {
-const msg = String(e?.message || '');
-const status = e?.statusCode || e?.response?.status;
-const selectProblem =
-(status === 400 && /Parameter name:\s*\$select/i.test(msg)) ||
-/Could not find a property named .* on type 'search\.document'/i.test(msg);
-const overload =
-/semanticPartialResponse|CapacityOverloaded/i.test(msg) ||
-status === 206 || status === 503;
-if (selectProblem) {
-console.warn('Invalid $select detected → retrying without select');
-const fallback = { ...opts };
-delete fallback.select;
-await run(fallback);
-} else if (USE_SEMANTIC_RANKER && overload) {
-console.warn('Semantic ranker overloaded → falling back to BM25 for this query.');
-const fallback = { ...opts };
-delete fallback.queryType;
-delete fallback.semanticConfiguration;
-delete fallback.queryLanguage;
-delete fallback.captions;
-delete fallback.answers;
-await run(fallback);
-} else {
-throw e;
-}
-}
-%%//%% 👉 If BM25 (non-vector) still returned 0, fallback to REST (we know it works)
-if (!vector && results.length === 0) {
-try {
-const restHits = await searchOnceRest({ query, top: TOP_K });
-if (restHits.length) return restHits;
-} catch (e) {
-console.warn('REST fallback failed:', e?.message || e);
-}
-}
-return results;
-}
 function collapseByParent(items, maxPerParent = 3) {
+  if (!PARENT_ID_FIELD) return items;
-if (!PARENT_ID_FIELD) return items;
+  const groups = new Map();
+  for (const it of items) {
-const groups = new Map();
+    const key = it.parentId || it.id;
+    if (!groups.has(key)) groups.set(key, []);
-for (const it of items) {
+    groups.get(key).push(it);
+  }
-const key = it.parentId || it.id;
+  const collapsed = [];
+  for (const arr of groups.values()) {
-if (!groups.has(key)) groups.set(key, []);
+    arr.sort((a, b) => (b.rerankScore ?? b.score ?? 0) - (a.rerankScore ?? a.score ?? 0));
+    collapsed.push(...arr.slice(0, maxPerParent));
-groups.get(key).push(it);
+  }
+  return collapsed.slice(0, TOP_K);
 }
-const collapsed = [];
-for (const arr of groups.values()) {
-arr.sort((a, b) => (b.rerankScore ?? b.score ?? 0) - (a.rerankScore ?? a.score ?? 0));
-collapsed.push(...arr.slice(0, maxPerParent));
-}
-return collapsed.slice(0, TOP_K);
-}
 async function retrieve(userQuery, { filter } = {}) {
+  const t0 = nowMs();
-const t0 = nowMs();
+  const lists = [];
-const lists = [];
+  // Core keyword/semantic
+  lists.push(await searchOnce({ query: userQuery, filter }));
+  // Vector on the raw query (memoized)
-%%//%% Core keyword/semantic
+  try {
+    const qvec = await embedMemo(userQuery);
-lists.push(await searchOnce({ query: userQuery, filter }));
+    lists.push(await searchOnce({ query: '', vector: qvec, filter }));
+  } catch (e) {
+    console.warn('vector(query) failed:', e?.response?.status || e?.message || e);
+  }
-%%//%% Vector on the raw query (memoized)
+  // Multi-query expansion (togglable)
-try {
+  let expansions = [];
+  if (RETRIEVER_MULTIQUERY_N > 0) {
-const qvec = await embedMemo(userQuery);
+    try {
+      expansions = await multiQueryExpand(userQuery, RETRIEVER_MULTIQUERY_N);
-lists.push(await searchOnce({ query: '', vector: qvec, filter }));
+      for (const q of expansions) {
+        lists.push(await searchOnce({ query: q, filter }));
-} catch (e) {
+      }
+    } catch (e) {
-console.warn('vector(query) failed:', e?.response?.status || e?.message || e);
+      console.warn('multiQueryExpand failed (continuing):', e?.response?.status || e?.message || e);
+    }
+  }
+  // HyDE (togglable)
+  if (RETRIEVER_USE_HYDE) {
+    try {
+      const pseudo = await hydeDoc(userQuery);
+      const pvec = await embedMemo(pseudo);
+      lists.push(await searchOnce({ query: '', vector: pvec, filter }));
+    } catch (e) {
+      console.warn('HyDE failed (continuing):', e?.response?.status || e?.message || e);
+    }
+  }
+  // Fuse & collapse
+  const fused = collapseByParent(rrfFuse(lists));
+  const latencyMs = nowMs() - t0;
+  return { items: fused.slice(0, TOP_K), latencyMs, expansions };
 }
-%%//%% Multi-query expansion (togglable)
-let expansions = [];
-if (RETRIEVER_MULTIQUERY_N > 0) {
-try {
-expansions = await multiQueryExpand(userQuery, RETRIEVER_MULTIQUERY_N);
-for (const q of expansions) {
-lists.push(await searchOnce({ query: q, filter }));
-}
-} catch (e) {
-console.warn('multiQueryExpand failed (continuing):', e?.response?.status || e?.message || e);
-}
-}
-%%//%% HyDE (togglable)
-if (RETRIEVER_USE_HYDE) {
-try {
-const pseudo = await hydeDoc(userQuery);
-const pvec = await embedMemo(pseudo);
-lists.push(await searchOnce({ query: '', vector: pvec, filter }));
-} catch (e) {
-console.warn('HyDE failed (continuing):', e?.response?.status || e?.message || e);
-}
-}
-%%//%% Fuse & collapse
-const fused = collapseByParent(rrfFuse(lists));
-const latencyMs = nowMs() - t0;
-return { items: fused.slice(0, TOP_K), latencyMs, expansions };
-}
 async function rerankWithLLM(query, items) {
+  if (!LLM_RERANK || !items.length) return items;
-if (!LLM_RERANK || !items.length) return items;
+  const sys = { role: 'system', content: 'Rank passages by relevance to the query. Return JSON array of {id, score} 0..100.' };
+  const passages = items.map(p => ({ id: p.id, title: p.title || '', url: p.url || '', content: (p.content || '').slice(0, 1200) }));
-const sys = { role: 'system', content: 'Rank passages by relevance to the query. Return JSON array of {id, score} 0..100.' };
+  const usr = { role: 'user', content: `Query: ${query}\nPassages:\n${JSON.stringify(passages, null, 2)}\nReturn only JSON array.` };
+  const res = await chat([sys, usr], { temperature: 0.0, max_tokens: 400 });
-const passages = items.map(p => ({ id: p.id, title: p.title || '', url: p.url || '', content: (p.content || '').slice(0, 1200) }));
+  let scores = [];
+  try { scores = JSON.parse(res.choices[0].message.content); } catch {}
-const usr = { role: 'user', content: `Query: ${query}\nPassages:\n${JSON.stringify(passages, null, 2)}\nReturn only JSON array.` };
+  const byId = new Map(scores.map(s => [String(s.id), Number(s.score) || 0]));
+  for (const it of items) it.rerankScore = byId.get(String(it.id)) ?? null;
-const res = await chat([sys, usr], { temperature: 0.0, max_tokens: 400 });
+  items.sort((a, b) => (b.rerankScore ?? 0) - (a.rerankScore ?? 0));
+  return items;
-let scores = [];
-try { scores = JSON.parse(res.choices[0].message.content); } catch {}
-const byId = new Map(scores.map(s => [String(s.id), Number(s.score) || 0]));
-for (const it of items) it.rerankScore = byId.get(String(it.id)) ?? null;
-items.sort((a, b) => (b.rerankScore ?? 0) - (a.rerankScore ?? 0));
-return items;
 }
 function toCitableChunks(items) {
+  return items.map((it, idx) => ({
-return items.map((it, idx) => ({
+    key: `[${idx + 1}]`,
+    id: it.id,
-key: `[${idx + 1}]`,
+    title: it.title || `Doc ${idx + 1}`,
+    url: it.url || null,
-id: it.id,
+    content: (it.content || '').slice(0, 2000)
+  }));
-title: it.title || `Doc ${idx + 1}`,
-url: it.url || null,
-content: (it.content || '').slice(0, 2000)
-}));
 }
 async function synthesizeAnswer(userQuery, chunks) {
+  const sys = { role: 'system', content:
-const sys = { role: 'system', content:
+`You answer using only the provided sources. Cite like [1], [2] inline.
-`You answer using only the provided sources. Cite like [1], [2] inline.
 If unsure or missing info, say you don't know.
 Return a JSON object: { "answer": string, "citations": [{"key":"[n]","id":"...","title":"...","url":"..."}] }.`};
+  const usr = { role: 'user', content: `Question:\n${userQuery}\n\nSources:\n${JSON.stringify(chunks, null, 2)}` };
-const usr = { role: 'user', content: `Question:\n${userQuery}\n\nSources:\n${JSON.stringify(chunks, null, 2)}` };
+  const data = await chat([sys, usr], { temperature: TEMPERATURE, max_tokens: MAX_TOKENS_ANSWER });
+  const raw = data.choices[0].message.content;
-const data = await chat([sys, usr], { temperature: TEMPERATURE, max_tokens: MAX_TOKENS_ANSWER });
+  let parsed;
+  try { parsed = JSON.parse(raw); } catch {
-const raw = data.choices[0].message.content;
+    parsed = { answer: raw, citations: chunks.map(c => ({ key: c.key, id: c.id, title: c.title, url: c.url })) };
+  }
-let parsed;
+  parsed.citations = parsed.citations || chunks.map(c => ({ key: c.key, id: c.id, title: c.title, url: c.url }));
+  return { ...parsed, usage: data.usage || null };
-try { parsed = JSON.parse(raw); } catch {
-parsed = { answer: raw, citations: chunks.map(c => ({ key: c.key, id: c.id, title: c.title, url: c.url })) };
 }
-parsed.citations = parsed.citations || chunks.map(c => ({ key: c.key, id: c.id, title: c.title, url: c.url }));
-return { ...parsed, usage: data.usage || null };
-}
 async function answerQuestion(userQuery, options = {}) {
+  const retrieval = await retrieve(userQuery, options);
-const retrieval = await retrieve(userQuery, options);
+  let items = retrieval.items;
+  items = await rerankWithLLM(userQuery, items);
-let items = retrieval.items;
+  const topChunks = toCitableChunks(items.slice(0, TOP_K));
-items = await rerankWithLLM(userQuery, items);
+  const t0 = nowMs();
+  const synthesis = await synthesizeAnswer(userQuery, topChunks);
+  const genLatencyMs = nowMs() - t0;
-const topChunks = toCitableChunks(items.slice(0, TOP_K));
+  return {
+    answer: synthesis.answer,
-const t0 = nowMs();
+    citations: synthesis.citations,
+    retrieved: items,
-const synthesis = await synthesizeAnswer(userQuery, topChunks);
+    metrics: {
+      retrievalLatencyMs: retrieval.latencyMs,
-const genLatencyMs = nowMs() - t0;
+      generationLatencyMs: genLatencyMs,
+      expansions: retrieval.expansions,
+      usage: synthesis.usage || null
+    }
-return {
+  };
-answer: synthesis.answer,
-citations: synthesis.citations,
-retrieved: items,
-metrics: {
-retrievalLatencyMs: retrieval.latencyMs,
-generationLatencyMs: genLatencyMs,
-expansions: retrieval.expansions,
-usage: synthesis.usage || null
 }
-};
-}
 module.exports = {
+  answerQuestion,
-answerQuestion,
+  _internals: { retrieve, rerankWithLLM, toCitableChunks, embed, embedMemo, chat, multiQueryExpand, hydeDoc, searchOnce }
-_internals: { retrieve, rerankWithLLM, toCitableChunks, embed, embedMemo, chat, multiQueryExpand, hydeDoc, searchOnce }
 };
+</code>
-package.json
+=== package.json ===
+<code>
 {
+    "name": "openai-bot",
-"name": "openai-bot",
+    "version": "1.0.0",
+    "description": "Testing openai bot",
-"version": "1.0.0",
+    "author": "Generated using Microsoft Bot Builder Yeoman generator v4.22.1",
+    "license": "MIT",
-"description": "Testing openai bot",
+    "main": "index.js",
+    "scripts": {
-"author": "Generated using Microsoft Bot Builder Yeoman generator v4.22.1",
+        "start": "node ./index.js",
+        "watch": "nodemon ./index.js",
-"license": "MIT",
+        "lint": "eslint .",
+        "test": "echo \"Error: no test specified\" && exit 1"
-"main": "index.js",
+    },
+    "repository": {
-"scripts": {
+        "type": "git",
+        "url": "https://github.com"
-"start": "node ./index.js",
+    },
+    "dependencies": {
-"watch": "nodemon ./index.js",
+        "@azure/search-documents": "^12.1.0",
+        "axios": "^1.5.0",
-"lint": "eslint .",
+        "botbuilder": "~4.22.1",
+        "dotenv": "~8.2.0",
-"test": "echo \"Error: no test specified\" && exit 1"
+        "mammoth": "^1.10.0",
+        "restify": "~11.1.0"
-},
+    },
+    "devDependencies": {
-"repository": {
+        "eslint": "^7.0.0",
+        "eslint-config-standard": "^14.1.1",
-"type": "git",
+        "eslint-plugin-import": "^2.20.2",
+        "eslint-plugin-node": "^11.1.0",
-"url": "https:%%//%%github.com"
+        "eslint-plugin-promise": "^4.2.1",
+        "eslint-plugin-standard": "^4.0.1",
-},
+        "nodemon": "^2.0.4"
+    },
-"dependencies": {
+    "engines": { "node": ">=20 <21" }
-"@azure/search-documents": "^12.1.0",
-"axios": "^1.5.0",
-"botbuilder": "~4.22.1",
-"dotenv": "~8.2.0",
-"mammoth": "^1.10.0",
-"restify": "~11.1.0"
-},
-"devDependencies": {
-"eslint": "^7.0.0",
-"eslint-config-standard": "^14.1.1",
-"eslint-plugin-import": "^2.20.2",
-"eslint-plugin-node": "^11.1.0",
-"eslint-plugin-promise": "^4.2.1",
-"eslint-plugin-standard": "^4.0.1",
-"nodemon": "^2.0.4"
-},
-"engines": { "node": ">=20 <21" }
 }
+</code>

Combined Cloud Managed Services

Site Tools

Differences

Page Tools