Scott.Idem/Cortex-Inara
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: token streaming for orchestrator final response

Browse Source 
Switches the orchestrator's final response from a fire-and-wait model to a
live SSE stream so text appears token-by-token as the model generates it.

- llm_client: complete() gains token_sink param; anthropic_api backend uses
  client.messages.stream(); local backend uses httpx SSE streaming; non-streaming
  backends (claude_cli, gemini_cli) emit the full text as one chunk
- orchestrator_engine + openai_orchestrator: token_sink threaded through run(),
  _run_from_contents(), _claude_handoff(), and _run_from_messages()
- routers/orchestrator: each job gets an asyncio.Queue; _on_progress and
  _token_sink write progress/token events to it; _finalize_job emits done,
  error handler emits error, confirmation gate emits confirm; new GET
  /orchestrate/{job_id}/stream SSE endpoint with 20s keepalive
- app.js: _doOrchestrate switches from 2s poll loop to EventSource; thinking
  bubble converts to a streaming message on first token; auto-scroll while
  streaming; confirm/error/done events handled; finalization unchanged

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Scott Idem
2026-06-16 23:22:50 -04:00
parent c31eba111f
commit 9cb2b0d9a5
6 changed files with 293 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
cortex/static/app.js
View File

@@ -1475,68 +1475,79 @@
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const { job_id } = await res.json();
// Poll until complete or stopped
let job;
while (true) {
if (activeController.signal.aborted) throw new DOMException('Aborted', 'AbortError');
// Stream events from the job via SSE
const job = await new Promise((resolve, reject) => {
const es = new EventSource(`/orchestrate/${job_id}/stream`);
let streamingStarted = false;
let accumulatedText = '';
await new Promise(r => setTimeout(r, 2000));
const abort = activeController.signal;
abort.addEventListener('abort', () => { es.close(); reject(new DOMException('Aborted', 'AbortError')); });
if (activeController.signal.aborted) throw new DOMException('Aborted', 'AbortError');
es.onmessage = async (e) => {
let event;
try { event = JSON.parse(e.data); } catch { return; }
const pollRes = await fetch(`/orchestrate/${job_id}`, {
signal: activeController.signal,
});
if (!pollRes.ok) throw new Error(`Poll failed: HTTP ${pollRes.status}`);
job = await pollRes.json();
if (event.type === 'connected' || event.type === 'keepalive') return;
if (job.status === 'queued' || job.status === 'running') {
const prog = job.progress;
const n = job.tool_calls?.length || 0;
if (prog) {
thinkingDiv.textContent = `⚡ ${prog}`;
} else {
thinkingDiv.textContent = n
? `⚡ working… (${n} tool${n !== 1 ? 's' : ''} used)`
: '⚡ working…';
if (event.type === 'progress') {
if (!streamingStarted) thinkingDiv.textContent = `⚡ ${event.text}`;
return;
}
continue;
}
if (job.status === 'awaiting_confirmation') {
const pc = job.pending_confirmation || {};
const toolNames = (pc.tools || []).map(t => t.name).join(', ');
thinkingDiv.className = 'message assistant';
thinkingDiv.innerHTML = `<div class="confirm-gate">
<p>${escapeHtml(pc.message || 'Confirm this action?')}</p>
<p class="confirm-tools">Tool${(pc.tools||[]).length !== 1 ? 's' : ''}: <code>${escapeHtml(toolNames)}</code></p>
<div class="confirm-actions">
<button class="confirm-btn">Confirm</button>
<button class="deny-btn">Deny</button>
</div>
</div>`;
if (event.type === 'token') {
if (!streamingStarted) {
streamingStarted = true;
thinkingDiv.className = 'message assistant';
thinkingDiv.innerHTML = '';
}
accumulatedText += event.text;
setMessageText(thinkingDiv, 'assistant', accumulatedText);
thinkingDiv.scrollIntoView({ behavior: 'smooth', block: 'end' });
return;
}
const confirmed = await new Promise(resolve => {
thinkingDiv.querySelector('.confirm-btn').onclick = () => resolve(true);
thinkingDiv.querySelector('.deny-btn').onclick = () => resolve(false);
});
if (event.type === 'confirm') {
const pc = event;
const toolNames = (pc.tools || []).map(t => t.name).join(', ');
thinkingDiv.className = 'message assistant';
thinkingDiv.innerHTML = `<div class="confirm-gate">
<p>${escapeHtml(pc.message || 'Confirm this action?')}</p>
<p class="confirm-tools">Tool${(pc.tools||[]).length !== 1 ? 's' : ''}: <code>${escapeHtml(toolNames)}</code></p>
<div class="confirm-actions">
<button class="confirm-btn">Confirm</button>
<button class="deny-btn">Deny</button>
</div>
</div>`;
const confirmed = await new Promise(r => {
thinkingDiv.querySelector('.confirm-btn').onclick = () => r(true);
thinkingDiv.querySelector('.deny-btn').onclick = () => r(false);
});
thinkingDiv.className = 'message assistant thinking';
thinkingDiv.textContent = confirmed ? '⚡ confirmed — continuing…' : '⚡ denied — finishing…';
streamingStarted = false;
accumulatedText = '';
const action = confirmed ? 'confirm' : 'deny';
await fetch(`/orchestrate/${job_id}/${action}`, { method: 'POST' });
return;
}
thinkingDiv.className = 'message assistant thinking';
thinkingDiv.textContent = confirmed ? '⚡ confirmed — continuing…' : '⚡ denied — finishing…';
if (event.type === 'error') {
es.close();
reject(new Error(event.message || 'Orchestrator failed'));
return;
}
const action = confirmed ? 'confirm' : 'deny';
const resumeRes = await fetch(`/orchestrate/${job_id}/${action}`, {
method: 'POST',
signal: activeController.signal,
});
if (!resumeRes.ok) throw new Error(`Resume failed: HTTP ${resumeRes.status}`);
continue;
}
if (event.type === 'done') {
es.close();
// If we received tokens, the response is already rendered —
// use accumulatedText; otherwise fall back to event.response.
resolve({ ...event, response: accumulatedText || event.response });
}
};
break;
}
if (job.status === 'error') throw new Error(job.error || 'Orchestrator failed');
es.onerror = () => { es.close(); reject(new Error('Stream connection lost')); };
});
// Update session so this turn is part of the resumable history
if (job.session_id) {
@@ -1548,6 +1559,7 @@
const userHistIdx = currentHistory.length - 1; // pushed before fetch
attachHistoryControls(userMsgDiv, userHistIdx);
// If tokens streamed, the div is already a message; if not, set text now.
thinkingDiv.className = 'message assistant';
setMessageText(thinkingDiv, 'assistant', job.response || '(no response)');
const assistHistIdx = currentHistory.length;