feat: usage tracking + knowledge import script

- usage_tracker.py: daily token/call buckets per user (home/{user}/usage.json)
- Hook into local backend (OpenAI usage field) and Gemini API (usage_metadata)
- Claude/Gemini CLI backends produce no structured token data and are not tracked
- Fix CLAUDE.md stale tool count (27 → 39) and refresh tool list
- scripts/import_knowledge.py: walk markdown dirs, chunk by H2, call local LLM
  for summaries, create AE journal entries with path-derived tags; resumable via
  state file; --dry-run and --limit flags for safe testing

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Scott Idem
2026-05-02 20:38:31 -04:00
parent eab92d876d
commit 8d4aa4094c
5 changed files with 524 additions and 3 deletions

View File

@@ -26,6 +26,8 @@ from google.genai import types
from config import settings
from llm_client import complete
from tools import TOOL_DECLARATIONS, call_tool, get_tools_for_role, CONFIRM_REQUIRED
import usage_tracker
from persona import _user
logger = logging.getLogger(__name__)
@@ -44,6 +46,25 @@ Keep your summary factual and complete. Include relevant URLs, data, and specifi
If no tools are needed, return an empty summary."""
def _track_gemini_usage(response, model_name: str | None) -> None:
meta = getattr(response, "usage_metadata", None)
if not meta:
return
prompt_tokens = getattr(meta, "prompt_token_count", 0) or 0
completion_tokens = getattr(meta, "candidates_token_count", 0) or 0
if prompt_tokens or completion_tokens:
try:
asyncio.create_task(usage_tracker.record(
username=_user.get(),
backend="gemini_api",
model_name=model_name or settings.orchestrator_model,
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
))
except Exception:
pass
@dataclass
class OrchestrateCheckpoint:
"""Saved execution state for a job paused at a confirmation gate."""
@@ -285,6 +306,7 @@ async def _run_from_contents(
system_instruction=_ORCHESTRATOR_SYSTEM,
),
)
_track_gemini_usage(response, model_name)
candidate = response.candidates[0]
parts = candidate.content.parts if candidate.content else []
@@ -348,6 +370,7 @@ async def _run_from_contents(
system_instruction=_ORCHESTRATOR_SYSTEM,
),
)
_track_gemini_usage(conf_response, model_name)
conf_parts = (
conf_response.candidates[0].content.parts
if conf_response.candidates and conf_response.candidates[0].content