diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -8,7 +8,6 @@ on: jobs: deploy: runs-on: ubuntu-latest - if: false # Temporarily disabled steps: - name: Deploy to server diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,9 @@ google.json # Application logs app.log + +# Pipeline test outputs +test_outputs/ +new-plans-*/ +.claude +/workflow-dashboard-cloud \ No newline at end of file diff --git a/app_tester/action_finder/claude_agent.py b/app_tester/action_finder/claude_agent.py new file mode 100644 --- /dev/null +++ b/app_tester/action_finder/claude_agent.py @@ -0,0 +1,214 @@ +""" +Claude Code Action Finder + +Uses GenericAgent with Claude SDK for action discovery. +Leverages Claude's built-in web search, file operations, and Composio's Firecrawl tool with processors. +""" + +# THIS IS NOT THE FINAL AGENT, USE THIS AS REFERENCE TO USE THE NEW PATTERNS + +from pathlib import Path +from typing import TYPE_CHECKING + +from app_tester.common.generic_agent import AgentProvider, create_generic_agent + +# Import GenericTool for Composio REST API integration +from app_tester.common.generic_tools import GenericTool +from app_tester.common.mercury_utils import get_all_actions +from app_tester.constant import mercury_path + +from .models import BatchActionFinderFilterResponse + + +if TYPE_CHECKING: + from app_tester.common.generic_agent import GenericAgent + + +def create_claude_action_finder( + app_name: str, + finder_instructions: str = "", + target_count: int = 1, +) -> "GenericAgent": + """ + Create Claude Code action finder agent using Composio tools. + + Args: + app_name: Name of the app to find actions for + finder_instructions: Optional specific instructions for what to find + target_count: Number of actions to find (default: 1) + + Returns: + GenericAgent configured for action finding + """ + # Get existing actions + current_actions = get_all_actions(app_name, load=True) + app_folder = Path(mercury_path) / "apps" / app_name + + # Build prompt (based on original ACTION_FINDER_PROMPT) + instructions = f"""You are an expert assistant that searches for official API documentation and actions for {app_name}. + + TARGET: Find UP TO {target_count} MOST IMPORTANT API endpoint{"s" if target_count > 1 else ""} that {"are" if target_count > 1 else "is"} NOT yet implemented. Stop when you reach {target_count} actions OR when there are no more valid endpoints to implement. + + CURRENT ACTIONS ALREADY IMPLEMENTED: + {', '.join(current_actions) if current_actions else 'None - This is the FIRST action for this app!'} + + APP FOLDER: {app_folder} + + YOUR AVAILABLE TOOLS: + - WebSearch: Search the web for official API documentation URLs + - WebFetch: Fetch and read documentation pages for detailed endpoint information + - FIRECRAWL_SCRAPE: Scrape and crawl entire documentation sites to extract comprehensive endpoint data + - Read: Check existing action files in apps/{app_name}/actions/ to understand implementation patterns + - Grep: Search codebase for similar implementations + - Write: Create new files or overwrite existing files (save research data, endpoint lists, etc.) + - Edit: Modify existing files + + YOUR TASK: + 1. **Find the main API documentation URL**: + - Use WebSearch ONCE to find "{app_name} official API documentation" or "{app_name} API reference" + - Identify the main documentation site URL from search results + + 2. **IMMEDIATELY use FIRECRAWL_SCRAPE to crawl the entire docs site**: + - **USE FIRECRAWL_SCRAPE RIGHT AWAY** on the main API documentation URL + - This will crawl the entire documentation site and extract ALL endpoint data in one comprehensive operation + - This gives you complete, structured data about all available API endpoints + + 3. **Systematically list ALL documented endpoints** from the Firecrawl results: + - Go through EVERY resource category and section from the crawled data + - Extract ALL endpoints (CRUD operations, file operations, messaging, user management, etc.) + - Include HTTP method, path, and brief description + - Verify against the crawled documentation data + + 4. **Cross-reference with existing actions**: + - Read existing action files using Read tool to understand patterns + - Use Grep to search for similar implementations + - Identify which documented endpoints are NOT yet implemented + + 5. **Rank endpoints by importance and select UP TO {target_count}**: + - IMPORTANCE CRITERIA (in priority order): + * Core CRUD operations (Create, Read, Update, Delete) + * Most frequently used endpoints (search, list, get details) + * High-value business operations (send, publish, execute) + * Prerequisites for other endpoints (auth, get ID, list resources) + - If this is the FIRST action: Prioritize foundational endpoints (get, list, create) + - If there are existing actions: Fill gaps in functionality + - Ensure each action is EXECUTABLE (all dependencies exist in current actions) + - Return UP TO {target_count} action{"s" if target_count > 1 else ""}, ranked from most to least important + - If there are fewer than {target_count} valid new endpoints, return only what you found (DO NOT make up or hallucinate endpoints) + + DECISION PRIORITY: + - **Highest:** Official API documentation from FIRECRAWL crawled data (source of truth) + - Medium: Cross-reference with similar apps + - Low: Inferred patterns + + THE NEW ACTION MUST BE COMPLETELY EXECUTABLE USING EXISTING ACTIONS. + ALL PREREQUISITE ACTIONS MUST EXIST IN THE CURRENT ACTIONS LIST. + + {'SPECIAL INSTRUCTIONS: ' + finder_instructions if finder_instructions else ''} + + CRITICAL OUTPUT REQUIREMENTS: + - Use one WebSearch to find the docs URL, then IMMEDIATELY use FIRECRAWL_SCRAPE to get all data + - **PRIORITIZE FIRECRAWL_SCRAPE** - use it as soon as you have the docs URL + - Read actual action files to understand the codebase + - Write your research findings to a file (e.g., `action_finder_research_{app_name}.md`) to save ALL discovered endpoints in a very structured manner + - Return UP TO {target_count} action{"s" if target_count > 1 else ""}, each with: action_name, description, API endpoint, documentation URL + - If you find fewer than {target_count} valid endpoints, that's OK - return only what exists + - Rank actions from MOST important to LEAST important + - Base your response on ACTUAL Firecrawl crawled data, not assumptions + - NEVER hallucinate or make up endpoints that don't exist in the official documentation + + Start by searching for "{app_name} API documentation" then IMMEDIATELY use FIRECRAWL_SCRAPE on the docs site. + """ + + # Create Composio tool using REST API - simple and clean + firecrawl_tool = GenericTool.from_composio( + tool_slug="FIRECRAWL_SCRAPE", user_id="samvit_test" + ) + + tools = [firecrawl_tool] + print(tools) + # Create agent with Composio Firecrawl tool + agent = create_generic_agent( + name=f"claude_action_finder__{app_name}", + model={ + AgentProvider.CLAUDE: "claude-sonnet-4-5", # Explicit model for consistency + AgentProvider.OPENAI: "gpt-5", # Fallback + }, + instructions=instructions, + tools=tools, # Composio Firecrawl tool with processors + output_type=BatchActionFinderFilterResponse, + max_turns=1000, # Allow thorough research + provider=AgentProvider.CLAUDE, + enable_web_search=True, # WebSearch + WebFetch + enable_file_read=True, # Read action files + enable_file_write=True, # Write files (create/overwrite) + enable_file_edit=True, # Edit existing files + ) + + return agent + + +def find_action_claude( + app_name: str, + finder_instructions: str = "", + target_count: int = 1, +) -> BatchActionFinderFilterResponse: + """ + Find new actions using Claude Code with Composio tools. + + Args: + app_name: Name of the app + finder_instructions: Optional specific instructions + target_count: Number of actions to find (default: 1) + + Returns: + BatchActionFinderFilterResponse with the found actions + """ + agent = create_claude_action_finder(app_name, finder_instructions, target_count) + + # Build query based on target count + if target_count == 1: + query = f"Find the MOST IMPORTANT new API endpoint for {app_name} that is not yet implemented." + else: + query = f"Find UP TO {target_count} of the MOST IMPORTANT new API endpoints for {app_name} that are not yet implemented, ranked by importance. Stop early if there are fewer than {target_count} valid new endpoints." + + if finder_instructions: + query += f"\n\nSpecific instructions: {finder_instructions}" + + response = agent.run(query) + + return response.final_output + + +# Simple CLI for testing +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Find actions using Claude Code") + parser.add_argument("app_name", help="Name of the app") + parser.add_argument( + "--instructions", default="", help="Specific finder instructions" + ) + parser.add_argument( + "--count", type=int, default=1, help="Number of actions to find (default: 1)" + ) + + args = parser.parse_args() + + action_word = "action" if args.count == 1 else f"{args.count} actions" + print(f"🔍 Finding {action_word} for {args.app_name}...") + result = find_action_claude(args.app_name, args.instructions, args.count) + + # Handle both structured output and string responses + if isinstance(result, BatchActionFinderFilterResponse): + print("\n✅ Found Actions:") + print(f" Success: {result.success}") + print(f" Total Found: {result.total_found}") + for i, action in enumerate(result.actions, 1): + print(f"\n Action {i}:") + print(f" Name: {action.action}") + print(f" Description: {action.description}") + print(f" Endpoint: {action.endpoint}") + print(f" Docs: {action.url}") + else: + print(f"\n⚠️ Response: {result}") diff --git a/app_tester/actionable_learnings/README.md b/app_tester/actionable_learnings/README.md --- a/app_tester/actionable_learnings/README.md +++ b/app_tester/actionable_learnings/README.md @@ -1,157 +1,154 @@ -# Actionable Learnings Pipeline +# Actionable Learnings Pipeline v2 -Automated pipeline for processing `rube_learnings` from ClickHouse and triggering actionable improvements. +Automated pipeline for processing missing tool suggestions from `rube_error_analyses` and triggering builds. ## Overview This pipeline: -1. **Fetches** new_tool and new_toolkit suggestions from ClickHouse (last 24 hours) -2. **Deduplicates** by `(toolkit, toolName)` with smart aggregation -3. **Ranks with LLM** (GPT-4-turbo) to select top 20-30 tools -4. **Triggers builds** via Builder API for approved tools -5. **Sends to Slack** formatted toolkit suggestions for human review +1. **Fetches** missing tool suggestions from ClickHouse (`rube_error_analyses` table) +2. **Filters** valid suggestions (`would_have_helped=true`, non-empty, not vetoed) +3. **Groups** by toolkit +4. **Ranks with LLM** (GPT-5.2 with web search) per-toolkit +5. **Triggers builds** via Builder API for approved tools +6. **Sends to Slack** summary of approved tools and new toolkits ## Architecture ``` -analyze_rube_sessions.py → ClickHouse (rube_learnings) - ↓ - Daily Cron @ 1 AM UTC - ↓ - Query last 24h → Deduplicate → Auto-veto - ↓ - ┌───────────────┴───────────────┐ - ↓ ↓ - new_tools (~1000) new_toolkits (~40) - ↓ ↓ - LLM ranks → top 25 LLM formats ALL - ↓ ↓ - Builder API (/build-action) Slack (C07847VBSF5) +rube_error_analyses (ClickHouse) + ↓ + missing_tools_impact[] extraction + ↓ + Filter (would_have_helped, vetoed, non-empty) + ↓ + Group by toolkit + ↓ + Split by existence (existing vs new toolkits) + ↓ + ┌─────────────────────┴─────────────────────┐ + ↓ ↓ +Existing Toolkits New Toolkits + ↓ ↓ +Per-toolkit LLM calls (parallel) Bulk LLM call + ↓ ↓ +Top 5 tools per toolkit Top 10 new toolkits + ↓ ↓ +Builder API (/build-action) Slack notification +``` + +## Data Source + +### Query +```sql +SELECT * +FROM rube_error_analyses +WHERE length(JSONExtractArrayRaw(tool_errors_json, 'missing_tools_impact')) > 0 + AND analyzed_at >= '{since_date}' +ORDER BY analyzed_at DESC +``` + +### Input Schema (`missing_tools_impact` item) +```json +{ + "suggestion_type": "new_tool", + "tool_or_toolkit_name": "GMAIL_BATCH_MODIFY_LABELS", + "toolkit": "gmail", + "use_case": "Batch modify labels on multiple emails", + "would_have_helped": true, + "impact_explanation": "Would have prevented the failure..." +} ``` ## Files -- **`learnings_processor.py`** - Fetch & deduplicate learnings -- **`llm_validator.py`** - LLM ranking and formatting +- **`models.py`** - Pydantic models for all pipeline stages +- **`learnings_processor.py`** - Fetch, filter, group logic +- **`llm_validator.py`** - Per-toolkit and bulk toolkit LLM ranking - **`action_handlers.py`** - Builder API & Slack integration - **`process_learnings.py`** - Main CLI orchestrator ## Usage -### Dry Run (Preview) -```bash -cd /Users/samvitjatia/integrator-1 -source .venv/bin/activate && source .env -python -m app_tester.actionable_learnings.process_learnings --dry-run -``` - ### Production Run ```bash -python -m app_tester.actionable_learnings.process_learnings +cd /path/to/integrator +source .venv/bin/activate && source .env +python -m app_tester.actionable_learnings.process_learnings --since 2025-12-20 ``` -### Custom Options +### Dry Run (Skip Builder, Still Send Slack) ```bash -# Custom lookback window -python -m app_tester.actionable_learnings.process_learnings --hours 48 - -# Custom target count -python -m app_tester.actionable_learnings.process_learnings --target-tools 30 - -# Combined -python -m app_tester.actionable_learnings.process_learnings --hours 12 --target-tools 20 --dry-run +python -m app_tester.actionable_learnings.process_learnings --since 2025-12-20 --dry-run ``` -## Cron Setup - -Run daily at 1 AM UTC: - +### Skip Slack Too ```bash -# Edit crontab -crontab -e - -# Add this line -0 1 * * * cd /Users/samvitjatia/integrator-1 && source .venv/bin/activate && source .env && python -m app_tester.actionable_learnings.process_learnings >> logs/actionable_learnings.log 2>&1 +python -m app_tester.actionable_learnings.process_learnings --since 2025-12-20 --dry-run --skip-slack ``` ## Configuration -### Featured Toolkits (119 toolkits) -Defined in `learnings_processor.py` - these get slight priority boost in LLM ranking. - -### Vetoed Toolkits (2 toolkits) -Auto-rejected competitors: +### Vetoed Toolkits +Auto-rejected (defined in `learnings_processor.py`): - `zapier` - `n8n` +- `remote_workbench` +- `composio` + +### LLM Model +- **Model:** `gpt-5.2` (with web search) +- **Per-toolkit calls:** Up to 5 tools approved per toolkit +- **New toolkits:** Up to 10 approved + +### LLM Rejection Criteria +Tools are rejected if they: +- Already exist with similar functionality +- Only add pagination, rate limiting, or validation +- Cannot be verified via web search +- Have no public API documentation ### Builder API - **URL:** `http://ec2-54-193-24-82.us-west-1.compute.amazonaws.com:8000/build-action` - **Branch:** `master` -- **Timeout:** 300s per tool -- **Tag:** `rube-learning` (identifies the source of the build trigger) -- **Color:** `purple` (GitHub PR label color) +- **Tag:** `rube-learning` +- **Color:** `purple` ### Slack Channel -- **Channel ID:** `C07847VBSF5` -- **Purpose:** New toolkit suggestions (informational only) - -## LLM Prompting Strategy - -### For new_tools: -- **Prioritize:** Frequency, Priority, Difficulty, Diversity, Essentiality -- **Be selective:** Only truly essential tools from top apps -- **Aim for:** 15-20 different toolkits (breadth over depth) -- **Model:** GPT-4-turbo (128K context) - -### For new_toolkits: -- **Verify:** Real apps with public APIs -- **Format:** Group by priority (High/Medium/Low) -- **Flag:** Suspicious/internal/no-API toolkits with ⚠️ -- **Model:** GPT-4-turbo - -## Typical Output +- **Channel ID:** `C09KB7HST9R` -### From last 24h: -- **Fetched:** ~1,400 raw learnings -- **Deduplicated:** ~1,060 unique (1,016 tools, 44 toolkits) -- **Auto-vetoed:** ~10 (Zapier/n8n) -- **LLM approved:** ~25-30 tools -- **Triggered:** 25-30 builds -- **Slack sent:** All 44 toolkits formatted +## Dry Run Behavior -### Success Rate: -- Build trigger: ~95% success -- Slack delivery: ~99% success +| Flag | Builder API | Slack | +|------|-------------|-------| +| (none) | ✅ Triggered | ✅ Sent | +| `--dry-run` | ❌ Skipped | ✅ Sent (with [DRY RUN] prefix) | +| `--dry-run --skip-slack` | ❌ Skipped | ❌ Skipped | -## Monitoring +## Logging -Logs are written to: -- **Cron:** `logs/actionable_learnings.log` -- **Manual:** stdout - -Check for: -- ClickHouse connection errors -- LLM timeout errors -- Builder API failures -- Slack delivery failures - -## Emergency Stops - -To pause the pipeline: -```bash -# Comment out the cron job -crontab -e -# Add # before the line - -# Or stop mid-run -pkill -f "process_learnings.py" +Pipeline logs detailed progress at each step: +``` +📥 Fetched 150 rows from rube_error_analyses +📦 Extracted 312 total suggestions from all rows +🔍 After filtering: 287 valid suggestions + 🚫 Vetoed: 15 + ⚠️ Empty toolkit/tool: 10 +🗂️ Grouped into 45 toolkit groups: + • gmail: 23 suggestions + • notion: 15 suggestions +📊 Split: 38 existing toolkits, 7 new toolkits +🤖 Processing 38 existing toolkit groups... + ✅ gmail: 3 approved, 20 rejected (12 similar exists, 8 low-value) +📋 Total approved tools: 47 + • [gmail] GMAIL_BATCH_MODIFY_LABELS + API: POST /gmail/v1/users/{userId}/messages/batchModify +🚀 Triggering 47 builds... +✅ gmail/GMAIL_BATCH_MODIFY_LABELS → Builder API (PID: 12345) ``` -## Future Enhancements - -- [ ] Add feedback loop (track which builds succeed/fail) -- [ ] Store processed learnings back to ClickHouse -- [ ] Add metrics/observability (Datadog) -- [ ] Implement rate limiting for Builder API -- [ ] Add email alerts for failures +## Error Handling +- Pipeline uses best-effort processing +- If one toolkit's LLM call fails, it continues with others +- Errors are collected and reported at the end +- Exit code 1 if any errors occurred, 0 otherwise diff --git a/app_tester/actionable_learnings/__init__.py b/app_tester/actionable_learnings/__init__.py --- a/app_tester/actionable_learnings/__init__.py +++ b/app_tester/actionable_learnings/__init__.py @@ -1,20 +1,39 @@ """ Actionable learnings pipeline. -Processes new_tool and new_toolkit suggestions from ClickHouse, -ranks them with LLM, and triggers appropriate actions. +Processes missing tool suggestions from rube_error_analyses, +ranks them with LLM (per-toolkit), and triggers builds. """ from .action_handlers import ToolBuilder, ToolkitNotifier -from .learnings_processor import FEATURED_TOOLKITS, LearningsProcessor, VETOED_TOOLKITS +from .learnings_processor import LearningsProcessor, VETOED_TOOLKITS from .llm_validator import LearningsRanker +from .models import ( + ApprovedNewToolkit, + ApprovedTool, + BuildResult, + MissingToolSuggestion, + NewToolkitRankingResult, + PipelineResult, + ReviewFile, + ToolkitGroup, + ToolkitRankingResult, +) __all__ = [ "ToolBuilder", "ToolkitNotifier", "LearningsProcessor", "LearningsRanker", - "FEATURED_TOOLKITS", "VETOED_TOOLKITS", + "ApprovedNewToolkit", + "ApprovedTool", + "BuildResult", + "MissingToolSuggestion", + "NewToolkitRankingResult", + "PipelineResult", + "ReviewFile", + "ToolkitGroup", + "ToolkitRankingResult", ] diff --git a/app_tester/actionable_learnings/action_handlers.py b/app_tester/actionable_learnings/action_handlers.py --- a/app_tester/actionable_learnings/action_handlers.py +++ b/app_tester/actionable_learnings/action_handlers.py @@ -1,118 +1,156 @@ """ Action handlers for approved learnings. -Sends approved tools to Builder API and toolkit suggestions to Slack. +Sends approved tools to Builder API and notifications to Slack. """ import asyncio import os -from typing import Any +import random +from datetime import datetime +from typing import Awaitable, Callable import httpx +from mercury.observability import logger from slack_sdk import WebClient from slack_sdk.errors import SlackApiError +from app_tester.actionable_learnings.models import ( + ApprovedNewToolkit, + ApprovedTool, + BuildResult, +) -BUILDER_API_URL = "http://ec2-54-193-24-82.us-west-1.compute.amazonaws.com:8000" -LEARNINGS_SLACK_CHANNEL = "C09KB7HST9R" + +# Type for callback that receives build result updates +BuildResultCallback = Callable[[BuildResult, int, int], Awaitable[None]] + + +BUILDER_API_URL = "https://integrations-api.composio.io/workflows/action-builder/run" +LEARNINGS_SLACK_CHANNEL = "C09V37E7B2L" class ToolBuilder: - """Sends approved tool suggestions to Builder API.""" + """Sends approved tool suggestions to Workflow Builder API.""" def __init__( self, api_url: str = BUILDER_API_URL, - timeout: float = 300.0, - tag: str = "rube-learning", - color: str = "", + timeout: float = 60.0, + labels: list[str] | None = None, ): self.api_url = api_url self.timeout = timeout - self.tag = tag - self.color = color + self.labels = labels or ["learning-pipeline"] + + def _build_instruction(self, tool: ApprovedTool) -> str: + """Build enriched instruction with all structured context.""" + parts = [tool.instruction] + + # Append structured metadata + parts.append("\n\n---\n**Tool Metadata (for reference):**") + parts.append(f"- Tool Name: {tool.tool_name}") + parts.append(f"- Description: {tool.description}") + parts.append(f"- API Endpoint: {tool.verified_api_endpoint}") + if tool.api_docs_url: + parts.append(f"- API Docs: {tool.api_docs_url}") + + return "\n".join(parts) async def build_tools( - self, approved_tools: list[dict[str, Any]] - ) -> list[dict[str, Any]]: + self, + approved_tools: list[ApprovedTool], + on_result: BuildResultCallback | None = None, + ) -> list[BuildResult]: """ - Send approved tools to Builder API for implementation. + Send approved tools to Workflow Builder API for implementation. Args: - approved_tools: List of tools approved by LLM with instructions + approved_tools: List of ApprovedTool objects + on_result: Optional callback called after each build with (result, index, total) Returns: - List of results with status for each tool + List of BuildResult objects """ if not approved_tools: return [] - results = [] + results: list[BuildResult] = [] + total = len(approved_tools) + async with httpx.AsyncClient(timeout=self.timeout) as client: - for tool in approved_tools: - toolkit = tool.get("toolkit", "unknown") - tool_name = tool.get("toolName", "unknown") + for i, tool in enumerate(approved_tools, 1): + logger.info( + f" [{i}/{total}] Triggering {tool.toolkit}/{tool.tool_name}..." + ) + + # Enrich instruction with structured context + enriched_instruction = self._build_instruction(tool) payload = { - "app_name": toolkit, - "base_branch": "master", - "instruction": tool.get("instruction", ""), - "tag": self.tag, - "color": self.color, + "app_name": tool.toolkit, + "instruction": enriched_instruction, + "labels": self.labels, } + result: BuildResult try: response = await client.post( - f"{self.api_url}/build-action", + self.api_url, json=payload, ) response.raise_for_status() result_data = response.json() - results.append( - { - "toolkit": toolkit, - "toolName": tool_name, - "status": "triggered", - "pid": result_data.get("pid"), - "response": result_data, - } - ) + workflow_id = result_data.get("workflow_id", "") - print( - f"✅ {toolkit}/{tool_name} → Builder API (PID: {result_data.get('pid')}) [tag: {self.tag}]" + result = BuildResult( + toolkit=tool.toolkit, + tool_name=tool.tool_name, + status="triggered", + workflow_id=workflow_id, ) - - # Rate limiting: wait 3 seconds between successful calls - await asyncio.sleep(3.0) + results.append(result) + logger.info(f" ✅ Workflow: {workflow_id}") except httpx.HTTPStatusError as e: - results.append( - { - "toolkit": toolkit, - "toolName": tool_name, - "status": "failed", - "error": f"HTTP {e.response.status_code}: {e.response.text[:200]}", - } + result = BuildResult( + toolkit=tool.toolkit, + tool_name=tool.tool_name, + status="failed", + error=f"HTTP {e.response.status_code}: {e.response.text[:500]}", + ) + results.append(result) + logger.error( + f"❌ {tool.toolkit}/{tool.tool_name}: HTTP {e.response.status_code} - {e.response.text[:200]}" ) - print(f"❌ {toolkit}/{tool_name}: HTTP {e.response.status_code}") except Exception as e: - results.append( - { - "toolkit": toolkit, - "toolName": tool_name, - "status": "failed", - "error": str(e)[:200], - } + result = BuildResult( + toolkit=tool.toolkit, + tool_name=tool.tool_name, + status="failed", + error=str(e)[:500], ) - print(f"❌ {toolkit}/{tool_name}: {str(e)[:100]}") + results.append(result) + logger.error(f"❌ {tool.toolkit}/{tool.tool_name}: {str(e)[:100]}") + + # Call callback if provided + if on_result: + try: + await on_result(result, i, total) + except Exception as e: + logger.warning(f"⚠️ Callback error (non-fatal): {e}") + + # Rate limiting with jitter: 2-5 seconds between calls + jitter = random.uniform(2.0, 5.0) + await asyncio.sleep(jitter) return results class ToolkitNotifier: - """Sends toolkit suggestions to Slack.""" + """Sends notifications to Slack.""" def __init__( self, @@ -123,45 +161,323 @@ def __init__( token=os.getenv("SLACK_TOKEN") or os.getenv("SLACK_BOT_TOKEN") ) self.channel_id = channel_id + self._thread_ts: str | None = None + + def start_build_thread( + self, + tools_count: int, + dry_run: bool = False, + ) -> str | None: + """ + Start a new Slack thread for build progress. + + Args: + tools_count: Number of tools to build + dry_run: Whether this is a dry run + + Returns: + Thread timestamp (ts) or None if failed + """ + prefix = "🧪 *[DRY RUN]* " if dry_run else "" + msg = f"{prefix}🚀 *Triggering builder for {tools_count} tools...*" - async def send_toolkit_suggestions(self, formatted_message: str) -> bool: + try: + response = self.slack_client.chat_postMessage( + channel=self.channel_id, + text=msg, + mrkdwn=True, + ) + if response.get("ok"): + self._thread_ts = response.get("ts") + logger.info(f"✅ Started Slack build thread (ts: {self._thread_ts})") + return self._thread_ts + else: + logger.error(f"❌ Failed to start Slack thread: {response}") + return None + except Exception as e: + logger.error(f"❌ Slack error starting thread: {e}") + return None + + def post_build_result( + self, + result: BuildResult, + index: int, + total: int, + thread_ts: str | None = None, + ) -> bool: """ - Send formatted toolkit suggestions to Slack channel. + Post a single build result to the thread. Args: - formatted_message: Slack-formatted message from LLM + result: BuildResult object + index: Current index (1-based) + total: Total number of builds + thread_ts: Thread timestamp (uses stored one if not provided) Returns: - True if successful, False otherwise + True if successful """ - if not self.slack_client: - print("⚠️ Slack client not configured - skipping notification") + ts = thread_ts or self._thread_ts + if not ts: + logger.warning("⚠️ No thread_ts available, skipping Slack update") return False - if not formatted_message or formatted_message.strip() == "": - print("⚠️ Empty message - skipping Slack notification") + if result.status == "triggered": + emoji = "✅" + status_text = ( + f"Triggered `{result.workflow_id}`" + if result.workflow_id + else "Triggered" + ) + elif result.status == "skipped_dry_run": + emoji = "⏭️" + status_text = "Skipped (dry run)" + else: + emoji = "❌" + error_preview = (result.error or "Unknown error")[:500] + status_text = f"Failed: {error_preview}" + + msg = f"{emoji} [{index}/{total}] `{result.toolkit}/{result.tool_name}` — {status_text}" + + try: + response = self.slack_client.chat_postMessage( + channel=self.channel_id, + text=msg, + thread_ts=ts, + mrkdwn=True, + ) + return response.get("ok", False) + except Exception as e: + logger.warning(f"⚠️ Failed to post build result to Slack: {e}") return False + def post_build_complete( + self, + results: list[BuildResult], + thread_ts: str | None = None, + ) -> bool: + """ + Post build completion summary to the thread. + + Args: + results: List of all BuildResult objects + thread_ts: Thread timestamp (uses stored one if not provided) + + Returns: + True if successful + """ + ts = thread_ts or self._thread_ts + if not ts: + return False + + triggered = sum(1 for r in results if r.status == "triggered") + failed = sum(1 for r in results if r.status == "failed") + skipped = sum(1 for r in results if r.status == "skipped_dry_run") + + parts = [] + if triggered: + parts.append(f"✅ {triggered} triggered") + if failed: + parts.append(f"❌ {failed} failed") + if skipped: + parts.append(f"⏭️ {skipped} skipped") + + msg = f"*Build complete:* {', '.join(parts)}" + try: response = self.slack_client.chat_postMessage( channel=self.channel_id, - text=formatted_message, + text=msg, + thread_ts=ts, mrkdwn=True, ) + return response.get("ok", False) + except Exception as e: + logger.warning(f"⚠️ Failed to post build complete to Slack: {e}") + return False - if response.get("ok"): - print( - f"✅ Sent toolkit suggestions to Slack (channel: {self.channel_id})" + def create_result_callback( + self, thread_ts: str | None = None + ) -> BuildResultCallback: + """ + Create a callback function for streaming build results to Slack. + + Args: + thread_ts: Thread timestamp to use + + Returns: + Async callback function compatible with ToolBuilder.build_tools() + """ + ts = thread_ts or self._thread_ts + + async def callback(result: BuildResult, index: int, total: int) -> None: + self.post_build_result(result, index, total, ts) + + return callback + + def _format_summary_message( + self, + approved_tools: list[ApprovedTool], + approved_toolkits: list[ApprovedNewToolkit], + suggestions_count: int, + filtered_count: int, + existing_groups: int, + new_groups: int, + dry_run: bool, + ) -> str: + """Format top-level summary message with metrics.""" + today = datetime.utcnow().strftime("%B %d, %Y") + prefix = "🧪 *[DRY RUN]* " if dry_run else "" + + lines = [f"{prefix}📋 *New Tool Learnings Report* • {today}", ""] + + # Metrics + lines.append("*Pipeline Metrics:*") + lines.append(f"• Suggestions extracted: {suggestions_count}") + lines.append(f"• After filtering: {filtered_count}") + lines.append(f"• Toolkit groups: {existing_groups} existing, {new_groups} new") + lines.append("") + + # Results + action_word = "Would build" if dry_run else "Building" + if approved_tools or approved_toolkits: + lines.append("*Results:*") + if approved_tools: + lines.append(f"• 📦 {action_word} *{len(approved_tools)}* new tools") + if approved_toolkits: + lines.append(f"• 🆕 Suggesting *{len(approved_toolkits)}* new toolkits") + lines.append("") + lines.append("_See thread for full details ↓_") + else: + lines.append("_No tools or toolkits approved in this run._") + + return "\n".join(lines) + + def _format_details_thread( + self, + approved_tools: list[ApprovedTool], + approved_toolkits: list[ApprovedNewToolkit], + build_results: list[BuildResult], + dry_run: bool, + ) -> str: + """Format detailed thread reply - clean and compact, with workflow IDs.""" + lines = [] + action_word = "Would build" if dry_run else "Building" + + # Create lookup for workflow IDs + workflow_lookup: dict[str, str] = {} + for r in build_results: + if r.status == "triggered" and r.workflow_id: + key = f"{r.toolkit}/{r.tool_name}" + workflow_lookup[key] = r.workflow_id + + # Tools section - grouped by toolkit, compact format + if approved_tools: + tools_by_toolkit: dict[str, list[ApprovedTool]] = {} + for tool in approved_tools: + if tool.toolkit not in tools_by_toolkit: + tools_by_toolkit[tool.toolkit] = [] + tools_by_toolkit[tool.toolkit].append(tool) + + lines.append(f"*📦 {action_word} {len(approved_tools)} new tools:*") + for toolkit, tools in sorted(tools_by_toolkit.items()): + lines.append(f"\n`{toolkit}` ({len(tools)}):") + for t in tools: + # Full description + tool_line = f"• *{t.tool_name}* — {t.description}" + # Add workflow ID if available + key = f"{t.toolkit}/{t.tool_name}" + if key in workflow_lookup: + tool_line += f" `[{workflow_lookup[key]}]`" + lines.append(tool_line) + + # New toolkits section + if approved_toolkits: + lines.append(f"\n*🆕 New toolkit suggestions ({len(approved_toolkits)}):*") + for tk in approved_toolkits: + lines.append( + f"• *{tk.toolkit_name}* ({tk.frequency}×) — <{tk.api_docs_url}|docs>" ) - return True - print(f"❌ Slack response not OK: {response}") - return False + return "\n".join(lines) + + async def send_summary( + self, + approved_tools: list[ApprovedTool], + approved_toolkits: list[ApprovedNewToolkit], + build_results: list[BuildResult], + suggestions_count: int, + filtered_count: int, + existing_groups: int, + new_groups: int, + dry_run: bool, + ) -> bool: + """ + Send summary to Slack: main message with metrics, details in thread. + + Args: + approved_tools: List of approved tools + approved_toolkits: List of approved new toolkits + build_results: Results from Builder API calls + suggestions_count: Total suggestions extracted + filtered_count: Suggestions after filtering + existing_groups: Number of existing toolkit groups + new_groups: Number of new toolkit groups + dry_run: Whether this is a dry run + + Returns: + True if successful + """ + summary_msg = self._format_summary_message( + approved_tools, + approved_toolkits, + suggestions_count, + filtered_count, + existing_groups, + new_groups, + dry_run, + ) + + try: + # Send main summary message + response = self.slack_client.chat_postMessage( + channel=self.channel_id, + text=summary_msg, + mrkdwn=True, + ) + + if not response.get("ok"): + logger.error(f"❌ Slack response not OK: {response}") + return False + + logger.info(f"✅ Sent summary to Slack (channel: {self.channel_id})") + thread_ts = response.get("ts") + + # Send details in thread if there are approved items + if approved_tools or approved_toolkits: + details_msg = self._format_details_thread( + approved_tools, approved_toolkits, build_results, dry_run + ) + + thread_response = self.slack_client.chat_postMessage( + channel=self.channel_id, + text=details_msg, + thread_ts=thread_ts, + mrkdwn=True, + ) + + if thread_response.get("ok"): + logger.info("✅ Sent details thread") + else: + logger.warning( + f"⚠️ Failed to send details thread: {thread_response}" + ) + + return True except SlackApiError as e: - print(f"❌ Slack API error: {e.response['error']}") + logger.error(f"❌ Slack API error: {e.response['error']}") return False - except Exception as e: - print(f"❌ Failed to send to Slack: {str(e)[:200]}") + logger.error(f"❌ Slack error (network/other): {e}") return False diff --git a/app_tester/actionable_learnings/learnings_processor.py b/app_tester/actionable_learnings/learnings_processor.py --- a/app_tester/actionable_learnings/learnings_processor.py +++ b/app_tester/actionable_learnings/learnings_processor.py @@ -1,427 +1,222 @@ """ -Learnings processor for deduplicating and aggregating tool suggestions. +Learnings processor for extracting and grouping missing tool suggestions. -Fetches learnings from ClickHouse and groups by (toolkit, toolName) to reduce noise. +Fetches from rube_error_analyses table and groups by toolkit. """ -from collections import Counter, defaultdict -from datetime import datetime, timedelta -from statistics import mean -from typing import Any, Dict +import json +import os +from collections import defaultdict +from mercury.observability import logger + +from app_tester.actionable_learnings.models import MissingToolSuggestion, ToolkitGroup from app_tester.agent_worker_fix_from_logs.services.clickhouse_service import ( ClickHouseService, ) +from app_tester.rube_learning.toolkit_checker import get_all_toolkits +from app_tester.rube_learning.utils import get_cached_toolkit_tools -FEATURED_TOOLKITS = { - "gmail", - "github", - "googlecalendar", - "notion", - "googlesheets", - "slack", - "supabase", - "outlook", - "twitter", - "linkedin", - "youtube", - "googledrive", - "googledocs", - "hubspot", - "linear", - "gemini", - "instagram", - "tiktok", - "figma", - "clickup", - "todoist", - "jira", - "zendesk", - "stripe", - "shopify", - "asana", - "trello", - "airtable", - "salesforce", - "intercom", - "discord", - "zoom", - "calendly", - "typeform", - "mailchimp", - "sendgrid", - "twilio", - "dropbox", - "box", - "onedrive", - "evernote", - "onenote", - "confluence", - "bitbucket", - "gitlab", - "circleci", - "jenkins", - "datadog", - "sentry", - "amplitude", - "mixpanel", - "segment", - "heap", - "fullstory", - "hotjar", - "zendesk", - "freshdesk", - "helpscout", - "drift", - "intercom", - "crisp", - "monday", - "wrike", - "basecamp", - "smartsheet", - "notion", - "coda", - "miro", - "lucidchart", - "whimsical", - "figma", - "sketch", - "invision", - "adobe", - "canva", - "piktochart", - "visme", - "crello", - "snappa", - "buffer", - "hootsuite", - "sproutsocial", - "later", - "planoly", - "tailwind", - "shopify", - "woocommerce", - "bigcommerce", - "magento", - "squarespace", - "wix", - "wordpress", - "webflow", - "bubble", - "make", - "aws", - "gcp", - "azure", - "heroku", - "vercel", - "netlify", - "postgres", - "mysql", - "mongodb", - "redis", - "elasticsearch", - "snowflake", - "tableau", - "looker", - "powerbi", - "metabase", - "redash", - "superset", - "quickbooks", - "xero", - "freshbooks", - "wave", - "zoho", - "sage", - "hubspot", - "pardot", - "marketo", - "activecampaign", - "convertkit", - "drip", -} - -# Toolkits to automatically reject (competitors/conflicts/internal) VETOED_TOOLKITS = { "zapier", "n8n", "remote_workbench", + "composio", + "googlesuper", } +TOOLKIT_CACHE_TTL_SECONDS = 6 * 60 * 60 # 6 hours + + class LearningsProcessor: - """Processes and deduplicates learnings from ClickHouse.""" + """Processes missing tool suggestions from rube_error_analyses.""" def __init__(self): self.clickhouse = ClickHouseService() - - def fetch_recent_learnings(self, hours_back: int = 24) -> list[dict[str, Any]]: + self._existing_toolkits: set[str] | None = None + self._existing_toolkits_timestamp: float = 0 + self._api_key = os.getenv("COMPOSIO_API_KEY", "") + + async def get_existing_toolkits(self) -> set[str]: + """Get all existing Composio toolkit slugs (cached for 6 hours).""" + import time + + now = time.time() + cache_valid = ( + self._existing_toolkits is not None + and (now - self._existing_toolkits_timestamp) < TOOLKIT_CACHE_TTL_SECONDS + ) + + if cache_valid: + return self._existing_toolkits # type: ignore + + toolkit_list = await get_all_toolkits(self._api_key) + self._existing_toolkits = {t.lower() for t in toolkit_list} + self._existing_toolkits_timestamp = now + logger.info( + f"📚 Loaded {len(self._existing_toolkits)} existing Composio toolkits" + ) + return self._existing_toolkits + + def get_toolkit_tools(self, toolkit: str) -> list[dict]: + """Get all tools for a toolkit from Composio API.""" + return get_cached_toolkit_tools(toolkit, self._api_key) + + def fetch_missing_tool_suggestions( + self, since_date: str + ) -> tuple[list[MissingToolSuggestion], int]: """ - Fetch new_tool and new_toolkit learnings from last N hours. + Fetch and extract all missing tool suggestions from rube_error_analyses. Args: - hours_back: Number of hours to look back + since_date: ISO date string (e.g., "2025-12-20") Returns: - List of learning records from ClickHouse + Tuple of (list of MissingToolSuggestion objects, row count) """ - since_timestamp = datetime.utcnow() - timedelta(hours=hours_back) - since_str = since_timestamp.strftime("%Y-%m-%d %H:%M:%S") - query = f""" SELECT - analysisId, - sessionId, - userId, - recommendationType, - recommendationName, - priority, - rationale, - orderIndex, - toolName, - description, - expectedUsage, - toolkit, - requiredApi, - implementationDifficulty, - apiDocs, - createdAt - FROM app_db.rube_learnings - WHERE recommendationType IN ('new_tool', 'new_toolkit') - AND createdAt >= '{since_str}' - ORDER BY createdAt DESC + tool_errors_json + FROM app_db.rube_error_analyses + WHERE length(JSONExtractArrayRaw(tool_errors_json, 'missing_tools_impact')) > 0 + AND analyzed_at >= '{since_date}' + ORDER BY analyzed_at DESC """ result = self.clickhouse._execute_query(query) - return result.get("data", []) + rows = result.get("data", []) + logger.info(f"📥 Fetched {len(rows)} rows from rube_error_analyses") - def deduplicate_learnings( - self, learnings: list[dict[str, Any]] - ) -> list[dict[str, Any]]: - """ - Deduplicate learnings by grouping on (toolkit, toolName). + suggestions: list[MissingToolSuggestion] = [] - For each unique pair: - - Average the priority scores - - Take most common difficulty level - - Select best rationale/description from highest priority entry - - Track frequency (demand signal) + for row in rows: + tool_errors_json_str = row.get("tool_errors_json", "{}") - Args: - learnings: Raw learning records + if isinstance(tool_errors_json_str, str): + tool_errors = json.loads(tool_errors_json_str) + else: + tool_errors = tool_errors_json_str - Returns: - Deduplicated learning records with aggregated stats + missing_tools_impact = tool_errors.get("missing_tools_impact", []) + + for item in missing_tools_impact: + # Skip items with missing required fields + toolkit = item.get("toolkit") or "" + tool_name = item.get("tool_or_toolkit_name") or "" + if not toolkit or not tool_name: + continue + + suggestion = MissingToolSuggestion( + suggestion_type=item.get("suggestion_type") or "new_tool", + tool_or_toolkit_name=tool_name, + toolkit=toolkit, + use_case=item.get("use_case") or "", + would_have_helped=item.get("would_have_helped", False), + impact_explanation=item.get("impact_explanation") or "", + ) + suggestions.append(suggestion) + + logger.info( + f"📦 Extracted {len(suggestions)} total suggestions from {len(rows)} rows" + ) + return suggestions, len(rows) + + def filter_valid_suggestions( + self, suggestions: list[MissingToolSuggestion] + ) -> list[MissingToolSuggestion]: """ - groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + Filter suggestions to only valid ones. + + Filters: + - would_have_helped must be true + - toolkit must be non-empty + - tool_or_toolkit_name must be non-empty + - toolkit must not be in VETOED_TOOLKITS + """ + valid: list[MissingToolSuggestion] = [] vetoed_count = 0 + empty_count = 0 + not_helpful_count = 0 + + for s in suggestions: + if not s.would_have_helped: + not_helpful_count += 1 + continue - for learning in learnings: - toolkit = (learning.get("toolkit") or "").lower().strip() - tool_name = (learning.get("toolName") or "").strip() + toolkit_lower = s.toolkit.lower().strip() - if not toolkit: + if not toolkit_lower or not s.tool_or_toolkit_name.strip(): + empty_count += 1 continue - # Auto-veto competitor toolkits - if toolkit in VETOED_TOOLKITS: + if toolkit_lower in VETOED_TOOLKITS: vetoed_count += 1 continue - groups[(toolkit, tool_name)].append(learning) + valid.append(s) + logger.info(f"🔍 After filtering: {len(valid)} valid suggestions") if vetoed_count > 0: - print( - f" 🚫 Auto-vetoed {vetoed_count} suggestions from blocked toolkits: {', '.join(VETOED_TOOLKITS)}" - ) - - unique_learnings = [] - for (toolkit, tool_name), group in groups.items(): - # Aggregate difficulty - use mode (most common) - difficulties = [ - item.get("implementationDifficulty") - for item in group - if item.get("implementationDifficulty") - ] - mode_difficulty = ( - Counter(difficulties).most_common(1)[0][0] if difficulties else "MEDIUM" - ) - - # Average priority - priorities: list[float] = [] - for item in group: - priority_val = item.get("priority") - if priority_val is not None: - priorities.append(float(priority_val)) - avg_priority: float = mean(priorities) if priorities else 5.0 - - # Pick best entry for text fields (highest priority) - best_entry = max(group, key=lambda x: x.get("priority", 0)) - - unique_learnings.append( - { - "recommendationType": best_entry.get("recommendationType"), - "toolkit": toolkit, - "toolName": tool_name, - "description": best_entry.get("description", ""), - "expectedUsage": best_entry.get("expectedUsage", ""), - "requiredApi": best_entry.get("requiredApi", ""), - "apiDocs": best_entry.get("apiDocs", ""), - "rationale": best_entry.get("rationale", ""), - "implementationDifficulty": mode_difficulty, - "avg_priority": round(avg_priority, 2), - "frequency": len(group), - "is_featured_toolkit": toolkit in FEATURED_TOOLKITS, - } - ) - - return unique_learnings - - def split_by_type( - self, learnings: list[dict[str, Any]] - ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: - """ - Split learnings into new_tool and new_toolkit lists. - - Args: - learnings: Deduplicated learning records - - Returns: - Tuple of (new_tool_learnings, new_toolkit_learnings) - """ - new_tools = [ - item for item in learnings if item.get("recommendationType") == "new_tool" - ] - new_toolkits = [ - item - for item in learnings - if item.get("recommendationType") == "new_toolkit" + logger.info(f" 🚫 Vetoed: {vetoed_count}") + if empty_count > 0: + logger.info(f" ⚠️ Empty toolkit/tool: {empty_count}") + if not_helpful_count > 0: + logger.info(f" ❌ would_have_helped=false: {not_helpful_count}") + + return valid + + def group_by_toolkit( + self, suggestions: list[MissingToolSuggestion] + ) -> list[ToolkitGroup]: + """Group suggestions by toolkit.""" + groups_dict: dict[str, list[MissingToolSuggestion]] = defaultdict(list) + + for s in suggestions: + toolkit_lower = s.toolkit.lower().strip() + groups_dict[toolkit_lower].append(s) + + groups = [ + ToolkitGroup(toolkit=toolkit, suggestions=sugs) + for toolkit, sugs in groups_dict.items() ] - return new_tools, new_toolkits + # Sort by frequency descending + groups.sort(key=lambda g: g.frequency, reverse=True) - def pass1_hard_filter(self, tools: list[dict[str, Any]]) -> list[dict[str, Any]]: - """ - Pass 1: Hard rules filter to remove obvious low-value suggestions. + logger.info(f"🗂️ Grouped into {len(groups)} toolkit groups:") + for g in groups[:20]: # Show top 20 + logger.info(f" • {g.toolkit}: {g.frequency} suggestions") + if len(groups) > 20: + logger.info(f" ... and {len(groups) - 20} more") - Rules: - - Auto-include if frequency >= 2 - - For frequency = 1: Must have priority >= 7, not HARD, and be featured toolkit + return groups - Args: - tools: Deduplicated tool suggestions + async def split_by_existence( + self, groups: list[ToolkitGroup] + ) -> tuple[list[ToolkitGroup], list[ToolkitGroup]]: + """ + Split groups into existing toolkit groups and new toolkit groups. Returns: - Filtered tools (~600-800 expected) + Tuple of (existing_toolkit_groups, new_toolkit_groups) """ - filtered = [] - - for tool in tools: - # Auto-include if frequency >= 2 (demand signal) - if tool.get("frequency", 0) >= 2: - filtered.append(tool) - continue - - # For frequency = 1, apply strict filters - if tool.get("frequency", 0) == 1: - if tool.get("implementationDifficulty") == "HARD": - continue + existing_toolkits = await self.get_existing_toolkits() - if tool.get("avg_priority", 0) < 7: - continue - - if not tool.get("is_featured_toolkit", False): - continue - - filtered.append(tool) - - return filtered + existing_groups: list[ToolkitGroup] = [] + new_groups: list[ToolkitGroup] = [] - def pass2_diversity_filter( - self, tools: list[dict[str, Any]], target: int = 250 - ) -> list[dict[str, Any]]: - """ - Pass 2: Diversity-aware reduction with frequency-based prioritization. - - Groups tools by frequency tier and enforces per-toolkit caps: - - Freq >= 6: Max 6 tools per toolkit - - Freq 3-5: Max 5 tools per toolkit (cumulative) - - Freq = 2: Max 3 tools per toolkit (cumulative) - - Freq = 1: Max 2 tools per toolkit (cumulative) + for group in groups: + if group.toolkit in existing_toolkits: + group.is_existing_toolkit = True + existing_groups.append(group) + else: + group.is_existing_toolkit = False + new_groups.append(group) - Args: - tools: Filtered tools from pass1 - target: Target number of tools to return + logger.info( + f"📊 Split: {len(existing_groups)} existing toolkits, {len(new_groups)} new toolkits" + ) - Returns: - Diversity-filtered tools (~250 expected) - """ - import math - from collections import defaultdict - - # Group by frequency tiers - very_high = [t for t in tools if t.get("frequency", 0) >= 6] - high = [t for t in tools if 3 <= t.get("frequency", 0) < 6] - medium = [t for t in tools if t.get("frequency", 0) == 2] - low = [t for t in tools if t.get("frequency", 0) == 1] - - # Scoring function - difficulty_weights = {"EASY": 1.2, "MEDIUM": 1.0, "HARD": 0.8} - - def calculate_score(tool): - frequency = tool.get("frequency", 1) - priority = tool.get("avg_priority", 5.0) - difficulty = tool.get("implementationDifficulty", "MEDIUM") - is_featured = tool.get("is_featured_toolkit", False) - - difficulty_weight = difficulty_weights.get(difficulty, 1.0) - featured_boost = 1.15 if is_featured else 1.0 - - return ( - frequency - * priority - * difficulty_weight - * featured_boost - * math.log(1 + frequency) - ) - - # Build final list with diversity enforcement - final = [] - toolkit_counts: Dict[str, int] = defaultdict(int) - - # Tier 1: Very high frequency (freq >= 6) - Max 6 per toolkit - for tool in sorted(very_high, key=calculate_score, reverse=True): - if toolkit_counts[tool["toolkit"]] < 6: - final.append(tool) - toolkit_counts[tool["toolkit"]] += 1 - - # Tier 2: High frequency (freq 3-5) - Max 5 cumulative per toolkit - for tool in sorted(high, key=calculate_score, reverse=True): - if toolkit_counts[tool["toolkit"]] < 5: - final.append(tool) - toolkit_counts[tool["toolkit"]] += 1 - - if len(final) >= target: - break - - # Tier 3: Medium frequency (freq = 2) - Max 3 cumulative per toolkit - for tool in sorted(medium, key=calculate_score, reverse=True): - if toolkit_counts[tool["toolkit"]] < 3: - final.append(tool) - toolkit_counts[tool["toolkit"]] += 1 - - if len(final) >= target: - break - - # Tier 4: Low frequency (freq = 1) - Max 2 cumulative per toolkit, fill to target - if len(final) < target: - for tool in sorted(low, key=calculate_score, reverse=True): - if toolkit_counts[tool["toolkit"]] < 2: - final.append(tool) - toolkit_counts[tool["toolkit"]] += 1 - - if len(final) >= target: - break - - return final[:target] + return existing_groups, new_groups diff --git a/app_tester/actionable_learnings/llm_validator.py b/app_tester/actionable_learnings/llm_validator.py --- a/app_tester/actionable_learnings/llm_validator.py +++ b/app_tester/actionable_learnings/llm_validator.py @@ -1,340 +1,297 @@ """ -LLM-based validation and ranking for tool and toolkit suggestions. +LLM-based ranking for tool and toolkit suggestions. -Uses GPT-4o to intelligently rank tools and format toolkit suggestions. +Uses OpenAI Agents SDK with WebSearchTool for API verification. """ -import json -import os -from datetime import datetime -from typing import Any +from agents import Agent, ModelSettings, Runner, WebSearchTool +from mercury.observability import logger +from openai.types.shared import Reasoning -from openai import AsyncOpenAI -from pydantic import BaseModel +from app_tester.actionable_learnings.models import ( + MissingToolSuggestion, + NewToolkitRankingResult, + ToolkitGroup, + ToolkitRankingResult, +) -from .learnings_processor import FEATURED_TOOLKITS, VETOED_TOOLKITS +# ============================================================================= +# LearningsRanker using Agents SDK +# ============================================================================= -class ToolkitInfo(BaseModel): - """Individual toolkit information.""" - - name: str - frequency: int - priority: float - use_case: str - is_valid: bool - concern: str | None = None +class LearningsRanker: + """Ranks tool and toolkit suggestions using OpenAI Agents SDK with web search.""" -class StructuredToolkitData(BaseModel): - """Structured toolkit categorization.""" + def __init__(self, model: str = "gpt-5.2"): + self.model = model - high_priority: list[ToolkitInfo] - medium_priority: list[ToolkitInfo] - needs_review: list[ToolkitInfo] + def _format_existing_tools(self, tools: list[dict]) -> str: + """ + Format existing tools for LLM context. + For large toolkits (>200 tools), only show important ones. + Excludes tools with mcpIgnore tag. + """ + lines: list[str] = [] + is_large = len(tools) > 200 -class ApprovedTool(BaseModel): - """Individual approved tool with reasoning.""" + if is_large: + lines.append( + f"(Showing only IMPORTANT tools - toolkit has {len(tools)} total)" + ) + lines.append("") - toolkit: str - toolName: str - instruction: str - reasoning: str + shown_count = 0 + for tool in tools: + slug = tool.get("slug", "") + description = tool.get("description", "") + tags = tool.get("tags", []) + # Skip tools marked as mcpIgnore + if "mcpIgnore" in tags: + continue -class ToolRankingResult(BaseModel): - """LLM tool ranking result.""" + is_important = "important" in tags - approved: list[ApprovedTool] - rejected_count: int - rejection_summary: str + if is_large and not is_important: + continue + desc_line = description.replace("\n", " ").strip()[:200] + important_marker = " ⭐" if is_important else "" + lines.append(f"- {slug}{important_marker}: {desc_line}") + shown_count += 1 -class LearningsRanker: - """Ranks and validates tool/toolkit suggestions using LLM.""" + if shown_count == 0: + lines.append("(No tools found for this toolkit)") - def __init__(self, model: str = "gpt-5-nano"): - self.model = model - self.client = AsyncOpenAI(api_key=os.getenv("OPENAI_API_KEY")) - self._existing_toolkits: list[str] | None = None + return "\n".join(lines) - def _get_existing_toolkits(self) -> list[str]: - """Fetch existing Composio toolkits from API.""" - if self._existing_toolkits is not None: - return self._existing_toolkits + def _format_suggestions( + self, suggestions: list[MissingToolSuggestion] + ) -> tuple[str, dict[str, int]]: + """ + Format suggestions for LLM prompt. - try: - import httpx + Returns: + Tuple of (formatted string, dict of tool_name -> count) + """ + # Count occurrences of each tool suggestion + tool_counts: dict[str, int] = {} + for s in suggestions: + name = s.tool_or_toolkit_name.upper() + tool_counts[name] = tool_counts.get(name, 0) + 1 + + # Deduplicate suggestions for display + seen: set[str] = set() + lines: list[str] = [] + idx = 0 + for s in suggestions: + name = s.tool_or_toolkit_name.upper() + if name in seen: + continue + seen.add(name) + idx += 1 + count = tool_counts[name] + count_str = f" (×{count})" if count > 1 else "" + lines.append(f"{idx}. Tool Name: {s.tool_or_toolkit_name}{count_str}") + lines.append(f" Use Case: {s.use_case}") + lines.append(f" Impact: {s.impact_explanation}") + lines.append("") + return "\n".join(lines), tool_counts - response = httpx.get( - "https://backend.composio.dev/api/v3/toolkits", - headers={"x-api-key": os.getenv("COMPOSIO_API_KEY", "")}, - timeout=10.0, - ) - data = response.json() - slugs = [item["slug"] for item in data.get("items", [])] - self._existing_toolkits = slugs - return slugs - except Exception: - # Fallback to FEATURED_TOOLKITS if API fails - return list(FEATURED_TOOLKITS) - - async def rank_tools( + async def rank_tools_for_toolkit( self, - tool_suggestions: list[dict[str, Any]], - target_count: int = 25, - ) -> list[dict[str, Any]]: + toolkit: str, + suggestions: list[MissingToolSuggestion], + existing_tools: list[dict], + ) -> ToolkitRankingResult: """ - Rank all tool suggestions and return top N most valuable. - - Args: - tool_suggestions: Deduplicated tool suggestions - target_count: Target number of tools to return (20-30 range) + Rank tool suggestions for a single toolkit. - Returns: - List of approved tools with build instructions + Uses Agents SDK with WebSearchTool to verify API endpoints. """ - if not tool_suggestions: - return [] - - featured_preview = ", ".join(sorted(FEATURED_TOOLKITS)[:40]) - today = datetime.utcnow().strftime("%B %d, %Y") - - prompt = f"""You have {len(tool_suggestions)} unique tool suggestions from the last 24 hours of production sessions. - -Your task: Rank them and return the TOP {target_count} most valuable tools to build TODAY ({today}). - -TOOL SUGGESTIONS DATA: -{json.dumps(tool_suggestions, indent=2)} - -PRIORITIZATION FACTORS (in order of importance): -1. **Frequency** (demand signal): Tools suggested many times indicate high user demand - - 30+ suggestions = extremely high demand - - 10-29 suggestions = high demand - - 5-9 suggestions = moderate demand - - 1-4 suggestions = low demand - -2. **Priority Score** (avg_priority): Higher scores indicate more critical user needs - - 8-10 = critical - - 6-7 = important - - 4-5 = nice to have - -3. **Implementation Difficulty**: Prefer easier wins - - EASY > MEDIUM > HARD - -4. **Toolkit Diversity & Essentiality**: Spread across different apps, only pick the best - - For top apps (Gmail, Notion, Supabase, Google Drive, etc.): Be VERY selective - - Only pick if tool is TRULY ESSENTIAL + EASY to implement - - Ask: "Is this a must-have core feature that makes this app 10x more useful?" - - Favor breadth - aim for 15-20 different toolkits in your selection - - If you see 5 Gmail tools, pick only the 1-2 most critical ones - - Avoid more than 2-3 tools from the same toolkit unless they're all genuinely essential - -5. **Featured Toolkits**: Give slight preference to these popular toolkits: - {featured_preview}... (and {len(FEATURED_TOOLKITS)} total featured toolkits) - -6. **API Validity**: Must have a real, documented API endpoint - - Reject if API endpoint is clearly fake, vague, or impossible - -7. **Clear Value Proposition**: Must solve a real, specific problem - - Reject if description is too vague or overly niche - -SELECTION STRATEGY: -- Easy + Essential > Hard + Nice-to-have -- Diverse toolkits > Concentrated in top apps -- Core functionality > Edge cases -- Example: For Gmail, pick "batch modify messages" (essential) over "get message 17th header" (edge case) - -REJECTION CRITERIA: -- Authentication/Authorization tools: NEVER pick tools related to auth, login, token exchange, OAuth, permissions verification, etc. - Examples to reject: "auth_test", "verify_token", "oauth_exchange", "check_permissions", "get_my_permissions" -- API endpoint doesn't exist or is clearly invalid -- Duplicate functionality of existing Composio tools -- Too niche (< 1% of users would need this) -- Impossible to implement (no API available) -- Vague or unclear description/rationale -- Competitor/internal toolkits (already filtered): {', '.join(sorted(VETOED_TOOLKITS))} - -OUTPUT FORMAT (strict JSON): -{{ - "approved": [ - {{ - "toolkit": "gmail", - "toolName": "GMAIL_BATCH_MODIFY_MESSAGES", - "instruction": "Build a new action: GMAIL_BATCH_MODIFY_MESSAGES\\n\\n\ -Description: Modify multiple Gmail messages at once\\n\\n\ -API Endpoint: POST /gmail/v1/users/{{userId}}/messages/batchModify\\n\\n\ -API Documentation: https://developers.google.com/gmail/api/reference/rest/v1/users.messages/batchModify\\n\\n\ -Expected Usage: Agents need to bulk-process emails\\n\\n\ -Rationale: Batch operations are essential for email management at scale.\\n\\n\ -Demand Signal: Suggested 32 times\\nPriority: 8.5/10\\nDifficulty: MEDIUM", - "reasoning": "HIGH DEMAND: 32× requests in 48h indicates critical user need. CORE LIMITATION: Current tools require individual message loops, causing timeouts and hitting rate limits. IMPLEMENTATION: EASY - Gmail API has native batchModify endpoint with clear docs. IMPACT: Enables email management at scale, reduces API calls 100:1." - }}, - ... (return exactly {target_count} tools, ranked best-to-worst) - ], - "rejected_count": 150, - "rejection_summary": "Rejected tools: 45 invalid APIs, 30 duplicates, 25 too niche, 50 vague descriptions" -}} - -IMPORTANT: -- Return exactly {target_count} tools (or fewer if not enough quality suggestions) -- Each tool's "instruction" field must be comprehensive - it goes DIRECTLY to the Builder API -- Include ALL context: description, API endpoint, docs, usage, rationale, demand signal, priority, difficulty -- Add "reasoning" field explaining demand, limitations, implementation difficulty, and impact -- Rank by overall value (balance all factors above) -""" - - response = await self.client.beta.chat.completions.parse( + logger.info(f"Evaluating {len(suggestions)} suggestions for toolkit: {toolkit}") + existing_tools_str = self._format_existing_tools(existing_tools) + suggestions_str, tool_counts = self._format_suggestions(suggestions) + + # Build set of existing tool slugs for is_new_api detection + existing_tool_slugs = { + tool.get("slug", "").upper() for tool in existing_tools if tool.get("slug") + } + + instructions = f"""You are a CONSERVATIVE API tool evaluator. Only approve tools that add SIGNIFICANT value. + +TOOLKIT: {toolkit} + +EXISTING TOOLS IN THIS TOOLKIT ({len(existing_tools)} tools): +{existing_tools_str} + +You are given some suggestions of NEW tools to be added. Evaluate which ones are HIGH-VALUE and should be added as new tools. + +EVALUATION RULES: + +1. CHECK EXISTING TOOLS FIRST + - If an existing tool provides SIMILAR functionality, only approve the new tool if it adds SUBSTANTIALLY more value + - Example: If "GMAIL_GET_MESSAGE" exists, don't approve "GMAIL_FETCH_EMAIL" unless it does something meaningfully different + +2. IGNORE LOW-VALUE SUGGESTIONS - REJECT any tool that merely adds: + - Pagination support (e.g., "list all with pagination") + - Rate limiting or retry handling + - Input validation or schema checking + - Error handling improvements + - Pure validation or summarization tools + - Webhook setup/management (webhooks are handled separately) + - Authentication or OAuth flows (auth is handled at connection level) + - Never design tools that take or return raw bytes. Bytes must be handled by clients via presigned URLs. Tools should only pass references/URLs, not binary data. + - Only adds different input/output format compared to an existing tool. + - DO NOT add tools very similar to existing tool and differing only in some input/output. Reject such suggestions - they should be tool improvement instead of new tool. + - DO NOT approve bulk action tools unless a bulk action API exists. The tool shouldn't be calling single APIs in a loop. + - DO NOT approve actions for upload/download from public URLs. + These should be improvements to existing tools, NOT new tools. + +3. ONLY APPROVE: + - Missing API endpoints that don't exist in current tools (verify with web search) + - High-value composite operations (batch/bulk APIs that combine multiple operations). + - Propose composite tools only when manual tool-chaining is error-prone and the operation is frequently used. Avoid really complex composite tools. + - New capabilities not covered by ANY existing tools and the capability is materially different that it cannot be clubbed into any existing tool. + - These tools are intended for LLM use. Do not add tools for simple summarization, formatting, extractions, or transformations that LLMs can perform natively. + - If there is a missing functionality that can be added to an existing tool, then do NOT propose a new tool (reject suggestion). Eg if send_email tool exists but doesn't have image support, then adding image parameter support to it is better than new tool send_email_with_image. + - NOTE: If you propose a new capability (e.g., a schedules feature), you must also propose the missing full CRUD tool set for it (Create/Read/Update/Delete) to ensure complete coverage. In this case, you may exceed the suggestions list and the usual output limit. + - ONLY approve a "composite" tool if it occurs in more than 5 distinct requests. A composite tool is defined as any tool that does not directly correspond to a single API call and instead abstracts, transforms, or chains existing tools (e.g., search_by_label wrapping a search endpoint, or upload_by_url wrapping an upload endpoint). + +4. WEB SEARCH VERIFICATION (REQUIRED) + - Use web search to find the API documentation for {toolkit} + - Confirm the endpoint EXISTS and is PUBLIC + - You MUST provide the verified API endpoint URL + - If you cannot verify the API exists, REJECT the suggestion + +IMPORTANT RULES: +- EVALUATE if the suggestion can be part of an existing tool or does it need to be a new tool. Reject suggestions where adding support to an existing tool is feasible. +- PRIORITIZE suggestions that appear multiple times over one-off suggestions. For very niche suggestions, avoid adding them unless they appear multiple times. +- Propose composite tools for common patterns eg trigger → action → poll status, apply → get result, etc IF it appears in >=5 requests. Avoid overly atomic tools that are not useful on their own. + +OUTPUT: Return up to 4 HIGH-VALUE approved tools (fewer is fine if not enough quality candidates). When in doubt, err toward rejecting the suggestion. Aim for very high confidence new tools only. + +For each approved tool, provide: +- tool_name: e.g. "TOOLKIT_ACTION_NAME" +- description: Clear one-line description of what the tool does (e.g. "Batch modify labels on multiple Gmail messages in a single API call") +- verified_api_endpoint: e.g. "POST /api/v1/resource/action" +- api_docs_url: Link to official API documentation +- instruction: Full build instruction including description, endpoint, docs, usage, rationale +- reasoning: Brief explanation of why this adds significant value""" + + agent = Agent( + name=f"ToolRanker_{toolkit}", model=self.model, - messages=[{"role": "user", "content": prompt}], - response_format=ToolRankingResult, - timeout=300.0, + instructions=instructions, + tools=[WebSearchTool()], + model_settings=ModelSettings(reasoning=Reasoning(effort="medium")), + output_type=ToolkitRankingResult, ) - result = response.choices[0].message.parsed - if not result: - return [] + prompt = f"""Evaluate these {len(suggestions)} candidate tool suggestions for {toolkit}: - # Convert back to dict format for compatibility - return [tool.model_dump() for tool in result.approved] +{suggestions_str} - async def get_structured_toolkit_data( - self, toolkit_suggestions: list[dict[str, Any]] - ) -> StructuredToolkitData: - """ - Get structured, validated toolkit data from LLM. +Use web search to verify API endpoints exist. Return your evaluation.""" - Args: - toolkit_suggestions: Deduplicated toolkit suggestions + result = await Runner.run(agent, prompt) + output: ToolkitRankingResult = result.final_output - Returns: - Structured toolkit data with categorization - """ - if not toolkit_suggestions: - return StructuredToolkitData( - high_priority=[], medium_priority=[], needs_review=[] - ) + # Set toolkit on the result and all approved tools + output.toolkit = toolkit + for tool in output.approved_tools: + tool.toolkit = toolkit + # Set suggestion_count from the counts we computed + tool_name_upper = tool.tool_name.upper() + tool.suggestion_count = tool_counts.get(tool_name_upper, 1) + # Set is_new_api - True if not in existing tools + tool.is_new_api = tool_name_upper not in existing_tool_slugs - existing_toolkits = self._get_existing_toolkits() - - prompt = f"""Analyze these {len(toolkit_suggestions)} new toolkit suggestions and return structured data. - -TOOLKIT SUGGESTIONS: -{json.dumps(toolkit_suggestions, indent=2)} - -⚠️ CRITICAL: EXISTING COMPOSIO TOOLKITS - DO NOT SUGGEST ANY OF THESE: -{', '.join(sorted(existing_toolkits))} - -Above are {len(existing_toolkits)} toolkits that ALREADY EXIST in Composio. If a suggestion matches ANY of these (even partially), you MUST exclude it completely. - -FILTERING RULES (STRICTLY ENFORCE): -1. CHECK EVERY suggestion against the existing toolkit list above - if it exists, REJECT IT -2. EXCLUDE internal/Composio tools (rube, composio_*, workbench, remote_*) -3. EXCLUDE redundant variations (aws vs aws_pricing - pick NONE if similar exists) -4. EXCLUDE ultra-niche services (< 5000 users) -5. ONLY include genuinely NEW third-party services with public APIs - -TASKS: -1. Verify each is a real, established service -2. Categorize by priority (high: 8+, medium: 6-7) -3. Order by frequency within tier (highest first) -4. Add brief, clear use case - -NOTE: Auto-vetoed: {', '.join(sorted(VETOED_TOOLKITS))} - -OUTPUT (JSON only): -{{ - "high_priority": [ - {{ - "name": "wordpress", - "frequency": 4, - "priority": 8.0, - "use_case": "CMS automation", - "is_valid": true - }} - ], - "medium_priority": [ - {{ - "name": "firebase", - "frequency": 1, - "priority": 7.0, - "use_case": "Backend services", - "is_valid": true - }} - ], - "needs_review": [ - {{ - "name": "custom_app", - "frequency": 1, - "priority": 5.0, - "use_case": "Internal tool", - "is_valid": false, - "concern": "May be internal/custom" - }} - ] -}} - -Only include real, valuable toolkits. Flag suspicious ones in needs_review. -""" - - response = await self.client.beta.chat.completions.parse( - model=self.model, - messages=[{"role": "user", "content": prompt}], - response_format=StructuredToolkitData, - timeout=120.0, - ) + return output - return response.choices[0].message.parsed or StructuredToolkitData( - high_priority=[], medium_priority=[], needs_review=[] - ) + async def rank_new_toolkits( + self, + toolkit_groups: list[ToolkitGroup], + existing_toolkits: set[str], + ) -> NewToolkitRankingResult: + """ + Rank new toolkit suggestions in a single bulk LLM call. - def format_slack_message(self, structured_data: StructuredToolkitData) -> str: - """Format structured toolkit data into clean Slack message.""" - today = datetime.utcnow().strftime("%B %d, %Y") + Uses Agents SDK with WebSearchTool to verify toolkit APIs exist. + """ + if not toolkit_groups: + return NewToolkitRankingResult( + approved_toolkits=[], + rejected_count=0, + already_exists_count=0, + no_public_api_count=0, + rejection_summary="No new toolkit suggestions to evaluate", + ) - total_count = ( - len(structured_data.high_priority) - + len(structured_data.medium_priority) - + len(structured_data.needs_review) + # Format existing toolkits + existing_list = ", ".join(sorted(existing_toolkits)) + + # Format candidate toolkits with their suggestions + candidates_lines: list[str] = [] + for group in toolkit_groups: + candidates_lines.append(f"- Toolkit: {group.toolkit}") + candidates_lines.append(f" Suggestions: {group.frequency} requests") + candidates_lines.append(" Use Cases:") + for s in group.suggestions[:5]: # Show up to 5 use cases + candidates_lines.append(f" • {s.use_case}") + if group.frequency > 5: + candidates_lines.append(f" ... and {group.frequency - 5} more") + candidates_lines.append("") + candidates_str = "\n".join(candidates_lines) + + instructions = f"""You are evaluating NEW TOOLKIT/INTEGRATION requests. Be VERY CONSERVATIVE. + +EXISTING COMPOSIO TOOLKITS ({len(existing_toolkits)} total): +{existing_list} + +EVALUATION RULES: + +1. CHECK IF ALREADY EXISTS + - Toolkit might exist under different name (e.g., "google_mail" = "gmail", "gsheets" = "googlesheets") + - If it exists under any name, REJECT + +2. WEB SEARCH VERIFICATION (REQUIRED) + - Use web search to verify: Is this a REAL company/service? + - Does it have a PUBLIC, documented API? + - Estimated user base (reject if clearly < 10K users) + +3. REJECT: + - Internal/custom tools (not real public services) + - Services without public APIs (enterprise-only, invite-only) + - Niche services with tiny user bases + - Already exists under different name in the existing toolkits list + +OUTPUT: Return TOP 10 (or fewer) new toolkits to build. + +For each approved toolkit, provide: +- toolkit_name: Canonical name for the toolkit +- api_docs_url: Link to official API documentation (REQUIRED) +- use_cases_summary: Summary of aggregated use cases +- frequency: Number of times this was suggested +- reasoning: Why this is valuable + API verification""" + + agent = Agent( + name="NewToolkitRanker", + model=self.model, + instructions=instructions, + tools=[WebSearchTool()], + model_settings=ModelSettings(reasoning=Reasoning(effort="medium")), + output_type=NewToolkitRankingResult, ) - lines = [f"🆕 *New Toolkit Requests* • {today}", ""] - lines.append(f"Found *{total_count} integrations* from user sessions:") - lines.append("") - - # High priority (sorted by frequency desc) - if structured_data.high_priority: - lines.append("🔥 *High Priority*") - high_sorted = sorted( - structured_data.high_priority, key=lambda x: x.frequency, reverse=True - ) - for toolkit in high_sorted: - lines.append( - f"• `{toolkit.name}` ({toolkit.frequency}×) — {toolkit.use_case}" - ) - lines.append("") - - # Medium priority (sorted by frequency desc) - if structured_data.medium_priority: - lines.append("📋 *Medium Priority*") - med_sorted = sorted( - structured_data.medium_priority, key=lambda x: x.frequency, reverse=True - ) - for toolkit in med_sorted: - lines.append( - f"• `{toolkit.name}` ({toolkit.frequency}×) — {toolkit.use_case}" - ) - lines.append("") + prompt = f"""Evaluate these {len(toolkit_groups)} candidate new toolkits: - # Needs review - if structured_data.needs_review: - lines.append("⚠️ *Needs Review*") - for toolkit in structured_data.needs_review: - concern = toolkit.concern or "Requires verification" - lines.append(f"• `{toolkit.name}` ({toolkit.frequency}×) — {concern}") - lines.append("") +{candidates_str} - lines.append("_Detected from Rube learning sessions_") +Use web search to verify each is a real service with a public API. Return your evaluation.""" - return "\n".join(lines) + result = await Runner.run(agent, prompt) + return result.final_output diff --git a/app_tester/actionable_learnings/models.py b/app_tester/actionable_learnings/models.py new file mode 100644 --- /dev/null +++ b/app_tester/actionable_learnings/models.py @@ -0,0 +1,179 @@ +""" +Data models for the actionable learnings pipeline. + +Clean Pydantic models for each stage of processing: +1. Raw extraction from ClickHouse +2. Grouped by toolkit +3. LLM ranking results +""" + +from typing import Literal + +from pydantic import BaseModel, Field + + +# ============================================================================= +# Stage 1: Extracted from ClickHouse (rube_error_analyses.tool_errors_json) +# ============================================================================= + + +class MissingToolSuggestion(BaseModel): + """Single suggestion extracted from missing_tools_impact array.""" + + suggestion_type: Literal["new_tool", "new_toolkit"] + tool_or_toolkit_name: str + toolkit: str + use_case: str + would_have_helped: bool + impact_explanation: str + + +# ============================================================================= +# Stage 2: Grouped by toolkit +# ============================================================================= + + +class ToolkitGroup(BaseModel): + """All suggestions grouped by a single toolkit.""" + + toolkit: str + is_existing_toolkit: bool = False # Set after checking Composio API + suggestions: list[MissingToolSuggestion] + + @property + def frequency(self) -> int: + return len(self.suggestions) + + +# ============================================================================= +# Stage 3: LLM Ranking Results +# ============================================================================= + + +class ApprovedTool(BaseModel): + """A tool approved by LLM for building. Used as LLM output schema.""" + + tool_name: str = Field(description="Name of the tool, e.g. GMAIL_BATCH_MODIFY") + description: str = Field( + description="What this tool does - clear one-line description" + ) + verified_api_endpoint: str = Field( + description="Verified API endpoint, e.g. POST /api/v1/messages/batch" + ) + api_docs_url: str | None = Field( + default=None, description="URL to API documentation" + ) + instruction: str = Field(description="Full build instruction for the tool") + reasoning: str = Field(description="Why this tool adds significant value") + toolkit: str = Field(default="", description="Toolkit name (set after LLM call)") + suggestion_count: int = Field( + default=0, description="Number of times this tool appeared in suggestions" + ) + is_new_api: bool = Field( + default=False, + description="Whether this is a new API endpoint not currently in Composio", + ) + + +class ToolkitRankingResult(BaseModel): + """Result of ranking tools for a single toolkit. Used as LLM output schema.""" + + approved_tools: list[ApprovedTool] = Field(default_factory=list) + rejected_count: int = Field(default=0) + similar_exists_count: int = Field( + default=0, description="Rejected because similar tool exists" + ) + low_value_count: int = Field(default=0, description="Rejected because low value") + rejection_summary: str = Field(default="") + toolkit: str = Field(default="", description="Toolkit name (set after LLM call)") + + +class ApprovedNewToolkit(BaseModel): + """A new toolkit approved by LLM for building. Used as LLM output schema.""" + + toolkit_name: str = Field(description="Canonical toolkit name") + api_docs_url: str = Field(description="URL to API documentation") + use_cases_summary: str = Field(description="Summary of use cases") + frequency: int = Field(description="Number of times suggested") + reasoning: str = Field(description="Why this toolkit is valuable") + + +class NewToolkitRankingResult(BaseModel): + """Result of ranking new toolkit suggestions. Used as LLM output schema.""" + + approved_toolkits: list[ApprovedNewToolkit] = Field(default_factory=list) + rejected_count: int = Field(default=0) + already_exists_count: int = Field( + default=0, description="Rejected because already exists" + ) + no_public_api_count: int = Field( + default=0, description="Rejected because no public API" + ) + rejection_summary: str = Field(default="") + + +# ============================================================================= +# Stage 4: Build Results +# ============================================================================= + + +class BuildResult(BaseModel): + """Result of triggering a build via Workflow Builder API.""" + + toolkit: str + tool_name: str + status: Literal["triggered", "failed", "skipped_dry_run"] + workflow_id: str | None = None + error: str | None = None + + +class PipelineResult(BaseModel): + """Final result of the entire pipeline run.""" + + rows_fetched: int + suggestions_extracted: int + suggestions_after_filter: int + toolkit_groups_count: int + existing_toolkit_groups: int + new_toolkit_groups: int + tools_approved: int + tools_build_triggered: int + tools_build_failed: int + new_toolkits_approved: int + slack_sent: bool + dry_run: bool + errors: list[str] = Field(default_factory=list) + + +# ============================================================================= +# Resumable Pipeline: Review File +# ============================================================================= + + +class ReviewFile(BaseModel): + """ + Review file for resumable pipeline. + + Written after analysis phase, user can review and remove entries, + then trigger build phase from this file. + """ + + # Metadata + created_at: str = Field(description="ISO timestamp when file was created") + since_date: str = Field(description="The --since date used for analysis") + suggestions_extracted: int = Field(description="Total suggestions before filtering") + suggestions_after_filter: int = Field(description="Suggestions after filtering") + existing_toolkit_groups: int = Field( + description="Number of existing toolkit groups" + ) + new_toolkit_groups: int = Field(description="Number of new toolkit groups") + + # Approved items for review + approved_tools: list[ApprovedTool] = Field( + default_factory=list, + description="Tools approved by LLM. Remove entries you don't want to build.", + ) + approved_toolkits: list[ApprovedNewToolkit] = Field( + default_factory=list, + description="New toolkits approved by LLM. Remove entries you don't want.", + ) diff --git a/app_tester/actionable_learnings/process_learnings.py b/app_tester/actionable_learnings/process_learnings.py --- a/app_tester/actionable_learnings/process_learnings.py +++ b/app_tester/actionable_learnings/process_learnings.py @@ -2,282 +2,825 @@ """ Process actionable learnings pipeline. -Fetches recent learnings from ClickHouse, deduplicates, ranks with LLM, -and triggers appropriate actions (Builder API for tools, Slack for toolkits). +Fetches missing tool suggestions from rube_error_analyses, ranks with LLM, +and triggers appropriate actions (Builder API for tools, Slack for notifications). Usage: - python process_learnings.py # Process last 48h - python process_learnings.py --dry-run # Preview without triggering - python process_learnings.py --hours 48 # Custom lookback window - python process_learnings.py --target-tools 30 # Request 30 tools from LLM - python process_learnings.py --skip-slack # Build tools but skip Slack + # Full pipeline (analyze + build) + python -m app_tester.actionable_learnings.process_learnings --hours 24 + python -m app_tester.actionable_learnings.process_learnings --hours 24 --dry-run + python -m app_tester.actionable_learnings.process_learnings --since 2025-12-20 --skip-slack + + # Resumable pipeline (two-step) + # Step 1: Analyze only, writes to temp/learnings_review_*.json + python -m app_tester.actionable_learnings.process_learnings --hours 24 --analyze-only + + # Step 2: Review the file, remove unwanted entries, then build + python -m app_tester.actionable_learnings.process_learnings --build-from-file temp/learnings_review_*.json + +Environment: + ACTIONABLE_LEARNINGS_DISABLE_SLACK=1 # Disable Slack globally """ import argparse import asyncio +import json +import os import sys from datetime import datetime +from pathlib import Path +from typing import Optional + +from mercury.observability import logger + +from app_tester.actionable_learnings.action_handlers import ToolBuilder, ToolkitNotifier +from app_tester.actionable_learnings.learnings_processor import LearningsProcessor +from app_tester.actionable_learnings.llm_validator import LearningsRanker +from app_tester.actionable_learnings.models import ( + ApprovedNewToolkit, + ApprovedTool, + BuildResult, + PipelineResult, + ReviewFile, + ToolkitGroup, + ToolkitRankingResult, +) + + +LLM_TIMEOUT_SECONDS = 600 # 10 minutes per toolkit +REVIEW_FILE_DIR = "temp" + +# Tools to always filter out after LLM approval (hardcoded blocklist) +BLOCKED_TOOLS = { + "GEMINI_GENERATE_TEXT", + "GEMINI_GENERATE_CONTENT", +} + +# Toolkits to always exclude from processing (hardcoded blocklist) +BLOCKED_TOOLKITS: set[str] = { + "gemini", +} + + +def write_review_file( + review_data: ReviewFile, + output_dir: str = REVIEW_FILE_DIR, +) -> str: + """ + Write review file for manual inspection before building. + + Args: + review_data: ReviewFile with approved tools and toolkits + output_dir: Directory to write to + + Returns: + Path to the written file + """ + Path(output_dir).mkdir(parents=True, exist_ok=True) + + timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S") + filename = f"learnings_review_{timestamp}.json" + filepath = Path(output_dir) / filename + + # Write with nice formatting for easy editing + with open(filepath, "w") as f: + json.dump(review_data.model_dump(), f, indent=2) + + logger.info(f"📝 Written review file: {filepath}") + return str(filepath) + + +def read_review_file(filepath: str) -> ReviewFile: + """ + Read review file for building. -from .action_handlers import ToolBuilder, ToolkitNotifier -from .learnings_processor import FEATURED_TOOLKITS, LearningsProcessor -from .llm_validator import LearningsRanker + Args: + filepath: Path to the review file + + Returns: + ReviewFile with approved tools and toolkits + """ + with open(filepath) as f: + data = json.load(f) + + review_file = ReviewFile(**data) + logger.info(f"📖 Loaded review file: {filepath}") + logger.info(f" • {len(review_file.approved_tools)} tools") + logger.info(f" • {len(review_file.approved_toolkits)} new toolkits") + return review_file + + +async def process_toolkit_group( + group: ToolkitGroup, + processor: LearningsProcessor, + ranker: LearningsRanker, +) -> ToolkitRankingResult | None: + """ + Process a single toolkit group through LLM ranking. + + Returns None if processing fails or times out (best effort). + """ + try: + existing_tools = processor.get_toolkit_tools(group.toolkit) + result = await asyncio.wait_for( + ranker.rank_tools_for_toolkit( + toolkit=group.toolkit, + suggestions=group.suggestions, + existing_tools=existing_tools, + ), + timeout=LLM_TIMEOUT_SECONDS, + ) + # Build approved tools list for display + approved_names = [t.tool_name for t in result.approved_tools] + approved_str = ", ".join(approved_names) if approved_names else "none" + + logger.info( + f" ✅ {group.toolkit}: {len(result.approved_tools)} approved, " + f"{result.rejected_count} rejected " + f"({result.similar_exists_count} similar exists, {result.low_value_count} low-value)" + ) + + if approved_names: + logger.info(f" 📦 Approved: {approved_str}") + for tool in result.approved_tools: + logger.info(f" • {tool.tool_name}: {tool.description}") + logger.info(f" API: {tool.verified_api_endpoint}") + if tool.reasoning: + logger.info(f" Reason: {tool.reasoning}") + + if result.rejection_summary: + summary_preview = result.rejection_summary[:150].replace("\n", " ") + logger.info(f" 📋 Rejections: {summary_preview}") + + return result + except asyncio.TimeoutError: + logger.warning( + f" ⏱️ {group.toolkit}: Timed out after {LLM_TIMEOUT_SECONDS}s - skipping" + ) + return None + except Exception as e: + logger.error(f" ❌ {group.toolkit}: Failed - {e}") + return None async def process_learnings( - hours_back: int = 48, - target_tool_count: int = 25, + since_date: str, dry_run: bool = False, skip_slack: bool = False, -) -> dict: + exclude_toolkits: set[str] | None = None, + analyze_only: bool = False, +) -> PipelineResult: """ Main pipeline for processing learnings. Args: - hours_back: Hours to look back for learnings - target_tool_count: Target number of tools to build (20-30 recommended) - dry_run: If True, skip actual API calls - skip_slack: If True, skip Slack notification (still builds tools) + since_date: ISO date string (e.g., "2025-12-20") + dry_run: If True, skip Builder API but still send to Slack + skip_slack: If True, also skip Slack notification + exclude_toolkits: Additional toolkits to exclude (merged with BLOCKED_TOOLKITS) + analyze_only: If True, only analyze and write to review file (no building) Returns: - Summary dict with counts and results + PipelineResult with summary """ - print("=" * 80) - print( + # Merge global blocked toolkits with CLI-provided exclusions + blocked_toolkits = BLOCKED_TOOLKITS | (exclude_toolkits or set()) + logger.info("=" * 80) + logger.info( f"ACTIONABLE LEARNINGS PIPELINE - {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')} UTC" ) - print("=" * 80) - mode = "DRY RUN" if dry_run else "PRODUCTION" - if skip_slack and not dry_run: + logger.info("=" * 80) + + if analyze_only: + mode = "ANALYZE ONLY (will write review file)" + elif dry_run: + mode = "DRY RUN" + else: + mode = "PRODUCTION" + if skip_slack and not analyze_only: mode += " (Skip Slack)" - print(f"Mode: {mode}") - print(f"Lookback: {hours_back} hours") - print(f"Target tools: {target_tool_count}") - print(f"Featured toolkits: {len(FEATURED_TOOLKITS)}") - print("=" * 80) + logger.info(f"Mode: {mode}") + logger.info(f"Since: {since_date}") + logger.info("=" * 80) - # Step 1: Fetch recent learnings - print(f"\n📥 Fetching learnings from last {hours_back} hours...") + errors: list[str] = [] processor = LearningsProcessor() + ranker = LearningsRanker() - try: - learnings = processor.fetch_recent_learnings(hours_back) - except Exception as e: - print(f"❌ Failed to fetch learnings: {e}") - return {"error": str(e), "stage": "fetch"} + # Step 1: Fetch suggestions + suggestions, rows_fetched = processor.fetch_missing_tool_suggestions(since_date) + + if not suggestions: + logger.info("⚠️ No suggestions found") + return PipelineResult( + rows_fetched=rows_fetched, + suggestions_extracted=0, + suggestions_after_filter=0, + toolkit_groups_count=0, + existing_toolkit_groups=0, + new_toolkit_groups=0, + tools_approved=0, + tools_build_triggered=0, + tools_build_failed=0, + new_toolkits_approved=0, + slack_sent=False, + dry_run=dry_run, + ) - print(f"✅ Fetched {len(learnings)} raw learnings") + # Step 2: Filter valid suggestions + valid_suggestions = processor.filter_valid_suggestions(suggestions) + + if not valid_suggestions: + logger.info("⚠️ No valid suggestions after filtering") + return PipelineResult( + rows_fetched=rows_fetched, + suggestions_extracted=len(suggestions), + suggestions_after_filter=0, + toolkit_groups_count=0, + existing_toolkit_groups=0, + new_toolkit_groups=0, + tools_approved=0, + tools_build_triggered=0, + tools_build_failed=0, + new_toolkits_approved=0, + slack_sent=False, + dry_run=dry_run, + ) - if not learnings: - print("\n⚠️ No learnings found in the specified time window") - return {"learnings_count": 0, "status": "no_data"} + # Step 3: Group by toolkit + groups = processor.group_by_toolkit(valid_suggestions) + + # Step 4: Split by existence + existing_groups, new_groups = await processor.split_by_existence(groups) + + # Step 4.5: Filter out blocked toolkits + if blocked_toolkits: + existing_before = len(existing_groups) + new_before = len(new_groups) + existing_groups = [ + g for g in existing_groups if g.toolkit.lower() not in blocked_toolkits + ] + new_groups = [ + g for g in new_groups if g.toolkit.lower() not in blocked_toolkits + ] + existing_filtered = existing_before - len(existing_groups) + new_filtered = new_before - len(new_groups) + if existing_filtered > 0 or new_filtered > 0: + logger.info( + f"🚫 Filtered out {existing_filtered + new_filtered} blocked toolkit(s): " + f"{existing_filtered} existing, {new_filtered} new" + ) - # Step 2: Deduplicate - print("\n🔄 Deduplicating learnings...") - deduplicated = processor.deduplicate_learnings(learnings) - new_tools, new_toolkits = processor.split_by_type(deduplicated) + # Step 5: Process existing toolkit groups (parallel LLM calls) + all_approved_tools: list[ApprovedTool] = [] - print(f"✅ Deduplicated to {len(deduplicated)} unique suggestions:") - print(f" • new_tool: {len(new_tools)}") - print(f" • new_toolkit: {len(new_toolkits)}") + if existing_groups: + logger.info(f"🤖 Processing {len(existing_groups)} existing toolkit groups...") - # Step 3: Two-pass filtering before LLM - print("\n🔍 Pass 1: Applying hard rules filter...") - pass1_filtered = processor.pass1_hard_filter(new_tools) - print(f"✅ Pass 1: {len(new_tools)} → {len(pass1_filtered)} tools") + # Run in parallel with semaphore to limit concurrency + semaphore = asyncio.Semaphore(20) - print("\n🔍 Pass 2: Applying diversity filter...") - pass2_filtered = processor.pass2_diversity_filter(pass1_filtered, target=250) - print(f"✅ Pass 2: {len(pass1_filtered)} → {len(pass2_filtered)} tools") + async def process_with_semaphore(group: ToolkitGroup): + async with semaphore: + return await process_toolkit_group(group, processor, ranker) - # Step 4: LLM Ranking (final selection) - print(f"\n🤖 LLM ranking {len(pass2_filtered)} tools...") - ranker = LearningsRanker() + results = await asyncio.gather( + *[process_with_semaphore(g) for g in existing_groups] + ) - try: - approved_tools = await ranker.rank_tools(pass2_filtered, target_tool_count) - except Exception as e: - print(f"❌ LLM ranking failed: {e}") - return {"error": str(e), "stage": "llm_ranking"} + for result in results: + if result is not None: + all_approved_tools.extend(result.approved_tools) + + # Filter out blocked tools + before_filter = len(all_approved_tools) + all_approved_tools = [ + t for t in all_approved_tools if t.tool_name not in BLOCKED_TOOLS + ] + blocked_count = before_filter - len(all_approved_tools) + if blocked_count > 0: + logger.info(f"🚫 Filtered out {blocked_count} blocked tool(s)") + + logger.info(f"📋 Total approved tools: {len(all_approved_tools)}") + for tool in all_approved_tools: + logger.info(f" • [{tool.toolkit}] {tool.tool_name}") + logger.info(f" Description: {tool.description}") + logger.info(f" API: {tool.verified_api_endpoint}") + logger.info(f" Reason: {tool.reasoning}") + + # Step 6: Process new toolkit groups + approved_toolkits: list[ApprovedNewToolkit] = [] + + if new_groups: + logger.info(f"🤖 Processing {len(new_groups)} new toolkit suggestions...") + try: + existing_toolkits = await processor.get_existing_toolkits() + toolkit_result = await asyncio.wait_for( + ranker.rank_new_toolkits(new_groups, existing_toolkits), + timeout=LLM_TIMEOUT_SECONDS * 2, # Allow more time for bulk call + ) + approved_toolkits = toolkit_result.approved_toolkits + + logger.info(f" ✅ {len(approved_toolkits)} new toolkits approved") + for tk in approved_toolkits: + logger.info(f" • {tk.toolkit_name} ({tk.frequency}×)") + logger.info(f" API Docs: {tk.api_docs_url}") + except asyncio.TimeoutError: + logger.warning( + f" ⏱️ New toolkit ranking timed out after {LLM_TIMEOUT_SECONDS * 2}s - skipping" + ) + errors.append("New toolkit ranking: timed out") + except Exception as e: + logger.error(f" ❌ New toolkit ranking failed: {e}") + errors.append(f"New toolkit ranking: {e}") + + # Step 7: If analyze_only, write review file, send to Slack, and return + if analyze_only: + review_data = ReviewFile( + created_at=datetime.utcnow().isoformat(), + since_date=since_date, + suggestions_extracted=len(suggestions), + suggestions_after_filter=len(valid_suggestions), + existing_toolkit_groups=len(existing_groups), + new_toolkit_groups=len(new_groups), + approved_tools=all_approved_tools, + approved_toolkits=approved_toolkits, + ) + review_file_path = write_review_file(review_data) + + # Send analysis results to Slack (no streaming needed, just summary) + slack_sent = False + if not skip_slack: + logger.info("📤 Sending analysis results to Slack...") + summary_notifier = ToolkitNotifier() + slack_sent = await summary_notifier.send_summary( + approved_tools=all_approved_tools, + approved_toolkits=approved_toolkits, + build_results=[], # No builds yet + suggestions_count=len(suggestions), + filtered_count=len(valid_suggestions), + existing_groups=len(existing_groups), + new_groups=len(new_groups), + dry_run=True, # Mark as pending review + ) - print(f"✅ LLM approved {len(approved_tools)} tools for building") + # Print summary for user + print() + print("=" * 80) + print("ANALYZE ONLY - REVIEW FILE WRITTEN") + print("=" * 80) + print(f"\n📝 Review file: {review_file_path}") + print(f"\n📦 Approved tools ({len(all_approved_tools)}):") + for tool in all_approved_tools: + print(f" • [{tool.toolkit}] {tool.tool_name}") + print(f"\n🆕 Approved new toolkits ({len(approved_toolkits)}):") + for tk in approved_toolkits: + print(f" • {tk.toolkit_name} ({tk.frequency}×)") + print() + print("Next steps:") + print(f" 1. Review and edit: {review_file_path}") + print(" 2. Remove any entries you don't want to build") + print( + f" 3. Run: python -m app_tester.actionable_learnings.process_learnings --build-from-file {review_file_path}" + ) + print("=" * 80) - # Step 5: LLM Structure + Format (Toolkits) - print(f"\n🤖 Getting structured data for {len(new_toolkits)} toolkits...") + return PipelineResult( + rows_fetched=rows_fetched, + suggestions_extracted=len(suggestions), + suggestions_after_filter=len(valid_suggestions), + toolkit_groups_count=len(groups), + existing_toolkit_groups=len(existing_groups), + new_toolkit_groups=len(new_groups), + tools_approved=len(all_approved_tools), + tools_build_triggered=0, + tools_build_failed=0, + new_toolkits_approved=len(approved_toolkits), + slack_sent=slack_sent, + dry_run=dry_run, + ) - try: - structured_data = await ranker.get_structured_toolkit_data(new_toolkits) - toolkit_message = ranker.format_slack_message(structured_data) - except Exception as e: - print(f"❌ LLM structuring failed: {e}") - return {"error": str(e), "stage": "llm_formatting"} + # Step 8: Trigger Builder API (with Slack streaming) + build_results: list[BuildResult] = [] + triggered_count = 0 + failed_count = 0 + slack_sent = False - print(f"✅ Generated clean Slack message ({len(toolkit_message)} chars)") + # Create notifier for streaming updates + notifier: Optional[ToolkitNotifier] = ToolkitNotifier() if not skip_slack else None - # Dry run - show what would happen - if dry_run: - print("\n" + "=" * 80) - print("DRY RUN SUMMARY") - print("=" * 80) + if all_approved_tools: + # Start Slack thread for build progress + if notifier: + notifier.start_build_thread(len(all_approved_tools), dry_run=dry_run) - print(f"\n📋 Would trigger {len(approved_tools)} tool builds:") - for i, tool in enumerate(approved_tools, 1): - toolkit = tool.get("toolkit", "unknown") - tool_name = tool.get("toolName", "unknown") - reasoning = tool.get("reasoning", "No reasoning provided") - print(f" {i:2d}. [{toolkit}] {tool_name}") - print(f" 💡 {reasoning}") - - print("\n📤 Would send to Slack (preview):") - print("-" * 80) - preview = toolkit_message[:800] - print(preview) - if len(toolkit_message) > 800: - print(f"\n... ({len(toolkit_message) - 800} more chars)") - print("-" * 80) - - return { - "dry_run": True, - "approved_tools_count": len(approved_tools), - "toolkit_message_length": len(toolkit_message), - } - - # Step 6: Trigger Builder API - print(f"\n🚀 Triggering {len(approved_tools)} builds via Builder API...") - print("\n📋 Approved tools:") - for i, tool in enumerate(approved_tools, 1): - toolkit = tool.get("toolkit", "unknown") - tool_name = tool.get("toolName", "unknown") - reasoning = tool.get("reasoning", "No reasoning provided") - print(f" {i:2d}. [{toolkit}] {tool_name}") - print(f" 💡 {reasoning}") - - builder = ToolBuilder(tag="rube-learning", color="purple") + if dry_run: + logger.info( + f"🏃 DRY RUN: Skipping Builder API for {len(all_approved_tools)} tools" + ) + for i, t in enumerate(all_approved_tools, 1): + result = BuildResult( + toolkit=t.toolkit, + tool_name=t.tool_name, + status="skipped_dry_run", + ) + build_results.append(result) + # Stream to Slack + if notifier: + notifier.post_build_result(result, i, len(all_approved_tools)) + else: + logger.info(f"🚀 Triggering {len(all_approved_tools)} builds...") + builder = ToolBuilder() + + # Create callback for Slack streaming + on_result = notifier.create_result_callback() if notifier else None + + build_results = await builder.build_tools( + all_approved_tools, on_result=on_result + ) + triggered_count = sum(1 for r in build_results if r.status == "triggered") + failed_count = sum(1 for r in build_results if r.status == "failed") + + # Log workflow IDs + if triggered_count > 0: + logger.info(f"📋 Triggered {triggered_count} workflow(s):") + for r in build_results: + if r.status == "triggered" and r.workflow_id: + logger.info(f" • {r.toolkit}/{r.tool_name}: {r.workflow_id}") + + # Post build completion summary to thread + if notifier and build_results: + notifier.post_build_complete(build_results) + + # Step 9: Send final summary to Slack (new toolkits, etc.) + if notifier: + logger.info("📤 Sending final summary to Slack...") + slack_sent = await notifier.send_summary( + approved_tools=all_approved_tools, + approved_toolkits=approved_toolkits, + build_results=build_results, + suggestions_count=len(suggestions), + filtered_count=len(valid_suggestions), + existing_groups=len(existing_groups), + new_groups=len(new_groups), + dry_run=dry_run, + ) + else: + logger.info("⏭️ Skipping Slack notification") - try: - build_results = await builder.build_tools(approved_tools) - except Exception as e: - print(f"❌ Builder API failed: {e}") - return {"error": str(e), "stage": "builder_api"} + # Final summary + logger.info("") + logger.info("=" * 80) + logger.info("PIPELINE COMPLETE") + logger.info("=" * 80) + logger.info(f"Suggestions extracted: {len(suggestions)}") + logger.info(f"After filtering: {len(valid_suggestions)}") + logger.info( + f"Toolkit groups: {len(groups)} ({len(existing_groups)} existing, {len(new_groups)} new)" + ) + logger.info(f"Tools approved: {len(all_approved_tools)}") + if not dry_run: + logger.info(f"Builds triggered: {triggered_count}, failed: {failed_count}") + else: + logger.info("Builds: skipped (dry run)") + logger.info(f"New toolkits approved: {len(approved_toolkits)}") + logger.info(f"Slack: {'sent' if slack_sent else 'not sent'}") + if errors: + logger.info(f"Errors: {len(errors)}") + for err in errors: + logger.info(f" • {err}") + logger.info("=" * 80) + + # Final report - use print for cleaner terminal output + print() + print("=" * 80) + print("FINAL REPORT - APPROVED ITEMS") + print("=" * 80) + + if all_approved_tools: + print(f"\n📦 NEW TOOLS TO BUILD ({len(all_approved_tools)}):") + # Group by toolkit for cleaner display + tools_by_toolkit: dict[str, list[ApprovedTool]] = {} + for tool in all_approved_tools: + if tool.toolkit not in tools_by_toolkit: + tools_by_toolkit[tool.toolkit] = [] + tools_by_toolkit[tool.toolkit].append(tool) + + for toolkit, tools in sorted(tools_by_toolkit.items()): + print(f"\n [{toolkit.upper()}] ({len(tools)} tools):") + for tool in tools: + print(f" • {tool.tool_name}") + print(f" {tool.description}") + print(f" Endpoint: {tool.verified_api_endpoint}") + if tool.api_docs_url: + print(f" Docs: {tool.api_docs_url}") + print(f" Reasoning: {tool.reasoning}") + else: + print("\n📦 No new tools approved.") + + if approved_toolkits: + print(f"\n🆕 NEW TOOLKIT SUGGESTIONS ({len(approved_toolkits)}):") + for tk in approved_toolkits: + print(f" • {tk.toolkit_name} ({tk.frequency}× requested)") + print(f" Docs: {tk.api_docs_url}") + print(f" Use cases: {tk.use_cases_summary[:100]}...") + print(f" Reasoning: {tk.reasoning}") + else: + print("\n🆕 No new toolkits approved.") - triggered_count = sum(1 for r in build_results if r["status"] == "triggered") - failed_count = len(build_results) - triggered_count + print() + print("=" * 80) - print( - f"\n✅ Build trigger results: {triggered_count} succeeded, {failed_count} failed" + return PipelineResult( + rows_fetched=rows_fetched, + suggestions_extracted=len(suggestions), + suggestions_after_filter=len(valid_suggestions), + toolkit_groups_count=len(groups), + existing_toolkit_groups=len(existing_groups), + new_toolkit_groups=len(new_groups), + tools_approved=len(all_approved_tools), + tools_build_triggered=triggered_count, + tools_build_failed=failed_count, + new_toolkits_approved=len(approved_toolkits), + slack_sent=slack_sent, + dry_run=dry_run, + errors=errors, ) - # Step 7: Send to Slack - if skip_slack: - print("\n📤 Skipping Slack notification (--skip-slack flag set)") - slack_success = True # Mark as successful skip - else: - print("\n📤 Sending toolkit suggestions to Slack...") - notifier = ToolkitNotifier() - try: - slack_success = await notifier.send_toolkit_suggestions(toolkit_message) - except Exception as e: - print(f"❌ Slack notification failed: {e}") - slack_success = False +async def build_from_review_file( + filepath: str, + dry_run: bool = False, + skip_slack: bool = False, +) -> PipelineResult: + """ + Build tools from a review file (resumable pipeline step 2). - # Final summary - print("\n" + "=" * 80) - print("PIPELINE COMPLETE") - print("=" * 80) - print(f"✅ Tools triggered: {triggered_count}/{len(approved_tools)}") + Args: + filepath: Path to the review file + dry_run: If True, skip Builder API but still send to Slack + skip_slack: If True, also skip Slack notification + + Returns: + PipelineResult with summary + """ + logger.info("=" * 80) + logger.info( + f"BUILD FROM REVIEW FILE - {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')} UTC" + ) + logger.info("=" * 80) + + mode = "DRY RUN" if dry_run else "PRODUCTION" if skip_slack: - print("⏭️ Slack notification: skipped") - else: - print( - f"{'✅' if slack_success else '❌'} Slack notification: {'sent' if slack_success else 'failed'}" + mode += " (Skip Slack)" + logger.info(f"Mode: {mode}") + logger.info(f"Review file: {filepath}") + logger.info("=" * 80) + + # Load review file + review_file = read_review_file(filepath) + all_approved_tools = review_file.approved_tools + approved_toolkits = review_file.approved_toolkits + + errors: list[str] = [] + + # Trigger Builder API (with Slack streaming) + build_results: list[BuildResult] = [] + triggered_count = 0 + failed_count = 0 + slack_sent = False + + # Create notifier for streaming updates + notifier: Optional[ToolkitNotifier] = ToolkitNotifier() if not skip_slack else None + + if all_approved_tools: + # Start Slack thread for build progress + if notifier: + notifier.start_build_thread(len(all_approved_tools), dry_run=dry_run) + + if dry_run: + logger.info( + f"🏃 DRY RUN: Skipping Builder API for {len(all_approved_tools)} tools" + ) + for i, t in enumerate(all_approved_tools, 1): + result = BuildResult( + toolkit=t.toolkit, + tool_name=t.tool_name, + status="skipped_dry_run", + ) + build_results.append(result) + # Stream to Slack + if notifier: + notifier.post_build_result(result, i, len(all_approved_tools)) + else: + logger.info(f"🚀 Triggering {len(all_approved_tools)} builds...") + builder = ToolBuilder() + + # Create callback for Slack streaming + on_result = notifier.create_result_callback() if notifier else None + + build_results = await builder.build_tools( + all_approved_tools, on_result=on_result + ) + triggered_count = sum(1 for r in build_results if r.status == "triggered") + failed_count = sum(1 for r in build_results if r.status == "failed") + + # Log workflow IDs + if triggered_count > 0: + logger.info(f"📋 Triggered {triggered_count} workflow(s):") + for r in build_results: + if r.status == "triggered" and r.workflow_id: + logger.info(f" • {r.toolkit}/{r.tool_name}: {r.workflow_id}") + + # Post build completion summary to thread + if notifier and build_results: + notifier.post_build_complete(build_results) + + # Send final summary to Slack (new toolkits, overall metrics) + if notifier: + logger.info("📤 Sending final summary to Slack...") + slack_sent = await notifier.send_summary( + approved_tools=all_approved_tools, + approved_toolkits=approved_toolkits, + build_results=build_results, + suggestions_count=review_file.suggestions_extracted, + filtered_count=review_file.suggestions_after_filter, + existing_groups=review_file.existing_toolkit_groups, + new_groups=review_file.new_toolkit_groups, + dry_run=dry_run, ) - print("=" * 80) + else: + logger.info("⏭️ Skipping Slack notification") - return { - "status": "success", - "learnings_fetched": len(learnings), - "deduplicated_count": len(deduplicated), - "new_tools_count": len(new_tools), - "new_toolkits_count": len(new_toolkits), - "approved_tools": len(approved_tools), - "builds_triggered": triggered_count, - "builds_failed": failed_count, - "slack_sent": slack_success, - "build_results": build_results, - } + # Final summary + logger.info("") + logger.info("=" * 80) + logger.info("BUILD FROM REVIEW FILE COMPLETE") + logger.info("=" * 80) + logger.info(f"Tools in review file: {len(all_approved_tools)}") + logger.info(f"New toolkits in review file: {len(approved_toolkits)}") + if not dry_run: + logger.info(f"Builds triggered: {triggered_count}, failed: {failed_count}") + else: + logger.info("Builds: skipped (dry run)") + logger.info(f"Slack: {'sent' if slack_sent else 'not sent'}") + logger.info("=" * 80) + + return PipelineResult( + rows_fetched=0, + suggestions_extracted=review_file.suggestions_extracted, + suggestions_after_filter=review_file.suggestions_after_filter, + toolkit_groups_count=review_file.existing_toolkit_groups + + review_file.new_toolkit_groups, + existing_toolkit_groups=review_file.existing_toolkit_groups, + new_toolkit_groups=review_file.new_toolkit_groups, + tools_approved=len(all_approved_tools), + tools_build_triggered=triggered_count, + tools_build_failed=failed_count, + new_toolkits_approved=len(approved_toolkits), + slack_sent=slack_sent, + dry_run=dry_run, + errors=errors, + ) def main(): """CLI entry point.""" parser = argparse.ArgumentParser( - description="Process actionable learnings from ClickHouse", + description="Process actionable learnings from rube_error_analyses", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: - %(prog)s # Process last 48h - %(prog)s --dry-run # Preview mode - %(prog)s --hours 48 # Last 48 hours - %(prog)s --target-tools 30 # Request 30 tools - %(prog)s --skip-slack # Build tools, skip Slack - %(prog)s --hours 12 --dry-run # Preview last 12h + # Full pipeline (analyze + build) + %(prog)s --hours 24 # Process last 24 hours + %(prog)s --hours 48 --dry-run # Preview last 48 hours + %(prog)s --since 2025-12-20 # Process since specific date + %(prog)s --hours 24 --skip-slack # Skip Slack notification + %(prog)s --hours 24 --exclude-toolkits slack,github # Exclude specific toolkits + + # Resumable pipeline (two-step) + %(prog)s --hours 24 --analyze-only # Step 1: Analyze, write review file + %(prog)s --build-from-file temp/learnings_review_*.json # Step 2: Build from file + +Environment: + ACTIONABLE_LEARNINGS_DISABLE_SLACK=1 # Disable Slack globally """, ) - parser.add_argument( + # Source of data: --since, --hours, or --build-from-file + source_group = parser.add_mutually_exclusive_group(required=True) + source_group.add_argument( + "--since", + type=str, + help="ISO date to fetch suggestions from (e.g., 2025-12-20)", + ) + source_group.add_argument( "--hours", type=int, - default=48, - help="Hours to look back for learnings (default: 48)", + help="Hours to look back (e.g., 24 for last 24 hours)", ) - - parser.add_argument( - "--target-tools", - type=int, - default=25, - help="Target number of tools to build (default: 25, range: 20-30)", + source_group.add_argument( + "--build-from-file", + type=str, + metavar="FILE", + help="Build from a review file (resumable pipeline step 2)", ) parser.add_argument( "--dry-run", action="store_true", - help="Preview mode - don't trigger actual builds or Slack", + help="Preview mode - skip Builder API but still send to Slack", ) parser.add_argument( "--skip-slack", action="store_true", - help="Skip Slack notification (still builds tools)", + help="Skip Slack notification", + ) + + parser.add_argument( + "--analyze-only", + action="store_true", + help="Only analyze and write review file, don't build (resumable pipeline step 1)", + ) + + parser.add_argument( + "--exclude-toolkits", + type=str, + default="", + help="Comma-separated list of toolkits to exclude (merged with BLOCKED_TOOLKITS)", ) args = parser.parse_args() - # Validate target-tools range - if not 10 <= args.target_tools <= 50: - print("⚠️ Warning: target-tools should be between 10-50 (recommended: 20-30)") + # Validate: --analyze-only requires --since or --hours + if args.analyze_only and args.build_from_file: + parser.error("--analyze-only cannot be used with --build-from-file") + + # Check env var to disable Slack globally + slack_disabled_by_env = os.getenv( + "ACTIONABLE_LEARNINGS_DISABLE_SLACK", "" + ).lower() in ("1", "true", "yes") + skip_slack = args.skip_slack or slack_disabled_by_env + + if slack_disabled_by_env and not args.skip_slack: + logger.info("📵 Slack disabled via ACTIONABLE_LEARNINGS_DISABLE_SLACK env var") try: - result = asyncio.run( - process_learnings( - hours_back=args.hours, - target_tool_count=args.target_tools, - dry_run=args.dry_run, - skip_slack=args.skip_slack, + # Mode 1: Build from review file + if args.build_from_file: + result = asyncio.run( + build_from_review_file( + filepath=args.build_from_file, + dry_run=args.dry_run, + skip_slack=skip_slack, + ) + ) + # Mode 2: Analyze (and optionally build) + else: + # Compute since_date from --hours if provided + if args.hours: + from datetime import timedelta + + since_dt = datetime.utcnow() - timedelta(hours=args.hours) + since_date = since_dt.strftime("%Y-%m-%d") + else: + since_date = args.since + + # Parse excluded toolkits from CLI + exclude_toolkits: set[str] = set() + if args.exclude_toolkits: + exclude_toolkits = { + t.strip().lower() + for t in args.exclude_toolkits.split(",") + if t.strip() + } + logger.info( + f"🚫 CLI excluded toolkits: {', '.join(sorted(exclude_toolkits))}" + ) + + result = asyncio.run( + process_learnings( + since_date=since_date, + dry_run=args.dry_run, + skip_slack=skip_slack, + exclude_toolkits=exclude_toolkits, + analyze_only=args.analyze_only, + ) ) - ) - if result.get("error"): - print(f"\n❌ Pipeline failed at stage: {result.get('stage')}") - print(f"Error: {result.get('error')}") + if result.errors: + logger.warning(f"Pipeline completed with {len(result.errors)} error(s)") sys.exit(1) sys.exit(0) except KeyboardInterrupt: - print("\n\n⚠️ Interrupted by user") + logger.info("\n⚠️ Interrupted by user") sys.exit(130) except Exception as e: - print(f"\n❌ Unexpected error: {e}") + logger.error(f"❌ Pipeline failed: {e}") import traceback traceback.print_exc() diff --git a/app_tester/actionable_learnings/setup_cron.sh b/app_tester/actionable_learnings/setup_cron.sh --- a/app_tester/actionable_learnings/setup_cron.sh +++ b/app_tester/actionable_learnings/setup_cron.sh @@ -15,11 +15,13 @@ LOG_DIR="$PROJECT_ROOT/logs" mkdir -p "$LOG_DIR" # Create the cron command -CRON_CMD="0 1 * * * cd $PROJECT_ROOT && source $VENV_PATH/bin/activate && source $ENV_FILE && python -m app_tester.actionable_learnings.process_learnings >> $LOG_DIR/actionable_learnings.log 2>&1" +# Uses date to dynamically calculate 2 days ago for --since +CRON_CMD="0 1 * * * cd $PROJECT_ROOT && source $VENV_PATH/bin/activate && source $ENV_FILE && python -m app_tester.actionable_learnings.process_learnings --since \$(date -u -d '2 days ago' +\%Y-\%m-\%d) >> $LOG_DIR/actionable_learnings.log 2>&1" echo "Setting up cron job for actionable learnings pipeline..." echo "" echo "Cron schedule: Daily at 1 AM UTC" +echo "Lookback: 2 days (dynamic --since)" echo "Project root: $PROJECT_ROOT" echo "Log file: $LOG_DIR/actionable_learnings.log" echo "" @@ -56,9 +58,8 @@ echo "" echo "To test manually (dry-run):" echo " cd $PROJECT_ROOT" echo " source $VENV_PATH/bin/activate && source $ENV_FILE" -echo " python -m app_tester.actionable_learnings.process_learnings --dry-run" +echo " python -m app_tester.actionable_learnings.process_learnings --since \$(date -u -d '2 days ago' +%Y-%m-%d) --dry-run" echo "" echo "To remove cron job:" echo " crontab -e" echo " # Delete the line containing 'actionable_learnings'" - diff --git a/app_tester/agent_worker_fix_from_logs/services/clickhouse_service.py b/app_tester/agent_worker_fix_from_logs/services/clickhouse_service.py --- a/app_tester/agent_worker_fix_from_logs/services/clickhouse_service.py +++ b/app_tester/agent_worker_fix_from_logs/services/clickhouse_service.py @@ -6,6 +6,7 @@ import time from datetime import datetime, timedelta from typing import Dict, List, Optional, Tuple +from urllib.parse import urlparse import requests @@ -15,6 +16,13 @@ import logging as _logging logger = _logging.getLogger(__name__) +try: + import clickhouse_connect + CLICKHOUSE_CONNECT_AVAILABLE = True +except ImportError: + CLICKHOUSE_CONNECT_AVAILABLE = False + logger.warning("clickhouse-connect not available, falling back to requests") + from openai import OpenAI from composio import ComposioToolSet from composio.exceptions import EnumMetadataNotFound, HTTPError @@ -29,6 +37,160 @@ from ...filter_actions import check_action_call from ...constant import ErrorType +# Fully-qualified table names for learning pipeline v2 +SESSION_ANALYSES_TABLE_FQ = "app_db.rube_session_analyses_v2" +WORKFLOW_ANALYSES_TABLE_FQ = "app_db.rube_workflow_analyses" + + +class QueryBuilder: + """Query builder for ClickHouse queries with centralized column definitions.""" + + # Centralized column definitions for v2 tables with JOINs + # Session analysis columns (prefixed with 's.' for JOIN queries) + RUBE_SESSION_ANALYSES_COLUMNS = [ + 's.id', + 's.client_session_id', + 's.user_id', + 's.client_name', + 's.session_start_time', + 's.session_end_time', + 's.session_duration_seconds', + 's.analyzed_at', + 's.tool_call_count', + 's.total_tool_duration_ms', + 's.log_ids', + 's.action_sequence', + 's.search_queries', + 's.overall_usecase', + 's.workflow_category', + 's.workflow_summary', + 's.task_difficulty', + 's.search_quality', + 's.workflow_quality', + 's.primary_failure_reason', + 's.overall_rationale', + 's.confidence_score', + 's.total_issues_count', + 's.active_bucket_count', + 's.tool_improvement_count', + 's.meta_tool_improvement_count', + 's.new_tool_suggestion_count', + 's.new_toolkit_suggestion_count', + 's.planner_issue_count', + 's.total_search_calls', + 's.total_cached_plans_returned', + 's.total_plans_generated', + 's.cached_plan_quality', + 's.cached_plan_quality_reason', + 's.overall_search_quality_score', + 's.workflow_analysis_id', + # Workflow analysis fields (joined from rube_workflow_analyses) + 'w.per_usecase_plans_json', + 'w.per_usecase_tool_analysis_json', + 'w.search_json', + 'w.search_quality_explanation', + ] + + # Extended columns for cached plan queries + RUBE_SESSION_ANALYSES_CACHED_COLUMNS = RUBE_SESSION_ANALYSES_COLUMNS + [ + 's.cached_plan_returned', + ] + + ERROR_CLASSIFICATION_COLUMNS = [ + 'logId', + 'appName', + 'actionName', + 'errorType', + 'reason', + 'createdAt', + 'updatedAt', + 'isToolError', + 'slackThreadTs', + ] + + @staticmethod + def build_select( + table: str, + columns: List[str], + where_clauses: Optional[List[str]] = None, + order_by: Optional[str] = None, + limit: Optional[int] = None, + join_clause: Optional[str] = None, + ) -> str: + """ + Build a SELECT query. + + Args: + table: Table name (can include database, e.g., 'app_db.table') + columns: List of column names + where_clauses: List of WHERE conditions (will be joined with AND) + order_by: ORDER BY clause (e.g., 's.analyzed_at DESC') + limit: LIMIT clause + join_clause: Optional JOIN clause (e.g., 'LEFT JOIN ...') + + Returns: + SQL SELECT query string + """ + columns_str = ',\n '.join(columns) + query = f"SELECT\n {columns_str}\n FROM {table}" + + if join_clause: + query += f"\n {join_clause}" + + if where_clauses: + where_clause = " AND ".join(where_clauses) + query += f"\n WHERE {where_clause}" + + if order_by: + query += f"\n ORDER BY {order_by}" + + if limit: + query += f"\n LIMIT {limit}" + + return query + + @staticmethod + def build_time_filter( + since: datetime, + until: Optional[datetime] = None, + column: str = 's.analyzed_at', + ) -> List[str]: + """ + Build time-based WHERE clause conditions. + + Args: + since: Start timestamp (inclusive) + until: End timestamp (exclusive, optional) + column: Column name to filter on (default: 's.analyzed_at' for v2 tables) + + Returns: + List of WHERE clause conditions + """ + conditions = [f"{column} >= '{since.strftime('%Y-%m-%d %H:%M:%S')}'"] + if until: + conditions.append(f"{column} < '{until.strftime('%Y-%m-%d %H:%M:%S')}'") + return conditions + + @staticmethod + def build_workflow_quality_filter(only_successful: bool) -> List[str]: + """Build workflow quality filter conditions.""" + if only_successful: + return ["s.workflow_quality IN ('SUCCESS', 'SUCCESS_WEAK')"] + return [] + + @staticmethod + def build_search_quality_filter(filter_good: bool) -> List[str]: + """Build search quality filter conditions. + + Supports both old enum values (GOOD, ACCEPTABLE, POOR) and + new enum values (EXCELLENT, AVERAGE, POOR). + """ + if filter_good: + # Include both old and new enum values for backwards compatibility + # Old: GOOD, ACCEPTABLE | New: EXCELLENT, AVERAGE + return ["s.search_quality IN ('GOOD', 'ACCEPTABLE', 'EXCELLENT', 'AVERAGE')"] + return [] + class ClickHouseService: """Service for polling ClickHouse for tool error logs""" @@ -48,6 +210,33 @@ def __init__(self): if not self.base_url or not self.username or not self.password: raise ValueError("Missing ClickHouse connection environment variables") + + # Initialize ClickHouse Connect client if available + self.client = None + self.use_clickhouse_connect = False + if CLICKHOUSE_CONNECT_AVAILABLE: + try: + parsed_url = urlparse(self.base_url) + host = parsed_url.hostname or parsed_url.netloc.split(':')[0] + port = parsed_url.port or (8443 if parsed_url.scheme == 'https' else 8123) + database = parsed_url.path.lstrip('/') or 'default' + + self.client = clickhouse_connect.get_client( + host=host, + port=port, + username=self.username, + password=self.password, + database=database, + secure=parsed_url.scheme == 'https', + ) + self.use_clickhouse_connect = True + logger.info("Using ClickHouse Connect client") + except Exception as e: + logger.warning(f"Failed to initialize ClickHouse Connect, falling back to requests: {e}") + self.use_clickhouse_connect = False + + # Initialize query builder + self.query_builder = QueryBuilder() def _setup_encryption_key(self): """Setup encryption key for decryption""" @@ -183,6 +372,71 @@ def _decrypt_log_fields(self, log: Dict) -> Dict: def _execute_query(self, query: str, max_retries: int = 3, base_delay: float = 1.0) -> Dict: """Execute a ClickHouse query and return the parsed JSON result with retry logic""" is_select = query.strip().lower().startswith("select") or query.strip().lower().startswith("with") + + # Use ClickHouse Connect if available + if self.use_clickhouse_connect and self.client: + return self._execute_with_clickhouse_connect(query, is_select, max_retries, base_delay) + + # Fallback to requests + return self._execute_with_requests(query, is_select, max_retries, base_delay) + + def _execute_with_clickhouse_connect( + self, query: str, is_select: bool, max_retries: int, base_delay: float + ) -> Dict: + """Execute query using ClickHouse Connect client.""" + assert self.client is not None # Checked by caller + last_exception = None + + for attempt in range(max_retries + 1): + try: + if is_select: + result = self.client.query(query) + # Convert result_rows (list of tuples) to list of dicts + column_names = result.column_names + data = [ + dict(zip(column_names, row)) + for row in result.result_rows + ] + return {"data": data} + else: + self.client.command(query) + return {"status": "ok"} + except Exception as e: + last_exception = e + is_connection_error = ( + isinstance(e, (ConnectionError, TimeoutError)) or + "Connection" in str(type(e).__name__) + ) + + if is_connection_error and attempt < max_retries: + delay = base_delay * (2 ** attempt) + logger.warning( + f"ClickHouse connection error (attempt {attempt + 1}/{max_retries + 1}): {e}. " + f"Retrying in {delay:.1f} seconds..." + ) + time.sleep(delay) + continue + else: + logger.error( + "Error executing ClickHouse query: {}\nQuery: {}, traceback: {}", + e, + query, + traceback.format_exc(), + ) + return {"data": []} + + if last_exception: + logger.error( + "Failed to execute ClickHouse query after {} retries: {}", + max_retries, + last_exception, + ) + return {"data": []} + + def _execute_with_requests( + self, query: str, is_select: bool, max_retries: int, base_delay: float + ) -> Dict: + """Execute query using requests library (fallback).""" if is_select: query = query.strip().rstrip(";") + "\nFORMAT JSON;" @@ -195,7 +449,7 @@ def _execute_query(self, query: str, max_retries: int = 3, base_delay: float = 1 data=query, auth=(self.username or "", self.password or ""), headers={"Content-Type": "text/plain"}, - timeout=300, # 300 second timeout + timeout=300, ) response.raise_for_status() @@ -220,15 +474,14 @@ def _execute_query(self, query: str, max_retries: int = 3, base_delay: float = 1 ) if is_connection_error and attempt < max_retries: - delay = base_delay * (2 ** attempt) # Exponential backoff + delay = base_delay * (2 ** attempt) logger.warning( f"ClickHouse connection error (attempt {attempt + 1}/{max_retries + 1}): {e}. " f"Retrying in {delay:.1f} seconds..." ) time.sleep(delay) continue else: - # Log the final error and return empty result logger.error( "Error executing ClickHouse query: {}\nResponse text: {}\nQuery: {}, traceback: {}", e, @@ -238,7 +491,6 @@ def _execute_query(self, query: str, max_retries: int = 3, base_delay: float = 1 ) return {"data": []} - # This should never be reached, but just in case if last_exception: logger.error( "Failed to execute ClickHouse query after {} retries: {}", @@ -491,8 +743,26 @@ def insert_error_type( ) -> None: """ Insert the error type, reason and optional Slack thread reference. + Uses parameterized queries to prevent SQL injection. """ - # Escape single quotes by replacing them with two single quotes (ClickHouse standard) + # Use ClickHouse Connect parameterized insert if available + if self.use_clickhouse_connect and self.client: + try: + # ClickHouse Connect handles NOW() automatically if we use raw SQL + # Use client.command() for INSERT with NOW() function + self.client.command( + f""" + INSERT INTO app_db.error_classification + (logId, errorType, reason, appName, actionName, isToolError, createdAt, updatedAt, slackThreadTs) + VALUES + ('{log_id.replace("'", "''")}', '{error_type.value.replace("'", "''")}', '{error_reason.replace("'", "''")}', '{app_name.replace("'", "''")}', '{action_name.replace("'", "''")}', {1 if is_tool_error else 0}, NOW(), NOW(), '{((thread_ts or "").replace("'", "''"))}') + """ + ) + return + except Exception as e: + logger.warning(f"ClickHouse Connect insert failed, falling back to query: {e}") + + # Fallback to manual escaping (still safer than f-strings with user input) escaped_reason = error_reason.replace("'", "''") escaped_app_name = app_name.replace("'", "''") escaped_action_name = action_name.replace("'", "''") @@ -549,3 +819,248 @@ def update_last_poll_timestamp(self, timestamp: datetime) -> None: # TODO: Implement persistent storage of last poll timestamp # For now, this is a no-op logger.info(f"Last poll timestamp updated to {timestamp}") + + def fetch_recent_session_analyses( + self, + hours_back: int = 12, + only_successful: bool = True, + filter_good_search_quality: bool = True + ) -> List[Dict]: + """ + Fetch session analyses from rube_session_analyses_v2 table for last N hours. + + Args: + hours_back: Number of hours to look back + only_successful: If True, only fetch workflows with workflow_quality IN ('SUCCESS', 'SUCCESS_WEAK') + filter_good_search_quality: If True, only fetch sessions with search_quality='GOOD' + + Returns: + List of session analysis records with all fields + """ + since_timestamp = datetime.utcnow() - timedelta(hours=hours_back) + + # Build WHERE clause using QueryBuilder + where_clauses = self.query_builder.build_time_filter( + since_timestamp, column="s.analyzed_at" + ) + where_clauses.extend(self.query_builder.build_workflow_quality_filter(only_successful)) + where_clauses.extend(self.query_builder.build_search_quality_filter(filter_good_search_quality)) + + # Build query using QueryBuilder with JOIN + query = self.query_builder.build_select( + table=f'{SESSION_ANALYSES_TABLE_FQ} AS s', + columns=self.query_builder.RUBE_SESSION_ANALYSES_COLUMNS, + join_clause=f"LEFT JOIN {WORKFLOW_ANALYSES_TABLE_FQ} AS w ON s.workflow_analysis_id = w.id", + where_clauses=where_clauses, + order_by='s.analyzed_at DESC', + ) + + logger.info(f"Fetching session analyses since {since_timestamp.strftime('%Y-%m-%d %H:%M:%S')} (only_successful={only_successful}, filter_good_search={filter_good_search_quality})") + result = self._execute_query(query) + data = result.get("data", []) + logger.info(f"Fetched {len(data)} session analyses") + + return data + + def fetch_session_analyses_time_range( + self, + hours_start: float, + hours_end: float, + only_successful: bool = True, + filter_good_search_quality: bool = True + ) -> List[Dict]: + """ + Fetch session analyses from rube_session_analyses_v2 table for a specific time range. + + Args: + hours_start: Start of time range (hours ago, e.g., 24 for 24 hours ago, can be fractional) + hours_end: End of time range (hours ago, e.g., 23 for 23 hours ago, can be fractional) + only_successful: If True, only fetch workflows with workflow_quality IN ('SUCCESS', 'SUCCESS_WEAK') + filter_good_search_quality: If True, only fetch sessions with search_quality='GOOD' + + Returns: + List of session analysis records with all fields + """ + start_timestamp = datetime.utcnow() - timedelta(hours=hours_start) + end_timestamp = datetime.utcnow() - timedelta(hours=hours_end) + + # Build WHERE clause using QueryBuilder + where_clauses = self.query_builder.build_time_filter( + start_timestamp, end_timestamp, column="s.analyzed_at" + ) + where_clauses.extend(self.query_builder.build_workflow_quality_filter(only_successful)) + where_clauses.extend(self.query_builder.build_search_quality_filter(filter_good_search_quality)) + + # Build query using QueryBuilder with JOIN + query = self.query_builder.build_select( + table=f'{SESSION_ANALYSES_TABLE_FQ} AS s', + columns=self.query_builder.RUBE_SESSION_ANALYSES_COLUMNS, + join_clause=f"LEFT JOIN {WORKFLOW_ANALYSES_TABLE_FQ} AS w ON s.workflow_analysis_id = w.id", + where_clauses=where_clauses, + order_by='s.analyzed_at DESC', + ) + + logger.info(f"Fetching session analyses from {start_timestamp.strftime('%Y-%m-%d %H:%M:%S')} to {end_timestamp.strftime('%Y-%m-%d %H:%M:%S')} (only_successful={only_successful}, filter_good_search={filter_good_search_quality})") + result = self._execute_query(query) + data = result.get("data", []) + logger.info(f"Fetched {len(data)} session analyses") + + return data + + def fetch_cached_only_session_analyses( + self, + hours_back: int = 12, + hours_start: Optional[float] = None, + hours_end: Optional[float] = None + ) -> List[Dict]: + """ + Fetch session analyses where ALL plans returned were cached plans. + No workflow success or search quality filters applied. + + Filter: total_cached_plans_returned > 0 AND total_cached_plans_returned = total_plans_generated + + Args: + hours_back: Number of hours to look back (if hours_start/end not provided) + hours_start: Start of time range (hours ago, e.g., 24 for 24 hours ago, can be fractional) + hours_end: End of time range (hours ago, e.g., 12 for 12 hours ago, can be fractional) + + Returns: + List of session analysis records where all plans were cached + """ + # Determine time range + if hours_start is not None and hours_end is not None: + start_timestamp = datetime.utcnow() - timedelta(hours=hours_start) + end_timestamp = datetime.utcnow() - timedelta(hours=hours_end) + else: + end_timestamp = datetime.utcnow() + start_timestamp = end_timestamp - timedelta(hours=hours_back) + + # Build WHERE clause - only time filter and cached plan filter + where_clauses = self.query_builder.build_time_filter( + start_timestamp, end_timestamp, column="s.analyzed_at" + ) + where_clauses.extend([ + "s.total_cached_plans_returned > 0", + "s.total_cached_plans_returned = s.total_plans_generated", + "s.workflow_quality != 'INCOMPLETE'" + ]) + + # Build query using QueryBuilder with extended columns and JOIN + query = self.query_builder.build_select( + table=f'{SESSION_ANALYSES_TABLE_FQ} AS s', + columns=self.query_builder.RUBE_SESSION_ANALYSES_CACHED_COLUMNS, + join_clause=f"LEFT JOIN {WORKFLOW_ANALYSES_TABLE_FQ} AS w ON s.workflow_analysis_id = w.id", + where_clauses=where_clauses, + order_by='s.analyzed_at DESC', + ) + + logger.info(f"Fetching cached-only session analyses from {start_timestamp.strftime('%Y-%m-%d %H:%M:%S')} to {end_timestamp.strftime('%Y-%m-%d %H:%M:%S')}") + logger.info("Filter: total_cached_plans_returned > 0 AND total_cached_plans_returned = total_plans_generated AND workflow_quality != 'INCOMPLETE'") + + result = self._execute_query(query) + data = result.get("data", []) + + logger.info(f"Fetched {len(data)} cached-only session analyses") + + return data + + def fetch_sessions_by_ids( + self, + session_ids: List[str], + earliest_timestamp: Optional[str] = None, + ) -> List[Dict]: + """ + Fetch specific sessions by client_session_id, optionally filtering by analyzed_at. + """ + if not session_ids: + return [] + + sanitized_parts: List[str] = [] + for session_id in session_ids: + if not session_id: + continue + sanitized = session_id.replace("'", "") + sanitized_parts.append(f"'{sanitized}'") + sanitized_ids = ",".join(sanitized_parts) + if not sanitized_ids: + return [] + + where_clauses = [f"s.client_session_id IN ({sanitized_ids})"] + + if earliest_timestamp: + try: + start_dt = datetime.fromisoformat(earliest_timestamp.replace("Z", "+00:00")) + except ValueError: + start_dt = None + if start_dt: + where_clauses.extend( + self.query_builder.build_time_filter( + since=start_dt, column="s.analyzed_at" + ) + ) + + query = self.query_builder.build_select( + table=f"{SESSION_ANALYSES_TABLE_FQ} AS s", + columns=self.query_builder.RUBE_SESSION_ANALYSES_COLUMNS, + join_clause=f"LEFT JOIN {WORKFLOW_ANALYSES_TABLE_FQ} AS w ON s.workflow_analysis_id = w.id", + where_clauses=where_clauses, + order_by="s.analyzed_at DESC", + ) + + result = self._execute_query(query) + return result.get("data", []) + + def fetch_sessions_by_clickhouse_ids( + self, + clickhouse_ids: List[str], + earliest_timestamp: Optional[str] = None, + ) -> List[Dict]: + """ + Fetch specific sessions by ClickHouse row ID, optionally filtering by analyzed_at. + + Args: + clickhouse_ids: List of ClickHouse row IDs (the 'id' column) + earliest_timestamp: Optional ISO timestamp to filter sessions created after + + Returns: + List of session dictionaries + """ + if not clickhouse_ids: + return [] + + sanitized_parts: List[str] = [] + for clickhouse_id in clickhouse_ids: + if not clickhouse_id: + continue + sanitized = clickhouse_id.replace("'", "") + sanitized_parts.append(f"'{sanitized}'") + sanitized_ids = ",".join(sanitized_parts) + if not sanitized_ids: + return [] + + where_clauses = [f"s.id IN ({sanitized_ids})"] + + if earliest_timestamp: + try: + start_dt = datetime.fromisoformat( + earliest_timestamp.replace("Z", "+00:00") + ) + except ValueError: + start_dt = None + if start_dt: + where_clauses.extend( + self.query_builder.build_time_filter( + since=start_dt, column="s.analyzed_at" + ) + ) + + query = self.query_builder.build_select( + table=f"{SESSION_ANALYSES_TABLE_FQ} AS s", + columns=self.query_builder.RUBE_SESSION_ANALYSES_COLUMNS, + join_clause=f"LEFT JOIN {WORKFLOW_ANALYSES_TABLE_FQ} AS w ON s.workflow_analysis_id = w.id", + where_clauses=where_clauses, + order_by="s.analyzed_at DESC", + ) + + result = self._execute_query(query) + return result.get("data", []) diff --git a/app_tester/analyze_rube_sessions.py b/app_tester/analyze_rube_sessions.py deleted file mode 100644 --- a/app_tester/analyze_rube_sessions.py +++ /dev/null @@ -1,1848 +0,0 @@ -#!/usr/bin/env python3 -""" -Analyze Rube user sessions to generate insights and recommendations. - -This script: -1. Fetches Rube logs from ClickHouse for the last hour -2. Groups logs by user ID and detects sessions -3. Analyzes sessions with LLM to generate recommendations -4. Outputs formatted recommendations - -Usage: - # Single run mode - python app_tester/analyze_rube_sessions.py - - # Polling mode (runs continuously with 60 minute intervals) - python app_tester/analyze_rube_sessions.py --poll-interval 60 - -Required environment variables: - - BACKEND_READONLY_URL: ClickHouse URL - - CLICKHOUSE_USERNAME: ClickHouse username - - CLICKHOUSE_PASSWORD: ClickHouse password - - OPENAI_API_KEY: OpenAI API key for GPT analysis - - ENCRYPTION_KEY: Key for decrypting request/response data - -Optional dependencies: - - rich: For beautiful formatted output (pip install rich) - Without it, the script will use basic text output - -The script will: -- Group Rube tool calls by user ID -- Detect sessions (new session starts with search tool if gap > 1 minute) -- Filter to sessions with 3+ tool calls (more substantial workflows) -- Log session sequences (e.g., search -> multi-execute -> workbench) -- Analyze each user's sessions with GPT-4 -- Output prioritized recommendations for tool improvements -""" - -import argparse -import asyncio -import json -import os -import sys -import time -from concurrent.futures import ThreadPoolExecutor, as_completed -from datetime import datetime, timedelta -from typing import Dict, List, Optional -from uuid import uuid4 - -# Load environment variables from .env file if it exists -from dotenv import load_dotenv - -# Import rich for better formatting -from rich.console import Console - -# Import Slack SDK for sending notifications -from slack_sdk import WebClient - -from app_tester.agent_worker_fix_from_logs.services.clickhouse_service import ( - ClickHouseService, -) -from app_tester.constant import azure_client -from app_tester.rube_learning import ( - AnalysisFormatter, - PrimarySessionAnalysis, - SearchQuality, - Session, - SessionAnalysis, - SessionDetector, - SessionInfo, - SlackNotifier, - run_tool_gap_analysis, -) - - -load_dotenv() - -SEND_TO_SLACK = True -SAVE_TO_CLICKHOUSE = True - - -class LLMAnalyzer: - """Analyzes sessions using LLM to generate recommendations""" - - def __init__(self): - self.client = azure_client - self.console = Console() - - def format_session_for_llm(self, session: Session) -> str: - """Format session data for LLM analysis""" - formatted_logs = [] - - for i, call in enumerate(session.calls, 1): - call_lines = [ - f"Call {i}:", - f"Action: {call.get('actionName', 'unknown')}", - f"Timestamp: {call.get('timestamp', 'unknown')}", - ] - - # Add duration if available - duration = call.get("duration") - if duration: - try: - duration_ms = float(duration) - call_lines.append(f"Duration: {duration_ms/1000:.2f}s") - except (ValueError, TypeError): - pass - - # Add full request without parsing - request = call.get("request", "") - call_lines.append(f"Request: {request}") - - # Add response or error - error = call.get("error", "") - if error and error not in ["", "null", "None"]: - call_lines.append(f"Error: {error}") - else: - response = call.get("response", "") - call_lines.append(f"Response: {response}") - - call_text = "\n".join(call_lines) - if ( - len(call_text) > 40000 - and call.get("actionName") != "COMPOSIO_SEARCH_TOOLS" - ): - call_text = ( - f"{call_text[:30000]}\n[..TRUNCATED..]\n{call_text[-10000:]}" - ) - - formatted_logs.append(call_text) - - return "\n\n".join(formatted_logs) - - workbench_tools_prompt = """ -4. Helper functions initialized in the workbench: - 1) `run_composio_tool(tool_slug: str, arguments: dict)`: Execute a known Composio **app** tool (from RUBE_SEARCH_TOOLS). Do not invent names; match the tool’s input schema. Suited for loops/parallel/bulk over datasets. - 1.1 run_composio_tools returns JSON with top-level "data". Parse carefully—structure may be nested. - 2) `invoke_llm(query: str)`: Invoke an LLM for semantic tasks. Pass MAX 200000 characters in input. - 3) `proxy_execute(method, endpoint, toolkit, params=None, body=None)`: Call a toolkit API directly when no Composio tool exists. - 4) `web_search(query: str)`: Search the web for information. - 5) `upload_local_file(*file_paths)`: Upload files to Composio S3/R2 storage. Use this when you need to upload/download any generated artifacts from the sandbox. - -### 1) run_composio_tool(tool_slug, arguments) -Executes a known Composio tool via backend API. Do NOT call RUBE_* meta tools to avoid cyclic calls. - - def run_composio_tool(tool_slug: str, arguments: Dict[str, Any]) -> tuple[Dict[str, Any], str] - # Returns: (tool_response_dict, error_message) - # Success: ({"data": {actual_data}}, "") - Note the top-level data - # Error: ({}, "error_message") or (response_data, "error_message") - - result, error = run_composio_tool("GMAIL_FETCH_EMAILS", {"max_results": 1, "user_id": "me"}) - if error: - print("GMAIL_FETCH_EMAILS error:", error); return - email_data = result.get("data", {}) - print("Fetched:", email_data) - -### 2) invoke_llm(query) -Calls Groq LLM for reasoning, analysis, and semantic tasks. Pass MAX 50000 characters input. - - def invoke_llm(query: str) -> tuple[str, str] - # Returns: (llm_response, error_message) - - resp, error = invoke_llm("Summarize the key points from this data") - if error: - print("invoke_llm error:", error); return - print("LLM:", resp) - - # Example: analyze tool response with LLM - tool_resp, err = run_composio_tool("GMAIL_FETCH_EMAILS", {"max_results": 5, "user_id": "me"}) - if err: - print("GMAIL_FETCH_EMAILS error:", err); return - parsed = tool_resp.get("data", {}) - resp, err = invoke_llm(f"Analyze these emails and summarize: {parsed}") - if err: - print("invoke_llm error:", err); return - print("LLM Gmail Summary:", resp) - - # TIP: batch prompts to reduce LLM calls. - -### 4) proxy_execute(method, endpoint, toolkit, params=None, body=None) -Direct API call to a connected toolkit service. - - from typing import Literal, Optional - - def proxy_execute( - method: Literal["GET","POST","PUT","DELETE","PATCH"], - endpoint: str, - toolkit: str, - params: Optional[list[Parameter]] = None, - body: Optional[object] = None, - ) -> tuple[ToolProxyResponse | None, str] - # Returns: (response_object, error_message) - - response, error = proxy_execute("GET", "https://api.github.com/repos/owner/repo", "github") - if error: - print("proxy_execute error:", error); return - print("Success:", response.data) - -### 5) web_search(query) -Searches the web via Exa AI. - - def web_search(query: str) -> tuple[str, str] - # Returns: (search_results_text, error_message) - - results, error = web_search("latest developments in AI") - if error: - print("web_search error:", error) - else: - print("Results:", results) - - ### Parallelize (4-min sandbox timeout) -Adjust concurrency so all tasks finish within 4 minutes. - - import concurrent.futures - - MAX_CONCURRENCY = 10 # Adjust as needed - - def send_bulk_emails(email_list): - def send_single(email): - result, error = run_composio_tool("GMAIL_SEND_EMAIL", { - "to": email["recipient"], "subject": email["subject"], "body": email["body"] - }) - if error: - print(f"Failed {email['recipient']}: {error}") - return {"status": "failed", "error": error} - return {"status": "sent", "data": result} - - results = [] - with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_CONCURRENCY) as ex: - futures = [ex.submit(send_single, e) for e in email_list] - for f in concurrent.futures.as_completed(futures): - results.append(f.result()) - return results - - # Example usage - email_list = [{"recipient": f"user{i}@example.com", "subject": "Test", "body": "Hello"} for i in range(1000)] - results = send_bulk_emails(email_list) - -### upload_local_file(*file_paths) -Uploads files to Composio S3/R2 storage. Single files upload directly, multiple files are auto-zipped. -Use this when you need to upload/download any generated artifacts from the sandbox. - - def upload_local_file(*file_paths) -> tuple[Dict[str, Any], str] - # Returns: (result_dict, error_string) - # Success: ({"s3_url": str, "uploaded_file": str, "type": str, "id": str, "key": str, "message": str}, "") - # Error: ({}, "error_message") - - # Single file - result, error = upload_local_file("/path/to/report.pdf") - - # Multiple files (auto-zipped) - result, error = upload_local_file("/home/user/doc1.txt", "/home/user/doc2.txt") - - # Always check for errors - if error: - print("Upload failed:", error) - return - print("Uploaded:", result["s3_url"]) - """ - - async def analyze_session( - self, session: Session - ) -> Optional[PrimarySessionAnalysis]: - """Analyze a session and generate workflow recommendations (no new tool suggestions)""" - try: - session_data = self.format_session_for_llm(session) - - prompt = f"""You are given a sequence of tool calls made by an LLM to Rube Universal MCP to accomplish a task. Your goal is to systematically analyze the workflow and suggest tool recommendations to improve quality, latency, and failures in workflow. - -IMPORTANT CONTEXT: These tool calls are part of a general-purpose AI agent (Claude, GPT, etc.) using Rube as a tool layer. The agent handles summarization, analysis, user followups, and responses outside of tool calls - but you only see tool execution logs, not the agent's reasoning or conversations or final output to users. -- Only flag 'incomplete_workflow' if logs show sudden abandonment mid-execution (user gave up), NOT for missing summarization/analysis steps (agent does that). -- If tools returned requested data successfully, assume the workflow succeeded. -- Similarly if confirmation was requested from user, we won't have access to those logs. Therefore, assume that permission was requested and granted if the tools flagged it. - - -Workflow tools used by LLM: -1) Search - this is the main meta tool to discover relevant app tools for a usecase and known_fields. It internally uses an LLM to find relevant app tools based on user usecase and known_fields. - a) Search internally takes tool names and description (first 200 chars) into context for finding relevant tools for the usecase. So the tool descriptions must be accurate and thorough for high-quality search. - b) It returns main/related tools and toolkits (apps) with their descriptions and request schema, their connection status, and any memory items. - c) In the end it mentions guidance whether to call planner or not and task difficulty (easy/medium/hard). Capture this info. - d) Search may return multiple apps for the usecase (eg gmail, outlook for sending email), and the caller can pick which they want to use. -2) Planner tool - this creates detailed plan for complex tasks (medium/hard). Skipped / no-op for easy tasks. IMPORTANT: If planner returns no-op/skip, treat as PLANNER_NOT_CALLED (not bad plan - it's correct behavior). -3) Multi execute - this can be used to execute upto 20 tools in parallel. If the response is too large, it gets saved to workbench, and the response structure and preview are returned inline. -4) Remote workbench tool - this can be used to execute code in a remote sandbox to process multi-execute responses or do bulk/parallel tool calling. It can use following helpers - a) run_composio_tool to call composio tools with auth handled automatically - b) proxy_execute to call the app's APIs directly with auth handled automatically (eg /api/search) - c) invoke_llm to analyze data with LLM in workbench; web_search to search web -5) Remote bash tool - this can be used for bash operations over files in the same sandbox. -Note: All steps of workflow may not be visible via logs, eg if LLM asked a question and user responded - it won't show up in logs. So keep that in mind and assume things got missed necessarily. - -{session_data} - - -Given this workflow - your job is to comprehensively analyze the workflow deeply for quality, inefficiencies, and suggest high impact improvements. Check if the workflow was finished completely - eg if summary of all emails was asked but it only fetched 100 emails, that's a red flag - did it paginate, get correct data, use correct summarization techniques, etc. - -Based on your analysis, suggest improvements in following categories: - -1) Improvements to existing app tool description or request schema/descriptions to add missing functionality or to document any pitfalls or details that can improve the search quality and llm usage - goal is to avoid running into these issues. - 1.1 Any constraints, rate-limits, usage patterns, quirks, or examples - these can go in description or schema edits. Propose concise edits to description / request schema where applicable. - 1.2 You may suggest adding new parameters or improvements to existing params to handle specific usecases. - 1.3 You may propose renaming a tool if it will significantly improve search quality - 1.4 A tool may fail due to a legit bug like auth issue - 1.5 Any edits to description should be under 300 characters - -2) Any improvements to meta tools (EXCEPT search-related improvements) like plan/workbench/multi-execute - improvements can be in meta tool descriptions, schemas, behavior, or even adding a new meta tool (in rare cases). - 2.1 Was multi-execute called efficiently utilizing the parallelism? - 2.2 If workbench code was written by LLM, check if there were any failures or inefficiencies (eg wrong code). Find out which specific app tool (if any) was called in workbench. Think if there's a systemic way to avoid these issues - eg if LLM misunderstands the response schema, then giving response schema in search can help. Or if it didnt't write parallel code then workbench tool description can encourage it more. Think of novel ideas too. - 2.3 Consider efficiency of the workflow - eg did it use workbench to process 20 emails when it should have used multi-execute? Or did it use multi-execute instead of workbench to send 1000 emails? Did it use workbench unnecessarily? - 2.4 To summarize/analyze response in the workbench, did the workflow use invoke_llm or print to stdout, or did it use naive filtering which may not be accurate? - -IMPORTANT: Do NOT suggest new tools to add OR search quality improvements - a separate tool gap analysis agent handles those. Only focus on improving existing app tools and non-search meta tools. - -Only give recommendations in category 1/2 if you think they are extremely relevant. You do not need to fill every category. - -You can find the description and request schema of app tools in search results that was given to LLM. The search response also contains difficulty level (easy/medium/hard) and planner guidance. Then multi execute may be called to execute some of those tools and responses are returned. For complex scripts workbench may be called. -In the request field, if it's JSON, look for the 'thought' field under 'params' to see what the LLM intended and what it got. Look for difficulty level in the search response. Consider failures, unexpected results, multiple steps / inefficiency that can be avoided, or pitfalls/notes that can be documented in tool or meta tools. - -IMPORTANT: -1. In the actual_workflow field, provide a concise summary of what happened in this session: - - What the user wanted to do - focus on input passed to Search tools to decide the intent. - - What tools were called/returned - - What worked or failed - here again focus on whether the usecase was completed or not. If some optional tools/apps were not executed that is fine as long as the original usecase completed. - Example: "User searched for 'reddit votes tool'. Search found no voting tools. User tried workbench with proxy_execute to /api/v1/vote, got 403 forbidden." - Example: "User searched for 'fetch my emails'. Search found gmail and outlook tools, gmail had an active connection. User executed gmail_fetch_email and succeeded." - Also note we only have data of tool calls made, but not the original client chat (LLM output or user input). Eg if LLM asked a question and user responded - it won't show up in logs. So keep that in mind and don't assume things got missed necessarily. - -2. In your rationale/overall_rationale, provide SPECIFIC DETAILS about what the user was trying to do and what went wrong. For example: - - "User searched for 'reddit votes tool' but no voting tools were found in Reddit toolkit" - - "User wanted to 'get user profile' but had to use workbench with proxy_execute instead" - - "Search returned GMAIL_SEND_EMAIL but user needed bulk send capability" - - "Tool failed because parameter 'user_id' expects email format but description says 'ID of user'" - -3. Was planner quality good or bad, and rationale. - -4. Workflow may have done decently well and may not need optimization. Note you do NOT have access to final response given to user to accurately judge workflow success. Hence keep recommendations conservative, only when something clearly needs improvement. Don't over optimize. - -5. PER USECASE PLANS (field per_usecase_plans): For EACH searched usecase in the session, analyze the entire workflow that followed and generate ONE comprehensive plan that captures the overall workflow and critical insights someone would need before attempting this same use case. Think of it as capturing the key details that made this workflow succeed (or fail). -IMPORTANT: There can be multiple usecases inside queries field within a single SEARCH tool call - eg {{"overall_usecase": "fetch recent emails and send an email with summary", "queries": [{{"use_case": "fetch recent emails"}}, {{"use_case": "send email", "known_fields": "recipient:John"}}]}}. In this case, you should generate a plan the overall usecase as well as for each individual usecase. - Also, there can be multiple SEARCH tool calls in the session - so consider overall usecase and individual usecase across all the search calls. - - As there can multiple usecases producing multiple plans, be very careful when matching the workflow execution with individual usecase plans. Do not mix the execution steps - eg don't put steps for sending email in plan for fetching email! - - For the overall usecase plan (if overall_usecase is present), you should generate a plan that captures all the individual usecases in the tool call. It should contain the details of individual usecase plans and any cross-usecase learnings as well. Eg if a search tool had 3 usecases and 1 overall usecase, you must generate 4 plans for this call. - - Number the plans as usecase_i_j or overall_i where i=search number, and j=usecase number within search. Eg usecase_1_1 for 1st usecase in 1st search, overall_1 for overall usecase in 1st search, usecase_2_3 for 3nd usecase in 2nd search, so on. - - Check the tool slug used in multi-execute or remote_workbench calls to help map which usecases would it correspond to. - - Your plan will be surfaced to any new user attempting a similar use case. So the Search Plan should include a crisp sequence of steps to follow and any gotchas to be careful about. Focus on the gap between expectation and reality - what seemed straightforward from the search but turned out complex, or what hidden constraints emerged during execution. - - Key aspects to analyze (but only mention what's most relevant to each search): - - a) Execution plan: - - What tools to call in which sequence, fallbacks tools, required inputs and output format. Capture the happy path and the fallback paths. Focus on app tools not meta tools - eg don't mention multi-execute vs workbench choice. - - If there's an existing plan, you should generate a plan that supersedes that existing plan. Otherwise generate a new plan. - - For each tool call, describe the INTENT - what data or outcome you want to achieve - in plain English. - - DO NOT specify exact parameters for the tools - let the executing LLM determine those based on the available context and tool schemas. - - Eg search query "download attachments from someone in my email since last week" -> Step 1) Call GMAIL_FETCH_EMAILS with query filter, paginate with cursor until all emails received Step 2) Call GMAIL_DOWNLOAD_ATTACHMENT in parallel with email IDs. - - You may include any non-obvious details that were not present in the Tool Schema and Description that was returned by Search. But only include if it is truly useful and broadly applicable to variants of the usecase. - - Rule - don't be overly prescriptive or verbose in the plan, the plan is used as a guidance rather than prescription, so keep it flexible. Focus on high-level strategy. - - ENSURE any tool or parameter names in the plan are accurate and not hallucinated. - - Key considerations in plan - - Would pagination need to be handled? Sequential vs parallel. - - Time Awareness: For time-sensitive operations (e.g., 'last week', 'last 24 hours'), mention in plan to use the UTC time (usually known from Search). Timezone may need to be handled (ask the user if not known). - - User confirmation: For irreversible/public-impact operations (sending messages, deleting data, sharing resources) - include a step in plan requiring user confirmation before execution. Read-only operations need no confirmation. - - Output verification: While you don't have access to the final output given to user, do your best to analyze from tool call responses whether the required data was found or not. Eg if someone searched for upcoming flights but tool call returned no flights, that's a red flag and you should assess why that may have happened (wrong filters, etc). - - Output formatting: Mention to always include clickable markdown links and inline references to sources (eg provide links to final thread/document instead of IDs). For summaries, provide detailed context with sections/examples, not just numbers. - - b) Data Volume & Performance Patterns: - - Did the workflow encounter unexpected API limits? (e.g., "GMAIL_FETCH_EMAILS caps at 500 messages - requires date-chunked iterations for 6-month audits") - - What pagination strategies were needed? Not just "needs pagination" but WHY (e.g., "Slack channels API returns max 100 per page, use next_cursor for complete list") - - Rate limits or throttling patterns (e.g., "GitHub allows 5000 requests/hour but LIST_COMMITS counts as multiple") - - c) Tool Sequencing & Dependencies: - - Which tools needed to be called in non-obvious orders? (e.g., "JIRA_GET_ISSUE doesn't include custom fields - need JIRA_GET_CUSTOM_FIELDS first, then map field IDs back to issues") - - What data transformations were required between tools? Any cross-app transformations or data merging? - - When did the workflow need tool A's output to properly configure tool B? - - d) Existing Plan's Analysis: There can be an existing plan that was surfaced in search to the user (if cached_plan_id is present). Analyze that plan and preserve the important bits - - If plan was present, did it correctly anticipate the workflow complexity? - - Did the plan identify non-obvious requirements (like needing to fetch user IDs before sending messages)? - - When the plan provided structure, did it prevent trial-and-error that would have occurred otherwise? - - Did execution follow the plan, or did real-world constraints force deviations? - - e) Error Patterns & Recovery: - - What failures occurred and what retry strategy worked? (e.g., "GitHub API returned 502 errors during peak hours - exponential backoff with 3 retries succeeded") - - What validation errors revealed tool input requirements? - - f) Workarounds & Missing Capabilities: - - When did the workflow need proxy_execute because no direct tool existed? - - What creative solutions emerged to work around tool limitations? - - g) Critical Realizations: - - What assumption about the tools proved incorrect? - - What undocumented behavior was discovered? - - What made the difference between this workflow succeeding vs failing? - - h) Workbench hints: - - IF the workbench was used, check if it was necessary. It is meant for processing large datasets or bulk scripting (e.g., 1000+ emails), not small ones (e.g., 10 emails). Flag common misuse cases. - - If the workbench ran too long, suggest breaking the task into smaller workbench runs. Typical runs should finish under 1 minute. - - Workbench use depends on data size - eg "summarize 10 emails" doesn't need it but "summarize 1000 emails" might. Avoid recommending workbench unless data size is explicitly mentioned in usecase. You may mention generic guidance like "if handling 100+ emails, switch to workbench" - - For parsing or any other workbench errors, you can suggest fixes: - - For response parsing error - suggest to call multi_execute first to inspect response schema. You can also provide any schema hints if known. - - For rate limits - sleep with backoff. - - For longer runs, break into smalller tasks - - IF workbench coding is involved, encourage good coding practices - breaking into smaller tasks/functions, adding print statements for debugging, parallelization, checkpointing. Avoid code comments as they eat up context. - - Each plan must be SPECIFIC to that search's use case with concrete details. Include actual numbers, tool names, error codes, and strategies used. For example: - - Bad: "Needed to handle rate limits" - - Good: "Shopify's GraphQL API allows 2 points/second but SHOPIFY_GET_PRODUCTS costs 3 points per call. For this inventory sync of 500 products, had to add 1.5s delays between calls in workbench, turning a 1-minute task into 12 minutes." - - IMPORTANT: For the per_usecase_plans field, return a list of UsecasePlan objects, one for each search query in the session. - - For each UsecasePlan: - - search_id: The id of the COMPOSIO_SEARCH_TOOLS call (eg usecase_1_1, overall_1, usecase_2_3, etc.) - - cleaned_search_usecase: Remove PII/names/identifiers/user-specific info, preserve app names, change to English if in another language. E.g., "send email to John via gmail" → "send email to someone via gmail" - - cleaned_known_fields: Only keep field keys, remove values. E.g., "recipient:John, count:10" → "recipient, count" - - toolkits_required: List of lowercase toolkit slugs required (e.g., ["gmail"], ["gmail", "outlook"]) - - usecase_plan: UsecasePlanDetails object with these fields: - * workflow_steps (List[str]): List of key steps in workflow execution, capture happy path and fallback paths (if applicable), keep high level. Each step should be a concise string. NEVER include PII or user-specific identifiers. - * difficulty (str): Difficulty assessment as "easy", "medium", or "hard" with short reason explaining why. Consider factors like number of steps, error handling needs, edge cases, etc. - * pitfalls (List[str]): List of gotchas and pitfalls in tool inputs/outputs/behavior - what can go wrong, rate limits, missing data handling, auth issues, edge cases, etc. Each pitfall should be concise and actionable. NEVER include PII or user-specific details. - DON'T REPEAT points between workflow_steps and pitfalls - mention once only. Keep concise and in English. - Note: If an existing plan was found (i.e., cached_plan_id is set), ensure that the new plan differs by **no more than 20%** from the existing one and the new plan is within the size limits mentioned. Do not remove any key elements from the previous plan unless you are certain they are unnecessary/redundant or incorrect. - - plan_changelog: Short changelog of what changed vs existing plan (e.g., "1) added rate limits, 2) added find projects dependency"). If no existing plan, just say "new". - - success_confidence: 0-10 score - lower if workflow incomplete/missing tools, higher if plan effectively addresses issues - - LIMITS: Keep each plan MAX under 10 lines and 500-1000 characters. Extend beyond this range only if the use case is unusually complex or involves multiple significant challenges. Easy usecases should have very concise plan. - If there are useful details in existing plan, preserve them as your plan supersedes existing plans. - Return empty list [] if no meaningful plans for any search. - - CRITICAL: Write plans as actionable insights for future workflows: - - Capture workflow steps using the given tools. - - MUST ENSURE the output is accurate and grounded in real data given to you. NEVER make up any step/parameter/suggestion based on your own knownledge as it can be stale. Base output ONLY on tool schemas and actual execution. - - Don't reference memory, meta tools (eg manage_connections), or any other non-app tool information in the plan. - - State the gotcha/constraint directly (e.g., "Slack search is day-level only with exclusive 'after:' parameter") - - Don't mention what "the planner did" or "the agent had to" - just state what needs to be done - - Avoid implementation details like sync_response_to_workbench or file paths - - Mention workbench only when the usecase clearly involves large data processing (100+ items) or complex scripting - for simple cases do not mention workbench as the behavior is automatically handled by client. - - If workbench is involved, mention insights about errors and code quality - e.g., "Parallel processing with ThreadPoolExecutor improves 100x over sequential calls", or "break code into smaller tasks with print statements for effective workflow" - - Focus on API quirks, data structures, rate limits, cross-app dependencies, and non-obvious requirements that future workflows must handle - - ONLY describe patterns you directly observed in THIS session's logs OR present in previously returned plan - do not invent or assume behaviors - - DON'T REPEAT any information that's already present in tool schemas clearly, and avoid unnecessary details to keep the plan concise (eg don't add unnecessary performance optimizations for simple usecases) - - For easy usecases, shorter and direct plans are more effective. For hard usecases covering different edge cases may be needed. Don't complicate simple plans with excessive details. - - NEVER put any PII or user-specific info in the plan - eg don't put column names, sheet names, etc. Keep the plan generic. - -NOTE: Always return responses in ENGLISH, even if the user's language is different. - -Give concrete recommendations on what to improve. Return upto 3 extremely high quality recommendations. You may return more if the workflow had lots of major issues. Make sure recommendations are EXTREMELY HIGH CONFIDENCE and highly targeted. If none meet this bar, return none. Do not repeat same reco in different steps. It is okay to not have any recommendations for a workflow (set has_recommendations to false). -Remember this is one of the many usecases so I strictly strictly don't want to overfit and don't want noisy/vague suggestions. Hence keep changes concise keeping generalizability in mind.""" - - # Log prompt length - print( - f"Total prompt length: {len(prompt)} characters, ~{len(prompt)/4} tokens" - ) - - model_name = "gpt-5" - response = await self.client.beta.chat.completions.parse( - model=model_name, - messages=[{"role": "user", "content": prompt}], - response_format=PrimarySessionAnalysis, - ) - - if response.choices[0].message.parsed: - analysis = response.choices[0].message.parsed - - # Sort recommendations by priority - analysis.tool_improvements.sort(key=lambda x: x.priority, reverse=True) - analysis.meta_tool_improvements.sort( - key=lambda x: x.priority, reverse=True - ) - - return analysis - - except Exception as e: - print(f"Failed to analyze session {session.session_key}: {e}") - - return None - - -class RubeSessionAnalyzer: - """Main analyzer that orchestrates the session analysis""" - - slack_notifier: Optional[SlackNotifier] - - def __init__( - self, - poll_interval_minutes: float = 5.0, - initial_lookback_minutes: float = 5.0, - last_log_timestamp: Optional[datetime] = None, - target_log_id: Optional[str] = None, - max_sessions: int = 10000, - slack_sampling_percent: float = 10.0, - ): - self.poll_interval_minutes = poll_interval_minutes - self.initial_lookback_minutes = initial_lookback_minutes - self.last_log_timestamp = last_log_timestamp - self.target_log_id = target_log_id - self.max_sessions = max_sessions - self.slack_sampling_percent = slack_sampling_percent - self.clickhouse = ClickHouseService() - self.session_detector = SessionDetector( - marked_session_threshold_minutes=20.0, - unmarked_session_threshold_minutes=10.0, - ) - self.llm_analyzer = LLMAnalyzer() - self.console = Console() - self.formatter = AnalysisFormatter(self.console) - - # Initialize Slack notifier - slack_token = os.getenv("SLACK_TOKEN") - if slack_token: - slack_client = WebClient(token=slack_token) - self.slack_notifier = SlackNotifier(slack_client) - else: - self.slack_notifier = None - print("SLACK_TOKEN not set - Slack notifications will be disabled") - - def fetch_rube_logs(self, since_timestamp: datetime) -> List[Dict]: - """Fetch Rube-related logs from ClickHouse""" - try: - since_str = since_timestamp.strftime("%Y-%m-%d %H:%M:%S") - - # Query for Rube actions - query = f""" - SELECT - id AS log_id, - provider AS app_name, - actionName, - request, - response, - errorRequest AS error, - createdAt AS timestamp, - duration, - userId, - sessionInfo, - connectedAccountId, - projectId - FROM app_db.tool_execution_logs - JOIN app_db.apollo_projects - ON tool_execution_logs.projectId = apollo_projects.id - WHERE createdAt >= '{since_str}' - AND actionName IN ( - 'COMPOSIO_SEARCH_TOOLS', - 'COMPOSIO_CREATE_PLAN', - 'COMPOSIO_MULTI_EXECUTE_TOOL', - 'COMPOSIO_REMOTE_WORKBENCH', - 'COMPOSIO_REMOTE_BASH_TOOL', - 'COMPOSIO_MANAGE_CONNECTIONS', - 'COMPOSIO_WAIT_FOR_CONNECTION' - ) - AND userId LIKE 'user_%' - AND apollo_projects.orgId = '3415d5cb-1dea-455a-a6a4-e66d764703e7' - ORDER BY createdAt ASC - """ - print(query) - - print(f"Fetching Rube logs since {since_str}") - result = self.clickhouse._execute_query(query) - logs = result.get("data", []) - - # Decrypt fields and parse timestamps - decrypted_logs = [] - for log in logs: - decrypted_log = self.clickhouse._decrypt_log_fields(log) - - # Parse timestamp string to datetime object - timestamp_str = decrypted_log.get("timestamp") - if timestamp_str: - try: - # ClickHouse returns datetime in format like "2025-09-01 12:34:56.123456" - if isinstance(timestamp_str, str): - decrypted_log["timestamp"] = datetime.strptime( - timestamp_str.split(".")[ - 0 - ], # Remove microseconds if present - "%Y-%m-%d %H:%M:%S", - ) - elif isinstance(timestamp_str, datetime): - decrypted_log["timestamp"] = timestamp_str - except ValueError as e: - print(f"Failed to parse timestamp '{timestamp_str}': {e}") - - decrypted_logs.append(decrypted_log) - - print(f"Found {len(decrypted_logs)} Rube logs") - return decrypted_logs - except Exception as e: - print(f"Error fetching logs from ClickHouse: {e}") - return [] - - def group_logs_by_user(self, logs: List[Dict]) -> Dict[str, List[Dict]]: - """Group logs by user ID""" - user_logs: Dict[str, List[Dict]] = {} - for log in logs: - user_id = log.get("userId", "unknown") - if user_id not in user_logs: - user_logs[user_id] = [] - user_logs[user_id].append(log) - return user_logs - - def extract_search_json(self, session: Session) -> str: - """ - Extract search tool calls and format into a clean JSON structure. - - Returns JSON string with: - - One entry per individual use case (not concatenated) - - Each entry has a search_id like "usecase_1_1", "usecase_1_2", "overall_1", etc. - - Request format: {"queries": [{"use_case": "...", "known_fields": "..."}], "overall_usecase": "..."} - Response format: {"data": {"results": [{"main_tool_slugs": [...], "related_tool_slugs": [...], "toolkits": [...]}]}} - """ - search_data = [] - search_call_count = 0 - - for i, call in enumerate(session.calls, 1): - action_name = call.get("actionName") - if action_name == "COMPOSIO_SEARCH_TOOLS": - search_call_count += 1 - try: - # Parse request and response - request_raw = call.get("request", "{}") - response_raw = call.get("response", "{}") - - # Handle both string and dict formats - if isinstance(request_raw, str): - request_data = json.loads(request_raw) if request_raw else {} - else: - request_data = request_raw or {} - - if isinstance(response_raw, str): - response_data = json.loads(response_raw) if response_raw else {} - else: - response_data = response_raw or {} - - # Extract overall_usecase if present - overall_usecase = request_data.get("overall_usecase", "") - - # Extract queries from request: {"queries": [{"use_case": "...", "known_fields": "..."}]} - queries = request_data.get("queries", []) - - # Parse response: {"data": {"results": [...]}} - results = [] - connected_apps = [] - - if isinstance(response_data, dict): - data = response_data.get("data", {}) - if isinstance(data, dict): - results = data.get("results", []) - - # Extract connected apps from toolkit_connection_statuses (shared across all queries) - toolkit_connection_statuses = data.get( - "toolkit_connection_statuses", [] - ) - if isinstance(toolkit_connection_statuses, list): - for conn in toolkit_connection_statuses: - if ( - isinstance(conn, dict) - and conn.get("active_connection") is True - ): - toolkit = conn.get("toolkit") - if toolkit and toolkit not in connected_apps: - connected_apps.append(toolkit) - connected_apps.sort() - - # Convert timestamp to string if it's a datetime object - timestamp = call.get("timestamp", "") - if hasattr(timestamp, "isoformat"): - timestamp = timestamp.isoformat() - - # Create entry for overall_usecase if present - if overall_usecase: - # For overall, aggregate toolkits and tools from all results - overall_toolkits = [] - overall_main_tools = [] - overall_related_tools = [] - - for result in results: - if isinstance(result, dict): - # Extract toolkits - result_toolkits = result.get("toolkits", []) - for tk in result_toolkits: - if ( - isinstance(tk, str) - and tk not in overall_toolkits - ): - overall_toolkits.append(tk) - - # Extract main tool slugs - main_tool_slugs = result.get("main_tool_slugs", []) - for slug in main_tool_slugs: - if slug and slug not in overall_main_tools: - overall_main_tools.append(slug) - - # Extract related tool slugs - related_tool_slugs = result.get( - "related_tool_slugs", [] - ) - for slug in related_tool_slugs: - if slug and slug not in overall_related_tools: - overall_related_tools.append(slug) - - overall_toolkits.sort() - - overall_entry = { - "search_id": f"overall_{search_call_count}", - "log_id": call.get("log_id", ""), - "timestamp": str(timestamp), - "request": { - "use_case": overall_usecase, - "known_fields": "", - }, - "response": { - "toolkits": overall_toolkits, - "main_tools": overall_main_tools, - "related_tools": overall_related_tools, - "connectedApps": connected_apps, - }, - } - search_data.append(overall_entry) - - # Create one entry per individual query/usecase - for query_idx, query in enumerate(queries, 1): - if isinstance(query, dict): - use_case = query.get("use_case", "") - known_fields = query.get("known_fields", "") - - # Get corresponding result (results array should match queries array) - result_toolkits = [] - result_main_tools = [] - result_related_tools = [] - cached_plan_id = None - - if query_idx <= len(results): - result = results[query_idx - 1] - if isinstance(result, dict): - # Extract toolkits - toolkits = result.get("toolkits", []) - for tk in toolkits: - if isinstance(tk, str): - result_toolkits.append(tk) - - # Extract main tool slugs - main_tool_slugs = result.get("main_tool_slugs", []) - result_main_tools = [ - slug for slug in main_tool_slugs if slug - ] - - # Extract related tool slugs - related_tool_slugs = result.get( - "related_tool_slugs", [] - ) - result_related_tools = [ - slug for slug in related_tool_slugs if slug - ] - - # Extract cached_plan_id if present - cached_plan_id = result.get("cached_plan_id") - - result_toolkits.sort() - - usecase_entry = { - "search_id": f"usecase_{search_call_count}_{query_idx}", - "log_id": call.get("log_id", ""), - "timestamp": str(timestamp), - "request": { - "use_case": use_case, - "known_fields": known_fields, - }, - "response": { - "toolkits": result_toolkits, - "main_tools": result_main_tools, - "related_tools": result_related_tools, - "connectedApps": connected_apps, - }, - } - - # Add cached_plan_id if present - if cached_plan_id: - usecase_entry["cached_plan_id"] = cached_plan_id - - search_data.append(usecase_entry) - - except Exception: - # If parsing fails, skip this search call - continue - - # Return as compact JSON string - search_json = json.dumps(search_data, separators=(",", ":")) - - # Print formatted version for debugging - if not search_data: - print("\n⚠️ WARNING: No search data extracted! Returning empty JSON.\n") - - return search_json - - def save_to_clickhouse(self, session: Session, analysis: SessionAnalysis) -> bool: - """ - Save session analysis results to ClickHouse. - - Args: - session: Session object with call details - analysis: SessionAnalysis object with recommendations - - Returns: - True if data was saved successfully, False otherwise - """ - try: - # Extract session metadata - log_ids = [call.get("log_id", "") for call in session.calls] - action_sequence = [] - - # Extract entity_id from sessionInfo of any call - entity_id = "" - client_name = session.source or "UNKNOWN" # Get source from session - - search_queries = [] # Collect all search queries - - # Calculate total duration of all tool calls - total_tool_duration_ms = 0 - for call in session.calls: - duration = call.get("duration") - if duration is not None: - try: - total_tool_duration_ms += int(duration) - except (ValueError, TypeError): - pass - - for call in session.calls: - session_info = SessionInfo.from_log(call) - if session_info: - if session_info.entity_id and not entity_id: - entity_id = session_info.entity_id - if session_info.source and not session.source: - client_name = session_info.source - - for call in session.calls: - action_name = call.get("actionName", "unknown") - - # Capture search queries - if action_name == "COMPOSIO_SEARCH_TOOLS": - request = call.get("request", "") - try: - if isinstance(request, str) and request: - request_json = json.loads(request) - params = request_json.get("params", request_json) - if isinstance(params, dict): - # Handle new format with queries array - queries = params.get("queries", []) - overall_usecase = params.get("overall_usecase", "") - - # Add overall usecase if present - if overall_usecase: - search_queries.append( - f"[Overall] {overall_usecase}" - ) - - # Add individual queries - for query in queries: - if isinstance(query, dict): - use_case = query.get("use_case", "") - known_fields = query.get("known_fields", "") - if use_case: - query_str = use_case - if known_fields: - query_str += ( - f" (known_fields: {known_fields})" - ) - search_queries.append(query_str) - except Exception: - pass - - # Build action sequence - if action_name == "COMPOSIO_SEARCH_TOOLS": - action_sequence.append("search") - elif action_name == "COMPOSIO_CREATE_PLAN": - action_sequence.append("plan") - elif action_name == "COMPOSIO_MULTI_EXECUTE_TOOL": - action_sequence.append("multi-execute") - elif action_name == "COMPOSIO_REMOTE_WORKBENCH": - action_sequence.append("workbench") - elif action_name == "COMPOSIO_REMOTE_BASH_TOOL": - action_sequence.append("bash") - elif action_name == "COMPOSIO_MANAGE_CONNECTIONS": - action_sequence.append("manage-conn") - elif action_name == "COMPOSIO_WAIT_FOR_CONNECTION": - action_sequence.append("wait-conn") - else: - action_sequence.append(action_name.lower()) - - # Count recommendations by type and priority - tool_improvement_count = len(analysis.tool_improvements) - new_tool_count = len(analysis.new_tool_suggestions) - new_toolkit_count = len(analysis.new_toolkit_suggestions) - meta_improvement_count = len(analysis.meta_tool_improvements) - - high_priority_count = ( - sum(1 for rec in analysis.tool_improvements if rec.priority >= 8) - + sum(1 for rec in analysis.new_tool_suggestions if rec.priority >= 8) - + sum( - 1 for rec in analysis.new_toolkit_suggestions if rec.priority >= 8 - ) - + sum(1 for rec in analysis.meta_tool_improvements if rec.priority >= 8) - ) - - # Generate analysis ID - analysis_id = str(uuid4()) - - # Prepare session analysis data - session_start_str = ( - session.start_time.strftime("%Y-%m-%d %H:%M:%S") - if session.start_time - else "1970-01-01 00:00:00" - ) - session_end_str = ( - session.end_time.strftime("%Y-%m-%d %H:%M:%S") - if session.end_time - else "1970-01-01 00:00:00" - ) - - # Create unique session ID: sessionId-entityId-startTime - start_time_str = ( - session.start_time.strftime("%Y%m%d_%H%M%S") - if session.start_time - else "19700101_000000" - ) - session_id = str( - session.session_key - ) # Convert to string to avoid escape_string errors - session_id_unique = f"{session_id}-{entity_id}-{start_time_str}" - - # Format search queries as "1) query1, 2) query2" - search_queries_formatted = ( - ", ".join(f"{i+1}) {query}" for i, query in enumerate(search_queries)) - if search_queries - else "" - ) - - # Escape strings for ClickHouse - def escape_string(s: str) -> str: - return s.replace("'", "''").replace("\\", "\\\\") - - # Insert session analysis - # Convert per_usecase_plans from List[UsecasePlan] to JSON string - if analysis.per_usecase_plans: - per_usecase_plans_json = json.dumps( - [plan.model_dump() for plan in analysis.per_usecase_plans] - ) - else: - per_usecase_plans_json = json.dumps([]) - - # plans_changelog is now part of each UsecasePlan, so we don't need a separate field - plans_changelog_json = json.dumps({}) - - # Convert per_usecase_tool_analysis from List[UsecaseToolAnalysis] to JSON string - if analysis.per_usecase_tool_analysis: - per_usecase_tool_analysis_json = json.dumps( - [ - tool_analysis.model_dump() - for tool_analysis in analysis.per_usecase_tool_analysis - ] - ) - else: - per_usecase_tool_analysis_json = json.dumps([]) - - analysis_query = f""" - INSERT INTO app_db.rube_session_analyses ( - id, sessionId, sessionIdUnique, clientName, userId, sessionStartTime, sessionEndTime, - sessionDurationSeconds, toolCallCount, totalToolDurationMs, logIds, actionSequence, searchQueries, - overallUsecase, workflowCategory, workflowSummary, failureCodes, - appsInvolvedStr, toolsInvolvedStr, - wasWorkflowSuccessful, taskDifficulty, workflowQuality, workflowQualityEnum, - overallRationale, hasRecommendations, - confidenceScore, toolImprovementCount, newToolSuggestionCount, - metaToolImprovementCount, highPriorityCount, - searchQuality, searchQualityExplanation, - perUsecasePlans, plansChangelog, perUsecaseToolAnalysis, - hoursLookback, analyzerVersion, searchJson - ) VALUES ( - '{analysis_id}', - '{escape_string(session_id)}', - '{escape_string(session_id_unique)}', - '{escape_string(client_name)}', - '{escape_string(session.user_id)}', - '{session_start_str}', - '{session_end_str}', - {session.duration_seconds()}, - {len(session.calls)}, - {total_tool_duration_ms}, - [{','.join(f"'{escape_string(lid)}'" for lid in log_ids)}], - [{','.join(f"'{escape_string(action)}'" for action in action_sequence)}], - '{escape_string(search_queries_formatted)}', - '{escape_string(analysis.overall_usecase)}', - '{escape_string(analysis.workflow_category)}', - '{escape_string(analysis.workflow_summary)}', - '{escape_string(", ".join(analysis.failure_codes))}', - '{escape_string(", ".join(analysis.apps_involved))}', - '{escape_string(", ".join(analysis.tools_involved))}', - {1 if analysis.was_workflow_successful else 0}, - '{analysis.task_difficulty.name.upper()}', - {analysis.workflow_quality}, - '{analysis.workflow_quality_enum.name.upper()}', - '{escape_string(analysis.overall_rationale)}', - {1 if analysis.has_recommendations else 0}, - {analysis.confidence_score}, - {tool_improvement_count}, - {new_tool_count + new_toolkit_count}, - {meta_improvement_count}, - {high_priority_count}, - '{analysis.search_quality.value}', - '{escape_string(analysis.search_quality_explanation)}', - '{escape_string(per_usecase_plans_json)}', - '{escape_string(plans_changelog_json)}', - '{escape_string(per_usecase_tool_analysis_json)}', - {self.poll_interval_minutes}, - '1.0', - '{escape_string(self.extract_search_json(session))}' - ) - """ - - self.clickhouse._execute_query(analysis_query) - - # Insert recommendations - order_index = 0 - - # Tool improvements - for rec in analysis.tool_improvements: - order_index += 1 - # Use tool_description_change for tool_description_change type, proposed_change for others - proposed_value = ( - rec.tool_description_change - if rec.improvement_type == "tool_description_change" - else rec.proposed_change - ) - - rec_query = f""" - INSERT INTO app_db.rube_learnings ( - analysisId, sessionId, userId, recommendationType, recommendationName, - priority, rationale, orderIndex, wasToolSuccessful, toolName, - improvementType, currentValue, proposedValue, failureCode, appsInvolvedStr - ) VALUES ( - '{analysis_id}', - '{escape_string(session_id)}', - '{escape_string(session.user_id)}', - 'tool_improvement_{escape_string(rec.improvement_type)}', - '{escape_string(rec.recommendation_name)}', - {rec.priority}, - '{escape_string(rec.rationale)}', - {order_index}, - {1 if rec.was_tool_successful else 0}, - '{escape_string(rec.tool_name)}', - '{escape_string(rec.improvement_type)}', - '{escape_string(rec.current_value)}', - '{escape_string(proposed_value)}', - '{escape_string(rec.failure_code)}', - '{escape_string(", ".join(analysis.apps_involved))}' - ) - """ - self.clickhouse._execute_query(rec_query) - - # New tool suggestions - for tool_rec in analysis.new_tool_suggestions: - order_index += 1 - rec_query = f""" - INSERT INTO app_db.rube_learnings ( - analysisId, sessionId, userId, recommendationType, recommendationName, - priority, rationale, orderIndex, toolName, description, expectedUsage, appsInvolvedStr, - requiredApi, toolkit, implementationDifficulty, apiDocs, - existsInComposio, searchFound, capabilityStatus, proxyExecuteAttempted - ) VALUES ( - '{analysis_id}', - '{escape_string(session_id)}', - '{escape_string(session.user_id)}', - 'new_tool', - '{escape_string(tool_rec.recommendation_name)}', - {tool_rec.priority}, - '{escape_string(tool_rec.rationale)}', - {order_index}, - '{escape_string(tool_rec.tool_name)}', - '{escape_string(tool_rec.description)}', - '{escape_string(tool_rec.expected_usage)}', - '{escape_string(", ".join(analysis.apps_involved))}', - {f"'{escape_string(tool_rec.required_api)}'" if tool_rec.required_api else 'NULL'}, - {f"'{escape_string(tool_rec.toolkit)}'" if tool_rec.toolkit else 'NULL'}, - {f"'{escape_string(tool_rec.implementation_difficulty)}'" if tool_rec.implementation_difficulty else 'NULL'}, - {f"'{escape_string(tool_rec.api_docs)}'" if tool_rec.api_docs else 'NULL'}, - {1 if tool_rec.exists_in_composio else 0 if tool_rec.exists_in_composio is not None else 'NULL'}, - {1 if tool_rec.search_found else 0 if tool_rec.search_found is not None else 'NULL'}, - {f"'{escape_string(tool_rec.capability_status)}'" if tool_rec.capability_status else 'NULL'}, - {1 if tool_rec.proxy_execute_attempted else 0 if tool_rec.proxy_execute_attempted is not None else 'NULL'} - ) - """ - self.clickhouse._execute_query(rec_query) - - # New toolkit suggestions - for toolkit_rec in analysis.new_toolkit_suggestions: - order_index += 1 - rec_query = f""" - INSERT INTO app_db.rube_learnings ( - analysisId, sessionId, userId, recommendationType, recommendationName, - priority, rationale, orderIndex, toolkit, apiDocs, expectedUsage, appsInvolvedStr - ) VALUES ( - '{analysis_id}', - '{escape_string(session_id)}', - '{escape_string(session.user_id)}', - 'new_toolkit', - '{escape_string(toolkit_rec.recommendation_name)}', - {toolkit_rec.priority}, - '{escape_string(toolkit_rec.reasoning)}', - {order_index}, - '{escape_string(toolkit_rec.toolkit_name)}', - {f"'{escape_string(toolkit_rec.api_docs_url)}'" if toolkit_rec.api_docs_url else 'NULL'}, - '{escape_string(toolkit_rec.expected_value)}', - '{escape_string(", ".join(analysis.apps_involved))}' - ) - """ - self.clickhouse._execute_query(rec_query) - - # Meta tool improvements - for meta_rec in analysis.meta_tool_improvements: - order_index += 1 - rec_query = f""" - INSERT INTO app_db.rube_learnings ( - analysisId, sessionId, userId, recommendationType, recommendationName, - priority, rationale, orderIndex, wasToolSuccessful, metaTool, - actualUsage, improvement, appsInvolvedStr - ) VALUES ( - '{analysis_id}', - '{escape_string(session_id)}', - '{escape_string(session.user_id)}', - 'meta_improvement', - '{escape_string(meta_rec.recommendation_name)}', - {meta_rec.priority}, - '{escape_string(meta_rec.rationale)}', - {order_index}, - {1 if meta_rec.was_tool_successful else 0}, - '{escape_string(meta_rec.meta_tool)}', - '{escape_string(meta_rec.actual_usage)}', - '{escape_string(meta_rec.improvement)}', - '{escape_string(", ".join(analysis.apps_involved))}' - ) - """ - self.clickhouse._execute_query(rec_query) - - print( - f"Successfully saved session analysis to ClickHouse (analysisId: {analysis_id})" - ) - return True - - except Exception as e: - print(f"Error saving session analysis to ClickHouse: {e}") - return False - - def _log_session_details(self, session: Session, session_num: int): - """Log details for a specific session""" - # Get short form of action names - action_sequence = [] - - for call in session.calls: - action_name = call.get("actionName", "unknown") - - if action_name == "COMPOSIO_SEARCH_TOOLS": - action_sequence.append("search") - elif action_name == "COMPOSIO_CREATE_PLAN": - action_sequence.append("plan") - elif action_name == "COMPOSIO_MULTI_EXECUTE_TOOL": - action_sequence.append("multi-execute") - elif action_name == "COMPOSIO_REMOTE_WORKBENCH": - action_sequence.append("workbench") - elif action_name == "COMPOSIO_REMOTE_BASH_TOOL": - action_sequence.append("bash") - elif action_name == "COMPOSIO_MANAGE_CONNECTIONS": - action_sequence.append("manage-conn") - elif action_name == "COMPOSIO_WAIT_FOR_CONNECTION": - action_sequence.append("wait-conn") - else: - action_sequence.append(action_name.lower()) - - sequence_str = " -> ".join(action_sequence) - duration_str = ( - f"{session.duration_seconds():.1f}s" - if session.duration_seconds() > 0 - else "0s" - ) - - session_display_id = session.session_key - print( - f"Session {session_num} ({session_display_id}, User: {session.user_id}): {sequence_str} (duration: {duration_str})" - ) - - # Get log IDs for the session - log_ids = [call.get("log_id", "?") for call in session.calls] - log_ids_str = ", ".join(log_ids) - print(f" Log IDs: {log_ids_str}") - - async def analyze(self): - """ - Main analysis function. - - Processes logs and analyzes completed sessions. - In polling mode, maintains state across cycles. - """ - # Show what we're analyzing - if self.target_log_id: - self.console.print( - f"[bold]Analyzing Rube sessions starting with log ID: {self.target_log_id}[/bold]" - ) - else: - lookback_minutes = ( - self.initial_lookback_minutes - if not self.last_log_timestamp - else self.poll_interval_minutes - ) - self.console.print( - f"[bold]Analyzing Rube sessions from the last {lookback_minutes} minutes[/bold]" - ) - self.console.print(f"[dim]Max sessions globally: {self.max_sessions}[/dim]") - - # Calculate since_timestamp based on last log or initial lookback - if self.last_log_timestamp: - since_timestamp = self.last_log_timestamp - print( - f"Continuing from last log timestamp: {since_timestamp.strftime('%Y-%m-%d %H:%M:%S')}" - ) - else: - # First run - look back by initial lookback period - since_timestamp = datetime.utcnow() - timedelta( - minutes=self.initial_lookback_minutes - ) - print(f"First run - looking back {self.initial_lookback_minutes} minutes") - logs = self.fetch_rube_logs(since_timestamp) - - if not logs: - self.console.print( - "[yellow]No Rube logs found in the specified time period[/yellow]" - ) - return # No new logs, keep the same timestamp - - # Update last_log_timestamp to the timestamp of the last log (query returns in ASC order) - if logs[-1].get("timestamp"): - self.last_log_timestamp = logs[-1]["timestamp"] - if self.last_log_timestamp: - print( - f"Updated last log timestamp to: {self.last_log_timestamp.strftime('%Y-%m-%d %H:%M:%S')}" - ) - - print(f"Processing {len(logs)} logs...") - - # Process all logs at once - detector handles grouping by user internally - # Returns only completed sessions (those that have timed out) - completed_sessions = self.session_detector.detect_sessions(logs) - - if not completed_sessions: - print("No completed sessions found in this cycle") - # Show active sessions status - active_count = sum( - len(sessions) - for sessions in self.session_detector.active_sessions.values() - ) - if active_count > 0: - print(f"Active sessions still in progress: {active_count}") - return - - print( - f"\nFound {len(completed_sessions)} completed sessions ready for analysis" - ) - - # If target_log_id is specified, filter sessions - if self.target_log_id: - matching_sessions = [] - for session in completed_sessions: - # Check if ANY log in the session has the target log ID - has_target_log = any( - call.get("log_id") == self.target_log_id for call in session.calls - ) - if has_target_log: - matching_sessions.append(session) - - if not matching_sessions: - print(f"No session found containing log ID {self.target_log_id}") - return - - completed_sessions = matching_sessions - print( - f"Found {len(matching_sessions)} session(s) containing log ID {self.target_log_id}" - ) - - # Filter sessions with more than 1 tool call - completed_sessions = [ - session for session in completed_sessions if len(session.calls) > 1 - ] - - if not completed_sessions: - print("No sessions with more than 1 tool call found") - return - - print(f"Found {len(completed_sessions)} sessions with > 1 tool call") - - # Sort sessions by start time - completed_sessions.sort(key=lambda s: s.start_time or datetime.min) - - # Apply max sessions limit - simple truncation - if len(completed_sessions) > self.max_sessions: - all_sessions = completed_sessions[: self.max_sessions] - print( - f"Limiting to {self.max_sessions} sessions (out of {len(completed_sessions)} available)" - ) - else: - all_sessions = completed_sessions - - print(f"\nAnalyzing {len(all_sessions)} sessions") - - # Log details for sessions we're analyzing - print("\nSession details for analysis:") - for i, session in enumerate(all_sessions, 1): - self._log_session_details(session, i) - - # Import random here for Slack sampling - import random - - # Track sessions sent to Slack - slack_sent_count = 0 - - # Function to analyze a single session and optionally send to Slack - def analyze_and_send_session(session, session_num, total_sessions): - nonlocal slack_sent_count - - user_id = session.user_id - import threading - import traceback - - thread_id = threading.current_thread().name - - try: - print( - f"\n[Thread-{thread_id}] STARTING session {session_num}/{total_sessions} for user {user_id}" - ) - - # Get recommendations for this session - # Run the async function in a new event loop since we're in a thread - analysis = asyncio.run(self.llm_analyzer.analyze_session(session)) - - print( - f"[Thread-{thread_id}] COMPLETED session analysis {session_num}/{total_sessions}" - ) - - if analysis: - # ALWAYS run tool gap analysis - print( - f"🔍 [Thread-{thread_id}] Running tool gap analysis for session {session.session_key}" - ) - tool_gap_result = None - try: - tool_gap_result = asyncio.run( - run_tool_gap_analysis(session, analysis, self.clickhouse) - ) - if tool_gap_result: - print( - f"✅ [Thread-{thread_id}] Tool gap analysis complete: search_quality={tool_gap_result.search_quality.value}, new_tools={len(tool_gap_result.new_tool_suggestions)}" - ) - except Exception as e: - print(f"⚠️ [Thread-{thread_id}] Tool gap analysis failed: {e}") - # Continue with defaults if tool gap analysis fails - - # Combine into full SessionAnalysis - combined_analysis = SessionAnalysis( - # Copy all fields from primary_analysis - overall_usecase=analysis.overall_usecase, - workflow_category=analysis.workflow_category, - workflow_summary=analysis.workflow_summary, - apps_involved=analysis.apps_involved, - tools_involved=analysis.tools_involved, - workflow_quality=analysis.workflow_quality, - was_workflow_successful=analysis.was_workflow_successful, - workflow_quality_enum=analysis.workflow_quality_enum, - task_difficulty=analysis.task_difficulty, - failure_codes=analysis.failure_codes, - overall_rationale=analysis.overall_rationale, - confidence_score=analysis.confidence_score, - tool_improvements=analysis.tool_improvements, - meta_tool_improvements=analysis.meta_tool_improvements, - per_usecase_plans=analysis.per_usecase_plans, - has_recommendations=analysis.has_recommendations, - # Add tool gap analysis results - per_usecase_tool_analysis=( - tool_gap_result.per_usecase_tool_analysis - if tool_gap_result - else [] - ), - new_tool_suggestions=( - tool_gap_result.new_tool_suggestions - if tool_gap_result - else [] - ), - new_toolkit_suggestions=( - tool_gap_result.new_toolkit_suggestions - if tool_gap_result - else [] - ), - search_quality=( - tool_gap_result.search_quality - if tool_gap_result - else SearchQuality.ACCEPTABLE - ), - search_quality_explanation=( - tool_gap_result.search_quality_explanation - if tool_gap_result - else "Tool gap analysis failed to complete" - ), - ) - - # Update has_recommendations if tool gap found new suggestions - if tool_gap_result and ( - tool_gap_result.new_tool_suggestions - or tool_gap_result.new_toolkit_suggestions - ): - combined_analysis.has_recommendations = True - - # Pass session number info for display - self.formatter.display_analysis( - user_id, - session_num, - total_sessions, - combined_analysis, - session.session_id, - ) - - # Save to ClickHouse - try: - if SAVE_TO_CLICKHOUSE: - self.save_to_clickhouse(session, combined_analysis) - except Exception as e: - print(f"Failed to save session analysis to ClickHouse: {e}") - - # Randomly decide whether to send to Slack - if ( - SEND_TO_SLACK - and self.slack_notifier - and random.random() * 100 < self.slack_sampling_percent - ): - try: - self.slack_notifier.send_session_analysis( - session, combined_analysis, session_num, total_sessions - ) - slack_sent_count += 1 - print(f"Sent session {session_num} to Slack") - except Exception as e: - print(f"Failed to send session {session_num} to Slack: {e}") - - return True - return False - - except Exception as e: - # Top-level catch to ensure thread doesn't crash - error_msg = f"{str(e)}\n{traceback.format_exc()}" - print( - f"\n❌ [Thread-{thread_id}] FAILED session {session_num}/{total_sessions}: {e}" - ) - print(f"Full traceback:\n{traceback.format_exc()}") - - # Send failure notification to Slack - if SEND_TO_SLACK and self.slack_notifier: - try: - self.slack_notifier.send_session_failure( - session, error_msg, session_num, total_sessions - ) - except Exception as slack_error: - print( - f"Failed to send failure notification to Slack: {slack_error}" - ) - - return False - - # Use ThreadPoolExecutor with max_workers=2 for parallelism - successful_analyses = 0 - with ThreadPoolExecutor(max_workers=4) as executor: - # Submit all tasks - futures = [] - for i, session in enumerate(all_sessions, 1): - future = executor.submit( - analyze_and_send_session, session, i, len(all_sessions) - ) - futures.append(future) - - # Wait for all to complete and count successes - for future in as_completed(futures): - try: - if future.result(): - successful_analyses += 1 - except Exception as e: - print(f"Failed to process session: {e}") - - print( - f"\nCompleted analysis of {successful_analyses}/{len(all_sessions)} sessions" - ) - if SEND_TO_SLACK: - print( - f"Sent {slack_sent_count} sessions to Slack ({self.slack_sampling_percent}% sampling)" - ) - - -def parse_arguments() -> argparse.Namespace: - """Parse command line arguments""" - parser = argparse.ArgumentParser( - description="Analyze Rube user sessions and generate insights" - ) - parser.add_argument( - "--log-id", - type=str, - help="Analyze only sessions beginning with this specific log ID", - ) - parser.add_argument( - "--test-session", - type=str, - help="Test mode: Analyze just this one session ID and exit (e.g., type_user_01K2KYB1M6JASCB0MW67K9J716)", - ) - parser.add_argument( - "--max-sessions", - type=int, - default=10000, - help="Maximum total number of sessions to analyze globally across all users in each poll cycle", - ) - parser.add_argument( - "--poll-interval", - type=int, - default=5, - help="Poll interval in minutes. If specified, runs continuously (default: single run)", - ) - parser.add_argument( - "--initial-lookback", - type=int, - default=5, - help="Minutes to look back on first run (default: 5)", - ) - parser.add_argument( - "--no-slack", - action="store_true", - help="Disable sending results to Slack", - ) - parser.add_argument( - "--no-clickhouse", - action="store_true", - help="Disable saving results to ClickHouse", - ) - parser.add_argument( - "--slack-sampling-percent", - type=float, - default=5.0, - help="Percentage of sessions to randomly sample for Slack (default: 5.0)", - ) - return parser.parse_args() - - -async def run_single_analysis(args, analyzer: RubeSessionAnalyzer) -> int: - """Run a single analysis cycle with the given analyzer""" - - try: - await analyzer.analyze() - return 0 - except Exception as e: - print(f"Analysis failed: {e}") - return 1 - - -async def test_single_session(session_id: str) -> int: - """Test mode: Analyze just one session by ID and exit""" - print(f"\n{'='*60}") - print(f"TEST MODE: Analyzing session {session_id}") - print(f"{'='*60}\n") - - # Create minimal analyzer - analyzer = RubeSessionAnalyzer( - poll_interval_minutes=5, - initial_lookback_minutes=10080, # 7 days lookback - last_log_timestamp=None, - target_log_id=session_id, - max_sessions=1, - slack_sampling_percent=0, # No slack in test mode - ) - - # Fetch logs for this specific session only - print(f"📥 Fetching logs for session {session_id}...") - - query = f""" - SELECT - id AS log_id, - provider AS app_name, - actionName, - request, - response, - errorRequest AS error, - createdAt AS timestamp, - duration, - userId, - sessionInfo, - connectedAccountId, - projectId - FROM app_db.tool_execution_logs - WHERE createdAt >= now() - INTERVAL 1 DAY - AND JSONExtractString(sessionInfo, 'session_id') = '{session_id}' - ORDER BY createdAt ASC - """ - - result = analyzer.clickhouse._execute_query(query) - raw_logs = result.get("data", []) - - if not raw_logs: - print(f"❌ No logs found for session {session_id} in last 7 days") - return 1 - - # Decrypt and process logs (same as fetch_rube_logs) - from datetime import datetime - - logs = [] - for log in raw_logs: - decrypted_log = analyzer.clickhouse._decrypt_log_fields(log) - - # Parse timestamp - timestamp_str = decrypted_log.get("timestamp") - if timestamp_str: - try: - if isinstance(timestamp_str, str): - decrypted_log["timestamp"] = datetime.strptime( - timestamp_str.split(".")[0], - "%Y-%m-%d %H:%M:%S", - ) - elif isinstance(timestamp_str, datetime): - decrypted_log["timestamp"] = timestamp_str - except ValueError as e: - print(f"Failed to parse timestamp '{timestamp_str}': {e}") - - logs.append(decrypted_log) - - print(f"✅ Found {len(logs)} logs for this session") - - # Build session directly from these logs - print("\n🔍 Building session from logs...") - detector = SessionDetector() - all_sessions = detector.detect_sessions(logs) - - if not all_sessions: - print("❌ Could not build session from logs") - return 1 - - # Should only be one session - target_session = all_sessions[0] - print(f"✅ Session built: {target_session.session_key}") - print(f" - Calls: {len(target_session.calls)}") - print(f" - Duration: {target_session.duration_seconds():.1f}s") - print(f" - User: {target_session.user_id}") - - # Analyze the session - print("\n🤖 Running primary LLM analysis...") - print(" This may take 30-60 seconds (calling GPT-5)...") - import time - - search_json = analyzer.extract_search_json(target_session) - import json - - print("SEARCH JSON:") - print(json.dumps(json.loads(search_json), indent=2, ensure_ascii=False)) - - start_time = time.time() - llm_analyzer = LLMAnalyzer() - primary_analysis = await llm_analyzer.analyze_session(target_session) - - if not primary_analysis: - print("❌ Primary analysis failed") - return 1 - - elapsed = time.time() - start_time - print(f"✅ Primary analysis complete (took {elapsed:.1f}s)") - print(f" - Use case: {primary_analysis.overall_usecase[:100]}...") - print(f" - Workflow successful: {primary_analysis.was_workflow_successful}") - print( - f" - Recommendations: {len(primary_analysis.tool_improvements)} tool, {len(primary_analysis.meta_tool_improvements)} meta" - ) - - # Run tool gap analysis - print("\n🔍 Running tool gap analysis...") - print(" This may take 30-90 seconds (calling GPT-5 + web search)...") - start_time = time.time() - tool_gap_result = await run_tool_gap_analysis( - target_session, primary_analysis, analyzer.clickhouse - ) - - if not tool_gap_result: - print("⚠️ Tool gap analysis failed, using defaults") - # Create a default ToolGapAnalysisOutput - from app_tester.rube_learning import ToolGapAnalysisOutput - - tool_gap_result = ToolGapAnalysisOutput( - per_usecase_tool_analysis=[], - search_quality=SearchQuality.ACCEPTABLE, - search_quality_explanation="Tool gap analysis failed", - new_tool_suggestions=[], - new_toolkit_suggestions=[], - ) - - elapsed = time.time() - start_time - print(f"✅ Tool gap analysis complete (took {elapsed:.1f}s)") - print(f" - Search quality: {tool_gap_result.search_quality.value}") - print(f" - New tool suggestions: {len(tool_gap_result.new_tool_suggestions)}") - - # Combine results - print("\n🔗 Combining results...") - combined_analysis = SessionAnalysis( - overall_usecase=primary_analysis.overall_usecase, - workflow_category=primary_analysis.workflow_category, - workflow_summary=primary_analysis.workflow_summary, - apps_involved=primary_analysis.apps_involved, - tools_involved=primary_analysis.tools_involved, - workflow_quality=primary_analysis.workflow_quality, - was_workflow_successful=primary_analysis.was_workflow_successful, - workflow_quality_enum=primary_analysis.workflow_quality_enum, - task_difficulty=primary_analysis.task_difficulty, - failure_codes=primary_analysis.failure_codes, - overall_rationale=primary_analysis.overall_rationale, - confidence_score=primary_analysis.confidence_score, - tool_improvements=primary_analysis.tool_improvements, - meta_tool_improvements=primary_analysis.meta_tool_improvements, - per_usecase_plans=primary_analysis.per_usecase_plans, - has_recommendations=primary_analysis.has_recommendations, - per_usecase_tool_analysis=tool_gap_result.per_usecase_tool_analysis, - new_tool_suggestions=tool_gap_result.new_tool_suggestions, - new_toolkit_suggestions=tool_gap_result.new_toolkit_suggestions, - search_quality=tool_gap_result.search_quality, - search_quality_explanation=tool_gap_result.search_quality_explanation, - ) - - # Display results - print(f"\n{'='*60}") - print("ANALYSIS RESULTS") - print(f"{'='*60}\n") - - formatter = AnalysisFormatter(Console()) - formatter.display_analysis( - target_session.user_id, 1, 1, combined_analysis, target_session.session_id - ) - - print(f"\n{'='*60}") - print("✅ TEST COMPLETE") - print(f"{'='*60}\n") - - return 0 - - -async def main(): - """Main entry point""" - args = parse_arguments() - - # Set global flags based on command line arguments - global SEND_TO_SLACK, SAVE_TO_CLICKHOUSE - SEND_TO_SLACK = not args.no_slack - SAVE_TO_CLICKHOUSE = not args.no_clickhouse - - # Validate environment variables - required_env_vars = [ - "BACKEND_READONLY_URL", - "CLICKHOUSE_USERNAME", - "CLICKHOUSE_PASSWORD", - "OPENAI_API_KEY", - ] - missing_vars = [var for var in required_env_vars if not os.getenv(var)] - - if missing_vars: - print(f"Missing required environment variables: {', '.join(missing_vars)}") - return 1 - - # TEST MODE: Analyze just one session and exit - if args.test_session: - return await test_single_session(args.test_session) - - # Create analyzer instance once (maintains state across polling cycles) - analyzer = RubeSessionAnalyzer( - poll_interval_minutes=args.poll_interval - or 5, # Default to 5 minutes poll interval - initial_lookback_minutes=args.initial_lookback, # Lookback period for first run - last_log_timestamp=None, # Will look back initial_lookback minutes on first run - target_log_id=args.log_id, - max_sessions=args.max_sessions, - slack_sampling_percent=args.slack_sampling_percent, - ) - - # Always run at least once - if args.poll_interval: - print( - f"Starting continuous analysis with {args.poll_interval} minute intervals" - ) - else: - print("Starting single analysis run") - - # Run continuous polling - cycle_count = 0 - while True: - cycle_count += 1 - cycle_start_time = time.time() - current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - - print(f"=== Analysis Cycle {cycle_count} started at {current_time} ===") - - try: - # Run the analysis - await run_single_analysis(args, analyzer) - print(f"Analysis cycle {cycle_count} completed successfully") - except KeyboardInterrupt: - print("\nReceived interrupt signal. Stopping.") - break - except Exception as e: - print(f"Analysis cycle {cycle_count} failed: {e}") - if cycle_count == 1: - # First run failed, exit - return 1 - print("Continuing to next cycle...") - - # If this was a single run (no continuous polling), exit after first cycle - if not args.poll_interval: - break - - # Calculate how long the cycle took - cycle_duration = time.time() - cycle_start_time - cycle_duration_minutes = cycle_duration / 60 - - # Calculate sleep time to maintain total interval - sleep_time = max(0, ((args.poll_interval or 0) * 60) - cycle_duration) - sleep_minutes = sleep_time / 60 - - if sleep_time > 0: - next_cycle_time = datetime.fromtimestamp(time.time() + sleep_time).strftime( - "%Y-%m-%d %H:%M:%S" - ) - print( - f"Cycle took {cycle_duration_minutes:.1f} minutes. Sleeping {sleep_minutes:.1f} minutes until next cycle at {next_cycle_time}" - ) - try: - await asyncio.sleep(sleep_time) - except KeyboardInterrupt: - print("\nReceived interrupt signal during sleep. Stopping.") - break - else: - print( - f"Warning: Cycle took {cycle_duration_minutes:.1f} minutes, which exceeds the {args.poll_interval} minute interval!" - ) - print("Starting next cycle immediately...") - - print() # Empty line for readability - - return 0 - - -if __name__ == "__main__": - sys.exit(asyncio.run(main())) diff --git a/app_tester/common/composio_client.py b/app_tester/common/composio_client.py new file mode 100644 --- /dev/null +++ b/app_tester/common/composio_client.py @@ -0,0 +1,39 @@ +"""Lightweight REST API client for Composio - no SDK needed.""" + +import os +from typing import Any, Dict, Optional + +import requests + + +class CustomComposioAPIClient: + """Direct REST API client for Composio.""" + + BASE_URL = "https://backend.composio.dev/api/v3" + + def __init__(self, api_key: Optional[str] = None): + self.api_key = api_key or os.getenv("COMPOSIO_API_KEY") + if not self.api_key: + raise ValueError("COMPOSIO_API_KEY environment variable required") + + self.headers = {"x-api-key": self.api_key, "Content-Type": "application/json"} + + def get_tool_schema(self, tool_slug: str) -> Dict[str, Any]: + """Fetch tool schema from Composio API.""" + url = f"{self.BASE_URL}/tools/{tool_slug}" + params = {"toolkit_versions": "latest"} + + response = requests.get(url, headers=self.headers, params=params) + response.raise_for_status() + return response.json() + + def execute_tool( + self, tool_slug: str, arguments: Dict[str, Any], user_id: str = "default" + ) -> Dict[str, Any]: + """Execute a Composio tool via REST API.""" + url = f"{self.BASE_URL}/tools/execute/{tool_slug}" + payload = {"arguments": arguments, "user_id": user_id} + + response = requests.post(url, headers=self.headers, json=payload) + response.raise_for_status() + return response.json() diff --git a/app_tester/common/generic_agent.py b/app_tester/common/generic_agent.py --- a/app_tester/common/generic_agent.py +++ b/app_tester/common/generic_agent.py @@ -160,7 +160,7 @@ def is_available(self) -> bool: return False def _convert_tools_for_openai(self, config: AgentConfig) -> list[Any]: - """Convert GenericTool instances to OpenAI format and add capability tools.""" + """Convert any tool type to OpenAI format and add capability tools.""" converted_tools: list[Any] = [] # Add capability-based tools first @@ -204,12 +204,10 @@ def _convert_tools_for_openai(self, config: AgentConfig) -> list[Any]: except ImportError: print("⚠️ File management tools not available") - # Convert GenericTool instances only + # Convert GenericTools to OpenAI format if config.tools: for tool in config.tools: - # Convert GenericTool to OpenAI format - openai_tool = tool.to_openai_tool() - converted_tools.append(openai_tool) + converted_tools.append(tool.to_openai_tool()) return converted_tools @@ -282,27 +280,11 @@ def __init__(self): def _convert_tools_for_claude( self, tools: t.Optional[List[GenericTool]] ) -> List[SdkMcpTool[t.Any]]: - """Convert GenericTool instances to Claude MCP format.""" + """Convert GenericTools to Claude MCP format.""" if not tools: return [] - claude_tools = [] - failed_conversions = [] - - for tool in tools: - try: - claude_tool = tool.to_claude_tool() - claude_tools.append(claude_tool) - except Exception as e: - failed_conversions.append( - f"{tool.definition.name} (conversion failed: {e})" - ) - - if failed_conversions: - print( - f"⚠️ Claude tool conversion failures: {', '.join(failed_conversions)}" - ) - + claude_tools = [tool.to_claude_tool() for tool in tools] print(f"🔧 Claude tools: Using {len(claude_tools)} custom MCP tools") return claude_tools @@ -337,7 +319,9 @@ def create_agent(self, config: AgentConfig) -> t.Any: for tool in converted_tools: if hasattr(tool, "name"): # Use MCP tool ID format: mcp____ - mcp_tool_id = f"mcp__custom__{tool.name}" + # Normalize tool name: replace spaces with underscores to match Claude's internal behavior + normalized_name = tool.name.replace(" ", "_") + mcp_tool_id = f"mcp__custom__{normalized_name}" allowed_tools.append(mcp_tool_id) print(f"🔧 Claude: Added custom tool: {tool.name} (ID: {mcp_tool_id})") diff --git a/app_tester/common/generic_tools.py b/app_tester/common/generic_tools.py --- a/app_tester/common/generic_tools.py +++ b/app_tester/common/generic_tools.py @@ -67,16 +67,40 @@ class GenericTool: OpenAI and Claude agent providers. """ - def __init__(self, definition: GenericToolDefinition): + def __init__( + self, + definition: GenericToolDefinition, + _composio_metadata: Optional[Dict[str, Any]] = None, + ): self.definition = definition - self._openai_tool: Optional["FunctionTool"] = None + self._composio_metadata = ( + _composio_metadata # Store Composio metadata for special handling + ) + self._openai_tool: Optional["FunctionTool"] = ( + None # Can be FunctionTool or OpenAI dict + ) self._claude_tool: Optional["SdkMcpTool[Any]"] = None + def is_composio_tool(self) -> bool: + """Check if this tool wraps a Composio tool.""" + return self._composio_metadata is not None + def to_openai_tool(self): """Convert to OpenAI function_tool format.""" if self._openai_tool is not None: return self._openai_tool + # Special handling for Composio tools + if self.is_composio_tool(): + from agents import function_tool + + # Disable strict mode for Composio tools (they have additionalProperties) + self._openai_tool = function_tool( + self.definition.function, strict_mode=False + ) + return self._openai_tool + + # Regular custom tool conversion (UNCHANGED) from agents import function_tool # OpenAI can use the function directly with @function_tool @@ -88,6 +112,12 @@ def to_claude_tool(self): if self._claude_tool is not None: return self._claude_tool + # Special handling for Composio tools + if self.is_composio_tool(): + self._claude_tool = self._create_composio_mcp_wrapper() + return self._claude_tool + + # Regular custom tool conversion (UNCHANGED) from claude_agent_sdk import tool # Build input schema for Claude (JSON Schema format) @@ -153,6 +183,158 @@ async def claude_wrapper(args): self._claude_tool = claude_tool return self._claude_tool + def _create_composio_mcp_wrapper(self) -> "SdkMcpTool[Any]": + """Create MCP wrapper for Composio tool that preserves modifiers.""" + import json + + from claude_agent_sdk import SdkMcpTool + + # Build schema from parameters + properties = {} + required = [] + for param in self.definition.parameters: + properties[param.name] = { + "type": param.type.value, + "description": param.description, + } + if param.required: + required.append(param.name) + + schema = {"type": "object", "properties": properties, "required": required} + + # Create handler that calls the wrapped function + async def mcp_handler(args: dict) -> dict: + try: + # Call wrapped function (which calls composio.tools.execute with modifiers) + result = self.definition.function(**args) + + # Format result for Claude + if isinstance(result, dict): + content = [{"type": "text", "text": json.dumps(result, indent=2)}] + elif isinstance(result, str): + content = [{"type": "text", "text": result}] + else: + content = [{"type": "text", "text": str(result)}] + + return {"content": content} + except Exception as e: + error_content = [{"type": "text", "text": f"Error: {str(e)}"}] + return {"content": error_content, "is_error": True} + + return SdkMcpTool( + name=self.definition.name, + description=self.definition.description, + input_schema=schema, + handler=mcp_handler, + ) + + @classmethod + def from_composio( + cls, + tool_slug: str, + user_id: str = "default", + api_key: Optional[str] = None, + ) -> "GenericTool": + """ + Create GenericTool from Composio tool using REST API. + + Args: + tool_slug: Tool identifier (e.g., 'FIRECRAWL_SCRAPE') + user_id: User ID for executing the tool + api_key: Optional Composio API key (defaults to COMPOSIO_API_KEY env var) + + Returns: + GenericTool wrapping the Composio tool + + Example: + ```python + from app_tester.common.generic_tools import GenericTool + + tool = GenericTool.from_composio( + tool_slug="FIRECRAWL_SCRAPE", + user_id="default" + ) + ``` + """ + from app_tester.common.composio_client import CustomComposioAPIClient + + # Get tool schema from REST API + client = CustomComposioAPIClient(api_key) + schema = client.get_tool_schema(tool_slug) + + # Extract parameters from API response + parameters = cls._extract_composio_parameters(schema) + + # Create execution wrapper + def composio_execute_wrapper(**kwargs) -> Any: + """Wrapper that calls Composio REST API.""" + result = client.execute_tool( + tool_slug=tool_slug, arguments=kwargs, user_id=user_id + ) + # Just return the full response - let Claude parse it + return result + + # Create definition + definition = GenericToolDefinition( + name=schema["name"], + description=schema["description"], + parameters=parameters, + function=composio_execute_wrapper, + return_type=None, + ) + + # Store metadata for provider conversions + return cls( + definition=definition, + _composio_metadata={ + "client": client, + "slug": tool_slug, + "user_id": user_id, + "schema": schema, # Raw API response for to_openai_tool() + }, + ) + + @classmethod + def _extract_composio_parameters( + cls, schema: Dict[str, Any] + ) -> List[GenericParameter]: + """Extract GenericParameter list from Composio API response.""" + parameters = [] + + input_params = schema.get("input_parameters", {}) + properties = input_params.get("properties", {}) + required = input_params.get("required", []) + + for param_name, param_def in properties.items(): + # Map JSON schema type to ParameterType + json_type = param_def.get("type", "string") + if json_type == "string": + param_type = ParameterType.STRING + elif json_type == "integer": + param_type = ParameterType.INTEGER + elif json_type == "number": + param_type = ParameterType.NUMBER + elif json_type == "boolean": + param_type = ParameterType.BOOLEAN + elif json_type == "array": + param_type = ParameterType.ARRAY + elif json_type == "object": + param_type = ParameterType.OBJECT + else: + param_type = ParameterType.STRING + + parameters.append( + GenericParameter( + name=param_name, + type=param_type, + description=param_def.get("description", ""), + required=param_name in required, + default=param_def.get("default"), + ) + ) + + return parameters + # Decorator for easy tool definition def generic_tool( diff --git a/app_tester/common/git_utils.py b/app_tester/common/git_utils.py --- a/app_tester/common/git_utils.py +++ b/app_tester/common/git_utils.py @@ -172,25 +172,48 @@ def _push_branch(cls) -> None: @classmethod def _add_label(cls, pr_url: str, label: str) -> None: - stdout, stderr = cls._run_command(f"gh pr edit {pr_url} --add-label '{label}'") - if stderr: - logger.error("Failed to add label '{}' to PR {}: {}", label, pr_url, stderr) - else: - logger.info("Successfully added label '{}' to PR {}", label, pr_url) + # Ensure label exists before adding (delegate to _add_labels) + cls._add_labels(pr_url, [label]) + + @classmethod + def _ensure_labels_exist(cls, labels: list[str]) -> None: + """Ensure all labels exist in the repository, create if missing.""" + if not labels: + return + + # Use gh CLI to create labels (idempotent - skips if exists) + for label in labels: + stdout, stderr = cls._run_command( + f'gh label create "{label}" --color 0075ca --description "Auto-created label: {label}" --repo {cls.owner}/{cls.repo} --force' + ) + # gh label create with --force updates if exists, so no error on duplicate + if stderr and "already exists" not in stderr.lower(): + logger.debug(f"Label '{label}' status: {stderr.strip()}") + else: + logger.debug(f"Ensured label '{label}' exists") @classmethod def _add_labels(cls, pr_url: str, labels: list[str]) -> None: - """Add multiple labels to a PR""" + """Add multiple labels to a PR sequentially (ensures labels exist first)""" if not labels: return - labels_str = ",".join(labels) - stdout, stderr = cls._run_command( - f'gh pr edit {pr_url} --add-label "{labels_str}"' - ) - if stderr: - logger.error("Failed to add labels {} to PR {}: {}", labels, pr_url, stderr) - else: - logger.info("Successfully added labels {} to PR {}", labels, pr_url) + + # Ensure all labels exist before adding + cls._ensure_labels_exist(labels) + + # Add labels one by one to avoid gh CLI comma-parsing issues + for label in labels: + stdout, stderr = cls._run_command( + f'gh pr edit {pr_url} --add-label "{label}"' + ) + if stderr: + logger.warning( + "Failed to add label '{}' to PR {}: {}", label, pr_url, stderr + ) + else: + logger.debug("Successfully added label '{}' to PR {}", label, pr_url) + + logger.info("Successfully added {} labels to PR {}", len(labels), pr_url) @classmethod def _create_pr( diff --git a/app_tester/common/metrics.py b/app_tester/common/metrics.py --- a/app_tester/common/metrics.py +++ b/app_tester/common/metrics.py @@ -160,6 +160,131 @@ def track_load_app(self, app_name: str, success: bool, context: str = "general") self.send_metric("create_app.load_app", tags=tags) + # Ingestion pipeline metrics + + def track_ingestion_started( + self, ingestion_id: str, hours_back: Optional[int], namespace: str + ): + """Track start of ingestion pipeline run.""" + tags = [f"ingestion_id:{ingestion_id}", f"namespace:{namespace}"] + if hours_back is not None: + tags.append(f"hours_back:{hours_back}") + + self.send_metric("rube_learning.ingestion.started", tags=tags) + + def track_ingestion_step( + self, + ingestion_id: str, + step_name: str, + duration_sec: float, + status: str, + count: Optional[int] = None, + ): + """Track completion of an ingestion pipeline step.""" + tags = [f"ingestion_id:{ingestion_id}", f"step:{step_name}", f"status:{status}"] + if count is not None: + tags.append(f"count:{count}") + + self.send_metric( + "rube_learning.ingestion.step_duration_sec", + value=duration_sec, + tags=tags, + metric_type="histogram", + ) + + def track_clustering_result( + self, + ingestion_id: str, + clusters_count: int, + plans_count: int, + compression_ratio: float, + ): + """Track clustering results.""" + tags = [f"ingestion_id:{ingestion_id}"] + + self.send_metric( + "rube_learning.ingestion.clusters_created", + value=clusters_count, + tags=tags, + metric_type="gauge", + ) + + self.send_metric( + "rube_learning.ingestion.compression_ratio", + value=compression_ratio, + tags=tags, + metric_type="gauge", + ) + + def track_turbopuffer_operation( + self, ingestion_id: str, operation: str, success: bool, count: int + ): + """Track TurboPuffer operations (search, upsert, update).""" + status = "success" if success else "failed" + tags = [ + f"ingestion_id:{ingestion_id}", + f"operation:{operation}", + f"status:{status}", + ] + + self.send_metric( + "rube_learning.ingestion.turbopuffer_operation", value=count, tags=tags + ) + + def track_tool_consolidation( + self, + ingestion_id: str, + cluster_id: str, + tools_before: int, + tools_after: int, + method: str, + ): + """Track tool consolidation results.""" + tags = [f"ingestion_id:{ingestion_id}", f"method:{method}"] + + reduction = tools_before - tools_after + self.send_metric( + "rube_learning.ingestion.tools_consolidated", + value=reduction, + tags=tags, + metric_type="histogram", + ) + + def track_plan_ingestion( + self, + ingestion_id: str, + plan_id: str, + operation: str, + success: bool, + source: str, + ): + """ + Track individual plan ingestion to TurboPuffer. + + This allows tracking how many times each plan ID was ingested, + whether it was a new plan or an update, and from which source. + + Args: + ingestion_id: Unique ID for this ingestion run + plan_id: TurboPuffer plan ID + operation: Either "create" (new plan) or "update" (existing plan) + success: Whether the ingestion succeeded + source: Source of the ingestion - "unmatched_cluster", "matched_cluster", or "cached_plan" + """ + status = "success" if success else "failed" + tags = [ + f"ingestion_id:{ingestion_id}", + f"plan_id:{plan_id}", + f"operation:{operation}", + f"status:{status}", + f"source:{source}", + ] + + self.send_metric( + "rube_learning.ingestion.plan_ingestion", + tags=tags, + ) + # Global metrics instance - now guaranteed singleton metrics = MetricsTracker() diff --git a/app_tester/common/tools.py b/app_tester/common/tools.py --- a/app_tester/common/tools.py +++ b/app_tester/common/tools.py @@ -34,7 +34,7 @@ class ActionSpec(BaseModel): - name: str = Field(description="Action name like 'SLACK_UPLOAD_FILE'") + name: str = Field(description="Action name like 'UPLOAD_FILE'") method: str = Field(description="HTTP method: GET, POST, PUT, DELETE") endpoint: str = Field(description="API endpoint path like '/api/files.upload'") description: str = Field(description="Detailed description with parameters") diff --git a/app_tester/constant.py b/app_tester/constant.py --- a/app_tester/constant.py +++ b/app_tester/constant.py @@ -229,39 +229,6 @@ def get_cloud_id(self, token: str) -> str: export default toolkitConfig """ -CONFIG_PY_TEMPLATE = """from mercury.tools.integration import ApiKeyAuthScheme, AppConfig, AuthSchemaField - - -config = AppConfig( - name="{name}", - unique_key="{name}", - docs="https://app.{name}.com/documentation", - description="Template description for {name}", - categories=["Some category"], - logo="some_logo_url", - get_current_user_endpoint="template_endpoint", - callback_url="template_callback_url", - auth_schemes=[ # type: ignore - ApiKeyAuthScheme( - name="api_key", - proxy={{ - "base_url": "template_base_url", - "params": {{"apikey": "{{api_key}}"}}, - }}, - fields=[ - AuthSchemaField( - name="api_key", - expected_from_customer=True, - display_name="API Key", - description="Your API key", - type="string", - required=True, - ), - ], - ), - ], -) -""" MESSAGE_ON_EMPTY_RESULT = "Do not produce an empty result. Please try again." @@ -278,20 +245,159 @@ class AzureConfig: azure_endpoint: str = os.getenv("AZURE_OPENAI_ENDPOINT_URL", "some_endpoint") api_key: str = os.getenv("AZURE_OPENAI_SUBSCRIPTION_KEY", "some_key") api_version: str = "2025-03-01-preview" + helicone_key: str = os.getenv("HELICONE_API_KEY", "") + helicone_org_id: str = os.getenv( + "HELICONE_ORG_ID", "bff0d211-66df-4329-aef6-27c1983b5dc9" + ) -azure_client = AsyncAzureOpenAI( - azure_endpoint=AzureConfig.azure_endpoint, - api_key=AzureConfig.api_key, - api_version=AzureConfig.api_version, -) +# Initialize Azure OpenAI client with conditional Helicone routing +# If Helicone is enabled, route through Helicone gateway +if AzureConfig.helicone_key: + # Route through Helicone proxy + azure_client = AsyncAzureOpenAI( + azure_endpoint="https://gateway.helicone.ai", # Route through Helicone + api_key=AzureConfig.api_key, + api_version=AzureConfig.api_version, + default_headers={ + "api-key": AzureConfig.api_key, + "Helicone-Auth": f"Bearer {AzureConfig.helicone_key}", + "Helicone-Target-Url": AzureConfig.azure_endpoint, # Tell Helicone where to proxy to + "Helicone-Org-Id": AzureConfig.helicone_org_id, + }, + ) +else: + # Direct to Azure without Helicone + azure_client = AsyncAzureOpenAI( + azure_endpoint=AzureConfig.azure_endpoint, + api_key=AzureConfig.api_key, + api_version=AzureConfig.api_version, + default_headers={ + "api-key": AzureConfig.api_key, + }, + ) -azure_client_sync = AzureOpenAI( - azure_endpoint=AzureConfig.azure_endpoint, - api_key=AzureConfig.api_key, - api_version=AzureConfig.api_version, -) +# Sync client with same Helicone routing logic +if AzureConfig.helicone_key: + azure_client_sync = AzureOpenAI( + azure_endpoint="https://gateway.helicone.ai", # Route through Helicone + api_key=AzureConfig.api_key, + api_version=AzureConfig.api_version, + default_headers={ + "api-key": AzureConfig.api_key, + "Helicone-Auth": f"Bearer {AzureConfig.helicone_key}", + "Helicone-Target-Url": AzureConfig.azure_endpoint, # Tell Helicone where to proxy to + "Helicone-Org-Id": AzureConfig.helicone_org_id, + }, + ) +else: + azure_client_sync = AzureOpenAI( + azure_endpoint=AzureConfig.azure_endpoint, + api_key=AzureConfig.api_key, + api_version=AzureConfig.api_version, + default_headers={ + "api-key": AzureConfig.api_key, + }, + ) model = Model.OPENAI MAX_RESPONSE_LENGTH = 100000 + + +def create_azure_client(service_name: str | None = None, is_async: bool = True): + """ + Create an Azure OpenAI client with optional Helicone tagging. + Routes through Helicone gateway if Helicone key is configured. + + Args: + service_name: Service identifier for Helicone tracking (e.g., "ingestion-clusterer") + is_async: Whether to create async (True) or sync (False) client + + Returns: + AsyncAzureOpenAI or AzureOpenAI instance with proper headers + """ + # Get environment for tagging + environment = os.getenv("ENVIRONMENT", "development") + + # Determine endpoint and build headers + if AzureConfig.helicone_key: + # Route through Helicone + endpoint = "https://gateway.helicone.ai" + headers = { + "api-key": AzureConfig.api_key, + "Helicone-Auth": f"Bearer {AzureConfig.helicone_key}", + "Helicone-Target-Url": AzureConfig.azure_endpoint, # Tell Helicone where to proxy + "Helicone-Org-Id": AzureConfig.helicone_org_id, + } + + # Add service-specific tags if provided + if service_name: + headers.update( + { + "Helicone-Property-Service": service_name, + "Helicone-Property-Environment": environment, + "Helicone-Tag": "integrator", + } + ) + else: + # Direct to Azure without Helicone + endpoint = AzureConfig.azure_endpoint + headers = {"api-key": AzureConfig.api_key} + + # Create and return the appropriate client + if is_async: + return AsyncAzureOpenAI( + azure_endpoint=endpoint, + api_key=AzureConfig.api_key, + api_version=AzureConfig.api_version, + default_headers=headers, + ) + else: + return AzureOpenAI( + azure_endpoint=endpoint, + api_key=AzureConfig.api_key, + api_version=AzureConfig.api_version, + default_headers=headers, + ) + + +def create_llm_client(service_name: str | None = None): + """ + Create an OpenAI Async client with optional Helicone routing and service tagging. + + Notes: + - If HELICONE_API_KEY is set, route via https://oai.helicone.ai/v1 and attach + Helicone headers as default_headers (plus service tags). + - Otherwise, use the default OpenAI base_url with no extra default headers. + """ + from openai import AsyncOpenAI + + environment = os.getenv("ENVIRONMENT", "development") + helicone_api_key = os.getenv("HELICONE_API_KEY") + helicone_org_id = os.getenv( + "HELICONE_ORG_ID", "bff0d211-66df-4329-aef6-27c1983b5dc9" + ) + + base_url = None + default_headers = None + if helicone_api_key: + base_url = "https://oai.helicone.ai/v1" + default_headers = { + "Helicone-Auth": f"Bearer {helicone_api_key}", + "Helicone-Org-Id": helicone_org_id, + } + if service_name: + default_headers.update( + { + "Helicone-Property-Service": service_name, + "Helicone-Property-Environment": environment, + "Helicone-Tag": "integrator", + } + ) + + return AsyncOpenAI( + api_key=os.getenv("OPENAI_API_KEY"), + base_url=base_url, + default_headers=default_headers if default_headers else None, + ) diff --git a/app_tester/prompts.py b/app_tester/prompts.py --- a/app_tester/prompts.py +++ b/app_tester/prompts.py @@ -399,20 +399,25 @@ def execute(self, request: UploadFileRequest, metadata: t.Dict) -> UploadFileRes ---ACTION_SEPARATOR--- ``` +**IMPORTANT NAMING RULES:** +- **DO NOT include the app name** in the action name (e.g., for Slack app, use `ADMIN_GET_APP_ACTIVITIES` NOT `SLACK_ADMIN_GET_APP_ACTIVITIES`) +- Action names should describe the operation, not repeat the app context +- Use clear, descriptive names in UPPER_SNAKE_CASE + **EXAMPLE OF CORRECT OUTPUT (if you found 3 actions):** -```SLACK_ADMIN_GET_APP_ACTIVITIES: Get logs for a specified team/org +```ADMIN_GET_APP_ACTIVITIES: Get logs for a specified team/org Method: GET Endpoint: /api/admin.apps.activities.list Parameters: app_id, team_id, limit DocsURL: https://api.slack.com/methods/admin.apps.activities.list ---ACTION_SEPARATOR--- -SLACK_ADMIN_APPROVE_APP: Approve an app for installation +ADMIN_APPROVE_APP: Approve an app for installation Method: POST Endpoint: /api/admin.apps.approve Parameters: app_id, team_id DocsURL: https://api.slack.com/methods/admin.apps.approve ---ACTION_SEPARATOR--- -SLACK_ADMIN_RESTRICT_APP: Restrict an app for installation +ADMIN_RESTRICT_APP: Restrict an app for installation Method: POST Endpoint: /api/admin.apps.restrict Parameters: app_id, team_id diff --git a/app_tester/rube_learning/ARCHITECTURE.md b/app_tester/rube_learning/ARCHITECTURE.md --- a/app_tester/rube_learning/ARCHITECTURE.md +++ b/app_tester/rube_learning/ARCHITECTURE.md @@ -1,6 +1,6 @@ # Rube Learning – Session Analysis Architecture -This document illustrates how `analyze_rube_sessions.py` and the extracted modules collaborate to ingest Rube MCP logs, analyse sessions with GPT-5, and surface product learnings. It also covers the new CLI helpers you can use for ad‑hoc testing. +This document illustrates how `pipeline.py` and the extracted modules collaborate to ingest Rube MCP logs, analyse sessions with GPT, and surface product learnings. --- @@ -15,8 +15,8 @@ This document illustrates how `analyze_rube_sessions.py` and the extracted modul fetch_rube_logs() │ decrypted JSON ▼ ┌──────────────────┐ - │ RubeSessionAnalyzer - │ (analyze_rube_sessions.py) + │ pipeline.py │ + │ (orchestrator) │ └──────┬──────────┘ │ ┌─────────▼────────┐ @@ -25,63 +25,69 @@ This document illustrates how `analyze_rube_sessions.py` and the extracted modul └────────┬─────────┘ │ Session objects ▼ - ┌──────────────┐ - │ LLMAnalyzer │ - │ (GPT‑5) │ - └────┬─────────┘ - │ SessionAnalysis model + ┌─────────────────────────────────┐ + │ Two-Stage Analysis │ + │ │ + │ Stage 1: workflow_analysis/ │ + │ - WorkflowAnalysisOutput │ + │ - Search quality │ + │ - Tool gap analysis │ + │ │ + │ Stage 2: error_analysis/ │ + │ - SessionErrorAnalysis │ + │ - Error buckets & weights │ + │ - Improvements │ + └─────────────┬───────────────────┘ + │ PipelineResult ┌─────────────┼─────────────┐ │ │ │ ▼ ▼ ▼ ┌───────────────┐┌─────────────┐┌───────────────┐ -│Formatter ││SlackNotifier││save_to_clickhouse│ -│rich panels ││priority ≥8 ││persist analysis │ +│ Formatter ││SlackNotifier││ database/ │ +│(rich panels) ││priority ≥8 ││ repository.py │ └───────────────┘└─────────────┘└───────────────┘ - -Test helpers hook into the same flow: - • test_session_analysis.py → Formatter + optional Slack/DB - • test_session_raw_output.py → raw SessionAnalysis JSON ``` ### Narrative 1. **ClickHouse** returns encrypted MCP tool logs. 2. **fetch_rube_logs** decrypts fields and hands them to **SessionDetector**, which groups calls into workflows keyed by user + session info. -3. Each completed session is summarised by **LLMAnalyzer** using GPT-5 and parsed directly into the `SessionAnalysis` Pydantic model (defined in `analysis_models.py`). +3. Each completed session is processed through **pipeline.py** which orchestrates: + - **Stage 1 (workflow_analysis)**: Understanding the workflow, analyzing tools, assessing search quality + - **Stage 2 (error_analysis)**: Identifying errors, assigning weights, suggesting improvements 4. Outputs fan out to: - - **AnalysisFormatter** – renders Rich panels for terminal UX. - - **SlackNotifier** – posts high-priority learnings to `#buzz-rube-learnings` (if enabled). - - **save_to_clickhouse** – persists the structured analysis into reporting tables. -5. You can short-circuit the full daemon by using the new **test scripts**, which call the same components against a single session. + - **AnalysisFormatter** – renders Rich panels for terminal UX (accepts `PipelineResult`) + - **SlackNotifier** – posts high-priority learnings to `#buzz-rube-learnings` (accepts `PipelineResult`) + - **database/repository.py** – persists the structured analysis into ClickHouse tables --- ## 2. Module Responsibilities | Module | Purpose | Key Symbols | | --- | --- | --- | -| `analyze_rube_sessions.py` | Orchestrator. Fetches logs, drives session detection, invokes GPT analysis, coordinates Slack/DB writes. | `RubeSessionAnalyzer`, `LLMAnalyzer`, CLI entrypoint | -| `rube_learning/analysis_models.py` | Pydantic schema for LLM outputs. Shared throughout pipeline. | `SessionAnalysis`, `ToolImprovement`, `NewSkillSuggestion`, `MetaToolImprovement`, enums | -| `rube_learning/session_detector.py` | Groups raw logs into logical sessions using timeouts + sessionInfo metadata. | `SessionDetector`, `Session` | -| `rube_learning/formatters.py` | Rich console rendering of analysis and recommendations. | `AnalysisFormatter` | -| `rube_learning/slack_notifier.py` | Slack message threading, filtering, file uploads. | `SlackNotifier` | -| `agent_worker_fix_from_logs/services/clickhouse_service.py` | Query + decrypt ClickHouse logs, helper for ClickHouse inserts. | `ClickHouseService` | - -Supporting constants/utilities live in `app_tester/constant.py`, `app_tester/filter_actions.py`, etc. +| `pipeline.py` | Orchestrator. Coordinates workflow_analysis → error_analysis stages | `process_session`, `PipelineResult`, `WorkflowAnalysisResult` | +| `run_pipeline.py` | CLI entry point for running the pipeline | CLI entrypoint | +| `workflow_analysis/models.py` | Models for workflow understanding, tool analysis, plans | `WorkflowAnalysisOutput`, `SearchQualityAnalysis`, `UsecasePlan`, `NewToolSuggestion` | +| `workflow_analysis/runner.py` | Runs the workflow analysis LLM call | `run_workflow_analysis` | +| `workflow_analysis/tool_gap.py` | Tool gap analysis (missing tools, proxy_execute detection) | `run_tool_gap_analysis`, `ToolGapAnalysisOutput` | +| `error_analysis/models/output.py` | Models for error analysis results | `SessionErrorAnalysis`, `ErrorAnalysisLLMOutput` | +| `error_analysis/runner.py` | Runs the error analysis LLM call | `run_error_analysis` | +| `error_analysis/models/improvements.py` | Improvement suggestions | `ToolImprovement`, `MetaToolImprovement` | +| `session_detector.py` | Groups raw logs into logical sessions | `SessionDetector`, `Session` | +| `formatters.py` | Rich console rendering of analysis | `AnalysisFormatter` | +| `slack_notifier.py` | Slack message threading, filtering | `SlackNotifier` | +| `database/repository.py` | ClickHouse persistence | `save_pipeline_result`, `SessionAnalysisRepository` | +| `common/enums.py` | Shared enums | `WorkflowStatus`, `PlanQuality`, `TaskDifficulty` | --- ## 3. Detailed Component Interaction ``` -CLI / test script +CLI / run_pipeline.py │ ▼ ┌─────────────────────┐ -│ RubeSessionAnalyzer │ -└────────┬────────────┘ - │ fetch_rube_logs() - ▼ -┌─────────────────────┐ │ ClickHouseService │ └────────┬────────────┘ │ decrypted logs @@ -92,39 +98,52 @@ CLI / test script │ completed sessions ▼ ┌─────────────────────┐ -│ LLMAnalyzer (GPT-5) │ +│ pipeline.py │ +│ process_session() │ └────────┬────────────┘ - │ SessionAnalysis model - ├──────────────┐ - │ │ - ▼ ▼ -┌────────────────┐ ┌──────────────────┐ -│Formatter │ │SlackNotifier │ -│(Rich panels) │ │(optional) │ -└────────────────┘ └──────────────────┘ + │ + ┌────┴────┐ + │ │ + ▼ ▼ +┌──────────┐ ┌──────────────┐ +│Stage 1: │ │Stage 2: │ +│workflow_ │ │error_ │ +│analysis │ │analysis │ +└────┬─────┘ └──────┬───────┘ + │ │ + └──────┬───────┘ + │ + ▼ PipelineResult + ┌───────┴───────┐ + │ │ + ▼ ▼ +┌────────────┐ ┌──────────────────┐ +│Formatter │ │SlackNotifier │ +│(Rich) │ │(optional) │ +└────────────┘ └──────────────────┘ │ ▼ ┌─────────────────────┐ -│save_to_clickhouse │ +│database/repository │ +│save_pipeline_result │ └─────────────────────┘ ``` Key design notes: -- **Async + Threads**: LLM analysis is async but invoked inside thread workers to parallelise session processing while respecting GPT call limits. -- **Filtering**: SlackNotifier only posts priority ≥ 8 recommendations to limit noise. -- **Persistence**: `save_to_clickhouse` flattens Pydantic models into ClickHouse columnar format, mirroring the analysis object. +- **Two-Stage Pipeline**: Workflow analysis provides context for error analysis +- **Async + Threads**: LLM analysis is async but can be parallelised +- **Filtering**: SlackNotifier only posts priority ≥ 8 recommendations +- **Persistence**: `save_pipeline_result` flattens Pydantic models into ClickHouse --- -## 4. CLI Utilities for Ad-hoc Testing +## 4. CLI Usage | Script | Usage | Notes | | --- | --- | --- | -| `python app_tester/test_session_analysis.py ` | Runs the full pipeline for a single session (Rich output). Optional `--slack`, `--clickhouse`. | Ideal for validating rendering and Slack notifications without daemon polling. | -| `python app_tester/test_session_raw_output.py ` | Logs the raw `SessionAnalysis` JSON (exact ClickHouse payload) and writes it to `/tmp/session_analysis_.json`. | Perfect for reviewing model fields / DB shape. | -| `python app_tester/analyze_rube_sessions.py --log-id ` | Original entry point; can still be used for targeted runs. | Supports `--no-slack` / `--no-clickhouse`. | - -Both helper scripts share infrastructure with the main analyzer (ClickHouse queries, `SessionDetector`, `LLMAnalyzer`), ensuring parity between test runs and production flows. +| `python -m app_tester.rube_learning.run_pipeline` | Run the full pipeline | Main entry point | +| `python -m app_tester.rube_learning.run_pipeline --test-session ` | Test a specific session | For debugging | +| `python -m app_tester.rube_learning.run_pipeline --poll-interval 5` | Polling mode | Continuous processing | --- @@ -134,28 +153,60 @@ Both helper scripts share infrastructure with the main analyzer (ClickHouse quer | --- | --- | | `BACKEND_READONLY_URL` | ClickHouse connection string | | `CLICKHOUSE_USERNAME` / `CLICKHOUSE_PASSWORD` | Auth for log ingestion | -| `OPENAI_API_KEY` | GPT-5 session analysis | +| `OPENAI_API_KEY` | GPT session analysis | | `SLACK_TOKEN` | (Optional) Post to #buzz-rube-learnings | | `ENCRYPTION_KEY` | Decrypt stored requests/responses | -Optional flags (`--no-slack`, `--no-clickhouse`) let you bypass integrations during local testing. - --- -## 6. Output Surfaces +## 6. Output Models + +### PipelineResult (from pipeline.py) +```python +PipelineResult: + - session_id: str + - workflow_result: WorkflowAnalysisResult (Stage 1) + - error_result: SessionErrorAnalysis (Stage 2) + - success: bool + - error: Optional[str] +``` +### WorkflowAnalysisOutput (from workflow_analysis/models.py) +```python +WorkflowAnalysisOutput: + - overall_usecase: str + - workflow_category: str + - workflow_summary: str + - task_difficulty: TaskDifficulty + - search_quality: SearchQualityAnalysis + - per_usecase_tool_analysis: List[UsecaseToolAnalysis] + - per_usecase_plans: List[UsecasePlan] + - new_tool_suggestions: List[NewToolSuggestion] + - new_toolkit_suggestions: List[NewToolkitSuggestion] ``` -SessionAnalysis model → terminal (Rich panels via AnalysisFormatter) -SessionAnalysis model → Slack (priority ≥8 via SlackNotifier) -SessionAnalysis model → ClickHouse tables -SessionAnalysis model → /tmp/session_analysis_.json (test script) + +### SessionErrorAnalysis (from error_analysis/models/output.py) +```python +SessionErrorAnalysis: + - workflow_quality_enum: WorkflowStatus + - overall_rationale: str + - llm_errors: LLMErrorBucket + - tool_errors: ToolErrorBucket + - search_quality: SearchQualityBucket + - connection_problems: ConnectionProblemsBucket + - workbench: WorkbenchBucket + - multi_execute: MultiExecuteBucket + - recipe: RecipeBucket + - tool_improvements: List[ToolImprovement] + - meta_tool_improvements: List[MetaToolImprovement] + - primary_failure_reason: Optional[str] ``` --- -### Summary -- The analyzer now cleanly separates **models**, **formatting**, **Slack delivery**, and **orchestration**. -- Helper scripts make it trivial to run a single session end-to-end, with or without integrations. -- The architecture is unchanged functionally; the refactor improves readability, reuse, and testing velocity. - +## 7. Summary +- The pipeline cleanly separates **workflow analysis** (Stage 1) and **error analysis** (Stage 2) +- `PipelineResult` is the unified output that flows to formatters, Slack, and database +- Models are organized by stage: `workflow_analysis/models.py` and `error_analysis/models/` +- Common enums live in `common/enums.py` for shared use diff --git a/app_tester/rube_learning/USECASE_INGESTION_README.md b/app_tester/rube_learning/USECASE_INGESTION_README.md new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/USECASE_INGESTION_README.md @@ -0,0 +1,271 @@ +# Rube Usecase Plan Ingestion Service + +This service ingests analyzed Rube session use cases into TurboPuffer for semantic search and plan retrieval. + +## Overview + +The service performs the following workflow: + +1. **Fetch** - Retrieves session analyses from ClickHouse (last N hours) +2. **Filter** - Removes sessions that used cached plans +3. **Cluster** - Uses GPT-4 to group similar use cases +4. **Extract** - Selects the best plan from each cluster +5. **Ingest** - Generates embeddings and stores in TurboPuffer + +## Architecture + +``` +ClickHouse (rube_session_analyses_v2 + rube_workflow_analyses) + ↓ +Filter (no cached_plan_id) + ↓ +LLM Clustering (GPT-5) + ↓ +Plan Selection (highest quality) + ↓ +Embedding Generation (Voyage AI voyage-3.5-lite) + ↓ +TurboPuffer (rube_usecase_plans_v1) +``` + +## TurboPuffer Schema + +**Namespace**: `rube_usecase_plans_v1` +**Vector Dimensions**: 3072 + +**Columns**: +- `vector` (3072-dim): Embedding of use case +- `usecase`: Representative use case text (PII removed) +- `plan`: JSON string of the best UsecasePlan +- `tools_involved`: Comma-separated tool names +- `toolkits_required`: Comma-separated toolkit slugs +- `workflow_category`: Category (e.g., "email management") +- `session_id`: Best session ID from cluster +- `session_ids_in_cluster`: All session IDs (comma-separated) +- `cluster_size`: Number of sessions in cluster +- `avg_workflow_quality`: Average quality score (0-100) +- `avg_success_confidence`: Average confidence (0-10) +- `search_log_id`: Search log ID from plan +- `created_at`: ISO timestamp + +## Usage + +### Command Line + +```bash +# Basic usage (12 hour lookback) +python -m app_tester.rube_learning.usecase_ingestion_service + +# Custom lookback period +python -m app_tester.rube_learning.usecase_ingestion_service --hours-back 24 + +# Custom namespace +python -m app_tester.rube_learning.usecase_ingestion_service \ + --hours-back 12 \ + --namespace rube_usecase_plans_v2 +``` + +### Programmatic Usage + +```python +from app_tester.rube_learning.usecase_ingestion_service import UsecaseIngestionService + +# Create service +service = UsecaseIngestionService(namespace="rube_usecase_plans_v1") + +# Run ingestion +success = service.run(hours_back=12) +``` + +## Environment Variables + +Required: +- `BACKEND_READONLY_URL`: ClickHouse connection URL +- `CLICKHOUSE_USERNAME`: ClickHouse username +- `CLICKHOUSE_PASSWORD`: ClickHouse password +- `ENCRYPTION_KEY`: Key for decrypting ClickHouse data +- `OPENAI_API_KEY`: OpenAI API key for embeddings +- `AZURE_OPENAI_ENDPOINT_URL`: Azure OpenAI endpoint (for GPT-4) +- `AZURE_OPENAI_SUBSCRIPTION_KEY`: Azure OpenAI key +- `TURBOPUFFER_API_TOKEN`: TurboPuffer API token + +Optional: +- `TURBOPUFFER_BASE_URL`: TurboPuffer base URL (default: https://api.turbopuffer.com) + +## Components + +### 1. ClickHouse Service Extension +**File**: `app_tester/agent_worker_fix_from_logs/services/clickhouse_service.py` + +Added method: +- `fetch_recent_session_analyses(hours_back: int)` - Fetches session analyses + +### 2. Data Models +**File**: `app_tester/rube_learning/usecase_cluster_models.py` + +Models: +- `UsecaseCluster` - Cluster of similar use cases +- `LLMClusteringResponse` - LLM clustering output +- `ClusteredUsecase` - Processed cluster with best plan +- `TurboPufferRow` - Row format for TurboPuffer + +### 3. Clustering Service +**File**: `app_tester/rube_learning/usecase_clustering.py` + +Class: `UsecaseClusterer` +- `cluster_usecases()` - Uses GPT-4 to cluster similar use cases +- `select_best_session_per_cluster()` - Picks highest quality session + +### 4. Ingestion Service +**File**: `app_tester/rube_learning/turbopuffer_ingestion.py` + +Class: `UsecasePlanIngestion` +- `generate_embeddings()` - Creates embeddings with OpenAI +- `build_turbopuffer_rows()` - Builds rows from clusters +- `upsert_to_turbopuffer()` - Inserts into TurboPuffer + +### 5. Main Orchestrator +**File**: `app_tester/rube_learning/usecase_ingestion_service.py` + +Class: `UsecaseIngestionService` +- `filter_sessions_without_cached_plans()` - Filters by cached_plan_id +- `extract_best_plan_from_session()` - Extracts plan from perUsecasePlans +- `build_clustered_usecases()` - Builds final objects +- `run()` - Main workflow execution + +## Example Output + +``` +============================================================ +Rube Usecase Plan Ingestion Service +============================================================ + +📥 Step 1: Fetching sessions from last 12 hours... + ✅ Fetched 243 sessions + +🔍 Step 2: Filtering sessions without cached plans... + 📊 Filtered 243 → 189 sessions (removed 54 with cached plans) + ✅ 189 sessions to process + +🤖 Step 3: Clustering similar use cases... + 🤖 Clustering 189 use cases with GPT-4... + ✅ Created 47 clusters from 189 use cases + ✅ Created 47 clusters + +🔨 Step 4: Building clustered use cases... + ✅ Built 47 clustered use cases + +📤 Step 5: Ingesting to TurboPuffer... + 🔨 Building TurboPuffer rows for 47 clusters... + Generating embeddings 1-47/47... + ✅ Built 47 TurboPuffer rows + 📤 Upserting 47 rows to TurboPuffer namespace 'rube_usecase_plans_v1'... + ✅ Successfully upserted 47 rows to TurboPuffer + +============================================================ +✅ Successfully ingested 47 use case plans! +============================================================ + +📊 Summary: + - Total sessions processed: 189 + - Clusters created: 47 + - Use cases ingested: 47 + - Average cluster size: 4.0 +``` + +## Data Flow Example + +**Input** (ClickHouse session): +```json +{ + "sessionId": "abc123", + "overallUsecase": "send email to John via gmail", + "perUsecasePlans": "[{\"usecase_plan\": {...}, \"success_confidence\": 9}]", + "toolsInvolvedStr": "GMAIL_SEND_EMAIL", + "workflowQuality": 90, + "searchJson": "[{\"cached_plan_id\": null}]" +} +``` + +**After Clustering**: +```python +ClusteredUsecase( + representative_usecase="send email to someone via gmail", # PII removed + cluster_size=3, + best_session_id="abc123", + plan={"workflow_steps": "...", "pitfalls": "..."}, + tools_involved="GMAIL_SEND_EMAIL", + avg_workflow_quality=87.5 +) +``` + +**TurboPuffer Storage**: +```json +{ + "id": "a3f2e1d4b5c6...", + "vector": [0.123, -0.456, ...], // 3072 dims + "usecase": "send email to someone via gmail", + "plan": "{\"workflow_steps\": \"...\", ...}", + "tools_involved": "GMAIL_SEND_EMAIL", + "cluster_size": 3, + "avg_workflow_quality": 87.5 +} +``` + +## Future Retrieval + +Plans can be retrieved via semantic search: + +```python +# Query TurboPuffer +query_vector = embed_text("send email via gmail") +results = turbopuffer.search( + namespace="rube_usecase_plans_v1", + vector=query_vector, + top_k=5 +) + +# Results contain: +# - Similar use cases +# - Their proven workflow plans +# - Tools involved +# - Quality metrics +``` + +## Troubleshooting + +### No sessions found +- Check ClickHouse connection +- Verify time range has sessions +- Check `BACKEND_READONLY_URL` environment variable + +### All sessions filtered out +- All sessions used cached plans +- Increase `hours_back` parameter +- Check `searchJson` field in database + +### Clustering fails +- Check Azure OpenAI credentials +- Verify `AZURE_OPENAI_ENDPOINT_URL` and key +- Check API quotas + +### TurboPuffer ingestion fails +- Verify `TURBOPUFFER_API_TOKEN` +- Check namespace permissions +- Verify vector dimensions (must be 3072) + +## Maintenance + +### Regular Runs +Recommended to run every 6-12 hours to keep TurboPuffer updated with latest use cases. + +### Deduplication +The service uses deterministic IDs (hash of use case text), so re-running will update existing entries rather than creating duplicates. + +### Monitoring +Key metrics to monitor: +- Number of sessions fetched +- Cluster count (should be ~20-30% of session count) +- Average cluster size (should be 2-5) +- Ingestion success rate + diff --git a/app_tester/rube_learning/__init__.py b/app_tester/rube_learning/__init__.py --- a/app_tester/rube_learning/__init__.py +++ b/app_tester/rube_learning/__init__.py @@ -1,55 +1,148 @@ """ -Rube Learning Module - For analyzing and learning from Rube session patterns +Rube Learning Module - For analyzing and learning from Rube session patterns. + +Main components: +- pipeline.py: Orchestrates the full analysis (workflow_analysis → error_analysis) +- workflow_analysis/: First stage - understand workflow, analyze tools, assess search quality +- error_analysis/: Second stage - identify errors, assign weights, suggest improvements """ -from .analysis_models import ( +# === Common Enums === +from .common import PlanQuality, TaskDifficulty, WorkflowStatus + +# === Database === +from .database import SessionAnalysisRepository, create_all_tables, save_pipeline_result + +# === Error Analysis === +from .error_analysis import ( # Buckets; Improvements; Static extraction + ConnectionProblemsBucket, + ErrorAnalysisLLMOutput, + LLMErrorBucket, MetaToolImprovement, + MultiExecuteBucket, + RecipeBucket, + SearchQualityBucket, + SessionErrorAnalysis, + StaticExtractionResult, + ToolErrorBucket, + ToolImprovement, + WorkbenchBucket, + extract_all_static_data, + run_error_analysis, +) + +# === Formatters === +from .formatters import AnalysisFormatter + +# === Usecase Ingestion === +from .ingestion_pipeline.toolkit_utils import ( + normalize_toolkits_list, + normalize_toolkits_string, + parse_toolkits_with_or, + toolkits_match, + toolkits_to_string, +) +from .ingestion_pipeline.turbopuffer_ingestion import UsecasePlanIngestion +from .ingestion_pipeline.usecase_cluster_models import ( + LLMClusteringResponse, + TurboPufferRow, + UsecaseCluster, +) +from .ingestion_pipeline.usecase_clustering import UsecaseClusterer +from .ingestion_pipeline.usecase_ingestion_service import UsecaseIngestionService + +# === Pipeline (main entry point) === +from .pipeline import ( + PipelineResult, + WorkflowAnalysisResult, + process_session, + process_session_simple, +) + +# === Session Detection === +from .session_detector import Session, SessionDetector, SessionInfo + +# === Slack Notifications === +from .slack_notifier import SlackNotifier + +# === Workflow Analysis === +from .workflow_analysis import ( NewToolSuggestion, NewToolkitSuggestion, - PlanQuality, - PrimarySessionAnalysis, SearchQuality, - SessionAnalysis, - TaskDifficulty, - ToolGapAnalysisOutput, - ToolImprovement, - ToolWithConfidence, - ToolingFailureCode, + SearchQualityAnalysis, + Tool, + UsecasePlan, UsecasePlanDetails, UsecaseToolAnalysis, - WorkflowStatus, + WorkflowAnalysisOutput, + analyze_search_results, + extract_tool_info, + run_workflow_analysis, ) -from .formatters import AnalysisFormatter -from .session_detector import Session, SessionDetector, SessionInfo -from .slack_notifier import SlackNotifier -from .tool_analysis_runner import run_tool_gap_analysis __all__ = [ - # Session detection - "Session", - "SessionDetector", - "SessionInfo", - # Analysis models - "PrimarySessionAnalysis", - "SessionAnalysis", + # === Pipeline (recommended entry point) === + "process_session", + "process_session_simple", + "PipelineResult", + "WorkflowAnalysisResult", + # === Workflow Analysis === + "run_workflow_analysis", + "WorkflowAnalysisOutput", + "SearchQualityAnalysis", "SearchQuality", - "ToolGapAnalysisOutput", - "ToolImprovement", + "analyze_search_results", + "extract_tool_info", "NewToolSuggestion", "NewToolkitSuggestion", - "MetaToolImprovement", - "ToolingFailureCode", - "ToolWithConfidence", + "Tool", "UsecaseToolAnalysis", + "UsecasePlan", "UsecasePlanDetails", + # === Error Analysis === + "run_error_analysis", + "SessionErrorAnalysis", + "ErrorAnalysisLLMOutput", "WorkflowStatus", + "LLMErrorBucket", + "ToolErrorBucket", + "SearchQualityBucket", + "ConnectionProblemsBucket", + "WorkbenchBucket", + "MultiExecuteBucket", + "RecipeBucket", + "ToolImprovement", + "MetaToolImprovement", + "extract_all_static_data", + "StaticExtractionResult", + # === Common Enums === "PlanQuality", "TaskDifficulty", - # Slack notifications - "SlackNotifier", - # Formatters + # === Session Detection === + "Session", + "SessionDetector", + "SessionInfo", + # === Formatters === "AnalysisFormatter", - # Tool analysis - "run_tool_gap_analysis", + # === Slack Notifications === + "SlackNotifier", + # === Usecase Ingestion === + "UsecaseIngestionService", + "UsecaseClusterer", + "UsecasePlanIngestion", + "LLMClusteringResponse", + "TurboPufferRow", + "UsecaseCluster", + # === Database === + "save_pipeline_result", + "SessionAnalysisRepository", + "create_all_tables", + # === Toolkit Utilities === + "normalize_toolkits_string", + "normalize_toolkits_list", + "parse_toolkits_with_or", + "toolkits_to_string", + "toolkits_match", ] diff --git a/app_tester/rube_learning/analysis_models.py b/app_tester/rube_learning/analysis_models.py deleted file mode 100644 --- a/app_tester/rube_learning/analysis_models.py +++ /dev/null @@ -1,446 +0,0 @@ -""" -Data models for Rube session analysis. - -These models define the structure of LLM analysis outputs and recommendations. -""" - -from enum import Enum -from typing import List, Optional - -from pydantic import BaseModel, Field - - -class SearchQuality(str, Enum): - """Search quality assessment - simple 3-level scale""" - - GOOD = "GOOD" # Search worked well, found needed tools - ACCEPTABLE = "ACCEPTABLE" # Search had issues but workflow could proceed - POOR = "POOR" # Search significantly failed, major gaps - - -class ToolingFailureCode(str, Enum): - """Classification of tooling failures for missing tool analysis""" - - SEARCH_MISSED_EXISTING_TOOL = "SEARCH_MISSED_EXISTING_TOOL" - TOOL_NOT_SUPPORTED = "TOOL_NOT_SUPPORTED" - PROXY_EXECUTE_WORKAROUND_USED = "PROXY_EXECUTE_WORKAROUND_USED" - TOOL_IMPOSSIBLE_TO_BUILD = "TOOL_IMPOSSIBLE_TO_BUILD" # Tool needed but priority < 6 or IMPOSSIBLE difficulty - - -class ToolImprovement(BaseModel): - recommendation_name: str = Field( - "Short enum name for recommendation, eg IMPROVE TOOL DESCRIPTION: GMAIL_SEND_EMAIL" - ) - tool_name: str = Field( - description="Name of the tool to improve (e.g., GMAIL_SEND_EMAIL, SLACK_SEARCH_MESSAGES)" - ) - was_tool_successful: bool = Field( - description="Whether or not the tool was successful" - ) - improvement_type: str = Field( - description="Type of improvement: 'tool_description_change' for tool description updates, 'param_description_change' for parameter descriptions, 'schema_change' for changes to input/output schemas except description, 'tool_rename' for tool name changes, 'tool_bug' if the tool is broken / auth issue" - ) - failure_code: str = Field( - description="Short code to classify the root cause of the issue based on your judgement - eg rate_limit_error, tool_logic_error, invalid_input_passed, unexpected_output_received, missing_params, wrong_tool_called, missing_tool, incomplete_workflow, missed_pagination, etc" - ) - current_value: str = Field( - description="Current value that needs improvement (e.g., param description 'user_id: ID of the user' or tool description 'Send email to recipient')" - ) - proposed_change: str = Field( - description="Proposed improved value for non tool-description changes (e.g., param description 'user_id: Email address of the recipient (e.g., user@example.com)' or param type max_results should handle int as well as string)." - ) - tool_description_change: str = Field( - description="""Proposed tool description change if applicable, empty otherwise. Keep it concise as it eats into caller's tokens. Eg gmail_fetch_email modify description to 'Send email with subject and body to recipient'. - Prefer to append to existing tool description unless there is something wrong with existing one. Output as 'append: example to get recent unread emails ..' v/s 'modify: '. - - Adding good input examples usually helps if LLM passed a wrong input. - - Any rate limits, input or output nuances can also be appended (but keep it concise). - - Do not add generic guidance like reconnect on 403 - these are already covered in system prompts. - - Keep everything in English, no other language. - Example: "append: Example to get recent tweets from a user: {{\\"query\\": \\"from:elonmusk -is:retweet -is:reply\\", \\"max_results\\": 50}}" - Example: "append: Sheets allows max 60 reads/min and 30 writes/min - avoid heavy parallelization, add backoff/retries." - """ - ) - rationale: str = Field( - description="Explanation of why this improvement would help (e.g., 'LLM passed user IDs instead of email addresses, causing validation errors'). Include specific examples of what went wrong that could've been prevented with your improvement." - ) - priority: int = Field( - description="Priority score 1-10 where higher is better (8-10: High priority, 5-7: Medium, 1-4: Low) based on potential impact and frequency of issues", - ge=1, - le=10, - ) - - -class NewToolSuggestion(BaseModel): - recommendation_name: str = Field( - "Short enum name for recommendation, eg NEW TOOL: GET_CALENDAR_EVENTS" - ) - tool_name: str = Field( - description="Proposed name for the new tool (e.g., GMAIL_BATCH_SEND_EMAILS)" - ) - description: str = Field( - description="Clear description of what the tool does and when to use it" - ) - expected_usage: str = Field( - description="Specific use cases where this tool would be valuable (e.g., 'Send personalized emails to 100+ recipients')" - ) - rationale: str = Field( - description="Why this tool is needed and what gap it fills (e.g., 'Current workflow requires 100 separate API calls for bulk email'). Include specific examples of what went wrong that could've been prevented with your improvement." - ) - priority: int = Field( - description="Priority score 1-10 where higher is better (8-10: High impact/frequent use, 5-7: Medium, 1-4: Low)", - ge=1, - le=10, - ) - # New fields for missing tool analysis metadata (map to ClickHouse columns) - required_api: Optional[str] = Field( - default=None, - description="The API endpoint needed (e.g., 'POST /dm_conversations/:id/messages')", - ) - toolkit: Optional[str] = Field( - default=None, - description="Which toolkit/app this belongs to (e.g., 'twitter', 'shopify')", - ) - implementation_difficulty: Optional[str] = Field( - default=None, - description="Implementation feasibility: EASY, MEDIUM, HARD, or IMPOSSIBLE", - ) - api_docs: Optional[str] = Field( - default=None, description="API documentation URL if tool is not supported" - ) - exists_in_composio: Optional[bool] = Field( - default=None, description="Whether Composio has a tool for this API" - ) - search_found: Optional[bool] = Field( - default=None, description="Whether the search returned this tool" - ) - capability_status: Optional[str] = Field( - default=None, - description="Status: AVAILABLE_AND_FOUND, AVAILABLE_BUT_MISSED, or NOT_SUPPORTED", - ) - proxy_execute_attempted: Optional[bool] = Field( - default=None, description="Whether proxy_execute was used as a workaround" - ) - - -class NewToolkitSuggestion(BaseModel): - """Suggestion for an entirely new toolkit/integration""" - - recommendation_name: str = Field( - description="Short enum name for recommendation, eg NEW TOOLKIT: STRIPE_INTEGRATION" - ) - toolkit_name: str = Field( - description="Name of the toolkit to add (e.g., 'stripe', 'airtable', 'salesforce')" - ) - priority: int = Field( - description="Priority score 1-10 (LLM decides based on implementation complexity, typically 6-8 for new integrations)", - ge=1, - le=10, - ) - reasoning: str = Field( - description="Why this toolkit is needed and what gap it fills" - ) - use_case: str = Field( - description="Specific workflow or use case that requires this toolkit" - ) - api_docs_url: Optional[str] = Field( - default=None, description="URL to official API documentation" - ) - expected_value: str = Field( - description="Expected value/impact of adding this toolkit" - ) - - -class MetaToolImprovement(BaseModel): - recommendation_name: str = Field( - "Short enum name for recommendation, eg META: ADD_HINTS_TO_SEARCH" - ) - meta_tool: str = Field( - description="Name of the meta tool to improve (e.g., 'search', 'multi_execute', 'workbench')." - ) - actual_usage: str = Field( - description="Describe what was done with meta tool, include details of any workbench helpers or app tools used - eg run_composio_tool(GMAIL_FETCH_EMAILS) was called in workbench in a loop 100 times, or 5 GMAIL_SEARCH calls were made in parallel via multi-execute" - ) - was_tool_successful: bool = Field( - description="Whether or not the tool was successful" - ) - improvement: str = Field( - description="Specific improvement suggestion (e.g., 'Add example of parallel execution pattern in workbench description')." - ) - rationale: str = Field( - description="Why this improvement would help (e.g., 'LLMs often write sequential code in workbench, causing timeouts'). Include specific examples of what went wrong that could've been prevented with your improvement." - ) - priority: int = Field( - description="Priority score 1-10 where higher is better (8-10: High impact on efficiency, 5-7: Medium, 1-4: Low)", - ge=1, - le=10, - ) - - -WorkflowStatus = Enum( - "WorkflowStatus", - [ - "SUCCESS", # Full smooth execution - "SUCCESS_WEAK", # Minor inefficiencies but outcome achieved - "PARTIAL_SUCCESS", # Major misses, only part of outcome achieved - "FAILURE", # Workflow failed badly - "INCOMPLETE", # Workflow failed badly - ], -) - -PlanQuality = Enum( - "PlanQuality", - [ - "PLANNER_NOT_CALLED", - "PLAN_POOR", - "PLAN_AVERAGE", - "PLAN_EXCELLENT", - ], -) - -TaskDifficulty = Enum( - "TaskDifficulty", - [ - "EASY", # Simple, straightforward tasks - "MEDIUM", # Moderate complexity - "HARD", # Complex, multi-step tasks - ], -) - - -class UsecasePlanDetails(BaseModel): - """Structured plan details for a usecase""" - - workflow_steps: List[str] = Field( - description="Key steps in workflow execution, capture happy path and fallback paths (if applicable), keep high level. Should be concise bullet points or numbered list. NEVER include PII or user-specific identifiers - keep it generic." - ) - difficulty: str = Field( - description="Difficulty assessment: easy/medium/hard with short reason explaining why. Consider factors like number of steps, error handling needs, edge cases, etc." - ) - pitfalls: List[str] = Field( - description="List of gotchas and pitfalls in tool inputs/outputs/behavior - what can go wrong, rate limits, missing data handling, auth issues, edge cases, etc. Keep concise and actionable. NEVER include PII or user-specific details. Never repeat the same point in both workflow_steps and pitfalls." - ) - - -class UsecasePlan(BaseModel): - search_id: str = Field( - description="The id of the search call, eg usecase_1_1, overall_1, usecase_2_3, etc." - ) - cleaned_search_usecase: str = Field( - description="Cleaned search query, remove all PII/names/identifiers/user-specific info (keep it generic). Preserve app names and stable identifiers, and keep language similar. Eg if original usecase was 'send email to John via gmail withing happy birthday', cleaned will be 'send email greeting to someone via gmail'" - ) - cleaned_known_fields: str = Field( - description="Cleaned known_fields field, only keep field keys, remove values. Eg 'recipient:John, count:10' becomes 'recipient, count'. Remove any user-specific identifiers if present in keys." - ) - toolkits_required: list[str] = Field( - description="""Sorted list of lowercase toolkit slugs required to fulfil this usecase. The toolkit slugs should come from the toolkits in search response, but also consider what was used in the workflow as search may return multiple options. - Eg usecase='send a gmail email', response.toolkits=['gmail'] -> toolkits_required=['gmail'] - Eg usecase='send an email', response.toolkits=['gmail', 'outlook'], but workflow used only gmail -> toolkits_required=['gmail' OR 'outlook'] -- use OR to denote any of the two toolkits can do the job and propose diverse options even if one was not used. - Eg usecase='send an email and create a doc', response.toolkits=['gmail', 'outlook', 'googledocs', 'gemini'], but workflow used only gmail and googledocs -> toolkits_required=['gmail' OR 'outlook', 'googledocs'] -- gemini or any related toolkit was not used at all so it was unnecessary, can be excluded - """ - ) - usecase_plan: UsecasePlanDetails = Field( - description="Structured plan for the usecase with workflow steps, difficulty assessment, and pitfalls. Should never contain any PII or user-specific identifiers or details - keep it generic." - ) - plan_changelog: str = Field(description="The changelog for the plan") - success_confidence: int = Field( - description=""" - On a scale of 0 to 10, how confident are you that the proposed plan will lead to success for this use case? - - A lower score should be given if the workflow is incomplete or missing key steps, since that introduces uncertainty about the outcome. - A higher score should be given if the plan effectively identifies and addresses inefficiencies or bugs in the workflow. - If the usecase is not possible (eg missing toolkit support or missing tool), again give lower confidence. Don't propose hacky plans as workarounds. - """ - ) - - -class PrimarySessionAnalysis(BaseModel): - """Primary LLM output - workflow analysis without new_tool_suggestions""" - - overall_usecase: str = Field( - description="Concisely tell what was user's usecase, including initial, followups etc (focusing on Search queries). Don't include PII (personally identifiable information) Eg 'Get summary of gmail emails and create google doc. Followup share doc with teammate'" - ) - workflow_category: str = Field( - description="Inferred category of the workflow, eg 'email management', 'social media marketing', 'twitter research', 'project management', 'other'" - ) - apps_involved: list[str] = Field( - description="List of apps involved in the workflow (apps that were used or should have been used), eg ['gmail', 'googledocs']" - ) - tools_involved: list[str] = Field( - description="List of tools involved in the workflow, either via multi-execute/workbench eg ['GMAIL_FETCH_EMAILS', 'GMAIL_SEND_EMAIL']" - ) - workflow_summary: str = Field( - description="Thorough summary of what happened: user intent → actions taken → outcomes. Keep concise, e.g., 'User asked to get Reddit upvotes. Search returned REDDIT_GET_POST tool. LLM ran multi execute but got 403 error, causing workflow failure.'" - ) - failure_codes: list[str] = Field( - description="""List of short codes classifying workflow issues: - - rate_limit_error, tool_logic_error, workbench_500_error - - workbench_invoke_llm_error, workbench_proxy_execute_error, workbench_run_composio_tool_error - - invalid_input_passed, unexpected_output_received - - missing_params, incomplete_workflow, missed_pagination, planner_issue - """ - ) - # was_planner_suggested: bool = Field( - # description="Whether or not planner tool was recommended in search guidance" - # ) - - # plan_quality_enum: PlanQuality = Field( - # description=( - # "PLANNER_NOT_CALLED if: (a) planner wasn't suggested by search, OR (b) planner was called " - # "but returned no-op/skip/empty response. PLAN_POOR (0-4): plan was incorrect, misleading, " - # "or omits key considerations. PLAN_EXCELLENT (8-10): comprehensive accurate plan that " - # "captures the full workflow, accounts for edge cases and nuances like pagination, but not " - # "over-engineered (eg doesn't suggest unnecessary complexity like workbench when " - # "multi-execute would have sufficed). PLAN_AVERAGE (5-7): in between excellent and bad, " - # "covers main steps but either misses some non-critical details or is over-engineered. " - # "CRITICAL: No-op responses should ALWAYS be PLANNER_NOT_CALLED, not PLAN_POOR." - # ) - # ) - # plan_quality_reason: str = Field( - # description="Why did you rate the plan like that - share details in concise form" - # ) - workflow_quality_enum: WorkflowStatus = Field( - description="Classify workflow quality into following categories: SUCCESS (full smooth execution), SUCCESS_WEAK (minor inefficiencies but outcome achieved), PARTIAL_SUCCESS (major misses, only part of outcome achieved), FAILURE (workflow failed badly), INCOMPLETE (if workflow was likely abandoned by user and relevant tool calls weren't made)" - ) - was_workflow_successful: bool = Field( - description="Whether or not the workflow was successful. Focus on whether the user's usecase succeeded or not. Any inefficiences can be highlighted separately." - ) - workflow_quality: int = Field( - description="Overall quality percent of the workflow 0-100, 100 being the highest quality, everything went smoothly. 0 if workflow failed badly. Minor misses should not be penalized heavily (eg user confirmation)" - ) - task_difficulty: TaskDifficulty = Field( - description="Difficulty of the task based on search response or planner request: EASY, MEDIUM, HARD. If not present in Search/Planner, infer from the workflow" - ) - overall_rationale: str = Field( - description="Thorough summary of patterns observed and why these specific recommendations were chosen. Include clearly what went wrong that could've been prevented with your recommendations." - ) - tool_improvements: List[ToolImprovement] = Field( - default_factory=list, - description="List of improvements to existing tool descriptions/schemas, ordered by priority (highest first)", - ) - meta_tool_improvements: List[MetaToolImprovement] = Field( - default_factory=list, - description="List of improvements to workflow meta tools, ordered by priority (highest first)", - ) - per_usecase_plans: Optional[List[UsecasePlan]] = Field( - default=None, - description="Per-search use case plans. Each searched usecase gets one UsecasePlan with actionable insights about that search's workflow patterns. Ordered in which the search queries were made.", - ) - has_recommendations: bool = Field( - description="Whether any actionable recommendations were found for this session" - ) - confidence_score: float = Field( - description="Overall confidence in these recommendations (0.0-1.0)", - ge=0.0, - le=1.0, - ) - - -class SessionAnalysis(PrimarySessionAnalysis): - """Complete analysis - extends PrimarySessionAnalysis with tool gap analysis results""" - - # Additional fields from tool gap analysis - per_usecase_tool_analysis: List["UsecaseToolAnalysis"] = Field( - default_factory=list, - description="Per-usecase tool refinement with confidence scores", - ) - new_tool_suggestions: List[NewToolSuggestion] = Field( - default_factory=list, - description="List of genuinely missing tools from tool gap analysis", - ) - new_toolkit_suggestions: List[NewToolkitSuggestion] = Field( - default_factory=list, - description="List of entirely new toolkit/integration suggestions", - ) - search_quality: SearchQuality = Field( - description="Search quality assessment from tool gap analysis: GOOD, ACCEPTABLE, or POOR" - ) - search_quality_explanation: str = Field( - description="Detailed explanation of search quality assessment" - ) - - -class ToolWithConfidence(BaseModel): - """Tool with confidence score for per-usecase analysis""" - - tool_slug: str = Field(description="Tool slug name (e.g., GMAIL_SEND_EMAIL)") - confidence: int = Field( - description="""Confidence score 0-10 indicating how necessary this tool is for the use case: - - 8-10: Essential/must-have - directly required to accomplish the core use case - - 6-8: Nice to have/helpful - supports the workflow but not strictly required - - 0-6: Not needed - tool doesn't add value or is redundant for this specific use case - - Base confidence on whether removing this tool would prevent accomplishing the use case.""", - ge=0, - le=10, - ) - - -class UsecaseToolAnalysis(BaseModel): - """Per-usecase tool refinement with confidence scores""" - - search_id: str = Field( - description="Search ID like 'usecase_1_1', 'overall_1', etc. Uniquely identifies the use case." - ) - final_main_tools: List[ToolWithConfidence] = Field( - description="""Refined list of MAIN TOOLS with confidence scores. - - Main Tools = Primary tools that DIRECTLY accomplish the use case. - These are the core tools needed to complete the user's request. - - Judge each tool by: Would removing it prevent accomplishing the use case? - - If yes → keep with confidence 8-10 - - If it helps but isn't essential → confidence 6-8 - - If redundant or unnecessary → confidence 0-6 (remove)""" - ) - final_related_tools: List[ToolWithConfidence] = Field( - description="""Refined list of RELATED TOOLS with confidence scores. - - Related Tools = Supplementary tools that support the main workflow: - - Prerequisites: Tools needed BEFORE main tools (e.g., get ID before update) - - Supporting tools: Tools that enhance or complement the main action - - Next steps: Tools useful AFTER main tools complete - - Fallbacks: Alternative tools if main approach fails - - Judge each tool by: Does it genuinely support the workflow or is it unnecessary? - - Essential prerequisite/support → confidence 8-10 - - Helpful but optional → confidence 6-8 - - Not relevant to this use case → confidence 0-6 (remove)""" - ) - final_toolkits: List[str] = Field( - description="""Refined list of required toolkits (app integrations). - - Only include toolkits whose tools are actually needed for this specific use case. - Remove toolkits that don't contribute (e.g., 'gemini' toolkit for 'send email' use case).""" - ) - reasoning: str = Field( - description="Detailed reasoning for EVERY change made (additions, removals, moves, reordering)" - ) - overall_confidence: int = Field( - description="0-10: How confident the refined list will accomplish the use case", - ge=0, - le=10, - ) - changes_made: int = Field( - description="Total count of changes made (additions + removals + moves + reclassifications). Must be ≤3 per use case." - ) - - -class ToolGapAnalysisOutput(BaseModel): - """Output from tool gap analysis agent""" - - per_usecase_tool_analysis: List[UsecaseToolAnalysis] = Field( - default_factory=list, - description="Per-usecase tool refinement with confidence scores", - ) - search_quality: SearchQuality = Field( - description="3-level assessment: GOOD (search worked well), ACCEPTABLE (had issues), POOR (major failures)" - ) - search_quality_explanation: str = Field( - description="Detailed explanation of search quality assessment" - ) - new_tool_suggestions: List[NewToolSuggestion] = Field( - default_factory=list, description="Genuinely missing tools in existing toolkits" - ) - new_toolkit_suggestions: List[NewToolkitSuggestion] = Field( - default_factory=list, description="Entirely new toolkit/integration suggestions" - ) diff --git a/app_tester/rube_learning/call_parser.py b/app_tester/rube_learning/call_parser.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/call_parser.py @@ -0,0 +1,269 @@ +""" +Call Parser - Parse and type raw log data once for reuse. + +This module provides: +1. ParsedCall - A typed representation of a tool call with pre-parsed JSON fields +2. parse_session_calls() - Parse all calls in a session once + +The key insight is that `response` contains successful responses, +and `errorRequest` contains error responses. We parse both once +and provide clean accessors. +""" + +import json +import logging +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from app_tester.rube_learning.session_detector import Session + + +logger = logging.getLogger(__name__) + + +@dataclass +class ParsedCall: + """ + A tool call with pre-parsed JSON fields. + + Raw log fields (strings) are parsed once into typed dicts. + Provides clean accessors for common patterns. + """ + + # Identity + log_id: str + action_name: str + + # Timing + timestamp: Optional[Any] = None + duration_ms: Optional[int] = None + + # Parsed JSON fields (parsed once from raw strings) + request: Optional[Dict[str, Any]] = None + response: Optional[Dict[str, Any]] = None + error_request: Optional[Dict[str, Any]] = None + + # Top-level error from the call (if any) + top_level_error: Optional[str] = None + + # Keep reference to raw call for any edge cases + _raw: Dict[str, Any] = field(default_factory=dict, repr=False) + + @property + def has_error(self) -> bool: + """Check if this call had an error (error_request populated or top_level_error set).""" + return self.error_request is not None or self.top_level_error is not None + + @property + def effective_response(self) -> Optional[Dict[str, Any]]: + """ + Get the effective response data. + + For errors: returns error_request (the actual response for failed calls) + For success: returns response + """ + if self.error_request is not None: + return self.error_request + return self.response + + @property + def error_message(self) -> Optional[str]: + """ + Get the error message from this call. + + Priority: + 1. top_level_error (from call.error field) + 2. error_request.error + 3. error_request.data.error + """ + if self.top_level_error: + return str(self.top_level_error) + + if self.error_request: + # Check top-level error in error_request + err = self.error_request.get("error") + if err: + return str(err) + + # Check data.error + data = self.error_request.get("data", {}) + if isinstance(data, dict): + data_err = data.get("error") + if data_err: + return str(data_err) + + return None + + @property + def is_successful(self) -> bool: + """Check if the call was successful (has response, no error_request).""" + return ( + self.response is not None + and self.error_request is None + and not self.top_level_error + ) + + def get_request_field(self, *keys: str, default: Any = None) -> Any: + """ + Get a field from the request, supporting nested paths. + + Example: call.get_request_field("params", "code_to_execute") + """ + if not self.request: + return default + + current: Any = self.request + for key in keys: + if not isinstance(current, dict): + return default + current = current.get(key) + if current is None: + return default + return current + + def get_response_field(self, *keys: str, default: Any = None) -> Any: + """ + Get a field from the effective response (error_request or response). + + Example: call.get_response_field("data", "results") + """ + resp = self.effective_response + if not resp: + return default + + current: Any = resp + for key in keys: + if not isinstance(current, dict): + return default + current = current.get(key) + if current is None: + return default + return current + + +def _parse_json_field(value: Any) -> Optional[Dict[str, Any]]: + """ + Parse a JSON field that might be a string or already a dict. + + Returns: + Parsed dict, or None if empty/invalid + """ + if value is None: + return None + + if isinstance(value, dict): + return value + + if isinstance(value, str): + stripped = value.strip() + if not stripped: + return None + try: + parsed = json.loads(stripped) + if isinstance(parsed, dict): + return parsed + # If it parsed to something else (list, string, etc.), wrap or return None + return None + except json.JSONDecodeError as e: + logger.debug(f"Failed to parse JSON: {e}. Value (truncated): {value[:100]}") + return None + + return None + + +def _get_duration_ms(raw_call: Dict[str, Any]) -> Optional[int]: + """Extract duration in milliseconds from raw call.""" + duration = raw_call.get("duration") + if duration is not None: + try: + return int(float(duration)) + except (ValueError, TypeError): + pass + return None + + +def parse_call(raw_call: Dict[str, Any]) -> ParsedCall: + """ + Parse a single raw call into a ParsedCall. + + This is where all JSON parsing happens - once per call. + """ + log_id = raw_call.get("log_id", "") + action_name = raw_call.get("actionName", "") + + # Parse JSON fields once + request = _parse_json_field(raw_call.get("request")) + response = _parse_json_field(raw_call.get("response")) + error_request = _parse_json_field(raw_call.get("errorRequest")) + + # Get top-level error from response.error (the error field is inside the response JSON, + # not a separate database column) + # Filter out literal "null" and "None" strings that backends sometimes return + # instead of actual null values - these are not real errors + top_level_error = None + if response and isinstance(response, dict): + top_level_error = response.get("error") + if top_level_error: + top_level_error = str(top_level_error) + if top_level_error in ("", "null", "None"): + top_level_error = None + + return ParsedCall( + log_id=log_id, + action_name=action_name, + timestamp=raw_call.get("timestamp"), + duration_ms=_get_duration_ms(raw_call), + request=request, + response=response, + error_request=error_request, + top_level_error=top_level_error if top_level_error else None, + _raw=raw_call, + ) + + +def parse_session_calls(session: Session) -> List[ParsedCall]: + """ + Parse all calls in a session into typed ParsedCall objects. + + This should be called once at the start of processing, + and the results reused by all extraction and formatting logic. + """ + return [parse_call(call) for call in session.calls] + + +@dataclass +class ParsedSession: + """ + A session with all calls pre-parsed. + + Use this as the central object for all extraction and formatting. + """ + + session_id: str + user_id: str + calls: List[ParsedCall] + + # Keep reference to original session for metadata + _original: Session = field(repr=False) + + @classmethod + def from_session(cls, session: Session) -> "ParsedSession": + """Create a ParsedSession from a raw Session.""" + return cls( + session_id=session.session_id, + user_id=session.user_id, + calls=parse_session_calls(session), + _original=session, + ) + + def get_calls_by_action(self, action_name: str) -> List[ParsedCall]: + """Get all calls with a specific action name.""" + return [c for c in self.calls if c.action_name == action_name] + + def get_error_calls(self) -> List[ParsedCall]: + """Get all calls that had errors.""" + return [c for c in self.calls if c.has_error] + + def get_success_calls(self) -> List[ParsedCall]: + """Get all calls that succeeded.""" + return [c for c in self.calls if c.is_successful] diff --git a/app_tester/rube_learning/common/__init__.py b/app_tester/rube_learning/common/__init__.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/common/__init__.py @@ -0,0 +1,15 @@ +""" +Common models and enums shared across the rube_learning module. + +This module contains enums and base models that are used by multiple +sub-modules (error_analysis, workflow_analysis, etc.) to avoid duplication. +""" + +from .enums import PlanQuality, TaskDifficulty, WorkflowStatus + + +__all__ = [ + "PlanQuality", + "TaskDifficulty", + "WorkflowStatus", +] diff --git a/app_tester/rube_learning/common/enums.py b/app_tester/rube_learning/common/enums.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/common/enums.py @@ -0,0 +1,42 @@ +""" +Shared enums for the rube_learning module. + +These enums are used across multiple sub-modules: +- error_analysis +- workflow_analysis +- analysis_models +""" + +from enum import Enum + + +class WorkflowStatus(str, Enum): + """Overall workflow quality assessment.""" + + SUCCESS = "SUCCESS" # Spotless execution, no issues + SUCCESS_WEAK = "SUCCESS_WEAK" # Outcome achieved but with minor issues + PARTIAL_SUCCESS = "PARTIAL_SUCCESS" # Only part of outcome achieved + FAILURE = "FAILURE" # Execution failed or unusable result + INCOMPLETE = "INCOMPLETE" # Workflow abandoned + UNKNOWN = "UNKNOWN" # Unable to determine + + +class PlanQuality(str, Enum): + """Quality assessment for cached plans.""" + + PLAN_NOT_AVAILABLE = "PLAN_NOT_AVAILABLE" # Plan wasn't returned or was ignored + PLAN_POOR = ( + "PLAN_POOR" # Plan was incorrect, misleading, or omits key considerations (0-4) + ) + PLAN_AVERAGE = "PLAN_AVERAGE" # Covers main steps but misses details or is over-engineered (5-7) + PLAN_EXCELLENT = ( + "PLAN_EXCELLENT" # Comprehensive, accurate, accounts for edge cases (8-10) + ) + + +class TaskDifficulty(str, Enum): + """Task difficulty level.""" + + EASY = "EASY" # Simple, straightforward tasks + MEDIUM = "MEDIUM" # Moderate complexity + HARD = "HARD" # Complex, multi-step tasks diff --git a/app_tester/rube_learning/config.py b/app_tester/rube_learning/config.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/config.py @@ -0,0 +1,65 @@ +""" +Configuration constants for the session analysis pipeline. +""" + +# Session Detection +MIN_CALLS_FOR_VALID_SESSION = 3 + +# Pipeline defaults +DEFAULT_MAX_SESSIONS = 100 +DEFAULT_POLL_INTERVAL_MINUTES = 5 +# Initial lookback must be > 20 min (session timeout) to find completed sessions +# 60 min gives a good balance: captures recent activity while ensuring sessions have "completed" +DEFAULT_INITIAL_LOOKBACK_MINUTES = 60 +# Maximum concurrent sessions to process in parallel +MAX_CONCURRENT_SESSIONS = 20 + +# Weight validation +WEIGHT_TOLERANCE = 0.05 # 5% tolerance for weight sum validation + +# LLM retry settings +MAX_LLM_RETRIES = 2 + +# ============================================================================= +# Truncation Limits for Data Extraction (extractors.py) +# ============================================================================= +# PHILOSOPHY: Extract once, use everywhere. These limits apply during extraction +# and the extracted data is then used for both session log formatting and counts. +# Don't truncate errors - they are critical context for analysis. + +# Workbench execution output limits (extracted once in extractors.py) +WORKBENCH_ERROR_OUTPUT_MAX_LENGTH = 15000 # Full error context - don't truncate errors +WORKBENCH_SUCCESS_OUTPUT_MAX_LENGTH = ( + 15000 # Increased to capture code patterns for workbench_code_snippets +) + +# Multi-execute error response limits (extracted once in extractors.py) +MULTI_EXECUTE_ERROR_RESPONSE_MAX_LENGTH = 8000 # Full context for multi-execute errors + +# ============================================================================= +# Truncation Limits for Session Log Formatting (runner.py) +# ============================================================================= +# These apply to generic calls (not workbench/multi-execute which use extracted data) + +# Session logs formatting - these go to LLM, don't truncate +SESSION_LOG_REQUEST_MAX_LENGTH = 5000 # Full request for LLM analysis +SESSION_LOG_RESPONSE_MAX_LENGTH = 1000 # Full response for LLM analysis +SESSION_LOG_ERROR_MAX_LENGTH = 5000 # Full error for LLM analysis + +# Workflow analysis formatting - these go to LLM, don't truncate +# NOTE: V1 used 30000 chars per call - matching that for plan quality +WORKFLOW_LOG_REQUEST_MAX_LENGTH = 30000 # Full request for LLM analysis +WORKFLOW_LOG_RESPONSE_MAX_LENGTH = 30000 # Full response for LLM analysis +WORKFLOW_LOG_ERROR_MAX_LENGTH = 15000 # Full error for LLM analysis + +# JSON summarization - for non-multi-execute requests +JSON_SUMMARY_MAX_LENGTH = 3000 # Generous limit for JSON summaries +JSON_KEY_VALUE_MAX_LENGTH = 500 # Show full values in JSON +JSON_MAX_KEYS_TO_SHOW = 20 # Show more keys + +# Field-specific limits +THOUGHT_MAX_LENGTH = 500 # Thought/purpose field - useful context +DESCRIPTION_MAX_LENGTH = 200 # Tool descriptions +PARAM_DESCRIPTION_MAX_LENGTH = 200 # Parameter descriptions - same as tool descriptions +MEMORY_FIELD_MAX_LENGTH = 500 # Memory field in requests +METADATA_FIELD_MAX_LENGTH = 200 # Metadata fields (current_step, etc.) diff --git a/app_tester/rube_learning/database/__init__.py b/app_tester/rube_learning/database/__init__.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/database/__init__.py @@ -0,0 +1,45 @@ +""" +Database module for session analysis storage. + +Handles saving and loading analysis results to/from ClickHouse. + +Tables: + - rube_session_analyses_v2: Session-level summary for dashboards + - rube_error_analyses: Full error bucket data + - rube_workflow_analyses: Workflow plans and suggestions +""" + +from .repository import SessionAnalysisRepository, save_pipeline_result +from .schema import ( + CREATE_ERROR_ANALYSES_TABLE, + CREATE_SESSION_ANALYSES_TABLE, + CREATE_WORKFLOW_ANALYSES_TABLE, + DATABASE_NAME, + ERROR_ANALYSES_TABLE, + ERROR_ANALYSES_TABLE_FQ, + SESSION_ANALYSES_TABLE, + SESSION_ANALYSES_TABLE_FQ, + WORKFLOW_ANALYSES_TABLE, + WORKFLOW_ANALYSES_TABLE_FQ, + create_all_tables, +) + + +__all__ = [ + # Table name constants + "DATABASE_NAME", + "SESSION_ANALYSES_TABLE", + "SESSION_ANALYSES_TABLE_FQ", + "ERROR_ANALYSES_TABLE", + "ERROR_ANALYSES_TABLE_FQ", + "WORKFLOW_ANALYSES_TABLE", + "WORKFLOW_ANALYSES_TABLE_FQ", + # Schema DDL + "CREATE_SESSION_ANALYSES_TABLE", + "CREATE_ERROR_ANALYSES_TABLE", + "CREATE_WORKFLOW_ANALYSES_TABLE", + "create_all_tables", + # Repository + "SessionAnalysisRepository", + "save_pipeline_result", +] diff --git a/app_tester/rube_learning/database/repository.py b/app_tester/rube_learning/database/repository.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/database/repository.py @@ -0,0 +1,866 @@ +""" +Repository for saving session analysis results to ClickHouse. + +Handles saving PipelineResult to the three normalized tables: + - rube_session_analyses_v2: Session-level summary (main table) + - rube_error_analyses: Full error bucket data + - rube_workflow_analyses: Workflow plans and suggestions + +Relationships: + - rube_session_analyses_v2.id is the primary key for joins + - Detail tables link back via session_analysis_id + - client_session_id is the unreliable session ID from client (for debugging) +""" + +import json +import logging +import uuid +from datetime import datetime +from typing import Optional + +from app_tester.agent_worker_fix_from_logs.services.clickhouse_service import ( + ClickHouseService, +) +from app_tester.rube_learning.error_analysis.models import SessionErrorAnalysis +from app_tester.rube_learning.workflow_analysis.models import ( + UsecasePlan, + WorkflowAnalysisOutput, +) + +from .schema import ( + ERROR_ANALYSES_TABLE_FQ, + SESSION_ANALYSES_TABLE_FQ, + WORKFLOW_ANALYSES_TABLE_FQ, +) + + +logger = logging.getLogger(__name__) + + +def _escape_string(value: Optional[str]) -> str: + """Escape a string for ClickHouse SQL insertion.""" + if value is None: + return "" + return str(value).replace("\\", "\\\\").replace("'", "\\'") + + +def _to_json(obj) -> str: + """Convert object to JSON string, handling Pydantic models.""" + if obj is None: + return "[]" + if hasattr(obj, "model_dump"): + return json.dumps(obj.model_dump()) + if isinstance(obj, list): + return json.dumps( + [item.model_dump() if hasattr(item, "model_dump") else item for item in obj] + ) + return json.dumps(obj) + + +def expand_plans_with_or_toolkits(plans: list[UsecasePlan]) -> list[UsecasePlan]: + """ + Expand plans with OR relations in toolkits_required. + + For each plan with OR relations in toolkits_required: + 1. Keep the original plan with OR relations + 2. If actually_used_toolkits is populated, create an additional plan with specific toolkits + + The specific variant gets a suffix added to search_id (e.g., "usecase_1_1" -> "usecase_1_1_specific") + to distinguish it from the original plan. + + Args: + plans: List of UsecasePlan objects + + Returns: + Expanded list of plans including _specific variants + """ + expanded = [] + for plan in plans: + # Always add the original plan + expanded.append(plan) + + # Check if any toolkit contains OR + has_or = any(" OR " in tk.upper() for tk in plan.toolkits_required) + + # If has OR relations and we know which toolkits were actually used, create specific variant + if has_or and plan.actually_used_toolkits: + # Create a deep copy and modify it + specific_plan = plan.model_copy(deep=True) + specific_plan.search_id = f"{plan.search_id}_specific" + specific_plan.toolkits_required = plan.actually_used_toolkits + expanded.append(specific_plan) + + return expanded + + +class SessionAnalysisRepository: + """Repository for session analysis data.""" + + def __init__(self, clickhouse: Optional[ClickHouseService] = None): + self.clickhouse = clickhouse or ClickHouseService() + + def save_error_analysis( + self, + session_analysis_id: uuid.UUID, + client_session_id: str, + client_name: str, + error_result: SessionErrorAnalysis, + ) -> uuid.UUID: + """ + Save error analysis to rube_error_analyses table. + + Args: + session_analysis_id: UUID of the parent session analysis record + client_session_id: Unreliable session ID from client (for debugging) + client_name: Name of the client/source + error_result: The error analysis result to save + + Returns the UUID of the saved record. + """ + record_id = uuid.uuid4() + + # Get bucket weights + weights = error_result.get_bucket_weights() + + # Get top issues for quick display + top_issues = error_result.get_top_issues(10) + top_issues_data = [ + {"bucket": b, "issue": i, "weight": w, "impact": impact} + for b, i, w, impact in top_issues + ] + + query = f""" + INSERT INTO {ERROR_ANALYSES_TABLE_FQ} ( + id, + session_analysis_id, + client_session_id, + client_name, + analyzed_at, + workflow_quality, + overall_rationale, + primary_failure_reason, + llm_errors_weight, + tool_errors_weight, + search_quality_weight, + connection_problems_weight, + workbench_weight, + multi_execute_weight, + recipe_weight, + other_errors_weight, + llm_errors_json, + tool_errors_json, + search_quality_json, + connection_problems_json, + workbench_json, + multi_execute_json, + recipe_json, + tool_improvements_json, + meta_tool_improvements_json, + top_issues_json + ) VALUES ( + '{record_id}', + '{session_analysis_id}', + '{_escape_string(client_session_id)}', + '{_escape_string(client_name)}', + now(), + '{_escape_string(error_result.workflow_quality_enum.value)}', + '{_escape_string(error_result.overall_rationale)}', + '{_escape_string(error_result.primary_failure_reason)}', + {weights.get('llm_errors', 0.0)}, + {weights.get('tool_errors', 0.0)}, + {weights.get('search_quality', 0.0)}, + {weights.get('connection_problems', 0.0)}, + {weights.get('workbench', 0.0)}, + {weights.get('multi_execute', 0.0)}, + {weights.get('recipe', 0.0)}, + {weights.get('other_errors', 0.0)}, + '{_escape_string(_to_json(error_result.llm_errors))}', + '{_escape_string(_to_json(error_result.tool_errors))}', + '{_escape_string(_to_json(error_result.search_quality))}', + '{_escape_string(_to_json(error_result.connection_problems))}', + '{_escape_string(_to_json(error_result.workbench))}', + '{_escape_string(_to_json(error_result.multi_execute))}', + '{_escape_string(_to_json(error_result.recipe))}', + '{_escape_string(_to_json(error_result.tool_improvements))}', + '{_escape_string(_to_json(error_result.meta_tool_improvements))}', + '{_escape_string(json.dumps(top_issues_data))}' + ) + """ + + self.clickhouse._execute_query(query) + logger.info( + f"Saved error analysis {record_id} for session_analysis {session_analysis_id}" + ) + return record_id + + def save_workflow_analysis( + self, + session_analysis_id: uuid.UUID, + client_session_id: str, + workflow_output: WorkflowAnalysisOutput, + search_data: Optional[dict] = None, + ) -> uuid.UUID: + """ + Save workflow analysis to rube_workflow_analyses table. + + Args: + session_analysis_id: UUID of the parent session analysis record + client_session_id: Unreliable session ID from client (for debugging) + workflow_output: The workflow analysis result to save + search_data: Raw search data from analyze_search_results() for search_json compatibility + + Returns the UUID of the saved record. + """ + record_id = uuid.uuid4() + + # Serialize wrong_tools_returned + wrong_tools_json = _to_json(workflow_output.search_quality.wrong_tools_returned) + + # Expand plans with OR toolkits to include _specific variants + expanded_plans = expand_plans_with_or_toolkits( + workflow_output.per_usecase_plans + ) + + # search_json is now built from raw session calls in save_pipeline_result + # and passed via search_data["search_json_entries"] + search_json_data = ( + search_data.get("search_json_entries", []) if search_data else [] + ) + search_json_str = json.dumps(search_json_data) + + query = f""" + INSERT INTO {WORKFLOW_ANALYSES_TABLE_FQ} ( + id, + session_analysis_id, + client_session_id, + analyzed_at, + overall_usecase, + workflow_category, + workflow_summary, + task_difficulty, + toolkits_involved, + tools_involved, + confidence_score, + search_quality, + search_quality_explanation, + wrong_tools_returned_json, + per_usecase_plans_json, + per_usecase_tool_analysis_json, + search_json, + new_tool_suggestions_json, + new_toolkit_suggestions_json, + planner_issues_json + ) VALUES ( + '{record_id}', + '{session_analysis_id}', + '{_escape_string(client_session_id)}', + now(), + '{_escape_string(workflow_output.overall_usecase)}', + '{_escape_string(workflow_output.workflow_category)}', + '{_escape_string(workflow_output.workflow_summary)}', + '{_escape_string(workflow_output.task_difficulty.value)}', + [{', '.join(f"'{_escape_string(a)}'" for a in workflow_output.toolkits_involved)}], + [{', '.join(f"'{_escape_string(t)}'" for t in workflow_output.tools_involved)}], + {workflow_output.confidence_score}, + '{_escape_string(workflow_output.search_quality.overall_quality.value)}', + '{_escape_string(workflow_output.search_quality.explanation)}', + '{_escape_string(wrong_tools_json)}', + '{_escape_string(_to_json(expanded_plans))}', + '{_escape_string(_to_json(workflow_output.per_usecase_tool_analysis))}', + '{_escape_string(search_json_str)}', + '{_escape_string(_to_json(workflow_output.new_tool_suggestions))}', + '{_escape_string(_to_json(workflow_output.new_toolkit_suggestions))}', + '{_escape_string(_to_json(workflow_output.planner_issues))}' + ) + """ + + self.clickhouse._execute_query(query) + logger.info( + f"Saved workflow analysis {record_id} for session_analysis {session_analysis_id}" + ) + return record_id + + def save_session_analysis( + self, + record_id: uuid.UUID, + client_session_id: str, + user_id: str, + client_name: str, + session_start_time: Optional[datetime], + session_end_time: Optional[datetime], + tool_call_count: int, + log_ids: list, + action_sequence: list, + workflow_output: WorkflowAnalysisOutput, + error_result: Optional[SessionErrorAnalysis], + error_analysis_id: Optional[uuid.UUID], + workflow_analysis_id: uuid.UUID, + total_tool_duration_ms: int = 0, + search_queries: Optional[list] = None, + total_cached_plans_returned: int = 0, + tool_router_session_id: Optional[str] = None, + project_uuid: Optional[str] = None, + ) -> uuid.UUID: + """ + Save session summary to rube_session_analyses_v2 table. + + Args: + record_id: UUID for this record (primary key, used for joins) + client_session_id: Unreliable session ID from client (for debugging) + user_id: User identifier + client_name: Name of the client/source + session_start_time: When the session started (optional, defaults to epoch) + session_end_time: When the session ended (optional, defaults to epoch) + tool_call_count: Number of tool calls in the session + log_ids: List of log IDs in the session + action_sequence: Sequence of actions in the session + workflow_output: Workflow analysis output + error_result: Error analysis result (optional) + error_analysis_id: UUID of the error analysis record + workflow_analysis_id: UUID of the workflow analysis record + total_tool_duration_ms: Total tool execution time in milliseconds + search_queries: List of search queries made during the session + total_cached_plans_returned: Count of search results with cached_plan_id (deterministic) + tool_router_session_id: Tool router session ID from sessionInfo.tool_router_session_id + project_uuid: Project UUID from ClickHouse projectId column + + Returns the UUID of the saved record (this is the primary key for joins). + """ + search_queries = search_queries or [] + + # Handle null timestamps - use epoch as fallback + epoch = datetime(1970, 1, 1, 0, 0, 0) + start_time = session_start_time if session_start_time else epoch + end_time = session_end_time if session_end_time else epoch + duration_seconds = ( + (end_time - start_time).total_seconds() + if session_start_time and session_end_time + else 0 + ) + + # Get summary stats from error analysis if available + stats = error_result.summary_stats() if error_result else {} + + # Extract metrics from error_result if available + metrics = error_result.metrics if error_result else None + + # Token usage metrics + total_request_tokens = ( + metrics.token_usage.total_request_tokens if metrics else 0 + ) + total_response_tokens = ( + metrics.token_usage.total_response_tokens if metrics else 0 + ) + large_request_count = metrics.token_usage.large_request_count if metrics else 0 + large_response_count = ( + metrics.token_usage.large_response_count if metrics else 0 + ) + + # Top token consumers (top 10 by total tokens) + top_token_consumers = [] + if metrics: + top_consumers = metrics.token_usage.get_top_consumers(10) + for tc in top_consumers: + top_token_consumers.append( + { + "log_id": tc.log_id, + "tool_name": tc.tool_name, + "toolkit_name": tc.toolkit_name, + "request_tokens": tc.request_tokens, + "response_tokens": tc.response_tokens, + "total_tokens": tc.total_tokens, + } + ) + top_token_consumers_json = json.dumps(top_token_consumers) + + # Workbench metrics (deterministic from logs) + workbench_total_calls = metrics.workbench.total_calls if metrics else 0 + workbench_total_time_ms = metrics.workbench.total_time_ms if metrics else 0 + workbench_error_count = metrics.workbench.error_count if metrics else 0 + + # Multi-execute metrics (deterministic from logs) + multi_execute_success_count = ( + metrics.multi_execute.success_count if metrics else 0 + ) + multi_execute_failure_count = ( + metrics.multi_execute.failure_count if metrics else 0 + ) + multi_execute_total_calls = metrics.multi_execute.total_calls if metrics else 0 + + # Recipe metrics + recipe_success_count = metrics.recipe.success_count if metrics else 0 + recipe_failure_count = metrics.recipe.failure_count if metrics else 0 + + # Connection metrics + manage_connection_calls = ( + metrics.connection.manage_connection_calls if metrics else 0 + ) + unique_toolkits_connected = ( + metrics.connection.unique_toolkits_connected if metrics else [] + ) + + # Search metrics + total_search_calls = metrics.search_quality.total_search_calls if metrics else 0 + cached_plan_returned = ( + 1 if metrics and metrics.search_quality.any_cached_plan_returned else 0 + ) + # Per-usecase cached plan quality is stored in per_usecase_plans_json + overall_search_quality_score = ( + metrics.search_quality.overall_search_quality_score if metrics else None + ) + + # Plan metrics + # total_plans_generated = number of plans saved (including _specific variants) + expanded_plans = expand_plans_with_or_toolkits( + workflow_output.per_usecase_plans + ) + total_plans_generated = len(expanded_plans) + + # total_cached_plans_returned is now passed as a parameter (computed deterministically + # from search_json_entries in save_pipeline_result) + + query = f""" + INSERT INTO {SESSION_ANALYSES_TABLE_FQ} ( + id, + client_session_id, + user_id, + client_name, + session_start_time, + session_end_time, + session_duration_seconds, + analyzed_at, + tool_call_count, + total_tool_duration_ms, + log_ids, + action_sequence, + search_queries, + overall_usecase, + workflow_category, + workflow_summary, + task_difficulty, + toolkits_involved, + tools_involved, + tasks_performed, + search_quality, + confidence_score, + workflow_quality, + primary_failure_reason, + overall_rationale, + total_issues_count, + active_bucket_count, + tool_improvement_count, + meta_tool_improvement_count, + new_tool_suggestion_count, + new_toolkit_suggestion_count, + planner_issue_count, + total_request_tokens, + total_response_tokens, + large_request_count, + large_response_count, + top_token_consumers_json, + workbench_total_calls, + workbench_total_time_ms, + workbench_error_count, + multi_execute_success_count, + multi_execute_failure_count, + multi_execute_total_calls, + recipe_success_count, + recipe_failure_count, + manage_connection_calls, + unique_toolkits_connected, + total_search_calls, + cached_plan_returned, + total_cached_plans_returned, + total_plans_generated, + overall_search_quality_score, + error_analysis_id, + workflow_analysis_id, + tool_router_session_id, + project_uuid + ) VALUES ( + '{record_id}', + '{_escape_string(client_session_id)}', + '{_escape_string(user_id)}', + '{_escape_string(client_name)}', + '{start_time.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]}', + '{end_time.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]}', + {duration_seconds}, + now(), + {tool_call_count}, + {total_tool_duration_ms}, + [{', '.join(f"'{_escape_string(lid)}'" for lid in log_ids)}], + [{', '.join(f"'{_escape_string(a)}'" for a in action_sequence)}], + [{', '.join(f"'{_escape_string(q)}'" for q in search_queries)}], + '{_escape_string(workflow_output.overall_usecase)}', + '{_escape_string(workflow_output.workflow_category)}', + '{_escape_string(workflow_output.workflow_summary)}', + '{_escape_string(workflow_output.task_difficulty.value)}', + [{', '.join(f"'{_escape_string(a)}'" for a in workflow_output.toolkits_involved)}], + [{', '.join(f"'{_escape_string(t)}'" for t in workflow_output.tools_involved)}], + [{', '.join(f"'{_escape_string(t)}'" for t in workflow_output.tasks_performed)}], + '{_escape_string(workflow_output.search_quality.overall_quality.value)}', + {workflow_output.confidence_score}, + '{_escape_string(error_result.workflow_quality_enum.value if error_result else "UNKNOWN")}', + '{_escape_string(error_result.primary_failure_reason if error_result else None)}', + '{_escape_string(error_result.overall_rationale if error_result else "")}', + {stats.get('total_issues', 0)}, + {stats.get('active_bucket_count', 0)}, + {stats.get('tool_improvements_count', 0)}, + {stats.get('meta_tool_improvements_count', 0)}, + {len(workflow_output.new_tool_suggestions)}, + {len(workflow_output.new_toolkit_suggestions)}, + {len(workflow_output.planner_issues.missing_prerequisite_warnings) + len(workflow_output.planner_issues.incorrect_suggestions)}, + {total_request_tokens}, + {total_response_tokens}, + {large_request_count}, + {large_response_count}, + '{_escape_string(top_token_consumers_json)}', + {workbench_total_calls}, + {workbench_total_time_ms}, + {workbench_error_count}, + {multi_execute_success_count}, + {multi_execute_failure_count}, + {multi_execute_total_calls}, + {recipe_success_count}, + {recipe_failure_count}, + {manage_connection_calls}, + [{', '.join(f"'{_escape_string(t)}'" for t in unique_toolkits_connected)}], + {total_search_calls}, + {cached_plan_returned}, + {total_cached_plans_returned}, + {total_plans_generated}, + {overall_search_quality_score if overall_search_quality_score is not None else "NULL"}, + {f"'{error_analysis_id}'" if error_analysis_id else "NULL"}, + '{workflow_analysis_id}', + {f"'{_escape_string(tool_router_session_id)}'" if tool_router_session_id else "NULL"}, + {f"'{_escape_string(project_uuid)}'" if project_uuid else "NULL"} + ) + """ + + self.clickhouse._execute_query(query) + logger.info( + f"Saved session analysis {record_id} (client_session: {client_session_id})" + ) + return record_id + + +def save_pipeline_result( + pipeline_result, # PipelineResult + session, # Session object with metadata + clickhouse: Optional[ClickHouseService] = None, +) -> Optional[uuid.UUID]: + """ + Save a complete PipelineResult to all three ClickHouse tables. + + This is the main entry point for saving analysis results. + + Save order: + 1. Generate session_analysis_id (UUID) - this is the primary key + 2. Save workflow analysis with session_analysis_id + 3. Save error analysis with session_analysis_id (if available) + 4. Save session analysis with references to workflow and error analysis + + Args: + pipeline_result: The result from process_session() + session: The original Session object (for metadata) + clickhouse: Optional ClickHouse service instance + + Returns: + UUID of the session analysis record (primary key), or None if save failed + """ + if not pipeline_result.success: + logger.warning( + f"Skipping save for failed pipeline result: {pipeline_result.error}" + ) + return None + + if not pipeline_result.workflow_result: + logger.warning("No workflow result to save") + return None + + repo = SessionAnalysisRepository(clickhouse) + + # Use the session_id from SessionDetector as our primary key + # This is the UUID we generated when detecting the session, NOT from the client + # It's already a UUID string, so we parse it + session_analysis_id = uuid.UUID(pipeline_result.session_id) + + # Get the actual client session ID (unreliable, for debugging only) + # This comes from session.client_session_id, which is what the client sent + client_session_id = getattr(session, "client_session_id", None) or "" + + try: + # First, extract log_ids, action_sequence, and compute total_tool_duration from calls + # Also build search_json and search_queries from COMPOSIO_SEARCH_TOOLS calls + log_ids = [] + action_sequence = [] + total_tool_duration_ms = 0 + search_queries = [] + search_json_entries = [] + search_call_count = 0 + + for c in session.calls: + if isinstance(c, dict): + log_ids.append(c.get("log_id", "")) + action_name = c.get("actionName", "") + action_sequence.append(action_name) + + # Sum up tool durations - field is "duration" in milliseconds + duration = c.get("duration") or c.get("durationMs") or 0 + if duration is not None: + try: + total_tool_duration_ms += int(duration) + except (ValueError, TypeError): + pass + + # Build search_json and search_queries from COMPOSIO_SEARCH_TOOLS calls + if action_name == "COMPOSIO_SEARCH_TOOLS": + search_call_count += 1 + try: + # Parse request and response + request_raw = c.get("request", "{}") + response_raw = c.get("response", "{}") + + request_data = ( + json.loads(request_raw) + if isinstance(request_raw, str) + else request_raw or {} + ) + response_data = ( + json.loads(response_raw) + if isinstance(response_raw, str) + else response_raw or {} + ) + + # Extract overall_usecase and queries + overall_usecase = request_data.get("overall_usecase", "") + queries = request_data.get("queries", []) + + # Parse response + data = ( + response_data.get("data", {}) + if isinstance(response_data, dict) + else {} + ) + results = ( + data.get("results", []) if isinstance(data, dict) else [] + ) + + # Extract connected apps from toolkit_connection_statuses + connected_apps = [] + toolkit_statuses = ( + data.get("toolkit_connection_statuses", []) + if isinstance(data, dict) + else [] + ) + for conn in toolkit_statuses: + if ( + isinstance(conn, dict) + and conn.get("active_connection") is True + ): + toolkit = conn.get("toolkit") + if toolkit and toolkit not in connected_apps: + connected_apps.append(toolkit) + connected_apps.sort() + + # Get timestamp + timestamp = c.get("timestamp", "") + if hasattr(timestamp, "isoformat"): + timestamp = timestamp.isoformat() + + # Add overall_usecase entry if present + if overall_usecase: + search_queries.append(f"[Overall] {overall_usecase}") + overall_toolkits = [] + overall_main_tools = [] + overall_related_tools = [] + for result in results: + if isinstance(result, dict): + for tk in result.get("toolkits", []): + if ( + isinstance(tk, str) + and tk not in overall_toolkits + ): + overall_toolkits.append(tk) + for slug in result.get( + "main_tool_slugs", + result.get("primary_tool_slugs", []), + ): + if slug and slug not in overall_main_tools: + overall_main_tools.append(slug) + for slug in result.get("related_tool_slugs", []): + if slug and slug not in overall_related_tools: + overall_related_tools.append(slug) + overall_toolkits.sort() + search_json_entries.append( + { + "search_id": f"overall_{search_call_count}", + "log_id": c.get("log_id", ""), + "timestamp": str(timestamp), + "request": { + "use_case": overall_usecase, + "known_fields": "", + }, + "response": { + "toolkits": overall_toolkits, + "main_tools": overall_main_tools, + "related_tools": overall_related_tools, + "connectedApps": connected_apps, + }, + } + ) + + # Create one entry per individual query + for query_idx, query in enumerate(queries, 1): + if isinstance(query, dict): + use_case = query.get("use_case", "") + known_fields = query.get("known_fields", "") + + # Add to search_queries + if use_case: + query_str = use_case + if known_fields: + query_str += f" (known_fields: {known_fields})" + search_queries.append(query_str) + + # Get corresponding result + result = ( + results[query_idx - 1] + if query_idx <= len(results) + else {} + ) + result_toolkits = [] + result_main_tools = [] + result_related_tools = [] + cached_plan_id = None + + if isinstance(result, dict): + result_toolkits = [ + tk + for tk in result.get("toolkits", []) + if isinstance(tk, str) + ] + result_toolkits.sort() + result_main_tools = [ + s + for s in result.get( + "main_tool_slugs", + result.get("primary_tool_slugs", []), + ) + if s + ] + result_related_tools = [ + s + for s in result.get("related_tool_slugs", []) + if s + ] + # API returns plan_id, but we store as cached_plan_id + # for compatibility with ingestion pipeline + cached_plan_id = result.get( + "plan_id" + ) or result.get("cached_plan_id") + + entry = { + "search_id": f"usecase_{search_call_count}_{query_idx}", + "log_id": c.get("log_id", ""), + "timestamp": str(timestamp), + "request": { + "use_case": use_case, + "known_fields": known_fields, + }, + "response": { + "toolkits": result_toolkits, + "main_tools": result_main_tools, + "related_tools": result_related_tools, + "connectedApps": connected_apps, + }, + } + if cached_plan_id: + entry["cached_plan_id"] = cached_plan_id + search_json_entries.append(entry) + except Exception as e: + # If parsing fails, skip this search call (keep pipeline resilient) + logger.debug( + "Failed to parse COMPOSIO_SEARCH_TOOLS call into search_json_entries: %s", + e, + ) + else: + log_ids.append(getattr(c, "log_id", "")) + action_sequence.append(getattr(c, "action_name", "")) + duration = getattr(c, "duration", 0) or 0 + if duration is not None: + try: + total_tool_duration_ms += int(duration) + except (ValueError, TypeError): + pass + + # Add search_json_entries to search_data for save_workflow_analysis + search_data = ( + getattr(pipeline_result.workflow_result, "search_data", None) or {} + ) + search_data["search_json_entries"] = search_json_entries + + # Count cached plans returned deterministically from search_json_entries + # A cached plan is returned when cached_plan_id is present in the search response + total_cached_plans_returned = sum( + 1 for entry in search_json_entries if entry.get("cached_plan_id") + ) + + # 1. Save workflow analysis (with session_analysis_id link) + # Include search_data for search_json compatibility with ingestion pipeline + workflow_analysis_id = repo.save_workflow_analysis( + session_analysis_id=session_analysis_id, + client_session_id=client_session_id, + workflow_output=pipeline_result.workflow_result.output, + search_data=search_data, + ) + + # 2. Save error analysis (if available, with session_analysis_id link) + error_analysis_id = None + client_name = getattr(session, "source", "") or "" + if pipeline_result.error_result: + error_analysis_id = repo.save_error_analysis( + session_analysis_id=session_analysis_id, + client_session_id=client_session_id, + client_name=client_name, + error_result=pipeline_result.error_result, + ) + + # 3. Save session summary (main table, with links to detail tables) + # Use the session_analysis_id we generated upfront for consistency + # Get tool_router_session_id and project_uuid from session + tool_router_session_id = getattr(session, "tool_router_session_id", None) + project_uuid = getattr(session, "project_uuid", None) + + repo.save_session_analysis( + record_id=session_analysis_id, + client_session_id=client_session_id, + user_id=session.user_id, + client_name=client_name, # Use source as client_name + session_start_time=session.start_time, + session_end_time=session.end_time, + tool_call_count=len(session.calls), + log_ids=log_ids, + action_sequence=action_sequence, + workflow_output=pipeline_result.workflow_result.output, + error_result=pipeline_result.error_result, + error_analysis_id=error_analysis_id, + workflow_analysis_id=workflow_analysis_id, + total_tool_duration_ms=total_tool_duration_ms, + search_queries=search_queries, + total_cached_plans_returned=total_cached_plans_returned, + tool_router_session_id=tool_router_session_id, + project_uuid=project_uuid, + ) + + logger.info( + f"Saved complete pipeline result: " + f"session_analysis={session_analysis_id}, workflow={workflow_analysis_id}, " + f"error={error_analysis_id}, client_session={client_session_id}" + ) + + return session_analysis_id + + except Exception as e: + logger.error(f"Failed to save pipeline result: {e}", exc_info=True) + return None diff --git a/app_tester/rube_learning/database/schema.py b/app_tester/rube_learning/database/schema.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/database/schema.py @@ -0,0 +1,252 @@ +""" +ClickHouse schema definitions for session analysis tables. + +Tables: + - rube_session_analyses_v2: Session-level summary (main table) + - rube_error_analyses: Full error bucket data (links back via session_analysis_id) + - rube_workflow_analyses: Workflow plans and suggestions (links back via session_analysis_id) + +Relationships: + - rube_session_analyses_v2.id is the primary key + - rube_session_analyses_v2.error_analysis_id -> rube_error_analyses.id + - rube_session_analyses_v2.workflow_analysis_id -> rube_workflow_analyses.id + - rube_error_analyses.session_analysis_id -> rube_session_analyses_v2.id + - rube_workflow_analyses.session_analysis_id -> rube_session_analyses_v2.id + +Note: client_session_id is the unreliable session ID from the client (for debugging only). + Use the UUID `id` field for reliable joins between tables. +""" + +from typing import Optional + +from app_tester.agent_worker_fix_from_logs.services.clickhouse_service import ( + ClickHouseService, +) + + +# ============================================================================= +# Table Names Configuration +# ============================================================================= +DATABASE_NAME = "app_db" +SESSION_ANALYSES_TABLE = "rube_session_analyses_v2" +ERROR_ANALYSES_TABLE = "rube_error_analyses" +WORKFLOW_ANALYSES_TABLE = "rube_workflow_analyses" + +# Fully qualified table names +SESSION_ANALYSES_TABLE_FQ = f"{DATABASE_NAME}.{SESSION_ANALYSES_TABLE}" +ERROR_ANALYSES_TABLE_FQ = f"{DATABASE_NAME}.{ERROR_ANALYSES_TABLE}" +WORKFLOW_ANALYSES_TABLE_FQ = f"{DATABASE_NAME}.{WORKFLOW_ANALYSES_TABLE}" + + +# ============================================================================= +# Table 1: Session Analyses (Summary) - MAIN TABLE +# ============================================================================= + +CREATE_SESSION_ANALYSES_TABLE = f""" +CREATE TABLE IF NOT EXISTS {SESSION_ANALYSES_TABLE_FQ} ( + -- Identity (id is the primary key used for joins) + id UUID, + client_session_id String, -- Unreliable session ID from client (for debugging) + user_id String, + client_name String, + + -- Timing + session_start_time DateTime64(3), + session_end_time DateTime64(3), + session_duration_seconds Float32, + analyzed_at DateTime64(3) DEFAULT now(), + + -- Session metadata + tool_call_count UInt16, + total_tool_duration_ms UInt32 DEFAULT 0, -- Total tool execution time in milliseconds + log_ids Array(String), + action_sequence Array(String), + search_queries Array(String) DEFAULT [], -- Search queries made during the session + + -- Workflow Analysis (Stage 1) - Summary fields + overall_usecase String, + workflow_category String, + workflow_summary String, + task_difficulty String, -- 'EASY', 'MEDIUM', 'HARD' + toolkits_involved Array(String), + tools_involved Array(String), + tasks_performed Array(String) DEFAULT [], -- High-level tasks performed in the workflow + search_quality String, -- 'GOOD', 'ACCEPTABLE', 'POOR' + confidence_score Float32, + + -- Error Analysis (Stage 2) - Summary fields + workflow_quality String, -- 'SUCCESS', 'PARTIAL_SUCCESS', 'FAILURE', 'INCOMPLETE' + primary_failure_reason Nullable(String), + overall_rationale String, + total_issues_count UInt16, + active_bucket_count UInt8, + + -- Counts for quick filtering + tool_improvement_count UInt16, + meta_tool_improvement_count UInt16, + new_tool_suggestion_count UInt16, + new_toolkit_suggestion_count UInt16, + planner_issue_count UInt16, + + -- Session Metrics (computed from logs, not LLM) + -- Token Usage + total_request_tokens UInt32 DEFAULT 0, + total_response_tokens UInt32 DEFAULT 0, + large_request_count UInt16 DEFAULT 0, + large_response_count UInt16 DEFAULT 0, + -- Top token consumers as JSON array: [{{log_id, tool_name, toolkit_name, request_tokens, response_tokens, total_tokens}}] + top_token_consumers_json String DEFAULT '[]', + + -- Workbench Metrics (deterministic from logs) + workbench_total_calls UInt16 DEFAULT 0, + workbench_total_time_ms UInt32 DEFAULT 0, + workbench_error_count UInt16 DEFAULT 0, -- Count of workbench calls with errors + + -- Multi-Execute Metrics (deterministic from logs) + multi_execute_success_count UInt16 DEFAULT 0, + multi_execute_failure_count UInt16 DEFAULT 0, + multi_execute_total_calls UInt16 DEFAULT 0, + + -- Recipe Metrics + recipe_success_count UInt16 DEFAULT 0, + recipe_failure_count UInt16 DEFAULT 0, + + -- Connection Metrics + manage_connection_calls UInt16 DEFAULT 0, + unique_toolkits_connected Array(String) DEFAULT [], + + -- Search Metrics + total_search_calls UInt16 DEFAULT 0, + cached_plan_returned UInt8 DEFAULT 0, + total_cached_plans_returned UInt16 DEFAULT 0, -- Count of cached plans returned by search + total_plans_generated UInt16 DEFAULT 0, -- Total number of plans generated by LLM pipeline + cached_plan_quality Nullable(String), + cached_plan_quality_reason Nullable(String), -- Why the cached plan was rated this way + overall_search_quality_score Nullable(Float32), + + -- Foreign keys to detail tables + error_analysis_id Nullable(UUID), -- Nullable because error analysis can be skipped or fail + workflow_analysis_id UUID, + + -- Project and session information + tool_router_session_id Nullable(String), -- Tool router session ID from sessionInfo.tool_router_session_id + project_uuid Nullable(String), -- Project UUID from ClickHouse projectId column + + -- Version + analyzer_version String DEFAULT '2.2' +) ENGINE = MergeTree() +ORDER BY (session_start_time, user_id, id) +""" + + +# ============================================================================= +# Table 2: Error Analyses (Full Buckets) +# ============================================================================= + +CREATE_ERROR_ANALYSES_TABLE = f""" +CREATE TABLE IF NOT EXISTS {ERROR_ANALYSES_TABLE_FQ} ( + -- Identity + id UUID, + session_analysis_id UUID, -- Links back to rube_session_analyses_v2.id + client_session_id String, -- Unreliable session ID from client (for debugging) + client_name String, -- Name of the client/source (denormalized for faster queries) + analyzed_at DateTime64(3) DEFAULT now(), + + -- Overall assessment + workflow_quality String, -- 'SUCCESS', 'PARTIAL_SUCCESS', 'FAILURE', 'INCOMPLETE' + overall_rationale String, + primary_failure_reason Nullable(String), + + -- Bucket weights (for quick aggregation without JSON parsing) + llm_errors_weight Float32, + tool_errors_weight Float32, + search_quality_weight Float32, + connection_problems_weight Float32, + workbench_weight Float32, + multi_execute_weight Float32, + recipe_weight Float32, + other_errors_weight Float32, + + -- Full bucket data as JSON + llm_errors_json String, + tool_errors_json String, + search_quality_json String, + connection_problems_json String, + workbench_json String, + multi_execute_json String, + recipe_json String, + + -- Improvements as JSON + tool_improvements_json String, + meta_tool_improvements_json String, + + -- Top issues (precomputed for quick display) + top_issues_json String +) ENGINE = MergeTree() +ORDER BY (analyzed_at, session_analysis_id, id) +""" + + +# ============================================================================= +# Table 3: Workflow Analyses (Plans & Suggestions) +# ============================================================================= + +CREATE_WORKFLOW_ANALYSES_TABLE = f""" +CREATE TABLE IF NOT EXISTS {WORKFLOW_ANALYSES_TABLE_FQ} ( + -- Identity + id UUID, + session_analysis_id UUID, -- Links back to rube_session_analyses_v2.id + client_session_id String, -- Unreliable session ID from client (for debugging) + analyzed_at DateTime64(3) DEFAULT now(), + + -- Workflow understanding + overall_usecase String, + workflow_category String, + workflow_summary String, + task_difficulty String, -- 'EASY', 'MEDIUM', 'HARD' + toolkits_involved Array(String), + tools_involved Array(String), + confidence_score Float32, + + -- Search quality details (no weights - from Stage 1) + search_quality String, -- 'GOOD', 'ACCEPTABLE', 'POOR' + search_quality_explanation String, + wrong_tools_returned_json String, -- JSON array of WrongToolsReturnedDetail + + -- Per-usecase data as JSON + per_usecase_plans_json String, + per_usecase_tool_analysis_json String, + + -- Raw search data (for ingestion pipeline compatibility) + -- Contains per-usecase search results as JSON array + search_json String DEFAULT '[]', + + -- Suggestions as JSON + new_tool_suggestions_json String, + new_toolkit_suggestions_json String, + planner_issues_json String +) ENGINE = MergeTree() +ORDER BY (analyzed_at, session_analysis_id, id) +""" + + +def create_all_tables(clickhouse: Optional[ClickHouseService] = None) -> None: + """ + Create all session analysis tables if they don't exist. + + Args: + clickhouse: ClickHouse service instance. If None, creates a new one. + """ + if clickhouse is None: + clickhouse = ClickHouseService() + + print("Creating rube_session_analyses_v2 table...") + clickhouse._execute_query(CREATE_SESSION_ANALYSES_TABLE) + + print("Creating rube_error_analyses table...") + clickhouse._execute_query(CREATE_ERROR_ANALYSES_TABLE) + + print("Creating rube_workflow_analyses table...") + clickhouse._execute_query(CREATE_WORKFLOW_ANALYSES_TABLE) + + print("All tables created successfully.") diff --git a/app_tester/rube_learning/error_analysis/__init__.py b/app_tester/rube_learning/error_analysis/__init__.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/__init__.py @@ -0,0 +1,74 @@ +""" +Error Analysis Pipeline (Call 2) + +This module implements the second LLM call in the session analysis pipeline. +It analyzes session errors, assigns weights, and generates improvement suggestions. + +Structure: + models/ - Pydantic models for error buckets + extractors.py - Functions to extract static data from logs + runner.py - Main error analysis runner + +Usage: + from app_tester.rube_learning.error_analysis import ( + run_error_analysis, + SessionErrorAnalysis, + ErrorAnalysisLLMOutput, + ) +""" + +from .extractors import ( + RawConnectionFailure, + RawExecutionError, + RawMultiExecuteFailure, + RawRecipeFailure, + RawWorkbenchCall, + StaticExtractionResult, + extract_all_static_data, +) +from .models import ( # Top-level output models; Bucket models; Improvement models + ConnectionProblemsBucket, + ErrorAnalysisLLMOutput, + LLMErrorBucket, + MetaToolImprovement, + MultiExecuteBucket, + RecipeBucket, + SearchQualityBucket, + SessionErrorAnalysis, + ToolErrorBucket, + ToolImprovement, + WorkbenchBucket, + WorkflowStatus, +) +from .runner import merge_llm_output_with_static_data, run_error_analysis + + +__all__ = [ + # Runner + "run_error_analysis", + "merge_llm_output_with_static_data", + # Extractors + "extract_all_static_data", + "StaticExtractionResult", + "RawExecutionError", + "RawConnectionFailure", + "RawMultiExecuteFailure", + "RawRecipeFailure", + "RawWorkbenchCall", + # Output models + "ErrorAnalysisLLMOutput", + "SessionErrorAnalysis", + "WorkflowStatus", + # Buckets + "LLMErrorBucket", + "ToolErrorBucket", + "SearchQualityBucket", + "SearchQualityBucket", + "ConnectionProblemsBucket", + "WorkbenchBucket", + "MultiExecuteBucket", + "RecipeBucket", + # Improvements + "ToolImprovement", + "MetaToolImprovement", +] diff --git a/app_tester/rube_learning/error_analysis/extractors.py b/app_tester/rube_learning/error_analysis/extractors.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/extractors.py @@ -0,0 +1,542 @@ +""" +Static Data Extractors - Extract raw facts from session logs. + +These functions extract ONLY static data that can be read directly from logs: +- Log IDs +- Action names +- Error messages (extracted ONCE here, used everywhere) +- Timestamps +- Durations +- Execution outputs + +NO inference, NO categorization - that's the LLM's job. + +The extracted data is used for: +1. Counts in static data summary +2. Full error details in session logs formatting +3. Tests to verify extraction works +""" + +import logging +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from app_tester.rube_learning.call_parser import ParsedCall, ParsedSession +from app_tester.rube_learning.config import ( + MULTI_EXECUTE_ERROR_RESPONSE_MAX_LENGTH, + WORKBENCH_ERROR_OUTPUT_MAX_LENGTH, + WORKBENCH_SUCCESS_OUTPUT_MAX_LENGTH, +) +from app_tester.rube_learning.session_detector import Session + + +logger = logging.getLogger(__name__) + + +@dataclass +class RawExecutionError: + """Raw execution error extracted from logs.""" + + action_name: str + log_id: Optional[str] + error_message: str # Full error message from call + timestamp: Optional[str] = None + duration_ms: Optional[int] = None + + +@dataclass +class RawConnectionFailure: + """Raw connection failure extracted from logs.""" + + toolkit_name: str + log_id: Optional[str] + error_message: str # Full error message from call + + +@dataclass +class RawMultiExecuteFailure: + """Raw multi-execute failure extracted from logs.""" + + log_id: Optional[str] + actions_attempted: List[str] + error_response: str # Full formatted error response (tool results, errors, etc.) + + +@dataclass +class RawRecipeFailure: + """Raw recipe failure extracted from logs.""" + + recipe_name: str + log_id: Optional[str] + error_message: str # Full error message from call + + +@dataclass +class RawWorkbenchCall: + """Raw workbench call data including code and output.""" + + log_id: str + duration_ms: Optional[int] + had_error: bool + error_message: Optional[str] = None # Error message if had_error is True + # Request fields + code_executed: Optional[str] = None + thought: Optional[str] = None + # Response/output - extracted once, used for both counts and session logs + execution_output: Optional[str] = None + + +@dataclass +class RawToolCallTokens: + """Token usage for a single tool call.""" + + log_id: str + tool_name: str + toolkit_name: Optional[str] + request_tokens: int + response_tokens: int + + +@dataclass +class StaticExtractionResult: + """All static data extracted from a session.""" + + # Raw errors (LLM will categorize and weight these) + execution_errors: List[RawExecutionError] = field(default_factory=list) + connection_failures: List[RawConnectionFailure] = field(default_factory=list) + multi_execute_failures: List[RawMultiExecuteFailure] = field(default_factory=list) + recipe_failures: List[RawRecipeFailure] = field(default_factory=list) + + # Raw workbench data + workbench_calls: List[RawWorkbenchCall] = field(default_factory=list) + + # Simple counts + multi_execute_success_count: int = 0 + recipe_success_count: int = 0 + recipes_called: List[str] = field(default_factory=list) + + # Connection metrics + manage_connection_calls: int = 0 + toolkits_connected: List[str] = field(default_factory=list) + + # Search metrics + search_calls: int = 0 + get_tool_schemas_calls: int = 0 + + # Token usage per tool call + tool_call_tokens: List[RawToolCallTokens] = field(default_factory=list) + + +def extract_all_static_data( + session: Session, count_tokens: bool = True +) -> StaticExtractionResult: + """ + Extract all static data from a session. + + Only extracts raw facts - no inference or categorization. + + Args: + session: The session to extract data from + count_tokens: Whether to count tokens for each tool call (default True) + """ + # Parse all calls once upfront + parsed_session = ParsedSession.from_session(session) + + result = StaticExtractionResult() + + for call in parsed_session.calls: + # Count tokens for all tool calls (not just errors) + # For MULTI_EXECUTE, we extract individual app tool tokens instead + if count_tokens and call.action_name: + if call.action_name == "COMPOSIO_MULTI_EXECUTE_TOOL": + # Extract tokens for individual app tools within multi-execute + _extract_multi_execute_app_tool_tokens(call, result) + else: + _extract_token_usage(call, result) + + # Route to appropriate extractor based on action name + if call.action_name == "COMPOSIO_MANAGE_CONNECTIONS": + _extract_connection_data(call, result) + elif call.action_name == "COMPOSIO_MULTI_EXECUTE_TOOL": + _extract_multi_execute_data(call, result) + elif call.action_name == "COMPOSIO_EXECUTE_RECIPE": + _extract_recipe_data(call, result) + elif call.action_name == "COMPOSIO_REMOTE_WORKBENCH": + _extract_workbench_data(call, result) + elif call.action_name == "COMPOSIO_SEARCH_TOOLS": + result.search_calls += 1 + elif call.action_name == "COMPOSIO_GET_TOOL_SCHEMAS": + result.get_tool_schemas_calls += 1 + elif not call.action_name.startswith("COMPOSIO_") and call.has_error: + # Regular tool execution error + result.execution_errors.append( + RawExecutionError( + action_name=call.action_name, + log_id=call.log_id, + error_message=call.error_message or "Unknown error", + timestamp=str(call.timestamp) if call.timestamp else None, + duration_ms=call.duration_ms, + ) + ) + + return result + + +def _extract_connection_data(call: ParsedCall, result: StaticExtractionResult) -> None: + """Extract connection-related data.""" + # Always count the call + result.manage_connection_calls += 1 + + toolkit_name = "unknown" + if call.request: + # Handle both single toolkit and multiple toolkits + toolkits = call.request.get("toolkits", []) + if toolkits: + # Batch format: collect all toolkit names + toolkit_names_in_batch = [] + for tk in toolkits: + tk_upper = tk.upper() if isinstance(tk, str) else "unknown" + toolkit_names_in_batch.append(tk_upper) + if tk_upper not in result.toolkits_connected: + result.toolkits_connected.append(tk_upper) + # Use comma-separated list for error reporting + if toolkit_names_in_batch: + toolkit_name = ",".join(toolkit_names_in_batch) + else: + toolkit_name = ( + call.request.get("app") or call.request.get("toolkit") or "unknown" + ) + tk_upper = toolkit_name.upper() + if tk_upper not in result.toolkits_connected: + result.toolkits_connected.append(tk_upper) + + if call.has_error: + result.connection_failures.append( + RawConnectionFailure( + toolkit_name=toolkit_name.upper(), + log_id=call.log_id, + error_message=call.error_message or "Unknown error", + ) + ) + + +def _extract_multi_execute_data( + call: ParsedCall, result: StaticExtractionResult +) -> None: + """Extract multi-execute data including full error response.""" + # Get actions attempted from request + actions: List[str] = [] + if call.request: + # Try "actions" field first, then "tools" field (newer format) + actions_list = call.request.get("actions", []) or call.request.get("tools", []) + for action in actions_list: + if isinstance(action, dict): + # Handle both "action" and "tool_slug" field names + action_name = ( + action.get("action") or action.get("tool_slug") or "unknown" + ) + actions.append(action_name) + elif isinstance(action, str): + actions.append(action) + + # Check for any failure and build error response + has_failure = call.has_error + error_response_parts = [] + + # Check errorRequest (populated in error cases) + if call.error_request: + has_failure = True + error_response_parts.extend( + _format_multi_execute_error_response( + call.error_request, call.top_level_error + ) + ) + + # Also check normal response for individual tool failures (partial success case) + if call.response: + data = call.response.get("data", call.response) + results = data.get("results", []) if isinstance(data, dict) else [] + for r in results: + if isinstance(r, dict) and r.get("error"): + has_failure = True + tool_slug = r.get("tool_slug", "unknown") + tool_error = r.get("error", "") + error_response_parts.append(f" - {tool_slug}: ERROR - {tool_error}") + + if has_failure: + error_response = "\n".join(error_response_parts) + # Apply max length + if len(error_response) > MULTI_EXECUTE_ERROR_RESPONSE_MAX_LENGTH: + error_response = ( + error_response[:MULTI_EXECUTE_ERROR_RESPONSE_MAX_LENGTH] + + "...[TRUNCATED]" + ) + result.multi_execute_failures.append( + RawMultiExecuteFailure( + log_id=call.log_id, + actions_attempted=actions, + error_response=error_response, + ) + ) + else: + result.multi_execute_success_count += 1 + + +def _format_multi_execute_error_response( + error_request: Dict[str, Any], top_error: Optional[str] +) -> List[str]: + """Format multi-execute error response into parts.""" + parts = [] + + # Get top-level error + error = error_request.get("error", "") or top_error + if error: + parts.append(f"Error: {error}") + + # Get successful flag + successful = error_request.get("successful", None) + if successful is not None: + parts.append(f"Successful: {successful}") + + # Extract data section for tool-level results + data = error_request.get("data", {}) + if isinstance(data, dict): + results = data.get("results", []) + if isinstance(results, list) and results: + tool_results = [] + for r in results: + if isinstance(r, dict): + tool_slug = r.get("tool_slug", "unknown") + tool_error = r.get("error", "") + tool_success = r.get("successful", None) + if tool_error: + tool_results.append(f" - {tool_slug}: ERROR - {tool_error}") + elif tool_success is False: + tool_results.append(f" - {tool_slug}: FAILED") + else: + tool_results.append(f" - {tool_slug}: OK") + if tool_results: + parts.append("Tool Results:\n" + "\n".join(tool_results)) + + # Get data.error if different from top-level + data_error = data.get("error", "") + if data_error and str(data_error) != str(error): + parts.append(f"Data Error: {data_error}") + + # If no specific error fields found but errorRequest exists, include raw summary + if not parts: + for key in ["successful", "version", "error"]: + if key in error_request: + parts.append(f"{key}: {error_request[key]}") + if data and isinstance(data, dict): + data_keys = list(data.keys())[:5] + if data_keys: + parts.append(f"data keys: {data_keys}") + + return parts + + +def _extract_recipe_data(call: ParsedCall, result: StaticExtractionResult) -> None: + """Extract recipe data.""" + recipe_name = "unknown" + if call.request: + recipe_name = ( + call.request.get("recipe_name") or call.request.get("recipe") or "unknown" + ) + + result.recipes_called.append(recipe_name) + + if call.has_error: + result.recipe_failures.append( + RawRecipeFailure( + recipe_name=recipe_name, + log_id=call.log_id, + error_message=call.error_message or "Unknown error", + ) + ) + else: + result.recipe_success_count += 1 + + +def _extract_workbench_data(call: ParsedCall, result: StaticExtractionResult) -> None: + """Extract workbench call data including full code and output.""" + # Get code_to_execute and thought from request + code_executed = None + thought = None + + if call.request: + # Handle both direct params and nested params structure + params = call.request.get("params", call.request) + if isinstance(params, dict): + code_executed = params.get("code_to_execute") + thought = params.get("thought") or params.get("thought_process") + + # Extract execution output from the effective response + execution_output = None + is_error_response = call.error_request is not None + + if call.error_request: + # Error case: combine results, stdout, and error message + data = call.error_request.get("data", {}) + if isinstance(data, dict): + error_parts = [] + + results = data.get("results", "") + if results: + error_parts.append(f"Results: {results}") + + stdout = data.get("stdout", "") + if stdout: + error_parts.append(f"Stdout: {stdout}") + + error_msg = data.get("error", "") + if error_msg: + error_parts.append(f"Error: {error_msg}") + + if error_parts: + execution_output = "\n\n".join(error_parts) + + elif call.response: + # Success case: look for output in standard fields + data = call.response.get("data", {}) + if isinstance(data, dict): + execution_output = ( + data.get("output") + or data.get("result") + or data.get("stdout") + or data.get("execution_result") + ) + elif data: + execution_output = str(data) + + # Also check for direct output at response level + if not execution_output: + execution_output = ( + call.response.get("output") + or call.response.get("result") + or call.response.get("stdout") + ) + + # Truncate based on whether it's an error or success + if execution_output: + execution_output = str(execution_output) + max_length = ( + WORKBENCH_ERROR_OUTPUT_MAX_LENGTH + if is_error_response + else WORKBENCH_SUCCESS_OUTPUT_MAX_LENGTH + ) + if len(execution_output) > max_length: + execution_output = execution_output[:max_length] + "...[TRUNCATED]" + + result.workbench_calls.append( + RawWorkbenchCall( + log_id=call.log_id or "", + duration_ms=call.duration_ms, + had_error=call.has_error, + error_message=call.error_message, + code_executed=code_executed, + thought=thought, + execution_output=execution_output if execution_output else None, + ) + ) + + +def _extract_token_usage(call: ParsedCall, result: StaticExtractionResult) -> None: + """Extract token usage for a tool call.""" + from .token_counter import count_request_response_tokens + + # Use raw request/response strings for token counting + request = call._raw.get("request") + response = call._raw.get("response") + + request_tokens, response_tokens = count_request_response_tokens(request, response) + + # Extract toolkit name from action name (e.g., GMAIL_SEND_EMAIL -> GMAIL) + toolkit_name = None + if "_" in call.action_name and not call.action_name.startswith("COMPOSIO_"): + toolkit_name = call.action_name.split("_")[0] + + result.tool_call_tokens.append( + RawToolCallTokens( + log_id=call.log_id or "", + tool_name=call.action_name, + toolkit_name=toolkit_name, + request_tokens=request_tokens, + response_tokens=response_tokens, + ) + ) + + +def _extract_multi_execute_app_tool_tokens( + call: ParsedCall, + result: StaticExtractionResult, +) -> None: + """ + Extract token usage for individual app tools within a multi-execute call. + + Instead of counting tokens for the entire multi-execute request/response, + we extract and count tokens for each individual app tool that was executed. + """ + from .token_counter import count_request_response_tokens + + if not call.request or not call.response: + return + + # Extract tools from request (params.tools or just tools) + params = call.request.get("params", call.request) + request_tools = params.get("tools", []) if isinstance(params, dict) else [] + + # Extract results from response (data.results) + data = call.response.get("data", call.response) + response_results = data.get("results", []) if isinstance(data, dict) else [] + + # Build a map of tool_slug -> request arguments for matching + request_by_index: Dict[int, Dict] = {} + request_by_slug: Dict[str, Dict] = {} + for idx, tool in enumerate(request_tools): + if isinstance(tool, dict): + tool_slug = tool.get("tool_slug", "") + request_by_index[idx] = tool + if tool_slug: + request_by_slug[tool_slug] = tool + + # Process each result and count tokens for the individual app tool + for resp_item in response_results: + if not isinstance(resp_item, dict): + continue + + tool_slug = resp_item.get("tool_slug", "") + if not tool_slug: + continue + + # Get the index to match with request + index = resp_item.get("index") + tool_response = resp_item.get("response", {}) + + # Find matching request - prefer by index, fallback to slug + tool_request = None + if index is not None and index in request_by_index: + tool_request = request_by_index[index] + elif tool_slug in request_by_slug: + tool_request = request_by_slug[tool_slug] + + # Count tokens for this individual app tool + tool_request_data = tool_request.get("arguments", {}) if tool_request else {} + request_tokens, response_tokens = count_request_response_tokens( + tool_request_data, tool_response + ) + + # Extract toolkit name from tool_slug + toolkit_name = None + if "_" in tool_slug: + toolkit_name = tool_slug.split("_")[0] + + result.tool_call_tokens.append( + RawToolCallTokens( + log_id=call.log_id or "", + tool_name=tool_slug, + toolkit_name=toolkit_name, + request_tokens=request_tokens, + response_tokens=response_tokens, + ) + ) diff --git a/app_tester/rube_learning/error_analysis/models/__init__.py b/app_tester/rube_learning/error_analysis/models/__init__.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/models/__init__.py @@ -0,0 +1,130 @@ +""" +Error bucket models for session analysis. + +Each file contains models for one error bucket category. +Models are split into: + - Item models (individual errors with weights) + - Bucket models (container with bucket-level weight) + - Metrics models (static metrics computed from logs, not by LLM) +""" + +from app_tester.rube_learning.common import PlanQuality + +from .connection_problems import ( + ConnectionErrorInToolExecution, + ConnectionProblemsBucket, + ManageConnectionFailed, + ManageConnectionNotCalled, +) +from .improvements import MetaToolImprovement, ToolImprovement +from .llm_errors import ( + HallucinatedToolCalled, + LLMErrorBucket, + MissedPagination, + MultiExecuteNotCalled, + MultiExecuteWronglyCalled, + WrongInputType, + WrongInputValue, +) +from .metrics import ( + ConnectionMetrics, + FalseConnectionStatus, + MultiExecuteMetrics, + PerUsecaseCachedPlanQuality, + RecipeMetrics, + SearchQualityMetrics, + SessionMetrics, + TokenUsageMetrics, + ToolCallTokenUsage, + WorkbenchMetrics, + WorkbenchTimeTaken, +) +from .multi_execute import MultiExecuteBucket, MultiExecuteMetaToolFailure +from .other_errors import OtherError, OtherErrorsBucket +from .output import ErrorAnalysisLLMOutput, SessionErrorAnalysis, WorkflowStatus +from .recipe import RecipeBucket, RecipeFailure +from .search_quality import SearchQualityBucket, WrongToolsReturned +from .tool_errors import ( + MissingFeatureImpact, + MissingToolImpact, + RequestSchemaProblem, + ResourceNotFound, + ResponseSchemaProblem, + ToolErrorBucket, + ToolImplementationProblem, +) +from .workbench import ( + CodeLogicError, + MetaToolFailure, + MissingPackage, + UnnecessaryWorkbenchCall, + WorkbenchBucket, + WorkbenchExecutionCount, + WorkbenchHelperToolProblem, +) + + +__all__ = [ + # LLM Errors + "WrongInputValue", + "WrongInputType", + "MultiExecuteWronglyCalled", + "MissedPagination", + "HallucinatedToolCalled", + "LLMErrorBucket", + # Tool Errors + "RequestSchemaProblem", + "ResponseSchemaProblem", + "ToolImplementationProblem", + "ResourceNotFound", + "MissingToolImpact", + "MissingFeatureImpact", + "ToolErrorBucket", + # Search Quality + "WrongToolsReturned", + "SearchQualityBucket", + "SearchQualityBucket", + "PlanQuality", + # Connection Problems + "ManageConnectionNotCalled", + "ManageConnectionFailed", + "ConnectionErrorInToolExecution", + "ConnectionProblemsBucket", + # Workbench + "CodeLogicError", + "WorkbenchHelperToolProblem", + "UnnecessaryWorkbenchCall", + "MetaToolFailure", + "MissingPackage", + "WorkbenchExecutionCount", + "WorkbenchBucket", + # Multi-Execute + "MultiExecuteNotCalled", + "MultiExecuteMetaToolFailure", + "MultiExecuteBucket", + # Recipe + "RecipeFailure", + "RecipeBucket", + # Other Errors (catch-all) + "OtherError", + "OtherErrorsBucket", + # Improvements + "ToolImprovement", + "MetaToolImprovement", + # Metrics (static, computed from logs, not by LLM) + "SessionMetrics", + "WorkbenchMetrics", + "WorkbenchTimeTaken", + "MultiExecuteMetrics", + "RecipeMetrics", + "ConnectionMetrics", + "FalseConnectionStatus", + "SearchQualityMetrics", + "PerUsecaseCachedPlanQuality", + "TokenUsageMetrics", + "ToolCallTokenUsage", + # Output + "ErrorAnalysisLLMOutput", + "SessionErrorAnalysis", + "WorkflowStatus", +] diff --git a/app_tester/rube_learning/error_analysis/models/base_bucket.py b/app_tester/rube_learning/error_analysis/models/base_bucket.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/models/base_bucket.py @@ -0,0 +1,99 @@ +""" +Base class for error bucket models. + +Provides common functionality for all bucket types: +- has_issues(): Check if bucket has any issues +- item_count(): Count total items in bucket +- get_item_weight_sum(): Sum all item weights in bucket + +Each bucket implements _get_item_lists() to return its specific item lists. +""" + +from typing import Any, Callable, List, Optional, Tuple + +from pydantic import BaseModel + + +class ErrorBucketBase(BaseModel): + """ + Base class for all error bucket models. + + Subclasses must implement _get_item_lists() to return a list of tuples: + (item_list, filter_predicate) + + Where: + - item_list: List of items (each item should have a 'weight' attribute) + - filter_predicate: Optional callable that filters items (default: None means no filter) + + Example: + def _get_item_lists(self): + return [ + (self.wrong_input_values, None), # None means no filter + (self.execution_counts, lambda x: x.was_excessive), # Custom filter + ] + """ + + weight: float = 0.0 + + def _get_item_lists( + self, + ) -> List[Tuple[List[Any], Optional[Callable[[Any], bool]]]]: + """ + Return list of (item_list, filter_predicate) tuples for this bucket. + + Each item_list should contain items with a 'weight' attribute. + filter_predicate is optional - if None, all items are included. + + Subclasses MUST implement this method. + """ + raise NotImplementedError( + "Subclasses must implement _get_item_lists() to return their item lists" + ) + + def has_issues(self) -> bool: + """Returns True if this bucket has any issues.""" + item_lists: List[Tuple[List[Any], Optional[Callable[[Any], bool]]]] = ( + self._get_item_lists() + ) + for item_list, filter_predicate in item_lists: + if filter_predicate is None: + # No filter - check if list has any items + if item_list: + return True + else: + # Apply filter and check if any items match + if any(filter_predicate(item) for item in item_list): + return True + return False + + def item_count(self) -> int: + """Total number of items in this bucket (after applying filters).""" + item_lists: List[Tuple[List[Any], Optional[Callable[[Any], bool]]]] = ( + self._get_item_lists() + ) + count = 0 + for item_list, filter_predicate in item_lists: + if filter_predicate is None: + # No filter - count all items + count += len(item_list) + else: + # Apply filter and count matching items + count += sum(1 for item in item_list if filter_predicate(item)) + return count + + def get_item_weight_sum(self) -> float: + """Sum of all item weights in this bucket (after applying filters).""" + item_lists: List[Tuple[List[Any], Optional[Callable[[Any], bool]]]] = ( + self._get_item_lists() + ) + total = 0.0 + for item_list, filter_predicate in item_lists: + if filter_predicate is None: + # No filter - sum all weights + total += sum(item.weight for item in item_list) + else: + # Apply filter and sum weights of matching items + total += sum( + item.weight for item in item_list if filter_predicate(item) + ) + return total diff --git a/app_tester/rube_learning/error_analysis/models/base_item.py b/app_tester/rube_learning/error_analysis/models/base_item.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/models/base_item.py @@ -0,0 +1,25 @@ +""" +Base class for error item models. + +All error items must have a weight field that represents their relative +importance within their bucket. +""" + +from pydantic import BaseModel, Field + + +class ErrorItemBase(BaseModel): + """ + Base class for all error item models. + + All items must have a weight field that represents their relative + contribution to the bucket's overall impact. + """ + + weight: float = Field( + 0.0, + ge=0.0, + le=1.0, + description="Weight relative to other items in this bucket. " + "All item weights within a bucket should sum to 1.0.", + ) diff --git a/app_tester/rube_learning/error_analysis/models/connection_problems.py b/app_tester/rube_learning/error_analysis/models/connection_problems.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/models/connection_problems.py @@ -0,0 +1,123 @@ +""" +Connection Problems Bucket - Issues with authentication and connection management. + +These track problems with COMPOSIO_MANAGE_CONNECTIONS and auth-related failures. +""" + +from typing import Any, Callable, List, Optional, Tuple + +from pydantic import Field + +from .base_bucket import ErrorBucketBase +from .base_item import ErrorItemBase +from .other_errors import OtherError + + +class ManageConnectionNotCalled(ErrorItemBase): + """COMPOSIO_MANAGE_CONNECTIONS was needed but not called.""" + + toolkit_name: str = Field( + ..., description="The toolkit that needed connection management" + ) + context: str = Field( + ..., + description="Context of when this should have been called (what was the LLM trying to do)", + ) + + +class ManageConnectionFailed(ErrorItemBase): + """COMPOSIO_MANAGE_CONNECTIONS was called but failed.""" + + toolkit_name: str = Field( + ..., description="The toolkit where connection management failed" + ) + log_id: Optional[str] = Field(None, description="Log ID where this occurred") + error_message: str = Field(..., description="The error message from the failure") + reason: str = Field(..., description="Analysis of why connection management failed") + + +class ConnectionErrorInToolExecution(ErrorItemBase): + """Tool execution failed due to auth/connection issues. + + CRITICAL: This is ONLY for **Composio app tool** execution failures, NOT for general workbench operations. + + Tool executions that belong here: + - Tools called via COMPOSIO_MULTI_EXECUTE_TOOL (called_in_workbench=False) + - Tools called via run_composio_tool helper in workbench (called_in_workbench=True) + + NOT for this bucket (these go to workbench bucket or other_errors): + - upload_local_file failures (S3/R2 uploads) - this is a workbench infrastructure helper, NOT a Composio tool + - proxy_execute failures for non-Composio APIs - these are direct API calls, NOT Composio tool executions + - invoke_llm failures - this is an LLM helper often used in workbench, NOT a Composio tool + - Any operation that is NOT a Composio app tool (GMAIL_*, SLACK_*, JIRA_*, etc.) + + The toolkit_name MUST be a valid Composio toolkit (e.g., 'GMAIL', 'SLACK', 'JIRA', 'GITHUB'). + Do NOT invent toolkit names for non-Composio operations (e.g., 'amazon_s3', 'aws', 'openai'). + """ + + toolkit_name: str = Field( + ..., + description="The Composio toolkit where execution failed (e.g., 'GMAIL', 'SLACK', 'JIRA'). " + "MUST be a valid Composio toolkit. Do NOT use made-up names like 'amazon_s3' or 'aws'.", + ) + action_name: str = Field( + ..., + description="The Composio action/tool that failed (e.g., 'GMAIL_SEND_EMAIL', 'SLACK_POST_MESSAGE'). " + "MUST be a tool called via multi_execute or run_composio_tool.", + ) + log_id: Optional[str] = Field(None, description="Log ID where this occurred") + error_message: str = Field(..., description="The error message") + error_type: str = Field( + ..., + description="Type of connection error: 'token_expired', 'permissions_revoked', " + "'api_key_invalid', 'rate_limited_auth', 'oauth_refresh_failed', " + "'missing_access' (access denied / 403 Forbidden), 'insufficient_scope' (OAuth scope missing), " + "'permission_denied', 'unauthorized' (401), 'unknown'", + ) + connection_was_active: bool = Field( + True, + description="Whether the connection was reported as active before the error", + ) + called_in_workbench: bool = Field( + False, + description="True if this error occurred during workbench execution " + "(via run_composio_tool), False if via direct multi_execute", + ) + reason: str = Field(..., description="Analysis of the connection error") + + +class ConnectionProblemsBucket(ErrorBucketBase): + """Container for all connection-related problems in a session.""" + + manage_connection_not_called: List[ManageConnectionNotCalled] = Field( + default_factory=list + ) + manage_connection_failed: List[ManageConnectionFailed] = Field(default_factory=list) + connection_errors_in_tool_execution: List[ConnectionErrorInToolExecution] = Field( + default_factory=list + ) + + # Catch-all for connection errors that don't fit above categories + other_errors: List[OtherError] = Field( + default_factory=list, + description="Connection-related errors that don't fit into predefined categories", + ) + + weight: float = Field( + 0.0, + ge=0.0, + le=1.0, + description="How much this bucket contributed to overall failure. " + "All bucket weights should sum to 1.0.", + ) + + def _get_item_lists( + self, + ) -> List[Tuple[List[Any], Optional[Callable[[Any], bool]]]]: + """Return item lists for this bucket.""" + return [ + (self.manage_connection_not_called, None), + (self.manage_connection_failed, None), + (self.connection_errors_in_tool_execution, None), + (self.other_errors, None), + ] diff --git a/app_tester/rube_learning/error_analysis/models/improvements.py b/app_tester/rube_learning/error_analysis/models/improvements.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/models/improvements.py @@ -0,0 +1,71 @@ +""" +Improvement Models - Suggestions for fixing tools and meta-tools. + +These are generated by Call 2 based on errors found in the session. +""" + +from typing import List + +from pydantic import BaseModel, Field + + +class ToolImprovement(BaseModel): + """ + Suggested improvement for an existing Composio tool. + + Use this for: + - Schema fixes, description improvements, bug fixes (always) + - Missing features when workflow SUCCEEDED (opportunistic suggestions) + + NOTE: If a missing feature CAUSED workflow failure, use tool_errors.missing_feature_impact + instead (which is weighted). You don't need to duplicate it here unless you want to add + additional context beyond what's in the error bucket. + """ + + action_name: str = Field(..., description="The action/tool to improve") + toolkit_name: str = Field(..., description="The toolkit (e.g., 'GMAIL')") + improvement_type: str = Field( + ..., + description="Type: 'schema_fix', 'description_improvement', 'add_parameter', " + "'remove_parameter', 'fix_bug', 'better_error_handling', 'output_format', " + "'missing_feature' (tool needs new capability - use for SUCCESS workflows only, " + "for failures use tool_errors.missing_feature_impact), " + "'other' (catch-all for improvements that don't fit other categories)", + ) + current_issue: str = Field(..., description="What's currently wrong or suboptimal") + suggested_fix: str = Field(..., description="Specific suggestion for improvement") + expected_impact: str = Field( + ..., description="How this improvement would help users/LLMs" + ) + priority: int = Field( + ..., ge=1, le=10, description="Priority 1-10 (10 = most critical)" + ) + evidence_log_ids: List[str] = Field( + default_factory=list, description="Log IDs that demonstrate this issue" + ) + + +class MetaToolImprovement(BaseModel): + """Suggested improvement for a Composio meta-tool.""" + + meta_tool_name: str = Field( + ..., + description="The meta-tool: 'COMPOSIO_SEARCH_TOOLS', 'COMPOSIO_MANAGE_CONNECTIONS', " + "'COMPOSIO_MULTI_EXECUTE_TOOL', 'COMPOSIO_REMOTE_WORKBENCH', 'COMPOSIO_EXECUTE_RECIPE'", + ) + improvement_type: str = Field( + ..., + description="Type: 'better_results', 'faster_execution', 'clearer_errors', " + "'new_capability', 'documentation', 'reliability'", + ) + current_issue: str = Field(..., description="What's currently wrong or suboptimal") + suggested_fix: str = Field(..., description="Specific suggestion for improvement") + expected_impact: str = Field( + ..., description="How this improvement would help users/LLMs" + ) + priority: int = Field( + ..., ge=1, le=10, description="Priority 1-10 (10 = most critical)" + ) + evidence_log_ids: List[str] = Field( + default_factory=list, description="Log IDs that demonstrate this issue" + ) diff --git a/app_tester/rube_learning/error_analysis/models/llm_errors.py b/app_tester/rube_learning/error_analysis/models/llm_errors.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/models/llm_errors.py @@ -0,0 +1,209 @@ +""" +LLM Error Bucket - Errors made by the LLM during tool invocation. + +These are mistakes the LLM made, NOT problems with Composio tools. +Examples: wrong values, wrong types, incorrect multi-execute usage, hallucinated tools. +""" + +from typing import Any, Callable, List, Optional, Tuple + +from pydantic import Field + +from .base_bucket import ErrorBucketBase +from .base_item import ErrorItemBase +from .other_errors import OtherError + + +class WrongInputValue(ErrorItemBase): + """LLM provided incorrect input values to a tool. + + ONLY use when schema was absolutely clear. If schema was unclear/ambiguous → tool_errors.request_schema_problems + """ + + action_name: str = Field( + ..., + description="The action/tool that received wrong input (e.g., 'GMAIL_SEND_EMAIL')", + ) + toolkit_name: str = Field(..., description="The toolkit (e.g., 'GMAIL')") + parameter_name: str = Field( + ..., description="The parameter that received wrong value" + ) + expected_value_description: str = Field( + ..., description="What the expected value should have been" + ) + actual_value_passed: str = Field( + ..., description="The actual value passed by the LLM" + ) + was_schema_clear_and_correct: bool = Field( + ..., + description="True ONLY if schema was both absolutely clear/unambiguous AND factually correct. " + "If False, use tool_errors.request_schema_problems instead.", + ) + schema_evidence: str = Field( + ..., + description="Quote exact schema description proving it was clear and correct. Required to justify this is LLM error not schema problem.", + ) + called_in_workbench: bool = Field( + False, + description="True if this error occurred in workbench code (run_composio_tool), " + "False if via multi_execute", + ) + reason: str = Field( + ..., description="Why the value was wrong and how it affected execution" + ) + + +class WrongInputType(ErrorItemBase): + """LLM provided wrong type for a parameter (e.g., string instead of array). + + ONLY use when schema was absolutely clear. If schema was unclear/ambiguous → tool_errors.request_schema_problems + """ + + action_name: str = Field( + ..., description="The action/tool that received wrong type" + ) + toolkit_name: str = Field(..., description="The toolkit (e.g., 'SLACK')") + parameter_name: str = Field( + ..., description="The parameter that received wrong type" + ) + expected_type: str = Field( + ..., description="Expected type (e.g., 'array', 'string', 'object')" + ) + actual_type_passed: str = Field( + ..., description="The actual type passed by the LLM" + ) + was_schema_clear_and_correct: bool = Field( + ..., + description="True ONLY if schema was both absolutely clear/unambiguous AND factually correct. " + "If False, use tool_errors.request_schema_problems instead.", + ) + schema_evidence: str = Field( + ..., + description="Quote exact schema description proving it was clear and correct. Required to justify this is LLM error not schema problem.", + ) + called_in_workbench: bool = Field( + False, + description="True if this error occurred in workbench code (run_composio_tool), " + "False if via multi_execute", + ) + reason: str = Field( + ..., description="Why the type was wrong and how it affected execution" + ) + + +class MultiExecuteWronglyCalled(ErrorItemBase): + """multi_execute called with actions that should have been sequential.""" + + log_id: Optional[str] = Field(None, description="Log ID where this occurred") + actions_called_together: List[str] = Field( + ..., description="Actions called together that shouldn't have been" + ) + dependency_reason: str = Field( + ..., description="Why these actions needed to be sequential (data dependency)" + ) + + +class MissedPagination(ErrorItemBase): + """LLM failed to paginate through results, missing available data.""" + + action_name: str = Field(..., description="Action where pagination was missed") + toolkit_name: str = Field(..., description="The toolkit (e.g., 'GMAIL')") + log_id: Optional[str] = Field(None, description="Log ID where this occurred") + pages_fetched: int = Field(..., description="Number of pages actually fetched") + more_data_available: bool = Field( + True, description="Whether more data was available" + ) + reason: str = Field( + ..., description="Why pagination was needed and how it affected workflow" + ) + + +class HallucinatedToolCalled(ErrorItemBase): + """LLM called a tool name that doesn't exist.""" + + attempted_tool_name: str = Field(..., description="The hallucinated tool name") + correct_tool_name: Optional[str] = Field( + None, description="Correct tool name if applicable" + ) + log_id: Optional[str] = Field(None, description="Log ID where this occurred") + called_in_workbench: bool = Field( + False, description="True if in workbench code, False if via multi_execute" + ) + reason: str = Field(..., description="Context and explanation of the hallucination") + + +class MultiExecuteNotCalled(ErrorItemBase): + """LLM should have used multi_execute but didn't (missed batching opportunity). + + IMPORTANT: This error applies ONLY when the LLM made multiple separate multi_execute + calls in sequence that could have been combined into a single multi_execute call. + + This is NOT an error when: + - The LLM uses the workbench with a for loop to make many tool calls (>5 calls). + The workbench is the correct approach for bulk operations because it allows + programmatic iteration, parallelization with ThreadPoolExecutor, and better + error handling for large datasets. + - The calls have data dependencies (output of one is input to another). + + Only flag this when you see consecutive multi_execute tool calls in the logs + that could have been batched into a single multi_execute call. + """ + + use_case: str = Field( + ..., description="The use case where multi_execute would have helped" + ) + actions_that_could_be_batched: List[str] = Field( + ..., description="Actions that could have been batched together" + ) + consecutive_multi_execute_calls: bool = Field( + ..., + description="True if the LLM made multiple separate multi_execute calls that " + "could have been combined. False if it used workbench for bulk operations " + "(which is valid behavior for >5 calls).", + ) + reason: str = Field( + ..., + description="Why multi_execute would have been better and what the impact was. " + "If consecutive_multi_execute_calls is False, explain why this is still an issue.", + ) + + +class LLMErrorBucket(ErrorBucketBase): + """Container for all LLM-related errors in a session.""" + + wrong_input_values: List[WrongInputValue] = Field(default_factory=list) + wrong_input_types: List[WrongInputType] = Field(default_factory=list) + multi_execute_wrongly_called: List[MultiExecuteWronglyCalled] = Field( + default_factory=list + ) + multi_execute_not_called: List[MultiExecuteNotCalled] = Field(default_factory=list) + missed_pagination: List[MissedPagination] = Field(default_factory=list) + hallucinated_tool_called: List[HallucinatedToolCalled] = Field(default_factory=list) + + # Catch-all for LLM errors that don't fit above categories + other_errors: List[OtherError] = Field( + default_factory=list, + description="LLM-related errors that don't fit into predefined categories", + ) + + weight: float = Field( + 0.0, + ge=0.0, + le=1.0, + description="How much this bucket contributed to overall failure. " + "All bucket weights should sum to 1.0.", + ) + + def _get_item_lists( + self, + ) -> List[Tuple[List[Any], Optional[Callable[[Any], bool]]]]: + """Return item lists for this bucket.""" + return [ + (self.wrong_input_values, None), + (self.wrong_input_types, None), + (self.multi_execute_wrongly_called, None), + (self.multi_execute_not_called, None), + (self.missed_pagination, None), + (self.hallucinated_tool_called, None), + (self.other_errors, None), + ] diff --git a/app_tester/rube_learning/error_analysis/models/metrics.py b/app_tester/rube_learning/error_analysis/models/metrics.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/models/metrics.py @@ -0,0 +1,361 @@ +""" +Session Metrics - Static metrics computed from logs, NOT requiring LLM analysis. + +These metrics are: +1. Computed post-hoc from session logs +2. Not shown to the LLM (they don't need LLM reasoning) +3. Useful for dashboards, aggregations, and identifying anomalies + +Separated from error buckets to keep LLM prompts clean and focused. +""" + +from typing import Dict, List, Optional, Union + +from pydantic import BaseModel, Field + + +# ============================================================================= +# Workbench Metrics +# ============================================================================= + + +class WorkbenchTimeTaken(BaseModel): + """Time taken for a single workbench execution.""" + + log_id: str = Field(..., description="Log ID of the workbench execution") + time_taken_ms: int = Field(..., description="Time taken in milliseconds") + was_slow: bool = Field(..., description="Whether this was slower than expected") + + +class WorkbenchMetrics(BaseModel): + """Metrics related to workbench usage in a session.""" + + total_calls: int = Field(0, description="Total number of workbench calls") + total_time_ms: int = Field(0, description="Total time spent in workbench (ms)") + error_count: int = Field( + 0, description="Number of workbench calls that had errors (deterministic)" + ) + time_per_call: List[WorkbenchTimeTaken] = Field( + default_factory=list, description="Time taken per workbench execution" + ) + + @property + def avg_time_ms(self) -> float: + """Average time per workbench call in ms.""" + if self.total_calls == 0: + return 0.0 + return self.total_time_ms / self.total_calls + + +# ============================================================================= +# Multi-Execute Metrics +# ============================================================================= + + +class MultiExecuteMetrics(BaseModel): + """Metrics related to multi_execute usage in a session.""" + + success_count: int = Field( + 0, description="Number of successful multi_execute calls" + ) + failure_count: int = Field(0, description="Number of failed multi_execute calls") + total_calls: int = Field(0, description="Total number of multi_execute calls") + + @property + def success_rate(self) -> float: + """Success rate as a percentage (0.0 to 1.0).""" + if self.total_calls == 0: + return 0.0 + return self.success_count / self.total_calls + + +# ============================================================================= +# Recipe Metrics +# ============================================================================= + + +class RecipeMetrics(BaseModel): + """Metrics related to recipe usage in a session.""" + + recipes_called: List[str] = Field( + default_factory=list, description="List of recipe names that were called" + ) + success_count: int = Field(0, description="Number of successful recipe calls") + failure_count: int = Field(0, description="Number of failed recipe calls") + + @property + def total_calls(self) -> int: + """Total number of recipe calls.""" + return len(self.recipes_called) + + @property + def success_rate(self) -> float: + """Success rate as a percentage (0.0 to 1.0).""" + if self.total_calls == 0: + return 0.0 + return self.success_count / self.total_calls + + +# ============================================================================= +# Connection Metrics +# ============================================================================= + + +class FalseConnectionStatus(BaseModel): + """ + Connection reported as active but actually wasn't. + + Computed post-hoc by comparing connection status with subsequent auth failures. + """ + + toolkit_name: str = Field( + ..., description="The toolkit with false connection status" + ) + reported_status: str = Field( + ..., description="Status that was reported (e.g., 'ACTIVE')" + ) + actual_status: str = Field(..., description="What the status actually was") + evidence: str = Field( + ..., description="Evidence that the reported status was incorrect" + ) + + +class ConnectionMetrics(BaseModel): + """Metrics related to connection management in a session.""" + + false_connection_statuses: List[FalseConnectionStatus] = Field( + default_factory=list, + description="Connections that reported active but weren't", + ) + manage_connection_calls: int = Field( + 0, description="Number of MANAGE_CONNECTIONS calls" + ) + unique_toolkits_connected: List[str] = Field( + default_factory=list, description="List of unique toolkits connected" + ) + + +# ============================================================================= +# Search Quality Metrics (from Call 1, not LLM) +# ============================================================================= + + +class PerUsecaseCachedPlanQuality(BaseModel): + """ + Cached plan quality assessment for a single usecase/search. + + Tracks whether a cached plan was returned and its quality for each + individual search query in the session. + + Note: was_cached_plan_followed is tracked in UsecasePlan (per_usecase_plans_json) + as it's a planner quality concern, not search quality. + """ + + search_id: str = Field( + ..., description="The search_id (e.g., 'usecase_1_1', 'overall_1')" + ) + was_cached_plan_returned: bool = Field( + False, description="Whether a cached plan was returned for this usecase" + ) + cached_plan_quality: Optional[str] = Field( + None, + description="Quality of the cached plan: PLAN_NOT_AVAILABLE, PLAN_POOR, " + "PLAN_AVERAGE, or PLAN_EXCELLENT", + ) + cached_plan_quality_reason: Optional[str] = Field( + None, description="Why the cached plan was rated this way" + ) + + +class SearchQualityMetrics(BaseModel): + """ + Metrics related to search quality - computed in Call 1, not by LLM in Call 2. + + These are assessments made during the initial log analysis, not during + the error weighting phase. + """ + + overall_search_quality_score: Optional[float] = Field( + None, ge=0.0, le=1.0, description="Overall search quality score (0.0 to 1.0)" + ) + search_quality_explanation: Optional[str] = Field( + None, description="Explanation of overall search quality" + ) + total_search_calls: int = Field(0, description="Total number of search calls") + total_get_tool_schemas_calls: int = Field( + 0, description="Total number of COMPOSIO_GET_TOOL_SCHEMAS calls" + ) + + # Per-usecase cached plan quality (replaces session-level aggregation) + per_usecase_cached_plan_quality: List[PerUsecaseCachedPlanQuality] = Field( + default_factory=list, + description="Cached plan quality assessment for each usecase/search in the session", + ) + + @property + def any_cached_plan_returned(self) -> bool: + """Whether any usecase had a cached plan returned.""" + return any( + p.was_cached_plan_returned for p in self.per_usecase_cached_plan_quality + ) + + @property + def cached_plans_returned_count(self) -> int: + """Number of usecases that had cached plans returned.""" + return sum( + 1 + for p in self.per_usecase_cached_plan_quality + if p.was_cached_plan_returned + ) + + +# ============================================================================= +# Token Usage Metrics +# ============================================================================= + + +class ToolCallTokenUsage(BaseModel): + """Token usage for a single tool call.""" + + log_id: str = Field(..., description="Log ID of the tool call") + tool_name: str = Field(..., description="Name of the tool called") + toolkit_name: Optional[str] = Field(None, description="Toolkit the tool belongs to") + request_tokens: int = Field(0, description="Estimated tokens in the request") + response_tokens: int = Field(0, description="Estimated tokens in the response") + + # Threshold for "large" token counts + LARGE_TOKEN_THRESHOLD: int = 2000 + + @property + def is_large_request(self) -> bool: + """Whether request exceeded 2k tokens.""" + return self.request_tokens > self.LARGE_TOKEN_THRESHOLD + + @property + def is_large_response(self) -> bool: + """Whether response exceeded 2k tokens.""" + return self.response_tokens > self.LARGE_TOKEN_THRESHOLD + + @property + def total_tokens(self) -> int: + """Total tokens for this call.""" + return self.request_tokens + self.response_tokens + + +class TokenUsageMetrics(BaseModel): + """ + Metrics related to token usage across all tool calls. + + Useful for identifying: + - Large tool calls that may be inefficient + - Tools with unexpectedly large schemas/responses + - Opportunities for optimization + """ + + tool_calls: List[ToolCallTokenUsage] = Field( + default_factory=list, description="Token usage per tool call" + ) + total_request_tokens: int = Field( + 0, description="Total request tokens across all calls" + ) + total_response_tokens: int = Field( + 0, description="Total response tokens across all calls" + ) + large_request_count: int = Field( + 0, description="Number of tool calls with >2k request tokens" + ) + large_response_count: int = Field( + 0, description="Number of tool calls with >2k response tokens" + ) + + @property + def total_tokens(self) -> int: + """Total tokens across all calls.""" + return self.total_request_tokens + self.total_response_tokens + + def get_large_calls(self, threshold: int = 2000) -> List[ToolCallTokenUsage]: + """Get tool calls exceeding the token threshold.""" + return [ + tc + for tc in self.tool_calls + if tc.request_tokens > threshold or tc.response_tokens > threshold + ] + + def get_top_consumers(self, n: int = 5) -> List[ToolCallTokenUsage]: + """Get top N tool calls by total token usage.""" + sorted_calls = sorted( + self.tool_calls, key=lambda x: x.total_tokens, reverse=True + ) + return sorted_calls[:n] + + +# ============================================================================= +# Session-Level Metrics Container +# ============================================================================= + + +class SessionMetrics(BaseModel): + """ + Container for all static metrics computed from a session. + + These metrics are: + - Computed post-hoc from logs (not requiring LLM analysis) + - Useful for dashboards, aggregations, and anomaly detection + - Kept separate from error buckets to avoid polluting LLM prompts + """ + + # Per-category metrics + workbench: WorkbenchMetrics = Field(default_factory=WorkbenchMetrics) # type: ignore[arg-type] + multi_execute: MultiExecuteMetrics = Field( + default_factory=MultiExecuteMetrics # type: ignore[arg-type] + ) + recipe: RecipeMetrics = Field(default_factory=RecipeMetrics) # type: ignore[arg-type] + connection: ConnectionMetrics = Field(default_factory=ConnectionMetrics) # type: ignore[arg-type] + search_quality: SearchQualityMetrics = Field( + default_factory=SearchQualityMetrics # type: ignore[arg-type] + ) + token_usage: TokenUsageMetrics = Field(default_factory=TokenUsageMetrics) # type: ignore[arg-type] + + # Session-level counts + total_tool_calls: int = Field( + 0, description="Total number of tool calls in session" + ) + total_meta_tool_calls: int = Field( + 0, + description="Total number of meta-tool calls (search, manage, multi_execute, workbench, recipe)", + ) + session_duration_ms: Optional[int] = Field( + None, description="Total session duration in milliseconds" + ) + + def summary(self) -> Dict[str, Union[int, float]]: + """Return a summary dict of key metrics.""" + return { + "total_tool_calls": self.total_tool_calls, + "total_meta_tool_calls": self.total_meta_tool_calls, + "session_duration_ms": self.session_duration_ms or 0, + # Workbench + "workbench_total_calls": self.workbench.total_calls, + "workbench_total_time_ms": self.workbench.total_time_ms, + "workbench_avg_time_ms": self.workbench.avg_time_ms, + # Multi-execute + "multi_execute_success_count": self.multi_execute.success_count, + "multi_execute_total_calls": self.multi_execute.total_calls, + "multi_execute_success_rate": self.multi_execute.success_rate, + # Recipe + "recipe_success_count": self.recipe.success_count, + "recipe_total_calls": self.recipe.total_calls, + # Connection + "false_connection_count": len(self.connection.false_connection_statuses), + "manage_connection_calls": self.connection.manage_connection_calls, + # Search + "search_quality_score": self.search_quality.overall_search_quality_score + or 0.0, + "total_search_calls": self.search_quality.total_search_calls, + "total_get_tool_schemas_calls": self.search_quality.total_get_tool_schemas_calls, + # Tokens + "total_request_tokens": self.token_usage.total_request_tokens, + "total_response_tokens": self.token_usage.total_response_tokens, + "large_request_count": self.token_usage.large_request_count, + "large_response_count": self.token_usage.large_response_count, + } diff --git a/app_tester/rube_learning/error_analysis/models/multi_execute.py b/app_tester/rube_learning/error_analysis/models/multi_execute.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/models/multi_execute.py @@ -0,0 +1,73 @@ +""" +Multi-Execute Bucket - Infrastructure issues with COMPOSIO_MULTI_EXECUTE_TOOL. + +This bucket tracks infrastructure failures (multi_execute meta-tool itself failing). + +NOTE: Missed batching opportunities (should have used multi_execute but didn't) are +tracked in LLMErrorBucket.multi_execute_not_called since that's an LLM decision error. + +NOTE: Individual app tool failures due to implementation bugs (not LLM wrong inputs) that happen +INSIDE multi_execute should go to ToolErrorBucket.tool_implementation_problems, NOT here. +""" + +from typing import Any, Callable, List, Optional, Tuple + +from pydantic import Field + +from .base_bucket import ErrorBucketBase +from .base_item import ErrorItemBase +from .other_errors import OtherError + + +class MultiExecuteMetaToolFailure(ErrorItemBase): + """ + The multi_execute meta-tool itself failed (500 error, timeout, etc.). + + This is for infrastructure-level failures where multi_execute couldn't even + dispatch the tools - NOT for individual tool failures within multi_execute. + """ + + log_id: Optional[str] = Field(None, description="Log ID where meta-tool failed") + error_code: Optional[str] = Field( + None, + description="Error code: 'EXECUTION_TIMEOUT', '500_ERROR', 'INVALID_REQUEST', etc.", + ) + actions_attempted: List[str] = Field( + default_factory=list, description="Actions that were attempted (if known)" + ) + reason: str = Field(..., description="Why the meta-tool failed") + + +class MultiExecuteBucket(ErrorBucketBase): + """ + Container for multi-execute infrastructure issues. + + NOTE: Individual app tool failures inside multi_execute go to ToolErrorBucket. + NOTE: Missed batching opportunities go to LLMErrorBucket.multi_execute_not_called. + This bucket is ONLY for multi_execute infrastructure failures (meta-tool itself failing). + """ + + meta_tool_failures: List[MultiExecuteMetaToolFailure] = Field(default_factory=list) + + # Catch-all for multi-execute errors that don't fit above categories + other_errors: List[OtherError] = Field( + default_factory=list, + description="Multi-execute related errors that don't fit into predefined categories", + ) + + weight: float = Field( + 0.0, + ge=0.0, + le=1.0, + description="How much this bucket contributed to overall failure. " + "All bucket weights should sum to 1.0.", + ) + + def _get_item_lists( + self, + ) -> List[Tuple[List[Any], Optional[Callable[[Any], bool]]]]: + """Return item lists for this bucket.""" + return [ + (self.meta_tool_failures, None), + (self.other_errors, None), + ] diff --git a/app_tester/rube_learning/error_analysis/models/other_errors.py b/app_tester/rube_learning/error_analysis/models/other_errors.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/models/other_errors.py @@ -0,0 +1,69 @@ +""" +Other Errors - Catch-all for errors that don't fit existing categories. + +This module provides: +1. OtherError - Generic error model used within buckets for uncategorized issues +2. OtherErrorsBucket - Top-level bucket for errors that don't fit any bucket +""" + +from typing import Any, Callable, List, Optional, Tuple + +from pydantic import Field + +from .base_bucket import ErrorBucketBase +from .base_item import ErrorItemBase + + +class OtherError(ErrorItemBase): + """ + Catch-all for errors that don't fit into predefined categories. + + Used both: + - Within buckets: for bucket-related errors that don't match specific types + - In OtherErrorsBucket: for errors that don't fit any bucket at all + """ + + toolkit_name: Optional[str] = Field( + None, description="The toolkit involved (e.g., 'GMAIL', 'SLACK')" + ) + tool_name: Optional[str] = Field( + None, description="The tool/action involved (e.g., 'GMAIL_SEND_EMAIL')" + ) + log_id: Optional[str] = Field(None, description="Log ID where this error occurred") + error_message: str = Field(..., description="The error message or description") + error_context: str = Field( + ..., + description="What was happening when this error occurred " + "(e.g., 'during email sending', 'while parsing response')", + ) + reason: str = Field( + ..., + description="Analysis of why this error occurred and how it affected the workflow", + ) + + +class OtherErrorsBucket(ErrorBucketBase): + """ + Top-level bucket for errors that don't fit any other bucket category. + + Use this for: + - Errors that span multiple categories + - Novel error types not covered by existing buckets + - Edge cases and unexpected failures + """ + + other_errors: List[OtherError] = Field(default_factory=list) + + weight: float = Field( + 0.0, + ge=0.0, + le=1.0, + description="How much this bucket contributed to overall failure. " + "All bucket weights should sum to 1.0.", + ) + + def _get_item_lists( + self, + ) -> List[Tuple[List[Any], Optional[Callable[[Any], bool]]]]: + """Return item lists for this bucket.""" + return [(self.other_errors, None)] diff --git a/app_tester/rube_learning/error_analysis/models/output.py b/app_tester/rube_learning/error_analysis/models/output.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/models/output.py @@ -0,0 +1,621 @@ +""" +Top-level Output Models for Error Analysis. + +ErrorAnalysisLLMOutput - What the LLM returns (no static metrics, uses slim bucket models) +SessionErrorAnalysis - Final model with static metrics and Call 1 data merged in +""" + +from typing import Dict, List, Optional, Tuple, Union + +from pydantic import BaseModel, Field + +from app_tester.rube_learning.common import WorkflowStatus + +from .base_bucket import ErrorBucketBase +from .connection_problems import ConnectionProblemsBucket +from .improvements import MetaToolImprovement, ToolImprovement +from .llm_errors import LLMErrorBucket +from .metrics import SessionMetrics +from .multi_execute import MultiExecuteBucket +from .other_errors import OtherErrorsBucket +from .recipe import RecipeBucket +from .search_quality import SearchQualityBucket +from .tool_errors import ToolErrorBucket +from .workbench import WorkbenchBucket + + +class ErrorAnalysisLLMOutput(BaseModel): + """ + What the LLM returns from Call 2. + + This is the stripped-down model containing only LLM-generated fields. + Uses SearchQualityBucketLLM (not full SearchQualityBucket) because: + - LLM only assigns weights to wrong_tools_returned items from Call 1 + - Cached plan quality fields are already computed in Call 1 + - Final model merges LLM weights with Call 1 data + """ + + # Workflow quality assessment + workflow_quality_enum: WorkflowStatus = Field( + ..., description="Overall workflow quality" + ) + overall_rationale: str = Field( + ..., description="Explanation of why this quality was assigned" + ) + + # Error buckets (all weights assigned by LLM) + # Note: search_quality uses slim LLM model - cached plan fields come from Call 1 + llm_errors: LLMErrorBucket = Field(default_factory=LLMErrorBucket) # type: ignore[arg-type] + tool_errors: ToolErrorBucket = Field(default_factory=ToolErrorBucket) # type: ignore[arg-type] + search_quality: SearchQualityBucket = Field( + default_factory=SearchQualityBucket # type: ignore[arg-type] + ) + connection_problems: ConnectionProblemsBucket = Field( + default_factory=ConnectionProblemsBucket # type: ignore[arg-type] + ) + workbench: WorkbenchBucket = Field(default_factory=WorkbenchBucket) # type: ignore[arg-type] + multi_execute: MultiExecuteBucket = Field( + default_factory=MultiExecuteBucket # type: ignore[arg-type] + ) + recipe: RecipeBucket = Field(default_factory=RecipeBucket) # type: ignore[arg-type] + + # Top-level catch-all for errors that don't fit any bucket + other_errors: OtherErrorsBucket = Field( + default_factory=OtherErrorsBucket # type: ignore[arg-type] + ) + + # Improvements based on errors found + tool_improvements: List[ToolImprovement] = Field(default_factory=list) + meta_tool_improvements: List[MetaToolImprovement] = Field(default_factory=list) + + # Summary + primary_failure_reason: Optional[str] = Field( + None, description="Single most important failure cause" + ) + + def get_active_buckets(self) -> Dict[str, ErrorBucketBase]: + """Returns dict of buckets that have issues.""" + buckets: Dict[str, ErrorBucketBase] = { + "llm_errors": self.llm_errors, + "tool_errors": self.tool_errors, + "search_quality": self.search_quality, + "connection_problems": self.connection_problems, + "workbench": self.workbench, + "multi_execute": self.multi_execute, + "recipe": self.recipe, + "other_errors": self.other_errors, + } + return {name: bucket for name, bucket in buckets.items() if bucket.has_issues()} + + def get_bucket_weights(self) -> Dict[str, float]: + """Returns dict of bucket name to weight.""" + return { + "llm_errors": self.llm_errors.weight, + "tool_errors": self.tool_errors.weight, + "search_quality": self.search_quality.weight, + "connection_problems": self.connection_problems.weight, + "workbench": self.workbench.weight, + "multi_execute": self.multi_execute.weight, + "recipe": self.recipe.weight, + "other_errors": self.other_errors.weight, + } + + def validate_bucket_weights(self) -> Tuple[bool, float]: + """ + Check if bucket weights sum to 1.0 (with tolerance). + Returns (is_valid, actual_sum). + """ + total = sum(self.get_bucket_weights().values()) + is_valid = abs(total - 1.0) < 0.01 # Allow 1% tolerance + return is_valid, total + + def validate_item_weights_within_buckets( + self, tolerance: float = 0.05 + ) -> Tuple[bool, Dict[str, float]]: + """ + Check if item weights within each non-empty bucket sum to 1.0. + + This validates the FLAT distribution rule: ALL items across ALL sub-lists + within a bucket should have weights summing to 1.0 TOGETHER. + + Args: + tolerance: Allowed deviation from 1.0 (default 5%) + + Returns: + Tuple of (all_valid, dict of bucket_name -> item_weight_sum for invalid buckets) + """ + buckets_to_check = { + "llm_errors": self.llm_errors, + "tool_errors": self.tool_errors, + "search_quality": self.search_quality, + "connection_problems": self.connection_problems, + "workbench": self.workbench, + "multi_execute": self.multi_execute, + "recipe": self.recipe, + "other_errors": self.other_errors, + } + + invalid_buckets: Dict[str, float] = {} + + for bucket_name, bucket in buckets_to_check.items(): + # Only check buckets that have issues + if not bucket.has_issues(): + continue + + item_weight_sum = bucket.get_item_weight_sum() + + # Item weights should sum to 1.0 for non-empty buckets + if abs(item_weight_sum - 1.0) > tolerance: + invalid_buckets[bucket_name] = item_weight_sum + + all_valid = len(invalid_buckets) == 0 + return all_valid, invalid_buckets + + def validate_no_zero_item_weights_within_buckets( + self, zero_tolerance: float = 1e-9 + ) -> Tuple[bool, Dict[str, Dict[str, int]]]: + """ + Ensure the LLM explicitly assigns (non-zero) weights to ALL items in each non-empty bucket. + + This catches the common failure mode where the LLM assigns weights to one sub-list, + but "forgets" later sub-lists and they default to 0.0 via Pydantic defaults. + + Returns: + (all_valid, invalid_buckets) where invalid_buckets maps: + bucket_name -> {"zero_weight_items": int, "total_items": int} + """ + + buckets_to_check = { + "llm_errors": self.llm_errors, + "tool_errors": self.tool_errors, + "search_quality": self.search_quality, + "connection_problems": self.connection_problems, + "workbench": self.workbench, + "multi_execute": self.multi_execute, + "recipe": self.recipe, + "other_errors": self.other_errors, + } + + invalid: Dict[str, Dict[str, int]] = {} + + for bucket_name, bucket in buckets_to_check.items(): + if not bucket.has_issues(): + continue + + item_lists = bucket._get_item_lists() + total_items = 0 + zero_items = 0 + for item_list, filter_predicate in item_lists: + if filter_predicate is None: + items = item_list + else: + items = [i for i in item_list if filter_predicate(i)] + + total_items += len(items) + zero_items += sum(1 for i in items if abs(i.weight) <= zero_tolerance) + + # Only validate when there are items (should be, if has_issues() is True) + if total_items > 0 and zero_items > 0: + invalid[bucket_name] = { + "zero_weight_items": zero_items, + "total_items": total_items, + } + + return len(invalid) == 0, invalid + + def check_missing_top_level_weights( + self, threshold: float = 0.7 + ) -> Tuple[bool, List[str]]: + """ + Check for buckets where sub-bucket weight sum > threshold but top-level weight is 0. + + This detects cases where the LLM populated sub-buckets with weights but forgot + to assign a top-level bucket weight. + + Args: + threshold: Minimum sub-bucket weight sum to consider (default 0.7) + + Returns: + Tuple of (has_issue, list_of_bucket_names_with_issue) + """ + buckets_to_check = { + "llm_errors": self.llm_errors, + "tool_errors": self.tool_errors, + "search_quality": self.search_quality, + "connection_problems": self.connection_problems, + "workbench": self.workbench, + "multi_execute": self.multi_execute, + "recipe": self.recipe, + "other_errors": self.other_errors, + } + + # Check if bucket appears in primary_failure_reason (case-insensitive) + primary_reason_lower = ( + self.primary_failure_reason.lower() if self.primary_failure_reason else "" + ) + + issues = [] + for bucket_name, bucket in buckets_to_check.items(): + # Check if bucket has issues OR appears in primary_failure_reason + has_issues = bucket.has_issues() + appears_in_reason = bucket_name.lower() in primary_reason_lower + + if has_issues or appears_in_reason: + item_weight_sum = bucket.get_item_weight_sum() + top_level_weight = bucket.weight + + # Check condition: item weights > threshold but top-level weight is 0 + if item_weight_sum > threshold and abs(top_level_weight) < 0.01: + issues.append(bucket_name) + + return len(issues) > 0, issues + + +class SessionErrorAnalysis(ErrorAnalysisLLMOutput): + """ + Final complete model - LLM output plus static metrics and Call 1 data. + + Created after Call 2 by merging: + - ErrorAnalysisLLMOutput (weights from LLM) + - Static metrics extracted from logs (in SessionMetrics) + - Call 1 data (cached plan quality, search quality explanation, etc.) + """ + + # Override search_quality to use full model + search_quality: SearchQualityBucket = Field( + default_factory=SearchQualityBucket # type: ignore[arg-type] + ) + + # Session identifiers (added post-LLM) + session_id: Optional[str] = Field(None, description="Unique session identifier") + user_id: Optional[str] = Field(None, description="User identifier") + + # Workflow context from Call 1 (for reference) + workflow_summary: Optional[str] = Field( + None, description="Workflow summary from Call 1" + ) + + # Static metrics - computed from logs, not by LLM + metrics: SessionMetrics = Field( + default_factory=SessionMetrics, # type: ignore[arg-type] + description="Static metrics computed from logs (not LLM-generated)", + ) + + def get_top_issues( # noqa: C901 + self, n: int = 5 + ) -> List[Tuple[str, str, float, float]]: + """ + Get top N issues by ABSOLUTE IMPACT across all buckets. + absolute_impact = bucket_weight × item_weight + + Returns list of (bucket_name, issue_description, item_weight, absolute_impact). + """ + all_issues: List[Tuple[str, str, float, float]] = [] + + # LLM errors + bw = self.llm_errors.weight + for wiv in self.llm_errors.wrong_input_values: + all_issues.append( + ( + "llm_errors", + f"{wiv.action_name}.{wiv.parameter_name} (wrong value)", + wiv.weight, + bw * wiv.weight, + ) + ) + for wit in self.llm_errors.wrong_input_types: + all_issues.append( + ( + "llm_errors", + f"{wit.action_name}.{wit.parameter_name} (wrong type)", + wit.weight, + bw * wit.weight, + ) + ) + for mewc in self.llm_errors.multi_execute_wrongly_called: + desc = ( + mewc.actions_called_together[0][:20] + if mewc.actions_called_together + else "unknown" + ) + all_issues.append( + ( + "llm_errors", + f"multi_execute misuse: {desc}", + mewc.weight, + bw * mewc.weight, + ) + ) + for mp in self.llm_errors.missed_pagination: + all_issues.append( + ( + "llm_errors", + f"{mp.action_name} (missed pagination)", + mp.weight, + bw * mp.weight, + ) + ) + for htc in self.llm_errors.hallucinated_tool_called: + all_issues.append( + ( + "llm_errors", + f"hallucinated: {htc.attempted_tool_name}", + htc.weight, + bw * htc.weight, + ) + ) + for llm_oe in self.llm_errors.other_errors: + tool_info = llm_oe.tool_name or llm_oe.toolkit_name or "unknown" + all_issues.append( + ( + "llm_errors", + f"other: {tool_info} - {llm_oe.error_context[:30]}", + llm_oe.weight, + bw * llm_oe.weight, + ) + ) + + # Tool errors + bw = self.tool_errors.weight + for rsp in self.tool_errors.request_schema_problems: + all_issues.append( + ( + "tool_errors", + f"{rsp.tool_name}.{rsp.parameter_name} (schema)", + rsp.weight, + bw * rsp.weight, + ) + ) + for tip in self.tool_errors.tool_implementation_problems: + all_issues.append( + ( + "tool_errors", + f"{tip.tool_name} (implementation bug)", + tip.weight, + bw * tip.weight, + ) + ) + for resp_sp in self.tool_errors.response_schema_problems: + all_issues.append( + ( + "tool_errors", + f"{resp_sp.tool_name}.{resp_sp.parameter_name} (response schema: {resp_sp.problem_type})", + resp_sp.weight, + bw * resp_sp.weight, + ) + ) + for rnf in self.tool_errors.resource_not_found: + all_issues.append( + ( + "tool_errors", + f"{rnf.tool_name} (resource not found: {rnf.resource_type})", + rnf.weight, + bw * rnf.weight, + ) + ) + for mti in self.tool_errors.missing_tools_impact: + all_issues.append( + ( + "tool_errors", + f"missing {mti.suggestion_type}: {mti.tool_or_toolkit_name}", + mti.weight, + bw * mti.weight, + ) + ) + for tool_oe in self.tool_errors.other_errors: + tool_info = tool_oe.tool_name or tool_oe.toolkit_name or "unknown" + all_issues.append( + ( + "tool_errors", + f"other: {tool_info} - {tool_oe.error_context[:30]}", + tool_oe.weight, + bw * tool_oe.weight, + ) + ) + + # Search quality + bw = self.search_quality.weight + for wtr in self.search_quality.wrong_tools_returned: + all_issues.append( + ( + "search_quality", + f"{wtr.use_case[:50]}", + wtr.weight, + bw * wtr.weight, + ) + ) + for sq_oe in self.search_quality.other_errors: + tool_info = sq_oe.tool_name or sq_oe.toolkit_name or "unknown" + all_issues.append( + ( + "search_quality", + f"other: {tool_info} - {sq_oe.error_context[:30]}", + sq_oe.weight, + bw * sq_oe.weight, + ) + ) + + # Connection problems + bw = self.connection_problems.weight + for mcnc in self.connection_problems.manage_connection_not_called: + all_issues.append( + ( + "connection_problems", + f"{mcnc.toolkit_name} (not called)", + mcnc.weight, + bw * mcnc.weight, + ) + ) + for mcf in self.connection_problems.manage_connection_failed: + all_issues.append( + ( + "connection_problems", + f"{mcf.toolkit_name} (failed)", + mcf.weight, + bw * mcf.weight, + ) + ) + for ceite in self.connection_problems.connection_errors_in_tool_execution: + all_issues.append( + ( + "connection_problems", + f"{ceite.toolkit_name} (exec error)", + ceite.weight, + bw * ceite.weight, + ) + ) + for conn_oe in self.connection_problems.other_errors: + tool_info = conn_oe.tool_name or conn_oe.toolkit_name or "unknown" + all_issues.append( + ( + "connection_problems", + f"other: {tool_info} - {conn_oe.error_context[:30]}", + conn_oe.weight, + bw * conn_oe.weight, + ) + ) + + # Workbench + bw = self.workbench.weight + for cle in self.workbench.code_logic_errors: + all_issues.append( + ( + "workbench", + f"code logic ({cle.log_id})", + cle.weight, + bw * cle.weight, + ) + ) + for htp in self.workbench.helper_tool_problems: + all_issues.append( + ( + "workbench", + f"helper tool problem: {htp.problem_type}", + htp.weight, + bw * htp.weight, + ) + ) + for uc in self.workbench.unnecessary_calls: + all_issues.append( + ( + "workbench", + f"unnecessary ({uc.log_id})", + uc.weight, + bw * uc.weight, + ) + ) + for mtf in self.workbench.meta_tool_failures: + all_issues.append( + ( + "workbench", + f"meta tool failure: {mtf.error_code}", + mtf.weight, + bw * mtf.weight, + ) + ) + for mpkg in self.workbench.missing_packages: + all_issues.append( + ( + "workbench", + f"missing: {mpkg.package_name}", + mpkg.weight, + bw * mpkg.weight, + ) + ) + for ec in self.workbench.execution_counts: + if ec.was_excessive: + all_issues.append( + ( + "workbench", + f"excessive executions: {ec.use_case[:30]}", + ec.weight, + bw * ec.weight, + ) + ) + for wb_oe in self.workbench.other_errors: + tool_info = wb_oe.tool_name or wb_oe.toolkit_name or "unknown" + all_issues.append( + ( + "workbench", + f"other: {tool_info} - {wb_oe.error_context[:30]}", + wb_oe.weight, + bw * wb_oe.weight, + ) + ) + + # Multi-execute + bw = self.multi_execute.weight + for me_mtf in self.multi_execute.meta_tool_failures: + all_issues.append( + ( + "multi_execute", + f"meta failure: {me_mtf.error_code or 'unknown'}", + me_mtf.weight, + bw * me_mtf.weight, + ) + ) + for me_oe in self.multi_execute.other_errors: + tool_info = me_oe.tool_name or me_oe.toolkit_name or "unknown" + all_issues.append( + ( + "multi_execute", + f"other: {tool_info} - {me_oe.error_context[:30]}", + me_oe.weight, + bw * me_oe.weight, + ) + ) + + # Recipe + bw = self.recipe.weight + for rf in self.recipe.recipe_failures: + all_issues.append( + ( + "recipe", + rf.recipe_name, + rf.weight, + bw * rf.weight, + ) + ) + for rec_oe in self.recipe.other_errors: + tool_info = rec_oe.tool_name or rec_oe.toolkit_name or "unknown" + all_issues.append( + ( + "recipe", + f"other: {tool_info} - {rec_oe.error_context[:30]}", + rec_oe.weight, + bw * rec_oe.weight, + ) + ) + + # Top-level other errors bucket + bw = self.other_errors.weight + for oe in self.other_errors.other_errors: + tool_info = oe.tool_name or oe.toolkit_name or "unknown" + all_issues.append( + ( + "other_errors", + f"{tool_info} - {oe.error_context[:40]}", + oe.weight, + bw * oe.weight, + ) + ) + + # Sort by absolute_impact descending + all_issues.sort(key=lambda x: x[3], reverse=True) + return all_issues[:n] + + def summary_stats(self) -> Dict[str, Union[int, float]]: + """Return summary statistics for this analysis.""" + active_buckets = self.get_active_buckets() + total_issues = sum(b.item_count() for b in active_buckets.values()) + + # Combine error analysis stats with metrics + stats: Dict[str, Union[int, float]] = { + "total_issues": total_issues, + "active_bucket_count": len(active_buckets), + "tool_improvements_count": len(self.tool_improvements), + "meta_tool_improvements_count": len(self.meta_tool_improvements), + } + # Add all metrics from SessionMetrics + stats.update(self.metrics.summary()) + return stats diff --git a/app_tester/rube_learning/error_analysis/models/recipe.py b/app_tester/rube_learning/error_analysis/models/recipe.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/models/recipe.py @@ -0,0 +1,56 @@ +""" +Recipe Bucket - Issues with COMPOSIO_EXECUTE_RECIPE usage. + +These track failures in pre-built workflow recipes. +""" + +from typing import Any, Callable, List, Optional, Tuple + +from pydantic import Field + +from .base_bucket import ErrorBucketBase +from .base_item import ErrorItemBase +from .other_errors import OtherError + + +class RecipeFailure(ErrorItemBase): + """A recipe was called but failed.""" + + recipe_name: str = Field(..., description="Name of the recipe that failed") + log_id: Optional[str] = Field(None, description="Log ID where this occurred") + error_message: str = Field(..., description="The error message from the failure") + error_category: str = Field( + ..., + description="Category: 'recipe_not_found', 'input_validation', " + "'execution_error', 'timeout', 'unknown'", + ) + reason: str = Field(..., description="Analysis of why the recipe failed") + + +class RecipeBucket(ErrorBucketBase): + """Container for all recipe-related issues in a session.""" + + recipe_failures: List[RecipeFailure] = Field(default_factory=list) + + # Catch-all for recipe errors that don't fit above categories + other_errors: List[OtherError] = Field( + default_factory=list, + description="Recipe-related errors that don't fit into predefined categories", + ) + + weight: float = Field( + 0.0, + ge=0.0, + le=1.0, + description="How much this bucket contributed to overall failure. " + "All bucket weights should sum to 1.0.", + ) + + def _get_item_lists( + self, + ) -> List[Tuple[List[Any], Optional[Callable[[Any], bool]]]]: + """Return item lists for this bucket.""" + return [ + (self.recipe_failures, None), + (self.other_errors, None), + ] diff --git a/app_tester/rube_learning/error_analysis/models/search_quality.py b/app_tester/rube_learning/error_analysis/models/search_quality.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/models/search_quality.py @@ -0,0 +1,87 @@ +""" +Search Quality Bucket - Issues with tool search results. + +These track cases where COMPOSIO_SEARCH_TOOLS returned wrong or missing tools. + +Flow: +- Call 1 identifies wrong_tools_returned issues and cached plan quality +- Call 2 receives wrong_tools_returned, assigns weights based on impact +- Final model merges LLM weights with Call 1's cached plan quality fields +""" + +from typing import Any, Callable, List, Optional, Tuple + +from pydantic import Field + +from .base_bucket import ErrorBucketBase +from .base_item import ErrorItemBase +from .other_errors import OtherError + + +class WrongToolsReturned(ErrorItemBase): + """Search returned wrong or missing tools for a use case. + + Fields come from Call 1, weight is assigned by Call 2. + """ + + search_id: str = Field( + ..., description="The search_id from Call 1 (e.g., 'usecase_1_1')" + ) + use_case: str = Field(..., description="The use case that was searched for") + tools_returned: List[str] = Field( + default_factory=list, + description="Tools that were returned by search. DO NOT MODIFY - this is a factual record.", + ) + tools_missing: List[str] = Field( + default_factory=list, + description="Tools that should have been returned but weren't. " + "You can ONLY REMOVE items from this list (if they didn't impact workflow). " + "NEVER ADD new tools - you don't have access to the tool catalog.", + ) + tools_irrelevant: List[str] = Field( + default_factory=list, + description="Tools that were returned but shouldn't have been. " + "You can ONLY REMOVE items from this list (if they were actually useful). " + "NEVER ADD new tools - you don't have access to the tool catalog. " + "Be lenient, only keep tools that are *clearly* irrelevant to the use case.", + ) + severity: str = Field( + ..., description="Severity: 'critical', 'high', 'medium', 'low'" + ) + reason: str = Field( + ..., description="Why this was a problem and how it affected the workflow" + ) + + +class SearchQualityBucket(ErrorBucketBase): + """ + LLM output model for search quality bucket. + + This is what Call 2's LLM returns - only wrong_tools_returned with weights. + Does NOT include cached plan fields (those come from Call 1). + """ + + wrong_tools_returned: List[WrongToolsReturned] = Field(default_factory=list) + + # Catch-all for search quality issues that don't fit above categories + other_errors: List[OtherError] = Field( + default_factory=list, + description="Search quality issues that don't fit into predefined categories", + ) + + weight: float = Field( + 0.0, + ge=0.0, + le=1.0, + description="How much this bucket contributed to overall failure. " + "All bucket weights should sum to 1.0.", + ) + + def _get_item_lists( + self, + ) -> List[Tuple[List[Any], Optional[Callable[[Any], bool]]]]: + """Return item lists for this bucket.""" + return [ + (self.wrong_tools_returned, None), + (self.other_errors, None), + ] diff --git a/app_tester/rube_learning/error_analysis/models/tool_errors.py b/app_tester/rube_learning/error_analysis/models/tool_errors.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/models/tool_errors.py @@ -0,0 +1,371 @@ +""" +Tool Error Bucket - Problems with Composio app tool definitions and execution. + +These are issues with **app tools** (GMAIL_SEND_EMAIL, SLACK_POST_MESSAGE, JIRA_CREATE_ISSUE, etc.), +NOT meta-tools (COMPOSIO_MULTI_EXECUTE_TOOL, COMPOSIO_REMOTE_WORKBENCH, etc.). + +IMPORTANT: App tool executions happen INSIDE meta-tools: +- Tools called via COMPOSIO_MULTI_EXECUTE_TOOL → failures go here (called_in_workbench=False) +- Tools called via run_composio_tool in COMPOSIO_REMOTE_WORKBENCH → failures go here (called_in_workbench=True) + +Meta-tool failures (the meta-tool itself failing, not the app tools inside it) go to their respective buckets: +- WorkbenchBucket.meta_tool_failures for workbench infrastructure failures +- MultiExecuteBucket.meta_tool_failures for multi_execute infrastructure failures + +Examples of what goes here: +- GMAIL_SEND_EMAIL has a bug in how it constructs the API request (tool_implementation_problems) +- SLACK_POST_MESSAGE has unclear channel_id description (request_schema_problems) +- JIRA_CREATE_ISSUE returns unexpected JSON format (unexpected_outputs) + +NOTE: If a tool fails because the LLM passed wrong inputs, that goes to llm_errors, NOT here. +""" + +from typing import Any, Callable, List, Optional, Tuple + +from pydantic import Field + +from .base_bucket import ErrorBucketBase +from .base_item import ErrorItemBase +from .other_errors import OtherError + + +class RequestSchemaProblem(ErrorItemBase): + """Problem with a tool's request schema that caused incorrect invocation. + + NOTE: Only use this when the schema was missing, unclear, or misleading. + If the schema was correct and descriptive but the LLM still made a mistake, + use WrongInputValue/WrongInputType in the LLM bucket instead. + """ + + tool_name: str = Field(..., description="The tool with the schema problem") + parameter_name: str = Field(..., description="The problematic parameter") + problem_type: str = Field( + ..., + description="Type: 'unclear_description', 'misleading_name', 'missing_constraints', " + "'ambiguous_type', 'missing_correlation' (params that must be together)", + ) + called_in_workbench: bool = Field( + False, + description="True if this schema problem was encountered in workbench code, " + "False if via multi_execute", + ) + reason: str = Field( + ..., description="What was wrong and how it led to incorrect invocation" + ) + suggested_fix: Optional[str] = Field( + None, description="Suggested improvement to the schema" + ) + + +class ToolImplementationProblem(ErrorItemBase): + """ + Tool implementation bug - the tool failed despite receiving correct inputs. + + ONLY use this when: + 1. The LLM provided valid, correct inputs (not wrong_input_values/wrong_input_types) + 2. The schema was clear and descriptive (not request_schema_problems) + 3. Auth/connection was working (not connection_problems) + 4. But the tool STILL failed due to a bug in the tool's implementation + + Examples: + - Tool code calls wrong API endpoint + - Tool incorrectly transforms/formats data before sending to API + - Tool has a bug in response parsing logic + - Tool doesn't handle a valid edge case properly + + NOT for this bucket: + - LLM passed invalid email format → goes to llm_errors.wrong_input_values + - Schema didn't explain required format → goes to request_schema_problems + - 401/403 auth errors → goes to connection_problems + - "missing access", "access denied", "permission denied" → goes to connection_problems + - Insufficient OAuth scopes → goes to connection_problems + """ + + tool_name: str = Field(..., description="The action/tool that failed") + log_id: Optional[str] = Field(None, description="Log ID where this occurred") + error_message: str = Field(..., description="The error message from execution") + error_category: str = Field( + ..., + description="Category: 'tool_bug' (bug in Composio tool code), " + "'external_api_issue' (API rate limit/downtime - not tool's fault), " + "'tier_mismatch' (API tier doesn't support operation), " + "'unknown' (unclear root cause)", + ) + reason: str = Field( + ..., + description="Detailed analysis explaining: (1) what inputs were provided and why they were correct, " + "(2) what the tool did wrong, (3) what the expected vs actual behavior was. " + "This should clearly demonstrate that the failure was NOT due to LLM mistakes or schema issues.", + ) + is_composio_bug: bool = Field( + ..., + description="True if this is a bug in Composio's tool implementation that needs fixing. " + "False if it's an external API issue or tier mismatch outside Composio's control.", + ) + called_in_workbench: bool = Field( + False, description="Whether tool was called from within workbench" + ) + + +class ResourceNotFound(ErrorItemBase): + """ + Resource not found despite valid inputs - benefit of doubt to user. + + Use this when the tool executed successfully (no error) but returned empty/not-found + results when the user provided seemingly valid inputs. We give benefit of doubt to + the user and assume the tool may have an issue rather than blaming user for wrong inputs. + + Examples: + - User asked to find "general" channel in Slack, tool returned empty results + - User provided a document ID, GOOGLE_DOCS_GET returned "not found" + - User searched for a contact by email, CRM tool found nothing + - User tried to list items in a folder, tool returned empty despite folder existing + + We're attributing the failure to potential tool issues (wrong API call, incorrect filtering, case sensitivity, etc.) + rather than blaming the user. + + NOT for this bucket: + - User clearly provided invalid/malformed inputs → llm_errors.wrong_input_values + - Tool returned an actual error message (500, auth error) → tool_implementation_problems + - Schema was unclear about expected format → request_schema_problems + - Tool returned data in wrong format → response_schema_problems + """ + + tool_name: str = Field( + ..., description="The tool name that couldn't find the resource" + ) + toolkit_name: str = Field( + ..., description="The toolkit (e.g., 'SLACK', 'GOOGLEDOCS')" + ) + log_id: Optional[str] = Field(None, description="Log ID where this occurred") + called_in_workbench: bool = Field( + False, description="Whether tool was called from within workbench" + ) + resource_type: str = Field( + ..., + description="Type of resource that wasn't found: 'channel', 'document', 'user', " + "'email', 'file', 'folder', 'record', 'item', 'contact', 'message', 'other'", + ) + user_provided_identifier: str = Field( + ..., + description="What the user provided to identify the resource (e.g., channel name, doc ID, email)", + ) + reason: str = Field( + ..., + description="Explanation of why we believe the user's input was valid and the tool " + "may have an issue. Include what was tried and why it's reasonable to expect " + "the resource should have been found.", + ) + attempts_made: int = Field( + 1, + description="Number of times the tool was called trying to find this resource", + ) + potential_tool_issue: Optional[str] = Field( + None, + description="Suspected issue with the tool: 'incorrect_api_endpoint', 'wrong_search_params', " + "'missing_pagination', 'case_sensitivity', 'encoding_issue', 'permission_scope', 'other'", + ) + + +class ResponseSchemaProblem(ErrorItemBase): + """Mismatch between a tool's DOCUMENTED response schema and its ACTUAL response. + + CRITICAL: This is about the tool's OWN schema accuracy, NOT about what the LLM/workbench + expected. The LLM can hallucinate expectations - that's irrelevant here. We ONLY care about: + 1. What does the tool's response schema DOCUMENT/ADVERTISE? + 2. What does the tool ACTUALLY return? + 3. Do these two match? + + If the documented schema says field X exists but the actual response doesn't have it, + that's a ResponseSchemaProblem - regardless of whether the LLM tried to use field X. + + Examples: + - Tool's documented schema says response has 'emails' field but actual response has 'messages' + - Tool's documented schema says 'id' is a string but actual response returns integer + - Tool's documented schema shows nested 'data.items' but actual response has flat 'items' + - Tool's documented schema is missing a field that's actually returned + + NOT for this bucket: + - LLM expected a field that isn't in the documented schema → NOT a schema problem + - Resource not found / empty results → resource_not_found + - Request parameter issues → request_schema_problems + - Tool execution errors (500, auth) → tool_implementation_problems + """ + + tool_name: str = Field(..., description="The tool with the response schema problem") + parameter_name: str = Field( + ..., + description="The response field/parameter where documented schema differs from actual response " + "(e.g., 'data.emails', 'response.id', 'items[].name')", + ) + problem_type: str = Field( + ..., + description="Type: 'missing_field' (documented in schema but not in actual response), " + "'extra_field' (in actual response but not documented in schema), " + "'wrong_type' (type mismatch between documented schema and actual response), " + "'wrong_structure' (nested structure in actual response differs from documented schema), " + "'wrong_key_name' (key name in actual response differs from documented schema, e.g., 'emails' vs 'messages')", + ) + called_in_workbench: bool = Field( + False, + description="True if this schema problem was encountered in workbench code, " + "False if via multi_execute", + ) + documented_schema: Optional[str] = Field( + None, + description="What the tool's response schema DOCUMENTS (e.g., 'string field \"id\"'). " + "This is the tool's advertised contract, not what LLM expected.", + ) + actual_response: Optional[str] = Field( + None, + description="What the tool ACTUALLY returned (e.g., 'integer field \"id\" with value 12345')", + ) + reason: str = Field( + ..., + description="Explanation of how the documented schema differs from actual response. " + "Focus on the tool's schema accuracy, not on LLM expectations.", + ) + suggested_fix: Optional[str] = Field( + None, + description="Suggested fix to the tool's response schema documentation to match actual behavior", + ) + + +class MissingToolImpact(ErrorItemBase): + """ + Impact assessment of a suggested missing tool/toolkit on workflow failure. + + Call 1 identifies tools/toolkits that don't exist but would be useful. + Call 2 assesses: if these existed, would the workflow have succeeded? + + NOT for: missing features in existing tools (use missing_feature_impact or tool_improvements). + + NOT missing toolkits: Using workbench/bash to fetch URLs (s3url, download links) is EXPECTED. + Downloads of files only works in workbench - doesn't make sense otherwise. + """ + + suggestion_type: str = Field( + ..., + description="Type: 'new_tool' (tool for existing toolkit) or 'new_toolkit' (entirely new integration)", + ) + tool_or_toolkit_name: str = Field( + ..., + description="Name of the suggested tool (e.g., 'GMAIL_ARCHIVE_EMAIL') or toolkit (e.g., 'airtable')", + ) + toolkit: Optional[str] = Field( + None, + description="For new_tool suggestions, the existing toolkit it belongs to (e.g., 'gmail')", + ) + use_case: str = Field(..., description="The use case that needed this tool/toolkit") + would_have_helped: bool = Field( + ..., + description="True if having this tool/toolkit would have made the workflow succeed or significantly improve", + ) + impact_explanation: str = Field( + ..., + description="How the absence of this tool/toolkit affected the workflow outcome", + ) + + +class MissingFeatureImpact(ErrorItemBase): + """ + Impact assessment when an existing tool lacks a feature needed for the workflow. + + Use this when: + - The tool EXISTS and works correctly for its current capabilities + - But it doesn't support a feature/capability the user's use case required + - The workflow FAILED or was incomplete because of this limitation + + Examples: + - GMAIL_SEND_EMAIL only supports attachment links, user needed inline/embedded content + - SLACK_POST_MESSAGE doesn't support scheduling messages for later + - JIRA_CREATE_ISSUE doesn't support custom field types the user needed + - GOOGLESHEETS_WRITE_DATA doesn't support writing to protected ranges + + NOT for this bucket: + - Tool has a bug → tool_implementation_problems + - Tool schema is unclear → request_schema_problems + - Need an entirely new tool → missing_tools_impact + - Workflow succeeded (use tool_improvements in PHASE 3 instead for opportunistic suggestions) + """ + + tool_name: str = Field( + ..., + description="The existing tool that lacks the feature (e.g., 'GMAIL_SEND_EMAIL')", + ) + toolkit_name: str = Field(..., description="The toolkit (e.g., 'GMAIL', 'SLACK')") + missing_feature: str = Field( + ..., + description="The feature/capability that was needed but not available " + "(e.g., 'inline/embedded content in email body', 'scheduled message posting')", + ) + use_case: str = Field(..., description="The use case that required this feature") + impact_explanation: str = Field( + ..., + description="How the lack of this feature affected the workflow outcome. " + "Explain what the user wanted to do and why the tool couldn't support it.", + ) + suggested_fix: Optional[str] = Field( + None, + description="Optional: How the tool could be improved to support this feature " + "(e.g., 'Add support for Content-ID headers for inline attachments')", + ) + priority: int = Field( + 0, + ge=1, + le=10, + description="Priority for adding this feature (1-10, higher = more important)", + ) + + +class ToolErrorBucket(ErrorBucketBase): + """Container for all tool-related errors in a session.""" + + request_schema_problems: List[RequestSchemaProblem] = Field(default_factory=list) + response_schema_problems: List[ResponseSchemaProblem] = Field(default_factory=list) + tool_implementation_problems: List[ToolImplementationProblem] = Field( + default_factory=list + ) + resource_not_found: List[ResourceNotFound] = Field(default_factory=list) + + # Missing tools impact - assesses if suggested NEW tools would have helped + missing_tools_impact: List[MissingToolImpact] = Field( + default_factory=list, + description="Impact assessment of suggested new tools/toolkits from Call 1. " + "Evaluates whether these missing tools contributed to workflow failure.", + ) + + # Missing feature impact - when EXISTING tools lack needed capabilities + missing_feature_impact: List[MissingFeatureImpact] = Field( + default_factory=list, + description="Impact assessment when existing tools lack features needed for the workflow. " + "Use when a tool works correctly but doesn't support a capability the use case required.", + ) + + # Catch-all for tool errors that don't fit above categories + other_errors: List[OtherError] = Field( + default_factory=list, + description="Tool-related errors that don't fit into predefined categories", + ) + + weight: float = Field( + 0.0, + ge=0.0, + le=1.0, + description="How much this bucket contributed to overall failure. " + "All bucket weights should sum to 1.0.", + ) + + def _get_item_lists( + self, + ) -> List[Tuple[List[Any], Optional[Callable[[Any], bool]]]]: + """Return item lists for this bucket.""" + return [ + (self.request_schema_problems, None), + (self.response_schema_problems, None), + (self.tool_implementation_problems, None), + (self.resource_not_found, None), + (self.missing_tools_impact, None), + (self.missing_feature_impact, None), + (self.other_errors, None), + ] diff --git a/app_tester/rube_learning/error_analysis/models/workbench.py b/app_tester/rube_learning/error_analysis/models/workbench.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/models/workbench.py @@ -0,0 +1,157 @@ +""" +Workbench Bucket - Issues with COMPOSIO_REMOTE_WORKBENCH code execution. + +These track problems with workbench code quality, tool usage within workbench, +unnecessary calls, and execution metrics. +""" + +from typing import Any, Callable, List, Optional, Tuple + +from pydantic import Field + +from .base_bucket import ErrorBucketBase +from .base_item import ErrorItemBase +from .other_errors import OtherError + + +class CodeLogicError(ErrorItemBase): + """Problems with the code logic written in workbench.""" + + log_id: str = Field(..., description="Log ID of the workbench execution") + is_optimized: bool = Field(..., description="Whether the code was optimized") + is_redundant: bool = Field( + ..., description="Whether the code was redundant/unnecessary" + ) + quality_score: float = Field( + ..., + ge=0.0, + le=1.0, + description="Code quality score (0.0 = poor, 1.0 = excellent)", + ) + issues: List[str] = Field( + default_factory=list, description="List of specific issues found in the code" + ) + reason: str = Field(..., description="Overall explanation of code quality problems") + + +class WorkbenchHelperToolProblem(ErrorItemBase): + """Problems when calling helper functions within workbench code. + + NOTE: Wrong input values/types are tracked in llm_errors bucket with called_in_workbench=True. + Request schema problems are tracked in tool_errors bucket with called_in_workbench=True. + This model tracks workbench-specific issues like response parsing errors and wrong inputs when using the helper functions. + + IMPORTANT: This bucket is ONLY for helper function errors (run_composio_tool, invoke_llm, proxy_execute, web_search). + Composite tool errors should go to tool_errors bucket, NOT here. Do NOT duplicate tool errors here. + + IMPORTANT: Error messages like "Helper function error: Internal server error: ..." can be misleading. + Analyze the actual request/response to determine the root cause: + - If the tool itself failed (e.g., API error, invalid tool inputs), classify in tool_errors bucket. + - Only classify here if the helper function wrapper itself failed (e.g., parsing issues, wrong helper usage). + """ + + log_id: str = Field(..., description="Log ID of the workbench execution") + problem_type: str = Field( + ..., + description="Type: 'response_parsing_error' (when the helper function failed to parse the response correctly), 'wrong_inputs' (when the helper function received wrong inputs), 'other_error' (when the helper function failed for other reasons)", + ) + tool_name: Optional[str] = Field( + None, + description="Optional tool name that the helper function was used to call, if applicable", + ) + toolkit_name: Optional[str] = Field( + None, description="Toolkit the tool belongs to, if applicable" + ) + helper_function: Optional[str] = Field( + None, + description="Helper function that the error occurred in: 'run_composio_tool', 'invoke_llm', " + "'proxy_execute', 'web_search', 'upload_local_file'", + ) + reason: str = Field( + ..., description="Explanation of what went wrong with the helper function call" + ) + + +class UnnecessaryWorkbenchCall(ErrorItemBase): + """Workbench was called when it wasn't needed.""" + + log_id: str = Field(..., description="Log ID of the unnecessary workbench call") + context: str = Field( + ..., description="What the workbench was used for and why it wasn't necessary" + ) + + +class MetaToolFailure(ErrorItemBase): + """The workbench meta-tool itself failed (500 error, timeout, etc.).""" + + log_id: Optional[str] = Field(None, description="Log ID where meta-tool failed") + error_code: Optional[str] = Field( + None, + description="Error code: 'EXECUTION_TIMEOUT', '500_ERROR', 'SANDBOX_ERROR', etc.", + ) + reason: str = Field(..., description="Why the meta-tool failed") + + +class MissingPackage(ErrorItemBase): + """Code failed because a required package wasn't available.""" + + log_id: str = Field(..., description="Log ID where package was missing") + package_name: str = Field(..., description="Name of the missing package") + context: Optional[str] = Field(None, description="What the package was needed for") + + +class WorkbenchExecutionCount(ErrorItemBase): + """Tracks if workbench was called excessively for a use case.""" + + use_case: str = Field(..., description="The use case these executions were for") + num_executions: int = Field(..., description="Number of workbench executions") + log_ids: List[str] = Field( + default_factory=list, description="Log IDs of all executions" + ) + was_excessive: bool = Field( + ..., description="Whether the number of executions was excessive" + ) + reason: str = Field( + ..., description="If excessive, why so many executions were needed" + ) + + +class WorkbenchBucket(ErrorBucketBase): + """Container for all workbench-related issues in a session.""" + + # LLM-analyzed items (with weights) + code_logic_errors: List[CodeLogicError] = Field(default_factory=list) + helper_tool_problems: List[WorkbenchHelperToolProblem] = Field(default_factory=list) + unnecessary_calls: List[UnnecessaryWorkbenchCall] = Field(default_factory=list) + meta_tool_failures: List[MetaToolFailure] = Field(default_factory=list) + missing_packages: List[MissingPackage] = Field(default_factory=list) + execution_counts: List[WorkbenchExecutionCount] = Field(default_factory=list) + + # Catch-all for workbench errors that don't fit above categories + other_errors: List[OtherError] = Field( + default_factory=list, + description="Workbench-related errors that don't fit into predefined categories", + ) + + weight: float = Field( + 0.0, + ge=0.0, + le=1.0, + description="How much this bucket contributed to overall failure. " + "All bucket weights should sum to 1.0.", + ) + + def _get_item_lists( + self, + ) -> List[Tuple[List[Any], Optional[Callable[[Any], bool]]]]: + """Return item lists for this bucket.""" + return [ + (self.code_logic_errors, None), + (self.helper_tool_problems, None), + (self.unnecessary_calls, None), + (self.meta_tool_failures, None), + (self.missing_packages, None), + # Only count execution_counts where was_excessive=True + (self.execution_counts, lambda x: x.was_excessive), + (self.other_errors, None), + ] diff --git a/app_tester/rube_learning/error_analysis/runner.py b/app_tester/rube_learning/error_analysis/runner.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/runner.py @@ -0,0 +1,1382 @@ +""" +Error Analysis Runner (Call 2) + +Runs the second LLM call to analyze errors, assign weights, and generate improvements. + +Input: + - Session data (full logs) + - workflow_summary (from Call 1) + - search quality data from Call 1 (wrong_tools_returned, cached plan quality, etc.) + - Static data extracted from logs + +Output: + - SessionErrorAnalysis (complete error analysis with weights) +""" + +import logging +import os +import time +from typing import Any, Dict, List, Optional + +from openai import AsyncOpenAI + +from app_tester.rube_learning.call_parser import ParsedCall, ParsedSession +from app_tester.rube_learning.config import ( + DESCRIPTION_MAX_LENGTH, + JSON_KEY_VALUE_MAX_LENGTH, + JSON_MAX_KEYS_TO_SHOW, + JSON_SUMMARY_MAX_LENGTH, + MEMORY_FIELD_MAX_LENGTH, + METADATA_FIELD_MAX_LENGTH, + PARAM_DESCRIPTION_MAX_LENGTH, + SESSION_LOG_ERROR_MAX_LENGTH, + SESSION_LOG_REQUEST_MAX_LENGTH, + SESSION_LOG_RESPONSE_MAX_LENGTH, + THOUGHT_MAX_LENGTH, +) +from app_tester.rube_learning.session_detector import Session + +from .extractors import ( + RawWorkbenchCall, + StaticExtractionResult, + extract_all_static_data, +) +from .models import ( + ConnectionMetrics, + ErrorAnalysisLLMOutput, + MultiExecuteMetrics, + PerUsecaseCachedPlanQuality, + RecipeMetrics, + SearchQualityMetrics, + SessionErrorAnalysis, + SessionMetrics, + TokenUsageMetrics, + ToolCallTokenUsage, + WorkbenchMetrics, + WorkbenchTimeTaken, +) + + +logger = logging.getLogger(__name__) + + +def build_error_analysis_prompt( + session: Session, + workflow_summary: str, + wrong_tools_returned: List[Dict[str, Any]], + static_data: StaticExtractionResult, + tool_info: Dict[str, Dict[str, Any]], + tool_suggestions: Optional[Dict[str, Any]] = None, +) -> str: + """ + Build the prompt for Call 2 (Error Analysis). + + Provides raw data to the LLM - the LLM does all categorization and weighting. + """ + # Format session logs - uses pre-extracted data to avoid duplicate extraction + session_logs = _format_session_logs(session, static_data) + + # Format raw static data + static_str = _format_static_data(static_data) + + # Format wrong tools from Call 1 + wrong_tools_str = _format_wrong_tools(wrong_tools_returned) + + # Format tool information (description + input schema) + tool_info_str = _format_tool_info(tool_info) + + # Format tool suggestions from Call 1 + tool_suggestions_str = _format_tool_suggestions(tool_suggestions) + + prompt = f"""# Error Analysis & Improvements (Call 2) + +You are analyzing errors from a Composio workflow to identify what went wrong and suggest improvements. + +## Important Context + +Composio is a universal tool/MCP layer that AI agents use. The agent calls meta-tools (Search, Multi-Execute, Workbench, etc.) which in turn execute app tools (GMAIL_SEND_EMAIL, SLACK_POST_MESSAGE, etc.). + +**Your goals:** +1. Identify and categorize ALL errors that occurred +2. Assign weights to quantify impact of each error +3. Suggest actionable improvements +4. Determine the primary failure reason and overall workflow quality + +## Workflow Context (from Call 1) + +Call 1 is a previous LLM call that analyzed this session's workflow, identified the user's intent, and assessed search quality. + +Summary of the workflow: + +{workflow_summary} +## Session Logs + +{session_logs} + +## Tool Information (from search results) + +{tool_info_str} + +## Extracted Data Summary from Logs (this can help ground your analysis since these are raw facts) + +{static_str} + +## wrong_tools_returned (from Call 1) - SUGGESTIONS TO VALIDATE + +These are Call 1's **suggestions** about potential search quality issues. See "Search Quality" in Phase 1 for how to handle them. + +{wrong_tools_str} + +## New Tool/Toolkit Suggestions (from Call 1) + +Call 1 identified tools or toolkits that don't exist in Composio but would have been useful for this workflow. +Your job is to assess: if these tools/toolkits existed, would the workflow have succeeded? +Populate `tool_errors.missing_tools_impact` with your assessment and assign weights. + +**Filter out tool improvements** (retry/pagination/validation/error handling for existing tools) - those belong in `tool_improvements` section. Only include genuinely new tools/toolkits here. + +{tool_suggestions_str} + +--- + +## PHASE 1: Error Bucket Analysis + +First, systematically analyze each error bucket and identify specific issues. You will assess overall workflow quality AFTER completing this analysis. + +**Each error belongs to exactly ONE bucket** - identify the root cause and categorize there only. + +### LLM Errors (mistakes the LLM made) + +- **wrong_input_values**: LLM passed incorrect values when the schema was correct and descriptive. Set `called_in_workbench=True` if error occurred in workbench code. +- **wrong_input_types**: Type mismatches when schema clearly specified the correct type. Set `called_in_workbench=True` if error occurred in workbench code. +- **multi_execute_wrongly_called**: Called multi-execute incorrectly (actions that should have been sequential) +- **multi_execute_not_called**: Should have used multi-execute but didn't (missed batching opportunity) + - **IMPORTANT**: This error applies ONLY when the LLM made multiple separate multi_execute calls in sequence that could have been combined into a single call. + - **NOT an error when**: The LLM uses workbench with a for loop to make many tool calls (>5 calls). Using workbench for bulk operations is the CORRECT approach because it allows programmatic iteration, parallelization with ThreadPoolExecutor, and better error handling. +- **missed_pagination**: Failed to paginate when needed +- **hallucinated_tool_called**: Called a tool that doesn't exist + +**LLM vs Schema Problem Decision:** +- If the tool's schema clearly described the expected value/type and the LLM still made a mistake → **llm_errors** (wrong_input_values/wrong_input_types) +- If the schema was missing, unclear, or misleading and caused the LLM to make a mistake → **tool_errors.request_schema_problems** + +**NOT LLM Errors:** Errors from workbench helpers (`run_composio_tool`, `proxy_execute`, `invoke_llm`) go in **workbench bucket helper_tool_problems**, NOT here. + +### Tool Errors (problems with Composio app tools) + +These are issues with **app tools** (GMAIL_SEND_EMAIL, SLACK_POST_MESSAGE, etc.), NOT meta-tools or workbench helpers. + +- **request_schema_problems**: Request schema was missing, unclear, or misleading causing wrong inputs. Set `called_in_workbench=True` if encountered in workbench code. +- **response_schema_problems**: Mismatch between tool's DOCUMENTED response schema and ACTUAL response. + - CRITICAL: This is about the tool's OWN schema accuracy, NOT about what the LLM/workbench expected + - LLMs can hallucinate expectations - that's IRRELEVANT here. We ONLY care about: + 1. What does the tool's response schema DOCUMENT/ADVERTISE? + 2. What does the tool ACTUALLY return? + 3. Do these two match? + - If the documented schema says field X exists but actual response doesn't have it → ResponseSchemaProblem + - NOT for this: LLM expected a field not in documented schema (→ NOT a schema problem), resource not found (→ resource_not_found), request issues (→ request_schema_problems) +- **tool_implementation_problems**: Tool failed due to a bug in its implementation, NOT due to LLM mistakes or schema issues. + - ONLY use this when: (1) LLM provided valid, correct inputs, (2) schema was clear, (3) auth was working, BUT the tool STILL failed + - Examples of tool bugs: wrong API endpoint, incorrect data transformation, bug in response parsing, unhandled valid edge case + - NOT for this bucket: LLM passed wrong inputs (→ llm_errors), schema was unclear (→ request_schema_problems), auth issues (→ connection_problems) + - **CRITICAL**: 401/403 errors, "access denied", "missing access", "permission denied", "insufficient scope" errors are ALL connection_problems, NOT tool_implementation_problems + - Set `is_composio_bug=True` if it's a bug in Composio's code, `False` if it's external API downtime/rate limits + - If an app tool fails INSIDE multi_execute, it still goes here (with called_in_workbench=False) + - If an app tool fails INSIDE workbench via run_composio_tool, it goes here (with called_in_workbench=True) +- **resource_not_found**: Resource not found despite user providing valid-looking inputs - GIVE BENEFIT OF DOUBT TO USER. + - Use this when a list/get/search operation returned empty or "not found" but the user's inputs appeared reasonable + - The tool executed successfully (no error) but just didn't find the resource + - NOT for this: User clearly provided invalid/malformed inputs (→ llm_errors), tool returned actual error (→ tool_implementation_problems), tool returned wrong format (→ response_schema_problems) +- **missing_tools_impact**: Impact assessment of suggested new tools/toolkits from Call 1. + - For each suggestion in "New Tool/Toolkit Suggestions" section, assess: + - `would_have_helped`: Would this tool/toolkit have made the workflow succeed or significantly improve? + - `impact_explanation`: How did the absence of this tool affect the outcome? + - Only include suggestions that would have meaningfully impacted the workflow + - If the workflow succeeded anyway or the suggestion wouldn't have helped, don't include it + - **CRITICAL**: Do NOT include missing features (retry logic, pagination, validation, etc.) in existing tools here - those go in `missing_feature_impact` below or in tool_improvements in PHASE 3 + - **NOT missing toolkits (REJECT these suggestions):** + - Using workbench/bash with curl/requests to fetch URLs (s3url, download links) is EXPECTED behavior + - File downloads only work in workbench - an HTTP_GET toolkit is NOT needed +- **missing_feature_impact**: When an EXISTING tool lacks a feature/capability needed for the workflow. + - Use this when: + - The tool EXISTS and works correctly for its current capabilities + - But it doesn't support a feature the user's use case required + - The workflow FAILED (or was SUCCESS_WEAK or PARTIAL_SUCCESS) because of this limitation + - Examples: + - SLACK_POST_MESSAGE doesn't support scheduling messages for later + - NOT for this bucket: + - Tool has a bug → tool_implementation_problems + - Tool schema is unclear → request_schema_problems + - Need an entirely NEW tool that doesn't exist → missing_tools_impact + - Workflow SUCCEEDED but you noticed the tool could be better → use `tool_improvements` in PHASE 3 instead + +**NOT Tool Errors:** Failures of workbench helpers themselves (`run_composio_tool` returning error, `proxy_execute` failing, `invoke_llm` error) go in **workbench bucket**, NOT here. + +### Search Quality + +The `wrong_tools_returned` items from the previous analysis (Call 1) are **suggestions**, not verified facts. You are the final arbiter. + +**CRITICAL CONSTRAINT - READ-ONLY MODE:** +- You do NOT have access to Composio's tool catalog. The previous analysis had the catalog and validated tool names against it. +- You can ONLY: + 1. **REJECT** entire `wrong_tools_returned` entries (by not including them in output) + 2. **REMOVE** items from `tools_missing` or `tools_irrelevant` lists (if you think they didn't actually impact the workflow) + 3. **ASSIGN WEIGHTS** to entries you include +- You **CANNOT**: + 1. ADD new tool names to `tools_missing` - you don't know what tools exist in Composio + 2. ADD new tool names to `tools_irrelevant` - you don't know what tools exist in Composio + 3. ADD new `wrong_tools_returned` entries - the previous analysis already identified all search quality issues + 4. MODIFY `tools_returned` - this is a factual record of what was returned + +**Your job:** +- **Filter**: Remove tools from `tools_missing`/`tools_irrelevant` that didn't actually impact the workflow +- **Assign weights**: Based on actual impact on workflow failure + +**Examples:** +- Call 1 says "SLACK_LIST_USERS was missing" but workflow succeeded without it → REJECT entire entry +- Call 1 says "GMAIL_SEARCH_EMAILS was irrelevant" but it was used successfully → REMOVE from tools_irrelevant +- Call 1 says "JIRA_CREATE_ISSUE was missing" and workflow failed because it couldn't create issues → INCLUDE with appropriate weight + + +### Connection Problems + +- **manage_connection_not_called**: Needed to call MANAGE_CONNECTION but didn't +- **manage_connection_failed**: MANAGE_CONNECTION was called but failed +- **connection_errors_in_tool_execution**: **Composio tool** execution failed due to auth/connection issues (via multi_execute or `run_composio_tool`). + - **ONLY for Composio tools** - toolkit_name must be a valid Composio toolkit (GMAIL, SLACK, JIRA, etc.) + - **NOT for non-Composio operations** like `upload_local_file`, `invoke_llm`, `web_search` - those go to `other_errors` below + - **CRITICAL - These are connection errors, NOT tool_errors:** + - "missing access" or "access denied" errors + - 401 Unauthorized / 403 Forbidden errors + - Insufficient OAuth scopes (user didn't grant required permissions) + - "Permission denied" or "insufficient permissions" errors + - API key lacks required access level + - Token expired or refresh failed + - Even if the tool technically "executed" and returned an error, if that error is about access/auth/permissions, it belongs HERE in connection_problems, NOT in tool_errors. +- **false_connection_status**: Connection reported as active but actually wasn't +- **other_errors**: Connection/auth errors from non-Composio operations (e.g., `upload_local_file` 403 errors, `invoke_llm` auth failures). Use this when there's an auth/connection error but it's NOT from a Composio tool execution. + +### Workbench Issues + +Analyze the **full code** and **execution output** provided in the "Workbench Calls (Full Details)" section. + +**When Workbench Usage is VALID (do NOT flag as unnecessary_calls):** +- Processing large datasets (100+ items) that would exceed multi-execute limits +- When multi-execute response was synced to workbench (`sync_response_to_workbench=true`) because data was too large +- Using `proxy_execute` for APIs without dedicated Composio tools (NOTE: `proxy_execute` usage signals a missing tool - add to `tool_errors.missing_tools_impact` if the workflow would benefit from a dedicated Composio tool) + +**When Workbench Usage is UNNECESSARY (flag as unnecessary_calls):** +- Simple operations on small datasets (< 20 items) that multi-execute could handle +- When the LLM wrote code just to call one tool with static arguments + +**Error Categories:** +- **code_logic_errors**: Bugs in generated code (syntax errors, logic errors, incorrect data processing) +- **helper_tool_problems**: Issues with workbench helper functions ONLY (`run_composio_tool`, `proxy_execute`, `invoke_llm`, `web_search` wrapper failures) + - IMPORTANT: This is ONLY for helper function wrapper failures, NOT for composite tool errors. Composite tool errors go in tool_errors bucket. + - This is the correct bucket for `run_composio_tool`, `proxy_execute` and other workbench helper failures when the HELPER ITSELF fails (parsing issues, wrong helper usage) + - NOTE: If the failure is due to LLM passing wrong values → llm_errors with `called_in_workbench=True` + - NOTE: If the failure is due to unclear schema → tool_errors.request_schema_problems with `called_in_workbench=True` + - NOTE: If the failure is due to connection/auth issues (token expired, permissions revoked, etc.) → connection_problems.connection_errors_in_tool_execution with `called_in_workbench=True` + - NOTE: If the underlying tool failed (API error, invalid tool inputs) → tool_errors bucket, NOT here +- **unnecessary_calls**: Workbench used when multi-execute would suffice (see criteria above) +- **meta_tool_failures**: Failures in workbench meta-tool itself. The COMPOSIO_REMOTE_WORKBENCH meta-tool itself failed. +- **missing_packages**: Required packages not available + +**How to analyze workbench code:** +1. Find all `run_composio_tool("TOOL_NAME", {...})` calls in the code - these are Composio tool executions +2. Check the execution output for success/failure of each tool call +3. If a tool failed, determine if it was due to: wrong arguments, API error, auth issue, etc. +4. Categorize tool failures appropriately (helper_tool_problems ONLY if the workbench helper wrapper itself failed, or tool_errors bucket if the underlying tool/API failed) + +### Multi-Execute Issues + +Infrastructure issues with COMPOSIO_MULTI_EXECUTE_TOOL. + +- **meta_tool_failures**: The multi_execute meta-tool itself failed (500 error, timeout before any tool ran, invalid request) + +**IMPORTANT**: +- If an individual app tool fails INSIDE multi_execute due to a tool implementation bug (not LLM wrong inputs), that goes in **tool_errors.tool_implementation_problems**, NOT here. +- Missed batching opportunities go in **llm_errors.multi_execute_not_called** (it's an LLM decision error). +- This bucket is ONLY for multi_execute infrastructure failures (meta-tool itself failing before dispatching tools). + +### Recipe Issues + +- **recipe_failures**: Recipe execution failures + +--- + +## PHASE 2: Weight Assignment + +**CRITICAL: Weights must be properly distributed!** + +**Bucket Weights:** +- If you found ZERO issues in Phase 1 (no items in any bucket), all bucket weights = 0.0 +- Otherwise, distribute weights across buckets that have issues - **MUST sum to exactly 1.0** +- Buckets with no issues should have weight 0.0 + +**Item Weights (FLAT distribution within each bucket):** +- Within each bucket, distribute weights **FLAT across ALL items from ALL sub-lists** +- This means if a bucket has items in multiple sub-lists (e.g., workbench has `code_logic_errors`, `helper_tool_problems`, `unnecessary_calls`), ALL items across ALL these sub-lists must have their weights sum to 1.0 **together** +- **DO NOT** have weights sum to 1.0 within each sub-list separately +- Higher weight = bigger contribution to overall failure +- **NO ZERO-WEIGHT ITEMS**: If you include an item in a bucket, you MUST assign it a weight > 0.0. Do NOT leave item weights unspecified (they default to 0.0 and will be treated as INVALID). + +**Procedure (MUST follow exactly):** +1. For a given bucket, build ONE combined list `ALL_ITEMS` by concatenating items from every sub-list in that bucket (e.g., for `tool_errors`: `request_schema_problems + response_schema_problems + tool_implementation_problems + resource_not_found + missing_tools_impact + other_errors`). +2. Let N = len(ALL_ITEMS). If N == 0: do not assign any item weights for that bucket (it should have no issues anyway). +3. Start with a flat baseline: every item weight starts at 1/N. +4. If you want to emphasize certain items, you may deviate from flat, BUT you must keep the sum across **ALL_ITEMS** exactly 1.0. +5. **FINAL STEP (REQUIRED)**: For each non-empty bucket, compute the sum of weights across **ALL_ITEMS**. If it is not exactly 1.0, renormalize across **ALL_ITEMS** so the total becomes 1.0. Do NOT renormalize within each sub-list. + +### Concrete Example + +**Session**: User tried to send email via Gmail. LLM provided correct inputs but the tool failed because the tool code incorrectly formatted the request body. + +**Bucket Analysis:** +- tool_errors: Gmail tool has a bug in how it constructs the API request (tool_implementation_problems) +- This is NOT an LLM error because the inputs were correct +- This is NOT a schema problem because the schema was clear + +**Weight Assignment:** +- tool_errors.bucket_weight = 1.0 (the tool bug was the only issue) + - tool_implementation_problems[0].weight = 1.0 (only one item in bucket, so it gets 100%) +- All other buckets = 0.0 + +**Result:** Bucket weights sum to 1.0. Each bucket's item weights sum to 1.0. + +### Multi-Item Example (Flat Distribution) + +**Session**: Workbench had multiple issues - code bugs, unnecessary calls, and tool problems. +THIS APPLIES TO ALL BUCKETS WITH SUB-BUCKETS. + +**Bucket Analysis:** +- workbench bucket (weight: 0.4) has 4 items across different sub-lists: + - 2 code_logic_errors + - 1 helper_tool_problems + - 1 unnecessary_calls + +**CORRECT Weight Assignment (flat across ALL items):** +- workbench.bucket_weight = 0.4 +- code_logic_errors[0].weight = 0.35 (major code bug) +- code_logic_errors[1].weight = 0.25 (minor code issue) +- helper_tool_problems[0].weight = 0.25 (parsing error) +- unnecessary_calls[0].weight = 0.15 (minor inefficiency) +- **Total item weights: 0.35 + 0.25 + 0.25 + 0.15 = 1.0** ✓ + +**WRONG Weight Assignment (DO NOT DO THIS):** +This is INCORRECT - each sub-list sums to 1.0 separately: +- code_logic_errors[0].weight = 0.6 <-- sub-list 1 +- code_logic_errors[1].weight = 0.4 <-- sub-list 1 total = 1.0 ❌ WRONG +- helper_tool_problems[0].weight = 1.0 <-- sub-list 2 total = 1.0 ❌ WRONG +- unnecessary_calls[0].weight = 1.0 <-- sub-list 3 total = 1.0 ❌ WRONG +- **Total item weights: 0.6 + 0.4 + 1.0 + 1.0 = 3.0** ❌ INVALID + +The TOTAL of ALL items in the bucket must be 1.0, NOT each sub-list separately! + +--- + +## PHASE 3: Generate Improvements + +Based on errors found, suggest specific improvements: + +### tool_improvements +For regular Composio app tools (GMAIL_SEND_EMAIL, SLACK_POST_MESSAGE, JIRA_CREATE_ISSUE, etc.). + +Include: +- **action_name**: The tool to improve +- **toolkit_name**: The toolkit (e.g., 'GMAIL') +- **improvement_type**: One of: + - `schema_fix`: Fix incorrect/misleading schema if it didn't cause workflow failure. if yes, it should be in `tool_errors.request(or response)_schema_problems` instead. + - `description_improvement`: Improve parameter descriptions + - `add_parameter`: Add a new parameter + - `remove_parameter`: Remove unnecessary parameter + - `fix_bug`: Fix a bug in tool implementation + - `better_error_handling`: Improve error messages/handling + - `output_format`: Fix response format issues + - `missing_feature`: Tool needs new capability (USE ONLY for SUCCESS workflows - if the missing feature CAUSED failure, it should be in `tool_errors.missing_feature_impact` instead) + - `other`: Catch-all for improvements that don't fit other categories +- **current_issue**: What's currently wrong +- **suggested_fix**: Specific fix +- **priority**: 1-10 (10 = most critical) +- **evidence_log_ids**: Log IDs demonstrating the issue + +**NOTE on missing_feature**: If you already captured a missing feature in `tool_errors.missing_feature_impact` (because it caused workflow failure), you don't need to duplicate it here. Only use `improvement_type='missing_feature'` for: +- Workflows that SUCCEEDED but you noticed the tool could be better +- Additional context/suggestions beyond what's in the error bucket + +### meta_tool_improvements +ONLY for Composio platform meta-tools: +- COMPOSIO_SEARCH_TOOLS: Tool search/discovery +- COMPOSIO_MANAGE_CONNECTIONS: Connection management +- COMPOSIO_MULTI_EXECUTE_TOOL: Batch tool execution +- COMPOSIO_REMOTE_WORKBENCH: Code execution environment +- COMPOSIO_EXECUTE_RECIPE: Recipe execution + +**IMPORTANT**: Do NOT add tools like COMPOSIO_MULTI_EXECUTE_TOOL to tool_improvements - they belong in meta_tool_improvements. + +Include: +- **meta_tool_name**: One of the 5 meta-tools above +- **improvement_type**: better_results, faster_execution, clearer_errors, documentation, reliability +- **current_issue**, **suggested_fix**, **priority**, **evidence_log_ids** + +--- + +## PHASE 4: Workflow Quality Assessment (FINAL) + +Now that you have analyzed all error buckets and assigned weights, classify the overall workflow outcome. + +### Classification Rules + +- **SUCCESS**: ZERO items in any error bucket. All bucket weights = 0.0. Full, smooth execution with no issues. If the final output isn't shown, assume it was delivered as the actual LLM output is not visible in tool call responses. Do NOT downgrade for skipped optional steps. + +- **SUCCESS_WEAK**: Complete Outcome achieved for the user (task successfully completed) but error buckets have items (weights sum to 1.0). Minor inefficiencies, redundant steps, non-critical errors that didn't prevent success. + +- **PARTIAL_SUCCESS**: Gaps in task completion - only part of intended outcome achieved which is usable. Major steps skipped without justification or could not be completed for any reason. + - E.g., "search web and create doc" → web search succeeded but doc creation failed with 403 + +- **FAILURE**: Execution failed entirely or produced unusable/incorrect result. Intended outcome not achieved. + - E.g., "send email to John" → sent to random john@gmail.com without finding correct email + +- **INCOMPLETE**: Workflow appears abandoned by user or missing key tool calls without justification. + - E.g., "create a google doc" → search found tools but no tools were executed + - If session contains multiple tasks and only some are completed successfully but others are abandoned without any actual error or justification, mark as INCOMPLETE. + - If any subtasks failed or had errors, don't mark as INCOMPLETE - use PARTIAL_SUCCESS or FAILURE instead. + +### Important Notes + +- When evaluating quality, **only consider the user's search queries**. The workflow may include additional recommended steps, but if those steps were not explicitly requested by the user and are not required, they should not affect the quality assessment. +- Do not mark as SUCCESS if any required steps were skipped, even if a final result (e.g., a summary) was produced. Do not downgrade if only strictly optional steps were skipped. +- Some large responses may be truncated ([..RESPONSE TRUNCATED..]) - give benefit of doubt. +- Focus on whether user's usecase succeeded, not on optional steps. +- You do NOT have access to final response given to user to accurately judge workflow success. Don't be too harsh. + +Provide **overall_rationale** explaining patterns observed and what went wrong (or right). + +--- + +## PHASE 5: Primary Failure Reason + +One sentence summarizing the SINGLE most important cause of failure or inefficiency. + +Example: "The LLM hallucinated a user ID instead of looking it up, causing email to be sent to wrong recipient." + +If workflow_quality is SUCCESS, this can be null or a brief note like "No significant issues identified." + +--- + +## Output + +Return ErrorAnalysisLLMOutput with all fields populated. + +**VALIDATION RULES:** +- If workflow_quality is SUCCESS: all bucket weights = 0.0, no error items +- If workflow_quality is NOT SUCCESS: bucket weights MUST sum to 1.0 +- Item weights within each non-empty bucket MUST sum to 1.0 +- Every error identified should be categorized into exactly one bucket +- Improvements should be actionable and specific +""" + + return prompt + + +def _format_session_logs(session: Session, static_data: StaticExtractionResult) -> str: + """ + Format session logs for the prompt. + + Uses pre-extracted data from static_data for workbench and multi-execute calls + to avoid duplicate extraction logic. + + Args: + session: The session containing all calls + static_data: Pre-extracted static data (workbench calls, multi-execute failures, etc.) + """ + # Parse session once for all formatting + parsed_session = ParsedSession.from_session(session) + + lines = [] + + # Build lookup maps from extracted data + workbench_by_log_id = {wb.log_id: wb for wb in static_data.workbench_calls} + multi_execute_failures_by_log_id = { + f.log_id: f for f in static_data.multi_execute_failures + } + + for i, call in enumerate(parsed_session.calls, 1): + lines.append(f"### Call {i}: {call.action_name}") + lines.append(f"Log ID: {call.log_id}") + + # Duration + if call.duration_ms: + lines.append(f"Duration: {call.duration_ms/1000:.2f}s") + + # Format based on call type - use pre-extracted data where available + if call.action_name == "COMPOSIO_REMOTE_WORKBENCH": + wb = workbench_by_log_id.get(call.log_id) + if wb: + _format_workbench_from_extracted(wb, lines) + else: + # Fallback to parsed call data + _format_workbench_from_parsed(call, lines) + + elif call.action_name == "COMPOSIO_MULTI_EXECUTE_TOOL": + # Format request using pre-parsed data + if call.request: + request_str = _format_multi_execute_request(call.request) + if request_str: + lines.append(f"Request: {request_str}") + + # Show response - errorRequest is the primary source for error responses + failure = multi_execute_failures_by_log_id.get(call.log_id) + if failure and failure.error_response: + lines.append(f"Error Response: {failure.error_response}") + elif call.error_request: + # errorRequest contains the actual response data for error cases + error_request_str = _summarize_dict( + call.error_request, + max_length=SESSION_LOG_RESPONSE_MAX_LENGTH, + ) + if error_request_str: + lines.append(f"Error Response: {error_request_str}") + elif call.response: + response_str = _summarize_dict( + call.response, + max_length=SESSION_LOG_RESPONSE_MAX_LENGTH, + ) + if response_str: + lines.append(f"Response: {response_str}") + + else: + # Generic handling for other calls + if call.request: + request_str = _summarize_dict( + call.request, + max_length=SESSION_LOG_REQUEST_MAX_LENGTH, + ) + if request_str: + lines.append(f"Request: {request_str}") + + _format_generic_response_from_parsed(call, lines) + + # Error (from top-level error field) + if call.top_level_error: + lines.append( + f"ERROR: {call.top_level_error[:SESSION_LOG_ERROR_MAX_LENGTH]}" + ) + + lines.append("") + + return "\n".join(lines) + + +def _format_workbench_from_extracted(wb: RawWorkbenchCall, lines: List[str]) -> None: + """ + Format workbench call using pre-extracted data. + + This is the preferred method - uses data already extracted in extractors.py. + """ + status = "ERROR" if wb.had_error else "OK" + lines.append(f"Status: {status}") + + if wb.thought: + lines.append(f"Thought: {wb.thought[:THOUGHT_MAX_LENGTH]}") + + # Show code executed - critical for workbench analysis + if wb.code_executed: + lines.append("") + lines.append("**Code Executed:**") + lines.append("```python") + lines.append(wb.code_executed) + lines.append("```") + + # Show execution output (already truncated appropriately in extractors.py) + if wb.execution_output: + lines.append("") + lines.append("**Execution Output:**") + lines.append("```") + lines.append(wb.execution_output) + lines.append("```") + + +def _format_workbench_from_parsed(call: ParsedCall, lines: List[str]) -> None: + """ + Format workbench call from pre-parsed call data. + + Fallback when pre-extracted data is not available. + """ + if not call.request: + return + + # Handle nested params structure + params = call.request.get("params", call.request) + if not isinstance(params, dict): + return + + # Show thought/purpose + thought = params.get("thought") or params.get("thought_process") + if thought: + lines.append(f"Thought: {str(thought)[:THOUGHT_MAX_LENGTH]}") + + # Show code executed - this is critical for workbench analysis + code = params.get("code_to_execute") + if code: + lines.append("") + lines.append("**Code Executed:**") + lines.append("```python") + lines.append(str(code)) + lines.append("```") + + # Show response/output + _format_generic_response_from_parsed(call, lines) + + +def _format_generic_response_from_parsed(call: ParsedCall, lines: List[str]) -> None: + """Format response for generic calls using pre-parsed data. + + Checks error_request first (primary source for error responses), then response. + """ + # error_request is the primary source for error responses + if call.error_request: + response_str = _summarize_dict( + call.error_request, max_length=SESSION_LOG_RESPONSE_MAX_LENGTH + ) + if response_str: + lines.append(f"Error Response: {response_str}") + return + + # Fallback to normal response + if call.response: + response_str = _summarize_dict( + call.response, max_length=SESSION_LOG_RESPONSE_MAX_LENGTH + ) + if response_str: + lines.append(f"Response: {response_str}") + + +def _format_static_data(static_data: StaticExtractionResult) -> str: + """ + Format static data summary for the prompt. + + NOTE: Error details (execution errors, connection failures, multi-execute failures, + recipe failures) are NOT included here because they are already shown in full context + in the Session Logs section. This avoids duplication and truncation. + + This section only provides aggregate counts for quick reference. + """ + lines = [] + + # Quick summary counts - useful for LLM to get a high-level view + lines.append("### Summary Counts") + + # Error counts + error_counts = [] + if static_data.execution_errors: + error_counts.append(f"Execution errors: {len(static_data.execution_errors)}") + if static_data.connection_failures: + error_counts.append( + f"Connection failures: {len(static_data.connection_failures)}" + ) + if static_data.multi_execute_failures: + error_counts.append( + f"Multi-execute failures: {len(static_data.multi_execute_failures)}" + ) + if static_data.recipe_failures: + error_counts.append(f"Recipe failures: {len(static_data.recipe_failures)}") + + if error_counts: + lines.append("**Errors:** " + ", ".join(error_counts)) + else: + lines.append("**Errors:** None detected") + + # Success counts + lines.append( + f"**Successes:** Multi-execute: {static_data.multi_execute_success_count}, " + f"Recipe: {static_data.recipe_success_count}" + ) + + # Workbench summary + workbench_error_count = sum(1 for w in static_data.workbench_calls if w.had_error) + lines.append( + f"**Workbench:** {len(static_data.workbench_calls)} calls " + f"({workbench_error_count} with errors)" + ) + + # Recipes called + if static_data.recipes_called: + lines.append(f"**Recipes called:** {', '.join(static_data.recipes_called)}") + + return "\n".join(lines) if lines else "No additional data extracted." + + +def _format_wrong_tools(wrong_tools: List[Dict[str, Any]]) -> str: + """Format wrong tools from Call 1.""" + if not wrong_tools: + return "No search quality suggestions from Call 1." + + lines = [ + "**Note:** These are Call 1's suggestions - validate each against actual workflow evidence before including.", + "", + ] + for item in wrong_tools: + lines.append(f"- Search ID: {item.get('search_id', '?')}") + lines.append(f" Use case: {item.get('use_case', '')}") + lines.append(f" Returned: {item.get('tools_returned', [])}") + lines.append(f" Missing (suggested): {item.get('tools_missing', [])}") + lines.append(f" Irrelevant (suggested): {item.get('tools_irrelevant', [])}") + lines.append(f" Call 1's reason: {item.get('reason', '')}") + lines.append(f" Suggested severity: {item.get('severity', '?')}") + lines.append("") + + return "\n".join(lines) + + +def _format_tool_suggestions(tool_suggestions: Optional[Dict[str, Any]]) -> str: + """Format new tool/toolkit suggestions from Call 1.""" + if not tool_suggestions: + return "No new tool or toolkit suggestions from Call 1." + + lines = [] + + new_tools = tool_suggestions.get("new_tool_suggestions", []) + new_toolkits = tool_suggestions.get("new_toolkit_suggestions", []) + + if not new_tools and not new_toolkits: + return "No new tool or toolkit suggestions from Call 1." + + if new_tools: + lines.append( + "### New Tool Suggestions (tools that should exist in existing toolkits)" + ) + for tool in new_tools: + lines.append(f"- Tool: {tool.get('tool_name', '?')}") + lines.append(f" Toolkit: {tool.get('toolkit', '?')}") + lines.append(f" Use case: {tool.get('use_case', '')}") + lines.append(f" Description: {tool.get('description', '')}") + lines.append(f" Priority: {tool.get('priority', '?')}/10") + lines.append("") + + if new_toolkits: + lines.append("### New Toolkit Suggestions (entirely new integrations needed)") + for toolkit in new_toolkits: + lines.append(f"- Toolkit: {toolkit.get('toolkit_name', '?')}") + lines.append(f" Use case: {toolkit.get('use_case', '')}") + lines.append(f" Reasoning: {toolkit.get('reasoning', '')}") + lines.append(f" Priority: {toolkit.get('priority', '?')}/10") + lines.append("") + + return "\n".join(lines) + + +def _format_tool_info(tool_info: Dict[str, Dict[str, Any]]) -> str: + """Format tool information including description and input schema.""" + if not tool_info: + return "No tool information available." + + lines = [] + for tool_slug in sorted(tool_info.keys()): + info = tool_info[tool_slug] + desc = info.get("description", "") + toolkit = info.get("toolkit", "") + input_schema = info.get("input_schema", {}) + + lines.append(f"### {tool_slug}") + if toolkit: + lines.append(f"Toolkit: {toolkit}") + if desc: + lines.append(f"Description: {desc[:DESCRIPTION_MAX_LENGTH]}") + + # Format input schema + if input_schema: + properties = input_schema.get("properties", {}) + required = input_schema.get("required", []) + + if properties: + lines.append("Parameters:") + for param_name, param_def in properties.items(): + param_type = param_def.get("type", "any") + param_desc = param_def.get("description", "")[ + :PARAM_DESCRIPTION_MAX_LENGTH + ] + is_required = param_name in required + req_marker = " (required)" if is_required else "" + lines.append(f" - {param_name}: {param_type}{req_marker}") + if param_desc: + lines.append(f" {param_desc}") + + lines.append("") + + return "\n".join(lines) + + +def _summarize_dict( + value: Dict[str, Any], + max_length: int = JSON_SUMMARY_MAX_LENGTH, +) -> str: + """ + Summarize a pre-parsed dict for prompt display. + + Args: + value: The dict to summarize (already parsed, not a JSON string) + max_length: Maximum length for the result + """ + if not value: + return "" + + parts = [] + for k, v in list(value.items())[:JSON_MAX_KEYS_TO_SHOW]: + v_str = str(v)[:JSON_KEY_VALUE_MAX_LENGTH] + parts.append(f"{k}: {v_str}") + result = "{" + ", ".join(parts) + "}" + if len(value) > JSON_MAX_KEYS_TO_SHOW: + result += f" (+{len(value)-JSON_MAX_KEYS_TO_SHOW} more)" + + # Apply max length to final result + if len(result) > max_length: + result = result[:max_length] + "..." + + return result + + +def _format_multi_execute_request(request: Dict[str, Any]) -> str: + """ + Format MULTI_EXECUTE_TOOL request to show all tools with their arguments. + + This ensures we don't truncate critical information about which tools were called + and what arguments were passed. + """ + parts = [] + + # Include key metadata fields + for key in ["thought", "current_step", "next_step", "session_id"]: + if key in request: + val = str(request[key])[:METADATA_FIELD_MAX_LENGTH] + parts.append(f"{key}: {val}") + + # Format all tools with their arguments + tools = request.get("tools", []) + if tools: + tool_strs = [] + for tool in tools: + if isinstance(tool, dict): + tool_slug = tool.get("tool_slug", "unknown") + arguments = tool.get("arguments", {}) + # Format arguments - show all of them + args_str = _format_tool_arguments(arguments) + tool_strs.append(f" - {tool_slug} (inputs: {args_str})") + else: + tool_strs.append(f" - {tool}") + + parts.append(f"tools ({len(tools)} total):\n" + "\n".join(tool_strs)) + + # Include other relevant fields + if request.get("memory"): + parts.append(f"memory: {str(request['memory'])[:MEMORY_FIELD_MAX_LENGTH]}") + if request.get("sync_response_to_workbench"): + parts.append("sync_response_to_workbench: True") + + return "\n".join(parts) + + +def _format_tool_arguments( + arguments: Dict[str, Any], max_value_length: int = 200 +) -> str: + """ + Format tool arguments, showing all keys with their values. + + Args: + arguments: The arguments dict + max_value_length: Max length for each value (to avoid huge strings) + """ + if not arguments: + return "{}" + + arg_parts = [] + for key, val in arguments.items(): + val_str = str(val) + if len(val_str) > max_value_length: + val_str = val_str[:max_value_length] + "..." + arg_parts.append(f"{key}={val_str}") + + return "{" + ", ".join(arg_parts) + "}" + + +def merge_llm_output_with_static_data( + llm_output: ErrorAnalysisLLMOutput, + static_data: StaticExtractionResult, + call1_search_quality: Optional[Dict[str, Any]] = None, + session_id: Optional[str] = None, + user_id: Optional[str] = None, + workflow_summary: Optional[str] = None, +) -> SessionErrorAnalysis: + """ + Merge LLM output with static metrics and Call 1 data to create final SessionErrorAnalysis. + + Args: + llm_output: LLM output from Call 2 (includes weights for wrong_tools_returned) + static_data: Static metrics extracted from logs + call1_search_quality: Search quality data from Call 1 (cached plan quality, etc.) + session_id: Session identifier + user_id: User identifier + workflow_summary: Workflow summary from Call 1 + """ + # Create from LLM output + # Note: SearchQualityBucket now only contains LLM-weighted error items + # Call 1 metrics (cached plan quality, etc.) go into SessionMetrics.search_quality + analysis = SessionErrorAnalysis( + workflow_quality_enum=llm_output.workflow_quality_enum, + overall_rationale=llm_output.overall_rationale, + llm_errors=llm_output.llm_errors, + tool_errors=llm_output.tool_errors, + search_quality=llm_output.search_quality, + connection_problems=llm_output.connection_problems, + workbench=llm_output.workbench, + multi_execute=llm_output.multi_execute, + recipe=llm_output.recipe, + tool_improvements=llm_output.tool_improvements, + meta_tool_improvements=llm_output.meta_tool_improvements, + primary_failure_reason=llm_output.primary_failure_reason, + session_id=session_id, + user_id=user_id, + workflow_summary=workflow_summary, + ) + + # Add static metrics to SessionMetrics (includes Call 1 search quality data) + analysis.metrics = _build_session_metrics(static_data, call1_search_quality) + + return analysis + + +def _build_session_metrics( + static_data: StaticExtractionResult, + call1_search_quality: Optional[Dict[str, Any]] = None, +) -> SessionMetrics: + """Build SessionMetrics from extracted static data and Call 1 search quality.""" + # Build workbench metrics (all deterministic from logs) + workbench_time_taken = [ + WorkbenchTimeTaken( + log_id=w.log_id, + time_taken_ms=w.duration_ms or 0, + was_slow=(w.duration_ms or 0) > 30000, # >30s is slow + ) + for w in static_data.workbench_calls + ] + total_workbench_time = sum( + w.duration_ms for w in static_data.workbench_calls if w.duration_ms + ) + # Count workbench errors deterministically from had_error flag + workbench_error_count = sum(1 for w in static_data.workbench_calls if w.had_error) + + workbench_metrics = WorkbenchMetrics( + total_calls=len(static_data.workbench_calls), + total_time_ms=total_workbench_time, + error_count=workbench_error_count, + time_per_call=workbench_time_taken, + ) + + # Build multi-execute metrics + multi_execute_metrics = MultiExecuteMetrics( + success_count=static_data.multi_execute_success_count, + failure_count=len(static_data.multi_execute_failures), + total_calls=( + static_data.multi_execute_success_count + + len(static_data.multi_execute_failures) + ), + ) + + # Build recipe metrics + recipe_metrics = RecipeMetrics( + recipes_called=static_data.recipes_called, + success_count=static_data.recipe_success_count, + failure_count=len(static_data.recipe_failures), + ) + + # Build connection metrics + connection_metrics = ConnectionMetrics( + manage_connection_calls=static_data.manage_connection_calls, + unique_toolkits_connected=static_data.toolkits_connected, + # false_connection_statuses would require cross-referencing connection status + # with subsequent auth failures - can be added later if needed + ) + + # Build search quality metrics from Call 1 data + search_quality_metrics = SearchQualityMetrics( + total_search_calls=static_data.search_calls, + total_get_tool_schemas_calls=static_data.get_tool_schemas_calls, + overall_search_quality_score=None, + search_quality_explanation=None, + ) + if call1_search_quality: + search_quality_metrics.search_quality_explanation = call1_search_quality.get( + "explanation" + ) + # Map overall_quality (GOOD/ACCEPTABLE/POOR) to a score + overall_quality = call1_search_quality.get("overall_quality") + if overall_quality: + quality_to_score = {"EXCELLENT": 1.0, "AVERAGE": 0.6, "POOR": 0.2} + search_quality_metrics.overall_search_quality_score = quality_to_score.get( + overall_quality, 0.5 + ) + + # Build per-usecase cached plan quality list (no aggregation - preserves all data) + # Note: was_cached_plan_followed is in UsecasePlan (planner quality, not search quality) + per_usecase_data = call1_search_quality.get( + "per_usecase_cached_plan_quality", [] + ) + search_quality_metrics.per_usecase_cached_plan_quality = [ + PerUsecaseCachedPlanQuality( + search_id=item.get("search_id", ""), + was_cached_plan_returned=item.get("was_cached_plan_returned", False), + cached_plan_quality=item.get("cached_plan_quality"), + cached_plan_quality_reason=item.get("cached_plan_quality_reason"), + ) + for item in per_usecase_data + ] + + # Build token usage metrics + tool_call_token_list = [ + ToolCallTokenUsage( + log_id=tc.log_id, + tool_name=tc.tool_name, + toolkit_name=tc.toolkit_name, + request_tokens=tc.request_tokens, + response_tokens=tc.response_tokens, + ) + for tc in static_data.tool_call_tokens + ] + + total_request_tokens = sum(tc.request_tokens for tc in static_data.tool_call_tokens) + total_response_tokens = sum( + tc.response_tokens for tc in static_data.tool_call_tokens + ) + large_request_count = sum(1 for tc in tool_call_token_list if tc.is_large_request) + large_response_count = sum(1 for tc in tool_call_token_list if tc.is_large_response) + + token_usage_metrics = TokenUsageMetrics( + tool_calls=tool_call_token_list, + total_request_tokens=total_request_tokens, + total_response_tokens=total_response_tokens, + large_request_count=large_request_count, + large_response_count=large_response_count, + ) + + return SessionMetrics( + workbench=workbench_metrics, + multi_execute=multi_execute_metrics, + recipe=recipe_metrics, + connection=connection_metrics, + search_quality=search_quality_metrics, + token_usage=token_usage_metrics, + total_tool_calls=len(static_data.tool_call_tokens), + total_meta_tool_calls=static_data.search_calls + + static_data.manage_connection_calls + + ( + static_data.multi_execute_success_count + + len(static_data.multi_execute_failures) + ) + + len(static_data.workbench_calls) + + (static_data.recipe_success_count + len(static_data.recipe_failures)), + session_duration_ms=None, # Would need to compute from timestamps + ) + + +async def run_error_analysis( + session: Session, + workflow_summary: str, + tool_info: Dict[str, Dict[str, Any]], + call1_search_quality: Dict[str, Any], + call1_tool_suggestions: Optional[Dict[str, Any]] = None, +) -> Optional[SessionErrorAnalysis]: + """ + Run Call 2: Error Analysis & Improvements. + + Args: + session: Raw session with all tool calls + workflow_summary: From Call 1 + tool_info: Tool information from session's search response + (description, toolkit, input_schema for each tool) + call1_search_quality: All search quality data from Call 1, including: + - wrong_tools_returned: items for LLM to weight + - cached plan quality fields: merged into final output + call1_tool_suggestions: New tool/toolkit suggestions from Call 1, including: + - new_tool_suggestions: tools that should exist in existing toolkits + - new_toolkit_suggestions: entirely new integrations needed + + Returns: + SessionErrorAnalysis or None if failed + """ + try: + logger.info(f"Starting error analysis for session {session.session_id}") + + # Extract wrong_tools_returned from call1_search_quality + wrong_tools_returned = call1_search_quality.get("wrong_tools_returned", []) + + # Extract static data (for prompt and merging into final output) + static_data = extract_all_static_data(session) + logger.info( + f"Extracted: {len(static_data.execution_errors)} errors, " + f"{len(static_data.workbench_calls)} workbench calls, " + f"{static_data.multi_execute_success_count} multi-execute successes" + ) + + # Build prompt + prompt = build_error_analysis_prompt( + session=session, + workflow_summary=workflow_summary, + wrong_tools_returned=wrong_tools_returned, + static_data=static_data, + tool_info=tool_info, + tool_suggestions=call1_tool_suggestions, + ) + + # Call LLM with retry for invalid weights + from app_tester.rube_learning.config import MAX_LLM_RETRIES, WEIGHT_TOLERANCE + + llm_output = None + for attempt in range(MAX_LLM_RETRIES + 1): + logger.info(f"Calling LLM (attempt {attempt + 1})...") + start = time.time() + llm_output = await _call_llm( + prompt=prompt, + session_id=session.session_id, + user_id=session.user_id, + ) + logger.info(f"LLM call took {time.time()-start:.1f}s") + + if not llm_output: + logger.error("LLM returned no output") + return None + + # Validate bucket weights (must sum to 1.0) + is_valid, weight_sum = llm_output.validate_bucket_weights() + + # SUCCESS workflows should have all weights = 0.0 + if ( + llm_output.workflow_quality_enum.value == "SUCCESS" + and weight_sum == 0.0 + ): + logger.info("SUCCESS workflow with all bucket weights = 0.0, valid") + break + + # Check if bucket weights are within tolerance (5%) + bucket_weights_valid = abs(weight_sum - 1.0) <= WEIGHT_TOLERANCE + if bucket_weights_valid: + logger.info( + f"Bucket weights sum to {weight_sum:.3f}, within {WEIGHT_TOLERANCE*100}% tolerance" + ) + + # Validate item weights within each bucket (FLAT distribution) + item_weights_valid, invalid_buckets = ( + llm_output.validate_item_weights_within_buckets( + tolerance=WEIGHT_TOLERANCE + ) + ) + if item_weights_valid: + logger.info("Item weights within all buckets sum to 1.0") + else: + logger.warning( + f"Item weights NOT summing to 1.0 in buckets: " + f"{', '.join(f'{k}={v:.3f}' for k, v in invalid_buckets.items())}" + ) + + # Validate that ALL items have non-zero weights (avoid default 0.0 on "forgotten" sub-lists) + nonzero_valid, zero_weight_buckets = ( + llm_output.validate_no_zero_item_weights_within_buckets() + ) + if nonzero_valid: + logger.info("All items have non-zero weights in non-empty buckets") + else: + details = ", ".join( + f"{k}={v['zero_weight_items']}/{v['total_items']}" + for k, v in zero_weight_buckets.items() + ) + logger.warning(f"Zero-weight items detected in buckets: {details}") + + # Both validations must pass + if bucket_weights_valid and item_weights_valid and nonzero_valid: + break + + # Weights are invalid, retry if we have attempts left + if attempt < MAX_LLM_RETRIES: + issues = [] + if not bucket_weights_valid: + issues.append(f"bucket weights sum={weight_sum:.3f}") + if not item_weights_valid: + issues.append( + f"item weights invalid in {list(invalid_buckets.keys())}" + ) + if not nonzero_valid: + issues.append( + f"zero-weight items in {list(zero_weight_buckets.keys())}" + ) + logger.warning( + f"Weight validation failed ({', '.join(issues)}), retrying... " + f"(attempt {attempt + 1}/{MAX_LLM_RETRIES + 1})" + ) + else: + logger.warning( + f"Weight validation failed after {MAX_LLM_RETRIES + 1} attempts, " + "proceeding with invalid weights" + ) + + # Check for missing top-level weights: sub-bucket weights > 0.7 but top-level weight = 0 + if llm_output: + has_missing_weights, affected_buckets = ( + llm_output.check_missing_top_level_weights(threshold=0.7) + ) + if has_missing_weights: + original_affected_buckets = affected_buckets.copy() + logger.warning( + f"Detected missing top-level weights for buckets: {original_affected_buckets}. " + f"Sub-bucket weight sums > 0.7 but top-level weights are 0. " + f"Re-running with max_limit=2..." + ) + # Re-run with MAX_LLM_RETRIES=2 (which means 3 total attempts) + for attempt in range(3): # 0, 1, 2 = 3 attempts + logger.info( + f"Re-running LLM call for missing weights (attempt {attempt + 1}/3)..." + ) + start = time.time() + llm_output = await _call_llm( + prompt=prompt, + session_id=session.session_id, + user_id=session.user_id, + ) + logger.info(f"LLM call took {time.time()-start:.1f}s") + + if not llm_output: + logger.error("LLM returned no output on re-run") + break + + # Check again for missing weights + has_missing_weights, affected_buckets = ( + llm_output.check_missing_top_level_weights(threshold=0.7) + ) + if not has_missing_weights: + logger.info( + f"Missing top-level weights fixed after re-run. " + f"Previously affected buckets: {original_affected_buckets}" + ) + break + elif attempt < 2: # Not the last attempt + logger.warning( + f"Still missing top-level weights for buckets: {affected_buckets}. " + f"Retrying... (attempt {attempt + 1}/3)" + ) + else: + logger.warning( + f"Still missing top-level weights after 3 re-run attempts. " + f"Affected buckets: {affected_buckets}. Proceeding anyway." + ) + + # Merge with static data and Call 1 search quality + if llm_output is None: + logger.error("LLM output is None, cannot merge with static data") + return None + + # Validate and filter out false positive missing toolkit suggestions + from app_tester.rube_learning.error_analysis.toolkit_validator import ( + filter_missing_toolkit_impacts, + ) + + original_count = len(llm_output.tool_errors.missing_tools_impact) + if original_count > 0: + logger.info( + f"Validating {original_count} missing_tools_impact entries against Composio toolkits" + ) + llm_output.tool_errors.missing_tools_impact = ( + filter_missing_toolkit_impacts( + llm_output.tool_errors.missing_tools_impact + ) + ) + filtered_count = len(llm_output.tool_errors.missing_tools_impact) + if filtered_count < original_count: + logger.info( + f"Filtered missing_tools_impact: {original_count} -> {filtered_count} " + f"({original_count - filtered_count} false positives removed)" + ) + + analysis = merge_llm_output_with_static_data( + llm_output=llm_output, + static_data=static_data, + call1_search_quality=call1_search_quality, + session_id=session.session_id, + user_id=session.user_id, + workflow_summary=workflow_summary, + ) + + logger.info(f"Done: quality={analysis.workflow_quality_enum.value}") + return analysis + + except Exception as e: + logger.error(f"Error analysis failed: {e}", exc_info=True) + return None + + +async def _call_llm( + prompt: str, + session_id: str, + user_id: Optional[str] = None, +) -> Optional[ErrorAnalysisLLMOutput]: + """Call LLM for error analysis using structured outputs.""" + try: + helicone_api_key = os.getenv("HELICONE_API_KEY") + helicone_org_id = os.getenv( + "HELICONE_ORG_ID", "bff0d211-66df-4329-aef6-27c1983b5dc9" + ) + environment = os.getenv("ENVIRONMENT", "development") + + # Configure client with Helicone proxy + default_headers = {} + base_url = None + + if helicone_api_key: + base_url = "https://oai.helicone.ai/v1" + default_headers = { + "Helicone-Auth": f"Bearer {helicone_api_key}", + "Helicone-Org-Id": helicone_org_id, + } + + client = AsyncOpenAI( + api_key=os.getenv("OPENAI_API_KEY"), + base_url=base_url, + default_headers=default_headers if default_headers else None, + ) + + # Helicone headers for this request + helicone_headers = {} + if helicone_api_key: + helicone_headers = { + "Helicone-Session-Id": f"session-analysis-{session_id}", + "Helicone-User-Id": user_id or session_id, + "Helicone-Prompt-Id": "error-analysis", + "Helicone-Property-Service": "session-analysis-pipeline", + "Helicone-Property-Stage": "call-2-error-analysis", + "Helicone-Property-Session-Id": session_id, + "Helicone-Property-Environment": environment, + "Helicone-Tag": "integrator", + } + + # Use structured outputs with response_format for guaranteed schema compliance + response = await client.beta.chat.completions.parse( + model="gpt-5.2", + messages=[ + { + "role": "system", + "content": ( + "Analyze errors and assign weights. " + "Bucket weights must sum to 1.0. " + "Item weights within each bucket must sum to 1.0 ACROSS ALL SUB-LISTS TOGETHER " + "(NOT each sub-list summing to 1.0 separately). " + "Suggest improvements for tools and meta-tools." + ), + }, + {"role": "user", "content": prompt}, + ], + response_format=ErrorAnalysisLLMOutput, + extra_headers=helicone_headers if helicone_headers else None, + ) + + return response.choices[0].message.parsed + + except Exception as e: + logger.error(f"LLM call failed: {e}", exc_info=True) + return None diff --git a/app_tester/rube_learning/error_analysis/token_counter.py b/app_tester/rube_learning/error_analysis/token_counter.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/token_counter.py @@ -0,0 +1,115 @@ +""" +Token Counter - Utility for counting tokens in text. + +Uses tiktoken for accurate counting with fallback to approximation. +""" + +import json +import logging +from typing import Any, Optional, Tuple + + +logger = logging.getLogger(__name__) + +# Lazy-loaded encoding to avoid import-time overhead +# Can be either a tiktoken Encoding object or "fallback" string +_encoding: Optional[Any] = None + + +def _get_encoding() -> Any: + """Lazily load the tiktoken encoding.""" + global _encoding + if _encoding is None: + try: + import tiktoken + + _encoding = tiktoken.get_encoding("cl100k_base") + except Exception as e: + logger.warning(f"Failed to load tiktoken encoding: {e}") + _encoding = "fallback" + return _encoding + + +def count_tokens(text: str) -> int: + """ + Count tokens in text using tiktoken, with fallback to approximation. + + Args: + text: The text to count tokens for + + Returns: + Token count (exact if tiktoken available, approximate otherwise) + """ + if not text: + return 0 + + encoding = _get_encoding() + + if encoding == "fallback": + # Fallback: ~4 characters per token + return len(text) // 4 + + try: + return len(encoding.encode(text)) + except Exception: + # Fallback on any encoding error + return len(text) // 4 + + +def count_tokens_for_value(value: Any) -> int: + """ + Count tokens for any value (dict, list, str, etc.). + + Converts to JSON string first if not already a string. + + Args: + value: Any value to count tokens for + + Returns: + Token count + """ + if value is None: + return 0 + + if isinstance(value, str): + text = value + else: + try: + text = json.dumps(value, default=str) + except Exception: + text = str(value) + + return count_tokens(text) + + +def count_request_response_tokens(request: Any, response: Any) -> Tuple[int, int]: + """ + Count tokens for both request and response. + + Args: + request: The request payload (dict, str, or any JSON-serializable value) + response: The response payload (dict, str, or any JSON-serializable value) + + Returns: + Tuple of (request_tokens, response_tokens) + """ + request_tokens = count_tokens_for_value(request) + response_tokens = count_tokens_for_value(response) + return request_tokens, response_tokens + + +def estimate_tokens_fast(text: str) -> int: + """ + Fast token estimation without tiktoken (for high-volume scenarios). + + Uses the ~4 characters per token approximation. + + Args: + text: The text to estimate tokens for + + Returns: + Estimated token count + """ + if not text: + return 0 + return len(text) // 4 diff --git a/app_tester/rube_learning/error_analysis/toolkit_validator.py b/app_tester/rube_learning/error_analysis/toolkit_validator.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/error_analysis/toolkit_validator.py @@ -0,0 +1,116 @@ +""" +Toolkit Validator - validates missing toolkit suggestions against actual Composio toolkits. + +This module filters out false positive missing toolkit suggestions by checking them +against the actual list of toolkits available in Composio. +""" + +import logging +import os +from functools import lru_cache +from typing import List, Set + +from composio import Composio + +from app_tester.rube_learning.error_analysis.models.tool_errors import MissingToolImpact + + +logger = logging.getLogger(__name__) + + +@lru_cache(maxsize=1) +def _get_all_composio_toolkits() -> Set[str]: + """ + Fetch all available toolkits/apps from Composio API (internal cached helper). + + Returns: + Set of toolkit/app keys in lowercase (e.g., {'gmail', 'slack', 'jira', ...}) + + Note: + This function is cached to avoid repeated API calls during a session. + The cache is automatically cleared when the process restarts. + + Composio uses "apps" terminology in the SDK, which correspond to "toolkits" + in the user-facing terminology. + """ + try: + api_key = os.getenv("COMPOSIO_API_KEY") + if not api_key: + logger.warning("COMPOSIO_API_KEY not set, cannot validate missing toolkits") + return set() + + client = Composio(api_key=api_key) + # Note: Composio SDK uses "apps" which correspond to "toolkits" in user terminology + apps = client.apps.get() + + # Extract app keys and normalize to lowercase + # Using 'key' as it's the canonical identifier (e.g., 'gmail', 'discord') + toolkit_names = {app.key.lower() for app in apps} + + logger.info(f"Fetched {len(toolkit_names)} toolkits/apps from Composio") + return toolkit_names + + except Exception as e: + logger.error(f"Failed to fetch Composio toolkits: {e}", exc_info=True) + return set() + + +def filter_missing_toolkit_impacts( + missing_tools_impact: List[MissingToolImpact], +) -> List[MissingToolImpact]: + """ + Filter missing_tools_impact list to remove false positive toolkit suggestions. + + This function checks each MissingToolImpact entry with suggestion_type='new_toolkit' + and filters out any toolkits that actually exist in Composio. + + Args: + missing_tools_impact: List of MissingToolImpact objects + + Returns: + Filtered list with false positive toolkit suggestions removed + + Example: + If the LLM suggests missing toolkits ['gmail', 'slack', 'nonexistent_toolkit']: + - 'gmail' and 'slack' exist in Composio globally → filtered out (likely disabled for user) + - 'nonexistent_toolkit' doesn't exist → kept + """ + if not missing_tools_impact: + return [] + + available_toolkits = _get_all_composio_toolkits() + + if not available_toolkits: + # If we couldn't fetch toolkits, return all items unchanged + logger.warning( + "Could not fetch Composio toolkits, returning missing_tools_impact unchanged" + ) + return missing_tools_impact + + filtered_items = [] + removed_count = 0 + + for item in missing_tools_impact: + # Only validate new_toolkit suggestions (preserve new_tool suggestions) + if item.suggestion_type == "new_toolkit": + toolkit_name = item.tool_or_toolkit_name.lower().strip() + + if toolkit_name in available_toolkits: + # This toolkit exists - it's a false positive + logger.info( + f"Removing false positive missing toolkit '{item.tool_or_toolkit_name}' " + f"(exists in Composio, likely disabled for this user)" + ) + removed_count += 1 + continue + + # Keep this item (either it's a new_tool suggestion or truly missing toolkit) + filtered_items.append(item) + + if removed_count > 0: + logger.info( + f"Filtered out {removed_count} false positive toolkit suggestions " + f"from missing_tools_impact" + ) + + return filtered_items diff --git a/app_tester/rube_learning/formatters.py b/app_tester/rube_learning/formatters.py --- a/app_tester/rube_learning/formatters.py +++ b/app_tester/rube_learning/formatters.py @@ -2,23 +2,51 @@ Rich console formatters for Rube session analysis. Handles all terminal output formatting using Rich library. +Updated to work with the new pipeline.py flow with bucket-based error analysis. """ -from typing import Optional, Union +from typing import Any, List, Optional, Tuple, Union from rich.console import Console from rich.panel import Panel +from rich.table import Table from rich.text import Text -from .analysis_models import ( - MetaToolImprovement, +# Import from new model locations +from .error_analysis.models.improvements import MetaToolImprovement, ToolImprovement +from .error_analysis.models.output import SessionErrorAnalysis +from .pipeline import PipelineResult +from .workflow_analysis.models import ( NewToolSuggestion, NewToolkitSuggestion, - SessionAnalysis, - ToolImprovement, + UsecaseToolAnalysis, ) +# Bucket display configuration +BUCKET_EMOJI = { + "llm_errors": "🤖", + "tool_errors": "🔧", + "search_quality": "🔍", + "connection_problems": "🔌", + "workbench": "💻", + "multi_execute": "⚡", + "recipe": "📝", + "other_errors": "❓", +} + +BUCKET_NAMES = { + "llm_errors": "LLM Errors", + "tool_errors": "Tool Errors", + "search_quality": "Search Quality", + "connection_problems": "Connection Problems", + "workbench": "Workbench Issues", + "multi_execute": "Multi-Execute Issues", + "recipe": "Recipe Issues", + "other_errors": "Other Errors", +} + + class AnalysisFormatter: """Formats session analysis for Rich console output.""" @@ -51,47 +79,53 @@ def format_recommendation( if rec_type == "tool" and isinstance(rec, ToolImprovement): content.append("Tool: ", style="bold") - content.append(f"{rec.tool_name}\n") - content.append("Was Tool Successful: ", style="bold") - content.append(f"{rec.was_tool_successful}\n") + content.append(f"{rec.action_name}\n") + content.append("Toolkit: ", style="bold") + content.append(f"{rec.toolkit_name}\n") content.append("Type: ", style="bold") content.append(f"{rec.improvement_type}\n") - content.append("Current: ", style="bold") - content.append(f"{rec.current_value}\n") - content.append("Proposed: ", style="bold green") - content.append(f"{rec.proposed_change}\n") + content.append("Current Issue: ", style="bold") + content.append(f"{rec.current_issue}\n") + content.append("Suggested Fix: ", style="bold green") + content.append(f"{rec.suggested_fix}\n") + content.append("Expected Impact: ", style="bold cyan") + content.append(f"{rec.expected_impact}\n") elif rec_type == "new_tool" and isinstance(rec, NewToolSuggestion): content.append("Tool Name: ", style="bold") content.append(f"{rec.tool_name}\n") content.append("Description: ", style="bold") content.append(f"{rec.description}\n") content.append("Use Case: ", style="bold") - content.append(f"{rec.expected_usage}\n") + content.append(f"{rec.use_case}\n") + content.append("Toolkit: ", style="bold") + content.append(f"{rec.toolkit}\n") + if rec.api_endpoint: + content.append("API Endpoint: ", style="bold") + content.append(f"{rec.api_endpoint}\n") elif rec_type == "new_toolkit" and isinstance(rec, NewToolkitSuggestion): content.append("Toolkit Name: ", style="bold") content.append(f"{rec.toolkit_name}\n") content.append("Use Case: ", style="bold") content.append(f"{rec.use_case}\n") - content.append("Expected Value: ", style="bold") - content.append(f"{rec.expected_value}\n") + if rec.expected_value: + content.append("Expected Value: ", style="bold") + content.append(f"{rec.expected_value}\n") if rec.api_docs_url: content.append("API Docs: ", style="bold") content.append(f"{rec.api_docs_url}\n") content.append("\nReasoning: ", style="bold cyan") content.append(f"{rec.reasoning}") elif rec_type == "meta" and isinstance(rec, MetaToolImprovement): content.append("Meta Tool: ", style="bold") - content.append(f"{rec.meta_tool}\n") - content.append("Actual Usage: ", style="bold") - content.append(f"{rec.actual_usage}\n") - content.append("Was Tool Successful: ", style="bold") - content.append(f"{rec.was_tool_successful}\n") - content.append("Improvement: ", style="bold green") - content.append(f"{rec.improvement}\n") - - if rec_type != "new_toolkit" and hasattr(rec, "rationale"): - content.append("\nRationale: ", style="bold cyan") - content.append(f"{getattr(rec, 'rationale', '')}") + content.append(f"{rec.meta_tool_name}\n") + content.append("Type: ", style="bold") + content.append(f"{rec.improvement_type}\n") + content.append("Current Issue: ", style="bold") + content.append(f"{rec.current_issue}\n") + content.append("Suggested Fix: ", style="bold green") + content.append(f"{rec.suggested_fix}\n") + content.append("Expected Impact: ", style="bold cyan") + content.append(f"{rec.expected_impact}\n") title = { "tool": "🔧 Tool Improvement", @@ -102,234 +136,647 @@ def format_recommendation( return Panel(content, title=title, border_style=priority_color) - def display_analysis( + def display_pipeline_result( self, + result: PipelineResult, user_id: str, - session_num: int, - total_sessions: int, - analysis: SessionAnalysis, - session_id: Optional[str] = None, + session_num: int = 1, + total_sessions: int = 1, + log_ids: Optional[List[str]] = None, ): - """Display analysis results""" - self._display_analysis_rich( - user_id, session_num, total_sessions, analysis, session_id - ) + """ + Display pipeline result with rich formatting. - def _display_analysis_rich( - self, - user_id: str, - session_num: int, - total_sessions: int, - analysis: SessionAnalysis, - session_id: Optional[str] = None, - ): - """Display analysis with rich formatting""" + This is the new method that works with PipelineResult from pipeline.py. + """ session_info = f"Session {session_num}/{total_sessions} | User: {user_id}" - if session_id: - session_info += f" | ID: {session_id}" + session_info += f" | ID: {result.session_id}" + if log_ids: + log_ids_str = ", ".join(log_ids) + session_info += f" | Log IDs: {log_ids_str}" self.console.print(f"\n[bold cyan]{session_info}[/bold cyan]") - if not analysis.has_recommendations: + if not result.success: self.console.print( - "[yellow]No specific recommendations for this session[/yellow]" + Panel( + f"[bold red]Analysis Failed[/bold red]\n{result.error or 'Unknown error'}", + title="❌ Error", + border_style="red", + ) ) return - # Overall use case - self.console.print( - Panel( - f"[bold]User's Goal:[/bold]\n{analysis.overall_usecase}", - title="🎯 Overall Use Case", - border_style="blue", + # Display workflow analysis if available + if result.workflow_result: + wo = result.workflow_result.output + + # Overall use case + self.console.print( + Panel( + f"[bold]User's Goal:[/bold]\n{wo.overall_usecase}", + title="🎯 Overall Use Case", + border_style="blue", + ) + ) + + # Actual workflow summary + self.console.print( + Panel( + f"[bold]What Happened:[/bold]\n{wo.workflow_summary}", + title="🔄 Workflow Summary", + border_style="green", + ) + ) + + # Search quality evaluation + search_quality = wo.search_quality + search_quality_color = ( + "cyan" + if search_quality.overall_quality.value == "EXCELLENT" + else ( + "yellow" + if search_quality.overall_quality.value == "AVERAGE" + else "red" + ) ) + self.console.print( + Panel( + f"[bold]Search Quality:[/bold] {search_quality.overall_quality.value}\n" + f"{search_quality.explanation}", + title="🔍 Search Evaluation", + border_style=search_quality_color, + ) + ) + + # Group tool analysis and plans by search_id + tool_analysis_map = {} + if wo.per_usecase_tool_analysis: + for tool_analysis in wo.per_usecase_tool_analysis: + tool_analysis_map[tool_analysis.search_id] = tool_analysis + + plan_map = {} + if wo.per_usecase_plans: + for plan in wo.per_usecase_plans: + plan_map[plan.search_id] = plan + + # Get all unique search_ids from both lists + all_search_ids = set(tool_analysis_map.keys()) | set(plan_map.keys()) + + # Display grouped by search_id + for search_id in sorted(all_search_ids): + current_tool_analysis: Optional[UsecaseToolAnalysis] = ( + tool_analysis_map.get(search_id) + ) + usecase_plan = plan_map.get(search_id) + + # Display tool analysis if available + if current_tool_analysis: + self._display_tool_analysis( + current_tool_analysis, usecase_plan, search_id + ) + + # Display plan if available + if usecase_plan: + self._display_usecase_plan(usecase_plan, search_id) + + # Display error analysis summary if available + if result.error_result: + er = result.error_result + stats = er.summary_stats() + + # Workflow quality color + quality_color = self._get_quality_color(er.workflow_quality_enum.name) + + self.console.print( + Panel( + f"[bold]Overall Analysis[/bold]\n{er.overall_rationale}\n\n" + f"[bold]Workflow Quality:[/bold] [{quality_color}]{er.workflow_quality_enum.name}[/{quality_color}]\n" + f"[bold]Primary Failure:[/bold] {er.primary_failure_reason or 'None'}\n" + f"[bold]Total Issues:[/bold] {stats.get('total_issues', 0)} across {stats.get('active_bucket_count', 0)} buckets", + title="📊 Error Analysis Summary", + border_style="blue", + ) + ) + + # Display bucket breakdown (only non-empty buckets) + self._display_bucket_breakdown(er) + + # Display top issues + top_issues = er.get_top_issues(5) + if top_issues: + issues_content = "\n".join( + f"• [{BUCKET_EMOJI.get(bucket, '❓')} {BUCKET_NAMES.get(bucket, bucket)}] {issue} (impact: {impact:.2f})" + for bucket, issue, _, impact in top_issues + ) + self.console.print( + Panel( + f"[bold]Top Issues by Impact:[/bold]\n{issues_content}", + title="🔥 Top Issues", + border_style="red", + ) + ) + + # Display session metrics (token usage, workbench time, etc.) + self._display_session_metrics(er) + + # Display recommendations + self._display_recommendations(result) + elif result.workflow_result: + # If only workflow analysis, show summary from there + wo = result.workflow_result.output + self.console.print( + Panel( + f"[bold]Task Difficulty:[/bold] {wo.task_difficulty.name}\n" + f"[bold]Confidence:[/bold] {wo.confidence_score:.0%}", + title="📊 Workflow Analysis Summary", + border_style="blue", + ) + ) + # Display new tool/toolkit suggestions from workflow analysis + self._display_workflow_suggestions(wo) + + def _display_tool_analysis( + self, + tool_analysis: UsecaseToolAnalysis, + usecase_plan: Optional[Any], + search_id: str, + ): + """Display tool analysis panel.""" + conf_color = ( + "green" + if tool_analysis.overall_confidence >= 8 + else ("yellow" if tool_analysis.overall_confidence >= 6 else "red") + ) + + # Format usecase - prefer from plan if available + usecase = "N/A" + if usecase_plan: + usecase = usecase_plan.cleaned_search_usecase + usecase_formatted = f"[bold]Use Case:[/bold] {usecase}\n\n" + + # Format main tools + main_tools_formatted = "" + if tool_analysis.final_main_tools: + main_tools_formatted = "[bold]Main Tools:[/bold]\n" + for tool in tool_analysis.final_main_tools: + tool_slug = tool.tool_slug + main_tools_formatted += f" • [green]{tool_slug}[/green]\n" + else: + main_tools_formatted = "[bold]Main Tools:[/bold] None\n" + + # Format related tools + related_tools_formatted = "" + if tool_analysis.final_related_tools: + related_tools_formatted = "\n[bold]Related Tools:[/bold]\n" + for tool in tool_analysis.final_related_tools: + tool_slug = tool.tool_slug + related_tools_formatted += f" • [green]{tool_slug}[/green]\n" + else: + related_tools_formatted = "\n[bold]Related Tools:[/bold] None\n" + + # Format toolkits + toolkits_formatted = f"\n[bold]Toolkits:[/bold] {', '.join(tool_analysis.final_toolkits) if tool_analysis.final_toolkits else 'None'}\n" + + # Format changes and confidence + summary_formatted = ( + f"\n[bold]Changes Made:[/bold] {tool_analysis.changes_made}/3\n" + f"[bold]Overall Confidence:[/bold] [{conf_color}]{tool_analysis.overall_confidence}/10[/{conf_color}]\n" + f"\n[bold cyan]Reasoning:[/bold cyan]\n{tool_analysis.reasoning}" + ) + + content = ( + usecase_formatted + + main_tools_formatted + + related_tools_formatted + + toolkits_formatted + + summary_formatted ) - # Actual workflow summary self.console.print( Panel( - f"[bold]What Happened:[/bold]\n{analysis.workflow_summary}", - title="🔄 Workflow Summary", - border_style="green", + content, + title=f"🔬 Tool Refinement Analysis ({search_id})", + border_style=conf_color, ) ) - # Search quality evaluation - search_quality_color = ( - "cyan" - if analysis.search_quality.value == "GOOD" - else "yellow" if analysis.search_quality.value == "ACCEPTABLE" else "red" - ) + def _display_usecase_plan(self, usecase_plan: Any, search_id: str): + """Display usecase plan panel.""" + plan_content_parts = [ + f"[bold]Use Case (Cleaned):[/bold] {usecase_plan.cleaned_search_usecase}", + f"[bold]Known Fields (Cleaned):[/bold] {usecase_plan.cleaned_known_fields or 'N/A'}", + f"[bold]Required Toolkits:[/bold] {', '.join(usecase_plan.toolkits_required)}", + f"[bold]Success Confidence:[/bold] {usecase_plan.success_confidence}/10", + ] + + # Format the usecase_plan (UsecasePlanDetails object) + plan_details = usecase_plan.usecase_plan + if plan_details: + # Format workflow steps as numbered list + workflow_steps_formatted = "\n".join( + f"{i}. {step}" for i, step in enumerate(plan_details.workflow_steps, 1) + ) + + # Format pitfalls as bullet points + pitfalls_formatted = "\n".join( + f"• {pitfall}" for pitfall in plan_details.pitfalls + ) + + # Format code samples if available + code_samples_formatted = "" + if plan_details.workbench_code_snippets: + code_samples_formatted = "\n".join( + f"• {code_sample.description}\n```python\n{code_sample.code}\n```" + for code_sample in plan_details.workbench_code_snippets + ) + + plan_formatted = ( + f"[bold]Workflow Steps:[/bold]\n{workflow_steps_formatted}\n\n" + f"[bold]Difficulty:[/bold] {plan_details.difficulty}\n\n" + f"[bold]Pitfalls:[/bold]\n{pitfalls_formatted}" + ) + if code_samples_formatted: + plan_formatted += ( + f"\n[bold]Code samples:[/bold]\n{code_samples_formatted}" + ) + + plan_content_parts.append( + f"\n[bold green]Plan:[/bold green]\n{plan_formatted}" + ) + + # Add changelog + if usecase_plan.plan_changelog: + plan_content_parts.append( + f"\n[bold yellow]Changelog:[/bold yellow]\n[italic]{usecase_plan.plan_changelog}[/italic]" + ) + + if usecase_plan.plan_reasoning: + plan_content_parts.append( + f"\n[bold yellow]Reasoning:[/bold yellow]\n[italic]{usecase_plan.plan_reasoning}[/italic]" + ) + + plan_content = "\n".join(plan_content_parts) + self.console.print( Panel( - f"[bold]Search Quality:[/bold] {analysis.search_quality.value}\n" - f"{analysis.search_quality_explanation}", - title="🔍 Search Evaluation", - border_style=search_quality_color, + plan_content, + title=f"📋 Use Case Plan (ID: {search_id})", + border_style="magenta", ) ) - # Per-usecase tool analysis - if analysis.per_usecase_tool_analysis: - for tool_analysis in analysis.per_usecase_tool_analysis: - # Confidence color based on overall confidence - conf_color = ( - "green" - if tool_analysis.overall_confidence >= 8 - else "yellow" if tool_analysis.overall_confidence >= 6 else "red" - ) - - # Format main tools - main_tools_formatted = "" - if tool_analysis.final_main_tools: - main_tools_formatted = "[bold]Main Tools:[/bold]\n" - for tool in tool_analysis.final_main_tools: - tool_conf_color = ( - "green" - if tool.confidence >= 8 - else "yellow" if tool.confidence >= 6 else "red" - ) - main_tools_formatted += f" • [{tool_conf_color}]{tool.tool_slug}[/{tool_conf_color}] (confidence: [{tool_conf_color}]{tool.confidence}/10[/{tool_conf_color}])\n" - else: - main_tools_formatted = "[bold]Main Tools:[/bold] None\n" - - # Format related tools - related_tools_formatted = "" - if tool_analysis.final_related_tools: - related_tools_formatted = "\n[bold]Related Tools:[/bold]\n" - for tool in tool_analysis.final_related_tools: - tool_conf_color = ( - "green" - if tool.confidence >= 8 - else "yellow" if tool.confidence >= 6 else "red" - ) - related_tools_formatted += f" • [{tool_conf_color}]{tool.tool_slug}[/{tool_conf_color}] (confidence: [{tool_conf_color}]{tool.confidence}/10[/{tool_conf_color}])\n" - else: - related_tools_formatted = "\n[bold]Related Tools:[/bold] None\n" - - # Format toolkits - toolkits_formatted = f"\n[bold]Toolkits:[/bold] {', '.join(tool_analysis.final_toolkits) if tool_analysis.final_toolkits else 'None'}\n" - - # Format changes and confidence - summary_formatted = ( - f"\n[bold]Changes Made:[/bold] {tool_analysis.changes_made}/3\n" - f"[bold]Overall Confidence:[/bold] [{conf_color}]{tool_analysis.overall_confidence}/10[/{conf_color}]\n" - f"\n[bold cyan]Reasoning:[/bold cyan]\n{tool_analysis.reasoning}" - ) - - content = ( - main_tools_formatted - + related_tools_formatted - + toolkits_formatted - + summary_formatted - ) + def _get_quality_color(self, quality: str) -> str: + """Get color for workflow quality status.""" + if quality in ["SUCCESS", "SUCCESS_STRONG"]: + return "green" + elif quality in ["SUCCESS_WEAK", "PARTIAL_SUCCESS"]: + return "yellow" + else: + return "red" - self.console.print( - Panel( - content, - title=f"🔬 Tool Refinement Analysis ({tool_analysis.search_id})", - border_style=conf_color, - ) + def _display_bucket_breakdown(self, er: SessionErrorAnalysis): + """Display breakdown of error buckets with weights.""" + active_buckets = er.get_active_buckets() + + if not active_buckets: + self.console.print( + Panel( + "[green]No issues found in any bucket![/green]", + title="✅ Bucket Analysis", + border_style="green", ) + ) + return - # Per-usecase plans - if analysis.per_usecase_plans: - for i, usecase_plan in enumerate(analysis.per_usecase_plans, 1): - # Format the plan content - plan_content_parts = [ - f"[bold]Use Case (Cleaned):[/bold] {usecase_plan.cleaned_search_usecase}", - f"[bold]Known Fields (Cleaned):[/bold] {usecase_plan.cleaned_known_fields or 'N/A'}", - f"[bold]Required Toolkits:[/bold] {', '.join(usecase_plan.toolkits_required)}", - f"[bold]Success Confidence:[/bold] {usecase_plan.success_confidence}/10", - ] + # Create a table for bucket breakdown + table = Table(title="Error Bucket Breakdown", show_header=True) + table.add_column("Bucket", style="bold") + table.add_column("Weight", justify="right") + table.add_column("Issues", justify="right") + table.add_column("Details", style="dim") + + # Sort buckets by weight (highest first) + bucket_weights = er.get_bucket_weights() + sorted_buckets = sorted( + active_buckets.items(), + key=lambda x: bucket_weights.get(x[0], 0), + reverse=True, + ) + + for bucket_name, bucket in sorted_buckets: + weight = bucket_weights.get(bucket_name, 0) + item_count = bucket.item_count() + emoji = BUCKET_EMOJI.get(bucket_name, "❓") + display_name = BUCKET_NAMES.get(bucket_name, bucket_name) - # Format the usecase_plan (UsecasePlanDetails object) - plan_details = usecase_plan.usecase_plan + # Get brief details for each bucket + details = self._get_bucket_details(bucket_name, bucket) - # Format workflow steps as numbered list - workflow_steps_formatted = "\n".join( - f"{i}. {step}" - for i, step in enumerate(plan_details.workflow_steps, 1) + # Color based on weight + weight_color = ( + "red" if weight >= 0.3 else "yellow" if weight >= 0.1 else "white" + ) + + table.add_row( + f"{emoji} {display_name}", + f"[{weight_color}]{weight:.0%}[/{weight_color}]", + str(item_count), + details, + ) + + self.console.print(table) + + # Show detailed bucket info for high-weight buckets + for bucket_name, bucket in sorted_buckets: + weight = bucket_weights.get(bucket_name, 0) + if weight >= 0.2: # Only show details for significant buckets + self._display_bucket_details(bucket_name, bucket, weight) + + def _get_bucket_details(self, bucket_name: str, bucket: Any) -> str: + """Get brief summary of bucket contents.""" + details = [] + + if bucket_name == "llm_errors": + if bucket.wrong_input_values: + details.append(f"{len(bucket.wrong_input_values)} wrong values") + if bucket.wrong_input_types: + details.append(f"{len(bucket.wrong_input_types)} wrong types") + if bucket.hallucinated_tool_called: + details.append(f"{len(bucket.hallucinated_tool_called)} hallucinated") + if bucket.missed_pagination: + details.append(f"{len(bucket.missed_pagination)} missed pagination") + elif bucket_name == "tool_errors": + if bucket.request_schema_problems: + details.append( + f"{len(bucket.request_schema_problems)} request schema issues" + ) + if bucket.response_schema_problems: + details.append( + f"{len(bucket.response_schema_problems)} response schema issues" + ) + if bucket.tool_implementation_problems: + details.append(f"{len(bucket.tool_implementation_problems)} tool bugs") + if bucket.resource_not_found: + details.append(f"{len(bucket.resource_not_found)} resources not found") + elif bucket_name == "search_quality": + if bucket.wrong_tools_returned: + details.append(f"{len(bucket.wrong_tools_returned)} wrong tools") + elif bucket_name == "connection_problems": + total = ( + len(getattr(bucket, "manage_connection_not_called", [])) + + len(getattr(bucket, "manage_connection_failed", [])) + + len(getattr(bucket, "connection_errors_in_tool_execution", [])) + ) + if total: + details.append(f"{total} connection issues") + elif bucket_name == "workbench": + if bucket.code_logic_errors: + details.append(f"{len(bucket.code_logic_errors)} code errors") + if bucket.helper_tool_problems: + details.append( + f"{len(bucket.helper_tool_problems)} helper tool problems" + ) + if bucket.meta_tool_failures: + details.append(f"{len(bucket.meta_tool_failures)} meta failures") + elif bucket_name == "multi_execute": + if getattr(bucket, "meta_tool_failures", []): + details.append(f"{len(bucket.meta_tool_failures)} meta failures") + elif bucket_name == "recipe": + if getattr(bucket, "recipe_failures", []): + details.append(f"{len(bucket.recipe_failures)} failures") + + return ", ".join(details) if details else "various issues" + + def _display_bucket_details(self, bucket_name: str, bucket: Any, weight: float): + """Display detailed information for a specific bucket.""" + emoji = BUCKET_EMOJI.get(bucket_name, "❓") + display_name = BUCKET_NAMES.get(bucket_name, bucket_name) + + content_parts = [] + + if bucket_name == "llm_errors": + for item in bucket.wrong_input_values[:3]: # Limit to top 3 + content_parts.append( + f"• [bold]{item.action_name}.{item.parameter_name}[/bold]: " + f"{item.reason[:100]}..." + ) + for item in bucket.hallucinated_tool_called[:2]: + content_parts.append( + f"• [red]Hallucinated:[/red] {item.attempted_tool_name}" + ) + elif bucket_name == "tool_errors": + for item in bucket.tool_implementation_problems[:3]: + content_parts.append( + f"• [bold]{item.tool_name}[/bold]: {item.error_category} - " + f"{item.error_message[:80]}..." + ) + for item in bucket.request_schema_problems[:2]: + content_parts.append( + f"• [yellow]Schema:[/yellow] {item.tool_name}.{item.parameter_name}: " + f"{item.problem_type}" + ) + elif bucket_name == "search_quality": + for item in bucket.wrong_tools_returned[:3]: + missing = ", ".join(item.tools_missing[:3]) + content_parts.append( + f"• [bold]{item.use_case[:50]}...[/bold]\n" + f" Missing: [red]{missing}[/red]" + ) + elif bucket_name == "workbench": + for item in bucket.code_logic_errors[:2]: + issues_str = "; ".join(item.issues[:2]) + content_parts.append(f"• [bold]Log {item.log_id}[/bold]: {issues_str}") + for item in bucket.meta_tool_failures[:2]: + content_parts.append( + f"• [red]Meta failure:[/red] {item.error_code or 'unknown'}" ) - # Format pitfalls as bullet points - pitfalls_formatted = "\n".join( - f"• {pitfall}" for pitfall in plan_details.pitfalls + if content_parts: + content = "\n".join(content_parts) + self.console.print( + Panel( + content, + title=f"{emoji} {display_name} Details (weight: {weight:.0%})", + border_style="yellow" if weight >= 0.3 else "white", ) + ) - plan_formatted = ( - f"[bold]Workflow Steps:[/bold]\n{workflow_steps_formatted}\n\n" - f"[bold]Difficulty:[/bold] {plan_details.difficulty}\n\n" - f"[bold]Pitfalls:[/bold]\n{pitfalls_formatted}" + def _display_session_metrics(self, er: SessionErrorAnalysis): + """Display session metrics including token usage and workbench time.""" + metrics = er.metrics + content_parts = [] + + # Overview stats + content_parts.append( + f"[bold]Total Tool Calls:[/bold] {metrics.total_tool_calls}" + ) + + # Workbench metrics + if metrics.workbench.total_calls > 0: + wb = metrics.workbench + content_parts.append( + f"[bold]Workbench:[/bold] {wb.total_calls} calls, " + f"{wb.total_time_ms / 1000:.1f}s total ({wb.avg_time_ms / 1000:.1f}s avg)" + ) + # Show slow workbench calls + slow_calls = [w for w in wb.time_per_call if w.was_slow] + if slow_calls: + content_parts.append( + f" [yellow]⚠ {len(slow_calls)} slow calls (>30s)[/yellow]" ) - plan_content_parts.append( - f"\n[bold green]Plan:[/bold green]\n{plan_formatted}" + # Multi-execute metrics + if metrics.multi_execute.total_calls > 0: + me = metrics.multi_execute + content_parts.append( + f"[bold]Multi-Execute:[/bold] {me.success_count}/{me.total_calls} success " + f"({me.success_rate:.0%})" + ) + + # Search quality metrics + if metrics.search_quality.total_search_calls > 0: + sq = metrics.search_quality + content_parts.append(f"[bold]Search Calls:[/bold] {sq.total_search_calls}") + if sq.any_cached_plan_returned: + content_parts.append( + f" Cached plans returned: {sq.cached_plans_returned_count}" ) + for pq in sq.per_usecase_cached_plan_quality: + if pq.was_cached_plan_returned: + quality = pq.cached_plan_quality or "unknown" + content_parts.append(f" [{pq.search_id}]: {quality}") + + # Token usage summary + tu = metrics.token_usage + if tu.total_tokens > 0: + content_parts.append("") + content_parts.append( + f"[bold]Token Usage:[/bold] {tu.total_request_tokens:,} request / " + f"{tu.total_response_tokens:,} response ({tu.total_tokens:,} total)" + ) - # Add changelog - if usecase_plan.plan_changelog: - plan_content_parts.append( - f"\n[bold yellow]Changelog:[/bold yellow]\n[italic]{usecase_plan.plan_changelog}[/italic]" + # Large calls section - show tool names with exact token counts + large_calls = tu.get_large_calls() + if large_calls: + content_parts.append("") + content_parts.append( + f"[bold red]Large Token Calls ({len(large_calls)}):[/bold red]" + ) + # Sort by total tokens descending + sorted_large = sorted( + large_calls, key=lambda x: x.total_tokens, reverse=True + ) + for tc in sorted_large[:10]: # Show top 10 + req_color = "red" if tc.is_large_request else "white" + resp_color = "red" if tc.is_large_response else "white" + content_parts.append( + f" • [bold]{tc.tool_name}[/bold] " + f"([{req_color}]{tc.request_tokens:,} req[/{req_color}] / " + f"[{resp_color}]{tc.response_tokens:,} resp[/{resp_color}])" ) - plan_content = "\n".join(plan_content_parts) + # Top token consumers (even if not "large") + top_consumers = tu.get_top_consumers(5) + if top_consumers and not large_calls: + content_parts.append("") + content_parts.append("[bold]Top Token Consumers:[/bold]") + for tc in top_consumers: + content_parts.append( + f" • {tc.tool_name}: {tc.total_tokens:,} tokens " + f"({tc.request_tokens:,} req / {tc.response_tokens:,} resp)" + ) - # Display each plan in its own panel - self.console.print( - Panel( - plan_content, - title=f"📋 Use Case Plan {i} (ID: {usecase_plan.search_id})", - border_style="magenta", + if content_parts: + self.console.print( + Panel( + "\n".join(content_parts), + title="📈 Session Metrics", + border_style="cyan", + ) + ) + + def _display_recommendations(self, result: PipelineResult): + """Display all recommendations from pipeline result.""" + all_recs: List[Tuple[int, str, Any, str]] = [] + + # Tool improvements from error analysis + if result.error_result: + for tool_rec in result.error_result.tool_improvements: + all_recs.append( + ( + tool_rec.priority, + f"Tool: {tool_rec.action_name}", + tool_rec, + "tool", + ) + ) + for meta_rec in result.error_result.meta_tool_improvements: + all_recs.append( + ( + meta_rec.priority, + f"Meta: {meta_rec.meta_tool_name}", + meta_rec, + "meta", ) ) - # Overall summary - self.console.print( - Panel( - f"[bold]Overall Analysis[/bold]\n{analysis.overall_rationale}\n\n" - f"[bold]Workflow Successful:[/bold] {'Yes' if analysis.was_workflow_successful else 'No'}\n" - f"[bold]Workflow Quality:[/bold] {analysis.workflow_quality}\n" - f"[bold]Difficulty:[/bold] {analysis.task_difficulty.name}\n" - f"[bold]Confidence:[/bold] {analysis.confidence_score:.0%}", - title="📊 Session Analysis Summary", - border_style="blue", - ) - ) + # New tool/toolkit suggestions from workflow analysis + if result.workflow_result: + wo = result.workflow_result.output + for new_tool_rec in wo.new_tool_suggestions: + all_recs.append( + ( + new_tool_rec.priority, + f"New Tool: {new_tool_rec.tool_name}", + new_tool_rec, + "new_tool", + ) + ) + for new_tk_rec in wo.new_toolkit_suggestions: + all_recs.append( + ( + new_tk_rec.priority, + f"New Toolkit: {new_tk_rec.toolkit_name}", + new_tk_rec, + "new_toolkit", + ) + ) - # Create ordered list by priority - from typing import Any, List, Tuple + if not all_recs: + return + # Sort by priority (highest first) + all_recs.sort(reverse=True, key=lambda x: x[0]) + + # Display recommendations ordered by priority + self.console.print("\n[bold]Recommendations (ordered by priority):[/bold]") + + for i, (priority, rec_name, rec, rec_type) in enumerate(all_recs, 1): + self.console.print(f"\n[bold cyan]{i}. {rec_name}[/bold cyan]") + panel = self.format_recommendation(rec, rec_type) + self.console.print(panel) + + def _display_workflow_suggestions(self, wo: Any): + """Display new tool/toolkit suggestions from workflow analysis.""" all_recs: List[Tuple[int, str, Any, str]] = [] - for rec in analysis.tool_improvements: - all_recs.append((rec.priority, rec.recommendation_name, rec, "tool")) - for new_tool_rec in analysis.new_tool_suggestions: + + for rec in wo.new_tool_suggestions: all_recs.append( - ( - new_tool_rec.priority, - new_tool_rec.recommendation_name, - new_tool_rec, - "new_tool", - ) + (rec.priority, f"New Tool: {rec.tool_name}", rec, "new_tool") ) - for new_toolkit_rec in analysis.new_toolkit_suggestions: + for rec in wo.new_toolkit_suggestions: all_recs.append( - ( - new_toolkit_rec.priority, - new_toolkit_rec.recommendation_name, - new_toolkit_rec, - "new_toolkit", - ) - ) - for meta_rec in analysis.meta_tool_improvements: - all_recs.append( - (meta_rec.priority, meta_rec.recommendation_name, meta_rec, "meta") + (rec.priority, f"New Toolkit: {rec.toolkit_name}", rec, "new_toolkit") ) + if not all_recs: + return + # Sort by priority (highest first) all_recs.sort(reverse=True, key=lambda x: x[0]) # Display recommendations ordered by priority - self.console.print("\n[bold]Recommendations (ordered by priority):[/bold]") + self.console.print("\n[bold]Suggestions (ordered by priority):[/bold]") for i, (priority, rec_name, rec, rec_type) in enumerate(all_recs, 1): self.console.print(f"\n[bold cyan]{i}. {rec_name}[/bold cyan]") diff --git a/app_tester/rube_learning/ingestion_pipeline/__init__.py b/app_tester/rube_learning/ingestion_pipeline/__init__.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/__init__.py @@ -0,0 +1,8 @@ +"""Ingestion pipeline package for usecase plan ingestion.""" + +from app_tester.rube_learning.ingestion_pipeline.usecase_ingestion_service import ( + UsecaseIngestionService, +) + + +__all__ = ["UsecaseIngestionService"] diff --git a/app_tester/rube_learning/ingestion_pipeline/cached_plan_processor.py b/app_tester/rube_learning/ingestion_pipeline/cached_plan_processor.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/cached_plan_processor.py @@ -0,0 +1,559 @@ +"""Cached plan processor for handling plans with cached_plan_id.""" + +import asyncio +import json +import uuid +from datetime import datetime +from typing import Any, Callable, Dict, List, Optional + +from app_tester.common.metrics import metrics +from app_tester.rube_learning.ingestion_pipeline.ingestion_config import ( + UsecaseIngestionConfig, +) +from app_tester.rube_learning.ingestion_pipeline.ingestion_logger import ( + IngestionTraceLogger, +) +from app_tester.rube_learning.ingestion_pipeline.ingestion_utils import ( + filter_plans_by_confidence, + run_with_concurrency_limit, +) +from app_tester.rube_learning.ingestion_pipeline.plan_consolidator import ( + PlanConsolidator, +) +from app_tester.rube_learning.ingestion_pipeline.tool_consolidator import ( + ToolConsolidator, +) +from app_tester.rube_learning.ingestion_pipeline.toolkit_utils import ( + normalize_toolkits_list, + normalize_toolkits_string, + parse_toolkits_with_or, + toolkits_to_string, +) +from app_tester.rube_learning.ingestion_pipeline.turbopuffer_ingestion import ( + UsecasePlanIngestion, +) +from app_tester.rube_learning.ingestion_pipeline.usecase_cluster_models import ( + PlanWithContext, +) + + +class CachedPlanProcessor: + """Handles processing of plans that have cached_plan_id.""" + + def __init__( + self, + config: UsecaseIngestionConfig, + ingestion_service: UsecasePlanIngestion, + tool_consolidator: ToolConsolidator, + plan_consolidator: PlanConsolidator, + trace_logger: IngestionTraceLogger, + ingestion_id: str, + debug: bool = False, + dry_run: bool = False, + log_updated_plan_func: Optional[ + Callable[[str, Dict, Dict, Dict[str, tuple]], None] + ] = None, + get_toolkits_from_tools_func: Optional[Callable[[List[str]], List[str]]] = None, + ): + """ + Initialize the cached plan processor. + + Args: + config: Ingestion configuration + ingestion_service: TurboPuffer ingestion service + tool_consolidator: Tool consolidator instance + plan_consolidator: Plan consolidator instance + trace_logger: Logger for tracing operations + ingestion_id: Ingestion ID for logging + debug: Enable debug mode + dry_run: Enable dry run mode + log_updated_plan_func: Callback function for logging updated plans + get_toolkits_from_tools_func: Function to get toolkits from tools list + """ + self.config = config + self.ingestion = ingestion_service + self.tool_consolidator = tool_consolidator + self.plan_consolidator = plan_consolidator + self.trace_logger = trace_logger + self.ingestion_id = ingestion_id + self.debug = debug + self.dry_run = dry_run + self._log_updated_plan = log_updated_plan_func + self._get_toolkits_from_tools = get_toolkits_from_tools_func + + @staticmethod + def _parse_update_counter(raw_value: Optional[Any]) -> Optional[int]: + """Convert raw update_counter values (str/int/None) into ints.""" + if raw_value is None: + return None + if isinstance(raw_value, bool): + return int(raw_value) + if isinstance(raw_value, (int, float)): + return int(raw_value) + if isinstance(raw_value, str): + raw_value = raw_value.strip() + if not raw_value: + return None + try: + return int(raw_value) + except ValueError: + try: + return int(float(raw_value)) + except ValueError: + return None + return None + + async def process_and_upsert_cached_plans( + self, plans_with_cached: List[PlanWithContext], source_label: str = "buffer" + ) -> tuple[int, int]: + """ + Process plans that have cached_plan_id by consolidating all plans + with the same cached_plan_id and upserting them directly to TurboPuffer. + + This replaces the existing plan in TurboPuffer with a consolidated version + that includes all important information from the new executions. + + Args: + plans_with_cached: List of PlanWithContext objects that have cached_plan_id + + Returns: + Tuple of (successful_upserts, failed_upserts) + """ + print( + f" [CachedPlanProcessor] process_and_upsert_cached_plans called with {len(plans_with_cached)} plans" + ) + if not plans_with_cached: + print(" [CachedPlanProcessor] No plans provided, returning (0, 0)") + return 0, 0 + + print( + f"\n🔄 Processing {len(plans_with_cached)} plans with cached_plan_id " + f"(source={source_label})..." + ) + + # Group plans by cached_plan_id + cached_plan_groups: dict[str, List[PlanWithContext]] = {} + for plan in plans_with_cached: + if plan.cached_plan_id: + if plan.cached_plan_id not in cached_plan_groups: + cached_plan_groups[plan.cached_plan_id] = [] + cached_plan_groups[plan.cached_plan_id].append(plan) + + print( + f" 📊 Found {len(cached_plan_groups)} unique cached plan IDs " + f"(source={source_label})" + ) + print( + f" [CachedPlanProcessor] Cached plan IDs: {list(cached_plan_groups.keys())}" + ) + + # Process in batches of 50 for parallelism + # Using centralized chunked function from ingestion_utils + + async def process_cached_plan_group( + cached_plan_id: str, group_plans: List[PlanWithContext] + ) -> dict: + """ + Process a single cached plan group: consolidate plans, merge tools, upsert to TurboPuffer. + + Returns dict with: + - success: bool + - cached_plan_id: str + - skipped: bool + """ + result = { + "success": False, + "cached_plan_id": cached_plan_id, + "skipped": False, + } + + print( + f"\n 🔍 Processing cached_plan_id: {cached_plan_id} " + f"({len(group_plans)} plans, source={source_label})" + ) + + if not group_plans: + print(" ⚠️ No plans found, skipping this group") + return result + + # Filter best plans by confidence criteria first + print( + f" [CachedPlanProcessor] Filtering {len(group_plans)} plans by confidence >= {self.config.plan_consolidation.min_success_confidence}" + ) + eligible_plans = filter_plans_by_confidence( + group_plans, self.config.plan_consolidation.min_success_confidence + ) + print( + f" [CachedPlanProcessor] {len(eligible_plans)} plans passed confidence filter" + ) + if not eligible_plans: + print( + f" ⚠️ No plans meet confidence criteria (conf>={self.config.plan_consolidation.min_success_confidence}), skipping this group" + ) + result["skipped"] = True + return result + + # Fetch cached plan to get existing tools and plan + print( + f" [CachedPlanProcessor] Fetching cached plan attributes for {cached_plan_id}" + ) + cached_plan_attrs = await asyncio.to_thread( + self.ingestion.get_plan_by_id, cached_plan_id + ) + + # None = error occurred (service down, network issue) + # {} = plan genuinely not found (successful query, no results) + if cached_plan_attrs is None: + # Error occurred - skip, will retry later + print( + f" ⚠️ Error fetching cached plan {cached_plan_id} from TurboPuffer. " + "Skipping - will retry in next buffer flush." + ) + result["skipped"] = True + return result + + if not cached_plan_attrs: + # Plan genuinely not found (deleted after enqueueing) + print( + f" ⚠️ Cached plan {cached_plan_id} not found in TurboPuffer. " + "Skipping - plan might have been deleted after enqueueing." + ) + result["skipped"] = True + return result + print( + f" [CachedPlanProcessor] Fetched cached plan, update_counter={cached_plan_attrs.get('update_counter')}, updated_at={cached_plan_attrs.get('updated_at')}" + ) + + existing_update_counter = self._parse_update_counter( + cached_plan_attrs.get("update_counter") + ) + + # Preserve existing metadata from cached plan (don't change stored values) + # These are not consolidated - we preserve what's already stored + usecase_text = cached_plan_attrs.get("usecase", "") + known_fields = cached_plan_attrs.get("known_fields", "") + search_log_id = cached_plan_attrs.get("search_log_id", "") + old_main_tools = cached_plan_attrs.get("main_tools", "") + old_related_tools = cached_plan_attrs.get("related_tools", "") + old_reasoning = cached_plan_attrs.get("tool_consolidation_reasoning", "") + # Note: task_difficulty will be extracted from consolidated plan's complexity_assessment + + # OPERATION 1: Tool Consolidation (do this FIRST so plan consolidation can filter steps) + # Use LLM-based tool consolidation (with cached plan tools as priority) + cached_plan_tools = { + "id": cached_plan_id, + "main_tools": old_main_tools, + "related_tools": old_related_tools, + "tool_consolidation_reasoning": old_reasoning, + } + + # Create a synthetic cluster for tool consolidation + synthetic_cluster = { + "representative_usecase": usecase_text, + "member_indices": list(range(len(group_plans))), + "rationale": "Plans with same cached_plan_id", + } + + print(" [CachedPlanProcessor] Starting tool consolidation...") + consolidated_tools_result = ( + await self.tool_consolidator.consolidate_tools_with_cached_plan( + cluster=synthetic_cluster, + cluster_plans=group_plans, + cached_plan_tools=cached_plan_tools, + ) + ) + consolidated_main_list = consolidated_tools_result.main_tools + consolidated_related_list = consolidated_tools_result.related_tools + tools_reasoning = consolidated_tools_result.reasoning + print( + f" [CachedPlanProcessor] Tool consolidation done: main={len(consolidated_main_list)}, related={len(consolidated_related_list)}" + ) + + # Infer toolkits from consolidated tools + if self._get_toolkits_from_tools: + toolkit_list = self._get_toolkits_from_tools( + consolidated_main_list + consolidated_related_list + ) + else: + # Fallback: empty list if function not provided + toolkit_list = [] + toolkit_str = toolkits_to_string(toolkit_list) + + # Determine allowed toolkits for _specific plans + allowed_toolkits: Optional[List[str]] = None + if group_plans and any( + isinstance(getattr(plan, "search_id", None), str) + and getattr(plan, "search_id").endswith("_specific") + for plan in group_plans + ): + allowed_set = set() + for plan in group_plans: + if plan.toolkits_required: + raw_toolkits = [ + t.strip() + for t in plan.toolkits_required.split(",") + if t.strip() + ] + expanded = parse_toolkits_with_or(raw_toolkits) + allowed_set.update(expanded) + if allowed_set: + allowed_toolkits = sorted(allowed_set) + + # OPERATION 2: Plan Consolidation (with consolidated tools to filter steps) + # Convert PlanWithContext to the format expected by consolidate_plans_with_llm + plans_for_consolidation = [] + for plan_ctx in group_plans: + plans_for_consolidation.append( + { + "plan": plan_ctx.plan, + "usecase": plan_ctx.usecase, + "known_fields": plan_ctx.known_fields, + "session_id": plan_ctx.session_id, + "cached_plan_quality_reason": plan_ctx.cached_plan_quality_reason, + } + ) + + # Consolidate plans with LLM (pass consolidated tools to filter plan steps) + # Compute toolkits_str for stats tracking from all plans in group + cluster_toolkits: set[str] = set() + for plan in group_plans: + if plan.toolkits_required: + cluster_toolkits.update( + t.strip().lower() + for t in plan.toolkits_required.split(",") + if t.strip() + ) + toolkits_str = ",".join(sorted(cluster_toolkits)) + + print(" [CachedPlanProcessor] Starting plan consolidation with LLM...") + consolidated_plan = await self.plan_consolidator.consolidate_plans_with_llm( + cached_plan_id, + plans_for_consolidation, + consolidated_main_tools=consolidated_main_list, + consolidated_related_tools=consolidated_related_list, + allowed_toolkits=allowed_toolkits, + tool_consolidation_reasoning=tools_reasoning, + representative_usecase=usecase_text, + toolkits_str=toolkits_str, + ) + + if not consolidated_plan: + print(" ⚠️ Plan consolidation failed, skipping this group") + return result + print( + " [CachedPlanProcessor] Plan consolidation done, got consolidated_plan" + ) + + # Extract task_difficulty from consolidated plan's complexity_assessment + task_difficulty_raw = consolidated_plan.get("complexity_assessment", "N/A") + # Extract level from "LEVEL: explanation" format + if ":" in task_difficulty_raw: + task_difficulty = task_difficulty_raw.split(":")[0].strip().upper() + else: + task_difficulty = task_difficulty_raw.strip().upper() + + main_tools = ",".join(consolidated_main_list) + related_tools = ",".join(consolidated_related_list) + toolkits = toolkit_str + + # Check if toolkits match the cached plan's toolkits + # This check happens AFTER consolidation so consolidate_plans_with_llm can fetch the plan + cached_toolkits_raw = cached_plan_attrs.get("toolkits", "") + if isinstance(cached_toolkits_raw, list): + cached_toolkits_normalized = normalize_toolkits_list( + cached_toolkits_raw + ) + else: + cached_toolkits_normalized = normalize_toolkits_string( + cached_toolkits_raw + ) + + current_toolkits_normalized = normalize_toolkits_string(toolkit_str) + + # Track if we're creating a new entry vs updating existing + is_new_entry = False + final_plan_id = cached_plan_id # Start with original cached_plan_id + + # If toolkits don't match, generate new UUID and create new entry instead of updating + if cached_toolkits_normalized != current_toolkits_normalized: + print(" ⚠️ Toolkit mismatch detected!") + print(f" Cached toolkits: {cached_toolkits_normalized}") + print(f" Current toolkits: {current_toolkits_normalized}") + print( + " → Generating new UUID (will create new entry, not update existing)" + ) + # Generate new UUID for this plan with different toolkits + final_plan_id = uuid.uuid4().hex + # Update result to reflect new ID + result["cached_plan_id"] = final_plan_id + is_new_entry = True + + # Log the update for debug (before upserting) + # Only log if updating existing entry (not creating new) + if self.debug and cached_plan_attrs and not is_new_entry: + old_plan_str = cached_plan_attrs.get("plan", "{}") + old_plan_dict = ( + json.loads(old_plan_str) + if isinstance(old_plan_str, str) + else old_plan_str + ) + + changes = {} + if old_plan_dict != consolidated_plan: + changes["plan"] = (old_plan_dict, consolidated_plan) + if old_main_tools != main_tools: + changes["main_tools"] = (old_main_tools, main_tools) + if old_related_tools != related_tools: + changes["related_tools"] = (old_related_tools, related_tools) + + old_data = { + "usecase": cached_plan_attrs.get("usecase", ""), + "plan": old_plan_dict, + "main_tools": old_main_tools, + "related_tools": old_related_tools, + "toolkits": cached_plan_attrs.get("toolkits", ""), + } + new_data = { + "usecase": usecase_text, + "plan": consolidated_plan, + "main_tools": main_tools, + "related_tools": related_tools, + "toolkits": toolkits, + "known_fields": known_fields, + "task_difficulty": task_difficulty, # Just the level + "search_log_id": search_log_id, + "updated_at": datetime.utcnow().isoformat() + "Z", + "tool_consolidation_reasoning": tools_reasoning, + } + if self._log_updated_plan: + self._log_updated_plan(final_plan_id, old_data, new_data, changes) + + # Upsert directly to TurboPuffer with final_plan_id (cached_plan_id or new UUID if toolkits differ) + if not self.dry_run: + update_counter_for_upsert = ( + None if is_new_entry else existing_update_counter + ) + # For new entries, don't pass existing_created_at so it gets a new timestamp + existing_created_at_raw = ( + None if is_new_entry else cached_plan_attrs.get("created_at") + ) + existing_created_at_value: Optional[str] + if ( + isinstance(existing_created_at_raw, str) + or existing_created_at_raw is None + ): + existing_created_at_value = existing_created_at_raw + else: + existing_created_at_value = str(existing_created_at_raw) + print( + f" [CachedPlanProcessor] Calling upsert_consolidated_plan for {final_plan_id}" + ) + print( + f" [CachedPlanProcessor] is_new_entry={is_new_entry}, update_counter_for_upsert={update_counter_for_upsert}" + ) + print( + f" [CachedPlanProcessor] existing_created_at_value={existing_created_at_value}" + ) + success = self.ingestion.upsert_consolidated_plan( + cached_plan_id=final_plan_id, + consolidated_plan=consolidated_plan, + usecase_text=usecase_text, + main_tools=main_tools, + related_tools=related_tools, + known_fields=known_fields, + toolkits=toolkits, + task_difficulty=task_difficulty, + existing_created_at=existing_created_at_value, + search_log_id=search_log_id, + tool_consolidation_reasoning=tools_reasoning, + existing_update_counter=update_counter_for_upsert, + ) + print( + f" [CachedPlanProcessor] upsert_consolidated_plan returned success={success}" + ) + else: + action = "create" if is_new_entry else "update" + print(f" 🔍 DRY RUN: Would {action} TurboPuffer ID: {final_plan_id}") + success = True + + if success: + result["success"] = True + action = "created" if is_new_entry else "updated" + print( + f" ✅ Successfully {action} TurboPuffer ID: {final_plan_id} " + f"(source={source_label})" + ) + # Track individual plan ingestion + metrics.track_plan_ingestion( + ingestion_id=self.ingestion_id, + plan_id=final_plan_id, + operation="create" if is_new_entry else "update", + success=True, + source="cached_plan", + ) + else: + print( + f" ❌ Failed to update TurboPuffer ID: {final_plan_id} " + f"(source={source_label})" + ) + # Track individual plan ingestion failure + metrics.track_plan_ingestion( + ingestion_id=self.ingestion_id, + plan_id=final_plan_id, + operation="create" if is_new_entry else "update", + success=False, + source="cached_plan", + ) + + return result + + # Process groups in batches of 50 for parallelism + cached_plan_parallelism = max(1, self.config.parallelism.cached_plan_groups) + successful_upserts = 0 + failed_upserts = 0 + + total_groups = len(cached_plan_groups) + _ = total_groups # used for logging context + processed = 0 + + async def _process_one_group(item: tuple[str, list[PlanWithContext]]) -> dict: + cached_plan_id, group_plans = item + return await process_cached_plan_group(cached_plan_id, group_plans) + + items = list(cached_plan_groups.items()) + if items: + print( + f" 📦 Processing {len(items)} cached plan group(s) " + f"with concurrency={cached_plan_parallelism} (source={source_label})..." + ) + + batch_results = await run_with_concurrency_limit( + items, _process_one_group, concurrency=cached_plan_parallelism + ) + + for result in batch_results: + if isinstance(result, BaseException): + print(f" ❌ Plan group processing failed with exception: {result}") + failed_upserts += 1 + processed += 1 + continue + + if not isinstance(result, dict): + processed += 1 + failed_upserts += 1 + continue + + if result.get("skipped"): + processed += 1 + continue + + if result.get("success"): + successful_upserts += 1 + else: + failed_upserts += 1 + + processed += 1 + + print(f" ✅ Processed {processed}/{total_groups} cached plan group(s)") + + return successful_upserts, failed_upserts diff --git a/app_tester/rube_learning/ingestion_pipeline/ingestion_config.py b/app_tester/rube_learning/ingestion_pipeline/ingestion_config.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/ingestion_config.py @@ -0,0 +1,295 @@ +"""Configuration management for usecase plan ingestion service.""" + +import os +from typing import Optional + +from pydantic import BaseModel, Field + + +class ToolConsolidationConfig(BaseModel): + """Configuration for tool consolidation logic.""" + + max_related_tools: int = Field( + default=6, + description="Maximum number of related tools to keep after consolidation. Main tools have no limit.", + ) + + high_confidence_threshold: float = Field( + default=0.8, + description="Minimum confidence threshold (0-1) for including tools from tool analysis", + ) + + +class PlanConsolidationConfig(BaseModel): + """Configuration for plan consolidation logic.""" + + top_plans_to_consolidate: int = Field( + default=6, description="Number of top-quality plans to consolidate per cluster" + ) + + min_success_confidence: float = Field( + default=8.0, + description="Minimum success confidence (0-10) for plans to be eligible for consolidation", + ) + + skip_on_consolidation_failure: bool = Field( + default=True, + description="If True, skip plans/clusters when consolidation fails. If False, use fallback behavior.", + ) + + +class CacheConfig(BaseModel): + """Configuration for TurboPuffer caching behavior.""" + + rerank_score_threshold: float = Field( + default=0.9, + description="Minimum rerank score (0-1) for considering a cached plan match", + ) + + early_candidate_threshold: int = Field( + default=3, + description=( + "Number of queued candidates required to trigger an update while the " + "plan update counter is <= 5." + ), + ) + + mature_candidate_threshold: int = Field( + default=5, + description=( + "Number of queued candidates required to trigger an update once the " + "plan update counter is > 5." + ), + ) + + mature_threshold_growth_rate: float = Field( + default=2.0, + description=( + "Exponential growth multiplier applied per update once the counter exceeds 5. " + "Set to 1.0 to keep the mature threshold static." + ), + ) + + max_mature_candidate_threshold: Optional[int] = Field( + default=None, + description=( + "Optional upper bound on the dynamically scaled mature candidate threshold." + ), + ) + + mature_min_hours_since_update: int = Field( + default=24, + description=( + "Minimum hours since the cached plan was last updated before allowing " + "another update when the update counter is > 5." + ), + ) + + ttl_threshold_hours: int = Field( + default=48, # 2 days + description=( + "Process plans whose buffer has existed longer than this many hours. " + "Plans are force-processed once their created_at timestamp exceeds this threshold, " + "ensuring every plan gets processed within X hours regardless of candidate activity." + ), + ) + + skip_ttl_approaching_plans: bool = Field( + default=False, + description="If True, skip processing TTL-approaching plans during ingestion runs.", + ) + + +class RedisConfig(BaseModel): + """Configuration for candidate tracking Redis buffer.""" + + url: str = Field( + default="redis://localhost:6379", + description="Redis connection URL (supports rediss:// for TLS).", + ) + key_prefix: str = Field( + default="ingestion_plan_candidates", + description="Prefix used for Redis keys that store plan candidate data.", + ) + ttl_hours: int = Field( + default=240, # 10 days + description="Time-to-live in hours for candidate keys to avoid orphaned data.", + ) + use_tls: bool = Field( + default=True, + description="Whether to force TLS when connecting (set False for local Redis).", + ) + username: Optional[str] = Field( + default=None, + description="Optional Redis username; overrides URL credentials if provided.", + ) + password: Optional[str] = Field( + default=None, + description="Optional Redis password; overrides URL credentials if provided.", + ) + processed_session_prefix: str = Field( + default="ingestion_session_id", + description="Prefix for keys that track processed ClickHouse session IDs.", + ) + + +class ParallelismConfig(BaseModel): + """Configuration for parallel execution limits across pipeline stages.""" + + clustering: int = Field( + default=10, + description="Maximum concurrent toolkit-group clustering tasks", + ) + + cached_plan_groups: int = Field( + default=10, + description="Maximum concurrent cached plan consolidation tasks", + ) + + cluster_search: int = Field( + default=30, + description="Maximum concurrent TurboPuffer searches for clusters", + ) + + unmatched_ingest: int = Field( + default=20, + description="Maximum concurrent unmatched cluster processing tasks", + ) + + cached_plan_processing: int = Field( + default=10, + description="Maximum TTL-forced cached plan updates processed concurrently", + ) + + tf_query_parallelism: int = Field( + default=10, + description="Maximum concurrent TurboPuffer query operations for cached plan lookups", + ) + + +class UsecaseIngestionConfig(BaseModel): + """Main configuration for usecase plan ingestion service.""" + + namespace: str = Field( + description="TurboPuffer namespace to use for storing plans", + ) + + tool_consolidation: ToolConsolidationConfig = Field( + default_factory=ToolConsolidationConfig + ) + + plan_consolidation: PlanConsolidationConfig = Field( + default_factory=PlanConsolidationConfig + ) + + cache: CacheConfig = Field(default_factory=CacheConfig) + + redis: RedisConfig = Field( + default_factory=RedisConfig, + description="Redis settings for candidate tracking buffer", + ) + + parallelism: ParallelismConfig = Field(default_factory=ParallelismConfig) + + # Validation thresholds for individual plans + min_plan_length: int = Field( + default=100, + description="Minimum allowed character length for a single usecase plan JSON.", + ) + max_plan_length: int = Field( + default=4000, + description="Maximum allowed character length for a single usecase plan JSON.", + ) + + dry_run: bool = Field( + default=False, description="If True, don't write to TurboPuffer" + ) + + debug: bool = Field(default=False, description="Enable debug logging") + + @classmethod + def from_env(cls) -> "UsecaseIngestionConfig": + """Load configuration from environment variables.""" + return cls( + namespace=os.getenv("INGESTION_NAMESPACE", "usecase_plans_prod_new_v1"), + tool_consolidation=ToolConsolidationConfig( + max_related_tools=int(os.getenv("INGESTION_MAX_RELATED_TOOLS", "6")), + high_confidence_threshold=float( + os.getenv("INGESTION_TOOL_CONFIDENCE_THRESHOLD", "0.8") + ), + ), + plan_consolidation=PlanConsolidationConfig( + top_plans_to_consolidate=int(os.getenv("INGESTION_TOP_PLANS", "3")), + min_success_confidence=float( + os.getenv("INGESTION_MIN_SUCCESS_CONFIDENCE", "8.0") + ), + skip_on_consolidation_failure=os.getenv( + "INGESTION_SKIP_ON_FAILURE", "true" + ).lower() + == "true", + ), + cache=CacheConfig( + rerank_score_threshold=float( + os.getenv("INGESTION_RERANK_THRESHOLD", "0.9") + ), + early_candidate_threshold=int( + os.getenv("INGESTION_EARLY_CANDIDATE_THRESHOLD", "3") + ), + mature_candidate_threshold=int( + os.getenv("INGESTION_MATURE_CANDIDATE_THRESHOLD", "5") + ), + mature_threshold_growth_rate=float( + os.getenv("INGESTION_MATURE_THRESHOLD_GROWTH_RATE", "2.0") + ), + max_mature_candidate_threshold=( + int(max_threshold) + if ( + max_threshold := os.getenv( + "INGESTION_MAX_MATURE_CANDIDATE_THRESHOLD" + ) + ) + else None + ), + mature_min_hours_since_update=int( + os.getenv("INGESTION_MATURE_MIN_HOURS", "24") + ), + ttl_threshold_hours=int( + os.getenv("INGESTION_TTL_THRESHOLD_HOURS", "48") + ), + ), + parallelism=ParallelismConfig( + clustering=int(os.getenv("INGESTION_PARALLELISM_CLUSTERING", "20")), + cached_plan_groups=int( + os.getenv("INGESTION_PARALLELISM_CACHED_GROUPS", "20") + ), + cluster_search=int( + os.getenv("INGESTION_PARALLELISM_CLUSTER_SEARCH", "20") + ), + unmatched_ingest=int( + os.getenv("INGESTION_PARALLELISM_UNMATCHED", "20") + ), + cached_plan_processing=int( + os.getenv("INGESTION_PARALLELISM_CACHED_PROCESSING", "10") + ), + ), + min_plan_length=int(os.getenv("INGESTION_MIN_PLAN_LENGTH", "100")), + max_plan_length=int(os.getenv("INGESTION_MAX_PLAN_LENGTH", "4000")), + dry_run=os.getenv("INGESTION_DRY_RUN", "false").lower() == "true", + debug=os.getenv("INGESTION_DEBUG", "false").lower() == "true", + redis=RedisConfig( + url=os.getenv( + "INGESTION_REDIS_URL", + "redis://localhost:6379", + ), + key_prefix=os.getenv( + "INGESTION_REDIS_KEY_PREFIX", "ingestion_plan_candidates" + ), + ttl_hours=int(os.getenv("INGESTION_REDIS_TTL_HOURS", "240")), + use_tls=os.getenv("INGESTION_REDIS_USE_TLS", "true").lower() == "true", + username=os.getenv("INGESTION_REDIS_USERNAME"), + password=os.getenv("INGESTION_REDIS_PASSWORD"), + processed_session_prefix=os.getenv( + "INGESTION_PROCESSED_SESSION_PREFIX", "ingestion_session_id" + ), + ), + ) diff --git a/app_tester/rube_learning/ingestion_pipeline/ingestion_logger.py b/app_tester/rube_learning/ingestion_pipeline/ingestion_logger.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/ingestion_logger.py @@ -0,0 +1,521 @@ +""" +Ingestion trace logger for usecase ingestion pipeline. + +Provides detailed tracing of ingestion runs with unique IDs for debugging and monitoring. +""" + +import json +import threading +import uuid +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional + + +class IngestionTraceLogger: + """ + Logger for tracing ingestion pipeline execution. + + Creates a unique ingestion ID per pipeline run and logs all operations + to a structured JSON log file in logs/.jsonl + """ + + def __init__(self, output_dir: str | Path | None = None): + """ + Initialize the trace logger. + + Args: + output_dir: Directory to write logs to (default: workspace_root/logs) + """ + # Generate unique ingestion ID + self.ingestion_id = str(uuid.uuid4()) + + # Setup log directory + if output_dir is None: + # Default to workspace root /logs + workspace_root = Path(__file__).parent.parent.parent + log_dir = workspace_root / "logs" + else: + log_dir = Path(output_dir) + + log_dir.mkdir(parents=True, exist_ok=True) + + # Create log file for this ingestion run + self.log_file: Path = log_dir / f"{self.ingestion_id}.jsonl" + self._file_lock = threading.Lock() + + # Print ingestion ID for easy reference + print(f"\n{'='*60}") + print(f"🔍 Ingestion Trace ID: {self.ingestion_id}") + print(f"📝 Trace log: {self.log_file}") + print(f"{'='*60}\n") + + # Log the start event + self.log_event( + step="initialization", + event_type="started", + data={ + "ingestion_id": self.ingestion_id, + "log_file": str(self.log_file), + "timestamp": datetime.utcnow().isoformat() + "Z", + }, + ) + + def log_event( + self, + step: str, + event_type: str, + data: Dict[str, Any], + turbopuffer_ids: Optional[List[str]] = None, + error: Optional[str] = None, + ): + """ + Log an event in the ingestion pipeline. + + Args: + step: Pipeline step (e.g., "fetch_sessions", "clustering", "turbopuffer_upsert") + event_type: Event within step (e.g., "started", "completed", "error") + data: Event-specific payload (sessions, plans, tools, etc.) + turbopuffer_ids: List of relevant TurboPuffer IDs (if applicable) + error: Error message if event_type is "error" + """ + log_entry = { + "timestamp": datetime.utcnow().isoformat() + "Z", + "ingestion_id": self.ingestion_id, + "step": step, + "event_type": event_type, + "turbopuffer_ids": turbopuffer_ids or [], + "data": data, + } + + if error: + log_entry["error"] = error + + # Write to file (thread-safe, one JSON object per line for JSONL format) + with self._file_lock: + with open(self.log_file, "a") as f: + # Write compact JSON (one line per event) + f.write(json.dumps(log_entry)) + f.write("\n") + + def log_step_start(self, step: str, data: Optional[Dict[str, Any]] = None): + """Log the start of a pipeline step.""" + self.log_event(step=step, event_type="started", data=data or {}) + + def log_step_complete( + self, + step: str, + data: Optional[Dict[str, Any]] = None, + duration_sec: Optional[float] = None, + turbopuffer_ids: Optional[List[str]] = None, + ): + """Log the completion of a pipeline step.""" + event_data = data or {} + if duration_sec is not None: + event_data["duration_sec"] = duration_sec + + self.log_event( + step=step, + event_type="completed", + data=event_data, + turbopuffer_ids=turbopuffer_ids, + ) + + def log_step_error( + self, + step: str, + error_msg: str, + data: Optional[Dict[str, Any]] = None, + turbopuffer_ids: Optional[List[str]] = None, + ): + """Log an error in a pipeline step.""" + self.log_event( + step=step, + event_type="error", + data=data or {}, + turbopuffer_ids=turbopuffer_ids, + error=error_msg, + ) + + # High-level logging methods for specific operations + + def log_sessions_fetched( + self, + hours_back: Optional[int], + hours_start: Optional[float], + hours_end: Optional[float], + session_count: int, + session_ids: List[str], + duration_sec: float, + ): + """Log session fetching results.""" + self.log_event( + step="fetch_sessions", + event_type="completed", + data={ + "hours_back": hours_back, + "hours_start": hours_start, + "hours_end": hours_end, + "session_count": session_count, + "session_ids": session_ids, + "duration_sec": duration_sec, + }, + ) + + def log_cached_sessions_fetched( + self, + count: int, + hours_back: Optional[int], + hours_start: Optional[float], + hours_end: Optional[float], + duration_sec: float, + ): + """Log fetching of cached-only sessions.""" + self.log_event( + step="fetch_cached_only_sessions", + event_type="completed", + data={ + "count": count, + "hours_back": hours_back, + "hours_start": hours_start, + "hours_end": hours_end, + "duration_sec": duration_sec, + "filter": "total_cached_plans_returned > 0 AND total_cached_plans_returned = total_plans_generated AND workflow_quality != 'INCOMPLETE'", + }, + ) + + def log_plans_flattened( + self, total_plans: int, plans_per_session: Dict[str, int], duration_sec: float + ): + """Log plan flattening results.""" + self.log_event( + step="flatten_plans", + event_type="completed", + data={ + "total_plans": total_plans, + "plans_per_session": plans_per_session, + "avg_plans_per_session": ( + total_plans / len(plans_per_session) if plans_per_session else 0 + ), + "duration_sec": duration_sec, + }, + ) + + def log_plan_separation( + self, cached_count: int, non_cached_count: int, cached_plan_ids: List[str] + ): + """Log separation of cached vs non-cached plans.""" + self.log_event( + step="separate_plans", + event_type="completed", + data={ + "cached_count": cached_count, + "non_cached_count": non_cached_count, + "sample_cached_plan_ids": cached_plan_ids[:10], # Sample only + }, + turbopuffer_ids=cached_plan_ids, + ) + + def log_session_merge_stats( + self, + successful_count: int, + cached_only_count: int, + overlap_count: int, + total_unique_count: int, + ): + """Log session merge statistics.""" + self.log_event( + step="merge_sessions", + event_type="completed", + data={ + "successful_count": successful_count, + "cached_only_count": cached_only_count, + "overlap_count": overlap_count, + "total_unique_count": total_unique_count, + "merge_efficiency": ( + (1 - overlap_count / (successful_count + cached_only_count)) + if (successful_count + cached_only_count) > 0 + else 1.0 + ), + }, + ) + + def log_cached_plan_consolidation( + self, + cached_plan_id: str, + plans_in_group: int, + session_ids: List[str], + original_tools: Dict[str, List[str]], + new_tools: Dict[str, List[str]], + consolidated_tools: Dict[str, List[str]], + plan_consolidation: Optional[Dict[str, Any]], + upsert_success: bool, + duration_sec: float, + ): + """Log cached plan consolidation details.""" + self.log_event( + step="cached_plan_consolidation", + event_type="completed", + data={ + "plans_in_group": plans_in_group, + "session_ids": session_ids, + "original_tools": original_tools, + "new_tools": new_tools, + "consolidated_tools": consolidated_tools, + "plan_consolidation": plan_consolidation, + "upsert_success": upsert_success, + "duration_sec": duration_sec, + }, + turbopuffer_ids=[cached_plan_id], + ) + + def log_clustering_started(self, plan_count: int): + """Log start of clustering.""" + self.log_event( + step="clustering", event_type="started", data={"plan_count": plan_count} + ) + + def log_clustering_complete( + self, + clusters: List[Dict[str, Any]], + duration_sec: float, + plans: Optional[List[Any]] = None, + llm_calls: Optional[int] = None, + ): + """Log clustering results with composite IDs for cluster members. + + Args: + clusters: List of cluster dictionaries with member_indices + duration_sec: Duration of clustering in seconds + plans: Optional list of plans to extract session_id and search_id for composite IDs + llm_calls: Optional number of LLM calls made during clustering + """ + cluster_summary = [] + + for c in clusters: + cluster_data = { + "cluster_id": c["cluster_id"], + "representative_usecase": c["representative_usecase"][:100], + "member_count": len(c.get("member_indices", [])), + "rationale": c.get("rationale", "")[:200], + } + + # Add composite IDs if plans are provided + if plans: + member_composite_ids = [] + for member_idx in c.get("member_indices", []): + if 0 <= member_idx < len(plans): + plan = plans[member_idx] + # Build composite ID from session_id and search_id + session_id = getattr(plan, "session_id", "unknown") + search_id = getattr(plan, "search_id", None) + + if search_id: + composite_id = f"{session_id}_{search_id}" + else: + composite_id = f"{session_id}_no_search_id" + + member_composite_ids.append(composite_id) + + cluster_data["member_composite_ids"] = member_composite_ids + + cluster_summary.append(cluster_data) + + self.log_event( + step="clustering", + event_type="completed", + data={ + "cluster_count": len(clusters), + "clusters": cluster_summary, + "llm_calls": llm_calls, + "duration_sec": duration_sec, + }, + ) + + def log_turbopuffer_search( + self, + cluster_id: str, + representative_usecase: str, + matched: bool, + cached_plan_id: Optional[str], + rerank_score: Optional[float], + duration_sec: float, + ): + """Log TurboPuffer search for a cluster.""" + self.log_event( + step="turbopuffer_search", + event_type="match_found" if matched else "no_match", + data={ + "cluster_id": cluster_id, + "representative_usecase": representative_usecase[:150], + "matched": matched, + "rerank_score": rerank_score, + "duration_sec": duration_sec, + }, + turbopuffer_ids=[cached_plan_id] if cached_plan_id else [], + ) + + def log_cluster_consolidation( + self, + cluster_id: str, + cached_plan_id: str, + matched_cluster: bool, + plans_in_cluster: int, + original_tools: Dict[str, List[str]], + new_tools: Dict[str, List[str]], + consolidated_tools: Dict[str, List[str]], + plan_consolidation: Optional[Dict[str, Any]], + upsert_success: bool, + duration_sec: float, + ): + """Log cluster consolidation (matched or unmatched).""" + self.log_event( + step="cluster_consolidation", + event_type="matched_cluster" if matched_cluster else "new_cluster", + data={ + "cluster_id": cluster_id, + "matched_cluster": matched_cluster, + "plans_in_cluster": plans_in_cluster, + "original_tools": original_tools, + "new_tools": new_tools, + "consolidated_tools": consolidated_tools, + "plan_consolidation": plan_consolidation, + "upsert_success": upsert_success, + "duration_sec": duration_sec, + }, + turbopuffer_ids=[cached_plan_id], + ) + + def log_best_plan_selection( + self, + cluster_id: str, + selected_plan_session_id: str, + quality_score: float, + candidates_count: int, + ): + """Log best plan selection for a cluster.""" + self.log_event( + step="best_plan_selection", + event_type="selected", + data={ + "cluster_id": cluster_id, + "selected_plan_session_id": selected_plan_session_id, + "quality_score": quality_score, + "candidates_count": candidates_count, + }, + ) + + def log_tool_consolidation( + self, + cluster_id: str, + tools_before: Dict[str, int], + tools_after: Dict[str, int], + method: str, + cached_plan_id: Optional[str] = None, + ): + """Log tool consolidation results.""" + self.log_event( + step="tool_consolidation", + event_type="completed", + data={ + "cluster_id": cluster_id, + "tools_before": tools_before, + "tools_after": tools_after, + "method": method, + }, + turbopuffer_ids=[cached_plan_id] if cached_plan_id else [], + ) + + def log_plan_consolidation( + self, + cluster_id: Optional[str], + cached_plan_id: str, + input_plans: List[Dict[str, Any]], + output_plan: Dict[str, Any], + consolidation_type: str = "multiple", + ): + """Log plan consolidation input and output. + + Args: + cluster_id: Optional cluster ID this consolidation belongs to + cached_plan_id: The cached plan ID (or synthetic ID for new clusters) + input_plans: List of input plans with context (plan, usecase, session_id) + output_plan: The consolidated plan output from LLM + consolidation_type: Either "multiple" (consolidating many plans) or "single" (filtering one plan) + """ + # Extract input plan summaries + input_plan_summaries = [] + for idx, plan_with_context in enumerate(input_plans): + input_plan_summaries.append( + { + "plan_index": idx, + "session_id": plan_with_context.get("session_id", "unknown"), + "usecase": plan_with_context.get("usecase", "")[ + :100 + ], # Truncate to 100 chars + "plan_keys": ( + list(plan_with_context.get("plan", {}).keys()) + if isinstance(plan_with_context.get("plan"), dict) + else [] + ), + } + ) + + # Extract output plan summary + output_plan_summary = { + "workflow_steps": output_plan.get("workflow_steps", "")[ + :500 + ], # Truncate to 500 chars + "difficulty": output_plan.get("difficulty", ""), + "pitfalls": output_plan.get("pitfalls", "")[:500], # Truncate to 500 chars + "final_plan_reasoning": output_plan.get("final_plan_reasoning", "")[ + :500 + ], # Truncate to 500 chars + "plan_keys": ( + list(output_plan.keys()) if isinstance(output_plan, dict) else [] + ), + } + + self.log_event( + step="plan_consolidation", + event_type="completed", + data={ + "cluster_id": cluster_id, + "cached_plan_id": cached_plan_id, + "consolidation_type": consolidation_type, + "input_plan_count": len(input_plans), + "input_plans": input_plan_summaries, + "output_plan": output_plan_summary, + }, + turbopuffer_ids=[cached_plan_id] if cached_plan_id else [], + ) + + def log_batch_ingestion( + self, + batch_num: int, + plans_count: int, + turbopuffer_ids: List[str], + success: bool, + ): + """Log batch ingestion to TurboPuffer.""" + self.log_event( + step="batch_ingestion", + event_type="completed" if success else "failed", + data={"batch_num": batch_num, "plans_count": plans_count}, + turbopuffer_ids=turbopuffer_ids, + ) + + def log_ingestion_summary( + self, summary_stats: Dict[str, Any], total_duration_sec: float + ): + """Log final ingestion summary.""" + self.log_event( + step="ingestion_complete", + event_type="summary", + data={**summary_stats, "total_duration_sec": total_duration_sec}, + ) + + print(f"\n{'='*60}") + print(f"✅ Ingestion trace complete: {self.ingestion_id}") + print(f"📝 Full trace: {self.log_file}") + print(f"{'='*60}\n") diff --git a/app_tester/rube_learning/ingestion_pipeline/ingestion_models.py b/app_tester/rube_learning/ingestion_pipeline/ingestion_models.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/ingestion_models.py @@ -0,0 +1,336 @@ +""" +Pydantic models for ingestion pipeline data structures. + +This module contains models for: +- TurboPuffer API requests and responses +- External API requests and responses (Voyage, Vector Search) +- Internal data structures (clusters, search results, metrics) +- Debug and logging structures +""" + +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, Field + + +# ============================================================================ +# TurboPuffer Models +# ============================================================================ + + +class TurboPufferUpsertRow(BaseModel): + """A single row to upsert to TurboPuffer.""" + + id: str = Field(description="Unique row ID") + vector: List[float] = Field(description="Embedding vector") + usecase: str = Field(description="Use case text") + plan: str = Field(description="Plan JSON as string") + main_tools: str = Field(default="", description="Comma-separated main tools") + related_tools: str = Field(default="", description="Comma-separated related tools") + toolkits: str = Field(description="Comma-separated toolkit slugs") + toolkits_array: List[str] = Field( + default_factory=list, + description="Toolkit slugs as sorted, deduplicated, lowercase array", + ) + known_fields: str = Field(default="", description="Comma-separated known fields") + task_difficulty: str = Field( + default="MEDIUM", description="Task difficulty: EASY, MEDIUM, or HARD" + ) + search_log_id: str = Field(default="", description="Search log ID") + enabled: bool = Field(default=True, description="Whether plan is enabled") + created_at: str = Field(description="ISO timestamp of creation") + updated_at: str = Field(description="ISO timestamp of last update") + update_counter: int = Field( + default=0, + description="Number of times the plan has been updated in TurboPuffer.", + ) + tool_consolidation_reasoning: Optional[str] = Field( + default=None, description="Tool consolidation reasoning" + ) + + +class TurboPufferUpsertPayload(BaseModel): + """Payload for TurboPuffer upsert operations.""" + + upsert_rows: List[TurboPufferUpsertRow] = Field(description="Rows to upsert") + distance_metric: str = Field( + default="cosine_distance", description="Distance metric for vector search" + ) + schema_config: Dict[str, Dict[str, Any]] = Field( + description="Schema definition for namespace fields as dict", + serialization_alias="schema", + ) + + def model_dump(self, **kwargs): + """Override to serialize schema_config as 'schema'.""" + data = super().model_dump(**kwargs) + if "schema_config" in data: + data["schema"] = data.pop("schema_config") + return data + + +class TurboPufferQueryFilter(BaseModel): + """Filter for TurboPuffer query operations.""" + + id: Optional[List[List[str]]] = Field( + default=None, description="ID filter (e.g., [['Eq', plan_id]])" + ) + + +class TurboPufferQueryPayload(BaseModel): + """Payload for TurboPuffer query operations.""" + + filters: Optional[TurboPufferQueryFilter] = Field( + default=None, description="Query filters" + ) + top_k: int = Field(default=1, description="Number of results to return") + include_attributes: bool = Field( + default=True, description="Include attributes in response" + ) + + +class TurboPufferQueryResponse(BaseModel): + """Response from TurboPuffer query operation.""" + + rows: List[Dict[str, Any]] = Field( + default_factory=list, description="Query result rows" + ) + performance: Optional[Dict[str, Any]] = Field( + default=None, description="Performance metrics" + ) + billing: Optional[Dict[str, Any]] = Field( + default=None, description="Billing information" + ) + + +class TurboPufferUpsertResponse(BaseModel): + """Response from TurboPuffer upsert operation.""" + + rows_upserted: Optional[int] = Field( + default=None, description="Number of rows upserted" + ) + + +# ============================================================================ +# Voyage AI Embedding Models +# ============================================================================ + + +class VoyageEmbeddingRequest(BaseModel): + """Request payload for Voyage AI embedding API.""" + + model: str = Field(default="voyage-3.5-lite", description="Embedding model name") + input: List[str] = Field(description="List of texts to embed") + input_type: str = Field( + default="document", description="Input type (e.g., 'document' for indexing)" + ) + + +class VoyageEmbeddingItem(BaseModel): + """Single embedding item from Voyage API response.""" + + embedding: List[float] = Field(description="Embedding vector") + index: Optional[int] = Field(default=None, description="Index in input list") + + +class VoyageEmbeddingResponse(BaseModel): + """Response from Voyage AI embedding API.""" + + data: List[VoyageEmbeddingItem] = Field(description="List of embeddings") + model: Optional[str] = Field(default=None, description="Model used") + usage: Optional[Dict[str, Any]] = Field( + default=None, description="Usage statistics" + ) + + +# ============================================================================ +# Vector Search API Models +# ============================================================================ + + +class VectorSearchRequest(BaseModel): + """Request payload for Apollo vector search API.""" + + use_case: str = Field(description="Use case text to search for") + known_fields: str = Field(default="", description="Known fields for the use case") + toolkits_exact: List[str] = Field( + description="List of exact toolkit slugs to match" + ) + tf_namespace: str = Field(description="TurboPuffer namespace name") + reranker_model: str = Field( + default="openai:gpt-5.2", description="Reranker model to use for search" + ) + + +class VectorSearchResponse(BaseModel): + """Response from Apollo vector search API.""" + + cache_hit: Optional[bool] = Field( + default=None, description="Whether a cache hit occurred" + ) + cached_plan_id: Optional[str] = Field( + default=None, description="ID of cached plan if matched" + ) + cached_plan: Optional[Dict[str, Any]] = Field( + default=None, description="Cached plan data if matched" + ) + match_quality: Optional[str] = Field( + default=None, + description="Match quality: excellent, average, poor (or legacy: GOOD, BAD, UNKNOWN)", + ) + rerank_score: Optional[float] = Field( + default=None, description="Rerank score (0-1)" + ) + toolkits: Optional[List[str]] = Field( + default=None, description="Toolkits from cached plan" + ) + + +# ============================================================================ +# Internal Data Models +# ============================================================================ + + +class ClusterDict(BaseModel): + """Cluster dictionary structure used throughout the pipeline.""" + + cluster_id: int = Field(description="Unique cluster ID") + representative_usecase: str = Field( + description="Representative use case for this cluster" + ) + member_indices: List[int] = Field( + description="Indices of plans belonging to this cluster" + ) + rationale: Optional[str] = Field( + default=None, description="Explanation of clustering" + ) + toolkit_key: Optional[str] = Field( + default=None, description="Toolkit combination key" + ) + + +class TurboPufferSearchResult(BaseModel): + """Result from TurboPuffer search operation.""" + + matched: bool = Field(description="Whether a match was found") + cached_plan_id: Optional[str] = Field( + default=None, description="ID of matched cached plan" + ) + cached_plan_record: Optional[Dict[str, Any]] = Field( + default=None, description="Full cached plan record" + ) + rerank_score: Optional[float] = Field( + default=None, description="Rerank score if matched" + ) + skip_reason: Optional[str] = Field( + default=None, description="Reason if search was skipped" + ) + + +# ============================================================================ +# Tool Metrics Models +# ============================================================================ + + +class ToolMetricsRecord(BaseModel): + """Record of tool metrics for a single plan.""" + + plan_id: Optional[str] = Field(default=None, description="Plan ID") + ingest_source: str = Field(description="Source of ingestion") + main_tools_count: int = Field(description="Number of main tools") + related_tools_count: int = Field(description="Number of related tools") + main_tools: List[str] = Field( + default_factory=list, description="List of main tools" + ) + related_tools: List[str] = Field( + default_factory=list, description="List of related tools" + ) + timestamp: str = Field(description="ISO timestamp of record") + + +class ToolMetricsSourceStats(BaseModel): + """Statistics for a single ingestion source.""" + + plans_tracked: int = Field(description="Number of plans tracked") + total_main_tools: int = Field(description="Total main tools across plans") + total_related_tools: int = Field(description="Total related tools across plans") + average_main_tools_per_plan: float = Field( + description="Average main tools per plan" + ) + average_related_tools_per_plan: float = Field( + description="Average related tools per plan" + ) + + +class ToolMetricsSummary(BaseModel): + """Summary of tool metrics across all plans.""" + + total_plans_tracked: int = Field(description="Total plans tracked") + total_main_tools: int = Field(description="Total main tools") + total_related_tools: int = Field(description="Total related tools") + average_main_tools_per_plan: float = Field( + description="Average main tools per plan" + ) + average_related_tools_per_plan: float = Field( + description="Average related tools per plan" + ) + sources: Dict[str, ToolMetricsSourceStats] = Field( + default_factory=dict, description="Statistics per source" + ) + + +class ToolMetricsExport(BaseModel): + """Export metadata for tool metrics.""" + + summary_path: str = Field(description="Path to summary JSON file") + per_plan_path: str = Field(description="Path to per-plan JSON file") + summary: ToolMetricsSummary = Field(description="Summary statistics") + + +# ============================================================================ +# Debug and Logging Models +# ============================================================================ + + +class DebugPlanEntry(BaseModel): + """Debug entry for a new plan being created.""" + + id: str = Field(description="Plan ID") + usecase: str = Field(description="Use case text") + plan: Dict[str, Any] = Field(description="Plan dictionary") + main_tools: str = Field(default="", description="Main tools") + related_tools: str = Field(default="", description="Related tools") + toolkits: str = Field(default="", description="Toolkits") + known_fields: str = Field(default="", description="Known fields") + task_difficulty: str = Field(default="", description="Task difficulty") + search_log_id: str = Field(default="", description="Search log ID") + created_at: Optional[str] = Field(default=None, description="Creation timestamp") + embedding_dimensions: Optional[int] = Field( + default=None, description="Embedding vector dimensions" + ) + embedding_sample: Optional[List[float]] = Field( + default=None, description="Sample of embedding vector (first 5 values)" + ) + tool_consolidation_reasoning: Optional[str] = Field( + default=None, description="Tool consolidation reasoning" + ) + + +class DebugPlanUpdateEntry(BaseModel): + """Debug entry for a plan being updated.""" + + id: str = Field(description="Plan ID") + usecase: str = Field(description="Use case text") + changes: Dict[str, Dict[str, Any]] = Field( + default_factory=dict, + description="Changed fields: {field: {'old': value, 'new': value}}", + ) + unchanged: Dict[str, Any] = Field( + default_factory=dict, description="Unchanged fields" + ) + current_state: Dict[str, Any] = Field( + description="Current state of all important fields" + ) + tool_consolidation_reasoning: Optional[str] = Field( + default=None, description="Tool consolidation reasoning" + ) diff --git a/app_tester/rube_learning/ingestion_pipeline/ingestion_stats_counter.py b/app_tester/rube_learning/ingestion_pipeline/ingestion_stats_counter.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/ingestion_stats_counter.py @@ -0,0 +1,712 @@ +""" +Global thread-safe stats counter for ingestion pipeline. + +This module provides a simple way to track metrics during ingestion runs. +Stats are reset at the start of each run/poll cycle and read at the end for reporting. + +Usage: + from .ingestion_stats_counter import reset_stats, record_tool_consolidation, format_stats_for_slack + + # At start of run/poll cycle + reset_stats() + + # During processing (called from consolidators) + record_tool_consolidation(...) + record_plan_consolidation(...) + + # At end of run/poll cycle + message = format_stats_for_slack() +""" + +from dataclasses import dataclass, field +from threading import Lock +from typing import Dict, List, Set + +from app_tester.common.metrics import metrics + + +@dataclass +class ToolConsolidationRecord: + """Record for a single tool consolidation operation.""" + + toolkits_str: str + main_tools_before: List[str] + main_tools_after: List[str] + related_tools_before: List[str] + related_tools_after: List[str] + is_new: bool + changes_made: int = 0 # From LLM - includes order changes + plan_id: str = "" + ingestion_id: str = "" + usecase: str = "" # Usecase for new plans + + +@dataclass +class PlanConsolidationRecord: + """Record for a single plan consolidation operation.""" + + toolkits_str: str + chars_before: int + chars_after: int + is_new: bool + changes_made: int = 0 + plan_id: str = "" + ingestion_id: str = "" + + +@dataclass +class ToolkitStats: + """Stats for a single toolkit (comma-separated toolkit string).""" + + plans_created: int = 0 + plans_updated: int = 0 + plans_no_tool_change: int = 0 + + # Main tool changes (using set diff) + main_tools_added: int = 0 + main_tools_removed: int = 0 + + # Related tool changes (using set diff) + related_tools_added: int = 0 + related_tools_removed: int = 0 + + # Plan content + plan_chars_delta: int = 0 # Can be negative + new_plans_total_chars: int = 0 + + # Usecases for new plans (for display) + new_plan_usecases: List[str] = field(default_factory=list) + + # Plan changes distribution (0, 1, 2, 3+ changes) + plan_changes_0: int = 0 + plan_changes_1: int = 0 + plan_changes_2: int = 0 + plan_changes_3plus: int = 0 + + # Tool changes distribution (0, 1, 2, 3+ changes) - from LLM, includes order changes + tool_changes_0: int = 0 + tool_changes_1: int = 0 + tool_changes_2: int = 0 + tool_changes_3plus: int = 0 + + # Tool counts after consolidation (for averages) + total_main_tools_after: int = 0 + total_related_tools_after: int = 0 + tool_records_count: int = 0 + + @property + def total_plans(self) -> int: + return self.plans_created + self.plans_updated + + @property + def avg_chars_delta(self) -> float: + """Average chars delta per updated plan.""" + if self.plans_updated == 0: + return 0.0 + return self.plan_chars_delta / self.plans_updated + + @property + def plan_changes_distribution(self) -> Dict[str, int]: + """Get plan changes distribution as dict.""" + return { + "0": self.plan_changes_0, + "1": self.plan_changes_1, + "2": self.plan_changes_2, + "3+": self.plan_changes_3plus, + } + + @property + def tool_changes_distribution(self) -> Dict[str, int]: + """Get tool changes distribution as dict (from LLM, includes order changes).""" + return { + "0": self.tool_changes_0, + "1": self.tool_changes_1, + "2": self.tool_changes_2, + "3+": self.tool_changes_3plus, + } + + @property + def avg_main_tools_after(self) -> float: + if self.tool_records_count == 0: + return 0.0 + return self.total_main_tools_after / self.tool_records_count + + @property + def avg_related_tools_after(self) -> float: + if self.tool_records_count == 0: + return 0.0 + return self.total_related_tools_after / self.tool_records_count + + +@dataclass +class IngestionStats: + """Aggregated stats for an ingestion cycle.""" + + # Per-toolkit stats + by_toolkit: Dict[str, ToolkitStats] = field(default_factory=dict) + + # Global tool tracking (for avg calculation) + total_main_tools: int = 0 + total_related_tools: int = 0 + total_plans_processed: int = 0 + + def get_or_create_toolkit(self, toolkits_str: str) -> ToolkitStats: + """Get or create stats for a toolkit.""" + key = toolkits_str or "unknown" + if key not in self.by_toolkit: + self.by_toolkit[key] = ToolkitStats() + return self.by_toolkit[key] + + # Aggregated getters + @property + def total_plans_created(self) -> int: + return sum(t.plans_created for t in self.by_toolkit.values()) + + @property + def total_plans_updated(self) -> int: + return sum(t.plans_updated for t in self.by_toolkit.values()) + + @property + def total_plans_no_tool_change(self) -> int: + return sum(t.plans_no_tool_change for t in self.by_toolkit.values()) + + @property + def total_main_tools_added(self) -> int: + return sum(t.main_tools_added for t in self.by_toolkit.values()) + + @property + def total_main_tools_removed(self) -> int: + return sum(t.main_tools_removed for t in self.by_toolkit.values()) + + @property + def total_related_tools_added(self) -> int: + return sum(t.related_tools_added for t in self.by_toolkit.values()) + + @property + def total_related_tools_removed(self) -> int: + return sum(t.related_tools_removed for t in self.by_toolkit.values()) + + @property + def total_chars_delta(self) -> int: + return sum(t.plan_chars_delta for t in self.by_toolkit.values()) + + @property + def total_new_plans_chars(self) -> int: + return sum(t.new_plans_total_chars for t in self.by_toolkit.values()) + + @property + def avg_tools_per_plan(self) -> float: + if self.total_plans_processed == 0: + return 0.0 + return ( + self.total_main_tools + self.total_related_tools + ) / self.total_plans_processed + + @property + def total_plan_changes_distribution(self) -> Dict[str, int]: + """Aggregate plan changes distribution across all toolkits.""" + return { + "0": sum(t.plan_changes_0 for t in self.by_toolkit.values()), + "1": sum(t.plan_changes_1 for t in self.by_toolkit.values()), + "2": sum(t.plan_changes_2 for t in self.by_toolkit.values()), + "3+": sum(t.plan_changes_3plus for t in self.by_toolkit.values()), + } + + @property + def total_tool_changes_distribution(self) -> Dict[str, int]: + """Aggregate tool changes distribution across all toolkits (from LLM).""" + return { + "0": sum(t.tool_changes_0 for t in self.by_toolkit.values()), + "1": sum(t.tool_changes_1 for t in self.by_toolkit.values()), + "2": sum(t.tool_changes_2 for t in self.by_toolkit.values()), + "3+": sum(t.tool_changes_3plus for t in self.by_toolkit.values()), + } + + +# Global state +_lock = Lock() +_stats: IngestionStats | None = None + + +def reset_stats() -> None: + """Reset all stats. Call at start of each run/poll cycle.""" + global _stats + with _lock: + _stats = IngestionStats() + + +def _ensure_initialized() -> IngestionStats: + """Ensure stats are initialized, create if needed.""" + global _stats + if _stats is None: + _stats = IngestionStats() + return _stats + + +def _set_diff(before: List[str], after: List[str]) -> tuple[Set[str], Set[str]]: + """Compute set difference for tools. + + Returns: + (added, removed) - sets of tool slugs + """ + before_set = set(before) + after_set = set(after) + added = after_set - before_set + removed = before_set - after_set + return added, removed + + +def record_tool_consolidation(record: ToolConsolidationRecord) -> None: + """ + Record stats from a single tool consolidation. + + Args: + record: ToolConsolidationRecord with tool lists + """ + # Pre-compute diffs once so we can both emit metrics and update counters consistently. + main_added, main_removed = _set_diff( + record.main_tools_before, record.main_tools_after + ) + related_added, related_removed = _set_diff( + record.related_tools_before, record.related_tools_after + ) + # IMPORTANT: For *new* plans, these "added" tools are the initial selection, + # not a "change from cached". Only count diffs for updated plans. + if record.is_new: + main_added = set() + main_removed = set() + related_added = set() + related_removed = set() + + tags = [] + if record.ingestion_id: + tags.append(f"ingestion_id:{record.ingestion_id}") + if record.toolkits_str: + tags.append(f"toolkits:{record.toolkits_str}") + tags.append(f"is_new:{str(record.is_new).lower()}") + + # Emit official metrics (DataDog StatsD if available, always logged) + metrics.send_metric("rube_learning.ingestion.tool_consolidation_event", tags=tags) + metrics.send_metric( + "rube_learning.ingestion.tool_consolidation_changes_made", + value=float(record.changes_made), + tags=tags, + metric_type="histogram", + ) + metrics.send_metric( + "rube_learning.ingestion.tool_consolidation_main_tools_added", + value=float(len(main_added)), + tags=tags, + metric_type="histogram", + ) + metrics.send_metric( + "rube_learning.ingestion.tool_consolidation_main_tools_removed", + value=float(len(main_removed)), + tags=tags, + metric_type="histogram", + ) + metrics.send_metric( + "rube_learning.ingestion.tool_consolidation_related_tools_added", + value=float(len(related_added)), + tags=tags, + metric_type="histogram", + ) + metrics.send_metric( + "rube_learning.ingestion.tool_consolidation_related_tools_removed", + value=float(len(related_removed)), + tags=tags, + metric_type="histogram", + ) + metrics.send_metric( + "rube_learning.ingestion.tool_consolidation_main_tools_count", + value=float(len(record.main_tools_after)), + tags=tags, + metric_type="gauge", + ) + metrics.send_metric( + "rube_learning.ingestion.tool_consolidation_related_tools_count", + value=float(len(record.related_tools_after)), + tags=tags, + metric_type="gauge", + ) + + if record.plan_id: + print( + f" 📈 [ToolConsolidation] plan_id={record.plan_id} " + f"toolkits={record.toolkits_str or 'unknown'} is_new={record.is_new} changes_made={record.changes_made}" + ) + with _lock: + stats = _ensure_initialized() + tk_stats = stats.get_or_create_toolkit(record.toolkits_str) + + # Categorize plan: new vs updated + if record.is_new: + tk_stats.plans_created += 1 + if record.usecase: + tk_stats.new_plan_usecases.append(record.usecase) + else: + tk_stats.plans_updated += 1 + + # Compute main tools diff (already computed above) + tk_stats.main_tools_added += len(main_added) + tk_stats.main_tools_removed += len(main_removed) + + # Compute related tools diff + # (already computed above) + tk_stats.related_tools_added += len(related_added) + tk_stats.related_tools_removed += len(related_removed) + + # Track avg tool counts per toolkit + tk_stats.total_main_tools_after += len(record.main_tools_after) + tk_stats.total_related_tools_after += len(record.related_tools_after) + tk_stats.tool_records_count += 1 + + # Track if any tool changes were made (for updated plans) + if not record.is_new: + has_changes = bool( + main_added or main_removed or related_added or related_removed + ) + if not has_changes: + tk_stats.plans_no_tool_change += 1 + + # Track tool changes distribution (from LLM - includes order changes) + # Only meaningful for updated plans (relative to an existing cached tool list). + if not record.is_new: + if record.changes_made == 0: + tk_stats.tool_changes_0 += 1 + elif record.changes_made == 1: + tk_stats.tool_changes_1 += 1 + elif record.changes_made == 2: + tk_stats.tool_changes_2 += 1 + else: + tk_stats.tool_changes_3plus += 1 + + # Track for global averages + stats.total_main_tools += len(record.main_tools_after) + stats.total_related_tools += len(record.related_tools_after) + stats.total_plans_processed += 1 + + +def record_plan_consolidation(record: PlanConsolidationRecord) -> None: + """ + Record stats from a single plan consolidation. + + Args: + record: PlanConsolidationRecord with char counts and toolkit + """ + tags = [] + if record.ingestion_id: + tags.append(f"ingestion_id:{record.ingestion_id}") + if record.toolkits_str: + tags.append(f"toolkits:{record.toolkits_str}") + tags.append(f"is_new:{str(record.is_new).lower()}") + + delta = record.chars_after - record.chars_before + metrics.send_metric("rube_learning.ingestion.plan_consolidation_event", tags=tags) + metrics.send_metric( + "rube_learning.ingestion.plan_consolidation_changes_made", + value=float(record.changes_made), + tags=tags, + metric_type="histogram", + ) + metrics.send_metric( + "rube_learning.ingestion.plan_consolidation_chars_delta", + value=float(delta), + tags=tags, + metric_type="histogram", + ) + metrics.send_metric( + "rube_learning.ingestion.plan_consolidation_chars_after", + value=float(record.chars_after), + tags=tags, + metric_type="gauge", + ) + + if record.plan_id: + delta_sign = "+" if delta >= 0 else "" + print( + f" 📈 [PlanConsolidation] plan_id={record.plan_id} " + f"toolkits={record.toolkits_str or 'unknown'} is_new={record.is_new} changes_made={record.changes_made} " + f"chars_delta={delta_sign}{delta}" + ) + with _lock: + stats = _ensure_initialized() + tk_stats = stats.get_or_create_toolkit(record.toolkits_str) + + if record.is_new: + tk_stats.new_plans_total_chars += record.chars_after + else: + # Track character delta for updates + delta = record.chars_after - record.chars_before + tk_stats.plan_chars_delta += delta + + # Track plan changes distribution + if record.changes_made == 0: + tk_stats.plan_changes_0 += 1 + elif record.changes_made == 1: + tk_stats.plan_changes_1 += 1 + elif record.changes_made == 2: + tk_stats.plan_changes_2 += 1 + else: + tk_stats.plan_changes_3plus += 1 + + +def get_stats() -> IngestionStats: + """Get current stats. Returns a copy to avoid mutation.""" + with _lock: + stats = _ensure_initialized() + # Return a copy + new_stats = IngestionStats( + total_main_tools=stats.total_main_tools, + total_related_tools=stats.total_related_tools, + total_plans_processed=stats.total_plans_processed, + ) + # Deep copy toolkit stats + for tk, tk_stats in stats.by_toolkit.items(): + new_stats.by_toolkit[tk] = ToolkitStats( + plans_created=tk_stats.plans_created, + plans_updated=tk_stats.plans_updated, + plans_no_tool_change=tk_stats.plans_no_tool_change, + main_tools_added=tk_stats.main_tools_added, + main_tools_removed=tk_stats.main_tools_removed, + related_tools_added=tk_stats.related_tools_added, + related_tools_removed=tk_stats.related_tools_removed, + plan_chars_delta=tk_stats.plan_chars_delta, + new_plans_total_chars=tk_stats.new_plans_total_chars, + new_plan_usecases=list(tk_stats.new_plan_usecases), + plan_changes_0=tk_stats.plan_changes_0, + plan_changes_1=tk_stats.plan_changes_1, + plan_changes_2=tk_stats.plan_changes_2, + plan_changes_3plus=tk_stats.plan_changes_3plus, + tool_changes_0=tk_stats.tool_changes_0, + tool_changes_1=tk_stats.tool_changes_1, + tool_changes_2=tk_stats.tool_changes_2, + tool_changes_3plus=tk_stats.tool_changes_3plus, + total_main_tools_after=tk_stats.total_main_tools_after, + total_related_tools_after=tk_stats.total_related_tools_after, + tool_records_count=tk_stats.tool_records_count, + ) + return new_stats + + +def format_stats_summary_for_slack() -> str: + """ + Format overall stats summary for Slack (main message). + + Returns: + Formatted string with overall stats only + """ + stats = get_stats() + + if stats.total_plans_processed == 0: + return "📊 No plans processed" + + lines = ["📊 *Ingestion Stats*"] + lines.append( + f"Plans: +{stats.total_plans_created} new, ↺{stats.total_plans_updated} updated" + ) + if stats.total_plans_updated > 0: + no_change_pct = ( + stats.total_plans_no_tool_change / stats.total_plans_updated + ) * 100 + lines.append( + f"No tool change: {stats.total_plans_no_tool_change} ({no_change_pct:.0f}%)" + ) + + # Tool changes + lines.append( + f"Main tools: +{stats.total_main_tools_added}/-{stats.total_main_tools_removed} | " + f"Related: +{stats.total_related_tools_added}/-{stats.total_related_tools_removed}" + ) + + # Plan chars + delta_sign = "+" if stats.total_chars_delta >= 0 else "" + lines.append(f"Plan chars Δ: {delta_sign}{stats.total_chars_delta:,}") + + # Avg tools + lines.append(f"Avg tools/plan: {stats.avg_tools_per_plan:.1f}") + + # Tool changes distribution (from LLM) + tool_dist = stats.total_tool_changes_distribution + if sum(tool_dist.values()) > 0: + lines.append( + f"Tool changes: 0:{tool_dist['0']} | 1:{tool_dist['1']} | 2:{tool_dist['2']} | 3+:{tool_dist['3+']}" + ) + + # Plan changes distribution + plan_dist = stats.total_plan_changes_distribution + if sum(plan_dist.values()) > 0: + lines.append( + f"Plan changes: 0:{plan_dist['0']} | 1:{plan_dist['1']} | 2:{plan_dist['2']} | 3+:{plan_dist['3+']}" + ) + + return "\n".join(lines) + + +def format_stats_details_for_slack() -> str: + """ + Format per-toolkit breakdown for Slack (thread reply). + + Returns: + Formatted string with toolkit details, or empty string if no toolkits + """ + stats = get_stats() + + if not stats.by_toolkit: + return "" + + lines = [] + + # Top 10 by total plans + sorted_toolkits = sorted( + stats.by_toolkit.items(), + key=lambda x: x[1].total_plans, + reverse=True, + ) + + lines.append("*Top 10 by Total Plans:*") + for tk, tk_stats in sorted_toolkits[:10]: + tk_display = tk[:35] + "..." if len(tk) > 35 else tk + lines.append(f"`{tk_display}`") + lines.append( + f" +{tk_stats.plans_created} new, ↺{tk_stats.plans_updated} upd" + + ( + f", ={tk_stats.plans_no_tool_change} unchanged" + if tk_stats.plans_no_tool_change + else "" + ) + ) + lines.append( + f" Main: +{tk_stats.main_tools_added}/-{tk_stats.main_tools_removed} | " + f"Rel: +{tk_stats.related_tools_added}/-{tk_stats.related_tools_removed}" + ) + if tk_stats.plans_updated > 0: + avg_delta = tk_stats.avg_chars_delta + delta_sign = "+" if avg_delta >= 0 else "" + lines.append(f" Avg Δchars: {delta_sign}{avg_delta:.0f}") + + # Top 5 by new plans with usecases + sorted_by_new = sorted( + stats.by_toolkit.items(), + key=lambda x: x[1].plans_created, + reverse=True, + ) + top_new = [(tk, s) for tk, s in sorted_by_new[:5] if s.plans_created > 0] + + if top_new: + lines.append("") + lines.append("*Top 5 by New Plans:*") + for tk, tk_stats in top_new: + tk_display = tk[:35] + "..." if len(tk) > 35 else tk + lines.append(f" `{tk_display}`: +{tk_stats.plans_created} new") + # Show up to 3 usecases per toolkit + for uc in tk_stats.new_plan_usecases[:3]: + uc_display = uc[:60] + "..." if len(uc) > 60 else uc + lines.append(f" • {uc_display}") + if len(tk_stats.new_plan_usecases) > 3: + lines.append(f" ... +{len(tk_stats.new_plan_usecases) - 3} more") + + return "\n".join(lines) + + +def format_stats_for_slack() -> str: + """ + Format current stats as a complete Slack message (summary + details). + + Returns: + Formatted string for Slack notification + """ + summary = format_stats_summary_for_slack() + details = format_stats_details_for_slack() + + if details: + return f"{summary}\n\n{details}" + return summary + + +def print_stats_summary() -> None: + """Print stats summary to console.""" + stats = get_stats() + + if stats.total_plans_processed == 0: + print("\n📊 No plans processed") + return + + print("\n" + "=" * 60) + print("📊 Ingestion Stats Summary") + print("=" * 60) + + print(f"\nTotal Plans: {stats.total_plans_processed}") + print(f" Created: {stats.total_plans_created}") + print(f" Updated: {stats.total_plans_updated}") + if stats.total_plans_updated > 0: + print(f" No tool change: {stats.total_plans_no_tool_change}") + + print("\nTool Changes:") + print( + f" Main: +{stats.total_main_tools_added} added, -{stats.total_main_tools_removed} removed" + ) + print( + f" Related: +{stats.total_related_tools_added} added, -{stats.total_related_tools_removed} removed" + ) + + print("\nPlan Content:") + delta_sign = "+" if stats.total_chars_delta >= 0 else "" + print(f" Total chars delta: {delta_sign}{stats.total_chars_delta:,}") + if stats.total_plans_created > 0 and stats.total_new_plans_chars > 0: + avg_new = stats.total_new_plans_chars // stats.total_plans_created + print(f" New plans avg size: {avg_new:,} chars") + + print(f"\nAvg tools/plan: {stats.avg_tools_per_plan:.1f}") + + # Tool changes distribution (from LLM) + tool_dist = stats.total_tool_changes_distribution + if sum(tool_dist.values()) > 0: + print("\nTool Changes Distribution (from LLM):") + print(f" 0 changes: {tool_dist['0']}") + print(f" 1 change: {tool_dist['1']}") + print(f" 2 changes: {tool_dist['2']}") + print(f" 3+ changes: {tool_dist['3+']}") + + # Plan changes distribution + plan_dist = stats.total_plan_changes_distribution + if sum(plan_dist.values()) > 0: + print("\nPlan Changes Distribution:") + print(f" 0 changes: {plan_dist['0']}") + print(f" 1 change: {plan_dist['1']}") + print(f" 2 changes: {plan_dist['2']}") + print(f" 3+ changes: {plan_dist['3+']}") + + # Per-toolkit breakdown + if stats.by_toolkit: + print("\n" + "-" * 40) + print("By Toolkit (sorted by total plans):") + print("-" * 40) + + sorted_toolkits = sorted( + stats.by_toolkit.items(), + key=lambda x: x[1].total_plans, + reverse=True, + ) + + for tk, tk_stats in sorted_toolkits[:15]: + print(f"\n {tk}") + print( + f" Plans: +{tk_stats.plans_created} new, ↺{tk_stats.plans_updated} updated" + ) + if tk_stats.plans_no_tool_change: + print(f" No tool change: {tk_stats.plans_no_tool_change}") + if tk_stats.tool_records_count: + print( + f" Avg tools: main {tk_stats.avg_main_tools_after:.1f}, related {tk_stats.avg_related_tools_after:.1f}" + ) + print( + f" Main tools: +{tk_stats.main_tools_added}/-{tk_stats.main_tools_removed}" + ) + print( + f" Related tools: +{tk_stats.related_tools_added}/-{tk_stats.related_tools_removed}" + ) + if tk_stats.plans_updated > 0: + print(f" Avg chars delta: {tk_stats.avg_chars_delta:.0f}") + + print("\n" + "=" * 60) diff --git a/app_tester/rube_learning/ingestion_pipeline/ingestion_utils.py b/app_tester/rube_learning/ingestion_pipeline/ingestion_utils.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/ingestion_utils.py @@ -0,0 +1,181 @@ +""" +Utilities for the usecase ingestion pipeline. + +Core helper functions used across the ingestion pipeline. +""" + +import asyncio +import hashlib +from itertools import islice +from typing import Any, Awaitable, Callable, Dict, Iterator, List, TypeVar + +from tenacity import ( + retry, + retry_if_exception_type, + stop_after_attempt, + wait_exponential, +) + +from app_tester.rube_learning.ingestion_pipeline.usecase_cluster_models import ( + PlanWithContext, +) + + +T = TypeVar("T") + + +def filter_plans_by_confidence( + plans: List[PlanWithContext], min_confidence: float +) -> List[PlanWithContext]: + """ + Filter plans by minimum confidence threshold. + + Args: + plans: List of plans with success_confidence attribute + min_confidence: Minimum confidence threshold (0-10 scale) + + Returns: + Filtered list of plans meeting confidence criteria + """ + return [p for p in plans if p.success_confidence >= min_confidence] + + +def chunked(iterable: Iterator[T], n: int) -> Iterator[List[T]]: + """ + Split iterable into chunks of size n. + + Args: + iterable: Any iterable to chunk + n: Size of each chunk + + Yields: + Lists of size n (last chunk may be smaller) + """ + it = iter(iterable) + while chunk := list(islice(it, n)): + yield chunk + + +async def run_with_concurrency_limit( + items: List[T], + worker_fn: Callable[[T], Awaitable[Any]], + *, + concurrency: int, + context: str = "", + log_exceptions: bool = True, +) -> List[Any]: + """ + Run N independent async jobs with a fixed concurrency limit (stdlib only). + + - No manual batching: schedule all tasks at once, guarded by a semaphore. + - Returns results in completion order (asyncio.as_completed). + - Exceptions are returned as Exception objects (callers can count/log). + """ + sem = asyncio.Semaphore(max(1, concurrency)) + + async def _guarded(item: T) -> Any: + async with sem: + return await worker_fn(item) + + task_to_item: Dict[asyncio.Task[Any], T] = {} + tasks: List[asyncio.Task[Any]] = [] + for item in items: + task = asyncio.create_task(_guarded(item)) + task_to_item[task] = item + tasks.append(task) + results: List[Any] = [] + for fut in asyncio.as_completed(tasks): + try: + results.append(await fut) + except Exception as exc: # pragma: no cover - defensive + if log_exceptions: + item = task_to_item.get(fut) # type: ignore[arg-type,call-overload] + item_repr = repr(item) if item is not None else "" + if len(item_repr) > 200: + item_repr = item_repr[:197] + "..." + prefix = f"[{context}] " if context else "" + print(f" ❌ {prefix}Task failed for item={item_repr}: {exc}") + import traceback + + traceback.print_exception(type(exc), exc, exc.__traceback__) + results.append(exc) + return results + + +def generate_plan_id(usecase: str, is_specific: bool = False) -> str: + """ + Generate a consistent plan ID from usecase text. + + Args: + usecase: The usecase text + is_specific: Whether this is a specific variant + + Returns: + 16-character hex ID with optional _specific suffix + """ + normalized_usecase = usecase.lower().strip() + hash_obj = hashlib.sha256(normalized_usecase.encode()) + base_id = hash_obj.hexdigest()[:16] + + if is_specific: + return f"{base_id}_specific" + return base_id + + +def get_representative_known_fields( + cluster: Dict, cluster_plans: List[PlanWithContext] +) -> str: + """ + Extract known_fields from the plan at representative_usecase_index. + + Args: + cluster: Cluster dict with representative_usecase_index and member_indices + cluster_plans: List of plans in the cluster (in same order as member_indices) + + Returns: + known_fields string from representative plan, or empty string if not found + """ + representative_usecase_index = cluster.get("representative_usecase_index") + if representative_usecase_index is None: + return "" + + cluster_plan_indices = cluster.get("member_indices", []) + if representative_usecase_index not in cluster_plan_indices: + return "" + + for plan_idx, plan in enumerate(cluster_plans): + if cluster_plan_indices[plan_idx] == representative_usecase_index: + return plan.known_fields or "" + + return "" + + +class RetryableOperation: + """ + Decorator for retryable operations with exponential backoff. + """ + + @staticmethod + def with_exponential_backoff( + max_attempts: int = 3, + min_wait: int = 2, + max_wait: int = 10, + retry_on: tuple = (Exception,), + ): + """ + Create a retry decorator with exponential backoff. + + Args: + max_attempts: Maximum number of attempts + min_wait: Minimum wait time in seconds + max_wait: Maximum wait time in seconds + retry_on: Tuple of exception types to retry on + + Returns: + Configured retry decorator + """ + return retry( + stop=stop_after_attempt(max_attempts), + wait=wait_exponential(multiplier=1, min=min_wait, max=max_wait), + retry=retry_if_exception_type(retry_on), + ) diff --git a/app_tester/rube_learning/ingestion_pipeline/matched_cluster_processor.py b/app_tester/rube_learning/ingestion_pipeline/matched_cluster_processor.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/matched_cluster_processor.py @@ -0,0 +1,209 @@ +"""Matched cluster processor for handling clusters that match TurboPuffer records.""" + +import asyncio +import time +from typing import Dict, List + +from app_tester.common.metrics import metrics +from app_tester.rube_learning.ingestion_pipeline.ingestion_config import ( + UsecaseIngestionConfig, +) +from app_tester.rube_learning.ingestion_pipeline.ingestion_logger import ( + IngestionTraceLogger, +) +from app_tester.rube_learning.ingestion_pipeline.ingestion_models import ( + TurboPufferSearchResult, +) +from app_tester.rube_learning.ingestion_pipeline.toolkit_utils import ( + normalize_toolkits_list, + normalize_toolkits_string, +) +from app_tester.rube_learning.ingestion_pipeline.turbopuffer_ingestion import ( + UsecasePlanIngestion, +) +from app_tester.rube_learning.ingestion_pipeline.usecase_cluster_models import ( + PlanWithContext, +) + + +class MatchedClusterProcessor: + """Handles searching for clusters in existing TurboPuffer records.""" + + def __init__( + self, + config: UsecaseIngestionConfig, + ingestion_service: UsecasePlanIngestion, + trace_logger: IngestionTraceLogger, + ingestion_id: str, + debug: bool = False, + ): + """ + Initialize the matched cluster processor. + + Args: + config: Ingestion configuration + ingestion_service: TurboPuffer ingestion service + trace_logger: Logger for tracing operations + ingestion_id: Ingestion ID for logging + debug: Whether debug mode is enabled + """ + self.config = config + self.ingestion_service = ingestion_service + self.trace_logger = trace_logger + self.ingestion_id = ingestion_id + self.debug = debug + + async def search_turbopuffer_optimized( + self, cluster: Dict, all_plans: List[PlanWithContext] + ) -> TurboPufferSearchResult: + """ + TurboPuffer search that combines multiple operations. + + This replaces 4 sequential operations with 1-2 operations: + 1. Search with all filters applied upfront + 2. Only fetch full plan if all criteria are met + + Args: + cluster: Cluster dict with representative_usecase, member_indices, etc. + all_plans: All plans (to extract cluster members) + + Returns: + TurboPufferSearchResult with matched status, cached_plan_id, cached_plan_record, + rerank_score, and skip_reason + """ + search_start = time.time() + result = TurboPufferSearchResult( + matched=False, + cached_plan_id=None, + cached_plan_record=None, + rerank_score=None, + skip_reason=None, + ) + + # Get cluster members + cluster_plan_indices = cluster.get("member_indices", []) + if not cluster_plan_indices: + result.skip_reason = "no_cluster_members" + return result + + # Get best plan from cluster for metadata + cluster_plans = [ + all_plans[idx] for idx in cluster_plan_indices if idx < len(all_plans) + ] + if not cluster_plans: + result.skip_reason = "no_valid_plans" + return result + + # Get best quality plan for metadata + best_plan = max(cluster_plans, key=lambda p: p.success_confidence * 10) + + # Normalize toolkits + toolkits = normalize_toolkits_string(best_plan.toolkits_required or "") + representative_usecase = cluster["representative_usecase"] + # Use representative_known_fields from cluster (set by LLM during clustering) + representative_known_fields = ( + cluster.get("representative_known_fields", "") or "" + ) + + if self.debug: + print( + f" 🔍 Searching TurboPuffer for cluster: {representative_usecase[:80]}..." + ) + print(f" Toolkits: {toolkits}") + if representative_known_fields: + print(f" Known Fields: {representative_known_fields}") + + try: + # OPTIMIZATION: Pass all filters to the search + search_result = await asyncio.to_thread( + self.ingestion_service.search_vector_cache, + representative_usecase, + representative_known_fields, + toolkits, + ) + except Exception as e: + print(f" ❌ TurboPuffer search failed: {e}") + result.skip_reason = f"search_error: {str(e)}" + return result + + # None = error occurred (service down, network issue) - skip cluster + if search_result is None: + print(" ❌ Vector search service error - skipping cluster") + result.skip_reason = "search_service_error" + return result + + # Check all criteria in one go + rerank_score = search_result.get("rerank_score") or 0 + cached_plan_id = search_result.get("cached_plan_id") + rerank_threshold = self.config.cache.rerank_score_threshold + + # Early exit if no match or low confidence + if not cached_plan_id or rerank_score <= rerank_threshold: + if self.debug: + print( + f" ℹ️ No high-confidence match found (cached_plan_id: {cached_plan_id}, score: {rerank_score:.3f})" + ) + result.skip_reason = "low_confidence" + result.rerank_score = rerank_score + return result + + # Check toolkit compatibility + cached_toolkits_raw = search_result.get("toolkits", []) + if isinstance(cached_toolkits_raw, list): + cached_toolkits = normalize_toolkits_list(cached_toolkits_raw) + else: + cached_toolkits = normalize_toolkits_string(cached_toolkits_raw) + + if cached_toolkits != toolkits: + if self.debug: + print(" ℹ️ No toolkit-compatible match found") + print(f" Current: {toolkits}") + print(f" Cached: {cached_toolkits}") + result.skip_reason = "toolkit_mismatch" + result.rerank_score = rerank_score + result.cached_plan_id = cached_plan_id + return result + + # Fetch plan record for later use + cached_plan_record = await asyncio.to_thread( + self.ingestion_service.get_plan_by_id, cached_plan_id + ) + + # None = error, {} = not found - either way, skip (don't create new entries) + if not cached_plan_record: + reason = "fetch_error" if cached_plan_record is None else "plan_not_found" + print( + f" ❌ Could not fetch cached plan from TurboPuffer ({reason})" + ) + result.skip_reason = "fetch_failed" + result.rerank_score = rerank_score + result.cached_plan_id = cached_plan_id + return result + + # All checks passed! + print( + f" ✅ Found TurboPuffer match! ID: {cached_plan_id}, score: {rerank_score:.3f}" + ) + + search_duration = time.time() - search_start + self.trace_logger.log_turbopuffer_search( + cluster_id=cluster["cluster_id"], + representative_usecase=representative_usecase, + matched=True, + cached_plan_id=cached_plan_id, + rerank_score=rerank_score, + duration_sec=search_duration, + ) + metrics.track_turbopuffer_operation( + ingestion_id=self.ingestion_id, + operation="search_match", + success=True, + count=1, + ) + + result.matched = True + result.cached_plan_id = cached_plan_id + result.cached_plan_record = cached_plan_record + result.rerank_score = rerank_score + + return result diff --git a/app_tester/rube_learning/ingestion_pipeline/plan_candidate_buffer.py b/app_tester/rube_learning/ingestion_pipeline/plan_candidate_buffer.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/plan_candidate_buffer.py @@ -0,0 +1,257 @@ +""" +Helper utilities for tracking plan update candidates in Redis. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import Any, Iterable, List, Optional, Protocol, Union + + +NameType = Union[str, bytes] + + +class RedisLike(Protocol): + """Protocol describing the minimal Redis client surface we need.""" + + def get(self, name: NameType) -> Optional[bytes]: ... + + def mget(self, names: Iterable[NameType]) -> List[Optional[bytes]]: ... + + def set(self, name: NameType, value: Any, *args: Any, **kwargs: Any) -> Any: ... + + def delete(self, *names: NameType) -> int: ... + + def ttl(self, name: NameType) -> int: ... + + def scan_iter(self, match: Optional[str] = None) -> Iterable[NameType]: ... + + +@dataclass(frozen=True) +class PlanCandidateRecord: + """Represents a single clickhouse_id-plan pair for update consideration.""" + + clickhouse_id: str + plan_id: str + session_timestamp: datetime + + @property + def serialized(self) -> str: + """Serialize to the clickhouse_id:plan_id format expected in Redis.""" + return f"{self.clickhouse_id}:{self.plan_id}" + + +class PlanCandidateBuffer: + """Stores per-plan candidate queues in Redis using comma-separated strings.""" + + def __init__( + self, + redis_client: RedisLike, + key_prefix: str, + ttl_seconds: int, + ) -> None: + self._redis = redis_client + self._key_prefix = key_prefix.strip() + self._ttl_seconds = ttl_seconds + + def _candidates_key(self, cached_plan_id: str) -> str: + if self._key_prefix: + return f"{self._key_prefix}:{cached_plan_id}" + return cached_plan_id + + def _oldest_ts_key(self, cached_plan_id: str) -> str: + return f"{self._candidates_key(cached_plan_id)}:oldest_ts" + + def _created_at_key(self, cached_plan_id: str) -> str: + return f"{self._candidates_key(cached_plan_id)}:created_at" + + def add_candidates( + self, cached_plan_id: str, records: Iterable[PlanCandidateRecord] + ) -> int: + """ + Add session-plan pairs for the given cached plan ID. + + Returns the total number of stored records after the operation. + """ + key = self._candidates_key(cached_plan_id) + existing_raw = self._redis.get(key) + is_new_buffer = existing_raw is None + entries: List[str] = [] + if existing_raw: + entries = [item for item in existing_raw.decode("utf-8").split(",") if item] + existing_set = set(entries) + + new_serialized = [] + min_timestamp: Optional[datetime] = None + for record in records: + serialized = record.serialized + if serialized in existing_set: + continue + existing_set.add(serialized) + new_serialized.append(serialized) + if min_timestamp is None or record.session_timestamp < min_timestamp: + min_timestamp = record.session_timestamp + + if not new_serialized: + if existing_raw: + self._redis.set(key, existing_raw.decode("utf-8"), ex=self._ttl_seconds) + return len(existing_set) + + combined = entries + new_serialized + payload = ",".join(combined) + self._redis.set(key, payload, ex=self._ttl_seconds) + + # Set created_at only when buffer is first created (not on subsequent adds) + if is_new_buffer: + self._set_created_at(cached_plan_id) + + if min_timestamp: + self._update_oldest_timestamp( + cached_plan_id=cached_plan_id, candidate_ts=min_timestamp + ) + return len(combined) + + def _update_oldest_timestamp( + self, cached_plan_id: str, candidate_ts: datetime + ) -> None: + key = self._oldest_ts_key(cached_plan_id) + existing_raw = self._redis.get(key) + raw_str: Optional[str] = None + if existing_raw: + raw_str = existing_raw.decode("utf-8") + try: + existing_dt = datetime.fromisoformat( + raw_str.replace("Z", "+00:00") + ).replace(tzinfo=None) + except ValueError: + existing_dt = None + else: + existing_dt = None + + if existing_dt and existing_dt <= candidate_ts and raw_str is not None: + self._redis.set(key, raw_str, ex=self._ttl_seconds) + return + + iso_value = candidate_ts.isoformat() + "Z" + self._redis.set(key, iso_value, ex=self._ttl_seconds) + + def _set_created_at(self, cached_plan_id: str) -> None: + """Set created_at timestamp when buffer is first created. Does NOT refresh on subsequent adds.""" + key = self._created_at_key(cached_plan_id) + iso_value = datetime.utcnow().isoformat() + "Z" + self._redis.set(key, iso_value, ex=self._ttl_seconds) + + def get_created_at(self, cached_plan_id: str) -> Optional[datetime]: + """Get the created_at timestamp for a plan's buffer.""" + raw = self._redis.get(self._created_at_key(cached_plan_id)) + if not raw: + return None + try: + return datetime.fromisoformat( + raw.decode("utf-8").replace("Z", "+00:00") + ).replace(tzinfo=None) + except ValueError: + return None + + def get_effective_created_at(self, cached_plan_id: str) -> Optional[datetime]: + """Get created_at with fallback to oldest_ts for legacy entries.""" + created_at = self.get_created_at(cached_plan_id) + if created_at is not None: + return created_at + + # Fallback to oldest_ts for legacy data + oldest_ts_str = self.get_oldest_timestamp(cached_plan_id) + if oldest_ts_str: + try: + return datetime.fromisoformat( + oldest_ts_str.replace("Z", "+00:00") + ).replace(tzinfo=None) + except ValueError: + pass + return None + + def get_candidates(self, cached_plan_id: str) -> List[str]: + raw = self._redis.get(self._candidates_key(cached_plan_id)) + if not raw: + return [] + return [item for item in raw.decode("utf-8").split(",") if item] + + def get_candidate_count(self, cached_plan_id: str) -> int: + return len(self.get_candidates(cached_plan_id)) + + def get_oldest_timestamp(self, cached_plan_id: str) -> Optional[str]: + raw = self._redis.get(self._oldest_ts_key(cached_plan_id)) + if not raw: + return None + return raw.decode("utf-8") + + def clear(self, cached_plan_id: str) -> None: + self._redis.delete( + self._candidates_key(cached_plan_id), + self._oldest_ts_key(cached_plan_id), + self._created_at_key(cached_plan_id), + ) + + def get_ttl(self, cached_plan_id: str) -> int: + """Get remaining TTL in seconds for a plan's candidates.""" + ttl = self._redis.ttl(self._candidates_key(cached_plan_id)) + return max(0, ttl) # Returns -2 if key doesn't exist, -1 if no TTL + + def get_all_plan_ids(self) -> List[str]: + """Get all cached plan IDs that have candidates in the buffer.""" + if self._key_prefix: + pattern = f"{self._key_prefix}:*" + prefix = f"{self._key_prefix}:" + else: + pattern = "*" + prefix = "" + + plan_ids: List[str] = [] + for key in self._redis.scan_iter(match=pattern): + key_str = key.decode("utf-8") if isinstance(key, bytes) else key + + # Skip metadata keys (oldest_ts and created_at are stored separately) + if key_str.endswith(":oldest_ts") or key_str.endswith(":created_at"): + continue + + if prefix: + if not key_str.startswith(prefix): + continue + candidate_token = key_str[len(prefix) :] + else: + candidate_token = key_str + + if not candidate_token: + continue + + if candidate_token.endswith(":candidates"): + candidate_token = candidate_token[: -len(":candidates")] + + if candidate_token: + plan_ids.append(candidate_token) + + return plan_ids + + def find_plans_past_age_threshold(self, age_threshold_hours: int = 24) -> List[str]: + """ + Find all plans whose buffer has existed longer than the age threshold. + + This ensures every plan gets processed within X hours of first being buffered, + regardless of how often new candidates are added. + + Args: + age_threshold_hours: Process plans whose buffer was created more than this many hours ago + + Returns: + List of cached_plan_ids that should be processed + """ + threshold_dt = datetime.utcnow() - timedelta(hours=age_threshold_hours) + aged_plans = [] + + for plan_id in self.get_all_plan_ids(): + created_at = self.get_effective_created_at(plan_id) + if created_at is None or created_at <= threshold_dt: + aged_plans.append(plan_id) + + return aged_plans diff --git a/app_tester/rube_learning/ingestion_pipeline/plan_consolidator.py b/app_tester/rube_learning/ingestion_pipeline/plan_consolidator.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/plan_consolidator.py @@ -0,0 +1,585 @@ +"""Plan consolidation module for merging multiple plans using LLM.""" + +import asyncio +import json +from typing import Dict, List, Optional + +from pydantic import BaseModel, Field + +from app_tester.rube_learning.ingestion_pipeline.ingestion_config import ( + UsecaseIngestionConfig, +) +from app_tester.rube_learning.ingestion_pipeline.ingestion_logger import ( + IngestionTraceLogger, +) +from app_tester.rube_learning.ingestion_pipeline.ingestion_stats_counter import ( + PlanConsolidationRecord, + record_plan_consolidation, +) +from app_tester.rube_learning.ingestion_pipeline.ingestion_utils import ( + RetryableOperation, +) + + +class WorkbenchCodeSnippet(BaseModel): + description: str = Field( + description="Description of the code snippet, eg 'Code snippet to parse large XYZ_TOOL response'" + ) + code: str = Field( + description="The python code snippet. Keep it concise, no comments or docstrings, only reference code" + ) + + +class ConsolidatedUsecasePlan(BaseModel): + """Schema for consolidated usecase plan output from LLM""" + + workflow_steps: List[str] = Field( + description="Key steps in workflow (happy path + fallback paths). Focus on tool sequences, dependencies, and pagination. Keep the consolidated list under 10 lines (~500-1000 characters) and emphasize the INTENT of each step, not exact parameters." + ) + complexity_assessment: str = Field( + description=( + "Difficulty level with very brief explanation. Format: 'LEVEL: explanation' where LEVEL is EASY, MEDIUM, or HARD (uppercase). " + "Follow with a colon and 1 sentence explanation. CRITICAL: Base complexity on the USE CASE INTENT itself, NOT on the detailed workflow steps. " + "The workflow steps include precautionary measures, error handling, and additional tools for robustness - ignore these when assessing complexity. " + "Focus on: What is the core user intent? How inherently complex is the use case? " + "EASY = simple, straightforward use case (e.g., 'send an email', 'get user info'); " + "MEDIUM = moderate complexity requiring coordination (e.g., 'create event with attendees', 'search and filter data'); " + "HARD = inherently complex multi-step use case (e.g., 'analyze data across multiple sources and generate report'). " + "Example: 'EASY: Simple email sending use case' (even if workflow includes verification steps)." + ) + ) + pitfalls: List[str] = Field( + description="Specific gotchas discovered. API limits (e.g., 'max 500 messages'), rate limits (e.g., '2 points/second'), tool-specific quirks. Don't repeat workflow_steps. Be specific with numbers, tool names, error codes." + ) + workbench_code_snippets: Optional[List[WorkbenchCodeSnippet]] = Field( + default=None, + description="Code snippets for workbench if present in source plans. Only include if highly relevant. Keep code anonymized and STRICTLY REMOVE PII.", + ) + final_plan_reasoning: str = Field( + description="Concise reasoning (2-3 sentences) explaining how plans were merged, what improvements were incorporated, and what learnings from new executions were added." + ) + changes_made: int = Field( + default=0, + description="Number of significant changes made vs the original cached plan (0=unchanged). Count: adding/removing workflow steps, changing complexity level, adding/removing pitfalls. If no cached plan, count total items as changes.", + ) + + +class PlanConsolidator: + """Handles LLM-based consolidation of multiple plans into a single best plan.""" + + def __init__( + self, + config: UsecaseIngestionConfig, + llm_client, + ingestion_service, # UsecasePlanIngestion instance + trace_logger: IngestionTraceLogger, + ingestion_id: str, + debug: bool = False, + ): + """ + Initialize the plan consolidator. + + Args: + config: Ingestion configuration + llm_client: OpenAI client for LLM calls + ingestion_service: TurboPuffer ingestion service for fetching plans + trace_logger: Logger for tracing operations + ingestion_id: Ingestion ID for logging + debug: Whether debug mode is enabled + """ + self.config = config + self.llm_client = llm_client + self.ingestion_service = ingestion_service + self.trace_logger = trace_logger + self.ingestion_id = ingestion_id + self.debug = debug + + @RetryableOperation.with_exponential_backoff( + max_attempts=3, min_wait=2, max_wait=10, retry_on=(Exception,) + ) + async def consolidate_plans_with_llm( + self, + cached_plan_id: str, + plans: List[Dict], + skip_original_fetch: bool = False, + consolidated_main_tools: Optional[List[str]] = None, + consolidated_related_tools: Optional[List[str]] = None, + cluster_id: Optional[str] = None, + allowed_toolkits: Optional[List[str]] = None, + tool_consolidation_reasoning: Optional[str] = None, + representative_usecase: Optional[str] = None, + toolkits_str: Optional[str] = None, + ) -> Optional[dict]: + """ + Use LLM to consolidate multiple plans into a single best plan. + + Args: + cached_plan_id: The cached plan ID these plans share + plans: List of plan dictionaries with context + skip_original_fetch: If True, skip fetching original plan from TurboPuffer + (useful for new clusters that don't exist yet) + consolidated_main_tools: Final list of main tools from tool consolidation + consolidated_related_tools: Final list of related tools from tool consolidation + cluster_id: Optional cluster identifier for Helicone telemetry + allowed_toolkits: Optional list of toolkit slugs that the final plan must stay within + tool_consolidation_reasoning: Reasoning from tool consolidation explaining why tools were included/excluded + representative_usecase: Optional representative usecase for the cluster (helps LLM understand overall intent) + + Returns: + Consolidated plan dictionary or None + """ + if not plans: + return None + + print( + f" 🤖 Consolidating {len(plans)} plans for cached_plan_id: {cached_plan_id}..." + ) + + # Fetch the original cached plan from TurboPuffer (if not skipped) + original_plan = None + original_plan_text = "" + original_plan_chars = 0 # For stats tracking + + if not skip_original_fetch: + original_plan = await asyncio.to_thread( + self.ingestion_service.get_plan_by_id, cached_plan_id + ) + + if original_plan: + print(" 📋 Found original cached plan in TurboPuffer") + # Extract the plan content + original_plan_content = original_plan.get("plan", "") + original_usecase = original_plan.get("usecase", "N/A") + original_known_fields = original_plan.get("known_fields", "") + + known_fields_section = "" + if original_known_fields: + known_fields_section = f"Known Fields: {original_known_fields}\n" + + if original_plan_content: + # Format plan content as JSON if it's a dict, otherwise use as-is + if isinstance(original_plan_content, dict): + formatted_plan_content = json.dumps( + original_plan_content, indent=2 + ) + original_plan_chars = len(formatted_plan_content) + else: + formatted_plan_content = original_plan_content + original_plan_chars = len(str(original_plan_content)) + original_plan_text = f""" +## Original Cached Plan (ID: {cached_plan_id}) +Use Case: {original_usecase} +{known_fields_section}#### Plan Details +{formatted_plan_content} + +This is the original plan that was cached. Your consolidated output should stay close to this structure +and intent while incorporating improvements from the new execution plans below. +""" + else: + print(" ⚠️ Could not fetch original plan from TurboPuffer") + + # Build prompt with all plans + plans_text = [] + for idx, plan_with_context in enumerate(plans): + plan = plan_with_context["plan"] + usecase = plan_with_context["usecase"] + known_fields = plan_with_context.get("known_fields", "") + + known_fields_section = "" + if known_fields: + known_fields_section = f"#### Known Fields\n{known_fields}\n" + + # Rename 'pitfalls' to 'observed_pitfalls' in the prompt to signal + # these are session observations, not exhaustive/definitive pitfalls + plan_for_prompt = dict(plan) + if "pitfalls" in plan_for_prompt: + plan_for_prompt["observed_pitfalls"] = plan_for_prompt.pop("pitfalls") + + plans_text.append( + f""" +### Plan {idx + 1} (real execution based): {usecase} +{known_fields_section}#### Plan Details +{json.dumps(plan_for_prompt, indent=2)} +""" + ) + + # Build tool filtering section if tools are provided + tool_filtering_section = "" + if consolidated_main_tools or consolidated_related_tools: + all_allowed_tools = [] + if consolidated_main_tools: + all_allowed_tools.extend(consolidated_main_tools) + if consolidated_related_tools: + all_allowed_tools.extend(consolidated_related_tools) + + toolkit_filtering_note = "" + if allowed_toolkits: + toolkit_names = ", ".join(sorted(allowed_toolkits)) + toolkit_filtering_note = f""" +### Toolkit Filtering Rule (for _specific plans) +- This plan is toolkit-specific and MUST ONLY mention tools from these toolkits: {toolkit_names} +- If any workflow_step or pitfall references a tool outside this set, remove or rewrite that step +- After drafting the plan, VERIFY every tool belongs to one of: {toolkit_names} +""" + + tool_consolidation_context = "" + if tool_consolidation_reasoning: + tool_consolidation_context = f""" +### Tool Consolidation Context +{tool_consolidation_reasoning} + +This reasoning explains why certain tools were included or excluded during the consolidation process. +Consider this context when filtering or adjusting plan steps. +""" + + tool_filtering_section = f""" +## Final Consolidated Tools +Main Tools: {', '.join(consolidated_main_tools) if consolidated_main_tools else 'None'} +Related Tools: {', '.join(consolidated_related_tools) if consolidated_related_tools else 'None'} +{toolkit_filtering_note} +{tool_consolidation_context} +## CRITICAL Tool Filtering Rules (Must Be Strictly Enforced) +- The consolidated plan MUST ONLY mention tools from the above list +- SCAN every workflow_step and pitfall for tool names (UPPERCASE_WITH_UNDERSCORES format) +- ALWAYS use FULL tool names - never use shorthand or truncated versions (e.g., use TELEGRAM_GET_CHAT not GET_CHAT) +- If you find ANY tool name NOT in the consolidated tools list above, you MUST: + 1. REMOVE that specific tool mention from the text, OR + 2. REMOVE the entire step/line if it depends on that tool +- DO NOT make up new tools or add tools not in the final list +- DO NOT assume a tool exists if it's not explicitly listed above +- After generating the plan, VERIFY that every tool mentioned exists in the consolidated tools list +- If removing a step breaks the workflow, adjust adjacent steps to maintain coherence +- EXCEPTION: COMPOSIO_REMOTE_WORKBENCH is ALWAYS ALLOWED - This is a Composio meta tool used for remote workbench execution when normal toolkits cannot perform certain tasks. Always preserve references to COMPOSIO_REMOTE_WORKBENCH in workflow steps even if it's not explicitly in the consolidated tools list above. + +EXAMPLES OF CORRECT FILTERING: +- Original step: "If needed, use TOOL_A or TOOL_B for backup" +- If TOOL_A is in list but TOOL_B is not → Change to: "If needed, use TOOL_A for backup" +- If neither is in list → REMOVE the entire step + +- Original step: "Poll via TOOL_C until complete" +- If TOOL_C is not in list → REMOVE the entire step AND adjust surrounding steps + +- Original step: "Use COMPOSIO_REMOTE_WORKBENCH to process large dataset" +- ALWAYS PRESERVE this step - COMPOSIO_REMOTE_WORKBENCH is always allowed +""" + + # Prepare reusable sections + original_plan_section = ( + original_plan_text + if original_plan_text.strip() + else "## Original Cached Plan\nNo cached plan available - rely on the execution plans below." + ) + + # Create instruction blocks based on whether consolidating multiple plans or filtering single plan + if len(plans) > 1: + # BLOCK 1: Consolidating multiple plans + approach_instructions = """## Consolidation Approach +1. Start with the CACHED PLAN's structure and core workflow as the foundation (60-70% of final plan) +2. Enhance it with valuable improvements and learnings from new plans (30-40% of final plan) +3. Keep the final plan coherent end-to-end so every step flows naturally +4. Remove redundant or duplicate information instead of repeating similar steps +5. Drop implementation-specific details that vary between plans (session IDs, timestamps, temp data) +6. Preserve workflow steps, metadata, and any high-confidence reasoning +7. Choose the highest quality/confidence values when plans disagree""" + + cluster_usecase_section = "" + if representative_usecase: + cluster_usecase_section = f"""## Cluster Representative Use Case +{representative_usecase} + +This represents the overall intent of the cluster. Use it as the lens when consolidating every plan. + +### Complexity Reminder +When determining complexity_assessment, focus on the use case intent. A simple use case like "send an email" should be EASY even if the workflow steps include verification, error handling, or pagination—those are implementation details, not inherent complexity.""" + + intro_text = f"""## Overview +You are consolidating multiple AI workflow plans that share a similar use case into a single, comprehensive, high-quality plan. + +{original_plan_section} + +{cluster_usecase_section} + +## Execution Inputs +You have {len(plans)} execution plans below. Your task is to create ONE consolidated plan. + +## Weightage Distribution +- CACHED PLAN (if provided): 60-70% weight — primary template and baseline +- NEW EXECUTION PLANS: 30-40% weight — improvements, learnings, nuances (these are observed session data, not exhaustive documentation) + +{tool_filtering_section} + +{approach_instructions}""" + else: + # BLOCK 2: Filtering single plan + approach_instructions = """## Filtering Approach +1. Start with the execution plan as the baseline +2. Filter the plan to only include steps referencing tools in the consolidated list +3. Keep the final plan coherent end-to-end after filtering +4. Adjust adjacent steps if removals would otherwise break the flow +5. Remove redundant or duplicate information +6. Drop implementation-specific details that vary between plans +7. Preserve workflow steps, metadata, and the plan's quality/confidence indicators""" + + cluster_usecase_section_filter = "" + if representative_usecase: + cluster_usecase_section_filter = f"""## Cluster Representative Use Case +{representative_usecase} + +This represents the overall intent of the cluster. Use it as guidance when filtering the plan below. + +### Complexity Reminder +When determining complexity_assessment, focus on the use case intent. A simple use case like "send an email" should be EASY even if the workflow steps include verification, error handling, or pagination. These are implementation details, not complexity indicators.""" + + intro_text = f"""## Overview +You are filtering a single AI workflow plan for this use case into a concise, high-quality plan. + +{original_plan_section} + +{cluster_usecase_section_filter} + +## Execution Inputs +You have {len(plans)} execution plan below. Your task is to create ONE filtered plan. + +## Weightage Distribution +- CACHED PLAN (if provided): 60-70% weight — primary baseline +- NEW EXECUTION PLANS: 30-40% weight — improvements, learnings, nuances (these are observed session data, not exhaustive documentation) + +{tool_filtering_section} + +{approach_instructions}""" + + # Common sections (same for both blocks) + has_cached_plan = bool(original_plan_text.strip()) + cached_rules = ( + """- CACHED PLAN is the PRIMARY template (60-70% weight) - stay very close to its structure and intent +- New plans contribute enhancements, edge cases, and improvements (30-40% weight) +- The consolidated plan should differ by NO MORE THAN 20-30% from the cached plan +- DO NOT remove important workflow steps or tool information from the cached plan +- DO NOT remove pitfalls or workflow steps from the cached plan just because they don't appear in new plans—the new plans are session observations, and absence may mean the guidance was followed successfully +""" + if has_cached_plan + else "" + ) + + prompt = f"""{intro_text} + +## Critical Rules +{cached_rules}- The final plan MUST make sense end-to-end as a coherent workflow (not a franken-plan) +- DO NOT oversimplify - keep the plan comprehensive +- DO keep only the tools/steps that add value. You do NOT need to force every tool from the source plans into the final workflow. HOWEVER, any tool that remains in the consolidated plan MUST come from the consolidated tool list / allowed_toolkits. If a tool falls outside that list, remove or rewrite the step. +- EXCEPTION: KNOWN FIELDS - If "Known Fields" are provided with a plan (e.g., "channel_id: C123"), those values are already available. In this case, it is fine to REMOVE workflow steps that fetch/obtain those fields, even if those tools appear in the consolidated tools list. +- DO preserve success confidence, difficulty, and quality indicators +- DO keep error handling and edge case information +- DO merge similar steps intelligently rather than duplicating them +- DO incorporate improvements and learnings from the new execution plans (don't replace existing content unless majority of new plans explicitly contradict it with reasoning) +- DO NOT include session-specific IDs or timestamps in the consolidated plan +- Keep the consolidated plan under ~10 lines and roughly 500-1000 characters (only exceed for inherently hard workflows) +- ALWAYS use FULL tool names (e.g., TELEGRAM_GET_CHAT, not GET_CHAT; VERCEL_LIST_ENV_VARIABLES, not LIST_ENV_VARIABLES) +""" + if allowed_toolkits: + toolkit_names = ", ".join(sorted(allowed_toolkits)) + prompt += f""" +### Toolkit Scope Enforcement +- ONLY use tools from these toolkits: {toolkit_names} +- REMOVE or rewrite any step that references tools outside this toolkit set +- EXCEPTION: COMPOSIO_REMOTE_WORKBENCH is ALWAYS ALLOWED - This is a Composio meta tool used for remote workbench execution when normal toolkits cannot perform certain tasks. Always preserve references to COMPOSIO_REMOTE_WORKBENCH even if it's outside the toolkit scope. +""" + + plans_section = ( + "These execution plans belong to the same cluster for the representative use case. " + "Analyze each one to understand how the tools and steps support the shared workflow.\n" + + "".join(plans_text) + ) + + prompt += f""" + +## Plan Design Guidelines + +- Surface a reusable workflow future users can follow, highlighting real execution learnings. +- Keep plans timeless—no session-specific IDs, dates, or PII. + +### Execution Plan +- Describe the tool sequence (happy path + fallback) focusing on intent, not raw parameters. +- Keep fallbacks concise: include at most one fallback per action unless multiple are absolutely necessary. +- Mention required inputs/outputs, pagination handling, and UTC/timezone needs. +- Call out user confirmation before irreversible/public-impact actions. +- Highlight output verification expectations (clickable links, inline references, validation steps). +- Include non-obvious details or schema quirks discovered in execution. +- Explicitly note any Known Fields or cached IDs that removed the need for extra lookup steps. + +### Data Volume & Performance Patterns +- Document concrete API caps, throttling behavior, Retry-After requirements, and why pagination/batching choices were made. +- Mention performance implications (e.g., “bulk run took 12 minutes due to 1.5s delays”). + +### Tool Sequencing & Dependencies +- Explain non-obvious ordering, cross-app dependencies, and data transformations. +- Call out opportunities where fewer tool calls would have sufficed. + +### Existing Plan Analysis +- If a cached plan existed, explain how the final plan supersedes it while preserving critical elements. +- Mention where execution deviated due to real-world constraints or new learnings. + +### Error Patterns & Recovery +- Capture failure modes, error codes, validation issues, and proven retry/backoff strategies. + +### Workarounds & Missing Capabilities +- Note when proxy_execute or other creative techniques were needed because a tool was missing/insufficient. + +### Critical Realizations +- List incorrect assumptions that changed, undocumented tool behaviors, or insights that determined success vs failure. + +### Workbench Hints +- Workbench is only Required when large data volumes or heavy scripting made it necessary; otherwise mark it Optional. +- When referencing workbench, stress good hygiene (parallel execution, chunking, checkpoints, print statements). +- Preserve concrete workbench learnings (e.g., parallelism gains, debugging tactics, data-shape gotchas) as optional guidance. + +## Formatting & Output Rules + +- EVERY workflow step MUST follow: "[Required/Optional (condition)] [Prerequisite/Step/Next Step/Fallback]: Action using TOOL_X (purpose)". + * Mention real tool slugs (UPPERCASE_WITH_UNDERSCORES). + * For Optional steps, state WHY it's optional - not just "Optional" but "Optional (if X unknown)" or "Optional (for write operations only)". + * Don't add unnecessary "Required" steps; prefer the shortest reliable workflow. + * Assume app connections already exist—don't add steps to reconnect unless execution proved it was needed. + * If workbench is mentioned, label it Optional unless the session clearly required it for huge datasets. + * Limit fallbacks: For any given action, include at most ONE fallback step unless multiple fallbacks are **absolutely necessary** for robustness. + * Examples: "Required Prerequisite (if project_id unknown): Fetch project ID using LINEAR_LIST_PROJECTS", "Required Step 1: Create draft using GMAIL_CREATE_DRAFT", "Optional Fallback (if FIRECRAWL_SEARCH returns 402): Use COMPOSIO_SEARCH_WEB as fallback", "Optional Next Step (for verification): Fetch email draft using GMAIL_GET_DRAFT". +- Pitfalls must follow "[TOOL_NAME] description" format (e.g., "[GMAIL_BATCH_MODIFY] max 1000 IDs per call", "[SLACK_SEARCH_MESSAGES] 429 after ~100 requests/minute on free tier"). + * BAD (generic): "Rate limit: throttle requests", "Response handling: parse carefully" + * GOOD (specific): "[DATADOG_LIST_LOGS] InvalidArgument when start_at exceeds window - omit pagination", "[SHOPIFY_GET_PRODUCTS] 3 points per call, 2 points/sec limit - add 1.5s delays" + * Numbers beat words: say "max 1000 IDs" not "batch appropriately". Include actual error codes, batch sizes, data paths, or behavioral quirks discovered. + * Pitfalls must not duplicate workflow steps. +- Focus ONLY on app tools; never mention meta tools (search, planner, multi_execute, manage_connections, etc.). +- Do NOT describe what “the planner” or “the agent” did—state what needs to happen. Avoid implementation details like file paths or sync_response_to_workbench. +- ALWAYS base instructions on actual cached/new plans—no hallucinated tools, parameters, or behaviors. +- Keep plans UNDER 10 lines and ~500-1000 characters (only exceed for inherently hard, multi-app workflows). +- State API quirks, data structures, rate limits, cross-app dependencies, and other non-obvious requirements explicitly. +- Use concise, concrete language with real tool names, error codes, and data sizes. Mention each insight only once (either in workflow_steps or pitfalls, not both). + +## Plans To Consolidate +{plans_section} + +## OUTPUT REQUIREMENTS: +You will return a structured JSON object with these fields: workflow_steps, complexity_assessment, pitfalls, workbench_code_snippets (if present in source plans), and final_plan_reasoning. + +CRITICAL requirements: +- Match the ORIGINAL CACHED PLAN's structure as closely as possible (differ by max 20%) if it exists, otherwise follow the new plans structure. +- workflow_steps: Merge all unique steps from original + new plans while keeping the final list under 10 lines (~500-1000 characters). + Every step MUST use the format "[Required/Optional (condition)] [Prerequisite/Step/Next Step/Fallback]: Action using TOOL_X (purpose)". + For Optional steps, state WHY it's optional (e.g., "Optional (if project_id unknown)"). + Cite exact tool slugs, avoid meta tools, and only describe actions grounded in the provided plans. Mark workbench as Optional unless massive data volumes made it mandatory. +- complexity_assessment: Format "LEVEL: explanation" where LEVEL is EASY/MEDIUM/HARD followed by colon and 1-2 sentence reason. + CRITICAL: Base complexity on the USE CASE INTENT, NOT on the workflow steps. The workflow steps include precautionary measures, + error handling, pagination, and additional tools for robustness - these should NOT inflate the complexity. + Ask yourself: "What is the core user intent? How inherently complex is the use case itself?" + Examples: "send an email" = EASY (even if workflow includes verification), + "create calendar event" = EASY/MEDIUM (depending on if attendees are known), + "analyze data across multiple APIs and generate report" = HARD. + Do NOT rate MEDIUM just because the workflow includes error handling or pagination - those are implementation details, not complexity indicators. +- pitfalls: Provide 3-5 concise entries formatted as "[TOOL_NAME] description" (e.g., "[GMAIL_BATCH_MODIFY] max 1000 IDs per call", "[SLACK_SEARCH_MESSAGES] 429 after ~100 requests/minute on free tier"). Avoid duplicating workflow steps; highlight gotchas, limits, validation errors, pagination quirks, or auth nuances with concrete numbers/tool names. +- workbench_code_snippets: If ANY source plan contains workbench_code_snippets, preserve them in the consolidated output - order by important/relevant snippets first. Only include highly relevant snippets that show code structure, fields, etc. Keep code anonymized and STRICTLY REMOVE PII. If no source plans have workbench_code_snippets, omit this field. +- final_plan_reasoning: Concise reasoning (2-3 sentences) explaining how plans were merged, what improvements were incorporated, and what learnings from new executions were added. +- Keep total plan length under ~500-1000 characters and ≤10 lines whenever possible; exceed only for inherently hard, multi-app workflows. +- Preserve all important details from the original plan—pitfalls should be retained even if not observed in new plans, unless majority of plans' reasoning mentions the pitfall is no longer relevant. Workflow steps should be retained and any new approaches incorporated or existing approaches amended if a new workflow is observed in majority of plans. +- Incorporate improvements discovered in new executions +- Be SPECIFIC with concrete numbers, tool names, error codes, and cite the source plan (cached vs Plan #N) when relevant +- Don't repeat between workflow_steps and pitfalls +- Focus on gaps between expectation and reality—highlight hidden constraints, rate limits, or dependencies that surfaced during execution. +- IMPORTANT: NEVER include PII or user-specific identifiers in the consolidated plan. If it was already there, remove it. Keep guidance timeless and avoid referencing meta tools or internal implementation details. +- Note: The final plan must be self-contained. Do not reference or mention other plans (e.g., 'from plan 1+3').""" + + try: + # Prepare Helicone headers for this specific plan consolidation request + consolidation_type = "multiple" if len(plans) > 1 else "single" + cluster_id_value = "" + if cluster_id is not None: + cluster_id_value = str(cluster_id).strip() + + helicone_user_id = cluster_id_value or str(cached_plan_id) + helicone_headers = { + "Helicone-Session-Id": f"ingestion-{self.ingestion_id}", + "Helicone-User-Id": helicone_user_id, + "Helicone-Prompt-Id": "plan-consolidation", + "Helicone-Property-Cached-Plan-Id": str(cached_plan_id), + "Helicone-Property-Ingestion-Id": str(self.ingestion_id), + "Helicone-Property-Operation": "plan-consolidation", + "Helicone-Property-Consolidation-Type": consolidation_type, + "Helicone-Property-Input-Plans-Count": str(len(plans)), + } + if cluster_id_value: + helicone_headers["Helicone-Property-Cluster-Id"] = cluster_id_value + + # Add consolidated tools info if available + if consolidated_main_tools: + helicone_headers["Helicone-Property-Main-Tools-Count"] = str( + len(consolidated_main_tools) + ) + if consolidated_related_tools: + helicone_headers["Helicone-Property-Related-Tools-Count"] = str( + len(consolidated_related_tools) + ) + + completion = await self.llm_client.beta.chat.completions.parse( + model="gpt-5.2", + messages=[ + { + "role": "system", + "content": "You are an expert at consolidating AI workflow plans. You preserve all important information while removing redundancy. You maintain consistency with original plan design, keep plans concise (≤10 lines, roughly 500-1000 chars), and ensure the consolidated plan differs by no more than 20% from the original cached plan. Think carefully!", + }, + {"role": "user", "content": prompt}, + ], + reasoning_effort="medium", + response_format=ConsolidatedUsecasePlan, + extra_headers=helicone_headers, + ) + + if completion.choices and completion.choices[0].message.parsed: + consolidated_plan_obj = completion.choices[0].message.parsed + # Convert Pydantic model to dict + consolidated_plan = consolidated_plan_obj.model_dump() + print(f" ✅ Successfully consolidated {len(plans)} plans") + + # Display final plan reasoning in debug mode + if self.debug and "final_plan_reasoning" in consolidated_plan: + reasoning = consolidated_plan["final_plan_reasoning"] + if reasoning: + print(" 📝 Final Plan Reasoning:") + print(f" {reasoning}") + + # Log the plan consolidation + consolidation_type = "multiple" if len(plans) > 1 else "single" + self.trace_logger.log_plan_consolidation( + cluster_id=None, # Will be set by caller if needed + cached_plan_id=cached_plan_id, + input_plans=plans, + output_plan=consolidated_plan, + consolidation_type=consolidation_type, + ) + + # Record plan consolidation stats + consolidated_plan_chars = len(json.dumps(consolidated_plan)) + changes_made = consolidated_plan.get("changes_made", 0) + record_plan_consolidation( + PlanConsolidationRecord( + plan_id=str(cached_plan_id), + ingestion_id=str(self.ingestion_id), + toolkits_str=toolkits_str or "", + chars_before=original_plan_chars, + chars_after=consolidated_plan_chars, + is_new=skip_original_fetch, + changes_made=changes_made, + ) + ) + + return consolidated_plan + else: + print(" ❌ LLM did not return valid plan consolidation") + # Return None to skip this cluster (callers will handle None) + return None + + except Exception as e: + print(f" ❌ Plan consolidation failed: {e}") + import traceback + + traceback.print_exc() + # Return None to skip this cluster (callers will handle None) + return None diff --git a/app_tester/rube_learning/ingestion_pipeline/plan_update_policy.py b/app_tester/rube_learning/ingestion_pipeline/plan_update_policy.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/plan_update_policy.py @@ -0,0 +1,91 @@ +""" +Centralized policy checks for deciding when to refresh a cached plan. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta +from math import ceil +from typing import Optional, Tuple + + +def _parse_timestamp(value: Optional[str]) -> Optional[datetime]: + if not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).replace(tzinfo=None) + except ValueError: + return None + + +@dataclass +class PlanUpdateInputs: + candidate_count: int + plan_updated_at: Optional[str] + update_counter: Optional[int] + now: datetime + mature_min_age: timedelta + early_threshold: int + mature_threshold: int + mature_threshold_growth_rate: float = 1.0 + max_mature_threshold: Optional[int] = None + + +MATURITY_COUNTER_THRESHOLD = 5 + + +def compute_candidate_threshold( + *, + update_counter: int, + early_threshold: int, + mature_threshold: int, + mature_threshold_growth_rate: float = 1.0, + max_mature_threshold: Optional[int] = None, +) -> Tuple[int, bool]: + """ + Determine the candidate threshold and whether the plan is in the mature phase. + + Returns: + Tuple of (required_candidates, is_mature_phase) + """ + if update_counter <= MATURITY_COUNTER_THRESHOLD: + return early_threshold, False + + growth_rate = max(1.0, mature_threshold_growth_rate) + exponent = update_counter - MATURITY_COUNTER_THRESHOLD + scaled_threshold = ceil(mature_threshold * (growth_rate**exponent)) + + if max_mature_threshold is not None: + scaled_threshold = min(max_mature_threshold, scaled_threshold) + + return max(mature_threshold, scaled_threshold), True + + +def should_update_plan(inputs: PlanUpdateInputs) -> Tuple[bool, str]: + """ + Determine whether a cached plan should be refreshed based on queued data. + + Only the candidate count (N) requirement is enforced. The time-based + requirement (mature_min_age) has been removed to allow faster ingestion. + """ + update_counter = inputs.update_counter or 0 + + required_candidates, is_mature = compute_candidate_threshold( + update_counter=update_counter, + early_threshold=inputs.early_threshold, + mature_threshold=inputs.mature_threshold, + mature_threshold_growth_rate=inputs.mature_threshold_growth_rate, + max_mature_threshold=inputs.max_mature_threshold, + ) + stage = "mature" if is_mature else "early" + + if inputs.candidate_count < required_candidates: + return False, ( + f"{stage} candidates={inputs.candidate_count} " f"< {required_candidates}" + ) + + # Candidate threshold met - allow update regardless of time since last update + return True, ( + f"{stage} candidates={inputs.candidate_count} " f">= {required_candidates}" + ) diff --git a/app_tester/rube_learning/ingestion_pipeline/processed_session_tracker.py b/app_tester/rube_learning/ingestion_pipeline/processed_session_tracker.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/processed_session_tracker.py @@ -0,0 +1,73 @@ +"""Utilities for tracking which ClickHouse sessions have been ingested.""" + +from __future__ import annotations + +from typing import Iterable, Set + +from app_tester.rube_learning.ingestion_pipeline.plan_candidate_buffer import RedisLike + + +class ProcessedSessionTracker: + """Tracks processed ClickHouse session IDs in Redis.""" + + def __init__( + self, redis_client: RedisLike, key_prefix: str = "ingestion_session_id" + ) -> None: + self._redis = redis_client + self._key_prefix = (key_prefix or "").strip() + + def _normalize(self, clickhouse_id: object) -> str | None: + if clickhouse_id is None: + return None + normalized = str(clickhouse_id).strip() + if not normalized: + return None + return normalized + + def _build_key(self, normalized_id: str) -> str: + if self._key_prefix: + return f"{self._key_prefix}:{normalized_id}" + return normalized_id + + def get_processed_ids(self, candidate_ids: Iterable[object]) -> Set[str]: + """Return the subset of candidate IDs that already exist in Redis.""" + normalized_ids: list[str] = [] + seen: set[str] = set() + for raw_id in candidate_ids: + normalized = self._normalize(raw_id) + if not normalized or normalized in seen: + continue + seen.add(normalized) + normalized_ids.append(normalized) + + if not normalized_ids: + return set() + + keys = [self._build_key(cid) for cid in normalized_ids] + values = self._redis.mget(keys) + processed: set[str] = { + cid + for cid, value in zip(normalized_ids, values) + if value not in (None, b"", "") + } + return processed + + def mark_processed(self, clickhouse_ids: Iterable[object]) -> int: + """Mark the provided ClickHouse IDs as processed.""" + stored = 0 + for raw_id in clickhouse_ids: + normalized = self._normalize(raw_id) + if not normalized: + continue + key = self._build_key(normalized) + self._redis.set(key, "1") + stored += 1 + return stored + + def is_processed(self, clickhouse_id: object) -> bool: + """Check whether the provided ClickHouse ID was already ingested.""" + normalized = self._normalize(clickhouse_id) + if not normalized: + return False + value = self._redis.get(self._build_key(normalized)) + return bool(value) diff --git a/app_tester/rube_learning/ingestion_pipeline/retryable_operations.py b/app_tester/rube_learning/ingestion_pipeline/retryable_operations.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/retryable_operations.py @@ -0,0 +1,102 @@ +""" +Retryable operations and utilities for the usecase ingestion pipeline. + +Core helper functions used across the ingestion pipeline. +Includes retry decorators and utility functions that will be refactored +into more appropriate modules in the future. +""" + +import hashlib +from itertools import islice +from typing import Iterator, List, TypeVar + +from tenacity import ( + retry, + retry_if_exception_type, + stop_after_attempt, + wait_exponential, +) + + +T = TypeVar("T") + + +def filter_plans_by_confidence(plans: List, min_confidence: float) -> List: + """ + Filter plans by minimum confidence threshold. + + Args: + plans: List of plans with success_confidence attribute + min_confidence: Minimum confidence threshold (0-10 scale) + + Returns: + Filtered list of plans meeting confidence criteria + """ + return [p for p in plans if p.success_confidence >= min_confidence] + + +def chunked(iterable: Iterator[T], n: int) -> Iterator[List[T]]: + """ + Split iterable into chunks of size n. + + Args: + iterable: Any iterable to chunk + n: Size of each chunk + + Yields: + Lists of size n (last chunk may be smaller) + """ + it = iter(iterable) + while chunk := list(islice(it, n)): + yield chunk + + +def generate_plan_id(usecase: str, is_specific: bool = False) -> str: + """ + Generate a consistent plan ID from usecase text. + + Args: + usecase: The usecase text + is_specific: Whether this is a specific variant + + Returns: + 16-character hex ID with optional _specific suffix + """ + normalized_usecase = usecase.lower().strip() + hash_obj = hashlib.sha256(normalized_usecase.encode()) + base_id = hash_obj.hexdigest()[:16] + + if is_specific: + return f"{base_id}_specific" + return base_id + + +class RetryableOperation: + """ + Decorator for retryable operations with exponential backoff. + """ + + @staticmethod + def with_exponential_backoff( + max_attempts: int = 3, + min_wait: int = 2, + max_wait: int = 10, + retry_on: tuple = (Exception,), + ): + """ + Create a retry decorator with exponential backoff. + + Args: + max_attempts: Maximum number of attempts + min_wait: Minimum wait time in seconds + max_wait: Maximum wait time in seconds + retry_on: Tuple of exception types to retry on + + Returns: + Configured retry decorator + """ + return retry( + stop=stop_after_attempt(max_attempts), + wait=wait_exponential(multiplier=1, min=min_wait, max=max_wait), + retry=retry_if_exception_type(retry_on), + ) diff --git a/app_tester/rube_learning/ingestion_pipeline/tests/__init__.py b/app_tester/rube_learning/ingestion_pipeline/tests/__init__.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for the ingestion pipeline service.""" diff --git a/app_tester/rube_learning/ingestion_pipeline/tests/fakes.py b/app_tester/rube_learning/ingestion_pipeline/tests/fakes.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/tests/fakes.py @@ -0,0 +1,88 @@ +"""Test doubles shared across ingestion pipeline tests.""" + +from __future__ import annotations + +import fnmatch +import time +from typing import Any, Dict, Iterable, Optional, Tuple + +from app_tester.rube_learning.ingestion_pipeline.plan_candidate_buffer import NameType + + +class FakeRedis: + """Minimal in-memory Redis replacement for unit tests.""" + + def __init__(self) -> None: + self._store: Dict[str, Tuple[str, Optional[float]]] = {} + + def _normalize_key(self, name: NameType) -> str: + return name.decode("utf-8") if isinstance(name, bytes) else name + + def _purge_expired(self, name: Optional[str] = None) -> None: + now = time.time() + keys: Iterable[str] + if name is None: + keys = list(self._store.keys()) + else: + keys = [name] + for key in list(keys): + value = self._store.get(key) + if not value: + continue + _, expiry = value + if expiry is not None and expiry <= now: + self._store.pop(key, None) + + def get(self, name: NameType) -> Optional[bytes]: + norm_name = self._normalize_key(name) + self._purge_expired(norm_name) + if norm_name not in self._store: + return None + value, _ = self._store[norm_name] + return value.encode("utf-8") + + def mget(self, names: Iterable[NameType]) -> list[Optional[bytes]]: + return [self.get(name) for name in names] + + def set(self, name: NameType, value: Any, *args: Any, **kwargs: Any) -> bool: + norm_name = self._normalize_key(name) + ex = kwargs.get("ex") + value_str = value.decode("utf-8") if isinstance(value, bytes) else str(value) + expiry = time.time() + ex if ex else None + self._store[norm_name] = (value_str, expiry) + return True + + def delete(self, *names: NameType) -> int: + removed = 0 + for raw_name in names: + name = self._normalize_key(raw_name) + if name in self._store: + removed += 1 + self._store.pop(name, None) + return removed + + def ttl(self, name: NameType) -> int: + norm_name = self._normalize_key(name) + self._purge_expired(norm_name) + if norm_name not in self._store: + return -2 + _, expiry = self._store[norm_name] + if expiry is None: + return -1 + remaining = int(expiry - time.time()) + return remaining if remaining >= 0 else 0 + + def scan_iter(self, match: Optional[str] = None) -> Iterable[NameType]: + self._purge_expired() + pattern = match or "*" + for key in list(self._store.keys()): + if fnmatch.fnmatch(key, pattern): + yield key + + # Test helper (not part of Redis API) + def force_ttl(self, name: NameType, ttl_seconds: int) -> None: + norm_name = self._normalize_key(name) + if norm_name not in self._store: + return + value, _ = self._store[norm_name] + self._store[norm_name] = (value, time.time() + ttl_seconds) diff --git a/app_tester/rube_learning/ingestion_pipeline/tests/test_cached_plan_integration.py b/app_tester/rube_learning/ingestion_pipeline/tests/test_cached_plan_integration.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/tests/test_cached_plan_integration.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +""" +Integration test for cached plan processing workflow. + +Tests the complete workflow for processing sessions with cached plans: +1. Flattening sessions to plans +2. Separating cached vs new plans +3. Processing cached plans (grouped by cached_plan_id) +""" + +import asyncio +import json + +import pytest + +from app_tester.rube_learning.ingestion_pipeline.ingestion_config import ( + UsecaseIngestionConfig, +) +from app_tester.rube_learning.ingestion_pipeline.usecase_ingestion_service import ( + UsecaseIngestionService, +) + + +@pytest.mark.asyncio +async def test_process_cached_sessions(): + """Test processing sessions with cached plans""" + config = UsecaseIngestionConfig( + namespace="rube_usecase_plans_test", + debug=True, + dry_run=True, + ) + service = UsecaseIngestionService(config=config) + + # Create mock sessions with cached plan IDs (using correct field structure for v2 tables) + mock_sessions = [ + { + "client_session_id": "session1", + "workflow_category": "email", + "task_difficulty": "MEDIUM", + "search_json": json.dumps( + [ + { + "search_id": "usecase_1_1", + "cached_plan_id": "plan_abc123", + "response": { + "main_tools": ["GMAIL_SEND_EMAIL"], + "related_tools": [], + "toolkits": ["gmail"], + }, + } + ] + ), + "per_usecase_plans_json": json.dumps( + [ + { + "cleaned_search_usecase": "Send email via Gmail", + "search_id": "usecase_1_1", + "cleaned_known_fields": "", + "usecase_plan": json.dumps( + {"workflow_steps": ["step1", "step2"]} + ), + "toolkits_required": ["gmail"], + "success_confidence": 8.5, + "search_log_id": "log_1", + } + ] + ), + }, + { + "client_session_id": "session2", + "workflow_category": "email", + "task_difficulty": "MEDIUM", + "search_json": json.dumps( + [ + { + "search_id": "usecase_2_1", + "cached_plan_id": "plan_abc123", # Same cached ID as session1 + "response": { + "main_tools": ["GMAIL_SEND_EMAIL"], + "related_tools": ["GMAIL_GET_USER"], + "toolkits": ["gmail"], + }, + } + ] + ), + "per_usecase_plans_json": json.dumps( + [ + { + "cleaned_search_usecase": "Send email notification", + "search_id": "usecase_2_1", + "cleaned_known_fields": "", + "usecase_plan": json.dumps( + { + "workflow_steps": [ + "Initialize Gmail client with appropriate scopes", + "Gather notification content from upstream service", + "Generate personalized subject and body segments", + "Queue email for review and validation", + "Dispatch notification and record delivery result", + ], + "pitfalls": [ + "Ensure upstream payloads include required metadata", + "Throttle notifications to avoid API quota issues", + "Log delivery status for downstream auditing", + ], + } + ), + "toolkits_required": ["gmail"], + "success_confidence": 9.0, + "search_log_id": "log_2", + } + ] + ), + }, + { + "client_session_id": "session3", + "workflow_category": "communication", + "task_difficulty": "MEDIUM", + "search_json": json.dumps( + [ + { + "search_id": "usecase_3_1", + "cached_plan_id": "plan_xyz789", # Different cached ID + "response": { + "main_tools": ["SLACK_SEND_MESSAGE"], + "related_tools": [], + "toolkits": ["slack"], + }, + } + ] + ), + "per_usecase_plans_json": json.dumps( + [ + { + "cleaned_search_usecase": "Fetch Slack messages", + "search_id": "usecase_3_1", + "cleaned_known_fields": "", + "usecase_plan": json.dumps( + {"workflow_steps": ["authenticate", "fetch messages"]} + ), + "toolkits_required": ["slack"], + "success_confidence": 8.0, + "search_log_id": "log_3", + } + ] + ), + }, + ] + + print("🔄 Testing processing of cached sessions...") + + # Step 1: Flatten sessions to plans + plans = service.flatten_sessions_to_plans(mock_sessions) + print(f" Extracted {len(plans)} plans from {len(mock_sessions)} sessions") + + # Step 2: Separate cached vs new plans + non_cached, cached = service.separate_cached_and_new_plans(plans) + print(f" Non-cached plans: {len(non_cached)}, Cached plans: {len(cached)}") + + # Step 3: Process cached plans (grouped by cached_plan_id) + if cached: + successful, failed = ( + await service.cached_plan_processor.process_and_upsert_cached_plans(cached) + ) + print(f"✅ Processed cached plans: {successful} successful, {failed} failed") + assert ( + successful > 0 or len(cached) == 0 + ), "Should process cached plans if any exist" + else: + print("⚠️ No cached plans found (all plans have cached_plan_id=None)") + + print("✅ Cached sessions processing test passed!\n") + + +async def main(): + """Run all tests""" + print("=" * 60) + print("Testing Cached Plan Processing Integration") + print("=" * 60 + "\n") + + try: + await test_process_cached_sessions() + + print("=" * 60) + print("✅ All tests passed!") + print("=" * 60) + + except Exception as e: + print(f"\n❌ Test failed with error: {e}") + import traceback + + traceback.print_exc() + return 1 + + return 0 + + +if __name__ == "__main__": + import sys + + sys.exit(asyncio.run(main())) diff --git a/app_tester/rube_learning/ingestion_pipeline/tests/test_cached_plan_processing.py b/app_tester/rube_learning/ingestion_pipeline/tests/test_cached_plan_processing.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/tests/test_cached_plan_processing.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +""" +Test cached plan processing in usecase ingestion service. + +This script tests the process_and_upsert_cached_plans method to ensure: +1. Plans with same cached_plan_id are grouped together +2. Plans are consolidated using LLM +3. Consolidated plans are upserted to TurboPuffer with the cached_plan_id +""" + +import asyncio +import json + +import pytest + +from app_tester.rube_learning.ingestion_pipeline.usecase_cluster_models import ( + PlanWithContext, +) +from app_tester.rube_learning.ingestion_pipeline.usecase_ingestion_service import ( + UsecaseIngestionService, +) + + +@pytest.mark.asyncio +async def test_cached_plan_processing(): + """Test the cached plan processing workflow""" + + print("=" * 80) + print("Testing Cached Plan Processing") + print("=" * 80) + print() + + # Create service instance with dry run mode + from app_tester.rube_learning.ingestion_pipeline.ingestion_config import ( + UsecaseIngestionConfig, + ) + + config = UsecaseIngestionConfig( + namespace="rube_usecase_plans_test", + debug=True, + dry_run=True, # Use dry run mode for testing + ) + service = UsecaseIngestionService(config=config) + + # Create sample plans with same cached_plan_id + cached_plan_id_1 = "test_cached_123" + cached_plan_id_2 = "test_cached_456" + + test_plans = [ + PlanWithContext( + plan={ + "workflow_steps": [ + "Authenticate to Gmail", + "Compose email body", + "Send email", + ], + "pitfalls": [ + "Token may expire", + "Large attachments can fail", + ], + "tools": [ + "GMAIL_SEND_EMAIL", + "GMAIL_CREATE_MESSAGE", + "GMAIL_CREATE_DRAFT", + ], + }, + usecase="Send email to user", + session_id="session_1", + workflow_category="email", + main_tools="GMAIL_SEND_EMAIL,GMAIL_CREATE_MESSAGE", + related_tools="GMAIL_GET_USER,GMAIL_CREATE_DRAFT", + toolkits_required="gmail", + known_fields="recipient:user@example.com", + cached_plan_id=cached_plan_id_1, + success_confidence=8.9, + search_log_id="search_log_1", + ), + PlanWithContext( + plan={ + "workflow_steps": [ + "Authenticate to Gmail", + "Gather notification payload", + "Personalize content", + "Queue for review", + "Deliver notification", + ], + "pitfalls": [ + "Ensure payload includes metadata", + "Throttle to avoid quota issues", + "Log delivery status", + ], + "tools": [ + "GMAIL_SEND_EMAIL", + "GMAIL_BATCH_SEND", + "GMAIL_CREATE_DRAFT", + ], + }, + usecase="Send email to customer", + session_id="session_2", + workflow_category="email", + main_tools="GMAIL_SEND_EMAIL,GMAIL_BATCH_SEND", + related_tools="GMAIL_GET_USER,GMAIL_GET_PROFILE,GMAIL_LIST_MESSAGES", + toolkits_required="gmail", + known_fields="recipient:customer@example.com", + cached_plan_id=cached_plan_id_1, + success_confidence=9.4, + search_log_id="search_log_2", + ), + PlanWithContext( + plan={ + "workflow_steps": [ + "Authenticate to GitHub", + "Fetch repository list", + "Parse response", + ], + "pitfalls": [ + "Handle pagination", + "Respect rate limits", + ], + "tools": [ + "GITHUB_LIST_REPOS", + "GITHUB_GET_ORG_REPOS", + "GITHUB_SEARCH_REPOS", + ], + }, + usecase="List GitHub repositories", + session_id="session_3", + workflow_category="developer_tools", + main_tools="GITHUB_LIST_REPOS,GITHUB_GET_ORG_REPOS", + related_tools="GITHUB_GET_USER,GITHUB_SEARCH_REPOS", + toolkits_required="github", + known_fields="owner:user", + cached_plan_id=cached_plan_id_2, + success_confidence=8.5, + search_log_id="search_log_3", + ), + ] + + print(f"📋 Created {len(test_plans)} test plans:") + print(f" - 2 plans with cached_plan_id: {cached_plan_id_1}") + print(f" - 1 plan with cached_plan_id: {cached_plan_id_2}") + print() + + # Provide cached plan attributes so processing doesn't fetch real TurboPuffer data + cached_plan_fixtures = { + cached_plan_id_1: { + "plan": { + "workflow_steps": [ + "Authenticate to Gmail", + "Compose reusable template", + "Send confirmation email", + ], + "pitfalls": [ + "Verify OAuth scopes", + "Monitor send limits", + ], + }, + "usecase": "Send email to user", + "main_tools": "GMAIL_SEND_EMAIL,GMAIL_CREATE_MESSAGE", + "related_tools": "GMAIL_GET_USER,GMAIL_CREATE_DRAFT", + "toolkits": "gmail", + "known_fields": "recipient:user@example.com", + "task_difficulty": "MEDIUM", + "updated_at": "2024-01-01T00:00:00Z", + "created_at": "2023-12-01T00:00:00Z", + }, + cached_plan_id_2: { + "plan": { + "workflow_steps": [ + "Authenticate to GitHub", + "Query repository list", + "Return structured response", + ], + "pitfalls": [ + "Handle pagination", + "Respect API rate limits", + ], + }, + "usecase": "List GitHub repositories", + "main_tools": "GITHUB_LIST_REPOS,GITHUB_GET_ORG_REPOS", + "related_tools": "GITHUB_GET_USER,GITHUB_SEARCH_REPOS", + "toolkits": "github", + "known_fields": "owner:user", + "task_difficulty": "MEDIUM", + "updated_at": "2024-01-01T00:00:00Z", + "created_at": "2023-12-01T00:00:00Z", + }, + } + + def mock_get_plan_by_id(plan_id: str): + print(f" 🔧 Mock get_plan_by_id called for: {plan_id}") + fixture = cached_plan_fixtures.get(plan_id) + if not fixture: + return None + return { + "plan": json.dumps(fixture["plan"]), + "usecase": fixture["usecase"], + "main_tools": fixture["main_tools"], + "related_tools": fixture["related_tools"], + "toolkits": fixture["toolkits"], + "known_fields": fixture["known_fields"], + "task_difficulty": fixture["task_difficulty"], + "updated_at": fixture["updated_at"], + "created_at": fixture["created_at"], + } + + service.get_plan_by_id = mock_get_plan_by_id # type: ignore[attr-defined] + service.cached_plan_processor.ingestion.get_plan_by_id = mock_get_plan_by_id # type: ignore[method-assign] + service.cached_plan_processor._get_toolkits_from_tools = lambda tools: [] # type: ignore[attr-defined] + + # Process cached plans + print("🔄 Processing cached plans...") + successful, failed = ( + await service.cached_plan_processor.process_and_upsert_cached_plans(test_plans) + ) + + print() + print("=" * 80) + print("Test Results") + print("=" * 80) + print(f"✅ Successful upserts: {successful}") + print(f"❌ Failed upserts: {failed}") + print() + + if successful > 0: + print("✅ Test PASSED: Cached plans were successfully processed and upserted") + return True + else: + print("❌ Test FAILED: No plans were successfully upserted") + return False + + +async def main(): + """Main test runner""" + try: + success = await test_cached_plan_processing() + return 0 if success else 1 + except Exception as e: + print(f"\n❌ Test failed with error: {e}") + import traceback + + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + import sys + + sys.exit(asyncio.run(main())) diff --git a/app_tester/rube_learning/ingestion_pipeline/tests/test_full_pipeline_integration.py b/app_tester/rube_learning/ingestion_pipeline/tests/test_full_pipeline_integration.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/tests/test_full_pipeline_integration.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +""" +Integration test for the complete usecase ingestion pipeline. + +Tests the full pipeline end-to-end with real ClickHouse data. +This is the main integration test that validates the entire ingestion workflow. +""" + +import asyncio +import os +import sys +from typing import cast + +from dotenv import load_dotenv + + +# Add project root to path for imports +project_root = os.path.abspath( + os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "..") +) +if project_root not in sys.path: + sys.path.insert(0, project_root) + +# Load environment variables +load_dotenv() + + +def test_full_pipeline(): + """Test the complete pipeline with real ClickHouse data""" + print("\n🧪 Full Pipeline Integration Test") + print("=" * 60) + + # Check environment variables + required_vars = [ + "BACKEND_READONLY_URL", + "CLICKHOUSE_USERNAME", + "CLICKHOUSE_PASSWORD", + "ENCRYPTION_KEY", + "OPENAI_API_KEY", + "AZURE_OPENAI_ENDPOINT_URL", + "AZURE_OPENAI_SUBSCRIPTION_KEY", + "TURBOPUFFER_API_TOKEN", + ] + + missing_vars = [var for var in required_vars if not os.getenv(var)] + + if missing_vars: + print( + f" ⚠️ Skipping - Missing environment variables: {', '.join(missing_vars)}" + ) + return True # Don't fail, just skip + + try: + # Create service with test namespace and debug enabled + from app_tester.rube_learning.ingestion_pipeline.ingestion_config import ( + UsecaseIngestionConfig, + ) + from app_tester.rube_learning.ingestion_pipeline.usecase_ingestion_service import ( + UsecaseIngestionService, + ) + + config = UsecaseIngestionConfig( + namespace="rube_usecase_plans_test", + debug=True, + dry_run=True, # Use dry run mode for testing + ) + service = UsecaseIngestionService( + config=config, output_dir="temp/full_pipeline_integration_test" + ) + + # Run with 5 minute lookback in dry run mode + print(" Running service with 5 minute lookback in DRY RUN mode...\n") + success = asyncio.run(service.run(hours_back=cast(int, 5 / 60))) + + if success: + print("\n ✅ Full pipeline test successful!") + return True + else: + print("\n ❌ Full pipeline test failed") + return False + + except Exception as e: + print(f" ❌ Test failed with error: {e}") + import traceback + + traceback.print_exc() + return False + + +def main(): + """Run the full pipeline integration test""" + print("\n" + "=" * 60) + print("USECASE INGESTION PIPELINE INTEGRATION TEST") + print("=" * 60) + + # Run full pipeline integration test + success = test_full_pipeline() + + if success: + print("\n🎉 Test passed!") + return 0 + else: + print("\n⚠️ Test failed") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/app_tester/rube_learning/ingestion_pipeline/tests/test_plan_candidate_buffer.py b/app_tester/rube_learning/ingestion_pipeline/tests/test_plan_candidate_buffer.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/tests/test_plan_candidate_buffer.py @@ -0,0 +1,109 @@ +from datetime import datetime, timedelta + +from app_tester.rube_learning.ingestion_pipeline.plan_candidate_buffer import ( + PlanCandidateBuffer, + PlanCandidateRecord, +) +from app_tester.rube_learning.ingestion_pipeline.tests.fakes import FakeRedis + + +def _record(suffix: str = "1") -> PlanCandidateRecord: + return PlanCandidateRecord( + clickhouse_id=f"clickhouse_{suffix}", + plan_id=f"plan_{suffix}", + session_timestamp=datetime.utcnow() - timedelta(hours=1), + ) + + +def test_get_all_plan_ids_ignores_oldest_ts_keys() -> None: + redis = FakeRedis() + buffer = PlanCandidateBuffer( + redis_client=redis, + key_prefix="ingestion_plan_candidates", + ttl_seconds=3600, + ) + + buffer.add_candidates("alpha", [_record("alpha")]) + + # Simulate historical suffix format and ensure it's handled + redis.set("ingestion_plan_candidates:legacy_plan:candidates", "legacy", ex=3600) + redis.set( + "ingestion_plan_candidates:legacy_plan:oldest_ts", + "2024-01-01T00:00:00Z", + ex=3600, + ) + + plan_ids = buffer.get_all_plan_ids() + assert set(plan_ids) == {"alpha", "legacy_plan"} + + +def test_find_plans_past_age_threshold_respects_threshold() -> None: + redis = FakeRedis() + buffer = PlanCandidateBuffer( + redis_client=redis, + key_prefix="ingestion_plan_candidates", + ttl_seconds=7200, + ) + + buffer.add_candidates("old_plan", [_record("old")]) + buffer.add_candidates("new_plan", [_record("new")]) + + # Manually set created_at to simulate age + old_created_at = (datetime.utcnow() - timedelta(hours=48)).isoformat() + "Z" + new_created_at = (datetime.utcnow() - timedelta(hours=1)).isoformat() + "Z" + redis.set(buffer._created_at_key("old_plan"), old_created_at, ex=7200) + redis.set(buffer._created_at_key("new_plan"), new_created_at, ex=7200) + + # Threshold of 24 hours - should find old_plan (48h old) but not new_plan (1h old) + aged = buffer.find_plans_past_age_threshold(age_threshold_hours=24) + + assert aged == ["old_plan"] + + +def test_find_plans_past_age_threshold_uses_oldest_ts_fallback() -> None: + redis = FakeRedis() + buffer = PlanCandidateBuffer( + redis_client=redis, + key_prefix="ingestion_plan_candidates", + ttl_seconds=7200, + ) + + buffer.add_candidates("legacy_plan", [_record("legacy")]) + + # Delete the created_at key to simulate legacy data + redis.delete(buffer._created_at_key("legacy_plan")) + + # Set oldest_ts to 48 hours ago (beyond 24h threshold) + old_ts = (datetime.utcnow() - timedelta(hours=48)).isoformat() + "Z" + redis.set(buffer._oldest_ts_key("legacy_plan"), old_ts, ex=7200) + + # Should include legacy plan using oldest_ts fallback + aged = buffer.find_plans_past_age_threshold(age_threshold_hours=24) + assert aged == ["legacy_plan"] + + # Set oldest_ts to 1 hour ago (within 24h threshold) + recent_ts = (datetime.utcnow() - timedelta(hours=1)).isoformat() + "Z" + redis.set(buffer._oldest_ts_key("legacy_plan"), recent_ts, ex=7200) + + # Should NOT include it now + aged = buffer.find_plans_past_age_threshold(age_threshold_hours=24) + assert aged == [] + + +def test_clear_removes_created_at_key() -> None: + redis = FakeRedis() + buffer = PlanCandidateBuffer( + redis_client=redis, + key_prefix="ingestion_plan_candidates", + ttl_seconds=7200, + ) + + buffer.add_candidates("test_plan", [_record("test")]) + + # Verify created_at exists + assert buffer.get_created_at("test_plan") is not None + + # Clear and verify it's gone + buffer.clear("test_plan") + assert buffer.get_created_at("test_plan") is None + assert buffer.get_candidates("test_plan") == [] diff --git a/app_tester/rube_learning/ingestion_pipeline/tests/test_plan_consolidation.py b/app_tester/rube_learning/ingestion_pipeline/tests/test_plan_consolidation.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/ingestion_pipeline/tests/test_plan_consolidation.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +""" +Unit test for LLM-based plan consolidation functionality. + +Tests the consolidate_plans_with_llm() method that merges multiple plans using GPT-5. +""" + +import asyncio + +import pytest + +from app_tester.rube_learning.ingestion_pipeline.ingestion_config import ( + UsecaseIngestionConfig, +) +from app_tester.rube_learning.ingestion_pipeline.usecase_ingestion_service import ( + UsecaseIngestionService, +) + + +@pytest.mark.asyncio +async def test_plan_consolidation(): + """Test LLM-based plan consolidation""" + config = UsecaseIngestionConfig( + namespace="rube_usecase_plans_test", + debug=True, + dry_run=True, + ) + service = UsecaseIngestionService(config=config) + + # Create mock plans with context + mock_plans = [ + { + "plan": { + "workflow_steps": [ + "Authenticate to Gmail using OAuth credentials", + "Compose the email body with dynamic placeholders", + "Attach relevant documents and inline resources", + "Preview and validate recipients before sending", + "Send email and verify response status", + ], + "pitfalls": [ + "OAuth token may expire mid-flow", + "Attachments larger than 20MB will fail", + "Handle rate limiting errors gracefully", + ], + "toolkits_required": ["gmail"], + "success_confidence": 8.5, + "difficulty": "easy", + }, + "session_id": "session1", + "usecase": "Send email via Gmail to team", + "known_fields": "recipient:team@example.com", + }, + { + "plan": { + "workflow_steps": [ + "Login to Gmail", + "Create message", + "Send to recipient", + "Handle errors", + ], + "pitfalls": [ + "Ensure upstream payloads include required metadata", + "Throttle notifications to avoid API quota issues", + "Log delivery status for downstream auditing", + ], + "toolkits_required": ["gmail"], + "success_confidence": 9.0, + "difficulty": "easy", + "error_handling": "Retry on failure", + }, + "session_id": "session2", + "usecase": "Send email notification through Gmail", + "known_fields": "recipient:notifications@example.com", + }, + ] + + print("🤖 Testing plan consolidation with LLM...") + consolidated = await service.plan_consolidator.consolidate_plans_with_llm( + "test_cached_plan_123", + mock_plans, + representative_usecase="Send email via Gmail", + ) + + if consolidated: + print("✅ Successfully consolidated plan:") + print(f" Keys: {list(consolidated.keys())}") + print(f" Consolidated plan snippet: {str(consolidated)[:200]}...") + else: + print("❌ Plan consolidation returned None") + + assert consolidated is not None, "Should return consolidated plan" + print("✅ Plan consolidation test passed!\n") + + +async def main(): + """Run all tests""" + print("=" * 60) + print("Testing Plan Consolidation") + print("=" * 60 + "\n") + + try: + await test_plan_consolidation() + + print("=" * 60) + print("✅ All tests passed!") + print("=" * 60) + + except Exception as e: + print(f"\n❌ Test failed with error: {e}") + import traceback + + traceback.print_exc() + return 1 + + return 0 + + +if __name__ == "__main__": + import sys + + sys.exit(asyncio.run(main())) diff --git a/app_tester/rube_learning/ingestion_pipeline/tests/test_plan_extraction.py b/app_tester/rube_learning/ingestion_pipeline/tests/test_plan_extraction.py new file mode 100644 diff --git a/app_tester/rube_learning/ingestion_pipeline/tests/test_plan_update_policy.py b/app_tester/rube_learning/ingestion_pipeline/tests/test_plan_update_policy.py new file mode 100644 diff --git a/app_tester/rube_learning/ingestion_pipeline/tests/test_processed_session_tracker.py b/app_tester/rube_learning/ingestion_pipeline/tests/test_processed_session_tracker.py new file mode 100644 diff --git a/app_tester/rube_learning/ingestion_pipeline/tests/test_tool_consolidation.py b/app_tester/rube_learning/ingestion_pipeline/tests/test_tool_consolidation.py new file mode 100644 diff --git a/app_tester/rube_learning/ingestion_pipeline/tool_consolidator.py b/app_tester/rube_learning/ingestion_pipeline/tool_consolidator.py new file mode 100644 diff --git a/app_tester/rube_learning/ingestion_pipeline/toolkit_utils.py b/app_tester/rube_learning/ingestion_pipeline/toolkit_utils.py new file mode 100644 diff --git a/app_tester/rube_learning/ingestion_pipeline/turbopuffer_ingestion.py b/app_tester/rube_learning/ingestion_pipeline/turbopuffer_ingestion.py new file mode 100644 diff --git a/app_tester/rube_learning/ingestion_pipeline/unmatched_cluster_processor.py b/app_tester/rube_learning/ingestion_pipeline/unmatched_cluster_processor.py new file mode 100644 diff --git a/app_tester/rube_learning/ingestion_pipeline/usecase_cluster_models.py b/app_tester/rube_learning/ingestion_pipeline/usecase_cluster_models.py new file mode 100644 diff --git a/app_tester/rube_learning/ingestion_pipeline/usecase_clustering.py b/app_tester/rube_learning/ingestion_pipeline/usecase_clustering.py new file mode 100644 diff --git a/app_tester/rube_learning/ingestion_pipeline/usecase_ingestion_service.py b/app_tester/rube_learning/ingestion_pipeline/usecase_ingestion_service.py new file mode 100644 diff --git a/app_tester/rube_learning/mercury_utils.py b/app_tester/rube_learning/mercury_utils.py diff --git a/app_tester/rube_learning/missing_tool_models.py b/app_tester/rube_learning/missing_tool_models.py deleted file mode 100644 diff --git a/app_tester/rube_learning/pipeline.py b/app_tester/rube_learning/pipeline.py new file mode 100644 diff --git a/app_tester/rube_learning/rube-search-enriched.csv b/app_tester/rube_learning/rube-search-enriched.csv deleted file mode 100644 diff --git a/app_tester/rube_learning/run_pipeline.py b/app_tester/rube_learning/run_pipeline.py new file mode 100644 diff --git a/app_tester/rube_learning/search_response_models.py b/app_tester/rube_learning/search_response_models.py new file mode 100644 diff --git a/app_tester/rube_learning/session_detector.py b/app_tester/rube_learning/session_detector.py diff --git a/app_tester/rube_learning/slack_notifier.py b/app_tester/rube_learning/slack_notifier.py diff --git a/app_tester/rube_learning/tests/__init__.py b/app_tester/rube_learning/tests/__init__.py new file mode 100644 diff --git a/app_tester/rube_learning/tests/test_extractors.py b/app_tester/rube_learning/tests/test_extractors.py new file mode 100644 diff --git a/app_tester/rube_learning/tests/test_multi_execute_token_extraction.py b/app_tester/rube_learning/tests/test_multi_execute_token_extraction.py new file mode 100644 diff --git a/app_tester/rube_learning/tests/test_pipeline.py b/app_tester/rube_learning/tests/test_pipeline.py new file mode 100644 diff --git a/app_tester/rube_learning/tests/test_pipeline_integration.py b/app_tester/rube_learning/tests/test_pipeline_integration.py new file mode 100644 diff --git a/app_tester/rube_learning/tool_analysis_runner.py b/app_tester/rube_learning/tool_analysis_runner.py deleted file mode 100644 diff --git a/app_tester/rube_learning/toolkit_checker.py b/app_tester/rube_learning/toolkit_checker.py diff --git a/app_tester/rube_learning/utils.py b/app_tester/rube_learning/utils.py new file mode 100644 diff --git a/app_tester/rube_learning/workflow_analysis/__init__.py b/app_tester/rube_learning/workflow_analysis/__init__.py new file mode 100644 diff --git a/app_tester/rube_learning/workflow_analysis/models.py b/app_tester/rube_learning/workflow_analysis/models.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/workflow_analysis/models.py @@ -0,0 +1,471 @@ +""" +Models for Workflow Analysis (first stage of session analysis). + +These models capture: +- Workflow understanding (usecase, summary, apps, tools, difficulty) +- Per-usecase tool analysis and plans +- Search quality analysis +- New tool/toolkit suggestions +""" + +from enum import Enum +from typing import List, Optional + +from pydantic import BaseModel, Field + +from app_tester.rube_learning.common import PlanQuality, TaskDifficulty + + +class SearchQuality(str, Enum): + """Overall search quality assessment.""" + + EXCELLENT = "EXCELLENT" # Search worked perfectly, found all needed tools + AVERAGE = "AVERAGE" # Search had issues but workflow could proceed + POOR = "POOR" + + +class WrongToolsReturnedDetail(BaseModel): + """Detail about wrong tools returned.""" + + search_id: Optional[str] = Field( + None, description="The search_id from the original search (e.g., 'usecase_1_1')" + ) + use_case: str = Field( + ..., description="The user's intent or use case that triggered the search" + ) + tools_returned: List[str] = Field( + ..., description="List of tool slugs that were actually returned" + ) + tools_missing: List[str] = Field( + ..., description="List of tool slugs that should have been returned but weren't" + ) + tools_irrelevant: List[str] = Field( + default_factory=list, + description="List of tool slugs that were returned but shouldn't have been", + ) + reason: str = Field( + ..., description="Explanation of why the returned tools were wrong" + ) + severity: str = Field( + ..., description="Impact severity: 'critical', 'high', 'medium', 'low'" + ) + + +class SearchQualityAnalysis(BaseModel): + """Structured search quality output.""" + + overall_quality: SearchQuality = Field( + ..., description="Overall quality: GOOD, ACCEPTABLE, or POOR" + ) + explanation: str = Field( + ..., description="High-level explanation of search quality" + ) + wrong_tools_returned: List[WrongToolsReturnedDetail] = Field( + default_factory=list, + description="List of cases where wrong tools were returned", + ) + + +class Tool(BaseModel): + """Tool reference with optional confidence.""" + + tool_slug: str = Field(..., description="The tool slug (e.g., 'GMAIL_SEND_EMAIL')") + toolkit: Optional[str] = Field(None, description="The toolkit name") + reasoning: Optional[str] = Field(None, description="Why this tool was selected") + + +class UsecaseToolAnalysis(BaseModel): + """Per-usecase tool analysis with refinements.""" + + search_id: str = Field( + ..., description="The search_id from the original search (e.g., 'usecase_1_1')" + ) + final_main_tools: List[Tool] = Field( + default_factory=list, description="Refined list of main tools for this use case" + ) + final_related_tools: List[Tool] = Field( + default_factory=list, description="Refined list of related tools" + ) + final_toolkits: List[str] = Field( + default_factory=list, description="Toolkits needed for this use case" + ) + reasoning: str = Field( + "", description="Explanation of changes made to the tool list" + ) + overall_confidence: int = Field( + 5, ge=0, le=10, description="Confidence that these tools would help (0-10)" + ) + changes_made: int = Field( + 0, ge=0, le=3, description="Number of changes made (max 3)" + ) + + +class WorkbenchCodeSnippet(BaseModel): + """Reusable code snippet for workbench operations.""" + + description: str = Field( + ..., + description="Description of the code snippet, e.g., 'Code snippet to parse large XYZ_TOOL response'", + ) + code: str = Field( + ..., + description="The python code snippet. Keep it concise, no comments or docstrings, only reference code", + ) + + +class UsecasePlanDetails(BaseModel): + """Details of a workflow plan for a use case.""" + + workflow_steps: List[str] = Field( + default_factory=list, + description=( + "Key steps in workflow execution. Each step MUST be tagged as: " + "[Required/Optional] [Prerequisite/Step/Next Step/Fallback]: [Action using TOOL_X]. " + "Examples: 'Required Prerequisite: Fetch email ID using GMAIL_GET_EMAIL', " + "'Required Step 1: Create draft using GMAIL_CREATE_DRAFT', " + "'Optional Fallback: Use OUTLOOK_SEND_EMAIL as fallback'. " + "Capture happy path and fallback paths. NEVER include PII or user-specific identifiers." + ), + ) + difficulty: str = Field( + "medium", + description=( + "Difficulty assessment as 'easy', 'medium', or 'hard' with short reason. " + "Consider number of steps, error handling needs, edge cases, etc. " + "E.g., 'medium - requires pagination and error handling'" + ), + ) + pitfalls: List[str] = Field( + default_factory=list, + description=( + "List of gotchas and pitfalls in tool inputs/outputs/behavior. " + "Tag as [Tag] [Pitfall Description]. Examples: " + "'Rate limit: throttle requests as app allows max 60/m', " + "'Response handling: The response may not contain private channels'. " + "Keep concise and actionable. NEVER include PII or user-specific details." + ), + ) + workbench_code_snippets: Optional[List[WorkbenchCodeSnippet]] = Field( + default=None, + description=( + "Python code snippets for workbench operations. ONLY include if BROADLY relevant " + "to future users. Keep anonymized, STRICTLY REMOVE PII. Show code structure, fields, etc. " + "Don't show exact file names used in this session. DO NOT REPEAT info in code samples and workflow steps." + ), + ) + + +class UsecasePlan(BaseModel): + """Plan for a specific use case.""" + + search_id: str = Field( + ..., + description="The id of the search call, e.g., usecase_1_1, overall_1, usecase_2_3, etc.", + ) + cleaned_search_usecase: str = Field( + ..., + description=( + "Cleaned search query, remove all PII/names/identifiers/user-specific info (keep it generic). " + "Preserve app names and stable identifiers, keep language similar. " + "E.g., 'send email to John via gmail with happy birthday' -> 'send email greeting to someone via gmail'" + ), + ) + cleaned_known_fields: Optional[str] = Field( + None, + description="Cleaned known_fields field, only keep field keys, remove values. E.g., 'recipient:John, count:10' becomes 'recipient, count'", + ) + toolkits_required: List[str] = Field( + default_factory=list, + description=( + "Sorted list of lowercase toolkit slugs required. Use 'OR' for alternatives. " + "E.g., ['gmail OR outlook', 'googledocs'] when either email toolkit works." + ), + ) + actually_used_toolkits: Optional[List[str]] = Field( + None, + description=( + "For plans where toolkits_required contains OR relations, specify which toolkit(s) were actually used. " + "E.g., if toolkits_required=['gmail OR outlook', 'gemini'] and gmail was used -> ['gmail', 'gemini']" + ), + ) + actually_used_tools: Optional[List[str]] = Field( + None, + description="List of tools actually used in the workflow. Only app tools (e.g., GMAIL_FETCH_EMAILS), not COMPOSIO_ meta tools", + ) + was_cached_plan_returned: bool = Field( + False, + description="Whether a cached plan was returned by search - check if plan_id/cached_plan_id, plan, and pitfalls are present in the usecase response", + ) + cached_plan_quality: PlanQuality = Field( + default=PlanQuality.PLAN_NOT_AVAILABLE, + description=( + "Quality of the cached plan for this usecase: " + "PLAN_NOT_AVAILABLE if plan wasn't returned or was ignored; " + "PLAN_POOR (0-4) if incorrect, misleading, or omits key considerations; " + "PLAN_AVERAGE (5-7) if covers main steps but misses details or over-engineered; " + "PLAN_EXCELLENT (8-10) if comprehensive, accurate, accounts for edge cases" + ), + ) + cached_plan_quality_reason: str = Field( + default="N/A", + description="Why the cached plan was rated this way - share details in concise form. N/A if no cached plan found", + ) + usecase_plan: UsecasePlanDetails = Field( + ..., + description="Workflow plan details with steps, difficulty, pitfalls, and optional code snippets", + ) + success_confidence: int = Field( + 5, + ge=0, + le=10, + description=( + "0-10 confidence that the proposed plan will lead to success. " + "Lower if workflow incomplete or missing tools. Higher if plan effectively addresses issues." + ), + ) + plan_changelog: str = Field( + ..., + description="Short changelog vs existing plan (e.g., '1) added rate limits, 2) added find projects dependency'). If no existing plan, say 'new'", + ) + plan_reasoning: str = Field( + ..., + description="Thorough reasoning why this plan was proposed. E.g., 'x tool did not work due to incorrect params so added a pitfall for it'", + ) + + +class NewToolSuggestion(BaseModel): + """ + Suggestion for a genuinely NEW tool (not improvements like retry/pagination/validation to existing tools). + """ + + tool_name: str = Field( + ..., + description="Suggested name for the new tool (e.g., 'GMAIL_ARCHIVE_EMAIL'). " + "Must be a genuinely new capability, NOT an improvement to an existing tool.", + ) + description: str = Field( + ..., + description="What the tool would do - must be a new capability not covered by existing tools", + ) + use_case: str = Field(..., description="The specific use case that needs this tool") + toolkit: str = Field(..., description="The existing toolkit this tool belongs to") + api_endpoint: Optional[str] = Field( + None, description="The API endpoint this maps to" + ) + api_docs: Optional[str] = Field(None, description="URL to API documentation") + implementation_difficulty: str = Field( + "MEDIUM", description="EASY, MEDIUM, HARD, or IMPOSSIBLE" + ) + priority: int = Field( + 5, ge=1, le=10, description="Priority (1-10, higher = more important)" + ) + + +class NewToolkitSuggestion(BaseModel): + """Suggestion for an entirely new toolkit integration.""" + + toolkit_name: str = Field( + ..., description="Name of the toolkit (e.g., 'stripe', 'airtable')" + ) + priority: int = Field(5, ge=1, le=10, description="Priority (1-10)") + reasoning: str = Field(..., description="Why this toolkit is needed") + use_case: str = Field(..., description="The use case that requires this toolkit") + api_docs_url: Optional[str] = Field( + None, description="URL to the toolkit's API documentation" + ) + expected_value: Optional[str] = Field( + None, description="Expected value from adding this integration" + ) + + +class MissingPrerequisiteWarning(BaseModel): + """Cases where the cached plan didn't warn about prerequisites that caused failures.""" + + search_id: str = Field( + ..., description="The search_id this warning applies to (e.g., 'usecase_1_1')" + ) + plan_id: str = Field( + ..., description="Which cached plan this refers to (from search response)" + ) + usecase: str = Field(..., description="The sub-task this plan was for") + missing_prerequisite: str = Field( + ..., + description="What should have been warned about (e.g., 'Gmail OAuth setup')", + ) + consequence: str = Field( + ..., description="What went wrong because of this missing warning" + ) + suggested_warning: str = Field(..., description="What the plan should have said") + + +class IncorrectPlanSuggestion(BaseModel): + """Cases where the cached plan's suggestions were wrong or didn't work.""" + + search_id: str = Field( + ..., description="The search_id this issue applies to (e.g., 'usecase_1_1')" + ) + plan_id: str = Field( + ..., description="Which cached plan this refers to (from search response)" + ) + usecase: str = Field(..., description="The sub-task this plan was for") + incorrect_suggestion: str = Field( + ..., description="What the plan suggested that was wrong" + ) + why_incorrect: str = Field( + ..., description="Why it was wrong (tool doesn't exist, wrong params, etc.)" + ) + actual_solution: Optional[str] = Field( + None, description="What actually worked (if known)" + ) + + +class PerUsecasePlanFollowing(BaseModel): + """Tracks whether a cached plan was followed for each usecase.""" + + search_id: str = Field( + ..., description="The search_id (e.g., 'usecase_1_1', 'overall_1')" + ) + was_cached_plan_followed: bool = Field( + False, + description=( + "Whether the workflow followed the cached plan's required steps. " + "False if no cached plan was returned or if LLM ignored it." + ), + ) + deviation_reason: Optional[str] = Field( + None, + description="If plan wasn't followed, brief explanation of why (e.g., 'LLM chose different approach', 'Plan steps were incorrect')", + ) + + +class PlannerIssueAnalysis(BaseModel): + """ + Analysis of cached plan failures - used to improve the planner. + + This captures cases where a cached plan was returned by search but failed + or was insufficient. NOT about tool selection - about plan quality. + Issues are tracked per-usecase via search_id in each warning/suggestion. + """ + + per_usecase_plan_following: List[PerUsecasePlanFollowing] = Field( + default_factory=list, + description="Per-usecase tracking of whether cached plans were followed", + ) + missing_prerequisite_warnings: List[MissingPrerequisiteWarning] = Field( + default_factory=list, + description="Cases where plan didn't warn about prerequisites. Each item includes search_id for per-usecase tracking.", + ) + incorrect_suggestions: List[IncorrectPlanSuggestion] = Field( + default_factory=list, + description="Cases where plan suggested wrong tools/steps. Each item includes search_id for per-usecase tracking.", + ) + overall_planner_feedback: Optional[str] = Field( + None, + description="General observations about the cached plan quality across all usecases", + ) + + +class WorkflowAnalysisOutput(BaseModel): + """ + Complete output from workflow analysis (first stage). + + This captures: + - Workflow understanding + - Tool analysis per use case + - Plans per use case + - Search quality assessment + - Tool/toolkit gap suggestions + - Planner issues + """ + + # === Workflow Understanding === + overall_usecase: str = Field( + ..., + description="High-level description of what the user was trying to accomplish", + ) + workflow_category: str = Field( + ..., + description="Category of workflow (e.g., 'email', 'project_management', 'social')", + ) + workflow_summary: str = Field( + ..., description="Detailed summary of what happened in the workflow" + ) + toolkits_involved: List[str] = Field( + default_factory=list, description="List of toolkits involved" + ) + tools_involved: List[str] = Field( + default_factory=list, + description="List of tools actually executed (not COMPOSIO_ meta tools)", + ) + task_difficulty: TaskDifficulty = Field( + TaskDifficulty.MEDIUM, description="Overall task difficulty" + ) + confidence_score: float = Field( + 0.5, ge=0.0, le=1.0, description="Confidence in the analysis (0.0 to 1.0)" + ) + tasks_performed: List[str] = Field( + default_factory=list, + description=( + "List of distinctive broad-level tasks performed in the workflow. " + "Tasks are high-level objectives, not the same as usecases (one task can have multiple search usecases). " + "Keep concise." + ), + ) + + # === Tool Analysis === + per_usecase_tool_analysis: List[UsecaseToolAnalysis] = Field( + default_factory=list, description="Tool analysis for each search use case" + ) + + # === Plans === + per_usecase_plans: List[UsecasePlan] = Field( + default_factory=list, description="Workflow plans for each use case" + ) + + # === Search Quality === + search_quality: SearchQualityAnalysis = Field( + default_factory=lambda: SearchQualityAnalysis( + overall_quality=SearchQuality.AVERAGE, explanation="Not yet analyzed" + ), + description="Structured search quality assessment", + ) + + # === Tool/Toolkit Gap Analysis === + new_tool_suggestions: List[NewToolSuggestion] = Field( + default_factory=list, + description="Suggestions for new tools in existing toolkits", + ) + new_toolkit_suggestions: List[NewToolkitSuggestion] = Field( + default_factory=list, + description="Suggestions for entirely new toolkit integrations", + ) + + # === Planner Issues (for cached plan quality - only when cached plan returned) === + planner_issues: PlannerIssueAnalysis = Field( + default_factory=PlannerIssueAnalysis, # type: ignore[arg-type] + description="Analysis of cached plan failures (only populated when was_cached_plan_returned is True)", + ) + + def get_wrong_tools_for_error_analysis(self) -> List[dict]: + """ + Extract wrong_tools_returned data for error_analysis. + + Converts SearchQualityAnalysis items to dicts for Call 2. + Only includes items with a valid search_id. + """ + result = [] + for item in self.search_quality.wrong_tools_returned: + # Skip items without a valid search_id + if not item.search_id: + continue + result.append( + { + "search_id": item.search_id, + "use_case": item.use_case, + "tools_returned": item.tools_returned, + "tools_missing": item.tools_missing, + "tools_irrelevant": item.tools_irrelevant, + "severity": item.severity, + } + ) + return result diff --git a/app_tester/rube_learning/workflow_analysis/runner.py b/app_tester/rube_learning/workflow_analysis/runner.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/workflow_analysis/runner.py @@ -0,0 +1,751 @@ +""" +Workflow Analysis Runner. + +Runs the first stage of session analysis: +- Workflow understanding +- Tool analysis per use case +- Plans per use case +- Search quality assessment +- Tool/toolkit gap suggestions +""" + +import logging +import time +from typing import Any, Dict, List, Optional + +from app_tester.rube_learning.call_parser import ParsedCall, ParsedSession +from app_tester.rube_learning.config import ( + DESCRIPTION_MAX_LENGTH, + JSON_KEY_VALUE_MAX_LENGTH, + JSON_MAX_KEYS_TO_SHOW, + WORKFLOW_LOG_ERROR_MAX_LENGTH, + WORKFLOW_LOG_REQUEST_MAX_LENGTH, + WORKFLOW_LOG_RESPONSE_MAX_LENGTH, +) +from app_tester.rube_learning.session_detector import Session +from app_tester.rube_learning.utils import get_tool_context_string + +from .models import WorkflowAnalysisOutput +from .search_utils import analyze_search_results + + +logger = logging.getLogger(__name__) + + +def build_workflow_analysis_prompt( + session: Session, + search_data: Dict[str, Any], + tool_catalog: str = "", +) -> str: + """Build the prompt for workflow analysis.""" + + # Format session logs with full search details inline + session_logs = _format_session_logs(session, search_data) + + # Build tool catalog section + tool_catalog_section = "" + if tool_catalog: + tool_catalog_section = f""" +### Composio Tool Catalog (All Available Tools for Involved Toolkits) + +This section lists ALL available tools in Composio for the toolkits involved in this session. +Use this to: +1. Identify if Search returned the right tools (compare the search responses in Session Logs vs this catalog) +2. Find better alternatives that Search may have missed +3. Assess search quality - ONLY reference tools from this catalog when identifying tools_missing. If Search returned the best available tools, tools_missing should be EMPTY. + +**IMPORTANT**: This catalog is the source of truth for what tools exist. NEVER invent or hallucinate tool names. + +{tool_catalog} +""" + + prompt = f"""# Workflow & Tool Analysis + +You are analyzing tool calls made by an LLM to Composio (a universal tool/MCP layer) to accomplish a task. Your goal is to systematically analyze the workflow and provide comprehensive insights. + +## Important Context + +These tool calls are from a general-purpose AI agent (Claude, GPT, etc.) using Composio as a tool layer. The agent handles summarization, analysis, user followups, and responses outside of tool calls - but you only see tool execution logs, not the agent's reasoning or conversations. + +**Key assumptions:** +- Only flag 'incomplete_workflow' if logs show sudden abandonment mid-execution (user gave up), NOT for missing summarization/analysis steps (agent does that) +- If tools returned requested data successfully, assume the workflow succeeded +- If confirmation was requested from user, assume permission was requested and granted +- All steps of workflow may not be visible via logs (e.g., if LLM asked a question and user responded - it won't show up in logs). Keep that in mind and don't assume things got missed necessarily. + +## Composio Meta Tools + +1. **Search** - Main meta tool to discover relevant app tools for a usecase. + - Internally uses tool names and descriptions (first 200 chars) to find relevant tools + - Returns main/related tools, toolkits with descriptions and request schema, connection status, memory items + - Returns difficulty level (easy/medium/hard) and planner guidance + - May return multiple apps for the usecase (e.g., gmail, outlook for email) + +2. **Multi Execute** - Execute up to 20 tools in parallel. Large responses get saved to workbench. + +3. **Remote Workbench** - Execute code in a remote sandbox for processing large datasets or bulk scripting. + - Helpers: `run_composio_tool`, `proxy_execute`, `invoke_llm`, `web_search`, `upload_local_file` + - Meant for large datasets (1000+ emails), NOT small ones (10 emails) + - Typical runs should finish under 1 minute + - More error-prone than native LLM path for simpler usecases + +4. **Remote Bash** - Bash operations over files in the sandbox. + +## Session Data + +### Session Logs +{session_logs} +{tool_catalog_section} +--- + +## PHASE 1: Workflow Understanding + +Understand what the user was trying to accomplish and summarize the workflow. + +### Output Fields + +- **overall_usecase**: What the user was trying to accomplish (from Search queries). No PII. +- **workflow_category**: 'email management', 'social media', 'project management', etc. +- **toolkits_involved**: Toolkits used or should have been used +- **tools_involved**: Tools actually executed (NOT COMPOSIO_ meta tools) +- **workflow_summary**: Concise summary: user intent → actions → outcomes +- **task_difficulty**: EASY (single tool), MEDIUM (moderate sequence), HARD (multi-step, pagination, cross-app) +- **confidence_score**: 0.0-1.0 confidence in your analysis + +--- + +## PHASE 2: Per-Usecase Tool Refinement + +For EACH search use case, refine the tool lists to return only tools actually needed. + +### Tool Selection & Improvement + +- **Use the Composio Tool Catalog** (provided in Session Data) to identify better alternatives that Search may have missed. +- Return tools that were executed + better alternatives from the catalog. Criteria for better tool: + - If a tool is marked ⭐ IMPORTANT in the catalog, or + - If it is more LLM/user friendly - e.g., a tool that searches by name can be better than a tool that searches by ID, or a tool that takes markdown instead of plaintext. + - However there may be more nuances in input/output schema that you are not aware of - that's why prefer to append instead of replace. +- You may add necessary tools, remove redundant tools, reclassify tools (main ↔ related - counts as a change). +- Give slight preference (in ordering) to NO-AUTH toolkits over toolkits that require auth, while retaining both options. +- Refer the per-usecase plans you are proposing later and align the tools and toolkits analysis with the plan analysis. +- EVALUATION NOTE: Assess what's required for full execution of the use case, considering the complete intended workflow. It's possible the current run may have incorrectly skipped some required steps — keep your analysis general and not overly biased by this specific execution. +- If workflow outcome was decent, make fewer changes and keep more tools. +- Be EXTREMELY conservative: only suggest changes for clear issues that could impede the use case, not minor improvements. +- Maximum 3 changes per use case (strictly enforced). + +### Main vs Related Tools + +- Don't put "nice to have" tools in main tools. Main tool list should be focused toward the main usecase. +- Related tools should include: + - Prerequisite tools: tools required to be executed to fetch required dependencies - e.g., tool to fetch linear projects before tool to create an issue (if the project id is not known) + - Fallback/alternative tools: E.g., tool to update single item if tool to bulk update fails + - Logical next steps: E.g., `update_document` tool is a related tool for `create_document` as it is a next step (update after create). +- Add high-quality useful related tools, and don't unnecessarily remove existing related tools even if they weren't used in the workflow run, unless the tool is completely irrelevant. Be conservative. +- IMPORTANT: For each main tool, have AT MOST 1 fallback tool. For logical next steps, have AT MOST 2 next step tools. Exceptions allowed but be conservative and avoid bloating number of tools. + +### Redundancy, Pruning & Dependencies + +- You may add cross-toolkit dependencies: Add 1-2 tools MAX and ONLY for direct workflow needs. Example: GOOGLEDRIVE_SHARE_FILE → add GMAIL_SEARCH_PEOPLE to find emails. NOT just because available - must fill specific gap. Explain dependency in reasoning. +- You may remove toolkits absolutely not needed (e.g., gemini for sending email). +- You may remove absolutely redundant tools: + - E.g., if one main tool and one fallback are sufficient and present, you may remove extra second fallback. + - Assume connection is done, don't add tools to check connection or get profile unless required. + - If there are too many tools for potential next steps, you may remove the least important of them. +- Avoid having too many tools that do the same thing, unless they offer different features or capabilities. One fallback is fine. + +### Tool List Size Constraints + +Aim for 2-5 main tools and 3-6 related tools depending on complexity. If you have more tools than this, remove the tools with the weakest utility/relevance. + +- Ensure main+related list has >= 3 tools (strictly enforced). The main and related list should be ordered by relevance. +- Ensure main+related list has <= 11 tools. Be frugal as more tools adds to cost and latency. Exceptions possible if the usecase is very complex or multi-step, but in most cases up to 11 tools are sufficient for a single usecase. + - Exception: If the usecase proposes multiple toolkits for diversity (e.g., gmail+outlook+sendgrid for email), you may have more tools but still be frugal. + +### Diversity Across Toolkits + +- When multiple toolkits serve the same use case (e.g., email: Gmail, Outlook, SendGrid), and any toolkit name is not mentioned in the usecase - preserve tools from different toolkits even if unused. +- This maintains diversity across toolkit integrations - it's valuable to have options. +- However, if unused tools are from the same toolkit, remove them as usual. +- **PRIORITIZE IMPORTANT TOOLS**: When selecting tools to include or keep, prioritize tools marked with ⭐ IMPORTANT tag - these are the most commonly used and valuable tools for each toolkit. +- Example: If Gmail was used for email, but Outlook and SendGrid tools were also returned → keep all three toolkits' tools for diversity. +- Example: If Gmail returned 5 tools but only 2 were used → remove the 3 unused Gmail tools (same toolkit, no diversity benefit). + +### Output per UseCase + +- **search_id**: CRITICAL - Copy the EXACT search_id from the header (e.g., "usecase_1_1", "usecase_2_1", etc.) +- **final_main_tools**, **final_related_tools**, **final_toolkits** +- Include tools that are required for the execution of usecase + better alternatives if available in Composio (apply diversity rules above) +- **reasoning**: Explain EVERY change (WHAT tool changed, WHY it was changed, IMPACT on use case) +- **overall_confidence**: 0-10 rating your confidence that chosen tools would help the workflow +- **changes_made**: Count (must be ≤3, including reclassifications) + +--- + +## PHASE 3: Per-Usecase Plans + +PER USECASE PLANS (field per_usecase_plans): For EACH searched usecase in the session, analyze the entire workflow that followed and generate ONE comprehensive plan that captures the overall workflow and critical insights someone would need before attempting this same use case. Think of it as capturing the key details that made this workflow succeed (or fail). +IMPORTANT: There can be multiple usecases inside queries field within a single SEARCH tool call - eg {{"overall_usecase": "fetch recent emails and send an email with summary", "queries": [{{"use_case": "fetch recent emails"}}, {{"use_case": "send email", "known_fields": "recipient:John"}}]}}. In this case, you should generate a plan the overall usecase as well as for each individual usecase. + Also, there can be multiple SEARCH tool calls in the session - so consider overall usecase and individual usecase across all the search calls. + - As there can multiple usecases producing multiple plans, be very careful when matching the workflow execution with individual usecase plans. Do not mix the execution steps - eg don't put steps for sending email in plan for fetching email! + - For the overall usecase plan (if overall_usecase is present), you should generate a plan that captures all the individual usecases in the tool call. It should contain the details of individual usecase plans and any cross-usecase learnings as well. Eg if a search tool had 3 usecases and 1 overall usecase, you must generate 4 plans for this call. + - Number the plans as usecase_i_j or overall_i where i=search number, and j=usecase number within search. Eg usecase_1_1 for 1st usecase in 1st search, overall_1 for overall usecase in 1st search, usecase_2_3 for 3nd usecase in 2nd search, so on. + - Check the tool slug used in multi-execute or remote_workbench calls to help map which usecases would it correspond to. + + Your plan will be surfaced to any new user attempting a similar use case. So the Search Plan should include a crisp sequence of steps to follow and any gotchas to be careful about. Focus on the gap between expectation and reality - what seemed straightforward from the search but turned out complex, or what hidden constraints emerged during execution. + + Key aspects to analyze (but only mention what's most relevant to each search): + + a) Execution plan: + - What tools to call in which sequence, fallbacks tools, required inputs and output format. Capture the happy path and the fallback paths. Focus on app tools not meta tools - eg don't mention multi-execute vs workbench choice. + - If there's an existing plan, you should generate a plan that supersedes that existing plan. Otherwise generate a new plan. + - For each tool call, describe the INTENT - what data or outcome you want to achieve - in plain English. + - DO NOT specify exact parameters for the tools - let the executing LLM determine those based on the available context and tool schemas. + - Eg search query "download attachments from someone in my email since last week" -> Step 1) Call GMAIL_FETCH_EMAILS with query filter, paginate with cursor until all emails received Step 2) Call GMAIL_DOWNLOAD_ATTACHMENT in parallel with email IDs. + - You may include any non-obvious details that were not present in the Tool Schema and Description that was returned by Search. But only include if it is truly useful and broadly applicable to variants of the usecase. + - Rule - don't be overly prescriptive or verbose in the plan, the plan is used as a guidance rather than prescription, so keep it flexible. Focus on high-level strategy. + - ENSURE any tool or parameter names in the plan are accurate and not hallucinated. DO NOT use shorthand or alias for parameter names, always use full parameter names. + + Key considerations in plan + - If pagination is needed, mention it explicitly in the workflow step (e.g., "paginate until cursor exhausted"). + - Time Awareness: For time-sensitive operations (e.g., 'last week', 'last 24 hours'), mention in plan to use the UTC time (usually known from Search). Timezone may need to be handled (ask the user if not known). + - User confirmation: For irreversible/public-impact operations (sending messages, deleting data, sharing resources) - include a step in plan requiring user confirmation before execution. Read-only operations need no confirmation. + - Output verification: While you don't have access to the final output given to user, do your best to analyze from tool call responses whether the required data was found or not. Eg if someone searched for upcoming flights but tool call returned no flights, that's a red flag and you should assess why that may have happened (wrong filters, etc). + - Output formatting: Mention to always include clickable markdown links and inline references to sources (eg provide links to final thread/document instead of IDs). For summaries, provide detailed context with sections/examples, not just numbers. + + b) Data Volume & Performance Patterns: + - Did the workflow encounter unexpected API limits? (e.g., "GMAIL_FETCH_EMAILS caps at 500 messages - requires date-chunked iterations for 6-month audits") + - What pagination strategies were needed? Not just "needs pagination" but WHY (e.g., "Slack channels API returns max 100 per page, use next_cursor for complete list") + - Rate limits or throttling patterns (e.g., "GitHub allows 5000 requests/hour but LIST_COMMITS counts as multiple") + + c) Tool Sequencing & Dependencies: + - Which tools needed to be called in non-obvious orders? (e.g., "JIRA_GET_ISSUE doesn't include custom fields - need JIRA_GET_CUSTOM_FIELDS first, then map field IDs back to issues") + - What data transformations were required between tools? Any cross-app transformations or data merging? + - When did the workflow need tool A's output to properly configure tool B? + - Look for inefficiences in the plan - could the same outcome have been achieved with fewer tool calls. + + d) Existing Plan's Analysis: There can be an existing plan that was surfaced in search to the user - if plan_id / cached_plan_id is present (along with plan/pitfalls). Analyze that plan and preserve the important bits + - If plan was present, did it correctly anticipate the workflow complexity? + - Did the plan identify non-obvious requirements (like needing to fetch user IDs before sending messages)? + - When the plan provided structure, did it prevent trial-and-error that would have occurred otherwise? + - Did execution follow the plan, or did real-world constraints force deviations? + + e) Error Patterns & Recovery: + - What failures occurred and what retry strategy worked? (e.g., "GitHub API returned 502 errors during peak hours - exponential backoff with 3 retries succeeded") + - What validation errors revealed tool input requirements? + + f) Workarounds & Missing Capabilities: + - When did the workflow need proxy_execute because no direct tool existed? + - What creative solutions emerged to work around tool limitations? + + g) Critical Realizations: + - What assumption about the tools proved incorrect? + - What undocumented behavior was discovered? + - What made the difference between this workflow succeeding vs failing? + + h) Workbench hints: + - IF the workbench was used, check if it was necessary. It is meant for processing large datasets or bulk scripting (e.g., 1000+ emails), not small ones (e.g., 10 emails). Flag common misuse cases. + - If the workbench ran too long, suggest breaking the task into smaller workbench runs. Typical runs should finish under 1 minute. + - Workbench use depends on data size - eg "summarize 10 emails" doesn't need it but "summarize 1000 emails" might. Avoid recommending workbench unless data size is explicitly mentioned in usecase. You may mention generic guidance like "if handling 100+ emails, switch to workbench" + - For parsing or any other workbench errors, you can suggest fixes: + - For response parsing error - suggest to call multi_execute first to inspect response schema. You can also provide any schema hints if known. + - For rate limits - sleep with backoff. + - For longer runs, break into smalller tasks + - IF workbench coding is involved, encourage good coding practices - breaking into smaller tasks/functions, adding print statements for debugging, parallelization, checkpointing. Avoid code comments as they eat up context. + + Each plan must be SPECIFIC to that search's use case with concrete details. Include actual numbers, tool names, error codes, and strategies used. For example: + - Bad: "Needed to handle rate limits" + - Good: "Shopify's GraphQL API allows 2 points/second but SHOPIFY_GET_PRODUCTS costs 3 points per call. For this inventory sync of 500 products, had to add 1.5s delays between calls in workbench, turning a 1-minute task into 12 minutes." + + IMPORTANT: For the per_usecase_plans field, return a list of UsecasePlan objects, one for each search query in the session. + + For each UsecasePlan: + - search_id: The id of the COMPOSIO_SEARCH_TOOLS call (eg usecase_1_1, overall_1, usecase_2_3, etc.) + - cleaned_search_usecase: Remove PII/names/identifiers/user-specific info, preserve app names, change to English if in another language. E.g., "send email to John via gmail" → "send email to someone via gmail" + - cleaned_known_fields: Only keep field keys, remove values. E.g., "recipient:John, count:10" → "recipient, count" + - toolkits_required: List of lowercase toolkit slugs required (e.g., ["gmail"], ["gmail", "outlook"]) + - actually_used_toolkits: Optional list that resolves OR relations in toolkits_required. For each element in toolkits_required that contains " OR ", determine which specific toolkit was actually used based on the tools called in the workflow (look at tool names like GMAIL_SEND_EMAIL -> gmail). Return a flat list with OR options resolved. If toolkits_required has no OR relations or if you cannot determine which was used, leave as None. + - was_cached_plan_returned: Whether the cached plan was returned by the search tool - check if plan_id / cached_plan_id, plan, and pitfalls are present in the usecase response + - cached_plan_quality: Assess quality of returned cached plan and pitfalls. + - usecase_plan: Proposed updated UsecasePlanDetails object with these fields: + * workflow_steps (List[str]): List of key steps in workflow execution, capture happy path and fallback paths (if applicable), keep high level. Each step should be a concise string. NEVER include PII or user-specific identifiers. See below for exact plan format. + * difficulty (str): Difficulty assessment as "easy", "medium", or "hard" with short reason explaining why. Consider factors like number of steps, error handling needs, edge cases, etc. + * pitfalls (List[str]): List of gotchas and pitfalls in tool inputs/outputs/behavior - what can go wrong, rate limits, missing data handling, auth issues, edge cases, etc. Each pitfall should be concise and actionable. NEVER include PII or user-specific details. + CRITICAL: Every pitfall MUST name a specific TOOL_NAME - generic advice like "handle errors" or "stay within limits" is noise. Only include pitfalls actually OBSERVED in this session, not theoretical ones. + DON'T REPEAT points between workflow_steps and pitfalls - mention once only. Keep concise and in English. + Note: If an existing plan was found (i.e., plan_id / cached_plan_id is set), ensure that the new plan captures key elements from the existing one and is within the size limits mentioned. Do not remove any key elements from the previous plan unless you are certain they are unnecessary/redundant or incorrect. + - plan_changelog: Short changelog of what changed vs existing plan (e.g., "1) added rate limits, 2) added find projects dependency"). If no existing plan, just say "new". + - success_confidence: 0-10 score - lower if workflow incomplete/missing tools, higher if plan effectively addresses issues + + LIMITS: Keep each plan MAX under 10 lines and 500-1000 characters. Extend beyond this range only if the use case is unusually complex or involves multiple significant challenges. Easy usecases should have very concise plan. + IMPORTANT: Preserve important details from existing plan as your plan supersedes existing plans. You are allowed to remove incorrect or unnecessary information, add/modify details, and improve the plan. The goal is to make a final solid plan. + Return empty list [] if no meaningful plans for any search. + + CRITICAL: Write plans as actionable insights for future workflows: + - PLAN FORMAT: Capture workflow steps using the given tools. + - IMPORTANT: MUST tag each step as: [Required/Optional (condition)] [Prerequisite/Step/Next Step/Fallback]: [Action using TOOL_X]. + - For Optional steps, state WHY it's optional - not just "Optional" but "Optional (if X unknown)" or "Optional (for write operations only)". + - Examples: "Required Prerequisite (if project_id unknown): SUPABASE_LIST_PROJECTS", "Required: GMAIL_SEND_EMAIL with recipient and body", "Optional (if schema unfamiliar): SUPABASE_SELECT_FROM_TABLE to inspect data shape", "Fallback (if FIRECRAWL_SEARCH returns 402): COMPOSIO_SEARCH_WEB". + - Always mention the tool name(s) used in each step. + - BE SPECIFIC in steps: include concrete details observed in execution. "GMAIL_FETCH_EMAILS returns max 100 per page via nextPageToken" beats "paginate through results". Include actual limits, cursors, field names when known. + - DON'T INCLUDE UNNECESSARY REQUIRED STEPS. Don't propose a longer/sub-optimal path when a shorter one exists. + - Assume app connections are already done, DON'T add steps to verify connection/profile unless necessary. + - Don't include too many optional steps, keep it concise. + - ALWAYS create the plan based on execution data, don't make up extra steps. Keep the plan concise and to the point. + - TOOL NAMES MUST BE REAL: Every tool name mentioned in workflow_steps MUST be a real tool that was either executed in this session OR exists in the Composio Tool Catalog. Do NOT invent or hallucinate tool names. New tool suggestions (from PHASE 6) are hypothetical tools that DO NOT EXIST yet - they must NEVER appear in workflow_steps. If a capability gap exists, describe the workaround that was actually used (e.g., proxy_execute), not a hypothetical tool. + - If mentioning any parameters in a step, always mention full parameter name, don't use an alias/shorthand. Correctness is very important. + - NOTE: If the existing plan is not in the above format, you should update the plan to match the format. + - For pitfalls, tag them as [TOOL_NAME] [Pitfall Description] - eg "[GMAIL_BATCH_MODIFY] max 1000 IDs per call", "[SLACK_SEARCH_MESSAGES] 429 after ~100 requests/minute on free tier". + BAD (generic): "Rate limit: throttle requests", "Response handling: parse carefully" + GOOD (specific): "[DATADOG_LIST_LOGS] InvalidArgument when start_at exceeds window - omit pagination", "Response structure: results[i]['response']['data']['logs']" + Numbers beat words: say "max 1000 IDs" not "batch appropriately". Include actual error codes, batch sizes, data paths from this session. + - MUST ENSURE the output is accurate and grounded in real data given to you. NEVER make up any step/parameter/suggestion based on your own knownledge as it can be stale. Base output ONLY on tool schemas and actual execution. + - NEVER make up parameter names that are not specified in tool schema. Always use full parameter names, no shortcut/alias. + - Don't reference memory, meta tools (eg manage_connections), or any other non-app tool information in the plan. + - State the gotcha/constraint directly (e.g., "Slack search is day-level only with exclusive 'after:' parameter") + - Don't mention what "the planner did" or "the agent had to" - just state what needs to be done + - Avoid implementation details like sync_response_to_workbench or file paths + - You may mention what calls can be done in parallel (no dependencies) vs which ones have to be sequential (eg if input of one call goes to next one) + - Keep an eye on existing plan contents - if you strongly think something is not relevant or outdated (eg any pitfall no longer relevant), you may remove it with proper reasoning. + + WORKBENCH GUIDANCE + - If workbench is involved, mention insights about errors and code quality - e.g., "Parallel processing with ThreadPoolExecutor improves 100x over sequential calls", or "break code into smaller tasks with print statements for effective workflow" or "emailId is in data.result.emailId if parsing large results in workbench" + - You may mention workbench tool names (eg `invoke_llm`) only when relevant. + - Workbench should be treated as optional by default. It is only required when data cannot be returned inline (e.g., large datasets, 80k+ characters, or workflows requiring hundreds of tool calls). So it's use is session-dependent - it may not be needed by everyone using the same usecase. It may be required for cases when data involved is highly likely to be massive, eg "archive all emails from past year". + - Include python code snippets ONLY if they help avoid a real mistake - e.g., complex response parsing (logs = [e['content']['message'] for e in response['data']['logs']]), non-obvious pagination, or tricky data transforms. Skip boilerplate like "result = tool.execute()". Code should reveal structure that's not obvious from docs. + - If you mention Workbench, clarify usage boundaries (e.g., "Use Workbench only if data wasn’t returned inline"), and avoid pushing users toward Workbench unless the scenario clearly requires it. Otherise clients tend to bias toward workbench on seeing code samples, but workbench is more error-prone and less optimal than native LLM path for simpler usecases. + - Don't include workbench steps or code snippets for very simple usecases, eg "get recent unread emails" (low data volume / single tool call) + + (continued) GENERAL RULES: + - Focus on API quirks, data structures, rate limits, cross-app dependencies, and non-obvious requirements that future workflows must handle + - ONLY describe patterns you directly observed in THIS session's logs OR present in previously returned plan - do not invent or assume behaviors + - DON'T REPEAT any information that's already present in tool schemas clearly, and avoid unnecessary details to keep the plan concise (eg don't add unnecessary performance optimizations for simple usecases) + - For easy usecases, shorter and direct plans are more effective. For hard usecases covering different edge cases may be needed. Don't complicate simple plans with excessive details. + - Refer the per-usecase tool analysis you proposed earlier and align the tools and toolkits analysis with the plan analysis. Any preference/ordering in tool analysis should reflect in plans as well. The plan should use tools only from the tool analysis - exception is workbench tools that aren't part of tool analysis. + - NEVER put any PII or user-specific info in the plan - eg don't put column names, sheet names, etc. Keep the plan generic. + - Create a plan that can be reused as a standard for future users. Keep the content general and timeless—avoid references to specific sessions, dates, or individuals. Maintain a clear, professional tone. + - GROUNDING RULE: Tool names in workflow_steps must be derived from REAL execution data. Do NOT hallucinate tool names that don't exist in the Composio Tool Catalog. + - CROSS-CUTTING CONCERNS ARE VALUABLE: Capture ALL learnings from the session including infrastructure issues (S3 errors, auth failures, network issues), workbench helper behaviors (upload_local_file, invoke_llm, proxy_execute), and cross-toolkit dependencies. These cross-cutting insights are often the most valuable content - do NOT omit them just because they're not about a specific Composio tool. + - QUALITY TEST: Before finalizing each pitfall, ask: (1) Does it name a specific tool? (2) Does it include specific details (numbers, error codes, field names, OR behavioral quirks discovered)? (3) Was this observed in this session? If not, cut it. + - HONESTY OVER OPTIMISM: If a capability doesn't exist, say so directly (e.g., "No X tools available - return for manual posting") instead of vague hedging ("may require manual intervention"). Confidence scores should match execution reality. + +--- + +## PHASE 4: Search Quality Assessment + +Compare what Search returned (in "Search Results") vs what's available (in "Composio Tool Catalog") to assess search quality. + +- **overall_quality**: EXCELLENT, AVERAGE, or POOR + EXCELLENT - results included all the best tools and dependencies needed to reliably execute the workflow. + AVERAGE - results had non-blocking issues - e.g., some useful tools missing, suboptimal tools chosen, or some irrelevant/bloated results. + POOR - blocking issues - eg critical tools missing or results largely unrelated to the workflow. +- **explanation**: High-level explanation +- **wrong_tools_returned**: List cases with: + - search_id, use_case, tools_returned, tools_missing, tools_irrelevant, reason, severity + - **CRITICAL for tools_missing**: ONLY list tools that EXIST in the Composio Tool Catalog but were NOT returned by Search. NEVER hallucinate or invent tool names. If Search returned the best available tools from the catalog, leave tools_missing EMPTY. + - **tools_irrelevant**: Tools that were returned by Search but are not useful for this use case + +NOTE: Cached plan quality is tracked per-usecase in PHASE 3 (per_usecase_plans) via was_cached_plan_returned, cached_plan_quality, cached_plan_quality_reason fields. + +--- + +## PHASE 5: Planner Issues (Cached Plan Analysis) + +Analyze quality issues with cached plans that were returned. Each issue is tracked per-usecase via search_id. + +- **per_usecase_plan_following**: For each usecase where a cached plan was returned, track whether it was followed: + - search_id: The usecase search_id + - was_cached_plan_followed: True if workflow followed the cached plan's required steps, False if LLM ignored it or deviated + - deviation_reason: If not followed, brief explanation (e.g., "LLM chose different approach", "Plan steps were incorrect") +- **missing_prerequisite_warnings**: Cases where plan didn't warn about prerequisites. Include search_id, plan_id, usecase, missing_prerequisite, consequence, suggested_warning. +- **incorrect_suggestions**: Cases where plan suggested wrong tools/steps. Include search_id, plan_id, usecase, incorrect_suggestion, why_incorrect, actual_solution. +- **overall_planner_feedback**: General observations about cached plan quality across all usecases. + +Only populate for usecases where was_cached_plan_returned is True in their per_usecase_plan. + +--- + +## PHASE 6: Tool/Toolkit Gaps (Product Feedback Only) + +This section captures gaps for FUTURE development - these are tools/toolkits that DO NOT EXIST today. +IMPORTANT: Suggestions here are strictly for product feedback. They must NEVER appear in per_usecase_plans or per_usecase_tool_analysis - those sections must only reference tools that actually exist and were executed. + +Only if there are CLEAR gaps (be conservative): + +- **new_tool_suggestions**: For existing toolkits missing obvious tools (these tools don't exist yet) +- **new_toolkit_suggestions**: For entirely missing integrations (these toolkits don't exist yet) + +**NOT missing tools** (these are improvements to existing tools, not new tools): +- Adding retry/backoff, pagination, input validation, better errors, or new parameters to existing tools + +**IS a missing tool**: A genuinely new API operation not covered by any existing tool + +--- + +## Output + +Return WorkflowAnalysisOutput with all fields populated. +- NEVER include PII in any output +- Always return responses in ENGLISH +""" + + return prompt + + +def _format_session_logs( + session: Session, search_data: Optional[Dict[str, Any]] = None +) -> str: + """Format session logs for the prompt. + + Uses ParsedSession for pre-parsed JSON data and proper errorRequest handling. + For COMPOSIO_SEARCH_TOOLS calls, shows full search results inline using search_data. + + Args: + session: The session containing all calls + search_data: Pre-analyzed search data from analyze_search_results() + """ + # Parse session once + parsed_session = ParsedSession.from_session(session) + + # Build lookup for search results by search_id + search_results_by_id = {} + if search_data: + for uc in search_data.get("per_usecase_results", []): + search_id = uc.get("search_id", "") + if search_id: + search_results_by_id[search_id] = uc + + lines = [] + search_call_count = 0 + + for i, call in enumerate(parsed_session.calls, 1): + lines.append(f"### Call {i}: {call.action_name}") + lines.append(f"Log ID: {call.log_id}") + + # Duration + if call.duration_ms: + lines.append(f"Duration: {call.duration_ms/1000:.2f}s") + + # Special handling for COMPOSIO_SEARCH_TOOLS - show full search results inline + if call.action_name == "COMPOSIO_SEARCH_TOOLS": + search_call_count += 1 + _format_search_call_inline( + call, search_call_count, search_results_by_id, lines + ) + else: + # Standard handling for other calls + if call.request: + request_str = _summarize_dict( + call.request, max_length=WORKFLOW_LOG_REQUEST_MAX_LENGTH + ) + if request_str: + lines.append(f"Request: {request_str}") + + # Response summary - check errorRequest first (primary source for errors) + if call.error_request: + response_str = _summarize_dict( + call.error_request, max_length=WORKFLOW_LOG_RESPONSE_MAX_LENGTH + ) + if response_str: + lines.append(f"Error Response: {response_str}") + elif call.response: + response_str = _summarize_dict( + call.response, max_length=WORKFLOW_LOG_RESPONSE_MAX_LENGTH + ) + if response_str: + lines.append(f"Response: {response_str}") + + # Error (from top-level error field) + if call.top_level_error: + lines.append( + f"ERROR: {call.top_level_error[:WORKFLOW_LOG_ERROR_MAX_LENGTH]}" + ) + + lines.append("") + + return "\n".join(lines) + + +def _format_search_call_inline( + call: ParsedCall, + search_call_number: int, + search_results_by_id: Dict[str, Dict[str, Any]], + lines: List[str], +) -> None: + """ + Format a COMPOSIO_SEARCH_TOOLS call with full search results inline. + + Shows: + - The search request (overall_usecase, queries) + - For each use case result: main tools, related tools, toolkits, cached plan info + """ + # Format request - show overall_usecase and queries + if call.request: + overall_usecase = call.request.get("overall_usecase", "") + queries = call.request.get("queries", []) + + if overall_usecase: + lines.append(f"Overall Use Case: {overall_usecase}") + + if queries: + lines.append(f"Queries ({len(queries)}):") + for q in queries: + if isinstance(q, dict): + use_case = q.get("use_case", "") + known_fields = q.get("known_fields", "") + if known_fields: + lines.append(f" - {use_case} (known: {known_fields})") + else: + lines.append(f" - {use_case}") + + # Format response - show full search results for each use case + lines.append("") + lines.append("**Search Results:**") + + # Find all results for this search call + # search_id format: "usecase_{search_call}_{usecase_index}" or "overall_{search_call}" + # e.g., usecase_1_1, usecase_1_2, overall_1 for search call 1 + # usecase_2_1, usecase_2_2, overall_2 for search call 2 + found_results = False + for search_id, uc in search_results_by_id.items(): + # Match "usecase_N_*" or "overall_N" where N is the search call number + is_usecase_match = search_id.startswith(f"usecase_{search_call_number}_") + is_overall_match = search_id == f"overall_{search_call_number}" + if is_usecase_match or is_overall_match: + found_results = True + _format_usecase_result(uc, lines) + + if not found_results: + # Fallback: show raw response if no structured results found + if call.error_request: + response_str = _summarize_dict( + call.error_request, max_length=WORKFLOW_LOG_RESPONSE_MAX_LENGTH + ) + if response_str: + lines.append(f"Error Response: {response_str}") + elif call.response: + response_str = _summarize_dict( + call.response, max_length=WORKFLOW_LOG_RESPONSE_MAX_LENGTH + ) + if response_str: + lines.append(f"Response: {response_str}") + + +def _format_usecase_result(uc: Dict[str, Any], lines: List[str]) -> None: + """Format a single use case result from search.""" + search_id = uc.get("search_id", "") + use_case = uc.get("use_case", "") + difficulty = uc.get("difficulty", "") + + lines.append(f"\n#### {search_id}: {use_case}") + if difficulty: + lines.append(f"Difficulty: {difficulty}") + + # Main tools + main_tools = uc.get("main_tools", []) + if main_tools: + lines.append(f"\nMain tools ({len(main_tools)}):") + for tool in main_tools: + tool_slug = tool.get("tool_slug", "") + toolkit = tool.get("toolkit", "") + description = tool.get("description", "") + lines.append(f" - {tool_slug} ({toolkit})") + if description: + desc = description.replace("\n", " ").strip()[:DESCRIPTION_MAX_LENGTH] + lines.append(f" {desc}") + + # Related tools + related_tools = uc.get("related_tools", []) + if related_tools: + lines.append(f"\nRelated tools ({len(related_tools)}):") + for tool in related_tools: + tool_slug = tool.get("tool_slug", "") + toolkit = tool.get("toolkit", "") + lines.append(f" - {tool_slug} ({toolkit})") + + # Toolkits + toolkits = uc.get("toolkits", []) + if toolkits: + lines.append(f"\nToolkits: {', '.join(toolkits)}") + + # Cached plan info + if uc.get("plan_id"): + lines.append(f"\nCached Plan ID: {uc.get('plan_id')}") + if uc.get("execution_guidance"): + lines.append(f"Execution Guidance: {uc.get('execution_guidance')}") + if uc.get("recommended_plan_steps"): + lines.append(f"Recommended Steps: {uc.get('recommended_plan_steps')}") + if uc.get("known_pitfalls"): + lines.append(f"Known Pitfalls: {uc.get('known_pitfalls')}") + + +def _summarize_dict( + value: Dict[str, Any], + max_length: int = WORKFLOW_LOG_REQUEST_MAX_LENGTH, +) -> str: + """ + Summarize a pre-parsed dict for prompt display. + + Args: + value: The dict to summarize (already parsed, not a JSON string) + max_length: Maximum length for the result + """ + if not value: + return "" + + parts = [] + for k, v in list(value.items())[:JSON_MAX_KEYS_TO_SHOW]: + v_str = str(v)[:JSON_KEY_VALUE_MAX_LENGTH] + parts.append(f"{k}: {v_str}") + result = "{" + ", ".join(parts) + "}" + if len(value) > JSON_MAX_KEYS_TO_SHOW: + result += f" (+{len(value)-JSON_MAX_KEYS_TO_SHOW} more)" + + # Apply max length to final result + if len(result) > max_length: + result = result[:max_length] + "..." + + return result + + +async def run_workflow_analysis( + session: Session, +) -> tuple[Optional[WorkflowAnalysisOutput], Dict[str, Any]]: + """ + Run workflow analysis on a session. + + This is the first stage of session analysis: + - Understands what the user was trying to do + - Analyzes tool selection per use case + - Creates workflow plans + - Evaluates search quality + - Identifies tool/toolkit gaps + + Args: + session: Session to analyze + + Returns: + Tuple of (WorkflowAnalysisOutput or None if failed, search_data dict) + search_data contains per_usecase_results with raw search tool responses + """ + search_data: Dict[str, Any] = {} + try: + logger.info(f"Starting workflow analysis for session {session.session_id}") + + # Extract search data + search_data = analyze_search_results(session) + logger.info( + f"Extracted {len(search_data.get('tools_found', []))} tools from search" + ) + + # Fetch tool catalog for involved toolkits + tool_catalog = "" + try: + tool_catalog = get_tool_context_string(session) + if tool_catalog: + logger.info("Fetched tool catalog for involved toolkits") + else: + logger.info("No toolkits found in session, skipping tool catalog") + except Exception as e: + logger.warning(f"Failed to fetch tool catalog: {e}") + + # Build prompt + prompt = build_workflow_analysis_prompt(session, search_data, tool_catalog) + + # Call LLM + logger.info("Calling LLM for workflow analysis...") + start = time.time() + output = await _call_llm( + prompt=prompt, + session_id=session.session_id, + user_id=session.user_id, + ) + logger.info(f"Workflow analysis took {time.time()-start:.1f}s") + + if not output: + logger.error("Workflow analysis returned no output") + return None, search_data + + logger.info( + f"Workflow analysis complete: category={output.workflow_category}, " + f"difficulty={output.task_difficulty.value}, " + f"search_quality={output.search_quality.overall_quality.value}" + ) + + return output, search_data + + except Exception as e: + logger.error(f"Workflow analysis failed: {e}", exc_info=True) + return None, search_data + + +async def _call_llm( + prompt: str, + session_id: str, + user_id: Optional[str] = None, +) -> Optional[WorkflowAnalysisOutput]: + """Call LLM for workflow analysis using structured outputs.""" + import os + + from openai import AsyncOpenAI + + try: + helicone_api_key = os.getenv("HELICONE_API_KEY") + helicone_org_id = os.getenv( + "HELICONE_ORG_ID", "bff0d211-66df-4329-aef6-27c1983b5dc9" + ) + environment = os.getenv("ENVIRONMENT", "development") + + # Configure client with Helicone proxy + default_headers = {} + base_url = None + + if helicone_api_key: + base_url = "https://oai.helicone.ai/v1" + default_headers = { + "Helicone-Auth": f"Bearer {helicone_api_key}", + "Helicone-Org-Id": helicone_org_id, + } + + client = AsyncOpenAI( + api_key=os.getenv("OPENAI_API_KEY"), + base_url=base_url, + default_headers=default_headers if default_headers else None, + ) + + # Helicone headers for this request + helicone_headers = {} + if helicone_api_key: + helicone_headers = { + "Helicone-Session-Id": f"session-analysis-{session_id}", + "Helicone-User-Id": user_id or session_id, + "Helicone-Prompt-Id": "workflow-analysis", + "Helicone-Property-Service": "session-analysis-pipeline", + "Helicone-Property-Stage": "call-1-workflow-analysis", + "Helicone-Property-Session-Id": session_id, + "Helicone-Property-Environment": environment, + "Helicone-Tag": "integrator", + } + + # Use structured outputs with response_format for guaranteed schema compliance + response = await client.beta.chat.completions.parse( + model="gpt-5.2", + messages=[ + { + "role": "system", + "content": ( + "Analyze the session workflow, tool usage, and search quality. " + "Provide detailed per-usecase analysis and plans." + ), + }, + {"role": "user", "content": prompt}, + ], + response_format=WorkflowAnalysisOutput, + reasoning_effort="high", + extra_headers=helicone_headers if helicone_headers else None, + ) + + return response.choices[0].message.parsed + + except Exception as e: + logger.error(f"LLM call failed: {e}", exc_info=True) + return None diff --git a/app_tester/rube_learning/workflow_analysis/search_utils.py b/app_tester/rube_learning/workflow_analysis/search_utils.py new file mode 100644 --- /dev/null +++ b/app_tester/rube_learning/workflow_analysis/search_utils.py @@ -0,0 +1,254 @@ +""" +Search utilities for workflow analysis. + +Extracts and analyzes COMPOSIO_SEARCH_TOOLS responses. +""" + +import json +import logging +from typing import Any, Dict + +from app_tester.rube_learning.search_response_models import parseSearchRequest +from app_tester.rube_learning.session_detector import Session +from app_tester.rube_learning.utils import parseSearchResponse + + +logger = logging.getLogger(__name__) + + +def analyze_search_results(session: Session) -> Dict[str, Any]: + """ + Analyze COMPOSIO_SEARCH_TOOLS logs to understand what was searched and returned. + + Returns a summary of: + - What use cases were searched for + - What toolkits were searched + - What tools were returned by search + - Per-usecase results for structured analysis + + Supports new format: + - Request: {"queries": [{"use_case": "...", "known_fields": "..."}], "overall_usecase": "..."} + - Response: {"data": {"results": [{"primary_tool_slugs": [...], "related_tool_slugs": [...], "toolkits": [...]}]}} + """ + search_data: Dict[str, Any] = { + "per_usecase_results": [], + "use_cases": [], + "toolkits_searched": [], + "tools_found": [], + "search_attempts": [], + "search_reasoning": "", + } + + search_call_count = 0 + + for call in session.calls: + if call.get("actionName") == "COMPOSIO_SEARCH_TOOLS": + search_call_count += 1 + try: + # Parse search request using typed parser + request_data = ( + json.loads(call.get("request", "{}")) if call.get("request") else {} + ) + search_request = parseSearchRequest(request_data) + queries = search_request.queries + overall_usecase = search_request.overall_usecase + + # Add overall usecase if present + if overall_usecase and overall_usecase not in search_data["use_cases"]: + search_data["use_cases"].append(overall_usecase) + + # Add individual queries using typed fields + for query in queries: + if ( + query.use_case + and query.use_case not in search_data["use_cases"] + ): + search_data["use_cases"].append(query.use_case) + + # Parse search response + response_data = ( + json.loads(call.get("response", "{}")) + if call.get("response") + else {} + ) + + parsed_response = None + if response_data: + try: + parsed_response = parseSearchResponse( + response_data, search_call_count, overall_usecase + ) + except Exception as e: + logger.debug(f"Failed to parse search response: {e}") + parsed_response = None + + # Extract tools returned by search + if parsed_response: + tool_schemas = parsed_response.data.tool_schemas + results = parsed_response.data.results + + # Process each result for per-usecase structure + for result in results: + search_id = result.search_id or "" + use_case_text = result.use_case or "" + + # Extract main tools with details + main_tools = [] + for slug in result.primary_tool_slugs: + if isinstance(slug, str) and slug: + schema = tool_schemas.get(slug, {}) + main_tools.append( + { + "tool_slug": slug, + "description": schema.get("description", ""), + "toolkit": schema.get("toolkit", ""), + } + ) + + # Extract related tools with details + related_tools = [] + for slug in result.related_tool_slugs: + if isinstance(slug, str) and slug: + schema = tool_schemas.get(slug, {}) + related_tools.append( + { + "tool_slug": slug, + "description": schema.get("description", ""), + "toolkit": schema.get("toolkit", ""), + } + ) + + # Add to per-usecase results + search_data["per_usecase_results"].append( + { + "search_id": search_id, + "index": result.index, + "use_case": use_case_text, + "main_tools": main_tools, + "related_tools": related_tools, + "toolkits": result.toolkits, + "difficulty": result.difficulty or "", + # Cached plan info (deterministically extracted) + "plan_id": result.plan_id, + "execution_guidance": result.execution_guidance, + "recommended_plan_steps": result.recommended_plan_steps, + "known_pitfalls": result.known_pitfalls, + } + ) + + # Extract toolkits and tools for backward compatibility + for result in results: + for tk in result.toolkits: + if ( + isinstance(tk, str) + and tk not in search_data["toolkits_searched"] + ): + search_data["toolkits_searched"].append(tk) + + reasoning = result.reasoning + if reasoning and not search_data["search_reasoning"]: + search_data["search_reasoning"] = reasoning + + # Extract main tool slugs + for slug in result.primary_tool_slugs: + if isinstance(slug, str) and slug: + schema = tool_schemas.get(slug, {}) + tool_info = { + "tool_slug": slug, + "toolkit": schema.get("toolkit") + or (result.toolkits[0] if result.toolkits else ""), + "description": schema.get("description", ""), + "enum": schema.get("enum", ""), + "tags": schema.get("tags", []), + "display_name": schema.get("display_name") or slug, + "tool_set": "main", + "input_schema": schema.get("input_schema", {}), + } + if not any( + t["tool_slug"] == slug + for t in search_data["tools_found"] + ): + search_data["tools_found"].append(tool_info) + + # Extract related tool slugs + for slug in result.related_tool_slugs: + if isinstance(slug, str) and slug: + schema = tool_schemas.get(slug, {}) + tool_info = { + "tool_slug": slug, + "toolkit": schema.get("toolkit") + or (result.toolkits[0] if result.toolkits else ""), + "description": schema.get("description", ""), + "enum": schema.get("enum", ""), + "tags": schema.get("tags", []), + "display_name": schema.get("display_name") or slug, + "tool_set": "related", + "input_schema": schema.get("input_schema", {}), + } + if not any( + t["tool_slug"] == slug + for t in search_data["tools_found"] + ): + search_data["tools_found"].append(tool_info) + + # Store the full search attempt for reference + search_data["search_attempts"].append( + { + "request": request_data, + "response": response_data, + "error": call.get("error"), + } + ) + + except Exception as e: + logger.debug(f"Failed to parse search log: {e}") + + return search_data + + +def was_cached_plan_returned(search_data: Dict[str, Any]) -> bool: + """ + Deterministically check if any cached plan was returned by search. + + Returns True if any per_usecase_result has a non-empty plan_id. + """ + for result in search_data.get("per_usecase_results", []): + if result.get("plan_id"): + return True + return False + + +def extract_tool_info(session: Session) -> Dict[str, Dict[str, Any]]: + """ + Extract tool information from COMPOSIO_SEARCH_TOOLS responses. + + Returns dict of tool_slug -> {description, toolkit, input_schema} for tools returned by search. + Used by error_analysis to provide context for tool improvements. + """ + tool_info: Dict[str, Dict[str, Any]] = {} + + for call in session.calls: + if call.get("actionName") == "COMPOSIO_SEARCH_TOOLS": + try: + response_data = ( + json.loads(call.get("response", "{}")) + if call.get("response") + else {} + ) + + # Get tool_schemas from response + data = response_data.get("data", {}) + tool_schemas = data.get("tool_schemas", {}) + + for tool_slug, schema in tool_schemas.items(): + if isinstance(schema, dict) and tool_slug not in tool_info: + tool_info[tool_slug] = { + "description": schema.get("description", ""), + "toolkit": schema.get("toolkit", ""), + "input_schema": schema.get("input_schema", {}), + } + + except (json.JSONDecodeError, Exception) as e: + logger.debug(f"Failed to parse search response: {e}") + + return tool_info diff --git a/app_tester/runners/action/create_app.py b/app_tester/runners/action/create_app.py --- a/app_tester/runners/action/create_app.py +++ b/app_tester/runners/action/create_app.py @@ -29,7 +29,6 @@ ) from app_tester.constant import ( BUILDER_NUM_THREADS, - CONFIG_PY_TEMPLATE, INTEGRATOR_PATH, TOOL_TEMPLATE, TOP_20_APPS, @@ -129,13 +128,6 @@ def _setup_repo(self) -> str: f.write(config_code) logger.info("Successfully wrote config.ts file") - # Write config.py file - config_py_path = app_path / "config.py" - logger.info("Writing config.py file: {}", config_py_path) - with open(config_py_path, "w", encoding="utf-8") as f: - f.write(CONFIG_PY_TEMPLATE.format(name=self.app_name.capitalize())) - logger.info("Successfully wrote config.py file") - # Create git branch logger.info( "Creating stacked git branch from base branch: {}", self.base_branch diff --git a/app_tester/runners/action/fix.py b/app_tester/runners/action/fix.py --- a/app_tester/runners/action/fix.py +++ b/app_tester/runners/action/fix.py @@ -31,9 +31,8 @@ def execute_main_operation(self) -> ActionTestResult: return action_test_result def get_pr_details(self, execution_result: ActionTestResult) -> PrDetails: - # Get provider info for labeling + # Get provider info for title provider = self.config.model_provider or "openai" - provider_label = f"provider:{provider}" issue_body = f"Fixed the action {self.config.action_name} by fixer-agent using **{provider.upper()}** provider.\n\nThe instruction was:\n```\n{self.config.instruction}\n```\n\n" @@ -66,9 +65,8 @@ def get_pr_details(self, execution_result: ActionTestResult) -> PrDetails: title = f"fix: fix action {self.config.action_name} by fixer-agent ({provider})" - return PrDetails( - title=title, issue_body=issue_body, labels=["fixer-agent", provider_label] - ) + # BaseRunner will auto-add provider label + return PrDetails(title=title, issue_body=issue_body, labels=["fixer-agent"]) @classmethod def parse_args(cls) -> argparse.Namespace: diff --git a/app_tester/runners/action/fix_from_instructions.py b/app_tester/runners/action/fix_from_instructions.py --- a/app_tester/runners/action/fix_from_instructions.py +++ b/app_tester/runners/action/fix_from_instructions.py @@ -65,7 +65,7 @@ def get_pr_details(self, execution_result: ActionTestResult) -> PrDetails: title = f"feat: fix action {self.config.action_name} from instructions" return PrDetails( - title=title, issue_body=issue_body, label="fix-from-instructions" + title=title, issue_body=issue_body, labels=["fix-from-instructions"] ) @classmethod diff --git a/app_tester/runners/action/lint.py b/app_tester/runners/action/lint.py --- a/app_tester/runners/action/lint.py +++ b/app_tester/runners/action/lint.py @@ -28,7 +28,7 @@ def get_pr_details(self, execution_result: AppLinterResponse) -> PrDetails: return PrDetails( title=f"feat: lint app {self.app_name} by lint-agent", issue_body=issue_body, - label="lint-agent", + labels=["lint-agent"], ) @classmethod diff --git a/app_tester/runners/action/response_schema_modifier.py b/app_tester/runners/action/response_schema_modifier.py --- a/app_tester/runners/action/response_schema_modifier.py +++ b/app_tester/runners/action/response_schema_modifier.py @@ -70,7 +70,7 @@ def get_pr_details( return PrDetails( title=f"feat: modify response schema for app {self.app_name} by response-schema-modifier", issue_body=issue_body, - label="response-schema-modifier", + labels=["response-schema-modifier"], ) @classmethod diff --git a/app_tester/runners/action/slack_fix.py b/app_tester/runners/action/slack_fix.py --- a/app_tester/runners/action/slack_fix.py +++ b/app_tester/runners/action/slack_fix.py @@ -123,7 +123,9 @@ def get_pr_details(self, execution_result: ActionTestResult) -> PrDetails: issue_body += "\n```\n\n" title = f"feat: fix action {self.config.action_name} by tool-error-fixer" - return PrDetails(title=title, issue_body=issue_body, label="tool-error-fixer") + return PrDetails( + title=title, issue_body=issue_body, labels=["tool-error-fixer"] + ) @classmethod def parse_args(cls) -> argparse.Namespace: diff --git a/app_tester/runners/action/test_and_fix.py b/app_tester/runners/action/test_and_fix.py --- a/app_tester/runners/action/test_and_fix.py +++ b/app_tester/runners/action/test_and_fix.py @@ -44,7 +44,7 @@ def get_pr_details(self, execution_result: TestAndFixRunnerResult) -> PrDetails: return PrDetails( title=f"feat: test and fix app {self.app_name} by test-and-fix-agent", issue_body=issue_body, - label="fixer-agent", + labels=["fixer-agent"], ) @classmethod diff --git a/app_tester/runners/base.py b/app_tester/runners/base.py --- a/app_tester/runners/base.py +++ b/app_tester/runners/base.py @@ -798,6 +798,13 @@ def run(self) -> RunnerResult: pr_details.issue_body += f"\n**Trace URL:**\n {trace_url}" pr_details.issue_body += f"\n**Langsmith Project:**\n {os.environ.get('LANGSMITH_PROJECT', 'default')}" + # Auto-add provider label to all PRs + provider_label = f"provider:{self.config.model_provider or 'openai'}" + if not pr_details.labels: + pr_details.labels = [] + if provider_label not in pr_details.labels: + pr_details.labels.append(provider_label) + # Add review information to PR if review_result: pr_details = self.add_review_to_pr( diff --git a/app_tester/runners/trigger/build.py b/app_tester/runners/trigger/build.py --- a/app_tester/runners/trigger/build.py +++ b/app_tester/runners/trigger/build.py @@ -87,7 +87,7 @@ def get_pr_details(self, execution_result: TestAndFixTriggerResponse) -> PrDetai return PrDetails( title=f"feat: build trigger {self.config.app_name} by trigger-builder-agent", issue_body=issue_body, - label="trigger-builder-agent", + labels=["trigger-builder-agent"], ) @classmethod diff --git a/app_tester/runners/trigger/fix.py b/app_tester/runners/trigger/fix.py --- a/app_tester/runners/trigger/fix.py +++ b/app_tester/runners/trigger/fix.py @@ -46,7 +46,7 @@ def get_pr_details(self, execution_result: TestAndFixTriggerResponse) -> PrDetai return PrDetails( title=f"feat: fix trigger {self.config.trigger_name} by trigger-fixer-agent", issue_body=issue_body, - label="trigger-fixer-agent", + labels=["trigger-fixer-agent"], ) @classmethod diff --git a/app_tester/test_session_analysis.py b/app_tester/test_session_analysis.py deleted file mode 100644 --- a/app_tester/test_session_analysis.py +++ /dev/null @@ -1,245 +0,0 @@ -#!/usr/bin/env python3 -""" -Quick test script for analyzing a single Rube session. - -Usage: - python app_tester/test_session_analysis.py [--slack] [--clickhouse] - -Examples: - # Test with terminal output only - python app_tester/test_session_analysis.py 2V3-1TRBN - - # Test with Slack notification - python app_tester/test_session_analysis.py 2V3-1TRBN --slack - - # Test with ClickHouse save - python app_tester/test_session_analysis.py 2V3-1TRBN --clickhouse - - # Test everything - python app_tester/test_session_analysis.py 2V3-1TRBN --slack --clickhouse -""" - -import argparse -import asyncio -import os -import sys - -from dotenv import load_dotenv -from rich.console import Console -from slack_sdk import WebClient - -from app_tester.agent_worker_fix_from_logs.services.clickhouse_service import ( - ClickHouseService, -) -from app_tester.rube_learning import AnalysisFormatter, SessionDetector, SlackNotifier - - -load_dotenv() - - -async def test_session(session_id: str, enable_slack: bool, enable_clickhouse: bool): - """Test analysis of a single session""" - console = Console() - - console.print( - f"\n[bold cyan]🧪 Testing Session Analysis for: {session_id}[/bold cyan]" - ) - console.print( - f"Slack: {'✅' if enable_slack else '❌'} | ClickHouse: {'✅' if enable_clickhouse else '❌'}\n" - ) - - # Initialize services - clickhouse = ClickHouseService() - formatter = AnalysisFormatter(console) - - # Initialize Slack if requested - slack_notifier = None - if enable_slack: - slack_token = os.getenv("SLACK_TOKEN") - if slack_token: - slack_notifier = SlackNotifier(WebClient(token=slack_token)) - console.print("[green]✅ Slack notifier initialized[/green]") - else: - console.print("[red]❌ SLACK_TOKEN not found in environment[/red]") - return 1 - - # Step 1: Fetch all logs containing this session ID - console.print( - f"[yellow]📊 Step 1: Fetching logs for session {session_id}...[/yellow]" - ) - - query = f""" - SELECT - id AS log_id, - provider AS app_name, - actionName, - request, - response, - errorRequest AS error, - createdAt AS timestamp, - duration, - userId, - sessionInfo, - connectedAccountId, - projectId - FROM app_db.tool_execution_logs - JOIN app_db.apollo_projects - ON tool_execution_logs.projectId = apollo_projects.id - WHERE actionName IN ( - 'COMPOSIO_SEARCH_TOOLS', - 'COMPOSIO_CREATE_PLAN', - 'COMPOSIO_MULTI_EXECUTE_TOOL', - 'COMPOSIO_REMOTE_WORKBENCH', - 'COMPOSIO_REMOTE_BASH_TOOL', - 'COMPOSIO_MANAGE_CONNECTIONS', - 'COMPOSIO_WAIT_FOR_CONNECTION' - ) - AND userId LIKE 'user_%' - AND apollo_projects.orgId = '3415d5cb-1dea-455a-a6a4-e66d764703e7' - AND ( - id = '{session_id}' - OR sessionInfo LIKE '%{session_id}%' - ) - ORDER BY createdAt ASC - """ - - result = clickhouse._execute_query(query) - logs = result.get("data", []) - - if not logs: - console.print(f"[red]❌ No logs found for session {session_id}[/red]") - return 1 - - console.print(f"[green]✅ Found {len(logs)} logs[/green]") - - # Decrypt logs - decrypted_logs = [] - for log in logs: - decrypted_log = clickhouse._decrypt_log_fields(log) - - # Parse timestamp - timestamp_str = decrypted_log.get("timestamp") - if timestamp_str and isinstance(timestamp_str, str): - from datetime import datetime - - decrypted_log["timestamp"] = datetime.strptime( - timestamp_str.split(".")[0], "%Y-%m-%d %H:%M:%S" - ) - - decrypted_logs.append(decrypted_log) - - # Step 2: Detect session - console.print("[yellow]📊 Step 2: Detecting session boundaries...[/yellow]") - - session_detector = SessionDetector( - marked_session_threshold_minutes=20.0, - unmarked_session_threshold_minutes=10.0, - ) - - sessions = session_detector.detect_sessions(decrypted_logs) - - if not sessions: - console.print( - "[red]❌ No valid sessions detected (must start with COMPOSIO_SEARCH_TOOLS)[/red]" - ) - return 1 - - console.print(f"[green]✅ Detected {len(sessions)} session(s)[/green]") - - # Step 3: Analyze with LLM - from app_tester.analyze_rube_sessions import LLMAnalyzer - - llm_analyzer = LLMAnalyzer() - - for i, session in enumerate(sessions, 1): - console.print( - f"\n[yellow]📊 Step 3: Analyzing session {i}/{len(sessions)} with GPT-5...[/yellow]" - ) - console.print(f"Session: {session.session_key}, Calls: {len(session.calls)}") - - primary_analysis = await llm_analyzer.analyze_session(session) - - if not primary_analysis: - console.print(f"[red]❌ Analysis failed for session {i}[/red]") - continue - - # Combine with default tool gap analysis for testing - from app_tester.rube_learning import SearchQuality, SessionAnalysis - - analysis = SessionAnalysis( - **primary_analysis.model_dump(), - new_tool_suggestions=[], - search_quality=SearchQuality.ACCEPTABLE, - search_quality_explanation="Test mode - tool gap analysis not run", - ) - - # Step 4: Display results - console.print("[yellow]📊 Step 4: Displaying results...[/yellow]") - formatter.display_analysis( - session.user_id, i, len(sessions), analysis, session.session_id - ) - - # Step 5: Send to Slack if enabled - if enable_slack and slack_notifier: - console.print("\n[yellow]📊 Step 5: Sending to Slack...[/yellow]") - success = slack_notifier.send_session_analysis( - session, analysis, i, len(sessions) - ) - if success: - console.print("[green]✅ Sent to Slack successfully[/green]") - else: - console.print("[red]❌ Failed to send to Slack[/red]") - - # Step 6: Save to ClickHouse if enabled - if enable_clickhouse: - console.print("\n[yellow]📊 Step 6: Saving to ClickHouse...[/yellow]") - from app_tester.analyze_rube_sessions import RubeSessionAnalyzer - - # Create temporary analyzer just to use its save method - temp_analyzer = RubeSessionAnalyzer() - success = temp_analyzer.save_to_clickhouse(session, analysis) - if success: - console.print("[green]✅ Saved to ClickHouse successfully[/green]") - else: - console.print("[red]❌ Failed to save to ClickHouse[/red]") - - console.print("\n[bold green]🎉 Test completed successfully![/bold green]") - return 0 - - -def main(): - parser = argparse.ArgumentParser( - description="Quick test script for analyzing a single Rube session" - ) - parser.add_argument( - "session_id", help="Session ID or log ID to analyze (e.g., 2V3-1TRBN)" - ) - parser.add_argument( - "--slack", - action="store_true", - help="Send results to Slack (requires SLACK_TOKEN)", - ) - parser.add_argument( - "--clickhouse", action="store_true", help="Save results to ClickHouse" - ) - - args = parser.parse_args() - - # Validate environment variables - required_vars = [ - "BACKEND_READONLY_URL", - "CLICKHOUSE_USERNAME", - "CLICKHOUSE_PASSWORD", - "OPENAI_API_KEY", - ] - - missing_vars = [var for var in required_vars if not os.getenv(var)] - if missing_vars: - print(f"❌ Missing required environment variables: {', '.join(missing_vars)}") - return 1 - - return asyncio.run(test_session(args.session_id, args.slack, args.clickhouse)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/app_tester/test_session_raw_output.py b/app_tester/test_session_raw_output.py deleted file mode 100644 --- a/app_tester/test_session_raw_output.py +++ /dev/null @@ -1,238 +0,0 @@ -#!/usr/bin/env python3 -""" -Test script that shows RAW SessionAnalysis output (DB format, not Rich formatting). -""" - -import argparse -import asyncio -import json -import sys - -from dotenv import load_dotenv -from rich.console import Console - -from app_tester.agent_worker_fix_from_logs.services.clickhouse_service import ( - ClickHouseService, -) -from app_tester.rube_learning import SessionDetector - - -load_dotenv() - - -async def test_session(session_id: str): - """Test analysis and show raw output""" - console = Console() - - console.print(f"\n[bold cyan]🧪 Testing Session: {session_id}[/bold cyan]\n") - - # Fetch logs - clickhouse = ClickHouseService() - - query = f""" - SELECT - id AS log_id, - provider AS app_name, - actionName, - request, - response, - errorRequest AS error, - createdAt AS timestamp, - duration, - userId, - sessionInfo, - connectedAccountId, - projectId - FROM app_db.tool_execution_logs - JOIN app_db.apollo_projects - ON tool_execution_logs.projectId = apollo_projects.id - WHERE actionName IN ( - 'COMPOSIO_SEARCH_TOOLS', - 'COMPOSIO_CREATE_PLAN', - 'COMPOSIO_MULTI_EXECUTE_TOOL', - 'COMPOSIO_REMOTE_WORKBENCH', - 'COMPOSIO_REMOTE_BASH_TOOL', - 'COMPOSIO_MANAGE_CONNECTIONS', - 'COMPOSIO_WAIT_FOR_CONNECTION' - ) - AND userId LIKE 'user_%' - AND apollo_projects.orgId = '3415d5cb-1dea-455a-a6a4-e66d764703e7' - AND ( - id = '{session_id}' - OR sessionInfo LIKE '%{session_id}%' - ) - ORDER BY createdAt ASC - """ - - result = clickhouse._execute_query(query) - logs = result.get("data", []) - - if not logs: - console.print("[red]❌ No logs found[/red]") - return 1 - - console.print(f"[green]✅ Found {len(logs)} logs[/green]") - - # Decrypt logs - decrypted_logs = [] - for log in logs: - decrypted_log = clickhouse._decrypt_log_fields(log) - timestamp_str = decrypted_log.get("timestamp") - if timestamp_str and isinstance(timestamp_str, str): - from datetime import datetime - - decrypted_log["timestamp"] = datetime.strptime( - timestamp_str.split(".")[0], "%Y-%m-%d %H:%M:%S" - ) - decrypted_logs.append(decrypted_log) - - # Detect session - session_detector = SessionDetector( - marked_session_threshold_minutes=20.0, - unmarked_session_threshold_minutes=10.0, - ) - sessions = session_detector.detect_sessions(decrypted_logs) - - if not sessions: - console.print("[red]❌ No sessions detected[/red]") - return 1 - - console.print(f"[green]✅ Detected {len(sessions)} session(s)[/green]\n") - - # Analyze with LLM - from app_tester.analyze_rube_sessions import LLMAnalyzer - - llm_analyzer = LLMAnalyzer() - - for i, session in enumerate(sessions, 1): - console.print( - f"[yellow]Analyzing session {i}/{len(sessions)} with GPT-5...[/yellow]" - ) - - analysis = await llm_analyzer.analyze_session(session) - - if not analysis: - console.print("[red]❌ Analysis failed[/red]") - continue - - # Show RAW SessionAnalysis object (DB format) - console.print("\n" + "=" * 80) - console.print("[bold cyan]RAW SessionAnalysis OUTPUT (DB Format):[/bold cyan]") - console.print("=" * 80 + "\n") - - # Convert to dict for display - analysis_dict = analysis.model_dump() - - console.print( - f"[bold]Overall Usecase:[/bold] {analysis_dict['overall_usecase']}" - ) - console.print( - f"[bold]Workflow Category:[/bold] {analysis_dict['workflow_category']}" - ) - console.print( - f"[bold]Apps Involved:[/bold] {', '.join(analysis_dict['apps_involved'])}" - ) - console.print( - f"[bold]Tools Involved:[/bold] {', '.join(analysis_dict['tools_involved'])}" - ) - console.print( - f"\n[bold]Workflow Summary:[/bold]\n{analysis_dict['workflow_summary']}" - ) - console.print( - f"\n[bold]Failure Codes:[/bold] {', '.join(analysis_dict['failure_codes'])}" - ) - console.print( - f"\n[bold]Workflow Quality Enum:[/bold] {analysis_dict['workflow_quality_enum']}" - ) - console.print( - f"[bold]Was Workflow Successful:[/bold] {analysis_dict['was_workflow_successful']}" - ) - console.print( - f"[bold]Workflow Quality:[/bold] {analysis_dict['workflow_quality']}" - ) - console.print( - f"[bold]Task Difficulty:[/bold] {analysis_dict['task_difficulty']}" - ) - - console.print( - f"\n[bold]Overall Rationale:[/bold]\n{analysis_dict['overall_rationale']}" - ) - - console.print("\n[bold yellow]RECOMMENDATIONS:[/bold yellow]") - console.print(f"Has Recommendations: {analysis_dict['has_recommendations']}") - console.print(f"Confidence Score: {analysis_dict['confidence_score']}") - - console.print( - f"\n[bold]Tool Improvements ({len(analysis_dict['tool_improvements'])}):[/bold]" - ) - for idx, rec in enumerate(analysis_dict["tool_improvements"], 1): - console.print( - f"\n {idx}. {rec['recommendation_name']} (Priority: {rec['priority']}/10)" - ) - console.print(f" Tool: {rec['tool_name']}") - console.print(f" Success: {rec['was_tool_successful']}") - console.print(f" Type: {rec['improvement_type']}") - console.print(f" Failure Code: {rec['failure_code']}") - console.print(f" Current: {rec['current_value'][:100]}...") - console.print(f" Proposed: {rec['proposed_change'][:100]}...") - console.print(f" Rationale: {rec['rationale'][:150]}...") - - console.print( - f"\n[bold]New Tool Suggestions ({len(analysis_dict['new_tool_suggestions'])}):[/bold]" - ) - for idx, rec in enumerate(analysis_dict["new_tool_suggestions"], 1): - console.print( - f"\n {idx}. {rec['recommendation_name']} (Priority: {rec['priority']}/10)" - ) - console.print(f" Tool Name: {rec['tool_name']}") - console.print(f" Description: {rec['description'][:100]}...") - console.print(f" Rationale: {rec['rationale'][:150]}...") - - console.print( - f"\n[bold]New Skill Suggestions ({len(analysis_dict['new_skill_suggestions'])}):[/bold]" - ) - for idx, rec in enumerate(analysis_dict["new_skill_suggestions"], 1): - console.print( - f"\n {idx}. {rec['recommendation_name']} (Priority: {rec['priority']}/10)" - ) - console.print(f" Skill Name: {rec['skill_name']}") - console.print(f" Description: {rec['description'][:100]}...") - - console.print( - f"\n[bold]Meta Tool Improvements ({len(analysis_dict['meta_tool_improvements'])}):[/bold]" - ) - for idx, rec in enumerate(analysis_dict["meta_tool_improvements"], 1): - console.print( - f"\n {idx}. {rec['recommendation_name']} (Priority: {rec['priority']}/10)" - ) - console.print(f" Meta Tool: {rec['meta_tool']}") - console.print(f" Success: {rec['was_tool_successful']}") - console.print(f" Actual Usage: {rec['actual_usage'][:100]}...") - console.print(f" Improvement: {rec['improvement'][:150]}...") - - console.print("\n[bold green]Recommendation Order:[/bold green]") - for idx, rec_name in enumerate(analysis_dict["recommendation_names_order"], 1): - console.print(f" {idx}. {rec_name}") - - # Save full JSON - output_file = f"/tmp/session_analysis_{session_id}.json" - with open(output_file, "w") as f: - json.dump(analysis_dict, f, indent=2, default=str) - - console.print( - f"\n[bold green]✅ Full JSON saved to: {output_file}[/bold green]" - ) - - return 0 - - -def main(): - parser = argparse.ArgumentParser(description="Show raw SessionAnalysis output") - parser.add_argument("session_id", help="Session ID to analyze") - args = parser.parse_args() - - return asyncio.run(test_session(args.session_id)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/cortex/agents/test_and_fix_agent/agent.py b/cortex/agents/test_and_fix_agent/agent.py new file mode 100644 --- /dev/null +++ b/cortex/agents/test_and_fix_agent/agent.py @@ -0,0 +1,453 @@ +from pathlib import Path +import shutil +import time +import typing as t + +from common.logger import logger +from cortex.common.composio_client import get_auth_data +from cortex.common.failure_summarizer import get_failure_details +from cortex.common.feature_flags import get_feature_flags +from cortex.common.git_utils import Git +from cortex.common.mercury_utils import get_all_action_slugs +from cortex.common.parallel import run_parallel_with_timeouts +from cortex.common.run_summary import run_summary +from cortex.common.sanitise import sanitise_app +from cortex.common.utils import AppTestingResponse, get_testing_dir_response +from cortex.constant import mercury_path +from cortex.generic import ( + AgentProvider, + GenericAgent, + GenericAgentRunner, + create_generic_agent, +) + +from .models import ( + ActionTestAndFixStatus, + ActionTestResult, + TestAndFixActionConfig, + TestAndFixAgentResponse, + TestAndFixAppConfig, + TestAndFixWithFixInstructionConfig, +) +from .prompt import FixActionPrompt, TestAndFixAgentPrompt +from .runner_helper import TestAndFixRunnerHelper + + +# Timeout for test-and-fix with fix instruction (1 hour) +# Shorter timeout - single action fix doesn't have dependency chains +FIX_INSTRUCTION_TIMEOUT_SECONDS = 60 * 60 # 1 hour + +# Timeout for parallel test-and-fix (3 hours) +# Long timeout because agents can depend on each other and wait for dependencies +TEST_AND_FIX_TIMEOUT_SECONDS = 180 * 60 # 3 hours + + +class TestAndFixWithFixInstructionRunner( + GenericAgentRunner[TestAndFixWithFixInstructionConfig, ActionTestResult] +): + max_turns: int = 1000 + + def __init__(self, config: TestAndFixWithFixInstructionConfig): + super().__init__(config) + self._helper = TestAndFixRunnerHelper(config) + + def create(self, *args: t.Any, **kwargs: t.Any) -> GenericAgent: + # Generate unique agent name + agent_name = "fix_action_agent" + + # Use fix tools and FixActionPrompt when fix instruction is provided + custom_tools = self._helper.tool_provider.get_fix_tools() + + status_file_path = self.config.action_status_tracker.status_file_path + + prompt = FixActionPrompt( + app_name=self.config.app_name, + action_name=self.config.action_name, + actions_directory_path=str( + Path(mercury_path) / "apps" / self.config.app_name / "actions" + ), + fix_instruction=self.config.fix_instruction, + retry_feedback=self.config.retry_feedback, + auth_metadata=get_auth_data( + self.config.app_name, self.config.connection_id, redact=True + ), + remove_deprecated_logic=self.config.remove_deprecated_logic, + status_file_path=status_file_path, + ) + + fix_agent = create_generic_agent( + name=agent_name, + prompt=prompt, + tools=custom_tools, + output_type=TestAndFixAgentResponse, + model={AgentProvider.CLAUDE: "claude-sonnet-4-5"}, + enable_web_search=True, + enable_file_read=True, + enable_file_edit=True, + enable_file_create=True, + ) + + return fix_agent + + def run_agent(self) -> ActionTestResult: + return self._helper._run_common( + self, + self.config.action_name, + repro_pending=True, + timeout_seconds=FIX_INSTRUCTION_TIMEOUT_SECONDS, + ) + + @classmethod + def create_and_run( + cls, + config: TestAndFixWithFixInstructionConfig, + *args: t.Any, + **kwargs: t.Any, + ) -> ActionTestResult: + testing_agents_folder_path = ( + Path(mercury_path) / "testing-agents" / config.app_name + ) + if testing_agents_folder_path.exists() and testing_agents_folder_path.is_dir(): + shutil.rmtree(testing_agents_folder_path) + + action_status_tracker = config.action_status_tracker + current_action_name = config.action_name.upper() + all_actions = get_all_action_slugs(config.app_name) + + for action_name in all_actions: + if action_name != current_action_name: + action_status_tracker[action_name] = ActionTestResult( + status=ActionTestAndFixStatus.SUCCESSFUL, + ) + else: + action_status_tracker[action_name] = ActionTestResult( + status=ActionTestAndFixStatus.TO_BE_TESTED, + ) + + runner = cls(config) + run_summary.event(f"FixAction: {config.action_name} starting") + start_time = time.time() + result = runner.run() + duration = time.time() - start_time + + if result.status == ActionTestAndFixStatus.SUCCESSFUL: + run_summary.event( + f"FixAction: {config.action_name} passed ({duration:.1f}s)", + details=result.result_summary, + ) + else: + if result.is_auth_issue: + run_summary.event( + f"[AUTH ISSUE] FixAction: {config.action_name} failed ({duration:.1f}s)", + details=result.result_summary, + ) + else: + run_summary.event( + f"FixAction: {config.action_name} failed ({duration:.1f}s)", + details=result.result_summary, + ) + if result.rate_limited: + run_summary.detail(f"FixAction: {config.action_name} rate limited") + return result + + +class TestAndFixRunner(GenericAgentRunner[TestAndFixActionConfig, ActionTestResult]): + max_turns: int = 500 + + def __init__(self, config: TestAndFixActionConfig): + super().__init__(config) + self._helper = TestAndFixRunnerHelper(config) + + def create(self, *args: t.Any, **kwargs: t.Any) -> GenericAgent: + # Generate unique agent name + agent_name = "comprehensive_test_fix_agent" + + # Always use test tools and TestAndFixAgentPrompt (no fix instruction) + custom_tools = self._helper.tool_provider.get_test_tools() + + # Get status file path for parallel coordination + status_file_path = self.config.action_status_tracker.status_file_path + + prompt = TestAndFixAgentPrompt( + app_name=self.config.app_name, + action_name=self.config.action_name, + test_instruction=self.config.test_instruction, + actions_directory_path=str( + Path(mercury_path) / "apps" / self.config.app_name / "actions" + ), + success_count=3, + auth_metadata=get_auth_data( + self.config.app_name, self.config.connection_id, redact=True + ), + status_file_path=status_file_path, + ) + + test_and_fix_agent = create_generic_agent( + name=agent_name, + prompt=prompt, + tools=custom_tools, + output_type=TestAndFixAgentResponse, + model={AgentProvider.CLAUDE: "claude-sonnet-4-5"}, + enable_file_read=True, + enable_file_edit=True, + enable_file_create=True, + enable_web_search=True, + ) + + return test_and_fix_agent + + def run_agent(self) -> ActionTestResult: + # Use common run logic with repro_pending=False + return self._helper._run_common( + self, + self.config.action_name, + repro_pending=False, + timeout_seconds=TEST_AND_FIX_TIMEOUT_SECONDS, + ) + + @classmethod + def _test_single_action( + cls, action_name: str, config: TestAndFixAppConfig + ) -> ActionTestResult: + """Test a single action (for parallel execution).""" + logger.info(f"[Parallel] Starting test for action: {action_name}") + run_summary.detail(f"TestAndFix: {action_name} starting") + start_time = time.time() + + # Atomically try to claim the action for testing + # This prevents multiple threads from testing the same action + claimed, current_status = config.action_status_tracker.try_claim(action_name) + if not claimed: + # Another thread already claimed it or it's in a terminal state + if current_status: + logger.info( + f"[Parallel] Skipping {action_name} - already " + f"{current_status.status.value}" + ) + return current_status + # Shouldn't happen - action not in tracker + logger.warning(f"[Parallel] Action {action_name} not found in tracker") + return ActionTestResult( + status=ActionTestAndFixStatus.FAILED, + action_name=action_name, + result_summary="Action not found in tracker", + ) + + try: + response = cls( + config=TestAndFixActionConfig( + app_name=config.app_name, + action_name=action_name, + test_instruction=config.test_instruction, + connection_id=config.connection_id, + action_status_tracker=config.action_status_tracker, + already_claimed=True, # We already claimed via try_claim + ) + ).run() + duration = time.time() - start_time + logger.info( + f"[Parallel] Completed action: {action_name} " + f"with status: {response.status.value}" + ) + # Log results to run_summary + if response.status == ActionTestAndFixStatus.SUCCESSFUL: + run_summary.detail( + f"TestAndFix: {action_name} passed ({duration:.1f}s)" + ) + else: + if response.is_auth_issue: + run_summary.detail( + f"[AUTH ISSUE] TestAndFix: {action_name} failed ({duration:.1f}s)" + ) + else: + run_summary.detail( + f"TestAndFix: {action_name} failed ({duration:.1f}s)" + ) + if response.rate_limited: + run_summary.detail(f"TestAndFix: {action_name} rate limited") + return response + except Exception as e: + duration = time.time() - start_time + logger.exception(f"[Parallel] Error testing action {action_name}: {e}") + run_summary.detail( + f"TestAndFix: {action_name} error ({duration:.1f}s) - {str(e)}" + ) + result = ActionTestResult( + status=ActionTestAndFixStatus.FAILED, + action_name=action_name, + result_summary=f"Error during parallel execution: {str(e)}", + write_to_file=True, + ) + config.action_status_tracker.set(action_name, result) + return result + + @classmethod + def test_app( + cls, config: TestAndFixAppConfig + ) -> tuple[bool, AppTestingResponse, t.Optional[str]]: + testing_agents_folder_path = ( + Path(mercury_path) / "testing-agents" / config.app_name + ) + is_dir = testing_agents_folder_path.is_dir() + if testing_agents_folder_path.exists() and is_dir: + shutil.rmtree(testing_agents_folder_path) + + all_actions = get_all_action_slugs(config.app_name) + actions_to_be_tested = ( + list(map(lambda x: x.upper(), config.action_names)) + if config.action_names + else all_actions + ) + run_summary.event(f"TestAndFix: testing {len(actions_to_be_tested)} actions") + logger.info(f"Actions to be tested: {actions_to_be_tested}") + for action_name in all_actions: + if action_name not in actions_to_be_tested: + config.action_status_tracker[action_name] = ActionTestResult( + status=ActionTestAndFixStatus.SUCCESSFUL, + action_name=action_name, + ) + else: + config.action_status_tracker[action_name] = ActionTestResult( + status=ActionTestAndFixStatus.TO_BE_TESTED, + action_name=action_name, + ) + + # Write initial status file for parallel mode + config.action_status_tracker.write_status_file() + + # Get all actions to test + to_be_tested = [ + name + for name, result in config.action_status_tracker.items() + if result.status == ActionTestAndFixStatus.TO_BE_TESTED + ] + logger.info( + "Starting parallel testing of {} actions: {}", + len(to_be_tested), + to_be_tested, + ) + + # Run all actions in parallel (capped by feature_flags.max_parallel_agents) + max_workers = ( + min(len(to_be_tested), get_feature_flags().max_parallel_agents) or 1 + ) + + def make_task(action_name: str) -> t.Callable[[], ActionTestResult]: + return lambda: cls._test_single_action(action_name, config) + + tasks = {action_name: make_task(action_name) for action_name in to_be_tested} + + for task_result in run_parallel_with_timeouts( + tasks=tasks, + timeout_seconds=TEST_AND_FIX_TIMEOUT_SECONDS, + max_workers=max_workers, + ): + action_name = task_result.key + if task_result.timed_out: + logger.exception( + "[Parallel] {} timed out after {} seconds", + action_name, + int(task_result.elapsed_seconds), + ) + run_summary.detail( + f"TestAndFix: {action_name} timed out ({TEST_AND_FIX_TIMEOUT_SECONDS}s)" + ) + config.action_status_tracker.set( + action_name, + ActionTestResult( + status=ActionTestAndFixStatus.FAILED, + action_name=action_name, + result_summary=f"Timed out after {int(task_result.elapsed_seconds)}s", + write_to_file=True, + ), + ) + elif task_result.error: + logger.exception( + "[Parallel] Unhandled exception for {}: {}", + action_name, + task_result.error, + ) + run_summary.detail( + f"TestAndFix: {action_name} exception - {task_result.error}" + ) + config.action_status_tracker.set( + action_name, + ActionTestResult( + status=ActionTestAndFixStatus.FAILED, + action_name=action_name, + result_summary=f"Exception: {task_result.error}", + write_to_file=True, + ), + ) + # Success case: _test_single_action updates the tracker internally + + # Check for deleted actions - use set() to update status file + all_actions_in_app = get_all_action_slugs(config.app_name) + for action in config.action_status_tracker: + if action not in all_actions_in_app: + config.action_status_tracker.set( + action, + ActionTestResult( + status=ActionTestAndFixStatus.FAILED, + action_name=action, + result_summary=( + f"Action {action} not in {config.app_name}. " + f"Available: {all_actions_in_app}. FAILED." + ), + write_to_file=True, + ), + ) + + # Batch operations at the end (after all parallel testing completes) + logger.info("All actions tested. Running batch sanitise + checkpoint.") + Git.checkpoint(f"fix: tested all actions in {config.app_name}") + sanitise_app(config.app_name) + Git.checkpoint(f"fix: sanitised actions in {config.app_name}") + + # App-level summary - only count actions that were actually tested + passed = sum( + 1 + for name, r in config.action_status_tracker.items() + if name in actions_to_be_tested + and r.status == ActionTestAndFixStatus.SUCCESSFUL + ) + total = len(actions_to_be_tested) + failed_count = total - passed + + # Build failure summary for details field using LLM + details: t.Optional[str] = None + if failed_count > 0: + failures: dict[str, str] = {} + for name, result in config.action_status_tracker.items(): + if ( + name in actions_to_be_tested + and result.status == ActionTestAndFixStatus.FAILED + ): + failures[name] = result.result_summary or "Unknown failure" + details = get_failure_details(failures) + + # Check if all failures are auth issues (only if there ARE failures) + failed_auth_flags = [ + result.is_auth_issue + for name, result in config.action_status_tracker.items() + if name in actions_to_be_tested + and result.status == ActionTestAndFixStatus.FAILED + ] + is_auth_issue = bool(failed_auth_flags) and all(failed_auth_flags) + if is_auth_issue: + logger.info("Test and Fix failed due to auth issue") + + # Determine log level: ERROR if all failed, WARNING if some failed, INFO if none + if passed == 0 and failed_count > 0: + level = "ERROR" + elif failed_count > 0: + level = "WARNING" + else: + level = "INFO" + + run_summary.event( + f"TestAndFix: {passed}/{total} actions passed", + level=level, + details=details, + ) + return is_auth_issue, get_testing_dir_response(config.app_name), details diff --git a/pyproject.toml b/pyproject.toml --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,7 @@ dependencies = [ "google-genai==1.19.0", "rich>=13.0.0", "langsmith[claude-agent-sdk]>=0.4.37", + "clickhouse-connect>=0.9.2,<1.0", ] [dependency-groups] @@ -96,6 +97,7 @@ dev = [ "black>=24.1.1", "isort>=5.13.2", "pytest==7.4.2", + "pytest-asyncio>=0.21.1,<0.22", "jurigged==0.5.8", "types-psutil>=7.0.0.20250601", "types-psycopg2>=2.9.21.20250516", diff --git a/pytest.ini b/pytest.ini new file mode 100644 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +markers = + slow: marks tests as slow (deselect with '-m "not slow"') +asyncio_mode = auto +