mcpengine/studio/packages/ai-pipeline/streaming/parser.ts

// Stream Parser — extracts structured PipelineEvents from Claude's streaming text

import type {
  PipelineEvent,
  ToolDefinition,
} from '../types';

/**
 * Parse Claude streaming text into PipelineEvent objects.
 * Scans for JSON blocks, code blocks, and status markers in the accumulated text.
 *
 * @param text - Accumulated text from Claude stream so far
 * @param phase - Current pipeline phase (analysis, generate, design, test)
 * @returns Array of events found in the text (deduplicated by caller)
 */
export function parseStreamEvents(
  text: string,
  phase: 'analysis' | 'generate' | 'design' | 'test'
): PipelineEvent[] {
  const events: PipelineEvent[] = [];

  switch (phase) {
    case 'analysis':
      events.push(...parseAnalysisEvents(text));
      break;
    case 'generate':
      events.push(...parseGenerateEvents(text));
      break;
    case 'design':
      events.push(...parseDesignEvents(text));
      break;
    case 'test':
      events.push(...parseTestEvents(text));
      break;
  }

  return events;
}

/** Look for tool definitions appearing in analysis stream */
function parseAnalysisEvents(text: string): PipelineEvent[] {
  const events: PipelineEvent[] = [];
  const toolPatterns = [
    // Match tool objects in JSON arrays
    /\{\s*"name"\s*:\s*"([^"]+)"\s*,\s*"description"\s*:\s*"([^"]+)"[^}]*"endpoint"\s*:\s*"([^"]+)"[^}]*"method"\s*:\s*"([^"]+)"/g,
  ];

  for (const pattern of toolPatterns) {
    let match;
    while ((match = pattern.exec(text)) !== null) {
      const partialTool: Partial<ToolDefinition> = {
        name: match[1],
        description: match[2],
        endpoint: match[3],
        method: match[4],
      };
      events.push({
        type: 'analysis:tool_found',
        tool: partialTool as ToolDefinition,
      });
    }
  }

  // Check for progress markers
  const progressMarkers = [
    { pattern: /analyzing\s+endpoint/i, step: 'Analyzing endpoints', percent: 20 },
    { pattern: /auth(?:entication|orization)\s+(?:flow|config|type)/i, step: 'Detecting auth flow', percent: 40 },
    { pattern: /tool\s+(?:group|definition|mapping)/i, step: 'Mapping tools', percent: 60 },
    { pattern: /app\s+candidate/i, step: 'Identifying app candidates', percent: 75 },
    { pattern: /rate\s+limit/i, step: 'Checking rate limits', percent: 85 },
  ];

  for (const marker of progressMarkers) {
    if (marker.pattern.test(text)) {
      events.push({
        type: 'analysis:progress',
        step: marker.step,
        percent: marker.percent,
      });
    }
  }

  return events;
}

/** Look for file blocks in generation stream */
function parseGenerateEvents(text: string): PipelineEvent[] {
  const events: PipelineEvent[] = [];

  // Match completed file code blocks
  const fileBlockRegex = /```(\w+)\s*\/\/\s*path:\s*(\S+)\s*\n([\s\S]*?)```/g;
  let match;
  while ((match = fileBlockRegex.exec(text)) !== null) {
    events.push({
      type: 'generate:file_ready',
      path: match[2].trim(),
      content: match[3].trim(),
    });
  }

  return events;
}

/** Look for HTML output in design stream */
function parseDesignEvents(text: string): PipelineEvent[] {
  const events: PipelineEvent[] = [];

  // Check if HTML block is being written
  if (text.includes('```html') && !text.includes('```html\n')) {
    // Still writing the opening
    return events;
  }

  const progressMarkers = [
    { pattern: /<!DOCTYPE|<html/i, step: 'Generating HTML structure', percent: 30 },
    { pattern: /<style|tailwind/i, step: 'Applying styles', percent: 50 },
    { pattern: /mcpRequest|data-mcp-tool/i, step: 'Wiring tool bindings', percent: 70 },
    { pattern: /<\/html>/i, step: 'Finalizing app', percent: 90 },
  ];

  for (const marker of progressMarkers) {
    if (marker.pattern.test(text)) {
      events.push({
        type: 'design:progress',
        app: 'current',
        percent: marker.percent,
      });
    }
  }

  return events;
}

/** Look for test result markers */
function parseTestEvents(text: string): PipelineEvent[] {
  const events: PipelineEvent[] = [];

  // Check for pass/fail indicators
  const passPattern = /✓|✅|PASS|passed/gi;
  const failPattern = /✗|❌|FAIL|failed/gi;

  const passes = (text.match(passPattern) || []).length;
  const failures = (text.match(failPattern) || []).length;

  if (passes + failures > 0) {
    // We can infer progress from test count
    events.push({
      type: 'analysis:progress',
      step: `Tests running: ${passes} passed, ${failures} failed`,
      percent: Math.min(80, (passes + failures) * 10),
    });
  }

  return events;
}

/**
 * Extract a JSON object from text, handling code blocks and raw JSON.
 */
export function extractJSON<T = unknown>(text: string): T | null {
  // Try code block first
  const codeBlockMatch = text.match(/```(?:json)?\s*\n([\s\S]*?)\n```/);
  if (codeBlockMatch) {
    try {
      return JSON.parse(codeBlockMatch[1]) as T;
    } catch {
      // fall through
    }
  }

  // Try raw JSON (find outermost braces)
  const braceStart = text.indexOf('{');
  const braceEnd = text.lastIndexOf('}');
  if (braceStart !== -1 && braceEnd > braceStart) {
    try {
      return JSON.parse(text.slice(braceStart, braceEnd + 1)) as T;
    } catch {
      return null;
    }
  }

  return null;
}