mcpengine/infra/factory-tools/scripts/validate-all.mjs

#!/usr/bin/env node
/**
 * MCP Factory — Batch Protocol Validation
 * Runs mcp-jest validate on all 30 servers, collects compliance scores.
 */

import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'fs';
import { execSync } from 'child_process';
import { resolve, dirname } from 'path';
import { fileURLToPath } from 'url';

const __dirname = dirname(fileURLToPath(import.meta.url));
const FACTORY_ROOT = resolve(__dirname, '..');
const registry = JSON.parse(readFileSync(resolve(FACTORY_ROOT, 'server-registry.json'), 'utf-8'));
const SERVERS_ROOT = resolve(FACTORY_ROOT, registry.servers_root);

const CONFIGS_DIR = resolve(FACTORY_ROOT, 'test-configs');
const REPORTS_DIR = resolve(FACTORY_ROOT, 'reports');
mkdirSync(REPORTS_DIR, { recursive: true });

const results = [];
let totalScore = 0;

for (const [name, meta] of Object.entries(registry.servers)) {
  const configPath = resolve(CONFIGS_DIR, `${name}.json`);

  if (!existsSync(configPath)) {
    console.log(`⚠️  ${name}: No config — run discover first`);
    results.push({ name, score: null, level: 'SKIPPED', issues: ['No config file'] });
    continue;
  }

  try {
    console.log(`🔬 Validating ${name}...`);
    const output = execSync(`mcp-jest validate --config "${configPath}" 2>&1`, {
      timeout: 30000,
      encoding: 'utf-8',
      stdio: ['pipe', 'pipe', 'pipe']
    });

    // Parse score from output
    const scoreMatch = output.match(/Score:\s*(\d+)\/100/);
    const levelMatch = output.match(/Level:\s*(\S+)/);
    const failedTests = [...output.matchAll(/❌\s*\[(\w+)\s*\]\s*(.+)/g)].map(m => m[2].trim());

    const score = scoreMatch ? parseInt(scoreMatch[1]) : null;
    const level = levelMatch ? levelMatch[1] : 'UNKNOWN';

    totalScore += score || 0;
    results.push({ name, score, level, issues: failedTests });

    const emoji = score >= 95 ? '🟢' : score >= 80 ? '🟡' : '🔴';
    console.log(`   ${emoji} ${name}: ${score}/100 (${level}) ${failedTests.length > 0 ? '— ' + failedTests.length + ' issue(s)' : ''}`);
  } catch (err) {
    // mcp-jest validate exits with code 1 for non-compliant, but still has output
    const output = err.stdout?.toString() || err.stderr?.toString() || '';

    const scoreMatch = output.match(/Score:\s*(\d+)\/100/);
    const levelMatch = output.match(/Level:\s*(\S+)/);
    const failedTests = [...output.matchAll(/❌\s*\[(\w+)\s*\]\s*(.+)/g)].map(m => m[2].trim());

    const score = scoreMatch ? parseInt(scoreMatch[1]) : 0;
    const level = levelMatch ? levelMatch[1] : 'ERROR';

    totalScore += score;
    results.push({ name, score, level, issues: failedTests.length > 0 ? failedTests : [output.split('\n')[0]] });

    const emoji = score >= 95 ? '🟢' : score >= 80 ? '🟡' : '🔴';
    console.log(`   ${emoji} ${name}: ${score}/100 (${level}) ${failedTests.length > 0 ? '— ' + failedTests.length + ' issue(s)' : ''}`);
  }
}

// Summary
const validResults = results.filter(r => r.score !== null);
const avgScore = validResults.length > 0 ? Math.round(totalScore / validResults.length) : 0;
const perfect = validResults.filter(r => r.score >= 95).length;
const good = validResults.filter(r => r.score >= 80 && r.score < 95).length;
const needsWork = validResults.filter(r => r.score < 80).length;

console.log('\n' + '═'.repeat(60));
console.log('        MCP FACTORY — COMPLIANCE REPORT');
console.log('═'.repeat(60));
console.log(`\nServers validated: ${validResults.length}/${results.length}`);
console.log(`Average score:    ${avgScore}/100`);
console.log(`🟢 Compliant (95+):  ${perfect}`);
console.log(`🟡 Near (80-94):     ${good}`);
console.log(`🔴 Needs work (<80): ${needsWork}`);

// Common issues
const allIssues = results.flatMap(r => r.issues);
const issueFreq = {};
for (const issue of allIssues) {
  issueFreq[issue] = (issueFreq[issue] || 0) + 1;
}
const sortedIssues = Object.entries(issueFreq).sort((a, b) => b[1] - a[1]);

if (sortedIssues.length > 0) {
  console.log('\nMost common issues:');
  for (const [issue, count] of sortedIssues.slice(0, 5)) {
    console.log(`  ${count}x — ${issue}`);
  }
}

// Write report
const report = {
  date: new Date().toISOString(),
  summary: { total: results.length, validated: validResults.length, avgScore, perfect, good, needsWork },
  commonIssues: sortedIssues,
  servers: results
};

const reportPath = resolve(REPORTS_DIR, `compliance-${new Date().toISOString().split('T')[0]}.json`);
writeFileSync(reportPath, JSON.stringify(report, null, 2));

// Also write markdown
let md = `# MCP Factory Compliance Report\n\n`;
md += `**Date:** ${new Date().toLocaleDateString()}\n`;
md += `**Average Score:** ${avgScore}/100\n\n`;
md += `| Server | Score | Level | Issues |\n|--------|-------|-------|--------|\n`;
for (const r of results) {
  const emoji = r.score >= 95 ? '🟢' : r.score >= 80 ? '🟡' : r.score === null ? '⚪' : '🔴';
  md += `| ${emoji} ${r.name} | ${r.score ?? '-'}/100 | ${r.level} | ${r.issues.join('; ') || 'None'} |\n`;
}

const mdPath = resolve(REPORTS_DIR, `compliance-${new Date().toISOString().split('T')[0]}.md`);
writeFileSync(mdPath, md);
console.log(`\nReports saved:\n  ${reportPath}\n  ${mdPath}`);