128 lines
5.1 KiB
JavaScript
128 lines
5.1 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* MCP Factory — Batch Protocol Validation
|
|
* Runs mcp-jest validate on all 30 servers, collects compliance scores.
|
|
*/
|
|
|
|
import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'fs';
|
|
import { execSync } from 'child_process';
|
|
import { resolve, dirname } from 'path';
|
|
import { fileURLToPath } from 'url';
|
|
|
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
const FACTORY_ROOT = resolve(__dirname, '..');
|
|
const registry = JSON.parse(readFileSync(resolve(FACTORY_ROOT, 'server-registry.json'), 'utf-8'));
|
|
const SERVERS_ROOT = resolve(FACTORY_ROOT, registry.servers_root);
|
|
|
|
const CONFIGS_DIR = resolve(FACTORY_ROOT, 'test-configs');
|
|
const REPORTS_DIR = resolve(FACTORY_ROOT, 'reports');
|
|
mkdirSync(REPORTS_DIR, { recursive: true });
|
|
|
|
const results = [];
|
|
let totalScore = 0;
|
|
|
|
for (const [name, meta] of Object.entries(registry.servers)) {
|
|
const configPath = resolve(CONFIGS_DIR, `${name}.json`);
|
|
|
|
if (!existsSync(configPath)) {
|
|
console.log(`⚠️ ${name}: No config — run discover first`);
|
|
results.push({ name, score: null, level: 'SKIPPED', issues: ['No config file'] });
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
console.log(`🔬 Validating ${name}...`);
|
|
const output = execSync(`mcp-jest validate --config "${configPath}" 2>&1`, {
|
|
timeout: 30000,
|
|
encoding: 'utf-8',
|
|
stdio: ['pipe', 'pipe', 'pipe']
|
|
});
|
|
|
|
// Parse score from output
|
|
const scoreMatch = output.match(/Score:\s*(\d+)\/100/);
|
|
const levelMatch = output.match(/Level:\s*(\S+)/);
|
|
const failedTests = [...output.matchAll(/❌\s*\[(\w+)\s*\]\s*(.+)/g)].map(m => m[2].trim());
|
|
|
|
const score = scoreMatch ? parseInt(scoreMatch[1]) : null;
|
|
const level = levelMatch ? levelMatch[1] : 'UNKNOWN';
|
|
|
|
totalScore += score || 0;
|
|
results.push({ name, score, level, issues: failedTests });
|
|
|
|
const emoji = score >= 95 ? '🟢' : score >= 80 ? '🟡' : '🔴';
|
|
console.log(` ${emoji} ${name}: ${score}/100 (${level}) ${failedTests.length > 0 ? '— ' + failedTests.length + ' issue(s)' : ''}`);
|
|
} catch (err) {
|
|
// mcp-jest validate exits with code 1 for non-compliant, but still has output
|
|
const output = err.stdout?.toString() || err.stderr?.toString() || '';
|
|
|
|
const scoreMatch = output.match(/Score:\s*(\d+)\/100/);
|
|
const levelMatch = output.match(/Level:\s*(\S+)/);
|
|
const failedTests = [...output.matchAll(/❌\s*\[(\w+)\s*\]\s*(.+)/g)].map(m => m[2].trim());
|
|
|
|
const score = scoreMatch ? parseInt(scoreMatch[1]) : 0;
|
|
const level = levelMatch ? levelMatch[1] : 'ERROR';
|
|
|
|
totalScore += score;
|
|
results.push({ name, score, level, issues: failedTests.length > 0 ? failedTests : [output.split('\n')[0]] });
|
|
|
|
const emoji = score >= 95 ? '🟢' : score >= 80 ? '🟡' : '🔴';
|
|
console.log(` ${emoji} ${name}: ${score}/100 (${level}) ${failedTests.length > 0 ? '— ' + failedTests.length + ' issue(s)' : ''}`);
|
|
}
|
|
}
|
|
|
|
// Summary
|
|
const validResults = results.filter(r => r.score !== null);
|
|
const avgScore = validResults.length > 0 ? Math.round(totalScore / validResults.length) : 0;
|
|
const perfect = validResults.filter(r => r.score >= 95).length;
|
|
const good = validResults.filter(r => r.score >= 80 && r.score < 95).length;
|
|
const needsWork = validResults.filter(r => r.score < 80).length;
|
|
|
|
console.log('\n' + '═'.repeat(60));
|
|
console.log(' MCP FACTORY — COMPLIANCE REPORT');
|
|
console.log('═'.repeat(60));
|
|
console.log(`\nServers validated: ${validResults.length}/${results.length}`);
|
|
console.log(`Average score: ${avgScore}/100`);
|
|
console.log(`🟢 Compliant (95+): ${perfect}`);
|
|
console.log(`🟡 Near (80-94): ${good}`);
|
|
console.log(`🔴 Needs work (<80): ${needsWork}`);
|
|
|
|
// Common issues
|
|
const allIssues = results.flatMap(r => r.issues);
|
|
const issueFreq = {};
|
|
for (const issue of allIssues) {
|
|
issueFreq[issue] = (issueFreq[issue] || 0) + 1;
|
|
}
|
|
const sortedIssues = Object.entries(issueFreq).sort((a, b) => b[1] - a[1]);
|
|
|
|
if (sortedIssues.length > 0) {
|
|
console.log('\nMost common issues:');
|
|
for (const [issue, count] of sortedIssues.slice(0, 5)) {
|
|
console.log(` ${count}x — ${issue}`);
|
|
}
|
|
}
|
|
|
|
// Write report
|
|
const report = {
|
|
date: new Date().toISOString(),
|
|
summary: { total: results.length, validated: validResults.length, avgScore, perfect, good, needsWork },
|
|
commonIssues: sortedIssues,
|
|
servers: results
|
|
};
|
|
|
|
const reportPath = resolve(REPORTS_DIR, `compliance-${new Date().toISOString().split('T')[0]}.json`);
|
|
writeFileSync(reportPath, JSON.stringify(report, null, 2));
|
|
|
|
// Also write markdown
|
|
let md = `# MCP Factory Compliance Report\n\n`;
|
|
md += `**Date:** ${new Date().toLocaleDateString()}\n`;
|
|
md += `**Average Score:** ${avgScore}/100\n\n`;
|
|
md += `| Server | Score | Level | Issues |\n|--------|-------|-------|--------|\n`;
|
|
for (const r of results) {
|
|
const emoji = r.score >= 95 ? '🟢' : r.score >= 80 ? '🟡' : r.score === null ? '⚪' : '🔴';
|
|
md += `| ${emoji} ${r.name} | ${r.score ?? '-'}/100 | ${r.level} | ${r.issues.join('; ') || 'None'} |\n`;
|
|
}
|
|
|
|
const mdPath = resolve(REPORTS_DIR, `compliance-${new Date().toISOString().split('T')[0]}.md`);
|
|
writeFileSync(mdPath, md);
|
|
console.log(`\nReports saved:\n ${reportPath}\n ${mdPath}`);
|