281 lines
11 KiB
Bash
Executable File
281 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ═══════════════════════════════════════════════════════════
|
|
# GooseFactory MCP Server — Tool Verification Script
|
|
#
|
|
# Tests all 12 MCP tools via JSON-RPC over stdio.
|
|
# Prerequisites:
|
|
# - API running at http://localhost:4000
|
|
# - MCP server compiled (packages/mcp-server/dist/index.js)
|
|
#
|
|
# Usage: ./scripts/test-mcp-tools.sh
|
|
# ═══════════════════════════════════════════════════════════
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
MCP_SERVER="$PROJECT_DIR/packages/mcp-server/dist/index.js"
|
|
API_URL="${FACTORY_API_URL:-http://localhost:4000}"
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
ERRORS=()
|
|
|
|
# ─── Preflight checks ────────────────────────────────────
|
|
echo -e "${BLUE}═══════════════════════════════════════════════════════${NC}"
|
|
echo -e "${BLUE} GooseFactory MCP Server — Tool Verification${NC}"
|
|
echo -e "${BLUE}═══════════════════════════════════════════════════════${NC}"
|
|
echo ""
|
|
|
|
# Check API health
|
|
echo -n "Checking API health... "
|
|
HEALTH=$(curl -s "$API_URL/health" 2>/dev/null || echo "FAIL")
|
|
if echo "$HEALTH" | grep -q '"status":"ok"'; then
|
|
echo -e "${GREEN}✅ API healthy${NC}"
|
|
else
|
|
echo -e "${RED}❌ API not reachable at $API_URL${NC}"
|
|
echo " Start it with: cd packages/api && DEV_MODE=true npx tsx src/index.ts"
|
|
exit 1
|
|
fi
|
|
|
|
# Check MCP server exists
|
|
if [ ! -f "$MCP_SERVER" ]; then
|
|
echo -e "${RED}❌ MCP server not compiled at $MCP_SERVER${NC}"
|
|
echo " Build it with: cd packages/mcp-server && npx tsc"
|
|
exit 1
|
|
fi
|
|
echo -e "MCP server: ${GREEN}$MCP_SERVER${NC}"
|
|
echo ""
|
|
|
|
# ─── Helper: call MCP tool via stdio ─────────────────────
|
|
call_mcp_tool() {
|
|
local tool_name="$1"
|
|
local arguments="$2"
|
|
|
|
printf '%s\n%s\n%s\n' \
|
|
'{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"mcp-test","version":"1.0"}}}' \
|
|
'{"jsonrpc":"2.0","method":"notifications/initialized"}' \
|
|
"{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"tools/call\",\"params\":{\"name\":\"$tool_name\",\"arguments\":$arguments}}" \
|
|
| node "$MCP_SERVER" 2>/dev/null | tail -1
|
|
}
|
|
|
|
# ─── Helper: run test ────────────────────────────────────
|
|
run_test() {
|
|
local num="$1"
|
|
local name="$2"
|
|
local args="$3"
|
|
local desc="$4"
|
|
|
|
echo -n " $num. $name... "
|
|
|
|
local result
|
|
result=$(call_mcp_tool "$name" "$args" 2>/dev/null)
|
|
|
|
if [ -z "$result" ]; then
|
|
echo -e "${RED}❌ NO RESPONSE${NC}"
|
|
FAIL=$((FAIL + 1))
|
|
ERRORS+=("$name: No response from MCP server")
|
|
return
|
|
fi
|
|
|
|
# Check for JSON-RPC error
|
|
if echo "$result" | grep -q '"error"'; then
|
|
local err_msg
|
|
err_msg=$(echo "$result" | sed 's/.*"message":"\([^"]*\)".*/\1/' | head -c 100)
|
|
echo -e "${RED}❌ RPC ERROR: $err_msg${NC}"
|
|
FAIL=$((FAIL + 1))
|
|
ERRORS+=("$name: $err_msg")
|
|
return
|
|
fi
|
|
|
|
# Check for tool-level isError
|
|
if echo "$result" | grep -q '"isError":true'; then
|
|
local err_text
|
|
err_text=$(echo "$result" | sed 's/.*"text":"\([^"]*\).*/\1/' | head -c 120)
|
|
echo -e "${RED}❌ $err_text${NC}"
|
|
FAIL=$((FAIL + 1))
|
|
ERRORS+=("$name: $err_text")
|
|
return
|
|
fi
|
|
|
|
# Extract first 100 chars of text content for display
|
|
local preview
|
|
preview=$(echo "$result" | sed 's/.*"text":"\([^"]*\).*/\1/' | head -c 100)
|
|
echo -e "${GREEN}✅${NC} ${preview}"
|
|
PASS=$((PASS + 1))
|
|
}
|
|
|
|
# ─── Get test data from API ──────────────────────────────
|
|
echo -e "${YELLOW}Fetching test data from API...${NC}"
|
|
|
|
# Get a pipeline ID
|
|
PIPELINE_ID=$(curl -s "$API_URL/v1/pipelines?limit=1" 2>/dev/null | sed 's/.*"id":"\([^"]*\)".*/\1/' | head -c 36)
|
|
echo " Pipeline ID: $PIPELINE_ID"
|
|
|
|
# Create a test pipeline for advance/deploy/test operations
|
|
NEW_PIPELINE=$(curl -s -X POST "$API_URL/v1/pipelines" \
|
|
-H 'Content-Type: application/json' \
|
|
-d '{"name":"mcp-test-run-'"$(date +%s)"'","platform":"test","priority":"low"}' 2>/dev/null)
|
|
TEST_PIPELINE_ID=$(echo "$NEW_PIPELINE" | sed 's/.*"id":"\([^"]*\)".*/\1/' | head -c 36)
|
|
echo " Test Pipeline ID: $TEST_PIPELINE_ID"
|
|
|
|
# Advance test pipeline to review (creates approval task with real UUID)
|
|
# intake -> scaffolding -> building -> testing (4 advances to get to review)
|
|
for i in 1 2 3; do
|
|
curl -s -X POST "$API_URL/v1/pipelines/$TEST_PIPELINE_ID/stages/advance" \
|
|
-H 'Content-Type: application/json' -d '{"skipValidation":true}' > /dev/null 2>&1
|
|
done
|
|
# This advance goes to review (requires approval) -> creates task
|
|
ADVANCE_RESULT=$(curl -s -X POST "$API_URL/v1/pipelines/$TEST_PIPELINE_ID/stages/advance" \
|
|
-H 'Content-Type: application/json' -d '{"skipValidation":true}' 2>/dev/null)
|
|
|
|
# Extract task ID from tasksCreated array - get the id after "tasksCreated"
|
|
TASK_ID=$(echo "$ADVANCE_RESULT" | node -e "
|
|
let d=''; process.stdin.on('data',c=>d+=c); process.stdin.on('end',()=>{
|
|
try {
|
|
const j=JSON.parse(d);
|
|
const tasks = j.data?.tasksCreated || [];
|
|
if (tasks.length > 0) { console.log(tasks[0].id); }
|
|
else { console.log(''); }
|
|
} catch(e) { console.log(''); }
|
|
});" 2>/dev/null)
|
|
|
|
if [ -z "$TASK_ID" ] || [ ${#TASK_ID} -lt 36 ]; then
|
|
echo " (No task from advance, checking pending tasks...)"
|
|
TASK_ID=$(curl -s "$API_URL/v1/tasks?status=pending&limit=5" | node -e "
|
|
let d=''; process.stdin.on('data',c=>d+=c); process.stdin.on('end',()=>{
|
|
try {
|
|
const j=JSON.parse(d);
|
|
// Find a task with a valid RFC4122 UUID (variant byte 8-b)
|
|
const t = j.data?.find(t => /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(t.id));
|
|
if (t) console.log(t.id); else console.log('');
|
|
} catch(e) { console.log(''); }
|
|
});" 2>/dev/null)
|
|
fi
|
|
echo " Task ID: $TASK_ID"
|
|
|
|
# Create a second pipeline for the reject task
|
|
NEW_PIPELINE2=$(curl -s -X POST "$API_URL/v1/pipelines" \
|
|
-H 'Content-Type: application/json' \
|
|
-d '{"name":"mcp-reject-test-'"$(date +%s)"'","platform":"test","priority":"low"}' 2>/dev/null)
|
|
TEST_PIPELINE2_ID=$(echo "$NEW_PIPELINE2" | node -e "
|
|
let d=''; process.stdin.on('data',c=>d+=c); process.stdin.on('end',()=>{
|
|
try { console.log(JSON.parse(d).data.id); } catch(e) { console.log(''); }
|
|
});" 2>/dev/null)
|
|
|
|
# Advance pipeline2 to review
|
|
for i in 1 2 3; do
|
|
curl -s -X POST "$API_URL/v1/pipelines/$TEST_PIPELINE2_ID/stages/advance" \
|
|
-H 'Content-Type: application/json' -d '{"skipValidation":true}' > /dev/null 2>&1
|
|
done
|
|
REJECT_RESULT=$(curl -s -X POST "$API_URL/v1/pipelines/$TEST_PIPELINE2_ID/stages/advance" \
|
|
-H 'Content-Type: application/json' -d '{"skipValidation":true}' 2>/dev/null)
|
|
REJECT_TASK_ID=$(echo "$REJECT_RESULT" | node -e "
|
|
let d=''; process.stdin.on('data',c=>d+=c); process.stdin.on('end',()=>{
|
|
try {
|
|
const j=JSON.parse(d);
|
|
const tasks = j.data?.tasksCreated || [];
|
|
if (tasks.length > 0) { console.log(tasks[0].id); }
|
|
else { console.log(''); }
|
|
} catch(e) { console.log(''); }
|
|
});" 2>/dev/null)
|
|
echo " Reject Task ID: $REJECT_TASK_ID"
|
|
echo ""
|
|
|
|
# ─── Run all 12 tests ────────────────────────────────────
|
|
echo -e "${BLUE}Running 12 tool tests...${NC}"
|
|
echo ""
|
|
|
|
# === Task Tools (3) ===
|
|
echo -e "${YELLOW}Task Tools:${NC}"
|
|
run_test "01" "factory_get_pending_tasks" \
|
|
'{"limit":5}' \
|
|
"Get pending tasks"
|
|
|
|
run_test "02" "factory_approve_task" \
|
|
"{\"task_id\":\"$TASK_ID\",\"notes\":\"Test approval via script\"}" \
|
|
"Approve a task"
|
|
|
|
run_test "03" "factory_reject_task" \
|
|
"{\"task_id\":\"${REJECT_TASK_ID:-$TASK_ID}\",\"reason\":\"Test rejection via script\",\"severity\":\"minor\"}" \
|
|
"Reject a task"
|
|
|
|
echo ""
|
|
|
|
# === Pipeline Tools (4) ===
|
|
echo -e "${YELLOW}Pipeline Tools:${NC}"
|
|
run_test "04" "factory_get_pipeline_status" \
|
|
'{"status":"active"}' \
|
|
"Get pipeline status"
|
|
|
|
run_test "05" "factory_advance_stage" \
|
|
"{\"pipeline_id\":\"$PIPELINE_ID\",\"skip_validation\":true,\"notes\":\"Test script advance\"}" \
|
|
"Advance pipeline stage"
|
|
|
|
run_test "06" "factory_create_pipeline" \
|
|
"{\"name\":\"test-script-pipeline-$(date +%s)\",\"platform\":\"test\",\"priority\":\"low\"}" \
|
|
"Create a new pipeline"
|
|
|
|
run_test "07" "factory_search" \
|
|
'{"query":"mcp","limit":5}' \
|
|
"Search across entities"
|
|
|
|
echo ""
|
|
|
|
# === Operations Tools (4) ===
|
|
echo -e "${YELLOW}Operations Tools:${NC}"
|
|
run_test "08" "factory_assign_priority" \
|
|
"{\"entity_type\":\"pipeline\",\"entity_id\":\"$PIPELINE_ID\",\"priority\":\"high\",\"reason\":\"Test script priority change\"}" \
|
|
"Assign priority"
|
|
|
|
run_test "09" "factory_get_blockers" \
|
|
'{}' \
|
|
"Get blockers"
|
|
|
|
run_test "10" "factory_run_tests" \
|
|
"{\"pipeline_id\":\"$PIPELINE_ID\",\"test_type\":\"all\"}" \
|
|
"Run tests"
|
|
|
|
run_test "11" "factory_deploy" \
|
|
"{\"pipeline_id\":\"$PIPELINE_ID\",\"target\":\"staging\",\"dry_run\":true}" \
|
|
"Deploy (dry run)"
|
|
|
|
echo ""
|
|
|
|
# === Review Tools (1) ===
|
|
echo -e "${YELLOW}Review Tools:${NC}"
|
|
run_test "12" "factory_request_review" \
|
|
"{\"pipeline_id\":\"$PIPELINE_ID\",\"modal_type\":\"traffic-light\"}" \
|
|
"Request review modal"
|
|
|
|
echo ""
|
|
|
|
# ─── Summary ─────────────────────────────────────────────
|
|
echo -e "${BLUE}═══════════════════════════════════════════════════════${NC}"
|
|
echo -e " Results: ${GREEN}$PASS passed${NC} / ${RED}$FAIL failed${NC} / 12 total"
|
|
echo -e "${BLUE}═══════════════════════════════════════════════════════${NC}"
|
|
|
|
if [ ${#ERRORS[@]} -gt 0 ]; then
|
|
echo ""
|
|
echo -e "${RED}Failures:${NC}"
|
|
for err in "${ERRORS[@]}"; do
|
|
echo -e " ${RED}•${NC} $err"
|
|
done
|
|
fi
|
|
|
|
echo ""
|
|
if [ $FAIL -eq 0 ]; then
|
|
echo -e "${GREEN}🎉 All 12 MCP tools are working!${NC}"
|
|
exit 0
|
|
else
|
|
echo -e "${YELLOW}⚠️ $FAIL tool(s) need attention${NC}"
|
|
exit 1
|
|
fi
|