mcpengine/infra/factory-tools/mcp-validator/ref_gh_actions/stdio-validation.yml

# Template for validating STDIO-based MCP servers
# Copy this file to your server's .github/workflows/ directory and customize as needed

name: MCP STDIO Protocol Validation

on:
  pull_request:
    branches: [ main, master ]  # Adjust branch names as needed
  workflow_dispatch:  # Allows manual triggering

# IMPORTANT: Before using this template:
# 1. Update SERVER_PATH below to point to your actual STDIO server file
# 2. Ensure your server file is executable and handles STDIO communication
# 3. Adjust environment variables and dependencies as needed for your server

jobs:
  validate-stdio:
    name: Validate STDIO MCP Server
    runs-on: ubuntu-latest
    
    strategy:
      matrix:
        # Test against multiple protocol versions including the latest 2025-06-18
        protocol-version: ["2025-03-26", "2025-06-18"]
        # Optionally test multiple Python versions
        # python-version: ['3.10', '3.11', '3.12']
    
    # Optional: Add environment variables needed by your server
    # env:
    #   API_KEY: ${{ secrets.API_KEY }}
    #   OTHER_VAR: value
    
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'  # Adjust version as needed
      
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          # Install your server's dependencies
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
          # Install the MCP validator from the repo
          pip install git+https://github.com/janix-ai/mcp-validator.git
      
      - name: Run MCP STDIO Compliance Tests
        id: stdio-test
        run: |
          mkdir -p reports
          
          # REQUIRED: Update this path to your actual server file!
          SERVER_PATH="path/to/your/stdio_server.py"  # <-- CHANGE THIS LINE
          if [ ! -f "$SERVER_PATH" ]; then
            echo "Error: Server file not found at $SERVER_PATH"
            exit 1
          fi
          chmod +x "$SERVER_PATH"
          
          # Run compliance tests with the STDIO server
          python -m mcp_testing.stdio.cli \
            "python $SERVER_PATH" \
            --protocol-version ${{ matrix.protocol-version }} \
            --output-dir reports \
            --timeout 30 \
            --debug
          
          # Store the exit code
          TEST_EXIT_CODE=$?
          
          # Additional options for 2025-06-18:
          # --test-mode tools         # Focus on testing tool functionality
          # --dynamic-only            # Auto-discover and test available tools
          # --required-tools tool1,tool2  # Test specific tools
          # --skip-tests test1,test2      # Skip specific tests
          # --skip-async                  # Skip async tool testing
          
          # Exit with the test exit code
          exit $TEST_EXIT_CODE
      
      - name: Upload test reports
        if: always()  # Upload reports even if tests fail
        uses: actions/upload-artifact@v4
        with:
          name: mcp-stdio-reports-${{ matrix.protocol-version }}
          path: reports/
          retention-days: 14
      
      - name: Post summary to PR
        if: github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const fs = require('fs');
            try {
              const reportFiles = fs.readdirSync('reports');
              const jsonReportFile = reportFiles.find(file => file.endsWith('.json'));
              
              let summary = `## MCP STDIO Validation Results (${{ matrix.protocol-version }})\n\n`;
              
              if (jsonReportFile) {
                const reportData = JSON.parse(fs.readFileSync(`reports/${jsonReportFile}`, 'utf8'));
                summary += `- Protocol Version: ${reportData.protocol_version || 'Unknown'}\n`;
                summary += `- Success Rate: ${reportData.success_rate || 'Unknown'}\n`;
                summary += `- Tests Run: ${reportData.total_tests || 0}\n`;
                
                // Add failed tests if any
                const failedTests = (reportData.test_cases || []).filter(tc => tc.status === 'failed');
                if (failedTests.length > 0) {
                  summary += `\n### Failed Tests (${failedTests.length}):\n\n`;
                  failedTests.forEach(test => {
                    summary += `- ${test.name}: ${test.error_message || 'No error message'}\n`;
                  });
                }
              } else {
                summary += "No test report found.\n";
              }
              
              github.rest.issues.createComment({
                issue_number: context.issue.number,
                owner: context.repo.owner,
                repo: context.repo.repo,
                body: summary
              });
            } catch (error) {
              console.error('Error creating PR comment:', error);
              
              github.rest.issues.createComment({
                issue_number: context.issue.number,
                owner: context.repo.owner,
                repo: context.repo.repo,
                body: "## MCP STDIO Validation\n\nError generating test results summary. Check the workflow logs for details."
              });
            }