clawdbot-workspace/goosefactory/packages/learning/src/analysis/dimensions.ts

/**
 * Dimension Score Trend Analysis
 *
 * Track per-dimension score trends over time:
 * - Is code quality improving?
 * - What dimensions are below threshold?
 * - What are the learned minimum thresholds (rejection boundaries)?
 */

import type {
  EnrichedFeedbackEvent,
  DimensionTrend,
  QualityDimension,
} from "../types.js";

/**
 * Compute per-dimension trends from feedback events.
 */
export async function computeDimensionTrends(
  events: EnrichedFeedbackEvent[]
): Promise<DimensionTrend[]> {
  // Collect scores by dimension
  const dimensionData = new Map<
    QualityDimension,
    {
      scores: Array<{ score: number; timestamp: string; approved: boolean }>;
    }
  >();

  for (const event of events) {
    const isApproved = event.feedback.decision?.decision === "approved";

    for (const score of event.feedback.scores ?? []) {
      if (!dimensionData.has(score.dimension)) {
        dimensionData.set(score.dimension, { scores: [] });
      }
      dimensionData.get(score.dimension)!.scores.push({
        score: score.score,
        timestamp: event.timestamp,
        approved: isApproved ?? false,
      });
    }
  }

  const trends: DimensionTrend[] = [];

  for (const [dimension, data] of dimensionData) {
    if (data.scores.length === 0) continue;

    // Sort by timestamp
    const sorted = [...data.scores].sort(
      (a, b) =>
        new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime()
    );

    // Overall average
    const avgScore =
      sorted.reduce((sum, s) => sum + s.score, 0) / sorted.length;

    // Trend: compare first third to last third
    const thirdLen = Math.max(1, Math.floor(sorted.length / 3));
    const firstThird = sorted.slice(0, thirdLen);
    const lastThird = sorted.slice(-thirdLen);

    const firstAvg =
      firstThird.reduce((sum, s) => sum + s.score, 0) / firstThird.length;
    const lastAvg =
      lastThird.reduce((sum, s) => sum + s.score, 0) / lastThird.length;

    const trendDelta = lastAvg - firstAvg;
    let trend: "improving" | "stable" | "declining";
    if (trendDelta > 0.5) trend = "improving";
    else if (trendDelta < -0.5) trend = "declining";
    else trend = "stable";

    // Compute minimum threshold: score below which rejection rate > 80%
    const minimumThreshold = computeMinimumThreshold(sorted);

    trends.push({
      dimension,
      avgScore: Math.round(avgScore * 100) / 100,
      trend,
      trendDelta: Math.round(trendDelta * 100) / 100,
      dataPoints: sorted.length,
      minimumThreshold,
    });
  }

  // Sort by data points (most data first)
  return trends.sort((a, b) => b.dataPoints - a.dataPoints);
}

/**
 * Find the score threshold below which Jake almost always rejects.
 * Returns undefined if insufficient data.
 */
function computeMinimumThreshold(
  scores: Array<{ score: number; approved: boolean }>
): number | undefined {
  if (scores.length < 10) return undefined;

  // Test each integer score as a potential threshold
  for (let threshold = 1; threshold <= 10; threshold++) {
    const belowThreshold = scores.filter((s) => s.score < threshold);
    if (belowThreshold.length < 3) continue;

    const rejectionRate =
      belowThreshold.filter((s) => !s.approved).length / belowThreshold.length;

    if (rejectionRate >= 0.8) {
      return threshold;
    }
  }

  return undefined;
}

/**
 * Get the top improving and declining dimensions.
 */
export function getDimensionHighlights(trends: DimensionTrend[]): {
  improving: DimensionTrend[];
  declining: DimensionTrend[];
  belowThreshold: DimensionTrend[];
} {
  return {
    improving: trends
      .filter((t) => t.trend === "improving")
      .sort((a, b) => b.trendDelta - a.trendDelta),
    declining: trends
      .filter((t) => t.trend === "declining")
      .sort((a, b) => a.trendDelta - b.trendDelta),
    belowThreshold: trends.filter(
      (t) =>
        t.minimumThreshold !== undefined && t.avgScore < t.minimumThreshold + 1
    ),
  };
}

/**
 * Get score distributions per dimension for dashboard display.
 */
export function getDimensionDistributions(
  events: EnrichedFeedbackEvent[]
): Record<
  QualityDimension,
  { avg: number; min: number; max: number; count: number }
> {
  const distributions: Record<
    string,
    { scores: number[] }
  > = {};

  for (const event of events) {
    for (const score of event.feedback.scores ?? []) {
      if (!distributions[score.dimension]) {
        distributions[score.dimension] = { scores: [] };
      }
      distributions[score.dimension]!.scores.push(score.score);
    }
  }

  const result: Record<
    string,
    { avg: number; min: number; max: number; count: number }
  > = {};

  for (const [dim, data] of Object.entries(distributions)) {
    if (data.scores.length === 0) continue;
    const sorted = [...data.scores].sort((a, b) => a - b);
    result[dim] = {
      avg:
        Math.round(
          (sorted.reduce((a, b) => a + b, 0) / sorted.length) * 100
        ) / 100,
      min: sorted[0]!,
      max: sorted[sorted.length - 1]!,
      count: sorted.length,
    };
  }

  return result as Record<
    QualityDimension,
    { avg: number; min: number; max: number; count: number }
  >;
}