feat: add observability for security agent

src/app/api/cron/cleanup-stale-analyses/route.ts

@@ -1,12 +1,20 @@
 import { NextResponse } from 'next/server';
+import { captureException } from '@sentry/nextjs';
 import { cleanupStaleAnalyses } from '@/lib/security-agent/db/security-analysis';
 import { sentryLogger } from '@/lib/utils.server';
-import { CRON_SECRET } from '@/lib/config.server';
+import { CRON_SECRET, SECURITY_CLEANUP_BETTERSTACK_HEARTBEAT_URL } from '@/lib/config.server';
 
 if (!CRON_SECRET) {
   throw new Error('CRON_SECRET is not configured in environment variables');
 }
 
+const log = sentryLogger('security-agent:cron-cleanup', 'info');
+const warn = sentryLogger('security-agent:cron-cleanup', 'warning');
+const cronWarn = sentryLogger('cron', 'warning');
+
+/** Threshold for alerting on abnormally high stale analysis counts */
+const STALE_ANOMALY_THRESHOLD = 10;
+
 /**
  * Cron job endpoint to cleanup stale security analyses
  *
@@ -26,26 +34,56 @@ export async function GET(request: Request) {
   // Vercel sends: Authorization: Bearer <CRON_SECRET>
   const expectedAuth = `Bearer ${CRON_SECRET}`;
   if (authHeader !== expectedAuth) {
-    sentryLogger(
-      'cron',
-      'warning'
-    )(
+    cronWarn(
       'SECURITY: Invalid CRON job authorization attempt: ' +
         (authHeader ? 'Invalid authorization header' : 'Missing authorization header')
     );
     return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
   }
 
-  // Execute cleanup - mark analyses running for more than 30 minutes as failed
-  const cleanedCount = await cleanupStaleAnalyses(30);
+  try {
+    // Execute cleanup - mark analyses running for more than 30 minutes as failed
+    const cleanedCount = await cleanupStaleAnalyses(30);
 
-  if (cleanedCount > 0) {
-    sentryLogger('cron', 'info')(`Cleaned up ${cleanedCount} stale security analyses`);
-  }
+    if (cleanedCount > 0) {
+      log(`Cleaned up ${cleanedCount} stale security analyses`);
+    }
+
+    // Alert if abnormally high number of stale analyses indicates a systemic issue
+    if (cleanedCount > STALE_ANOMALY_THRESHOLD) {
+      warn(
+        `Abnormally high stale analysis count: ${cleanedCount} (threshold: ${STALE_ANOMALY_THRESHOLD}). This may indicate a systemic problem with analysis completion.`,
+        { cleanedCount, threshold: STALE_ANOMALY_THRESHOLD }
+      );
+    }
+
+    // Send heartbeat to BetterStack on success
+    if (SECURITY_CLEANUP_BETTERSTACK_HEARTBEAT_URL) {
+      await fetch(SECURITY_CLEANUP_BETTERSTACK_HEARTBEAT_URL, { signal: AbortSignal.timeout(5000) }).catch(() => {});
+    }
 
-  return NextResponse.json({
-    success: true,
-    cleanedCount,
-    timestamp: new Date().toISOString(),
-  });
+    return NextResponse.json({
+      success: true,
+      cleanedCount,
+      timestamp: new Date().toISOString(),
+    });
+  } catch (error) {
+    captureException(error, {
+      tags: { endpoint: 'cron/cleanup-stale-analyses' },
+    });
+
+    // Send failure heartbeat to BetterStack
+    if (SECURITY_CLEANUP_BETTERSTACK_HEARTBEAT_URL) {
+      await fetch(`${SECURITY_CLEANUP_BETTERSTACK_HEARTBEAT_URL}/fail`, { signal: AbortSignal.timeout(5000) }).catch(() => {});
+    }
+
+    return NextResponse.json(
+      {
+        success: false,
+        error: 'Failed to cleanup stale analyses',
+        message: error instanceof Error ? error.message : 'Unknown error',
+      },
+      { status: 500 }
+    );
+  }
 }

src/app/api/cron/sync-security-alerts/route.ts

@@ -1,11 +1,13 @@
 import { NextResponse } from 'next/server';
 import type { NextRequest } from 'next/server';
 import { captureException } from '@sentry/nextjs';
-import { CRON_SECRET } from '@/lib/config.server';
+import { CRON_SECRET, SECURITY_SYNC_BETTERSTACK_HEARTBEAT_URL } from '@/lib/config.server';
 import { runFullSync } from '@/lib/security-agent/services/sync-service';
+import { sentryLogger } from '@/lib/utils.server';
 
-// TODO: Create BetterStack heartbeat for security alerts sync
-// const BETTERSTACK_HEARTBEAT_URL = 'https://uptime.betterstack.com/api/v1/heartbeat/...';
+const log = sentryLogger('security-agent:cron-sync', 'info');
+const cronWarn = sentryLogger('cron', 'warning');
+const logError = sentryLogger('security-agent:cron-sync', 'error');
 
 /**
  * Vercel Cron Job: Sync Security Alerts
@@ -19,10 +21,14 @@ export async function GET(request: NextRequest) {
   try {
     const authHeader = request.headers.get('authorization');
     if (!CRON_SECRET || authHeader !== `Bearer ${CRON_SECRET}`) {
+      cronWarn(
+        'SECURITY: Invalid CRON job authorization attempt: ' +
+          (authHeader ? 'Invalid authorization header' : 'Missing authorization header')
+      );
       return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
     }
 
-    console.log('[sync-security-alerts] Starting security alerts sync...');
+    log('Starting security alerts sync...');
     const startTime = Date.now();
 
     const result = await runFullSync();
@@ -37,23 +43,27 @@ export async function GET(request: NextRequest) {
       timestamp: new Date().toISOString(),
     };
 
-    console.log('[sync-security-alerts] Sync completed:', summary);
+    log('Sync completed', summary);
 
-    // TODO: Send heartbeat to BetterStack on success
-    // await fetch(BETTERSTACK_HEARTBEAT_URL);
+    // Send heartbeat to BetterStack on success
+    if (SECURITY_SYNC_BETTERSTACK_HEARTBEAT_URL) {
+      await fetch(SECURITY_SYNC_BETTERSTACK_HEARTBEAT_URL, { signal: AbortSignal.timeout(5000) }).catch(() => {});
+    }
 
     return NextResponse.json(summary);
   } catch (error) {
-    console.error('[sync-security-alerts] Error syncing security alerts:', error);
+    logError('Error syncing security alerts', { error });
     captureException(error, {
       tags: { endpoint: 'cron/sync-security-alerts' },
       extra: {
         action: 'syncing_security_alerts',
       },
     });
 
-    // TODO: Send failure heartbeat to BetterStack
-    // await fetch(`${BETTERSTACK_HEARTBEAT_URL}/fail`);
+    // Send failure heartbeat to BetterStack
+    if (SECURITY_SYNC_BETTERSTACK_HEARTBEAT_URL) {
+      await fetch(`${SECURITY_SYNC_BETTERSTACK_HEARTBEAT_URL}/fail`, { signal: AbortSignal.timeout(5000) }).catch(() => {});
+    }
 
     return NextResponse.json(
       {

src/lib/config.server.ts

@@ -151,3 +151,11 @@ export const CREDIT_CATEGORIES_ENCRYPTION_KEY = getEnvVariable('CREDIT_CATEGORIE
 // Agent observability ingest service
 export const O11Y_SERVICE_URL = getEnvVariable('O11Y_SERVICE_URL') || '';
 export const O11Y_KILO_GATEWAY_CLIENT_SECRET = getEnvVariable('O11Y_KILO_GATEWAY_CLIENT_SECRET');
+
+// Security agent BetterStack heartbeat URLs
+export const SECURITY_SYNC_BETTERSTACK_HEARTBEAT_URL = getEnvVariable(
+  'SECURITY_SYNC_BETTERSTACK_HEARTBEAT_URL'
+);
+export const SECURITY_CLEANUP_BETTERSTACK_HEARTBEAT_URL = getEnvVariable(
+  'SECURITY_CLEANUP_BETTERSTACK_HEARTBEAT_URL'
+);

src/lib/security-agent/core/types.ts

@@ -284,6 +284,8 @@ export type SecurityFindingAnalysis = {
   modelUsed?: string;
   /** User ID who triggered the analysis (for audit tracking) */
   triggeredByUserId?: string;
+  /** Correlation ID for tracing across triage → sandbox → extraction → auto-dismiss */
+  correlationId?: string;
 };
 
 /**

src/lib/security-agent/github/dependabot-api.ts

@@ -7,6 +7,11 @@
 import { Octokit } from '@octokit/rest';
 import { generateGitHubInstallationToken } from '@/lib/integrations/platforms/github/adapter';
 import type { DependabotAlertRaw, DependabotAlertState } from '../core/types';
+import { sentryLogger } from '@/lib/utils.server';
+
+const log = sentryLogger('security-agent:dependabot-api', 'info');
+const warn = sentryLogger('security-agent:dependabot-api', 'warning');
+const logError = sentryLogger('security-agent:dependabot-api', 'error');
 
 /**
  * Dependabot alert from GitHub API
@@ -111,40 +116,61 @@ export async function fetchAllDependabotAlerts(
   owner: string,
   repo: string
 ): Promise<DependabotAlertRaw[]> {
-  console.log(
-    `[dependabot-api] Fetching alerts for ${owner}/${repo} (installationId=${installationId})`
-  );
+  log(`Fetching alerts for ${owner}/${repo}`, { installationId });
 
+  const apiStartTime = performance.now();
   const tokenData = await generateGitHubInstallationToken(installationId);
   const octokit = new Octokit({ auth: tokenData.token });
 
   try {
-    console.log(`[dependabot-api] Fetching all alerts for ${owner}/${repo} using pagination...`);
     // Use Octokit's paginate helper which handles cursor-based pagination automatically
     // The Dependabot API does not support the `page` parameter
-    const data = await octokit.paginate(octokit.rest.dependabot.listAlertsForRepo, {
-      owner,
-      repo,
-      per_page: 100,
-      // No state filter - get all alerts including fixed/dismissed
-    });
+    const data = await octokit.paginate(
+      octokit.rest.dependabot.listAlertsForRepo,
+      {
+        owner,
+        repo,
+        per_page: 100,
+        // No state filter - get all alerts including fixed/dismissed
+      },
+      response => {
+        // Track rate limit on each page response
+        const remaining = response.headers['x-ratelimit-remaining'];
+        const limit = response.headers['x-ratelimit-limit'];
+        if (remaining !== undefined && Number(remaining) < 100) {
+          warn(`GitHub API rate limit low: ${remaining}/${limit} remaining`, {
+            repo: `${owner}/${repo}`,
+          });
+        }
+        return response.data;
+      }
+    );
 
+    const apiDurationMs = Math.round(performance.now() - apiStartTime);
     const alerts = data.map(alert => toInternalAlert(alert as unknown as GitHubDependabotAlert));
-    console.log(`[dependabot-api] Total alerts fetched for ${owner}/${repo}: ${alerts.length}`);
+
+    log(`Alerts fetched for ${owner}/${repo}`, {
+      alertCount: alerts.length,
+      durationMs: apiDurationMs,
+    });
+
     return alerts;
   } catch (error) {
+    const apiDurationMs = Math.round(performance.now() - apiStartTime);
     const status = (error as { status?: number }).status;
     const message = (error as { message?: string }).message;
 
     // Handle case where Dependabot alerts are disabled for the repository
     if (status === 403 && message?.includes('Dependabot alerts are disabled')) {
-      console.log(`[dependabot-api] Dependabot alerts are disabled for ${owner}/${repo}, skipping`);
+      log(`Dependabot alerts are disabled for ${owner}/${repo}, skipping`);
       return [];
     }
 
-    console.error(
-      `[dependabot-api] Error fetching alerts for ${owner}/${repo}: status=${status}, message=${message}`
-    );
+    logError(`Error fetching alerts for ${owner}/${repo}`, {
+      status,
+      message,
+      durationMs: apiDurationMs,
+    });
     throw error;
   }
 }