feat(phase-2): workstream 7 — Prometheus + Grafana Monitoring

- Add prom-client 15; shared registry in src/metrics/registry.ts (7 metrics)
- HTTP request counter + duration histogram via metricsMiddleware
- DB query duration histogram wrapping pg Pool.query
- Redis command duration histogram via typed instrumentRedisMethod wrapper
- agentidp_tokens_issued_total in OAuth2Service
- agentidp_agents_registered_total in AgentService
- GET /metrics unauthenticated endpoint (Prometheus text format)
- docker-compose.monitoring.yml overlay (Prometheus + Grafana)
- Grafana auto-provisioned datasource + pre-built AgentIdP dashboard
- docs/devops/operations.md monitoring section added
- 36/36 unit tests passing, 100% coverage on new metrics code
- Fix pre-existing unused import in tests/integration/agents.test.ts

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

tests/unit/middleware/metrics.test.ts

@@ -0,0 +1,190 @@
+/**
+ * Unit tests for src/middleware/metrics.ts
+ *
+ * Verifies that metricsMiddleware increments agentidp_http_requests_total
+ * and records agentidp_http_request_duration_seconds with the correct labels
+ * (method, route, status_code) on each request's 'finish' event.
+ */
+
+import { Request, Response, NextFunction } from 'express';
+import { metricsMiddleware } from '../../../src/middleware/metrics';
+import { metricsRegistry } from '../../../src/metrics/registry';
+
+/**
+ * prom-client 15 MetricValue does not expose `metricName` in its TypeScript
+ * types, but histogram entries carry it at runtime to distinguish _count/_sum
+ * from _bucket rows. This local interface allows the cast below.
+ */
+interface HistogramMetricValue {
+  labels: Record<string, string>;
+  value: number;
+  metricName?: string;
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+// Helpers
+// ────────────────────────────────────────────────────────────────────────────
+
+/** Build a minimal mock Express Request. */
+function makeMockRequest(overrides: Partial<Request> = {}): Request {
+  return {
+    method: 'GET',
+    path: '/test',
+    baseUrl: '',
+    route: undefined,
+    originalUrl: '/test',
+    ...overrides,
+  } as unknown as Request;
+}
+
+/**
+ * Build a minimal mock Express Response that captures 'finish' callbacks
+ * so we can trigger them manually.
+ */
+function makeMockResponse(statusCode = 200): { res: Response; triggerFinish: () => void } {
+  const finishCallbacks: Array<() => void> = [];
+
+  const res = {
+    statusCode,
+    on: (event: string, cb: () => void) => {
+      if (event === 'finish') {
+        finishCallbacks.push(cb);
+      }
+    },
+  } as unknown as Response;
+
+  return {
+    res,
+    triggerFinish: () => finishCallbacks.forEach((cb) => cb()),
+  };
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+// Tests
+// ────────────────────────────────────────────────────────────────────────────
+
+describe('metricsMiddleware', () => {
+  let next: jest.MockedFunction<NextFunction>;
+
+  beforeEach(async () => {
+    // Reset all metric values between tests to avoid cross-test pollution.
+    metricsRegistry.resetMetrics();
+    next = jest.fn();
+  });
+
+  it('calls next() immediately', () => {
+    const req = makeMockRequest();
+    const { res } = makeMockResponse();
+
+    metricsMiddleware(req, res, next);
+
+    expect(next).toHaveBeenCalledTimes(1);
+  });
+
+  it('does NOT increment counter before finish event fires', async () => {
+    const req = makeMockRequest();
+    const { res } = makeMockResponse();
+
+    metricsMiddleware(req, res, next);
+
+    const metricsBefore = await metricsRegistry.getMetricsAsJSON();
+    const counterEntry = metricsBefore.find((e) => e.name === 'agentidp_http_requests_total');
+    // No values recorded yet — values array will be empty
+    expect(counterEntry?.values ?? []).toHaveLength(0);
+  });
+
+  it('increments agentidp_http_requests_total after finish event', async () => {
+    const req = makeMockRequest({ method: 'POST', path: '/api/v1/agents' });
+    const { res, triggerFinish } = makeMockResponse(201);
+
+    metricsMiddleware(req, res, next);
+    triggerFinish();
+
+    const metricsJson = await metricsRegistry.getMetricsAsJSON();
+    const counterEntry = metricsJson.find((e) => e.name === 'agentidp_http_requests_total');
+    expect(counterEntry).toBeDefined();
+    expect(counterEntry!.values).toHaveLength(1);
+
+    const recorded = counterEntry!.values[0];
+    expect(recorded.labels['method']).toBe('POST');
+    expect(recorded.labels['status_code']).toBe('201');
+    expect(recorded.value).toBe(1);
+  });
+
+  it('records agentidp_http_request_duration_seconds after finish event', async () => {
+    const req = makeMockRequest({ method: 'GET', path: '/health' });
+    const { res, triggerFinish } = makeMockResponse(200);
+
+    metricsMiddleware(req, res, next);
+    triggerFinish();
+
+    const metricsJson = await metricsRegistry.getMetricsAsJSON();
+    const histEntry = metricsJson.find(
+      (e) => e.name === 'agentidp_http_request_duration_seconds',
+    );
+    expect(histEntry).toBeDefined();
+    // Histogram produces _bucket, _count and _sum entries — count must be 1
+    const countEntry = (histEntry!.values as HistogramMetricValue[]).find(
+      (v) => v.metricName === 'agentidp_http_request_duration_seconds_count',
+    );
+    expect(countEntry).toBeDefined();
+    expect(countEntry!.value).toBe(1);
+  });
+
+  it('uses matched route pattern when req.route.path is available', async () => {
+    const req = makeMockRequest({
+      method: 'GET',
+      path: '/api/v1/agents/some-uuid',
+      baseUrl: '/api/v1/agents',
+      route: { path: '/:agentId' } as Request['route'],
+    });
+    const { res, triggerFinish } = makeMockResponse(200);
+
+    metricsMiddleware(req, res, next);
+    triggerFinish();
+
+    const metricsJson = await metricsRegistry.getMetricsAsJSON();
+    const counterEntry = metricsJson.find((e) => e.name === 'agentidp_http_requests_total');
+    expect(counterEntry).toBeDefined();
+    const recorded = counterEntry!.values[0];
+    // Route should be baseUrl + route.path = '/api/v1/agents/:agentId'
+    expect(recorded.labels['route']).toBe('/api/v1/agents/:agentId');
+  });
+
+  it('replaces UUID segments when no route pattern is available', async () => {
+    const uuid = '123e4567-e89b-12d3-a456-426614174000';
+    const req = makeMockRequest({
+      method: 'DELETE',
+      path: `/api/v1/agents/${uuid}`,
+      baseUrl: '',
+      route: undefined,
+    });
+    const { res, triggerFinish } = makeMockResponse(204);
+
+    metricsMiddleware(req, res, next);
+    triggerFinish();
+
+    const metricsJson = await metricsRegistry.getMetricsAsJSON();
+    const counterEntry = metricsJson.find((e) => e.name === 'agentidp_http_requests_total');
+    expect(counterEntry).toBeDefined();
+    const recorded = counterEntry!.values[0];
+    expect(recorded.labels['route']).toBe('/api/v1/agents/:id');
+    expect(recorded.labels['method']).toBe('DELETE');
+    expect(recorded.labels['status_code']).toBe('204');
+  });
+
+  it('increments counter multiple times for multiple requests', async () => {
+    for (let i = 0; i < 3; i++) {
+      const req = makeMockRequest({ method: 'GET', path: '/health' });
+      const { res, triggerFinish } = makeMockResponse(200);
+      metricsMiddleware(req, res, next);
+      triggerFinish();
+    }
+
+    const metricsJson = await metricsRegistry.getMetricsAsJSON();
+    const counterEntry = metricsJson.find((e) => e.name === 'agentidp_http_requests_total');
+    expect(counterEntry).toBeDefined();
+    const recorded = counterEntry!.values[0];
+    expect(recorded.value).toBe(3);
+  });
+});