feat(phase-4): WS1 — Production Hardening (Redis rate limiting, DB pool, health endpoint, k6)

Rate limiting: - Replace in-memory express-rate-limit with ioredis + rate-limiter-flexible (sliding window) - Graceful fallback to RateLimiterMemory when Redis unreachable - RATE_LIMIT_WINDOW_MS / RATE_LIMIT_MAX_REQUESTS env var config - Retry-After header on 429 responses - agentidp_rate_limit_hits_total Prometheus counter Database pool: - Explicit pg.Pool config via DB_POOL_MAX/MIN/IDLE_TIMEOUT_MS/CONNECTION_TIMEOUT_MS - Defaults: max=20, min=2, idle=30s, conn timeout=5s - agentidp_db_pool_active_connections + agentidp_db_pool_waiting_requests gauges Health endpoint: - GET /health/detailed — per-service status (database, Redis, Vault, OPA) - healthy / degraded (>1000ms) / unreachable classification - HTTP 200 (all healthy) / 207 (any degraded) / 503 (any unreachable) Load tests: - tests/load/ with k6 scenarios for agent registration (100 VUs), token issuance (1000 VUs), credential rotation (50 VUs) - npm run load-test script Tests: 586 passing, zero TypeScript errors Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-02 04:20:37 +00:00
parent b0f70b7ac4
commit 1b682c22b2
16 changed files with 1467 additions and 113 deletions
--- a/tests/unit/middleware/rateLimit.test.ts
+++ b/tests/unit/middleware/rateLimit.test.ts
@@ -1,93 +1,241 @@
 /**
 * Unit tests for src/middleware/rateLimit.ts
+ *
+ * Covers:
+ *   - Redis path: RateLimiterRedis honours limit, sets headers, calls next()
+ *   - Fallback path: RateLimiterMemory used when Redis is disabled
+ *   - 429 response shape: Retry-After header, RateLimitError passed to next()
+ *   - Prometheus counter incremented on rejection
 */

 import { Request, Response, NextFunction } from 'express';
 import { RateLimitError } from '../../../src/utils/errors';

-const mockIncr = jest.fn();
-const mockExpire = jest.fn();
+// ── Mocks ─────────────────────────────────────────────────────────────────────

-jest.mock('../../../src/cache/redis', () => ({
-  getRedisClient: jest.fn().mockResolvedValue({
-    incr: mockIncr,
-    expire: mockExpire,
-  }),
+/** Controls whether the mocked ioredis client is returned (Redis enabled path). */
+let mockRedisEnabled = true;
+
+const mockIoredisClient = { status: 'ready' };
+
+jest.mock('../../../src/infrastructure/redisClient', () => ({
+  getRateLimitRedisClient: jest.fn(() => (mockRedisEnabled ? mockIoredisClient : null)),
 }));

-import { rateLimitMiddleware } from '../../../src/middleware/rateLimit';
+/** Tracks the last RateLimiterRedis / RateLimiterMemory consume call. */
+const mockConsume = jest.fn();

-function buildMocks(clientId?: string): {
+/** Factory stubs — return the same mock consume regardless of class. */
+jest.mock('rate-limiter-flexible', () => {
+  class MockRateLimiterRedis {
+    readonly points: number = 100;
+    readonly consume = mockConsume;
+  }
+
+  class MockRateLimiterMemory {
+    readonly points: number = 100;
+    readonly consume = mockConsume;
+  }
+
+  class MockRateLimiterRes extends Error {
+    remainingPoints: number;
+    msBeforeNext: number;
+    consumedPoints: number;
+    isFirstInDuration: boolean;
+    constructor(opts?: Partial<{ remainingPoints: number; msBeforeNext: number }>) {
+      super('Too Many Requests');
+      this.remainingPoints = opts?.remainingPoints ?? 0;
+      this.msBeforeNext = opts?.msBeforeNext ?? 30000;
+      this.consumedPoints = 101;
+      this.isFirstInDuration = false;
+    }
+  }
+
+  return {
+    RateLimiterRedis: MockRateLimiterRedis,
+    RateLimiterMemory: MockRateLimiterMemory,
+    RateLimiterRes: MockRateLimiterRes,
+    RateLimiterAbstract: class {},
+  };
+});
+
+/** Stub for the Prometheus counter so we can assert increments. */
+const mockCounterInc = jest.fn();
+jest.mock('../../../src/metrics/registry', () => ({
+  rateLimitHitsTotal: { inc: (...args: unknown[]) => mockCounterInc(...args) },
+}));
+
+// ── Import after mocks are in place ──────────────────────────────────────────
+
+import { rateLimitMiddleware, _resetRateLimiterForTests } from '../../../src/middleware/rateLimit';
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+function buildMocks(path = '/api/v1/agents'): {
  req: Partial<Request>;
  res: Partial<Response>;
-  next: NextFunction;
+  next: jest.Mock;
 } {
-  const res: Partial<Response> = {
-    setHeader: jest.fn(),
-  };
  return {
    req: {
-      user: clientId ? { client_id: clientId, sub: clientId, scope: '', jti: '', iat: 0, exp: 0 } : undefined,
+      user: { client_id: 'agent-123', sub: 'agent-123', scope: '', jti: 'jti', iat: 0, exp: 0 },
      ip: '127.0.0.1',
+      path,
    },
-    res,
-    next: jest.fn() as NextFunction,
+    res: {
+      setHeader: jest.fn(),
+    },
+    next: jest.fn(),
  };
 }

-describe('rateLimitMiddleware', () => {
-  beforeEach(() => {
-    jest.clearAllMocks();
-    mockExpire.mockResolvedValue(1);
-  });
+/** Builds a successful RateLimiterRes result (request allowed). */
+function makeAllowedResult(remaining = 99, msBeforeNext = 30000): Record<string, unknown> {
+  return {
+    remainingPoints: remaining,
+    msBeforeNext,
+    consumedPoints: 100 - remaining,
+    isFirstInDuration: false,
+  };
+}

-  it('should set X-RateLimit-* headers and call next() when counter is under the limit', async () => {
-    mockIncr.mockResolvedValue(1);
-    const { req, res, next } = buildMocks('agent-123');
+/** Returns the MockRateLimiterRes class from the mock module. */
+function getMockRateLimiterRes(): new (opts?: { msBeforeNext?: number; remainingPoints?: number }) => Error {
+  return (jest.requireMock('rate-limiter-flexible') as {
+    RateLimiterRes: new (opts?: { msBeforeNext?: number; remainingPoints?: number }) => Error;
+  }).RateLimiterRes;
+}

-    await rateLimitMiddleware(req as Request, res as Response, next);
+// ── Tests ─────────────────────────────────────────────────────────────────────
+
+beforeEach(() => {
+  jest.clearAllMocks();
+  _resetRateLimiterForTests();
+  mockRedisEnabled = true;
+  process.env['RATE_LIMIT_WINDOW_MS'] = '60000';
+  process.env['RATE_LIMIT_MAX_REQUESTS'] = '100';
+});
+
+describe('rateLimitMiddleware — Redis path (REDIS_RATE_LIMIT_ENABLED=true)', () => {
+  it('calls next() without error when request is under the limit', async () => {
+    mockConsume.mockResolvedValue(makeAllowedResult(99));
+    const { req, res, next } = buildMocks();
+
+    await rateLimitMiddleware(req as Request, res as Response, next as NextFunction);

-    expect(res.setHeader).toHaveBeenCalledWith('X-RateLimit-Limit', 100);
-    expect(res.setHeader).toHaveBeenCalledWith('X-RateLimit-Remaining', 99);
-    expect(res.setHeader).toHaveBeenCalledWith('X-RateLimit-Reset', expect.any(Number));
    expect(next).toHaveBeenCalledWith();
    expect(next).not.toHaveBeenCalledWith(expect.any(Error));
  });

-  it('should call next(RateLimitError) when counter equals 100', async () => {
-    mockIncr.mockResolvedValue(101);
-    const { req, res, next } = buildMocks('agent-456');
+  it('sets X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset headers on success', async () => {
+    mockConsume.mockResolvedValue(makeAllowedResult(50));
+    const { req, res, next } = buildMocks();

-    await rateLimitMiddleware(req as Request, res as Response, next);
+    await rateLimitMiddleware(req as Request, res as Response, next as NextFunction);
+
+    expect(res.setHeader).toHaveBeenCalledWith('X-RateLimit-Limit', expect.any(Number));
+    expect(res.setHeader).toHaveBeenCalledWith('X-RateLimit-Remaining', 50);
+    expect(res.setHeader).toHaveBeenCalledWith('X-RateLimit-Reset', expect.any(Number));
+  });
+
+  it('keys the limiter by client_id from req.user', async () => {
+    mockConsume.mockResolvedValue(makeAllowedResult(99));
+    const { req, res, next } = buildMocks();
+
+    await rateLimitMiddleware(req as Request, res as Response, next as NextFunction);
+
+    expect(mockConsume).toHaveBeenCalledWith('agent-123');
+  });
+
+  it('falls back to req.ip when req.user is not set', async () => {
+    mockConsume.mockResolvedValue(makeAllowedResult(99));
+    const { req, res, next } = buildMocks();
+    req.user = undefined;
+
+    await rateLimitMiddleware(req as Request, res as Response, next as NextFunction);
+
+    expect(mockConsume).toHaveBeenCalledWith('127.0.0.1');
+  });
+});
+
+describe('rateLimitMiddleware — 429 response shape', () => {
+  it('calls next(RateLimitError) when limit is exceeded', async () => {
+    const MockRateLimiterRes = getMockRateLimiterRes();
+    mockConsume.mockRejectedValue(new MockRateLimiterRes({ msBeforeNext: 45000 }));
+    const { req, res, next } = buildMocks();
+
+    await rateLimitMiddleware(req as Request, res as Response, next as NextFunction);

    expect(next).toHaveBeenCalledWith(expect.any(RateLimitError));
  });

-  it('should use req.ip as key when req.user is not set', async () => {
-    mockIncr.mockResolvedValue(5);
-    const { req, res, next } = buildMocks(); // no clientId → no req.user
+  it('sets Retry-After header on rejection', async () => {
+    const MockRateLimiterRes = getMockRateLimiterRes();
+    mockConsume.mockRejectedValue(new MockRateLimiterRes({ msBeforeNext: 30000 }));
+    const { req, res, next } = buildMocks();

-    await rateLimitMiddleware(req as Request, res as Response, next);
+    await rateLimitMiddleware(req as Request, res as Response, next as NextFunction);

-    expect(mockIncr).toHaveBeenCalledWith(expect.stringContaining('127.0.0.1'));
-    expect(next).toHaveBeenCalledWith();
+    expect(res.setHeader).toHaveBeenCalledWith('Retry-After', 30);
  });

-  it('should set expire TTL only on first request (count === 1)', async () => {
-    mockIncr.mockResolvedValue(1);
-    const { req, res, next } = buildMocks('agent-789');
+  it('sets X-RateLimit-Remaining to 0 on rejection', async () => {
+    const MockRateLimiterRes = getMockRateLimiterRes();
+    mockConsume.mockRejectedValue(new MockRateLimiterRes({ msBeforeNext: 30000 }));
+    const { req, res, next } = buildMocks();

-    await rateLimitMiddleware(req as Request, res as Response, next);
+    await rateLimitMiddleware(req as Request, res as Response, next as NextFunction);

-    expect(mockExpire).toHaveBeenCalledWith(expect.any(String), 60);
+    expect(res.setHeader).toHaveBeenCalledWith('X-RateLimit-Remaining', 0);
  });

-  it('should not call expire on subsequent requests (count > 1)', async () => {
-    mockIncr.mockResolvedValue(50);
-    const { req, res, next } = buildMocks('agent-789');
+  it('increments agentidp_rate_limit_hits_total with endpoint label on rejection', async () => {
+    const MockRateLimiterRes = getMockRateLimiterRes();
+    mockConsume.mockRejectedValue(new MockRateLimiterRes({ msBeforeNext: 10000 }));
+    const { req, res, next } = buildMocks('/api/v1/agents');

-    await rateLimitMiddleware(req as Request, res as Response, next);
+    await rateLimitMiddleware(req as Request, res as Response, next as NextFunction);

-    expect(mockExpire).not.toHaveBeenCalled();
+    expect(mockCounterInc).toHaveBeenCalledWith({ endpoint: '/api/v1/agents' });
+  });
+});
+
+describe('rateLimitMiddleware — fallback path (Redis disabled)', () => {
+  beforeEach(() => {
+    mockRedisEnabled = false;
+    _resetRateLimiterForTests();
+  });
+
+  it('calls next() without error when request is under the limit (memory limiter)', async () => {
+    mockConsume.mockResolvedValue(makeAllowedResult(99));
+    const { req, res, next } = buildMocks();
+
+    await rateLimitMiddleware(req as Request, res as Response, next as NextFunction);
+
+    expect(next).toHaveBeenCalledWith();
+    expect(next).not.toHaveBeenCalledWith(expect.any(Error));
+  });
+
+  it('calls next(RateLimitError) when memory limiter is exceeded', async () => {
+    const MockRateLimiterRes = getMockRateLimiterRes();
+    mockConsume.mockRejectedValue(new MockRateLimiterRes({ msBeforeNext: 60000 }));
+    const { req, res, next } = buildMocks();
+
+    await rateLimitMiddleware(req as Request, res as Response, next as NextFunction);
+
+    expect(next).toHaveBeenCalledWith(expect.any(RateLimitError));
+  });
+});
+
+describe('rateLimitMiddleware — unexpected errors', () => {
+  it('passes non-RateLimiterRes errors to next() as-is', async () => {
+    const unexpectedError = new Error('Redis network failure');
+    mockConsume.mockRejectedValue(unexpectedError);
+    const { req, res, next } = buildMocks();
+
+    await rateLimitMiddleware(req as Request, res as Response, next as NextFunction);
+
+    expect(next).toHaveBeenCalledWith(unexpectedError);
+    expect(next).not.toHaveBeenCalledWith(expect.any(RateLimitError));
  });
 });