rateLimit.ts

mcp-server/src/lib/rateLimit.ts

1. /**
2. * Rate limiting for the Plinth API.
3. *
4. * The API enforces a hard limit of 10 requests / 10 seconds; exceeding it
5. * returns {"error":"Too many requests. Slow down and retry."}. Because a single
6. * model can issue parallel tool calls and every user shares one API budget, the
7. * limit is enforced server-side: every Plinth call passes through a limiter that
8. * delays rather than fails, so heavy pulls slow down instead of erroring. The
9. * client awaits each tool result, so it needs no pacing of its own.
10. *
11. * The scheduling math (`reserveSlot`) is a pure function shared by both limiter
12. * implementations:
13. * - InMemoryRateLimiter: a single Node process (the stdio dev entry).
14. * - the Durable-Object-backed limiter on Workers (src/rateLimiterDO.ts),
15. * which routes every session through ONE shared instance so the global
16. * budget holds across all isolates and parallel calls.
17. */
19. /** The Plinth hard cap is 10/10s; schedule to 9/10s to leave headroom. */
20. export const PLINTH_RATE_LIMIT = { max: 9, windowMs: 10_000 } as const;
22. /** Backoff before the single retry when a RateLimitError still slips through. */
23. export const RATE_LIMIT_RETRY_MS = PLINTH_RATE_LIMIT.windowMs;
25. export interface RateLimiter {
26. /** Resolves when it is safe to issue one more request (may delay first). */
27. acquire(): Promise<void>;
28. }
30. export function sleep(ms: number): Promise<void> {
31. return new Promise((resolve) => setTimeout(resolve, ms));
32. }
34. export interface SlotDecision {
35. /** Milliseconds the caller must wait before proceeding (0 = go now). */
36. delayMs: number;
37. /** Reserved-slot timestamps to persist for the next call. */
38. slots: number[];
39. }
41. /**
42. * Pure sliding-window scheduler. Reserves a slot for one new request such that
43. * no `windowMs` interval ever holds more than `max` requests, spreading bursts
44. * across time: each request is scheduled no earlier than `windowMs` after the
45. * `max`-th most recent reservation.
46. *
47. * Retains only the most recent `max` reservations (older ones can never
48. * constrain a future request), so the returned array stays bounded under any
49. * burst size.
50. */
51. export function reserveSlot(
52. slots: readonly number[],
53. now: number,
54. max: number,
55. windowMs: number,
56. ): SlotDecision {
57. // Drop reservations that have aged out of the window (they no longer constrain).
58. const live = slots.filter((t) => t > now - windowMs).sort((a, b) => a - b);
59. const reserved =
60. live.length < max ? now : Math.max(now, live[live.length - max]! + windowMs);
61. const next = [...live, reserved].slice(-max);
62. return { delayMs: reserved - now, slots: next };
63. }
65. /** In-memory limiter for a single process (the Node/stdio dev entry). */
66. export class InMemoryRateLimiter implements RateLimiter {
67. private slots: number[] = [];
68. constructor(
69. private readonly max: number = PLINTH_RATE_LIMIT.max,
70. private readonly windowMs: number = PLINTH_RATE_LIMIT.windowMs,
71. ) {}
73. async acquire(): Promise<void> {
74. // Reserve synchronously (atomic in a single-threaded runtime) BEFORE the
75. // sleep, so concurrent acquire() calls are spaced apart, not stacked.
76. const { delayMs, slots } = reserveSlot(this.slots, Date.now(), this.max, this.windowMs);
77. this.slots = slots;
78. if (delayMs > 0) await sleep(delayMs);
79. }
80. }