paginate.ts

packages/ninety-client/src/paginate.ts

1. /**
2. * Pagination and response-shape normalization.
3. *
4. * This file exists because Ninety's API is not internally consistent. Per a
5. * third-party SDK's docs (verified live only where marked):
6. *
7. * Issues, KPIs -> { items: [], totalCount: n }
8. * To-Dos, Teams -> bare JSON array
9. * Rocks -> object keyed by teamId
10. *
11. * and pagination splits three ways:
12. *
13. * Rocks, Issues -> pageIndex (0-based), sortField/sortDirection
14. * To-Dos -> page (1-based, verified), sort/order
15. * Company To-Dos -> opaque cursor (verified, see fromCursor below)
16. *
17. * Everything above this layer speaks one dialect: 0-based `page`, and `Page<T>`.
18. * Cursor endpoints cannot be expressed that way (you cannot seek to page 4 of a
19. * cursor stream), so they get `CursorPage<T>` and `iterateCursor` instead of a
20. * fake page number. Do not paper over the difference; a synthetic pageIndex on a
21. * cursor endpoint is a lie that eventually gets indexed into.
22. *
23. * If a shape turns out wrong, this file changes and nothing else does. That is
24. * the point of isolating it.
25. */
27. import { NinetySchemaError, type NinetyErrorContext } from './errors.js'
29. /** The one paginated shape the rest of the codebase knows about. */
30. export interface Page<T> {
31. items: T[]
32. /** null when Ninety does not report a total (bare-array endpoints). */
33. total: number | null
34. /** Always 0-based, regardless of what the endpoint wanted. */
35. pageIndex: number
36. pageSize: number
37. hasMore: boolean
38. }
40. /**
41. * Cursor-paginated shape. No page index, no total; those do not exist here.
42. * `nextToken` is opaque: pass it back verbatim, never parse or construct one.
43. */
44. export interface CursorPage<T> {
45. items: T[]
46. /** null means the set is exhausted. Verified: a complete set returns null. */
47. nextToken: string | null
48. }
50. /** The one request shape callers use. Translated per-resource on the way out. */
51. export interface PageRequest {
52. /** 0-based. Translated to 1-based for endpoints that need it. */
53. page?: number
54. pageSize?: number
55. sortBy?: string
56. sortDir?: 'asc' | 'desc'
57. }
59. // --- shape adapters ---------------------------------------------------------
61. /** `{ items, totalCount }` -> Page. Issues, KPIs. */
62. export function fromEnvelope<T>(
63. raw: unknown,
64. req: Required<Pick<PageRequest, 'page' | 'pageSize'>>,
65. ctx: NinetyErrorContext,
66. ): Page<T> {
67. if (typeof raw !== 'object' || raw === null || !('items' in raw)) {
68. throw new NinetySchemaError(ctx, `expected {items, totalCount}, got ${describe(raw)}`)
69. }
70. const items = (raw as { items: unknown }).items
71. if (!Array.isArray(items)) {
72. throw new NinetySchemaError(ctx, `expected .items to be an array, got ${describe(items)}`)
73. }
74. const totalRaw = (raw as { totalCount?: unknown }).totalCount
75. const total = typeof totalRaw === 'number' ? totalRaw : null
76. return {
77. items: items as T[],
78. total,
79. pageIndex: req.page,
80. pageSize: req.pageSize,
81. hasMore:
82. total === null ? items.length === req.pageSize : (req.page + 1) * req.pageSize < total,
83. }
84. }
86. /**
87. * Bare array -> Page. To-Dos, Teams.
88. *
89. * No total is reported, so `hasMore` is inferred from a full page. That means
90. * one wasted request when the item count is an exact multiple of pageSize.
91. * Correctness over cleverness; the alternative is silently truncating.
92. */
93. export function fromArray<T>(
94. raw: unknown,
95. req: Required<Pick<PageRequest, 'page' | 'pageSize'>>,
96. ctx: NinetyErrorContext,
97. ): Page<T> {
98. if (!Array.isArray(raw)) {
99. throw new NinetySchemaError(ctx, `expected a bare array, got ${describe(raw)}`)
100. }
101. return {
102. items: raw as T[],
103. total: null,
104. pageIndex: req.page,
105. pageSize: req.pageSize,
106. hasMore: raw.length >= req.pageSize,
107. }
108. }
110. /**
111. * `{ [teamId]: T[] }` -> Page. Rocks.
112. *
113. * Flattens and stamps each item with the teamId it was filed under, since that
114. * key is otherwise the only place the association exists.
115. */
116. export function fromTeamKeyed<T extends object>(
117. raw: unknown,
118. req: Required<Pick<PageRequest, 'page' | 'pageSize'>>,
119. ctx: NinetyErrorContext,
120. ): Page<T & { teamId: string }> {
121. if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
122. throw new NinetySchemaError(ctx, `expected an object keyed by teamId, got ${describe(raw)}`)
123. }
124. const items: (T & { teamId: string })[] = []
125. for (const [teamId, group] of Object.entries(raw as Record<string, unknown>)) {
126. if (!Array.isArray(group)) {
127. throw new NinetySchemaError(ctx, `expected an array under teamId "${teamId}", got ${describe(group)}`)
128. }
129. for (const item of group) {
130. items.push({ ...(item as T), teamId })
131. }
132. }
133. return {
134. items,
135. total: null,
136. pageIndex: req.page,
137. pageSize: req.pageSize,
138. hasMore: items.length >= req.pageSize,
139. }
140. }
142. /**
143. * `{ data: T[], pagination: { nextToken } }` -> CursorPage. Company To-Dos.
144. *
145. * Verified against GET /todos/company: top-level keys are exactly
146. * `['data', 'pagination']`. A complete result set
147. * returns `pagination.nextToken: null`, so null is "done", not "unknown".
148. *
149. * Note this is a different envelope from `{items, totalCount}`: `data` not
150. * `items`, and no total is reported at all. Do not try to compute a percentage
151. * complete from it; there is no denominator.
152. */
153. export function fromCursor<T>(raw: unknown, ctx: NinetyErrorContext): CursorPage<T> {
154. if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
155. throw new NinetySchemaError(ctx, `expected {data, pagination}, got ${describe(raw)}`)
156. }
157. const data = (raw as { data?: unknown }).data
158. if (!Array.isArray(data)) {
159. throw new NinetySchemaError(ctx, `expected .data to be an array, got ${describe(data)}`)
160. }
161. const pagination = (raw as { pagination?: unknown }).pagination
162. let nextToken: string | null = null
163. if (pagination !== undefined && pagination !== null) {
164. if (typeof pagination !== 'object' || Array.isArray(pagination)) {
165. throw new NinetySchemaError(ctx, `expected .pagination to be an object, got ${describe(pagination)}`)
166. }
167. const token = (pagination as { nextToken?: unknown }).nextToken
168. if (typeof token === 'string' && token.length > 0) nextToken = token
169. }
170. return { items: data as T[], nextToken }
171. }
173. /**
174. * Walks a cursor stream to exhaustion.
175. *
176. * The safety cap matters more here than on numbered pages: a server that keeps
177. * echoing the same nextToken produces an infinite loop that looks like healthy
178. * traffic. We also detect a repeated token explicitly, because that is the
179. * failure mode most likely to actually happen.
180. */
181. export async function* iterateCursor<T>(
182. fetchPage: (nextToken: string | undefined) => Promise<CursorPage<T>>,
183. opts: IterateOptions = {},
184. ): AsyncGenerator<T, void, undefined> {
185. const maxPages = opts.maxPages ?? 500
186. let token: string | undefined
187. const seen = new Set<string>()
189. for (let n = 0; n < maxPages; n++) {
190. const page = await fetchPage(token)
191. for (const item of page.items) yield item
192. if (!page.nextToken) return
193. if (seen.has(page.nextToken)) {
194. throw new Error(
195. `iterateCursor received the same nextToken twice ("${page.nextToken.slice(0, 12)}..."). ` +
196. `Ninety is not advancing the cursor; stopping rather than looping forever.`,
197. )
198. }
199. seen.add(page.nextToken)
200. token = page.nextToken
201. }
202. throw new Error(
203. `iterateCursor stopped at the ${maxPages}-page safety cap. Either raise maxPages or ` +
204. `check that nextToken is being threaded back correctly.`,
205. )
206. }
208. // --- iteration --------------------------------------------------------------
210. export interface IterateOptions {
211. /**
212. * Hard stop. Without it a pagination bug on Ninety's side becomes an infinite
213. * loop against a rate-limited API.
214. */
215. maxPages?: number
216. }
218. /**
219. * Walks every page. Callers should almost always use this rather than paging by
220. * hand, so the 1-based/0-based translation stays in one place.
221. */
222. export async function* iterateAll<T>(
223. fetchPage: (req: Required<Pick<PageRequest, 'page' | 'pageSize'>>) => Promise<Page<T>>,
224. req: Required<Pick<PageRequest, 'page' | 'pageSize'>>,
225. opts: IterateOptions = {},
226. ): AsyncGenerator<T, void, undefined> {
227. const maxPages = opts.maxPages ?? 500
228. let page = req.page
229. for (let n = 0; n < maxPages; n++) {
230. const result = await fetchPage({ page, pageSize: req.pageSize })
231. for (const item of result.items) yield item
232. if (!result.hasMore || result.items.length === 0) return
233. page++
234. }
235. throw new Error(
236. `iterateAll stopped at the ${maxPages}-page safety cap. Either raise maxPages or ` +
237. `check that hasMore is being computed correctly for this endpoint.`,
238. )
239. }
241. /** Convenience: drain an iterator into an array. */
242. export async function collect<T>(it: AsyncGenerator<T, void, undefined>): Promise<T[]> {
243. const out: T[] = []
244. for await (const item of it) out.push(item)
245. return out
246. }
248. function describe(v: unknown): string {
249. if (v === null) return 'null'
250. if (Array.isArray(v)) return `array(${v.length})`
251. return typeof v
252. }