mattrguerra/meridian-ops-boardmainPrivate on GitHub, published here in full

gmail.test.ts

Language TypeScript34.0 KB759 lines

packages/gmail-client/test/gmail.test.ts

/**
* The Gmail reader.
*
* What is worth testing here is not "does it call the API"; it is the three
* things that would quietly corrupt the board: an address parsed wrong so a
* filter matches the wrong sender, a dead mailbox rendered as an empty one, and
* a message body reaching KV.
*/
import { describe, expect, it, vi } from 'vitest'
import { SA } from './sa.js'
import {
accessTokenFor,
cleanSnippet,
GMAIL_SCOPE,
GMAIL_SCOPES,
GMAIL_SEND_SCOPE,
GmailAuthError,
scopesGranted,
normalizeAddress,
parseFrom,
parseServiceAccountSecret,
readMailbox,
buildBatchBody,
parseBatchResponse,
} from '../src/index.js'
const CREDS = SA
const json = (body: unknown, status = 200) =>
new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } })
describe('parseFrom', () => {
it('splits a display name from the address', () => {
expect(parseFrom('Upscaler Labs <news@upscaler.vendor.example>')).toEqual({
sender: 'news@upscaler.vendor.example',
senderName: 'Upscaler Labs',
})
})
it('handles a bare address', () => {
expect(parseFrom('noreply@qrcodes.vendor.example')).toEqual({ sender: 'noreply@qrcodes.vendor.example', senderName: '' })
})
it('handles a quoted name containing a comma', () => {
// Splitting on ',' or on the first '<' both get this wrong, and the result
// is a filter entry that matches nothing.
expect(parseFrom('"Labs, Upscaler" <news@upscaler.vendor.example>')).toEqual({
sender: 'news@upscaler.vendor.example',
senderName: 'Labs, Upscaler',
})
})
})
describe('normalizeAddress', () => {
it('lowercases, so a filter matches whatever case the header used', () => {
// Live: TrackingUpdates@courier.vendor.example. Case-sensitive matching would let a
// sender slip the filter by changing capitalisation.
expect(normalizeAddress('TrackingUpdates@courier.vendor.example')).toBe('trackingupdates@courier.vendor.example')
})
it('strips a plus-tag so a per-account sender is one row, not hundreds', () => {
// Live: failed-payments+acct_1ExampleAcct0000@stripe.com. Left whole, every
// Stripe account becomes its own "sender" and the button applies to one.
expect(normalizeAddress('failed-payments+acct_1ExampleAcct0000@stripe.com')).toBe(
'failed-payments@stripe.com',
)
})
it('leaves a + in the domain alone', () => {
expect(normalizeAddress('a@b+c.com')).toBe('a@b+c.com')
})
})
describe('parseServiceAccountSecret: read the secret however it was pasted', () => {
// The secret often holds the whole downloaded key JSON, not the bare
// private_key. Pasting the file is the natural move at a prompt that says
// "enter a secret value", so the natural move has to work.
const KEY_JSON = JSON.stringify({
type: 'service_account',
client_email: 'meridian-dashboard@meridian-ops.iam.gserviceaccount.com',
private_key: SA.privateKey,
token_uri: 'https://oauth2.googleapis.com/token',
})
it('accepts the entire key JSON and yields both key and email', () => {
const out = parseServiceAccountSecret(KEY_JSON)
expect(out.email).toBe('meridian-dashboard@meridian-ops.iam.gserviceaccount.com')
expect(out.privateKey).toBe(SA.privateKey)
})
it('accepts a bare PEM, with no email', () => {
// Trailing whitespace is trimmed; the key material must survive intact.
expect(parseServiceAccountSecret(SA.privateKey)).toEqual({
email: null,
privateKey: SA.privateKey.trim(),
})
})
it('unwraps a PEM someone quoted', () => {
expect(parseServiceAccountSecret(`"${SA.privateKey.trim()}"`).privateKey).toBe(SA.privateKey.trim())
})
it('rejects an OAuth client secret with a message that names the confusion', () => {
// A plausible mistake: GMAIL_SA_KEY holding the GOCSPX OAuth client secret,
// because it was on screen when the prompt appeared. Nothing can recover a
// key that was never uploaded, so the error's job is to say which
// credential this is and which it needs.
expect(() => parseServiceAccountSecret('GOCSPX-AbCdEfGhIjKlMnOp')).toThrow(/OAuth client secret/)
})
it('names a truncated-JSON paste and the file-redirect fix', () => {
// Half a paste is the other likely failure, and "not valid base64" pointed
// nowhere. The error must carry the actionable command.
expect(() => parseServiceAccountSecret('{"client_email": "a@b.c", "priv')).toThrow(/< key\.json/)
})
it('says when the JSON is not a key file at all', () => {
expect(() => parseServiceAccountSecret('{"foo": 1}')).toThrow(/no private_key/)
})
it('a full-JSON secret is a complete configuration for the token exchange', async () => {
const parsed = parseServiceAccountSecret(KEY_JSON)
const f = (async () => new Response(JSON.stringify({ access_token: 'tok', expires_in: 3599 }), { status: 200 })) as unknown as typeof fetch
await expect(
accessTokenFor('hello@meridiandisplay.example', { email: parsed.email!, privateKey: parsed.privateKey }, f),
).resolves.toMatchObject({ accessToken: 'tok' })
})
})
describe('token exchange', () => {
it('signs a real JWT and impersonates the right mailbox', async () => {
// The claim set is the whole mechanism: `sub` is the mailbox being read and
// is what domain-wide delegation authorizes. A token minted with the wrong
// sub reads the wrong person's mail.
let sent = ''
const f = (async (_u: string | URL, init?: RequestInit) => {
sent = String(init?.body)
return json({ access_token: 'tok', expires_in: 3599 })
}) as unknown as typeof fetch
await accessTokenFor('accounts@meridiandisplay.example', SA, f)
const assertion = new URLSearchParams(sent).get('assertion')!
expect(new URLSearchParams(sent).get('grant_type')).toBe(
'urn:ietf:params:oauth:grant-type:jwt-bearer',
)
const [h, c, sig] = assertion.split('.')
const dec = (p: string) => JSON.parse(Buffer.from(p, 'base64url').toString('utf8'))
expect(dec(h!)).toEqual({ alg: 'RS256', typ: 'JWT' })
const claims = dec(c!)
expect(claims.sub).toBe('accounts@meridiandisplay.example')
expect(claims.iss).toBe(SA.email)
expect(claims.scope).toBe(GMAIL_SCOPE)
expect(claims.aud).toBe('https://oauth2.googleapis.com/token')
expect(claims.exp - claims.iat).toBe(3600)
// base64url, not base64: a '+' or '/' in the signature makes Google reject
// the assertion with an error that names none of this.
expect(sig).toMatch(/^[A-Za-z0-9_-]+$/)
})
it('names the causes of unauthorized_client rather than picking one', async () => {
// Rule 1. This is the error that WILL happen during setup, and
// Google says only "unauthorized_client" for all of: client ID not added in
// Admin, scope string mismatched, propagation not finished.
const f = vi.fn(async () => json({ error: 'unauthorized_client' }, 401)) as unknown as typeof fetch
await expect(accessTokenFor('hello@meridiandisplay.example', SA, f)).rejects.toThrow(GmailAuthError)
await expect(accessTokenFor('hello@meridiandisplay.example', SA, f)).rejects.toThrow(/Domain-wide delegation/)
})
it('names the causes of invalid_grant, including a skewed clock', async () => {
const f = vi.fn(async () => json({ error: 'invalid_grant' }, 400)) as unknown as typeof fetch
await expect(accessTokenFor('hello@meridiandisplay.example', SA, f)).rejects.toThrow(/suspended/)
await expect(accessTokenFor('hello@meridiandisplay.example', SA, f)).rejects.toThrow(/clock/)
})
it('accepts a PEM whose newlines were flattened to backslash-n', async () => {
// The key survives a shell, an env var and a secrets UI on its way to the
// Worker, and very often arrives with literal \\n. Rejecting it produces a
// base64 error that names nothing.
const flattened = { ...SA, privateKey: SA.privateKey.replace(/\n/g, '\\n') }
const f = (async () => json({ access_token: 'tok', expires_in: 3599 })) as unknown as typeof fetch
await expect(accessTokenFor('hello@meridiandisplay.example', flattened, f)).resolves.toMatchObject({
accessToken: 'tok',
})
})
it('reports an unreadable key as a key problem, not a network one', async () => {
const bad = { ...SA, privateKey: '-----BEGIN PRIVATE KEY-----\nnot base64!!\n-----END PRIVATE KEY-----' }
const f = (async () => json({ access_token: 'x' })) as unknown as typeof fetch
await expect(accessTokenFor('hello@meridiandisplay.example', bad, f)).rejects.toThrow(/private key could not be read/)
})
it('treats a 200 with no access_token as a failure', async () => {
// Returning '' would surface as a 401 on the next call and send somebody
// debugging the wrong request.
const f = vi.fn(async () => json({ expires_in: 3599 })) as unknown as typeof fetch
await expect(accessTokenFor('hello@meridiandisplay.example', CREDS, f)).rejects.toThrow(/no access_token/)
})
})
describe('cleanSnippet', () => {
it('decodes the entities Gmail encodes snippets with', () => {
expect(cleanSnippet('Tom & Jerry's "sale"')).toBe('Tom & Jerry\'s "sale"')
})
it('strips the zero-width padding marketing mail games previews with', () => {
expect(cleanSnippet('Real text' + '\u200c\u00ad\u200b\ufeff'.repeat(80))).toBe('Real text')
})
it('caps at 140 with an ellipsis, not a mid-word chop artifact', () => {
const out = cleanSnippet('word '.repeat(60))
expect(out.length).toBeLessThanOrEqual(140)
expect(out.endsWith('…')).toBe(true)
})
it('is empty for an absent snippet, never undefined', () => {
expect(cleanSnippet(undefined)).toBe('')
})
})
describe('readMailbox', () => {
/**
* Gmail's batch endpoint returns multipart/mixed, so the fake has to speak it.
* That is deliberate rather than a nuisance: it means these tests exercise the
* real parser instead of a mock of it, which is where a transport change like
* this one would otherwise hide its bugs.
*/
const batchResponse = (msgs: (Record<string, unknown> | undefined)[], boundary = 'testbatch') => {
const parts = msgs
.map(
(m) =>
`--${boundary}\r\nContent-Type: application/http\r\n\r\n` +
(m
? `HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n${JSON.stringify(m)}\r\n`
: `HTTP/1.1 500 Internal Server Error\r\nContent-Type: application/json\r\n\r\n${JSON.stringify({ error: { message: 'boom' } })}\r\n`),
)
.join('')
return new Response(`${parts}--${boundary}--\r\n`, {
headers: { 'content-type': `multipart/mixed; boundary=${boundary}` },
})
}
/** The ids a batch body asked for, in order. */
const idsIn = (body: string): string[] =>
[...body.matchAll(/messages\/([^?]+)\?/g)].map((m) => decodeURIComponent(m[1]!))
const okFetch = (messages: Record<string, unknown>[]) =>
(async (url: string | URL, init?: RequestInit) => {
const u = String(url)
if (u.includes('oauth2')) return json({ access_token: 'tok', expires_in: 3599 })
if (u.includes('/batch/')) {
const ids = idsIn(String(init?.body ?? ''))
return batchResponse(ids.map((id) => messages.find((m) => m.id === id)))
}
if (u.includes('/messages?')) {
return json({
messages: messages.map((m) => ({ id: m.id })),
resultSizeEstimate: messages.length,
})
}
return json({}, 404)
}) as unknown as typeof fetch
const msg = (id: string, from: string, subject: string) => ({
id,
threadId: `t-${id}`,
internalDate: '1788210063000',
labelIds: ['INBOX', 'CATEGORY_PROMOTIONS'],
payload: {
headers: [
{ name: 'From', value: from },
{ name: 'Subject', value: subject },
],
},
})
it('normalizes a message into metadata and nothing else', async () => {
const r = await readMailbox('hello@meridiandisplay.example', CREDS, {
paceMs: 0,
fetchImpl: okFetch([msg('m1', 'Upscaler Labs <news@upscaler.vendor.example>', 'What is new this month')]),
})
expect(r.ok).toBe(true)
const m = r.messages[0]!
expect(m.sender).toBe('news@upscaler.vendor.example')
expect(m.subject).toBe('What is new this month')
expect(m.mailbox).toBe('hello@meridiandisplay.example')
// The shape carries a snippet but no body: a one-line preview at rest,
// capped at 140 chars, for Gmail-style rows. Bodies are fetch-on-open only; a `text` or `body` field appearing here still
// has to be justified to seven people.
expect(Object.keys(m).sort()).toEqual(
['date', 'id', 'labels', 'mailbox', 'sender', 'senderName', 'snippet', 'subject', 'threadId', 'unread'],
)
expect(m.snippet.length).toBeLessThanOrEqual(140)
})
it('never asks Gmail for a body in the first place', async () => {
// The sub-requests live in the batch body. format=metadata plus a header
// allow-list means the body is not merely ignored; it never crosses the
// wire, and seven people's mail never lands in this Worker's memory.
let batchBody = ''
const f = (async (url: string | URL, init?: RequestInit) => {
const u = String(url)
if (u.includes('oauth2')) return json({ access_token: 'tok', expires_in: 3599 })
if (u.includes('/batch/')) {
batchBody = String(init?.body ?? '')
return batchResponse([msg('m1', 'a@b.com', 's')])
}
return json({ messages: [{ id: 'm1' }], resultSizeEstimate: 1 })
}) as unknown as typeof fetch
await readMailbox('hello@meridiandisplay.example', CREDS, { fetchImpl: f, paceMs: 0 })
expect(batchBody).toContain('format=metadata')
expect(batchBody).not.toContain('format=full')
expect(batchBody).toContain('metadataHeaders=From')
})
it('returns a failed read rather than throwing, so one dead token keeps six alive', async () => {
// The day a role mailbox changes hands its token stops working. Losing the
// other six inboxes with it would be the worse outcome, and it is the exact
// scenario this tab was built for.
const f = (async (url: string | URL) =>
String(url).includes('oauth2') ? json({ error: 'invalid_grant' }, 400) : json({})) as unknown as typeof fetch
const r = await readMailbox('accounts@meridiandisplay.example', CREDS, { fetchImpl: f, paceMs: 0 })
expect(r.ok).toBe(false)
expect(r.messages).toEqual([])
expect(r.error).toMatch(/invalid_grant/)
})
it('keeps a mailbox when one message fails, and COUNTS the one it lost', async () => {
// The old per-message loop did `if (!res.ok) continue`, which is the right
// call: one unreadable message is not a failed mailbox. But silently
// returning 1 of 2 makes a truncated read look complete. The skip survives;
// the silence does not.
const f = (async (url: string | URL, init?: RequestInit) => {
const u = String(url)
if (u.includes('oauth2')) return json({ access_token: 'tok', expires_in: 3599 })
if (u.includes('/batch/')) {
const ids = idsIn(String(init?.body ?? ''))
return batchResponse(ids.map((id) => (id === 'm1' ? msg('m1', 'a@b.com', 'fine') : undefined)))
}
return json({ messages: [{ id: 'm1' }, { id: 'm2' }], resultSizeEstimate: 2 })
}) as unknown as typeof fetch
const r = await readMailbox('hello@meridiandisplay.example', CREDS, { fetchImpl: f, paceMs: 0 })
expect(r.ok).toBe(true)
expect(r.messages).toHaveLength(1)
expect(r.failedFetches).toBe(1)
})
it('reads the inbox window, not the whole archive', async () => {
const seen: string[] = []
const f = (async (url: string | URL) => {
seen.push(String(url))
const u = String(url)
if (u.includes('oauth2')) return json({ access_token: 'tok', expires_in: 3599 })
return json({ messages: [] })
}) as unknown as typeof fetch
await readMailbox('hello@meridiandisplay.example', CREDS, { fetchImpl: f, paceMs: 0 })
expect(decodeURIComponent(seen.find((u) => u.includes('/messages?'))!)).toContain('in:inbox')
})
})
/**
* The scopes, and how the board learns whether a token can send.
*
* A refresh token carries the scopes it was consented with, so a token minted
* before the send scope was added still reads and cannot send. The board must
* be able to see that rather than infer it: a Reply button
* that fails is worse than no Reply button, and "read-only token", "we could
* not ask" and "the mailbox failed" need three different fixes.
*/
describe('scopes', () => {
it('asks for read AND send, space-joined for the OAuth parameter', () => {
expect(GMAIL_SCOPES).toContain('https://www.googleapis.com/auth/gmail.readonly')
expect(GMAIL_SCOPES).toContain(GMAIL_SEND_SCOPE)
expect(GMAIL_SCOPE).toBe(GMAIL_SCOPES.join(' '))
})
it('compares a grant as a SET, because Google returns its own order', () => {
// The old `granted !== requested` string compare fired on perfectly valid
// grants, which is how a real warning stops being read.
const reversed = [...GMAIL_SCOPES].reverse().join(' ')
expect(scopesGranted(reversed).ok).toBe(true)
expect(scopesGranted('https://www.googleapis.com/auth/gmail.readonly')).toEqual({
ok: false,
missing: [GMAIL_SEND_SCOPE],
})
})
it('reports the scopes a token actually carries, or null when Google said nothing', async () => {
const withScope = (async () =>
json({ access_token: 't', expires_in: 3599, scope: GMAIL_SCOPE })) as unknown as typeof fetch
await expect(accessTokenFor('hello@meridiandisplay.example', SA, withScope)).resolves.toMatchObject({
scopes: [...GMAIL_SCOPES],
})
const without = (async () => json({ access_token: 't', expires_in: 3599 })) as unknown as typeof fetch
await expect(accessTokenFor('hello@meridiandisplay.example', SA, without)).resolves.toMatchObject({
scopes: null,
})
})
})
describe('readMailbox reports whether the mailbox can send', () => {
const withScopes = (scope: string | null) =>
(async (url: string | URL) => {
const u = String(url)
if (u.includes('oauth2')) {
return json(scope === null ? { access_token: 't', expires_in: 3599 } : { access_token: 't', expires_in: 3599, scope })
}
if (u.includes('/messages?')) return json({ messages: [] })
return json({})
}) as unknown as typeof fetch
it('true when Google reported gmail.send', async () => {
const r = await readMailbox('hello@meridiandisplay.example', CREDS, { fetchImpl: withScopes(GMAIL_SCOPE) })
expect(r.canSend).toBe(true)
})
it('false when Google reported the scopes and send was not among them', async () => {
const r = await readMailbox('hello@meridiandisplay.example', CREDS, {
paceMs: 0,
fetchImpl: withScopes('https://www.googleapis.com/auth/gmail.readonly'),
})
expect(r.canSend).toBe(false)
})
it('null when Google reported no scopes at all, not false', async () => {
// The prime invariant, applied to a permission: "we do not know" is its own
// state and must not be rendered as "it cannot".
const r = await readMailbox('hello@meridiandisplay.example', CREDS, { fetchImpl: withScopes(null) })
expect(r.canSend).toBeNull()
})
it('null on a failed read, because we never got far enough to ask', async () => {
const dead = (async () => json({ error: 'invalid_grant' }, 400)) as unknown as typeof fetch
const r = await readMailbox('hello@meridiandisplay.example', CREDS, { fetchImpl: dead })
expect(r.ok).toBe(false)
expect(r.canSend).toBeNull()
})
})
/**
* The batch transport. One fetch per chunk of messages, rather than one per
* message, keeps Cloudflare subrequests from capping how much of a mailbox the
* board can hold.
*
* What is worth testing is the encoding and the failure accounting, because a
* malformed part is rejected with an error naming nothing, and a dropped part
* would quietly shrink a mailbox.
*/
describe('buildBatchBody', () => {
it('closes every part and terminates the body, which Gmail requires', () => {
const body = buildBatchBody(['a', 'b'], 'B')
expect(body.startsWith('--B\r\n')).toBe(true)
expect(body.endsWith('--B--\r\n')).toBe(true)
expect(body.match(/Content-Type: application\/http/g)).toHaveLength(2)
})
it('leaves the blank line after each request line', () => {
// A part without it is rejected as malformed, with an error that names
// nothing useful. Easy to lose in a refactor; impossible to debug from the
// response.
expect(buildBatchBody(['a'], 'B')).toMatch(/GET \/gmail\/v1[^\r]*\r\n\r\n/)
})
it('asks only for metadata, and escapes the id', () => {
const body = buildBatchBody(['a/b?c'], 'B')
expect(body).toContain('format=metadata')
expect(body).not.toContain('format=full')
// An id spliced raw into the path would change the request it belongs to.
expect(body).toContain(encodeURIComponent('a/b?c'))
})
})
describe('parseBatchResponse', () => {
const part = (json: string, status = '200 OK') =>
`--X\r\nContent-Type: application/http\r\n\r\nHTTP/1.1 ${status}\r\nContent-Type: application/json\r\n\r\n${json}\r\n`
it('reads every message out of a multipart body', () => {
const body = part('{"id":"m1"}') + part('{"id":"m2"}') + '--X--\r\n'
const r = parseBatchResponse('multipart/mixed; boundary=X', body)
expect(r.messages.map((m) => m.id)).toEqual(['m1', 'm2'])
expect(r.failed).toBe(0)
})
it('matches by the id in the body, not by arrival order', () => {
// Google documents that parts may come back in any order. Every successful
// body already carries its own id, so the mapping is free and cannot drift.
const body = part('{"id":"m2"}') + part('{"id":"m1"}') + '--X--\r\n'
const r = parseBatchResponse('multipart/mixed; boundary=X', body)
expect(new Set(r.messages.map((m) => m.id))).toEqual(new Set(['m1', 'm2']))
})
it('counts an errored part rather than dropping it', () => {
const body = part('{"id":"m1"}') + part('{"error":{"message":"nope"}}', '404 Not Found') + '--X--\r\n'
const r = parseBatchResponse('multipart/mixed; boundary=X', body)
expect(r.messages).toHaveLength(1)
expect(r.failed).toBe(1)
})
it('counts an unparseable part rather than dropping it', () => {
const body = part('{"id":"m1"}') + '--X\r\nContent-Type: application/http\r\n\r\nHTTP/1.1 200 OK\r\n\r\n{not json\r\n' + '--X--\r\n'
const r = parseBatchResponse('multipart/mixed; boundary=X', body)
expect(r.messages).toHaveLength(1)
expect(r.failed).toBe(1)
})
it('handles a quoted boundary', () => {
const body = part('{"id":"m1"}') + '--X--\r\n'
expect(parseBatchResponse('multipart/mixed; boundary="X"', body).messages).toHaveLength(1)
})
it('invents nothing when the response is not multipart at all', () => {
// An error page or a shape change. Returning a message here would be worse
// than returning none.
const r = parseBatchResponse('text/html', '<html>500</html>')
expect(r.messages).toEqual([])
})
})
describe('readMailbox pages the list', () => {
const page = (ids: string[], next?: string, estimate = 0) =>
json({ messages: ids.map((id) => ({ id })), nextPageToken: next, resultSizeEstimate: estimate })
const paged = (pages: { ids: string[]; next?: string }[], estimate: number) =>
(async (url: string | URL, init?: RequestInit) => {
const u = String(url)
if (u.includes('oauth2')) return json({ access_token: 'tok', expires_in: 3599 })
if (u.includes('/batch/')) {
const ids = [...String(init?.body ?? '').matchAll(/messages\/([^?]+)\?/g)].map((m) => m[1]!)
const parts = ids
.map((id) => `--X\r\nContent-Type: application/http\r\n\r\nHTTP/1.1 200 OK\r\n\r\n${JSON.stringify({ id, threadId: 't', internalDate: '1788210063000', labelIds: ['INBOX'], payload: { headers: [] } })}\r\n`)
.join('')
return new Response(`${parts}--X--\r\n`, { headers: { 'content-type': 'multipart/mixed; boundary=X' } })
}
const token = new URL(u).searchParams.get('pageToken')
const i = token ? pages.findIndex((p) => p.next === token) + 1 : 0
const pg = pages[Math.max(0, i)]!
return page(pg.ids, pg.next, estimate)
}) as unknown as typeof fetch
it('follows nextPageToken past the first page', async () => {
// One list call would stop at whatever the first page held.
const r = await readMailbox('hello@meridiandisplay.example', CREDS, {
paceMs: 0,
maxMessages: 500,
fetchImpl: paged([{ ids: ['a', 'b'], next: 'p2' }, { ids: ['c'] }], 3),
})
expect(r.messages).toHaveLength(3)
expect(r.truncated).toBe(false)
})
it('stops FETCHING at maxMessages, but still counts the whole window', async () => {
// The list is walked to the end because ids are cheap; only the fetch is
// capped. That is what makes windowTotal a count rather than a guess.
const r = await readMailbox('hello@meridiandisplay.example', CREDS, {
paceMs: 0,
maxMessages: 2,
fetchImpl: paged([{ ids: ['a', 'b'], next: 'p2' }, { ids: ['c'] }], 9),
})
expect(r.messages).toHaveLength(2)
expect(r.truncated).toBe(true)
expect(r.windowTotal).toBe(3)
})
it('counts the window rather than trusting resultSizeEstimate', async () => {
// Gmail's resultSizeEstimate is capped near the requested page size (about
// 501 at maxResults=500 for any larger window), a number that looks
// authoritative and is not. The fake below returns a deliberately wrong
// estimate; the counted answer must ignore it.
const r = await readMailbox('hello@meridiandisplay.example', CREDS, {
paceMs: 0,
maxMessages: 500,
fetchImpl: paged([{ ids: ['a', 'b', 'c'] }], 99999),
})
expect(r.windowTotal).toBe(3)
expect(r.truncated).toBe(false)
})
})
/**
* Gmail refusing part of a chunk.
*
* `messages.get` costs 5 quota units against a 250-per-second-per-user ceiling.
* A chunk of 100 (Gmail's documented maximum) spends 500 at once, twice the
* limit, and the overflow is refused: about a third of every large mailbox.
* The documented maximum is a limit on the request, not permission to spend
* the quota. Refused parts must be retried and, if still refused, counted.
*/
describe('batching survives Gmail refusing part of a chunk', () => {
const okMsg = (id: string) =>
JSON.stringify({ id, threadId: 't', internalDate: '1788210063000', labelIds: ['INBOX'], payload: { headers: [] } })
/** Refuses the first `refuseFirst` ids it is ever asked for, then relents. */
const flaky = (ids: string[], refuseFirst: number) => {
const refused = new Set(ids.slice(0, refuseFirst))
return (async (url: string | URL, init?: RequestInit) => {
const u = String(url)
if (u.includes('oauth2')) return json({ access_token: 'tok', expires_in: 3599 })
if (u.includes('/batch/')) {
const asked = [...String(init?.body ?? '').matchAll(/messages\/([^?]+)\?/g)].map((m) => m[1]!)
const parts = asked
.map((id) => {
if (refused.has(id)) {
refused.delete(id) // relent on the retry, like a rate limit does
return `--X\r\nContent-Type: application/http\r\n\r\nHTTP/1.1 429 Too Many Requests\r\n\r\n${JSON.stringify({ error: { message: 'rateLimitExceeded' } })}\r\n`
}
return `--X\r\nContent-Type: application/http\r\n\r\nHTTP/1.1 200 OK\r\n\r\n${okMsg(id)}\r\n`
})
.join('')
return new Response(`${parts}--X--\r\n`, { headers: { 'content-type': 'multipart/mixed; boundary=X' } })
}
return json({ messages: ids.map((id) => ({ id })) })
}) as unknown as typeof fetch
}
it('re-asks for refused messages and gets them, losing none', async () => {
const ids = Array.from({ length: 30 }, (_, i) => `m${i}`)
const r = await readMailbox('hello@meridiandisplay.example', CREDS, {
paceMs: 0,
maxMessages: 100,
fetchImpl: flaky(ids, 12),
})
expect(r.messages).toHaveLength(30)
expect(r.failedFetches).toBe(0)
})
it('reports what is still missing after the retries rather than shrinking quietly', async () => {
// A mailbox that returns 18 of 30 and says nothing looks exactly like a
// mailbox with 18 messages in it.
const ids = Array.from({ length: 30 }, (_, i) => `m${i}`)
const alwaysRefuse = (async (url: string | URL, init?: RequestInit) => {
const u = String(url)
if (u.includes('oauth2')) return json({ access_token: 'tok', expires_in: 3599 })
if (u.includes('/batch/')) {
const asked = [...String(init?.body ?? '').matchAll(/messages\/([^?]+)\?/g)].map((m) => m[1]!)
const parts = asked
.map((id) =>
Number(id.slice(1)) < 10
? `--X\r\nContent-Type: application/http\r\n\r\nHTTP/1.1 200 OK\r\n\r\n${okMsg(id)}\r\n`
: `--X\r\nContent-Type: application/http\r\n\r\nHTTP/1.1 429\r\n\r\n${JSON.stringify({ error: {} })}\r\n`,
)
.join('')
return new Response(`${parts}--X--\r\n`, { headers: { 'content-type': 'multipart/mixed; boundary=X' } })
}
return json({ messages: ids.map((id) => ({ id })) })
}) as unknown as typeof fetch
const r = await readMailbox('hello@meridiandisplay.example', CREDS, { paceMs: 0, maxMessages: 100, fetchImpl: alwaysRefuse })
expect(r.messages).toHaveLength(10)
expect(r.failedFetches).toBe(20)
expect(r.ok).toBe(true) // still a usable mailbox, just an incomplete one
})
it('survives a whole chunk being refused at the HTTP level', async () => {
const ids = Array.from({ length: 5 }, (_, i) => `m${i}`)
let calls = 0
const f = (async (url: string | URL) => {
const u = String(url)
if (u.includes('oauth2')) return json({ access_token: 'tok', expires_in: 3599 })
if (u.includes('/batch/')) {
calls++
return calls === 1 ? new Response('nope', { status: 503 }) : new Response(
ids.map((id) => `--X\r\nContent-Type: application/http\r\n\r\nHTTP/1.1 200 OK\r\n\r\n${okMsg(id)}\r\n`).join('') + '--X--\r\n',
{ headers: { 'content-type': 'multipart/mixed; boundary=X' } },
)
}
return json({ messages: ids.map((id) => ({ id })) })
}) as unknown as typeof fetch
const r = await readMailbox('hello@meridiandisplay.example', CREDS, { paceMs: 0, maxMessages: 100, fetchImpl: f })
expect(r.messages).toHaveLength(5)
expect(r.failedFetches).toBe(0)
})
})
/**
* The quota is per minute, so a short backoff does nothing.
*
* Gmail says so in its own words: "Quota exceeded for quota metric 'Total
* Query Cost' and limit 'Units per MINUTE per user'". Retries a few seconds
* apart land inside the same exhausted minute and are refused for the same
* reason, which is indistinguishable from a permanent failure unless you read
* the error. So the error text is kept.
*/
describe('a refused batch reports Gmail\'s own reason', () => {
/** Every part refused with the real quota error Gmail returns. */
const quotaRefusal = (ids: string[]) =>
(async (url: string | URL, init?: RequestInit) => {
const u = String(url)
if (u.includes('oauth2')) return json({ access_token: 'tok', expires_in: 3599 })
if (u.includes('/batch/')) {
const asked = [...String(init?.body ?? '').matchAll(/messages\/([^?]+)\?/g)].map((m) => m[1]!)
const parts = asked
.map(
() =>
`--X\r\nContent-Type: application/http\r\n\r\nHTTP/1.1 403 Forbidden\r\n\r\n${JSON.stringify({
error: {
code: 403,
status: 'PERMISSION_DENIED',
message:
"Quota exceeded for quota metric 'Total Query Cost' and limit 'Units per minute per user'",
},
})}\r\n`,
)
.join('')
return new Response(`${parts}--X--\r\n`, { headers: { 'content-type': 'multipart/mixed; boundary=X' } })
}
return json({ messages: ids.map((id) => ({ id })) })
}) as unknown as typeof fetch
it('carries the quota message, not just a count', async () => {
// An identical number of failures on every run cannot say why; the message
// can. "Units per MINUTE per user" is the difference between lengthening a
// retry delay and rewriting the fetcher.
const ids = ['m1', 'm2']
const r = await readMailbox('hello@meridiandisplay.example', CREDS, {
paceMs: 0,
maxMessages: 10,
fetchImpl: quotaRefusal(ids),
})
expect(r.failedFetches).toBe(2)
expect(r.failureSample.join(' ')).toContain('Units per minute per user')
expect(r.failureSample.join(' ')).toContain('403')
})
it('deduplicates and caps the sample, so it is a diagnosis and not a wall', async () => {
// Every refused part in a chunk carries the identical quota text. Three
// distinct reasons is useful; two hundred copies of one is noise that
// hides any second reason underneath it.
const ids = Array.from({ length: 60 }, (_, i) => `m${i}`)
const r = await readMailbox('hello@meridiandisplay.example', CREDS, {
paceMs: 0,
maxMessages: 100,
fetchImpl: quotaRefusal(ids),
})
expect(r.failedFetches).toBe(60)
expect(r.failureSample.length).toBeLessThanOrEqual(3)
expect(new Set(r.failureSample).size).toBe(r.failureSample.length)
})
it('reports nothing when nothing failed', async () => {
// An empty sample beside a zero count; never a stale reason from a prior
// pass, which would send someone chasing a limit they are not hitting.
const clean = (async (url: string | URL, init?: RequestInit) => {
const u = String(url)
if (u.includes('oauth2')) return json({ access_token: 'tok', expires_in: 3599 })
if (u.includes('/batch/')) {
const asked = [...String(init?.body ?? '').matchAll(/messages\/([^?]+)\?/g)].map((m) => m[1]!)
const parts = asked
.map(
(id) =>
`--X\r\nContent-Type: application/http\r\n\r\nHTTP/1.1 200 OK\r\n\r\n${JSON.stringify({ id, threadId: 't', internalDate: '1788210063000', labelIds: ['INBOX'], payload: { headers: [] } })}\r\n`,
)
.join('')
return new Response(`${parts}--X--\r\n`, { headers: { 'content-type': 'multipart/mixed; boundary=X' } })
}
return json({ messages: [{ id: 'm1' }] })
}) as unknown as typeof fetch
const r = await readMailbox('hello@meridiandisplay.example', CREDS, { paceMs: 0, fetchImpl: clean })
expect(r.failedFetches).toBe(0)
expect(r.failureSample).toEqual([])
})
})