body.ts

packages/gmail-client/src/body.ts

1. /**
2. * Message bodies: fetch one, and turn Gmail's MIME tree into readable text.
3. *
4. * Fetched on demand, not stored. The list in KV stays metadata-only and a body
5. * is pulled from Gmail when someone opens that message. Three reasons, in order
6. * of how badly each would bite:
7. *
8. * A KV value caps at 25 MiB and the whole value is read on every /inbox
9. * request. Seven mailboxes × 100 messages with bodies would blow past the cap
10. * and make the list page slow for everyone to serve one person's read.
11. *
12. * Bodies at rest in the board's own storage is a much larger thing to hold
13. * than a subject line. On demand means the board never accumulates seven
14. * people's mail.
15. *
16. * A body read now is the body as it is now.
17. *
18. * HTML is carried, but only the frame route may render it. A sender's markup
19. * is an XSS surface handed to anyone who can email the company, so the HTML is
20. * never injected into a board page. It is served
21. * by `/inbox/message/frame` as its own document with its own CSP (no scripts,
22. * no forms) and displayed through a sandboxed iframe without `allow-scripts`.
23. * Anything that puts `html` into a page the board's scripts run on
24. * reintroduces exactly that surface. The text conversion
25. * stays as the fallback for plain messages and failed frames.
26. */
28. /** Gmail's URL-safe base64, which atob() does not accept as-is. */
29. export function decodeB64Url(data: string): string {
30. const b64 = data.replace(/-/g, '+').replace(/_/g, '/')
31. const padded = b64 + '='.repeat((4 - (b64.length % 4)) % 4)
32. const bin = atob(padded)
33. // The payload is UTF-8; atob gives latin-1 bytes, so re-decode or accented
34. // characters and emoji arrive as mojibake.
35. return new TextDecoder('utf-8').decode(Uint8Array.from(bin, (c) => c.charCodeAt(0)))
36. }
38. /**
39. * The same base64url, decoded to bytes rather than text.
40. *
41. * Separate from decodeB64Url because an attachment is not text: running a PDF
42. * through TextDecoder('utf-8') replaces every invalid sequence with U+FFFD and
43. * hands back a file that opens as garbage.
44. */
45. export function decodeB64UrlBytes(data: string): Uint8Array {
46. const b64 = data.replace(/-/g, '+').replace(/_/g, '/')
47. const padded = b64 + '='.repeat((4 - (b64.length % 4)) % 4)
48. const bin = atob(padded)
49. const out = new Uint8Array(bin.length)
50. for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i)
51. return out
52. }
54. /** Tags whose content is not readable text and must go with them. */
55. const DROP_BLOCKS = /<(script|style|head|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi
57. /**
58. * HTML to text. Same shape as stripHtml in the Ninety client, plus the two
59. * things email needs that a to-do description does not.
60. */
61. export function htmlToText(html: string): string {
62. return html
63. // Drop script/style CONTENT, not just their tags. Stripping tags alone
64. // leaves a wall of CSS in the middle of the message.
65. .replace(DROP_BLOCKS, ' ')
66. .replace(/<\s*(br|\/li|\/p|\/div|\/tr|\/h[1-6])\s*\/?>/gi, '\n')
67. .replace(/<\s*li[^>]*>/gi, '- ')
68. .replace(/<[^>]+>/g, '')
69. .replace(/ | | /gi, ' ')
70. .replace(/&/gi, '&')
71. .replace(/&lt;/gi, '<')
72. .replace(/>/gi, '>')
73. .replace(/"/gi, '"')
74. .replace(/'|'/gi, "'")
75. // Marketing HTML is padded with zero-width and soft-hyphen filler to game
76. // preview text: the "͏ ͏ ͏" in every snippet. Left in, a message reads as
77. // hundreds of blank lines.
78. .replace(/[­​-‏⁠]/g, '')
79. .replace(/[ \t]+/g, ' ')
80. .replace(/\n[ \t]+/g, '\n')
81. .replace(/\n{3,}/g, '\n\n')
82. .trim()
83. }
85. interface Part {
86. mimeType?: string
87. filename?: string
88. body?: { data?: string; size?: number; attachmentId?: string }
89. parts?: Part[]
90. headers?: { name: string; value: string }[]
91. }
93. /**
94. * One attachment on an incoming message.
95. *
96. * More than a name, because a download route needs three things a name cannot
97. * carry: Gmail's `attachmentId` to fetch it, the size to decide whether it is
98. * inside the cap before fetching anything, and the mime type.
99. *
100. * `size` and `attachmentId` obey the prime invariant. Gmail reports a size for
101. * every part it has one for; when it reports none the field is `null` and the
102. * page says "size not reported", never 0, since a 0 would read as an empty
103. * file and would pass a size check it was never measured against.
104. */
105. export interface Attachment {
106. /**
107. * Position in the MIME walk, from 0: the download route's address.
108. *
109. * The route takes this and not `attachmentId`: Gmail's ids are hundreds of
110. * characters and change between reads of the same message, so a URL built
111. * from one is long and stale. The index is stable for a given message
112. * because the walk order is, and the route re-reads the message and re-walks
113. * it before fetching anything.
114. */
115. index: number
116. /** The filename as the sender spelled it. Never trusted as a path. */
117. name: string
118. mimeType: string
119. /** Bytes, from Gmail's `body.size`. Null when Gmail reported none. */
120. size: number | null
121. /** Gmail's handle for the bytes. Null when the part carries none. */
122. attachmentId: string | null
123. /**
124. * An inline part: a `cid:` image the HTML references, not a file anyone
125. * attached. Signature logos arrive as several of these, and counting them as
126. * attachments on the message page would be true and useless.
127. */
128. inline: boolean
129. }
131. /**
132. * Every attachment part, in walk order.
133. *
134. * Inline is detected, not guessed: `Content-Disposition: inline` or the
135. * presence of a `Content-ID` is what a mail client uses, and both come free at
136. * format=full. A part with a filename and neither is a real attachment.
137. */
138. export function collectAttachments(payload: Part | undefined): Attachment[] {
139. const out: Attachment[] = []
140. const walk = (p: Part): void => {
141. if (p.filename) {
142. const header = (n: string): string =>
143. p.headers?.find((h) => h.name.toLowerCase() === n)?.value ?? ''
144. out.push({
145. index: out.length,
146. name: p.filename,
147. mimeType: (p.mimeType ?? 'application/octet-stream').toLowerCase(),
148. // `?? null`, not `?? 0`: see the interface. Gmail sends 0 for an empty
149. // part and omits the field when it has nothing to say; those are
150. // different facts and the page renders them differently.
151. size: typeof p.body?.size === 'number' ? p.body.size : null,
152. attachmentId: p.body?.attachmentId ?? null,
153. inline: /(^|;|\s)inline(;|$|\s)/i.test(header('content-disposition')) || Boolean(header('content-id')),
154. })
155. }
156. for (const child of p.parts ?? []) walk(child)
157. }
158. if (payload) walk(payload)
159. return out
160. }
162. /**
163. * Walk the MIME tree for the best readable body.
164. *
165. * Prefers text/plain, falls back to text/html converted to text. Skips anything
166. * with a filename; that is an attachment, not the message.
167. */
168. export function extractBody(
169. payload: Part | undefined,
170. ): { text: string; kind: 'plain' | 'html' | 'none'; html: string } {
171. if (!payload) return { text: '', kind: 'none', html: '' }
172. let plain = ''
173. let html = ''
175. const walk = (p: Part): void => {
176. if (p.filename) return // attachment
177. const type = (p.mimeType ?? '').toLowerCase()
178. const data = p.body?.data
179. if (data) {
180. if (type === 'text/plain' && !plain) plain = decodeB64Url(data)
181. else if (type === 'text/html' && !html) html = decodeB64Url(data)
182. }
183. for (const child of p.parts ?? []) walk(child)
184. }
185. walk(payload)
187. // The raw HTML rides along even when text/plain wins the text field: the
188. // frame route renders the HTML part, the text is the fallback and the search
189. // haystack. See the header for why the HTML must only ever reach the frame.
190. if (plain.trim()) return { text: plain.trim(), kind: 'plain', html: html.trim() }
191. if (html.trim()) return { text: htmlToText(html), kind: 'html', html: html.trim() }
192. return { text: '', kind: 'none', html: '' }
193. }
195. /** How much of a body to keep. Enough to read; not enough to be a mail client. */
196. export const BODY_CAP = 20_000
198. /**
199. * The HTML cap is much larger because markup is mostly overhead: a marketing
200. * email's 200KB is tables and inline CSS around two paragraphs, and cutting it
201. * at text size would truncate mid-tag and render garbage.
202. */
203. export const HTML_CAP = 400_000
205. export interface MessageBody {
206. id: string
207. mailbox: string
208. sender: string
209. senderName: string
210. subject: string
211. date: string
212. to: string
213. text: string
214. kind: 'plain' | 'html' | 'none'
215. /** True when the body was cut. Said on the page, never silently truncated. */
216. truncated: boolean
217. /**
218. * The sender's HTML, capped, or null when the message has no HTML part.
219. * Only the /inbox/message/frame route may put this in a response; see the
220. * file header. Board pages link to the frame; they never embed this string.
221. */
222. html: string | null
223. htmlTruncated: boolean
224. /** Gmail's own UNREAD label; the open-message page offers mark-read from it. */
225. unread: boolean
226. /**
227. * Every attachment part, with the size and handle a download needs. Empty
228. * array means "we read the MIME tree and there are none": a real zero, not
229. * an absence.
230. */
231. attachments: Attachment[]
232. /**
233. * Gmail's thread id, meaningful only inside this mailbox.
234. *
235. * This and the fields below are threading and reply-addressing. They are
236. * read from the original message on the server and are the only source of
237. * the reply's recipient, subject and thread; the compose form carries no To,
238. * no Subject and no thread id. A form field a person can edit is a form field
239. * an attacker can edit, and the one irreversible thing this board does must
240. * not take its recipient from the request that triggers it.
241. *
242. * Thread ids are per-mailbox, so passing `hello@`'s id on a send through
243. * `accounts@`'s token is rejected by Gmail. See decideReply(): it is sent
244. * only when replying from the mailbox the message arrived in.
245. */
246. threadId: string
247. /** The Message-ID header, angle brackets as sent. Null when absent (rare). */
248. messageId: string | null
249. /** The References header verbatim, or null. */
250. references: string | null
251. /** Reply-To verbatim, '' if absent. Beats From as the reply recipient. */
252. replyTo: string
253. /**
254. * Cc verbatim, '' if absent.
255. *
256. * Also the source of a reply-all's Cc list, derived on the server by
257. * `decideReply` from this field and `to`, and never from the form.
258. */
259. cc: string
260. }