fix(gmail): encode body parts as base64 and decode numeric HTML entities

Author: waleedlatif1

apps/sim/tools/gmail/utils.test.ts

@@ -15,6 +15,20 @@ function decodeSimpleMessage(encoded: string): string {
   return Buffer.from(encoded, 'base64url').toString('utf-8')
 }
 
+/**
+ * Extract and base64-decode the body of a specific MIME part identified by its
+ * Content-Type prefix (e.g. `text/plain`, `text/html`). Returns the decoded
+ * UTF-8 string.
+ */
+function decodePart(mime: string, contentTypePrefix: string): string {
+  const partRegex = new RegExp(
+    `Content-Type: ${contentTypePrefix}[^\\n]*\\nContent-Transfer-Encoding: base64\\n\\n([\\s\\S]*?)\\n\\n--`
+  )
+  const match = mime.match(partRegex)
+  if (!match) throw new Error(`No ${contentTypePrefix} part found`)
+  return Buffer.from(match[1].replace(/\n/g, ''), 'base64').toString('utf-8')
+}
+
 describe('encodeRfc2047', () => {
   it('returns ASCII text unchanged', () => {
     expect(encodeRfc2047('Simple ASCII Subject')).toBe('Simple ASCII Subject')
@@ -81,6 +95,12 @@ describe('htmlToPlainText', () => {
       '&lt; is the literal < entity'
     )
   })
+
+  it('decodes decimal and hexadecimal numeric entities', () => {
+    expect(htmlToPlainText('<p>&#8220;hi&#8221; &#160;and&#x2019;s</p>')).toBe(
+      '\u201chi\u201d \u00a0and\u2019s'
+    )
+  })
 })
 
 describe('buildSimpleEmailMessage', () => {
@@ -96,8 +116,21 @@ describe('buildSimpleEmailMessage', () => {
     const htmlIdx = decoded.indexOf('text/html')
     expect(plainIdx).toBeGreaterThan(-1)
     expect(htmlIdx).toBeGreaterThan(plainIdx)
-    expect(decoded).toContain('Hi Janice,')
-    expect(decoded).toContain('<p>Hi Janice,</p>')
+    expect(decodePart(decoded, 'text/plain')).toBe('Hi Janice,\n\nQuick question.')
+    expect(decodePart(decoded, 'text/html')).toContain('<p>Hi Janice,</p>')
+  })
+
+  it('encodes bodies as base64 so UTF-8 (emoji, accents) round-trips cleanly', () => {
+    const body = 'Café 🎉 — résumé'
+    const encoded = buildSimpleEmailMessage({
+      to: 'a@example.com',
+      subject: 'Hi',
+      body,
+    })
+    const decoded = decodeSimpleMessage(encoded)
+    expect(decoded).toContain('Content-Transfer-Encoding: base64')
+    expect(decodePart(decoded, 'text/plain')).toBe(body)
+    expect(decodePart(decoded, 'text/html')).toContain('Café 🎉 — résumé')
   })
 
   it('uses the supplied HTML body and derives a plain-text fallback when contentType is html', () => {
@@ -108,8 +141,8 @@ describe('buildSimpleEmailMessage', () => {
       contentType: 'html',
     })
     const decoded = decodeSimpleMessage(encoded)
-    expect(decoded).toContain('<p>Hello <b>there</b></p>')
-    expect(decoded).toContain('Hello there')
+    expect(decodePart(decoded, 'text/html')).toBe('<p>Hello <b>there</b></p>')
+    expect(decodePart(decoded, 'text/plain')).toBe('Hello there')
   })
 
   it('includes threading headers when replying', () => {
@@ -142,7 +175,8 @@ describe('buildMimeMessage', () => {
     expect(message).toMatch(/Content-Type: multipart\/mixed; boundary="([^"]+)"/)
     expect(message).toMatch(/Content-Type: multipart\/alternative; boundary="([^"]+)"/)
     expect(message).toContain('Content-Disposition: attachment; filename="note.txt"')
-    expect(message).toContain('<p>Hello</p>')
+    expect(decodePart(message, 'text/plain')).toBe('Hello')
+    expect(decodePart(message, 'text/html')).toContain('<p>Hello</p>')
   })
 
   it('emits multipart/alternative without multipart/mixed when no attachments', () => {

apps/sim/tools/gmail/utils.ts

@@ -359,7 +359,8 @@ export function htmlToPlainText(html: string): string {
     .replace(/&lt;/g, '<')
     .replace(/&gt;/g, '>')
     .replace(/&quot;/g, '"')
-    .replace(/&#39;/g, "'")
+    .replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16)))
+    .replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number.parseInt(dec, 10)))
     .replace(/&amp;/g, '&')
     .replace(/\n{3,}/g, '\n\n')
     .trim()
@@ -381,6 +382,16 @@ export function buildBodyAlternatives(
   return { plain: body, html: plainTextToHtml(body) }
 }
 
+/**
+ * Encode a text body as base64 with RFC 2045 line wrapping (max 76 chars).
+ * Using base64 lets us safely transport arbitrary UTF-8 (emoji, accented
+ * characters, etc.) — `7bit` is only valid for strict 7-bit ASCII.
+ */
+function encodeBodyBase64(content: string): string[] {
+  const base64 = Buffer.from(content, 'utf-8').toString('base64')
+  return base64.match(/.{1,76}/g) || ['']
+}
+
 /**
  * Render the inner part of a `multipart/alternative` section (text/plain
  * followed by text/html, per RFC 2046 — clients pick the last format they
@@ -390,15 +401,15 @@ function renderAlternativeParts(plain: string, html: string, boundary: string):
   return [
     `--${boundary}`,
     'Content-Type: text/plain; charset="UTF-8"',
-    'Content-Transfer-Encoding: 7bit',
+    'Content-Transfer-Encoding: base64',
     '',
-    plain,
+    ...encodeBodyBase64(plain),
     '',
     `--${boundary}`,
     'Content-Type: text/html; charset="UTF-8"',
-    'Content-Transfer-Encoding: 7bit',
+    'Content-Transfer-Encoding: base64',
     '',
-    html,
+    ...encodeBodyBase64(html),
     '',
     `--${boundary}--`,
   ]