sentences.py

transcription_pipeline/sentences.py

1. """Sentence splitting, ported from the original JavaScript splitter.
3. Splits on . ? ! followed by whitespace, with carve-outs for abbreviations, single
4. initials, decimals, closing quotes and brackets, and an ellipsis followed by a
5. lowercase word (a pause mid-thought, not the end of a sentence). Each sentence
6. comes back with the character offset where it starts, which is what lets the
7. pipeline map sentences onto timed words. The output must match
8. tests/fixtures/sentences-golden.json exactly.
9. """
11. from __future__ import annotations
13. import re
15. # Words that end in a period but do not end a sentence.
16. ABBREVIATIONS = {
17. "mr", "mrs", "ms", "dr", "prof", "sr", "jr", "st", "mt",
18. "inc", "ltd", "llc", "co", "corp", "dept", "est", "fig",
19. "vs", "etc", "ie", "eg", "approx", "no", "vol", "jan", "feb",
20. "mar", "apr", "jun", "jul", "aug", "sep", "sept", "oct", "nov", "dec",
21. "a.m", "p.m", "u.s", "u.k",
22. }
23. TERMINATORS = ".?!"
24. CLOSERS = "\"'”’)]"
26. _WS = re.compile(r"\s")
27. _LAST_WORD = re.compile(r"([A-Za-z][A-Za-z.]*)$")
30. def split_sentences(text: str) -> list[dict]:
31. """Return [{"text": sentence, "offset": start_index}, ...]."""
32. out: list[dict] = []
33. start = 0
34. i = 0
35. n = len(text)
36. while i < n:
37. ch = text[i]
38. if ch not in TERMINATORS:
39. i += 1
40. continue
42. # Absorb runs of terminators and any closing quote/bracket.
43. end = i
44. while end + 1 < n and text[end + 1] in TERMINATORS:
45. end += 1
46. while end + 1 < n and text[end + 1] in CLOSERS:
47. end += 1
49. # A sentence break needs whitespace (or end of text) after it.
50. if end + 1 < n and not _WS.match(text[end + 1]):
51. i = end + 1
52. continue
54. # A trailing ellipsis followed by a lowercase word is a pause, not an end.
55. if end > i and text[i:end + 1] == "." * (end - i + 1):
56. if re.match(r"\s*[a-z]", text[end + 1:]):
57. i = end + 1
58. continue
60. if ch == ".":
61. before = text[start:i]
62. m = _LAST_WORD.search(before)
63. if m:
64. bare = m.group(1).lower()
65. if bare.endswith("."):
66. bare = bare[:-1]
67. # Known abbreviation, or a single initial like "J."
68. if bare in ABBREVIATIONS or re.fullmatch(r"[a-z]", bare):
69. i = end + 1
70. continue
71. # A decimal point: a digit before and after.
72. if re.search(r"\d$", before) and re.match(r"\s*\d", text[end + 1:]):
73. i = end + 1
74. continue
76. raw = text[start:end + 1]
77. trimmed = raw.strip()
78. if trimmed:
79. out.append({"text": trimmed, "offset": start + (len(raw) - len(raw.lstrip()))})
80. start = end + 1
81. i = end + 1
83. tail = text[start:]
84. if tail.strip():
85. out.append({"text": tail.strip(), "offset": start + (len(tail) - len(tail.lstrip()))})
86. return out