Sentences

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
// `Sentences` topical primitive — UAX #29 sentence segmentation
// over UTF-8.

primitive Sentences
  fun count(s: String box): (USize | InvalidUtf8) =>
    """
    Number of UAX #29 sentence segments in `s`. Sentences break at
    paragraph separators (SB4) and after end-of-sentence terminators
    like `.`, `!`, `?` followed by appropriate trailing context
    (SB11). Default is NO break — single text run with no sentence
    terminators yields one segment.
    """
    match Bytes.first_bad_utf8_offset(s)
    | AllValid => _count(s)
    | let off: USize => InvalidUtf8(off)
    end

  fun ranges(s: String box)
    : (Iterator[(USize, USize)] | InvalidUtf8)
  =>
    """
    Iterator over byte ranges `[start, end_exclusive)` for each
    sentence segment.
    """
    match Bytes.first_bad_utf8_offset(s)
    | AllValid => _ranges(s)
    | let off: USize => InvalidUtf8(off)
    end

  fun iter(s: String val)
    : (Iterator[String val] | InvalidUtf8)
  =>
    """
    Iterator yielding each sentence as a `String val` slice.
    """
    match Bytes.first_bad_utf8_offset(s)
    | AllValid => _iter(s)
    | let off: USize => InvalidUtf8(off)
    end

  // ============================================================
  // Workhorses (assume valid UTF-8)
  // ============================================================

  fun _count(utf8: String box): USize =>
    let cursor = _SentenceBreakCursor(utf8)
    var n: USize = 0
    var done: Bool = false
    while not done do
      match cursor.next_range()
      | (_, _) => n = n + 1
      | None => done = true
      end
    end
    n

  fun _ranges(utf8: String box): Iterator[(USize, USize)] =>
    _SentenceRangeIterator(utf8)

  fun _iter(utf8: String val): Iterator[String val] =>
    _SentenceSliceIterator(utf8)