unicode_segmentation/
sentence.rs1use core::cmp;
12use core::iter::Filter;
13
14mod fwd {
16 use tables::sentence::SentenceCat;
17 use core::cmp;
18
19 #[derive(Clone, Copy, PartialEq, Eq)]
22 enum StatePart {
23 Sot,
24 Eot,
25 Other,
26 CR,
27 LF,
28 Sep,
29 ATerm,
30 UpperLower,
31 ClosePlus,
32 SpPlus,
33 STerm
34 }
35
36 #[derive(Clone, PartialEq, Eq)]
37 struct SentenceBreaksState(pub [StatePart; 4]);
38
39 const INITIAL_STATE: SentenceBreaksState = SentenceBreaksState([
40 StatePart::Sot,
41 StatePart::Sot,
42 StatePart::Sot,
43 StatePart::Sot
44 ]);
45
46 #[derive(Clone)]
47 pub struct SentenceBreaks<'a> {
48 pub string: &'a str,
49 pos: usize,
50 state: SentenceBreaksState
51 }
52
53 impl SentenceBreaksState {
54 fn next(&self, cat: SentenceCat) -> SentenceBreaksState {
57 let &SentenceBreaksState(parts) = self;
58 let parts = match (parts[3], cat) {
59 (StatePart::ClosePlus, SentenceCat::SC_Close) => parts,
60 (StatePart::SpPlus, SentenceCat::SC_Sp) => parts,
61 _ => [
62 parts[1],
63 parts[2],
64 parts[3],
65 match cat {
66 SentenceCat::SC_CR => StatePart::CR,
67 SentenceCat::SC_LF => StatePart::LF,
68 SentenceCat::SC_Sep => StatePart::Sep,
69 SentenceCat::SC_ATerm => StatePart::ATerm,
70 SentenceCat::SC_Upper |
71 SentenceCat::SC_Lower => StatePart::UpperLower,
72 SentenceCat::SC_Close => StatePart::ClosePlus,
73 SentenceCat::SC_Sp => StatePart::SpPlus,
74 SentenceCat::SC_STerm => StatePart::STerm,
75 _ => StatePart::Other
76 }
77 ]
78 };
79 SentenceBreaksState(parts)
80 }
81
82 fn end(&self) -> SentenceBreaksState {
83 let &SentenceBreaksState(parts) = self;
84 SentenceBreaksState([
85 parts[1],
86 parts[2],
87 parts[3],
88 StatePart::Eot
89 ])
90 }
91
92 fn match1(&self, part: StatePart) -> bool {
94 let &SentenceBreaksState(parts) = self;
95 part == parts[3]
96 }
97
98 fn match2(&self, part1: StatePart, part2: StatePart) -> bool {
101 let &SentenceBreaksState(parts) = self;
102 part1 == parts[2] && part2 == parts[3]
103 }
104 }
105
106 fn match_sb8(state: &SentenceBreaksState, ahead: &str) -> bool {
109 let &SentenceBreaksState(parts) = state;
110 let mut idx = if parts[3] == StatePart::SpPlus { 2 } else { 3 };
111 if parts[idx] == StatePart::ClosePlus { idx -= 1 }
112
113 if parts[idx] == StatePart::ATerm {
114 use tables::sentence as se;
115
116 for next_char in ahead.chars() {
117 match se::sentence_category(next_char) {
119 se::SC_Lower => return true,
120 se::SC_OLetter |
121 se::SC_Upper |
122 se::SC_Sep | se::SC_CR | se::SC_LF |
123 se::SC_STerm | se::SC_ATerm => return false,
124 _ => continue
125 }
126 }
127 }
128
129 false
130 }
131
132 fn match_sb8a(state: &SentenceBreaksState) -> bool {
134 let &SentenceBreaksState(parts) = state;
136 let mut idx = if parts[3] == StatePart::SpPlus { 2 } else { 3 };
137 if parts[idx] == StatePart::ClosePlus { idx -= 1 }
138 parts[idx] == StatePart::STerm || parts[idx] == StatePart::ATerm
139 }
140
141 fn match_sb9(state: &SentenceBreaksState) -> bool {
143 let &SentenceBreaksState(parts) = state;
145 let idx = if parts[3] == StatePart::ClosePlus { 2 } else { 3 };
146 parts[idx] == StatePart::STerm || parts[idx] == StatePart::ATerm
147 }
148
149 fn match_sb11(state: &SentenceBreaksState) -> bool {
151 let &SentenceBreaksState(parts) = state;
153 let mut idx = match parts[3] {
154 StatePart::Sep |
155 StatePart::CR |
156 StatePart::LF => 2,
157 _ => 3
158 };
159
160 if parts[idx] == StatePart::SpPlus { idx -= 1 }
161 if parts[idx] == StatePart::ClosePlus { idx -= 1}
162
163 parts[idx] == StatePart::STerm || parts[idx] == StatePart::ATerm
164 }
165
166 impl<'a> Iterator for SentenceBreaks<'a> {
167 type Item = usize;
169
170 #[inline]
171 fn size_hint(&self) -> (usize, Option<usize>) {
172 let slen = self.string.len();
173 (cmp::min(slen, 2), Some(slen + 1))
175 }
176
177 #[inline]
178 fn next(&mut self) -> Option<usize> {
179 use tables::sentence as se;
180
181 for next_char in self.string[self.pos..].chars() {
182 let position_before = self.pos;
183 let state_before = self.state.clone();
184
185 let next_cat = se::sentence_category(next_char);
186
187 self.pos += next_char.len_utf8();
188 self.state = self.state.next(next_cat);
189
190 match next_cat {
191 _ if state_before.match1(StatePart::Sot) =>
193 return Some(position_before),
194
195 SentenceCat::SC_LF if state_before.match1(StatePart::CR) =>
199 continue,
200
201 _ if state_before.match1(StatePart::Sep)
203 || state_before.match1(StatePart::CR)
204 || state_before.match1(StatePart::LF)
205 => return Some(position_before),
206
207 SentenceCat::SC_Extend |
209 SentenceCat::SC_Format => self.state = state_before,
210
211 SentenceCat::SC_Numeric if state_before.match1(StatePart::ATerm) =>
213 continue,
214
215 SentenceCat::SC_Upper if state_before.match2(StatePart::UpperLower, StatePart::ATerm) =>
217 continue,
218
219 _ if match_sb8(&state_before, &self.string[position_before..]) =>
221 continue,
222
223 SentenceCat::SC_SContinue |
225 SentenceCat::SC_STerm |
226 SentenceCat::SC_ATerm if match_sb8a(&state_before) =>
227 continue,
228
229 SentenceCat::SC_Close |
231 SentenceCat::SC_Sp |
232 SentenceCat::SC_Sep |
233 SentenceCat::SC_CR |
234 SentenceCat::SC_LF if match_sb9(&state_before) =>
235 continue,
236
237 SentenceCat::SC_Sp |
239 SentenceCat::SC_Sep |
240 SentenceCat::SC_CR |
241 SentenceCat::SC_LF if match_sb8a(&state_before) =>
242 continue,
243
244 _ if match_sb11(&state_before) =>
246 return Some(position_before),
247
248 _ => continue
250 }
251 }
252
253 if self.state.match1(StatePart::Sot) {
255 None
256 } else if self.state.match1(StatePart::Eot) {
257 None
258 } else {
259 self.state = self.state.end();
260 Some(self.pos)
261 }
262 }
263 }
264
265 pub fn new_sentence_breaks<'a>(source: &'a str) -> SentenceBreaks<'a> {
266 SentenceBreaks { string: source, pos: 0, state: INITIAL_STATE }
267 }
268
269}
270
271#[derive(Clone)]
278pub struct UnicodeSentences<'a> {
279 inner: Filter<USentenceBounds<'a>, fn(&&str) -> bool>,
280}
281
282#[derive(Clone)]
285pub struct USentenceBounds<'a> {
286 iter: fwd::SentenceBreaks<'a>,
287 sentence_start: Option<usize>
288}
289
290#[derive(Clone)]
292pub struct USentenceBoundIndices<'a> {
293 start_offset: usize,
294 iter: USentenceBounds<'a>,
295}
296
297#[inline]
298pub fn new_sentence_bounds<'a>(source: &'a str) -> USentenceBounds<'a> {
299 USentenceBounds {
300 iter: fwd::new_sentence_breaks(source),
301 sentence_start: None
302 }
303}
304
305#[inline]
306pub fn new_sentence_bound_indices<'a>(source: &'a str) -> USentenceBoundIndices<'a> {
307 USentenceBoundIndices {
308 start_offset: source.as_ptr() as usize,
309 iter: new_sentence_bounds(source)
310 }
311}
312
313#[inline]
314pub fn new_unicode_sentences<'b>(s: &'b str) -> UnicodeSentences<'b> {
315 use super::UnicodeSegmentation;
316 use tables::util::is_alphanumeric;
317
318 fn has_alphanumeric(s: &&str) -> bool { s.chars().any(|c| is_alphanumeric(c)) }
319 let has_alphanumeric: fn(&&str) -> bool = has_alphanumeric; UnicodeSentences { inner: s.split_sentence_bounds().filter(has_alphanumeric) }
322}
323
324impl<'a> Iterator for UnicodeSentences<'a> {
325 type Item = &'a str;
326
327 #[inline]
328 fn next(&mut self) -> Option<&'a str> { self.inner.next() }
329}
330
331impl<'a> Iterator for USentenceBounds<'a> {
332 type Item = &'a str;
333
334 #[inline]
335 fn size_hint(&self) -> (usize, Option<usize>) {
336 let (lower, upper) = self.iter.size_hint();
337 (cmp::max(0, lower - 1), upper.map(|u| cmp::max(0, u - 1)))
338 }
339
340 #[inline]
341 fn next(&mut self) -> Option<&'a str> {
342 if self.sentence_start == None {
343 if let Some(start_pos) = self.iter.next() {
344 self.sentence_start = Some(start_pos)
345 } else {
346 return None
347 }
348 }
349
350 if let Some(break_pos) = self.iter.next() {
351 let start_pos = self.sentence_start.unwrap();
352 let sentence = &self.iter.string[start_pos..break_pos];
353 self.sentence_start = Some(break_pos);
354 Some(sentence)
355 } else {
356 None
357 }
358 }
359}
360
361impl<'a> Iterator for USentenceBoundIndices<'a> {
362 type Item = (usize, &'a str);
363
364 #[inline]
365 fn next(&mut self) -> Option<(usize, &'a str)> {
366 self.iter.next().map(|s| (s.as_ptr() as usize - self.start_offset, s))
367 }
368
369 #[inline]
370 fn size_hint(&self) -> (usize, Option<usize>) {
371 self.iter.size_hint()
372 }
373}