1#[cfg(span_locations)]
2use std::cell::RefCell;
3#[cfg(span_locations)]
4use std::cmp;
5use std::fmt;
6use std::iter;
7use std::ops::RangeBounds;
8#[cfg(procmacro2_semver_exempt)]
9use std::path::Path;
10use std::path::PathBuf;
11use std::str::FromStr;
12use std::vec;
13
14use crate::strnom::{block_comment, skip_whitespace, whitespace, word_break, Cursor, PResult};
15use crate::{Delimiter, Punct, Spacing, TokenTree};
16use unicode_xid::UnicodeXID;
17
18#[derive(Clone)]
19pub struct TokenStream {
20 inner: Vec<TokenTree>,
21}
22
23#[derive(Debug)]
24pub struct LexError;
25
26impl TokenStream {
27 pub fn new() -> TokenStream {
28 TokenStream { inner: Vec::new() }
29 }
30
31 pub fn is_empty(&self) -> bool {
32 self.inner.len() == 0
33 }
34}
35
36#[cfg(span_locations)]
37fn get_cursor(src: &str) -> Cursor {
38 SOURCE_MAP.with(|cm| {
40 let mut cm = cm.borrow_mut();
41 let name = format!("<parsed string {}>", cm.files.len());
42 let span = cm.add_file(&name, src);
43 Cursor {
44 rest: src,
45 off: span.lo,
46 }
47 })
48}
49
50#[cfg(not(span_locations))]
51fn get_cursor(src: &str) -> Cursor {
52 Cursor { rest: src }
53}
54
55impl FromStr for TokenStream {
56 type Err = LexError;
57
58 fn from_str(src: &str) -> Result<TokenStream, LexError> {
59 let cursor = get_cursor(src);
61
62 match token_stream(cursor) {
63 Ok((input, output)) => {
64 if skip_whitespace(input).len() != 0 {
65 Err(LexError)
66 } else {
67 Ok(output)
68 }
69 }
70 Err(LexError) => Err(LexError),
71 }
72 }
73}
74
75impl fmt::Display for TokenStream {
76 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
77 let mut joint = false;
78 for (i, tt) in self.inner.iter().enumerate() {
79 if i != 0 && !joint {
80 write!(f, " ")?;
81 }
82 joint = false;
83 match *tt {
84 TokenTree::Group(ref tt) => {
85 let (start, end) = match tt.delimiter() {
86 Delimiter::Parenthesis => ("(", ")"),
87 Delimiter::Brace => ("{", "}"),
88 Delimiter::Bracket => ("[", "]"),
89 Delimiter::None => ("", ""),
90 };
91 if tt.stream().into_iter().next().is_none() {
92 write!(f, "{} {}", start, end)?
93 } else {
94 write!(f, "{} {} {}", start, tt.stream(), end)?
95 }
96 }
97 TokenTree::Ident(ref tt) => write!(f, "{}", tt)?,
98 TokenTree::Punct(ref tt) => {
99 write!(f, "{}", tt.as_char())?;
100 match tt.spacing() {
101 Spacing::Alone => {}
102 Spacing::Joint => joint = true,
103 }
104 }
105 TokenTree::Literal(ref tt) => write!(f, "{}", tt)?,
106 }
107 }
108
109 Ok(())
110 }
111}
112
113impl fmt::Debug for TokenStream {
114 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
115 f.write_str("TokenStream ")?;
116 f.debug_list().entries(self.clone()).finish()
117 }
118}
119
120#[cfg(use_proc_macro)]
121impl From<proc_macro::TokenStream> for TokenStream {
122 fn from(inner: proc_macro::TokenStream) -> TokenStream {
123 inner
124 .to_string()
125 .parse()
126 .expect("compiler token stream parse failed")
127 }
128}
129
130#[cfg(use_proc_macro)]
131impl From<TokenStream> for proc_macro::TokenStream {
132 fn from(inner: TokenStream) -> proc_macro::TokenStream {
133 inner
134 .to_string()
135 .parse()
136 .expect("failed to parse to compiler tokens")
137 }
138}
139
140impl From<TokenTree> for TokenStream {
141 fn from(tree: TokenTree) -> TokenStream {
142 TokenStream { inner: vec![tree] }
143 }
144}
145
146impl iter::FromIterator<TokenTree> for TokenStream {
147 fn from_iter<I: IntoIterator<Item = TokenTree>>(streams: I) -> Self {
148 let mut v = Vec::new();
149
150 for token in streams.into_iter() {
151 v.push(token);
152 }
153
154 TokenStream { inner: v }
155 }
156}
157
158impl iter::FromIterator<TokenStream> for TokenStream {
159 fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
160 let mut v = Vec::new();
161
162 for stream in streams.into_iter() {
163 v.extend(stream.inner);
164 }
165
166 TokenStream { inner: v }
167 }
168}
169
170impl Extend<TokenTree> for TokenStream {
171 fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, streams: I) {
172 self.inner.extend(streams);
173 }
174}
175
176impl Extend<TokenStream> for TokenStream {
177 fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I) {
178 self.inner
179 .extend(streams.into_iter().flat_map(|stream| stream));
180 }
181}
182
183pub type TokenTreeIter = vec::IntoIter<TokenTree>;
184
185impl IntoIterator for TokenStream {
186 type Item = TokenTree;
187 type IntoIter = TokenTreeIter;
188
189 fn into_iter(self) -> TokenTreeIter {
190 self.inner.into_iter()
191 }
192}
193
194#[derive(Clone, PartialEq, Eq)]
195pub struct SourceFile {
196 path: PathBuf,
197}
198
199impl SourceFile {
200 pub fn path(&self) -> PathBuf {
202 self.path.clone()
203 }
204
205 pub fn is_real(&self) -> bool {
206 false
208 }
209}
210
211impl fmt::Debug for SourceFile {
212 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
213 f.debug_struct("SourceFile")
214 .field("path", &self.path())
215 .field("is_real", &self.is_real())
216 .finish()
217 }
218}
219
220#[derive(Clone, Copy, Debug, PartialEq, Eq)]
221pub struct LineColumn {
222 pub line: usize,
223 pub column: usize,
224}
225
226#[cfg(span_locations)]
227thread_local! {
228 static SOURCE_MAP: RefCell<SourceMap> = RefCell::new(SourceMap {
229 files: vec![{
232 #[cfg(procmacro2_semver_exempt)]
233 {
234 FileInfo {
235 name: "<unspecified>".to_owned(),
236 span: Span { lo: 0, hi: 0 },
237 lines: vec![0],
238 }
239 }
240
241 #[cfg(not(procmacro2_semver_exempt))]
242 {
243 FileInfo {
244 span: Span { lo: 0, hi: 0 },
245 lines: vec![0],
246 }
247 }
248 }],
249 });
250}
251
252#[cfg(span_locations)]
253struct FileInfo {
254 #[cfg(procmacro2_semver_exempt)]
255 name: String,
256 span: Span,
257 lines: Vec<usize>,
258}
259
260#[cfg(span_locations)]
261impl FileInfo {
262 fn offset_line_column(&self, offset: usize) -> LineColumn {
263 assert!(self.span_within(Span {
264 lo: offset as u32,
265 hi: offset as u32
266 }));
267 let offset = offset - self.span.lo as usize;
268 match self.lines.binary_search(&offset) {
269 Ok(found) => LineColumn {
270 line: found + 1,
271 column: 0,
272 },
273 Err(idx) => LineColumn {
274 line: idx,
275 column: offset - self.lines[idx - 1],
276 },
277 }
278 }
279
280 fn span_within(&self, span: Span) -> bool {
281 span.lo >= self.span.lo && span.hi <= self.span.hi
282 }
283}
284
285#[cfg(span_locations)]
287fn lines_offsets(s: &str) -> Vec<usize> {
288 let mut lines = vec![0];
289 let mut prev = 0;
290 while let Some(len) = s[prev..].find('\n') {
291 prev += len + 1;
292 lines.push(prev);
293 }
294 lines
295}
296
297#[cfg(span_locations)]
298struct SourceMap {
299 files: Vec<FileInfo>,
300}
301
302#[cfg(span_locations)]
303impl SourceMap {
304 fn next_start_pos(&self) -> u32 {
305 self.files.last().unwrap().span.hi + 1
310 }
311
312 fn add_file(&mut self, name: &str, src: &str) -> Span {
313 let lines = lines_offsets(src);
314 let lo = self.next_start_pos();
315 let span = Span {
317 lo,
318 hi: lo + (src.len() as u32),
319 };
320
321 #[cfg(procmacro2_semver_exempt)]
322 self.files.push(FileInfo {
323 name: name.to_owned(),
324 span,
325 lines,
326 });
327
328 #[cfg(not(procmacro2_semver_exempt))]
329 self.files.push(FileInfo { span, lines });
330 let _ = name;
331
332 span
333 }
334
335 fn fileinfo(&self, span: Span) -> &FileInfo {
336 for file in &self.files {
337 if file.span_within(span) {
338 return file;
339 }
340 }
341 panic!("Invalid span with no related FileInfo!");
342 }
343}
344
345#[derive(Clone, Copy, PartialEq, Eq)]
346pub struct Span {
347 #[cfg(span_locations)]
348 lo: u32,
349 #[cfg(span_locations)]
350 hi: u32,
351}
352
353impl Span {
354 #[cfg(not(span_locations))]
355 pub fn call_site() -> Span {
356 Span {}
357 }
358
359 #[cfg(span_locations)]
360 pub fn call_site() -> Span {
361 Span { lo: 0, hi: 0 }
362 }
363
364 #[cfg(procmacro2_semver_exempt)]
365 pub fn def_site() -> Span {
366 Span::call_site()
367 }
368
369 #[cfg(procmacro2_semver_exempt)]
370 pub fn resolved_at(&self, _other: Span) -> Span {
371 *self
375 }
376
377 #[cfg(procmacro2_semver_exempt)]
378 pub fn located_at(&self, other: Span) -> Span {
379 other
380 }
381
382 #[cfg(procmacro2_semver_exempt)]
383 pub fn source_file(&self) -> SourceFile {
384 SOURCE_MAP.with(|cm| {
385 let cm = cm.borrow();
386 let fi = cm.fileinfo(*self);
387 SourceFile {
388 path: Path::new(&fi.name).to_owned(),
389 }
390 })
391 }
392
393 #[cfg(span_locations)]
394 pub fn start(&self) -> LineColumn {
395 SOURCE_MAP.with(|cm| {
396 let cm = cm.borrow();
397 let fi = cm.fileinfo(*self);
398 fi.offset_line_column(self.lo as usize)
399 })
400 }
401
402 #[cfg(span_locations)]
403 pub fn end(&self) -> LineColumn {
404 SOURCE_MAP.with(|cm| {
405 let cm = cm.borrow();
406 let fi = cm.fileinfo(*self);
407 fi.offset_line_column(self.hi as usize)
408 })
409 }
410
411 #[cfg(not(span_locations))]
412 pub fn join(&self, _other: Span) -> Option<Span> {
413 Some(Span {})
414 }
415
416 #[cfg(span_locations)]
417 pub fn join(&self, other: Span) -> Option<Span> {
418 SOURCE_MAP.with(|cm| {
419 let cm = cm.borrow();
420 if !cm.fileinfo(*self).span_within(other) {
422 return None;
423 }
424 Some(Span {
425 lo: cmp::min(self.lo, other.lo),
426 hi: cmp::max(self.hi, other.hi),
427 })
428 })
429 }
430}
431
432impl fmt::Debug for Span {
433 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
434 #[cfg(procmacro2_semver_exempt)]
435 return write!(f, "bytes({}..{})", self.lo, self.hi);
436
437 #[cfg(not(procmacro2_semver_exempt))]
438 write!(f, "Span")
439 }
440}
441
442pub fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) {
443 if cfg!(procmacro2_semver_exempt) {
444 debug.field("span", &span);
445 }
446}
447
448#[derive(Clone)]
449pub struct Group {
450 delimiter: Delimiter,
451 stream: TokenStream,
452 span: Span,
453}
454
455impl Group {
456 pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group {
457 Group {
458 delimiter,
459 stream,
460 span: Span::call_site(),
461 }
462 }
463
464 pub fn delimiter(&self) -> Delimiter {
465 self.delimiter
466 }
467
468 pub fn stream(&self) -> TokenStream {
469 self.stream.clone()
470 }
471
472 pub fn span(&self) -> Span {
473 self.span
474 }
475
476 pub fn span_open(&self) -> Span {
477 self.span
478 }
479
480 pub fn span_close(&self) -> Span {
481 self.span
482 }
483
484 pub fn set_span(&mut self, span: Span) {
485 self.span = span;
486 }
487}
488
489impl fmt::Display for Group {
490 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
491 let (left, right) = match self.delimiter {
492 Delimiter::Parenthesis => ("(", ")"),
493 Delimiter::Brace => ("{", "}"),
494 Delimiter::Bracket => ("[", "]"),
495 Delimiter::None => ("", ""),
496 };
497
498 f.write_str(left)?;
499 self.stream.fmt(f)?;
500 f.write_str(right)?;
501
502 Ok(())
503 }
504}
505
506impl fmt::Debug for Group {
507 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
508 let mut debug = fmt.debug_struct("Group");
509 debug.field("delimiter", &self.delimiter);
510 debug.field("stream", &self.stream);
511 #[cfg(procmacro2_semver_exempt)]
512 debug.field("span", &self.span);
513 debug.finish()
514 }
515}
516
517#[derive(Clone)]
518pub struct Ident {
519 sym: String,
520 span: Span,
521 raw: bool,
522}
523
524impl Ident {
525 fn _new(string: &str, raw: bool, span: Span) -> Ident {
526 validate_ident(string);
527
528 Ident {
529 sym: string.to_owned(),
530 span,
531 raw,
532 }
533 }
534
535 pub fn new(string: &str, span: Span) -> Ident {
536 Ident::_new(string, false, span)
537 }
538
539 pub fn new_raw(string: &str, span: Span) -> Ident {
540 Ident::_new(string, true, span)
541 }
542
543 pub fn span(&self) -> Span {
544 self.span
545 }
546
547 pub fn set_span(&mut self, span: Span) {
548 self.span = span;
549 }
550}
551
552#[inline]
553fn is_ident_start(c: char) -> bool {
554 ('a' <= c && c <= 'z')
555 || ('A' <= c && c <= 'Z')
556 || c == '_'
557 || (c > '\x7f' && UnicodeXID::is_xid_start(c))
558}
559
560#[inline]
561fn is_ident_continue(c: char) -> bool {
562 ('a' <= c && c <= 'z')
563 || ('A' <= c && c <= 'Z')
564 || c == '_'
565 || ('0' <= c && c <= '9')
566 || (c > '\x7f' && UnicodeXID::is_xid_continue(c))
567}
568
569fn validate_ident(string: &str) {
570 let validate = string;
571 if validate.is_empty() {
572 panic!("Ident is not allowed to be empty; use Option<Ident>");
573 }
574
575 if validate.bytes().all(|digit| digit >= b'0' && digit <= b'9') {
576 panic!("Ident cannot be a number; use Literal instead");
577 }
578
579 fn ident_ok(string: &str) -> bool {
580 let mut chars = string.chars();
581 let first = chars.next().unwrap();
582 if !is_ident_start(first) {
583 return false;
584 }
585 for ch in chars {
586 if !is_ident_continue(ch) {
587 return false;
588 }
589 }
590 true
591 }
592
593 if !ident_ok(validate) {
594 panic!("{:?} is not a valid Ident", string);
595 }
596}
597
598impl PartialEq for Ident {
599 fn eq(&self, other: &Ident) -> bool {
600 self.sym == other.sym && self.raw == other.raw
601 }
602}
603
604impl<T> PartialEq<T> for Ident
605where
606 T: ?Sized + AsRef<str>,
607{
608 fn eq(&self, other: &T) -> bool {
609 let other = other.as_ref();
610 if self.raw {
611 other.starts_with("r#") && self.sym == other[2..]
612 } else {
613 self.sym == other
614 }
615 }
616}
617
618impl fmt::Display for Ident {
619 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
620 if self.raw {
621 "r#".fmt(f)?;
622 }
623 self.sym.fmt(f)
624 }
625}
626
627impl fmt::Debug for Ident {
628 #[cfg(not(procmacro2_semver_exempt))]
630 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
631 let mut debug = f.debug_tuple("Ident");
632 debug.field(&format_args!("{}", self));
633 debug.finish()
634 }
635
636 #[cfg(procmacro2_semver_exempt)]
641 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
642 let mut debug = f.debug_struct("Ident");
643 debug.field("sym", &format_args!("{}", self));
644 debug.field("span", &self.span);
645 debug.finish()
646 }
647}
648
649#[derive(Clone)]
650pub struct Literal {
651 text: String,
652 span: Span,
653}
654
655macro_rules! suffixed_numbers {
656 ($($name:ident => $kind:ident,)*) => ($(
657 pub fn $name(n: $kind) -> Literal {
658 Literal::_new(format!(concat!("{}", stringify!($kind)), n))
659 }
660 )*)
661}
662
663macro_rules! unsuffixed_numbers {
664 ($($name:ident => $kind:ident,)*) => ($(
665 pub fn $name(n: $kind) -> Literal {
666 Literal::_new(n.to_string())
667 }
668 )*)
669}
670
671impl Literal {
672 fn _new(text: String) -> Literal {
673 Literal {
674 text,
675 span: Span::call_site(),
676 }
677 }
678
679 suffixed_numbers! {
680 u8_suffixed => u8,
681 u16_suffixed => u16,
682 u32_suffixed => u32,
683 u64_suffixed => u64,
684 u128_suffixed => u128,
685 usize_suffixed => usize,
686 i8_suffixed => i8,
687 i16_suffixed => i16,
688 i32_suffixed => i32,
689 i64_suffixed => i64,
690 i128_suffixed => i128,
691 isize_suffixed => isize,
692
693 f32_suffixed => f32,
694 f64_suffixed => f64,
695 }
696
697 unsuffixed_numbers! {
698 u8_unsuffixed => u8,
699 u16_unsuffixed => u16,
700 u32_unsuffixed => u32,
701 u64_unsuffixed => u64,
702 u128_unsuffixed => u128,
703 usize_unsuffixed => usize,
704 i8_unsuffixed => i8,
705 i16_unsuffixed => i16,
706 i32_unsuffixed => i32,
707 i64_unsuffixed => i64,
708 i128_unsuffixed => i128,
709 isize_unsuffixed => isize,
710 }
711
712 pub fn f32_unsuffixed(f: f32) -> Literal {
713 let mut s = f.to_string();
714 if !s.contains(".") {
715 s.push_str(".0");
716 }
717 Literal::_new(s)
718 }
719
720 pub fn f64_unsuffixed(f: f64) -> Literal {
721 let mut s = f.to_string();
722 if !s.contains(".") {
723 s.push_str(".0");
724 }
725 Literal::_new(s)
726 }
727
728 pub fn string(t: &str) -> Literal {
729 let mut text = String::with_capacity(t.len() + 2);
730 text.push('"');
731 for c in t.chars() {
732 if c == '\'' {
733 text.push(c);
735 } else {
736 text.extend(c.escape_default());
737 }
738 }
739 text.push('"');
740 Literal::_new(text)
741 }
742
743 pub fn character(t: char) -> Literal {
744 let mut text = String::new();
745 text.push('\'');
746 if t == '"' {
747 text.push(t);
749 } else {
750 text.extend(t.escape_default());
751 }
752 text.push('\'');
753 Literal::_new(text)
754 }
755
756 pub fn byte_string(bytes: &[u8]) -> Literal {
757 let mut escaped = "b\"".to_string();
758 for b in bytes {
759 match *b {
760 b'\0' => escaped.push_str(r"\0"),
761 b'\t' => escaped.push_str(r"\t"),
762 b'\n' => escaped.push_str(r"\n"),
763 b'\r' => escaped.push_str(r"\r"),
764 b'"' => escaped.push_str("\\\""),
765 b'\\' => escaped.push_str("\\\\"),
766 b'\x20'..=b'\x7E' => escaped.push(*b as char),
767 _ => escaped.push_str(&format!("\\x{:02X}", b)),
768 }
769 }
770 escaped.push('"');
771 Literal::_new(escaped)
772 }
773
774 pub fn span(&self) -> Span {
775 self.span
776 }
777
778 pub fn set_span(&mut self, span: Span) {
779 self.span = span;
780 }
781
782 pub fn subspan<R: RangeBounds<usize>>(&self, _range: R) -> Option<Span> {
783 None
784 }
785}
786
787impl fmt::Display for Literal {
788 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
789 self.text.fmt(f)
790 }
791}
792
793impl fmt::Debug for Literal {
794 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
795 let mut debug = fmt.debug_struct("Literal");
796 debug.field("lit", &format_args!("{}", self.text));
797 #[cfg(procmacro2_semver_exempt)]
798 debug.field("span", &self.span);
799 debug.finish()
800 }
801}
802
803fn token_stream(mut input: Cursor) -> PResult<TokenStream> {
804 let mut trees = Vec::new();
805 loop {
806 let input_no_ws = skip_whitespace(input);
807 if input_no_ws.rest.len() == 0 {
808 break;
809 }
810 if let Ok((a, tokens)) = doc_comment(input_no_ws) {
811 input = a;
812 trees.extend(tokens);
813 continue;
814 }
815
816 let (a, tt) = match token_tree(input_no_ws) {
817 Ok(p) => p,
818 Err(_) => break,
819 };
820 trees.push(tt);
821 input = a;
822 }
823 Ok((input, TokenStream { inner: trees }))
824}
825
826#[cfg(not(span_locations))]
827fn spanned<'a, T>(
828 input: Cursor<'a>,
829 f: fn(Cursor<'a>) -> PResult<'a, T>,
830) -> PResult<'a, (T, crate::Span)> {
831 let (a, b) = f(skip_whitespace(input))?;
832 Ok((a, ((b, crate::Span::_new_stable(Span::call_site())))))
833}
834
835#[cfg(span_locations)]
836fn spanned<'a, T>(
837 input: Cursor<'a>,
838 f: fn(Cursor<'a>) -> PResult<'a, T>,
839) -> PResult<'a, (T, crate::Span)> {
840 let input = skip_whitespace(input);
841 let lo = input.off;
842 let (a, b) = f(input)?;
843 let hi = a.off;
844 let span = crate::Span::_new_stable(Span { lo, hi });
845 Ok((a, (b, span)))
846}
847
848fn token_tree(input: Cursor) -> PResult<TokenTree> {
849 let (rest, (mut tt, span)) = spanned(input, token_kind)?;
850 tt.set_span(span);
851 Ok((rest, tt))
852}
853
854named!(token_kind -> TokenTree, alt!(
855 map!(group, |g| TokenTree::Group(crate::Group::_new_stable(g)))
856 |
857 map!(literal, |l| TokenTree::Literal(crate::Literal::_new_stable(l))) |
859 map!(op, TokenTree::Punct)
860 |
861 symbol_leading_ws
862));
863
864named!(group -> Group, alt!(
865 delimited!(
866 punct!("("),
867 token_stream,
868 punct!(")")
869 ) => { |ts| Group::new(Delimiter::Parenthesis, ts) }
870 |
871 delimited!(
872 punct!("["),
873 token_stream,
874 punct!("]")
875 ) => { |ts| Group::new(Delimiter::Bracket, ts) }
876 |
877 delimited!(
878 punct!("{"),
879 token_stream,
880 punct!("}")
881 ) => { |ts| Group::new(Delimiter::Brace, ts) }
882));
883
884fn symbol_leading_ws(input: Cursor) -> PResult<TokenTree> {
885 symbol(skip_whitespace(input))
886}
887
888fn symbol(input: Cursor) -> PResult<TokenTree> {
889 let raw = input.starts_with("r#");
890 let rest = input.advance((raw as usize) << 1);
891
892 let (rest, sym) = symbol_not_raw(rest)?;
893
894 if !raw {
895 let ident = crate::Ident::new(sym, crate::Span::call_site());
896 return Ok((rest, ident.into()));
897 }
898
899 if sym == "_" {
900 return Err(LexError);
901 }
902
903 let ident = crate::Ident::_new_raw(sym, crate::Span::call_site());
904 Ok((rest, ident.into()))
905}
906
907fn symbol_not_raw(input: Cursor) -> PResult<&str> {
908 let mut chars = input.char_indices();
909
910 match chars.next() {
911 Some((_, ch)) if is_ident_start(ch) => {}
912 _ => return Err(LexError),
913 }
914
915 let mut end = input.len();
916 for (i, ch) in chars {
917 if !is_ident_continue(ch) {
918 end = i;
919 break;
920 }
921 }
922
923 Ok((input.advance(end), &input.rest[..end]))
924}
925
926fn literal(input: Cursor) -> PResult<Literal> {
927 let input_no_ws = skip_whitespace(input);
928
929 match literal_nocapture(input_no_ws) {
930 Ok((a, ())) => {
931 let start = input.len() - input_no_ws.len();
932 let len = input_no_ws.len() - a.len();
933 let end = start + len;
934 Ok((a, Literal::_new(input.rest[start..end].to_string())))
935 }
936 Err(LexError) => Err(LexError),
937 }
938}
939
940named!(literal_nocapture -> (), alt!(
941 string
942 |
943 byte_string
944 |
945 byte
946 |
947 character
948 |
949 float
950 |
951 int
952));
953
954named!(string -> (), alt!(
955 quoted_string
956 |
957 preceded!(
958 punct!("r"),
959 raw_string
960 ) => { |_| () }
961));
962
963named!(quoted_string -> (), do_parse!(
964 punct!("\"") >>
965 cooked_string >>
966 tag!("\"") >>
967 option!(symbol_not_raw) >>
968 (())
969));
970
971fn cooked_string(input: Cursor) -> PResult<()> {
972 let mut chars = input.char_indices().peekable();
973 while let Some((byte_offset, ch)) = chars.next() {
974 match ch {
975 '"' => {
976 return Ok((input.advance(byte_offset), ()));
977 }
978 '\r' => {
979 if let Some((_, '\n')) = chars.next() {
980 } else {
982 break;
983 }
984 }
985 '\\' => match chars.next() {
986 Some((_, 'x')) => {
987 if !backslash_x_char(&mut chars) {
988 break;
989 }
990 }
991 Some((_, 'n')) | Some((_, 'r')) | Some((_, 't')) | Some((_, '\\'))
992 | Some((_, '\'')) | Some((_, '"')) | Some((_, '0')) => {}
993 Some((_, 'u')) => {
994 if !backslash_u(&mut chars) {
995 break;
996 }
997 }
998 Some((_, '\n')) | Some((_, '\r')) => {
999 while let Some(&(_, ch)) = chars.peek() {
1000 if ch.is_whitespace() {
1001 chars.next();
1002 } else {
1003 break;
1004 }
1005 }
1006 }
1007 _ => break,
1008 },
1009 _ch => {}
1010 }
1011 }
1012 Err(LexError)
1013}
1014
1015named!(byte_string -> (), alt!(
1016 delimited!(
1017 punct!("b\""),
1018 cooked_byte_string,
1019 tag!("\"")
1020 ) => { |_| () }
1021 |
1022 preceded!(
1023 punct!("br"),
1024 raw_string
1025 ) => { |_| () }
1026));
1027
1028fn cooked_byte_string(mut input: Cursor) -> PResult<()> {
1029 let mut bytes = input.bytes().enumerate();
1030 'outer: while let Some((offset, b)) = bytes.next() {
1031 match b {
1032 b'"' => {
1033 return Ok((input.advance(offset), ()));
1034 }
1035 b'\r' => {
1036 if let Some((_, b'\n')) = bytes.next() {
1037 } else {
1039 break;
1040 }
1041 }
1042 b'\\' => match bytes.next() {
1043 Some((_, b'x')) => {
1044 if !backslash_x_byte(&mut bytes) {
1045 break;
1046 }
1047 }
1048 Some((_, b'n')) | Some((_, b'r')) | Some((_, b't')) | Some((_, b'\\'))
1049 | Some((_, b'0')) | Some((_, b'\'')) | Some((_, b'"')) => {}
1050 Some((newline, b'\n')) | Some((newline, b'\r')) => {
1051 let rest = input.advance(newline + 1);
1052 for (offset, ch) in rest.char_indices() {
1053 if !ch.is_whitespace() {
1054 input = rest.advance(offset);
1055 bytes = input.bytes().enumerate();
1056 continue 'outer;
1057 }
1058 }
1059 break;
1060 }
1061 _ => break,
1062 },
1063 b if b < 0x80 => {}
1064 _ => break,
1065 }
1066 }
1067 Err(LexError)
1068}
1069
1070fn raw_string(input: Cursor) -> PResult<()> {
1071 let mut chars = input.char_indices();
1072 let mut n = 0;
1073 while let Some((byte_offset, ch)) = chars.next() {
1074 match ch {
1075 '"' => {
1076 n = byte_offset;
1077 break;
1078 }
1079 '#' => {}
1080 _ => return Err(LexError),
1081 }
1082 }
1083 for (byte_offset, ch) in chars {
1084 match ch {
1085 '"' if input.advance(byte_offset + 1).starts_with(&input.rest[..n]) => {
1086 let rest = input.advance(byte_offset + 1 + n);
1087 return Ok((rest, ()));
1088 }
1089 '\r' => {}
1090 _ => {}
1091 }
1092 }
1093 Err(LexError)
1094}
1095
1096named!(byte -> (), do_parse!(
1097 punct!("b") >>
1098 tag!("'") >>
1099 cooked_byte >>
1100 tag!("'") >>
1101 (())
1102));
1103
1104fn cooked_byte(input: Cursor) -> PResult<()> {
1105 let mut bytes = input.bytes().enumerate();
1106 let ok = match bytes.next().map(|(_, b)| b) {
1107 Some(b'\\') => match bytes.next().map(|(_, b)| b) {
1108 Some(b'x') => backslash_x_byte(&mut bytes),
1109 Some(b'n') | Some(b'r') | Some(b't') | Some(b'\\') | Some(b'0') | Some(b'\'')
1110 | Some(b'"') => true,
1111 _ => false,
1112 },
1113 b => b.is_some(),
1114 };
1115 if ok {
1116 match bytes.next() {
1117 Some((offset, _)) => {
1118 if input.chars().as_str().is_char_boundary(offset) {
1119 Ok((input.advance(offset), ()))
1120 } else {
1121 Err(LexError)
1122 }
1123 }
1124 None => Ok((input.advance(input.len()), ())),
1125 }
1126 } else {
1127 Err(LexError)
1128 }
1129}
1130
1131named!(character -> (), do_parse!(
1132 punct!("'") >>
1133 cooked_char >>
1134 tag!("'") >>
1135 (())
1136));
1137
1138fn cooked_char(input: Cursor) -> PResult<()> {
1139 let mut chars = input.char_indices();
1140 let ok = match chars.next().map(|(_, ch)| ch) {
1141 Some('\\') => match chars.next().map(|(_, ch)| ch) {
1142 Some('x') => backslash_x_char(&mut chars),
1143 Some('u') => backslash_u(&mut chars),
1144 Some('n') | Some('r') | Some('t') | Some('\\') | Some('0') | Some('\'') | Some('"') => {
1145 true
1146 }
1147 _ => false,
1148 },
1149 ch => ch.is_some(),
1150 };
1151 if ok {
1152 match chars.next() {
1153 Some((idx, _)) => Ok((input.advance(idx), ())),
1154 None => Ok((input.advance(input.len()), ())),
1155 }
1156 } else {
1157 Err(LexError)
1158 }
1159}
1160
1161macro_rules! next_ch {
1162 ($chars:ident @ $pat:pat $(| $rest:pat)*) => {
1163 match $chars.next() {
1164 Some((_, ch)) => match ch {
1165 $pat $(| $rest)* => ch,
1166 _ => return false,
1167 },
1168 None => return false
1169 }
1170 };
1171}
1172
1173fn backslash_x_char<I>(chars: &mut I) -> bool
1174where
1175 I: Iterator<Item = (usize, char)>,
1176{
1177 next_ch!(chars @ '0'..='7');
1178 next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F');
1179 true
1180}
1181
1182fn backslash_x_byte<I>(chars: &mut I) -> bool
1183where
1184 I: Iterator<Item = (usize, u8)>,
1185{
1186 next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F');
1187 next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F');
1188 true
1189}
1190
1191fn backslash_u<I>(chars: &mut I) -> bool
1192where
1193 I: Iterator<Item = (usize, char)>,
1194{
1195 next_ch!(chars @ '{');
1196 next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F');
1197 loop {
1198 let c = next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F' | '_' | '}');
1199 if c == '}' {
1200 return true;
1201 }
1202 }
1203}
1204
1205fn float(input: Cursor) -> PResult<()> {
1206 let (mut rest, ()) = float_digits(input)?;
1207 if let Some(ch) = rest.chars().next() {
1208 if is_ident_start(ch) {
1209 rest = symbol_not_raw(rest)?.0;
1210 }
1211 }
1212 word_break(rest)
1213}
1214
1215fn float_digits(input: Cursor) -> PResult<()> {
1216 let mut chars = input.chars().peekable();
1217 match chars.next() {
1218 Some(ch) if ch >= '0' && ch <= '9' => {}
1219 _ => return Err(LexError),
1220 }
1221
1222 let mut len = 1;
1223 let mut has_dot = false;
1224 let mut has_exp = false;
1225 while let Some(&ch) = chars.peek() {
1226 match ch {
1227 '0'..='9' | '_' => {
1228 chars.next();
1229 len += 1;
1230 }
1231 '.' => {
1232 if has_dot {
1233 break;
1234 }
1235 chars.next();
1236 if chars
1237 .peek()
1238 .map(|&ch| ch == '.' || is_ident_start(ch))
1239 .unwrap_or(false)
1240 {
1241 return Err(LexError);
1242 }
1243 len += 1;
1244 has_dot = true;
1245 }
1246 'e' | 'E' => {
1247 chars.next();
1248 len += 1;
1249 has_exp = true;
1250 break;
1251 }
1252 _ => break,
1253 }
1254 }
1255
1256 let rest = input.advance(len);
1257 if !(has_dot || has_exp || rest.starts_with("f32") || rest.starts_with("f64")) {
1258 return Err(LexError);
1259 }
1260
1261 if has_exp {
1262 let mut has_exp_value = false;
1263 while let Some(&ch) = chars.peek() {
1264 match ch {
1265 '+' | '-' => {
1266 if has_exp_value {
1267 break;
1268 }
1269 chars.next();
1270 len += 1;
1271 }
1272 '0'..='9' => {
1273 chars.next();
1274 len += 1;
1275 has_exp_value = true;
1276 }
1277 '_' => {
1278 chars.next();
1279 len += 1;
1280 }
1281 _ => break,
1282 }
1283 }
1284 if !has_exp_value {
1285 return Err(LexError);
1286 }
1287 }
1288
1289 Ok((input.advance(len), ()))
1290}
1291
1292fn int(input: Cursor) -> PResult<()> {
1293 let (mut rest, ()) = digits(input)?;
1294 if let Some(ch) = rest.chars().next() {
1295 if is_ident_start(ch) {
1296 rest = symbol_not_raw(rest)?.0;
1297 }
1298 }
1299 word_break(rest)
1300}
1301
1302fn digits(mut input: Cursor) -> PResult<()> {
1303 let base = if input.starts_with("0x") {
1304 input = input.advance(2);
1305 16
1306 } else if input.starts_with("0o") {
1307 input = input.advance(2);
1308 8
1309 } else if input.starts_with("0b") {
1310 input = input.advance(2);
1311 2
1312 } else {
1313 10
1314 };
1315
1316 let mut len = 0;
1317 let mut empty = true;
1318 for b in input.bytes() {
1319 let digit = match b {
1320 b'0'..=b'9' => (b - b'0') as u64,
1321 b'a'..=b'f' => 10 + (b - b'a') as u64,
1322 b'A'..=b'F' => 10 + (b - b'A') as u64,
1323 b'_' => {
1324 if empty && base == 10 {
1325 return Err(LexError);
1326 }
1327 len += 1;
1328 continue;
1329 }
1330 _ => break,
1331 };
1332 if digit >= base {
1333 return Err(LexError);
1334 }
1335 len += 1;
1336 empty = false;
1337 }
1338 if empty {
1339 Err(LexError)
1340 } else {
1341 Ok((input.advance(len), ()))
1342 }
1343}
1344
1345fn op(input: Cursor) -> PResult<Punct> {
1346 let input = skip_whitespace(input);
1347 match op_char(input) {
1348 Ok((rest, '\'')) => {
1349 symbol(rest)?;
1350 Ok((rest, Punct::new('\'', Spacing::Joint)))
1351 }
1352 Ok((rest, ch)) => {
1353 let kind = match op_char(rest) {
1354 Ok(_) => Spacing::Joint,
1355 Err(LexError) => Spacing::Alone,
1356 };
1357 Ok((rest, Punct::new(ch, kind)))
1358 }
1359 Err(LexError) => Err(LexError),
1360 }
1361}
1362
1363fn op_char(input: Cursor) -> PResult<char> {
1364 if input.starts_with("//") || input.starts_with("/*") {
1365 return Err(LexError);
1367 }
1368
1369 let mut chars = input.chars();
1370 let first = match chars.next() {
1371 Some(ch) => ch,
1372 None => {
1373 return Err(LexError);
1374 }
1375 };
1376 let recognized = "~!@#$%^&*-=+|;:,<.>/?'";
1377 if recognized.contains(first) {
1378 Ok((input.advance(first.len_utf8()), first))
1379 } else {
1380 Err(LexError)
1381 }
1382}
1383
1384fn doc_comment(input: Cursor) -> PResult<Vec<TokenTree>> {
1385 let mut trees = Vec::new();
1386 let (rest, ((comment, inner), span)) = spanned(input, doc_comment_contents)?;
1387 trees.push(TokenTree::Punct(Punct::new('#', Spacing::Alone)));
1388 if inner {
1389 trees.push(Punct::new('!', Spacing::Alone).into());
1390 }
1391 let mut stream = vec![
1392 TokenTree::Ident(crate::Ident::new("doc", span)),
1393 TokenTree::Punct(Punct::new('=', Spacing::Alone)),
1394 TokenTree::Literal(crate::Literal::string(comment)),
1395 ];
1396 for tt in stream.iter_mut() {
1397 tt.set_span(span);
1398 }
1399 let group = Group::new(Delimiter::Bracket, stream.into_iter().collect());
1400 trees.push(crate::Group::_new_stable(group).into());
1401 for tt in trees.iter_mut() {
1402 tt.set_span(span);
1403 }
1404 Ok((rest, trees))
1405}
1406
1407named!(doc_comment_contents -> (&str, bool), alt!(
1408 do_parse!(
1409 punct!("//!") >>
1410 s: take_until_newline_or_eof!() >>
1411 ((s, true))
1412 )
1413 |
1414 do_parse!(
1415 option!(whitespace) >>
1416 peek!(tag!("/*!")) >>
1417 s: block_comment >>
1418 ((s, true))
1419 )
1420 |
1421 do_parse!(
1422 punct!("///") >>
1423 not!(tag!("/")) >>
1424 s: take_until_newline_or_eof!() >>
1425 ((s, false))
1426 )
1427 |
1428 do_parse!(
1429 option!(whitespace) >>
1430 peek!(tuple!(tag!("/**"), not!(tag!("*")))) >>
1431 s: block_comment >>
1432 ((s, false))
1433 )
1434));