proc_macro2/
fallback.rs

1#[cfg(span_locations)]
2use std::cell::RefCell;
3#[cfg(span_locations)]
4use std::cmp;
5use std::fmt;
6use std::iter;
7use std::ops::RangeBounds;
8#[cfg(procmacro2_semver_exempt)]
9use std::path::Path;
10use std::path::PathBuf;
11use std::str::FromStr;
12use std::vec;
13
14use crate::strnom::{block_comment, skip_whitespace, whitespace, word_break, Cursor, PResult};
15use crate::{Delimiter, Punct, Spacing, TokenTree};
16use unicode_xid::UnicodeXID;
17
18#[derive(Clone)]
19pub struct TokenStream {
20    inner: Vec<TokenTree>,
21}
22
23#[derive(Debug)]
24pub struct LexError;
25
26impl TokenStream {
27    pub fn new() -> TokenStream {
28        TokenStream { inner: Vec::new() }
29    }
30
31    pub fn is_empty(&self) -> bool {
32        self.inner.len() == 0
33    }
34}
35
36#[cfg(span_locations)]
37fn get_cursor(src: &str) -> Cursor {
38    // Create a dummy file & add it to the source map
39    SOURCE_MAP.with(|cm| {
40        let mut cm = cm.borrow_mut();
41        let name = format!("<parsed string {}>", cm.files.len());
42        let span = cm.add_file(&name, src);
43        Cursor {
44            rest: src,
45            off: span.lo,
46        }
47    })
48}
49
50#[cfg(not(span_locations))]
51fn get_cursor(src: &str) -> Cursor {
52    Cursor { rest: src }
53}
54
55impl FromStr for TokenStream {
56    type Err = LexError;
57
58    fn from_str(src: &str) -> Result<TokenStream, LexError> {
59        // Create a dummy file & add it to the source map
60        let cursor = get_cursor(src);
61
62        match token_stream(cursor) {
63            Ok((input, output)) => {
64                if skip_whitespace(input).len() != 0 {
65                    Err(LexError)
66                } else {
67                    Ok(output)
68                }
69            }
70            Err(LexError) => Err(LexError),
71        }
72    }
73}
74
75impl fmt::Display for TokenStream {
76    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
77        let mut joint = false;
78        for (i, tt) in self.inner.iter().enumerate() {
79            if i != 0 && !joint {
80                write!(f, " ")?;
81            }
82            joint = false;
83            match *tt {
84                TokenTree::Group(ref tt) => {
85                    let (start, end) = match tt.delimiter() {
86                        Delimiter::Parenthesis => ("(", ")"),
87                        Delimiter::Brace => ("{", "}"),
88                        Delimiter::Bracket => ("[", "]"),
89                        Delimiter::None => ("", ""),
90                    };
91                    if tt.stream().into_iter().next().is_none() {
92                        write!(f, "{} {}", start, end)?
93                    } else {
94                        write!(f, "{} {} {}", start, tt.stream(), end)?
95                    }
96                }
97                TokenTree::Ident(ref tt) => write!(f, "{}", tt)?,
98                TokenTree::Punct(ref tt) => {
99                    write!(f, "{}", tt.as_char())?;
100                    match tt.spacing() {
101                        Spacing::Alone => {}
102                        Spacing::Joint => joint = true,
103                    }
104                }
105                TokenTree::Literal(ref tt) => write!(f, "{}", tt)?,
106            }
107        }
108
109        Ok(())
110    }
111}
112
113impl fmt::Debug for TokenStream {
114    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
115        f.write_str("TokenStream ")?;
116        f.debug_list().entries(self.clone()).finish()
117    }
118}
119
120#[cfg(use_proc_macro)]
121impl From<proc_macro::TokenStream> for TokenStream {
122    fn from(inner: proc_macro::TokenStream) -> TokenStream {
123        inner
124            .to_string()
125            .parse()
126            .expect("compiler token stream parse failed")
127    }
128}
129
130#[cfg(use_proc_macro)]
131impl From<TokenStream> for proc_macro::TokenStream {
132    fn from(inner: TokenStream) -> proc_macro::TokenStream {
133        inner
134            .to_string()
135            .parse()
136            .expect("failed to parse to compiler tokens")
137    }
138}
139
140impl From<TokenTree> for TokenStream {
141    fn from(tree: TokenTree) -> TokenStream {
142        TokenStream { inner: vec![tree] }
143    }
144}
145
146impl iter::FromIterator<TokenTree> for TokenStream {
147    fn from_iter<I: IntoIterator<Item = TokenTree>>(streams: I) -> Self {
148        let mut v = Vec::new();
149
150        for token in streams.into_iter() {
151            v.push(token);
152        }
153
154        TokenStream { inner: v }
155    }
156}
157
158impl iter::FromIterator<TokenStream> for TokenStream {
159    fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
160        let mut v = Vec::new();
161
162        for stream in streams.into_iter() {
163            v.extend(stream.inner);
164        }
165
166        TokenStream { inner: v }
167    }
168}
169
170impl Extend<TokenTree> for TokenStream {
171    fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, streams: I) {
172        self.inner.extend(streams);
173    }
174}
175
176impl Extend<TokenStream> for TokenStream {
177    fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I) {
178        self.inner
179            .extend(streams.into_iter().flat_map(|stream| stream));
180    }
181}
182
183pub type TokenTreeIter = vec::IntoIter<TokenTree>;
184
185impl IntoIterator for TokenStream {
186    type Item = TokenTree;
187    type IntoIter = TokenTreeIter;
188
189    fn into_iter(self) -> TokenTreeIter {
190        self.inner.into_iter()
191    }
192}
193
194#[derive(Clone, PartialEq, Eq)]
195pub struct SourceFile {
196    path: PathBuf,
197}
198
199impl SourceFile {
200    /// Get the path to this source file as a string.
201    pub fn path(&self) -> PathBuf {
202        self.path.clone()
203    }
204
205    pub fn is_real(&self) -> bool {
206        // XXX(nika): Support real files in the future?
207        false
208    }
209}
210
211impl fmt::Debug for SourceFile {
212    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
213        f.debug_struct("SourceFile")
214            .field("path", &self.path())
215            .field("is_real", &self.is_real())
216            .finish()
217    }
218}
219
220#[derive(Clone, Copy, Debug, PartialEq, Eq)]
221pub struct LineColumn {
222    pub line: usize,
223    pub column: usize,
224}
225
226#[cfg(span_locations)]
227thread_local! {
228    static SOURCE_MAP: RefCell<SourceMap> = RefCell::new(SourceMap {
229        // NOTE: We start with a single dummy file which all call_site() and
230        // def_site() spans reference.
231        files: vec![{
232            #[cfg(procmacro2_semver_exempt)]
233            {
234                FileInfo {
235                    name: "<unspecified>".to_owned(),
236                    span: Span { lo: 0, hi: 0 },
237                    lines: vec![0],
238                }
239            }
240
241            #[cfg(not(procmacro2_semver_exempt))]
242            {
243                FileInfo {
244                    span: Span { lo: 0, hi: 0 },
245                    lines: vec![0],
246                }
247            }
248        }],
249    });
250}
251
252#[cfg(span_locations)]
253struct FileInfo {
254    #[cfg(procmacro2_semver_exempt)]
255    name: String,
256    span: Span,
257    lines: Vec<usize>,
258}
259
260#[cfg(span_locations)]
261impl FileInfo {
262    fn offset_line_column(&self, offset: usize) -> LineColumn {
263        assert!(self.span_within(Span {
264            lo: offset as u32,
265            hi: offset as u32
266        }));
267        let offset = offset - self.span.lo as usize;
268        match self.lines.binary_search(&offset) {
269            Ok(found) => LineColumn {
270                line: found + 1,
271                column: 0,
272            },
273            Err(idx) => LineColumn {
274                line: idx,
275                column: offset - self.lines[idx - 1],
276            },
277        }
278    }
279
280    fn span_within(&self, span: Span) -> bool {
281        span.lo >= self.span.lo && span.hi <= self.span.hi
282    }
283}
284
285/// Computesthe offsets of each line in the given source string.
286#[cfg(span_locations)]
287fn lines_offsets(s: &str) -> Vec<usize> {
288    let mut lines = vec![0];
289    let mut prev = 0;
290    while let Some(len) = s[prev..].find('\n') {
291        prev += len + 1;
292        lines.push(prev);
293    }
294    lines
295}
296
297#[cfg(span_locations)]
298struct SourceMap {
299    files: Vec<FileInfo>,
300}
301
302#[cfg(span_locations)]
303impl SourceMap {
304    fn next_start_pos(&self) -> u32 {
305        // Add 1 so there's always space between files.
306        //
307        // We'll always have at least 1 file, as we initialize our files list
308        // with a dummy file.
309        self.files.last().unwrap().span.hi + 1
310    }
311
312    fn add_file(&mut self, name: &str, src: &str) -> Span {
313        let lines = lines_offsets(src);
314        let lo = self.next_start_pos();
315        // XXX(nika): Shouild we bother doing a checked cast or checked add here?
316        let span = Span {
317            lo,
318            hi: lo + (src.len() as u32),
319        };
320
321        #[cfg(procmacro2_semver_exempt)]
322        self.files.push(FileInfo {
323            name: name.to_owned(),
324            span,
325            lines,
326        });
327
328        #[cfg(not(procmacro2_semver_exempt))]
329        self.files.push(FileInfo { span, lines });
330        let _ = name;
331
332        span
333    }
334
335    fn fileinfo(&self, span: Span) -> &FileInfo {
336        for file in &self.files {
337            if file.span_within(span) {
338                return file;
339            }
340        }
341        panic!("Invalid span with no related FileInfo!");
342    }
343}
344
345#[derive(Clone, Copy, PartialEq, Eq)]
346pub struct Span {
347    #[cfg(span_locations)]
348    lo: u32,
349    #[cfg(span_locations)]
350    hi: u32,
351}
352
353impl Span {
354    #[cfg(not(span_locations))]
355    pub fn call_site() -> Span {
356        Span {}
357    }
358
359    #[cfg(span_locations)]
360    pub fn call_site() -> Span {
361        Span { lo: 0, hi: 0 }
362    }
363
364    #[cfg(procmacro2_semver_exempt)]
365    pub fn def_site() -> Span {
366        Span::call_site()
367    }
368
369    #[cfg(procmacro2_semver_exempt)]
370    pub fn resolved_at(&self, _other: Span) -> Span {
371        // Stable spans consist only of line/column information, so
372        // `resolved_at` and `located_at` only select which span the
373        // caller wants line/column information from.
374        *self
375    }
376
377    #[cfg(procmacro2_semver_exempt)]
378    pub fn located_at(&self, other: Span) -> Span {
379        other
380    }
381
382    #[cfg(procmacro2_semver_exempt)]
383    pub fn source_file(&self) -> SourceFile {
384        SOURCE_MAP.with(|cm| {
385            let cm = cm.borrow();
386            let fi = cm.fileinfo(*self);
387            SourceFile {
388                path: Path::new(&fi.name).to_owned(),
389            }
390        })
391    }
392
393    #[cfg(span_locations)]
394    pub fn start(&self) -> LineColumn {
395        SOURCE_MAP.with(|cm| {
396            let cm = cm.borrow();
397            let fi = cm.fileinfo(*self);
398            fi.offset_line_column(self.lo as usize)
399        })
400    }
401
402    #[cfg(span_locations)]
403    pub fn end(&self) -> LineColumn {
404        SOURCE_MAP.with(|cm| {
405            let cm = cm.borrow();
406            let fi = cm.fileinfo(*self);
407            fi.offset_line_column(self.hi as usize)
408        })
409    }
410
411    #[cfg(not(span_locations))]
412    pub fn join(&self, _other: Span) -> Option<Span> {
413        Some(Span {})
414    }
415
416    #[cfg(span_locations)]
417    pub fn join(&self, other: Span) -> Option<Span> {
418        SOURCE_MAP.with(|cm| {
419            let cm = cm.borrow();
420            // If `other` is not within the same FileInfo as us, return None.
421            if !cm.fileinfo(*self).span_within(other) {
422                return None;
423            }
424            Some(Span {
425                lo: cmp::min(self.lo, other.lo),
426                hi: cmp::max(self.hi, other.hi),
427            })
428        })
429    }
430}
431
432impl fmt::Debug for Span {
433    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
434        #[cfg(procmacro2_semver_exempt)]
435        return write!(f, "bytes({}..{})", self.lo, self.hi);
436
437        #[cfg(not(procmacro2_semver_exempt))]
438        write!(f, "Span")
439    }
440}
441
442pub fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) {
443    if cfg!(procmacro2_semver_exempt) {
444        debug.field("span", &span);
445    }
446}
447
448#[derive(Clone)]
449pub struct Group {
450    delimiter: Delimiter,
451    stream: TokenStream,
452    span: Span,
453}
454
455impl Group {
456    pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group {
457        Group {
458            delimiter,
459            stream,
460            span: Span::call_site(),
461        }
462    }
463
464    pub fn delimiter(&self) -> Delimiter {
465        self.delimiter
466    }
467
468    pub fn stream(&self) -> TokenStream {
469        self.stream.clone()
470    }
471
472    pub fn span(&self) -> Span {
473        self.span
474    }
475
476    pub fn span_open(&self) -> Span {
477        self.span
478    }
479
480    pub fn span_close(&self) -> Span {
481        self.span
482    }
483
484    pub fn set_span(&mut self, span: Span) {
485        self.span = span;
486    }
487}
488
489impl fmt::Display for Group {
490    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
491        let (left, right) = match self.delimiter {
492            Delimiter::Parenthesis => ("(", ")"),
493            Delimiter::Brace => ("{", "}"),
494            Delimiter::Bracket => ("[", "]"),
495            Delimiter::None => ("", ""),
496        };
497
498        f.write_str(left)?;
499        self.stream.fmt(f)?;
500        f.write_str(right)?;
501
502        Ok(())
503    }
504}
505
506impl fmt::Debug for Group {
507    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
508        let mut debug = fmt.debug_struct("Group");
509        debug.field("delimiter", &self.delimiter);
510        debug.field("stream", &self.stream);
511        #[cfg(procmacro2_semver_exempt)]
512        debug.field("span", &self.span);
513        debug.finish()
514    }
515}
516
517#[derive(Clone)]
518pub struct Ident {
519    sym: String,
520    span: Span,
521    raw: bool,
522}
523
524impl Ident {
525    fn _new(string: &str, raw: bool, span: Span) -> Ident {
526        validate_ident(string);
527
528        Ident {
529            sym: string.to_owned(),
530            span,
531            raw,
532        }
533    }
534
535    pub fn new(string: &str, span: Span) -> Ident {
536        Ident::_new(string, false, span)
537    }
538
539    pub fn new_raw(string: &str, span: Span) -> Ident {
540        Ident::_new(string, true, span)
541    }
542
543    pub fn span(&self) -> Span {
544        self.span
545    }
546
547    pub fn set_span(&mut self, span: Span) {
548        self.span = span;
549    }
550}
551
552#[inline]
553fn is_ident_start(c: char) -> bool {
554    ('a' <= c && c <= 'z')
555        || ('A' <= c && c <= 'Z')
556        || c == '_'
557        || (c > '\x7f' && UnicodeXID::is_xid_start(c))
558}
559
560#[inline]
561fn is_ident_continue(c: char) -> bool {
562    ('a' <= c && c <= 'z')
563        || ('A' <= c && c <= 'Z')
564        || c == '_'
565        || ('0' <= c && c <= '9')
566        || (c > '\x7f' && UnicodeXID::is_xid_continue(c))
567}
568
569fn validate_ident(string: &str) {
570    let validate = string;
571    if validate.is_empty() {
572        panic!("Ident is not allowed to be empty; use Option<Ident>");
573    }
574
575    if validate.bytes().all(|digit| digit >= b'0' && digit <= b'9') {
576        panic!("Ident cannot be a number; use Literal instead");
577    }
578
579    fn ident_ok(string: &str) -> bool {
580        let mut chars = string.chars();
581        let first = chars.next().unwrap();
582        if !is_ident_start(first) {
583            return false;
584        }
585        for ch in chars {
586            if !is_ident_continue(ch) {
587                return false;
588            }
589        }
590        true
591    }
592
593    if !ident_ok(validate) {
594        panic!("{:?} is not a valid Ident", string);
595    }
596}
597
598impl PartialEq for Ident {
599    fn eq(&self, other: &Ident) -> bool {
600        self.sym == other.sym && self.raw == other.raw
601    }
602}
603
604impl<T> PartialEq<T> for Ident
605where
606    T: ?Sized + AsRef<str>,
607{
608    fn eq(&self, other: &T) -> bool {
609        let other = other.as_ref();
610        if self.raw {
611            other.starts_with("r#") && self.sym == other[2..]
612        } else {
613            self.sym == other
614        }
615    }
616}
617
618impl fmt::Display for Ident {
619    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
620        if self.raw {
621            "r#".fmt(f)?;
622        }
623        self.sym.fmt(f)
624    }
625}
626
627impl fmt::Debug for Ident {
628    // Ident(proc_macro), Ident(r#union)
629    #[cfg(not(procmacro2_semver_exempt))]
630    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
631        let mut debug = f.debug_tuple("Ident");
632        debug.field(&format_args!("{}", self));
633        debug.finish()
634    }
635
636    // Ident {
637    //     sym: proc_macro,
638    //     span: bytes(128..138)
639    // }
640    #[cfg(procmacro2_semver_exempt)]
641    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
642        let mut debug = f.debug_struct("Ident");
643        debug.field("sym", &format_args!("{}", self));
644        debug.field("span", &self.span);
645        debug.finish()
646    }
647}
648
649#[derive(Clone)]
650pub struct Literal {
651    text: String,
652    span: Span,
653}
654
655macro_rules! suffixed_numbers {
656    ($($name:ident => $kind:ident,)*) => ($(
657        pub fn $name(n: $kind) -> Literal {
658            Literal::_new(format!(concat!("{}", stringify!($kind)), n))
659        }
660    )*)
661}
662
663macro_rules! unsuffixed_numbers {
664    ($($name:ident => $kind:ident,)*) => ($(
665        pub fn $name(n: $kind) -> Literal {
666            Literal::_new(n.to_string())
667        }
668    )*)
669}
670
671impl Literal {
672    fn _new(text: String) -> Literal {
673        Literal {
674            text,
675            span: Span::call_site(),
676        }
677    }
678
679    suffixed_numbers! {
680        u8_suffixed => u8,
681        u16_suffixed => u16,
682        u32_suffixed => u32,
683        u64_suffixed => u64,
684        u128_suffixed => u128,
685        usize_suffixed => usize,
686        i8_suffixed => i8,
687        i16_suffixed => i16,
688        i32_suffixed => i32,
689        i64_suffixed => i64,
690        i128_suffixed => i128,
691        isize_suffixed => isize,
692
693        f32_suffixed => f32,
694        f64_suffixed => f64,
695    }
696
697    unsuffixed_numbers! {
698        u8_unsuffixed => u8,
699        u16_unsuffixed => u16,
700        u32_unsuffixed => u32,
701        u64_unsuffixed => u64,
702        u128_unsuffixed => u128,
703        usize_unsuffixed => usize,
704        i8_unsuffixed => i8,
705        i16_unsuffixed => i16,
706        i32_unsuffixed => i32,
707        i64_unsuffixed => i64,
708        i128_unsuffixed => i128,
709        isize_unsuffixed => isize,
710    }
711
712    pub fn f32_unsuffixed(f: f32) -> Literal {
713        let mut s = f.to_string();
714        if !s.contains(".") {
715            s.push_str(".0");
716        }
717        Literal::_new(s)
718    }
719
720    pub fn f64_unsuffixed(f: f64) -> Literal {
721        let mut s = f.to_string();
722        if !s.contains(".") {
723            s.push_str(".0");
724        }
725        Literal::_new(s)
726    }
727
728    pub fn string(t: &str) -> Literal {
729        let mut text = String::with_capacity(t.len() + 2);
730        text.push('"');
731        for c in t.chars() {
732            if c == '\'' {
733                // escape_default turns this into "\'" which is unnecessary.
734                text.push(c);
735            } else {
736                text.extend(c.escape_default());
737            }
738        }
739        text.push('"');
740        Literal::_new(text)
741    }
742
743    pub fn character(t: char) -> Literal {
744        let mut text = String::new();
745        text.push('\'');
746        if t == '"' {
747            // escape_default turns this into '\"' which is unnecessary.
748            text.push(t);
749        } else {
750            text.extend(t.escape_default());
751        }
752        text.push('\'');
753        Literal::_new(text)
754    }
755
756    pub fn byte_string(bytes: &[u8]) -> Literal {
757        let mut escaped = "b\"".to_string();
758        for b in bytes {
759            match *b {
760                b'\0' => escaped.push_str(r"\0"),
761                b'\t' => escaped.push_str(r"\t"),
762                b'\n' => escaped.push_str(r"\n"),
763                b'\r' => escaped.push_str(r"\r"),
764                b'"' => escaped.push_str("\\\""),
765                b'\\' => escaped.push_str("\\\\"),
766                b'\x20'..=b'\x7E' => escaped.push(*b as char),
767                _ => escaped.push_str(&format!("\\x{:02X}", b)),
768            }
769        }
770        escaped.push('"');
771        Literal::_new(escaped)
772    }
773
774    pub fn span(&self) -> Span {
775        self.span
776    }
777
778    pub fn set_span(&mut self, span: Span) {
779        self.span = span;
780    }
781
782    pub fn subspan<R: RangeBounds<usize>>(&self, _range: R) -> Option<Span> {
783        None
784    }
785}
786
787impl fmt::Display for Literal {
788    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
789        self.text.fmt(f)
790    }
791}
792
793impl fmt::Debug for Literal {
794    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
795        let mut debug = fmt.debug_struct("Literal");
796        debug.field("lit", &format_args!("{}", self.text));
797        #[cfg(procmacro2_semver_exempt)]
798        debug.field("span", &self.span);
799        debug.finish()
800    }
801}
802
803fn token_stream(mut input: Cursor) -> PResult<TokenStream> {
804    let mut trees = Vec::new();
805    loop {
806        let input_no_ws = skip_whitespace(input);
807        if input_no_ws.rest.len() == 0 {
808            break;
809        }
810        if let Ok((a, tokens)) = doc_comment(input_no_ws) {
811            input = a;
812            trees.extend(tokens);
813            continue;
814        }
815
816        let (a, tt) = match token_tree(input_no_ws) {
817            Ok(p) => p,
818            Err(_) => break,
819        };
820        trees.push(tt);
821        input = a;
822    }
823    Ok((input, TokenStream { inner: trees }))
824}
825
826#[cfg(not(span_locations))]
827fn spanned<'a, T>(
828    input: Cursor<'a>,
829    f: fn(Cursor<'a>) -> PResult<'a, T>,
830) -> PResult<'a, (T, crate::Span)> {
831    let (a, b) = f(skip_whitespace(input))?;
832    Ok((a, ((b, crate::Span::_new_stable(Span::call_site())))))
833}
834
835#[cfg(span_locations)]
836fn spanned<'a, T>(
837    input: Cursor<'a>,
838    f: fn(Cursor<'a>) -> PResult<'a, T>,
839) -> PResult<'a, (T, crate::Span)> {
840    let input = skip_whitespace(input);
841    let lo = input.off;
842    let (a, b) = f(input)?;
843    let hi = a.off;
844    let span = crate::Span::_new_stable(Span { lo, hi });
845    Ok((a, (b, span)))
846}
847
848fn token_tree(input: Cursor) -> PResult<TokenTree> {
849    let (rest, (mut tt, span)) = spanned(input, token_kind)?;
850    tt.set_span(span);
851    Ok((rest, tt))
852}
853
854named!(token_kind -> TokenTree, alt!(
855    map!(group, |g| TokenTree::Group(crate::Group::_new_stable(g)))
856    |
857    map!(literal, |l| TokenTree::Literal(crate::Literal::_new_stable(l))) // must be before symbol
858    |
859    map!(op, TokenTree::Punct)
860    |
861    symbol_leading_ws
862));
863
864named!(group -> Group, alt!(
865    delimited!(
866        punct!("("),
867        token_stream,
868        punct!(")")
869    ) => { |ts| Group::new(Delimiter::Parenthesis, ts) }
870    |
871    delimited!(
872        punct!("["),
873        token_stream,
874        punct!("]")
875    ) => { |ts| Group::new(Delimiter::Bracket, ts) }
876    |
877    delimited!(
878        punct!("{"),
879        token_stream,
880        punct!("}")
881    ) => { |ts| Group::new(Delimiter::Brace, ts) }
882));
883
884fn symbol_leading_ws(input: Cursor) -> PResult<TokenTree> {
885    symbol(skip_whitespace(input))
886}
887
888fn symbol(input: Cursor) -> PResult<TokenTree> {
889    let raw = input.starts_with("r#");
890    let rest = input.advance((raw as usize) << 1);
891
892    let (rest, sym) = symbol_not_raw(rest)?;
893
894    if !raw {
895        let ident = crate::Ident::new(sym, crate::Span::call_site());
896        return Ok((rest, ident.into()));
897    }
898
899    if sym == "_" {
900        return Err(LexError);
901    }
902
903    let ident = crate::Ident::_new_raw(sym, crate::Span::call_site());
904    Ok((rest, ident.into()))
905}
906
907fn symbol_not_raw(input: Cursor) -> PResult<&str> {
908    let mut chars = input.char_indices();
909
910    match chars.next() {
911        Some((_, ch)) if is_ident_start(ch) => {}
912        _ => return Err(LexError),
913    }
914
915    let mut end = input.len();
916    for (i, ch) in chars {
917        if !is_ident_continue(ch) {
918            end = i;
919            break;
920        }
921    }
922
923    Ok((input.advance(end), &input.rest[..end]))
924}
925
926fn literal(input: Cursor) -> PResult<Literal> {
927    let input_no_ws = skip_whitespace(input);
928
929    match literal_nocapture(input_no_ws) {
930        Ok((a, ())) => {
931            let start = input.len() - input_no_ws.len();
932            let len = input_no_ws.len() - a.len();
933            let end = start + len;
934            Ok((a, Literal::_new(input.rest[start..end].to_string())))
935        }
936        Err(LexError) => Err(LexError),
937    }
938}
939
940named!(literal_nocapture -> (), alt!(
941    string
942    |
943    byte_string
944    |
945    byte
946    |
947    character
948    |
949    float
950    |
951    int
952));
953
954named!(string -> (), alt!(
955    quoted_string
956    |
957    preceded!(
958        punct!("r"),
959        raw_string
960    ) => { |_| () }
961));
962
963named!(quoted_string -> (), do_parse!(
964    punct!("\"") >>
965    cooked_string >>
966    tag!("\"") >>
967    option!(symbol_not_raw) >>
968    (())
969));
970
971fn cooked_string(input: Cursor) -> PResult<()> {
972    let mut chars = input.char_indices().peekable();
973    while let Some((byte_offset, ch)) = chars.next() {
974        match ch {
975            '"' => {
976                return Ok((input.advance(byte_offset), ()));
977            }
978            '\r' => {
979                if let Some((_, '\n')) = chars.next() {
980                    // ...
981                } else {
982                    break;
983                }
984            }
985            '\\' => match chars.next() {
986                Some((_, 'x')) => {
987                    if !backslash_x_char(&mut chars) {
988                        break;
989                    }
990                }
991                Some((_, 'n')) | Some((_, 'r')) | Some((_, 't')) | Some((_, '\\'))
992                | Some((_, '\'')) | Some((_, '"')) | Some((_, '0')) => {}
993                Some((_, 'u')) => {
994                    if !backslash_u(&mut chars) {
995                        break;
996                    }
997                }
998                Some((_, '\n')) | Some((_, '\r')) => {
999                    while let Some(&(_, ch)) = chars.peek() {
1000                        if ch.is_whitespace() {
1001                            chars.next();
1002                        } else {
1003                            break;
1004                        }
1005                    }
1006                }
1007                _ => break,
1008            },
1009            _ch => {}
1010        }
1011    }
1012    Err(LexError)
1013}
1014
1015named!(byte_string -> (), alt!(
1016    delimited!(
1017        punct!("b\""),
1018        cooked_byte_string,
1019        tag!("\"")
1020    ) => { |_| () }
1021    |
1022    preceded!(
1023        punct!("br"),
1024        raw_string
1025    ) => { |_| () }
1026));
1027
1028fn cooked_byte_string(mut input: Cursor) -> PResult<()> {
1029    let mut bytes = input.bytes().enumerate();
1030    'outer: while let Some((offset, b)) = bytes.next() {
1031        match b {
1032            b'"' => {
1033                return Ok((input.advance(offset), ()));
1034            }
1035            b'\r' => {
1036                if let Some((_, b'\n')) = bytes.next() {
1037                    // ...
1038                } else {
1039                    break;
1040                }
1041            }
1042            b'\\' => match bytes.next() {
1043                Some((_, b'x')) => {
1044                    if !backslash_x_byte(&mut bytes) {
1045                        break;
1046                    }
1047                }
1048                Some((_, b'n')) | Some((_, b'r')) | Some((_, b't')) | Some((_, b'\\'))
1049                | Some((_, b'0')) | Some((_, b'\'')) | Some((_, b'"')) => {}
1050                Some((newline, b'\n')) | Some((newline, b'\r')) => {
1051                    let rest = input.advance(newline + 1);
1052                    for (offset, ch) in rest.char_indices() {
1053                        if !ch.is_whitespace() {
1054                            input = rest.advance(offset);
1055                            bytes = input.bytes().enumerate();
1056                            continue 'outer;
1057                        }
1058                    }
1059                    break;
1060                }
1061                _ => break,
1062            },
1063            b if b < 0x80 => {}
1064            _ => break,
1065        }
1066    }
1067    Err(LexError)
1068}
1069
1070fn raw_string(input: Cursor) -> PResult<()> {
1071    let mut chars = input.char_indices();
1072    let mut n = 0;
1073    while let Some((byte_offset, ch)) = chars.next() {
1074        match ch {
1075            '"' => {
1076                n = byte_offset;
1077                break;
1078            }
1079            '#' => {}
1080            _ => return Err(LexError),
1081        }
1082    }
1083    for (byte_offset, ch) in chars {
1084        match ch {
1085            '"' if input.advance(byte_offset + 1).starts_with(&input.rest[..n]) => {
1086                let rest = input.advance(byte_offset + 1 + n);
1087                return Ok((rest, ()));
1088            }
1089            '\r' => {}
1090            _ => {}
1091        }
1092    }
1093    Err(LexError)
1094}
1095
1096named!(byte -> (), do_parse!(
1097    punct!("b") >>
1098    tag!("'") >>
1099    cooked_byte >>
1100    tag!("'") >>
1101    (())
1102));
1103
1104fn cooked_byte(input: Cursor) -> PResult<()> {
1105    let mut bytes = input.bytes().enumerate();
1106    let ok = match bytes.next().map(|(_, b)| b) {
1107        Some(b'\\') => match bytes.next().map(|(_, b)| b) {
1108            Some(b'x') => backslash_x_byte(&mut bytes),
1109            Some(b'n') | Some(b'r') | Some(b't') | Some(b'\\') | Some(b'0') | Some(b'\'')
1110            | Some(b'"') => true,
1111            _ => false,
1112        },
1113        b => b.is_some(),
1114    };
1115    if ok {
1116        match bytes.next() {
1117            Some((offset, _)) => {
1118                if input.chars().as_str().is_char_boundary(offset) {
1119                    Ok((input.advance(offset), ()))
1120                } else {
1121                    Err(LexError)
1122                }
1123            }
1124            None => Ok((input.advance(input.len()), ())),
1125        }
1126    } else {
1127        Err(LexError)
1128    }
1129}
1130
1131named!(character -> (), do_parse!(
1132    punct!("'") >>
1133    cooked_char >>
1134    tag!("'") >>
1135    (())
1136));
1137
1138fn cooked_char(input: Cursor) -> PResult<()> {
1139    let mut chars = input.char_indices();
1140    let ok = match chars.next().map(|(_, ch)| ch) {
1141        Some('\\') => match chars.next().map(|(_, ch)| ch) {
1142            Some('x') => backslash_x_char(&mut chars),
1143            Some('u') => backslash_u(&mut chars),
1144            Some('n') | Some('r') | Some('t') | Some('\\') | Some('0') | Some('\'') | Some('"') => {
1145                true
1146            }
1147            _ => false,
1148        },
1149        ch => ch.is_some(),
1150    };
1151    if ok {
1152        match chars.next() {
1153            Some((idx, _)) => Ok((input.advance(idx), ())),
1154            None => Ok((input.advance(input.len()), ())),
1155        }
1156    } else {
1157        Err(LexError)
1158    }
1159}
1160
1161macro_rules! next_ch {
1162    ($chars:ident @ $pat:pat $(| $rest:pat)*) => {
1163        match $chars.next() {
1164            Some((_, ch)) => match ch {
1165                $pat $(| $rest)*  => ch,
1166                _ => return false,
1167            },
1168            None => return false
1169        }
1170    };
1171}
1172
1173fn backslash_x_char<I>(chars: &mut I) -> bool
1174where
1175    I: Iterator<Item = (usize, char)>,
1176{
1177    next_ch!(chars @ '0'..='7');
1178    next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F');
1179    true
1180}
1181
1182fn backslash_x_byte<I>(chars: &mut I) -> bool
1183where
1184    I: Iterator<Item = (usize, u8)>,
1185{
1186    next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F');
1187    next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F');
1188    true
1189}
1190
1191fn backslash_u<I>(chars: &mut I) -> bool
1192where
1193    I: Iterator<Item = (usize, char)>,
1194{
1195    next_ch!(chars @ '{');
1196    next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F');
1197    loop {
1198        let c = next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F' | '_' | '}');
1199        if c == '}' {
1200            return true;
1201        }
1202    }
1203}
1204
1205fn float(input: Cursor) -> PResult<()> {
1206    let (mut rest, ()) = float_digits(input)?;
1207    if let Some(ch) = rest.chars().next() {
1208        if is_ident_start(ch) {
1209            rest = symbol_not_raw(rest)?.0;
1210        }
1211    }
1212    word_break(rest)
1213}
1214
1215fn float_digits(input: Cursor) -> PResult<()> {
1216    let mut chars = input.chars().peekable();
1217    match chars.next() {
1218        Some(ch) if ch >= '0' && ch <= '9' => {}
1219        _ => return Err(LexError),
1220    }
1221
1222    let mut len = 1;
1223    let mut has_dot = false;
1224    let mut has_exp = false;
1225    while let Some(&ch) = chars.peek() {
1226        match ch {
1227            '0'..='9' | '_' => {
1228                chars.next();
1229                len += 1;
1230            }
1231            '.' => {
1232                if has_dot {
1233                    break;
1234                }
1235                chars.next();
1236                if chars
1237                    .peek()
1238                    .map(|&ch| ch == '.' || is_ident_start(ch))
1239                    .unwrap_or(false)
1240                {
1241                    return Err(LexError);
1242                }
1243                len += 1;
1244                has_dot = true;
1245            }
1246            'e' | 'E' => {
1247                chars.next();
1248                len += 1;
1249                has_exp = true;
1250                break;
1251            }
1252            _ => break,
1253        }
1254    }
1255
1256    let rest = input.advance(len);
1257    if !(has_dot || has_exp || rest.starts_with("f32") || rest.starts_with("f64")) {
1258        return Err(LexError);
1259    }
1260
1261    if has_exp {
1262        let mut has_exp_value = false;
1263        while let Some(&ch) = chars.peek() {
1264            match ch {
1265                '+' | '-' => {
1266                    if has_exp_value {
1267                        break;
1268                    }
1269                    chars.next();
1270                    len += 1;
1271                }
1272                '0'..='9' => {
1273                    chars.next();
1274                    len += 1;
1275                    has_exp_value = true;
1276                }
1277                '_' => {
1278                    chars.next();
1279                    len += 1;
1280                }
1281                _ => break,
1282            }
1283        }
1284        if !has_exp_value {
1285            return Err(LexError);
1286        }
1287    }
1288
1289    Ok((input.advance(len), ()))
1290}
1291
1292fn int(input: Cursor) -> PResult<()> {
1293    let (mut rest, ()) = digits(input)?;
1294    if let Some(ch) = rest.chars().next() {
1295        if is_ident_start(ch) {
1296            rest = symbol_not_raw(rest)?.0;
1297        }
1298    }
1299    word_break(rest)
1300}
1301
1302fn digits(mut input: Cursor) -> PResult<()> {
1303    let base = if input.starts_with("0x") {
1304        input = input.advance(2);
1305        16
1306    } else if input.starts_with("0o") {
1307        input = input.advance(2);
1308        8
1309    } else if input.starts_with("0b") {
1310        input = input.advance(2);
1311        2
1312    } else {
1313        10
1314    };
1315
1316    let mut len = 0;
1317    let mut empty = true;
1318    for b in input.bytes() {
1319        let digit = match b {
1320            b'0'..=b'9' => (b - b'0') as u64,
1321            b'a'..=b'f' => 10 + (b - b'a') as u64,
1322            b'A'..=b'F' => 10 + (b - b'A') as u64,
1323            b'_' => {
1324                if empty && base == 10 {
1325                    return Err(LexError);
1326                }
1327                len += 1;
1328                continue;
1329            }
1330            _ => break,
1331        };
1332        if digit >= base {
1333            return Err(LexError);
1334        }
1335        len += 1;
1336        empty = false;
1337    }
1338    if empty {
1339        Err(LexError)
1340    } else {
1341        Ok((input.advance(len), ()))
1342    }
1343}
1344
1345fn op(input: Cursor) -> PResult<Punct> {
1346    let input = skip_whitespace(input);
1347    match op_char(input) {
1348        Ok((rest, '\'')) => {
1349            symbol(rest)?;
1350            Ok((rest, Punct::new('\'', Spacing::Joint)))
1351        }
1352        Ok((rest, ch)) => {
1353            let kind = match op_char(rest) {
1354                Ok(_) => Spacing::Joint,
1355                Err(LexError) => Spacing::Alone,
1356            };
1357            Ok((rest, Punct::new(ch, kind)))
1358        }
1359        Err(LexError) => Err(LexError),
1360    }
1361}
1362
1363fn op_char(input: Cursor) -> PResult<char> {
1364    if input.starts_with("//") || input.starts_with("/*") {
1365        // Do not accept `/` of a comment as an op.
1366        return Err(LexError);
1367    }
1368
1369    let mut chars = input.chars();
1370    let first = match chars.next() {
1371        Some(ch) => ch,
1372        None => {
1373            return Err(LexError);
1374        }
1375    };
1376    let recognized = "~!@#$%^&*-=+|;:,<.>/?'";
1377    if recognized.contains(first) {
1378        Ok((input.advance(first.len_utf8()), first))
1379    } else {
1380        Err(LexError)
1381    }
1382}
1383
1384fn doc_comment(input: Cursor) -> PResult<Vec<TokenTree>> {
1385    let mut trees = Vec::new();
1386    let (rest, ((comment, inner), span)) = spanned(input, doc_comment_contents)?;
1387    trees.push(TokenTree::Punct(Punct::new('#', Spacing::Alone)));
1388    if inner {
1389        trees.push(Punct::new('!', Spacing::Alone).into());
1390    }
1391    let mut stream = vec![
1392        TokenTree::Ident(crate::Ident::new("doc", span)),
1393        TokenTree::Punct(Punct::new('=', Spacing::Alone)),
1394        TokenTree::Literal(crate::Literal::string(comment)),
1395    ];
1396    for tt in stream.iter_mut() {
1397        tt.set_span(span);
1398    }
1399    let group = Group::new(Delimiter::Bracket, stream.into_iter().collect());
1400    trees.push(crate::Group::_new_stable(group).into());
1401    for tt in trees.iter_mut() {
1402        tt.set_span(span);
1403    }
1404    Ok((rest, trees))
1405}
1406
1407named!(doc_comment_contents -> (&str, bool), alt!(
1408    do_parse!(
1409        punct!("//!") >>
1410        s: take_until_newline_or_eof!() >>
1411        ((s, true))
1412    )
1413    |
1414    do_parse!(
1415        option!(whitespace) >>
1416        peek!(tag!("/*!")) >>
1417        s: block_comment >>
1418        ((s, true))
1419    )
1420    |
1421    do_parse!(
1422        punct!("///") >>
1423        not!(tag!("/")) >>
1424        s: take_until_newline_or_eof!() >>
1425        ((s, false))
1426    )
1427    |
1428    do_parse!(
1429        option!(whitespace) >>
1430        peek!(tuple!(tag!("/**"), not!(tag!("*")))) >>
1431        s: block_comment >>
1432        ((s, false))
1433    )
1434));