syn/
token.rs

1//! Tokens representing Rust punctuation, keywords, and delimiters.
2//!
3//! The type names in this module can be difficult to keep straight, so we
4//! prefer to use the [`Token!`] macro instead. This is a type-macro that
5//! expands to the token type of the given token.
6//!
7//! [`Token!`]: ../macro.Token.html
8//!
9//! # Example
10//!
11//! The [`ItemStatic`] syntax tree node is defined like this.
12//!
13//! [`ItemStatic`]: ../struct.ItemStatic.html
14//!
15//! ```
16//! # use syn::{Attribute, Expr, Ident, Token, Type, Visibility};
17//! #
18//! pub struct ItemStatic {
19//!     pub attrs: Vec<Attribute>,
20//!     pub vis: Visibility,
21//!     pub static_token: Token![static],
22//!     pub mutability: Option<Token![mut]>,
23//!     pub ident: Ident,
24//!     pub colon_token: Token![:],
25//!     pub ty: Box<Type>,
26//!     pub eq_token: Token![=],
27//!     pub expr: Box<Expr>,
28//!     pub semi_token: Token![;],
29//! }
30//! ```
31//!
32//! # Parsing
33//!
34//! Keywords and punctuation can be parsed through the [`ParseStream::parse`]
35//! method. Delimiter tokens are parsed using the [`parenthesized!`],
36//! [`bracketed!`] and [`braced!`] macros.
37//!
38//! [`ParseStream::parse`]: ../parse/struct.ParseBuffer.html#method.parse
39//! [`parenthesized!`]: ../macro.parenthesized.html
40//! [`bracketed!`]: ../macro.bracketed.html
41//! [`braced!`]: ../macro.braced.html
42//!
43//! ```
44//! use syn::{Attribute, Result};
45//! use syn::parse::{Parse, ParseStream};
46//! #
47//! # enum ItemStatic {}
48//!
49//! // Parse the ItemStatic struct shown above.
50//! impl Parse for ItemStatic {
51//!     fn parse(input: ParseStream) -> Result<Self> {
52//!         # use syn::ItemStatic;
53//!         # fn parse(input: ParseStream) -> Result<ItemStatic> {
54//!         Ok(ItemStatic {
55//!             attrs: input.call(Attribute::parse_outer)?,
56//!             vis: input.parse()?,
57//!             static_token: input.parse()?,
58//!             mutability: input.parse()?,
59//!             ident: input.parse()?,
60//!             colon_token: input.parse()?,
61//!             ty: input.parse()?,
62//!             eq_token: input.parse()?,
63//!             expr: input.parse()?,
64//!             semi_token: input.parse()?,
65//!         })
66//!         # }
67//!         # unimplemented!()
68//!     }
69//! }
70//! ```
71//!
72//! # Other operations
73//!
74//! Every keyword and punctuation token supports the following operations.
75//!
76//! - [Peeking] — `input.peek(Token![...])`
77//!
78//! - [Parsing] — `input.parse::<Token![...]>()?`
79//!
80//! - [Printing] — `quote!( ... #the_token ... )`
81//!
82//! - Construction from a [`Span`] — `let the_token = Token![...](sp)`
83//!
84//! - Field access to its span — `let sp = the_token.span`
85//!
86//! [Peeking]: ../parse/struct.ParseBuffer.html#method.peek
87//! [Parsing]: ../parse/struct.ParseBuffer.html#method.parse
88//! [Printing]: https://docs.rs/quote/1.0/quote/trait.ToTokens.html
89//! [`Span`]: https://docs.rs/proc-macro2/1.0/proc_macro2/struct.Span.html
90
91use std;
92#[cfg(feature = "extra-traits")]
93use std::cmp;
94#[cfg(feature = "extra-traits")]
95use std::fmt::{self, Debug};
96#[cfg(feature = "extra-traits")]
97use std::hash::{Hash, Hasher};
98use std::ops::{Deref, DerefMut};
99
100#[cfg(feature = "parsing")]
101use proc_macro2::Delimiter;
102#[cfg(any(feature = "parsing", feature = "printing"))]
103use proc_macro2::Ident;
104use proc_macro2::Span;
105#[cfg(feature = "printing")]
106use proc_macro2::TokenStream;
107#[cfg(feature = "printing")]
108use quote::{ToTokens, TokenStreamExt};
109
110use self::private::WithSpan;
111#[cfg(feature = "parsing")]
112use crate::buffer::Cursor;
113#[cfg(feature = "parsing")]
114use crate::error::Result;
115#[cfg(any(feature = "full", feature = "derive"))]
116#[cfg(feature = "parsing")]
117use crate::lifetime::Lifetime;
118#[cfg(any(feature = "full", feature = "derive"))]
119#[cfg(feature = "parsing")]
120use crate::lit::{Lit, LitBool, LitByte, LitByteStr, LitChar, LitFloat, LitInt, LitStr};
121#[cfg(feature = "parsing")]
122use crate::lookahead;
123#[cfg(feature = "parsing")]
124use crate::parse::{Parse, ParseStream};
125use crate::span::IntoSpans;
126
127/// Marker trait for types that represent single tokens.
128///
129/// This trait is sealed and cannot be implemented for types outside of Syn.
130#[cfg(feature = "parsing")]
131pub trait Token: private::Sealed {
132    // Not public API.
133    #[doc(hidden)]
134    fn peek(cursor: Cursor) -> bool;
135
136    // Not public API.
137    #[doc(hidden)]
138    fn display() -> &'static str;
139}
140
141mod private {
142    use proc_macro2::Span;
143
144    #[cfg(feature = "parsing")]
145    pub trait Sealed {}
146
147    /// Support writing `token.span` rather than `token.spans[0]` on tokens that
148    /// hold a single span.
149    #[repr(C)]
150    pub struct WithSpan {
151        pub span: Span,
152    }
153}
154
155#[cfg(feature = "parsing")]
156impl private::Sealed for Ident {}
157
158#[cfg(any(feature = "full", feature = "derive"))]
159#[cfg(feature = "parsing")]
160fn peek_impl(cursor: Cursor, peek: fn(ParseStream) -> bool) -> bool {
161    use crate::parse::Unexpected;
162    use std::cell::Cell;
163    use std::rc::Rc;
164
165    let scope = Span::call_site();
166    let unexpected = Rc::new(Cell::new(Unexpected::None));
167    let buffer = crate::parse::new_parse_buffer(scope, cursor, unexpected);
168    peek(&buffer)
169}
170
171#[cfg(any(feature = "full", feature = "derive"))]
172macro_rules! impl_token {
173    ($name:ident $display:expr) => {
174        #[cfg(feature = "parsing")]
175        impl Token for $name {
176            fn peek(cursor: Cursor) -> bool {
177                fn peek(input: ParseStream) -> bool {
178                    <$name as Parse>::parse(input).is_ok()
179                }
180                peek_impl(cursor, peek)
181            }
182
183            fn display() -> &'static str {
184                $display
185            }
186        }
187
188        #[cfg(feature = "parsing")]
189        impl private::Sealed for $name {}
190    };
191}
192
193#[cfg(any(feature = "full", feature = "derive"))]
194impl_token!(Lifetime "lifetime");
195#[cfg(any(feature = "full", feature = "derive"))]
196impl_token!(Lit "literal");
197#[cfg(any(feature = "full", feature = "derive"))]
198impl_token!(LitStr "string literal");
199#[cfg(any(feature = "full", feature = "derive"))]
200impl_token!(LitByteStr "byte string literal");
201#[cfg(any(feature = "full", feature = "derive"))]
202impl_token!(LitByte "byte literal");
203#[cfg(any(feature = "full", feature = "derive"))]
204impl_token!(LitChar "character literal");
205#[cfg(any(feature = "full", feature = "derive"))]
206impl_token!(LitInt "integer literal");
207#[cfg(any(feature = "full", feature = "derive"))]
208impl_token!(LitFloat "floating point literal");
209#[cfg(any(feature = "full", feature = "derive"))]
210impl_token!(LitBool "boolean literal");
211
212// Not public API.
213#[doc(hidden)]
214#[cfg(feature = "parsing")]
215pub trait CustomToken {
216    fn peek(cursor: Cursor) -> bool;
217    fn display() -> &'static str;
218}
219
220#[cfg(feature = "parsing")]
221impl<T: CustomToken> private::Sealed for T {}
222
223#[cfg(feature = "parsing")]
224impl<T: CustomToken> Token for T {
225    fn peek(cursor: Cursor) -> bool {
226        <Self as CustomToken>::peek(cursor)
227    }
228
229    fn display() -> &'static str {
230        <Self as CustomToken>::display()
231    }
232}
233
234macro_rules! define_keywords {
235    ($($token:tt pub struct $name:ident #[$doc:meta])*) => {
236        $(
237            #[cfg_attr(feature = "clone-impls", derive(Copy, Clone))]
238            #[$doc]
239            ///
240            /// Don't try to remember the name of this type &mdash; use the
241            /// [`Token!`] macro instead.
242            ///
243            /// [`Token!`]: crate::token
244            pub struct $name {
245                pub span: Span,
246            }
247
248            #[doc(hidden)]
249            #[allow(non_snake_case)]
250            pub fn $name<S: IntoSpans<[Span; 1]>>(span: S) -> $name {
251                $name {
252                    span: span.into_spans()[0],
253                }
254            }
255
256            impl std::default::Default for $name {
257                fn default() -> Self {
258                    $name {
259                        span: Span::call_site(),
260                    }
261                }
262            }
263
264            #[cfg(feature = "extra-traits")]
265            impl Debug for $name {
266                fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
267                    f.write_str(stringify!($name))
268                }
269            }
270
271            #[cfg(feature = "extra-traits")]
272            impl cmp::Eq for $name {}
273
274            #[cfg(feature = "extra-traits")]
275            impl PartialEq for $name {
276                fn eq(&self, _other: &$name) -> bool {
277                    true
278                }
279            }
280
281            #[cfg(feature = "extra-traits")]
282            impl Hash for $name {
283                fn hash<H: Hasher>(&self, _state: &mut H) {}
284            }
285
286            #[cfg(feature = "printing")]
287            impl ToTokens for $name {
288                fn to_tokens(&self, tokens: &mut TokenStream) {
289                    printing::keyword($token, self.span, tokens);
290                }
291            }
292
293            #[cfg(feature = "parsing")]
294            impl Parse for $name {
295                fn parse(input: ParseStream) -> Result<Self> {
296                    Ok($name {
297                        span: parsing::keyword(input, $token)?,
298                    })
299                }
300            }
301
302            #[cfg(feature = "parsing")]
303            impl Token for $name {
304                fn peek(cursor: Cursor) -> bool {
305                    parsing::peek_keyword(cursor, $token)
306                }
307
308                fn display() -> &'static str {
309                    concat!("`", $token, "`")
310                }
311            }
312
313            #[cfg(feature = "parsing")]
314            impl private::Sealed for $name {}
315        )*
316    };
317}
318
319macro_rules! impl_deref_if_len_is_1 {
320    ($name:ident/1) => {
321        impl Deref for $name {
322            type Target = WithSpan;
323
324            fn deref(&self) -> &Self::Target {
325                unsafe { &*(self as *const Self as *const WithSpan) }
326            }
327        }
328
329        impl DerefMut for $name {
330            fn deref_mut(&mut self) -> &mut Self::Target {
331                unsafe { &mut *(self as *mut Self as *mut WithSpan) }
332            }
333        }
334    };
335
336    ($name:ident/$len:tt) => {};
337}
338
339macro_rules! define_punctuation_structs {
340    ($($token:tt pub struct $name:ident/$len:tt #[$doc:meta])*) => {
341        $(
342            #[cfg_attr(feature = "clone-impls", derive(Copy, Clone))]
343            #[repr(C)]
344            #[$doc]
345            ///
346            /// Don't try to remember the name of this type &mdash; use the
347            /// [`Token!`] macro instead.
348            ///
349            /// [`Token!`]: crate::token
350            pub struct $name {
351                pub spans: [Span; $len],
352            }
353
354            #[doc(hidden)]
355            #[allow(non_snake_case)]
356            pub fn $name<S: IntoSpans<[Span; $len]>>(spans: S) -> $name {
357                $name {
358                    spans: spans.into_spans(),
359                }
360            }
361
362            impl std::default::Default for $name {
363                fn default() -> Self {
364                    $name {
365                        spans: [Span::call_site(); $len],
366                    }
367                }
368            }
369
370            #[cfg(feature = "extra-traits")]
371            impl Debug for $name {
372                fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
373                    f.write_str(stringify!($name))
374                }
375            }
376
377            #[cfg(feature = "extra-traits")]
378            impl cmp::Eq for $name {}
379
380            #[cfg(feature = "extra-traits")]
381            impl PartialEq for $name {
382                fn eq(&self, _other: &$name) -> bool {
383                    true
384                }
385            }
386
387            #[cfg(feature = "extra-traits")]
388            impl Hash for $name {
389                fn hash<H: Hasher>(&self, _state: &mut H) {}
390            }
391
392            impl_deref_if_len_is_1!($name/$len);
393        )*
394    };
395}
396
397macro_rules! define_punctuation {
398    ($($token:tt pub struct $name:ident/$len:tt #[$doc:meta])*) => {
399        $(
400            define_punctuation_structs! {
401                $token pub struct $name/$len #[$doc]
402            }
403
404            #[cfg(feature = "printing")]
405            impl ToTokens for $name {
406                fn to_tokens(&self, tokens: &mut TokenStream) {
407                    printing::punct($token, &self.spans, tokens);
408                }
409            }
410
411            #[cfg(feature = "parsing")]
412            impl Parse for $name {
413                fn parse(input: ParseStream) -> Result<Self> {
414                    Ok($name {
415                        spans: parsing::punct(input, $token)?,
416                    })
417                }
418            }
419
420            #[cfg(feature = "parsing")]
421            impl Token for $name {
422                fn peek(cursor: Cursor) -> bool {
423                    parsing::peek_punct(cursor, $token)
424                }
425
426                fn display() -> &'static str {
427                    concat!("`", $token, "`")
428                }
429            }
430
431            #[cfg(feature = "parsing")]
432            impl private::Sealed for $name {}
433        )*
434    };
435}
436
437macro_rules! define_delimiters {
438    ($($token:tt pub struct $name:ident #[$doc:meta])*) => {
439        $(
440            #[cfg_attr(feature = "clone-impls", derive(Copy, Clone))]
441            #[$doc]
442            pub struct $name {
443                pub span: Span,
444            }
445
446            #[doc(hidden)]
447            #[allow(non_snake_case)]
448            pub fn $name<S: IntoSpans<[Span; 1]>>(span: S) -> $name {
449                $name {
450                    span: span.into_spans()[0],
451                }
452            }
453
454            impl std::default::Default for $name {
455                fn default() -> Self {
456                    $name {
457                        span: Span::call_site(),
458                    }
459                }
460            }
461
462            #[cfg(feature = "extra-traits")]
463            impl Debug for $name {
464                fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
465                    f.write_str(stringify!($name))
466                }
467            }
468
469            #[cfg(feature = "extra-traits")]
470            impl cmp::Eq for $name {}
471
472            #[cfg(feature = "extra-traits")]
473            impl PartialEq for $name {
474                fn eq(&self, _other: &$name) -> bool {
475                    true
476                }
477            }
478
479            #[cfg(feature = "extra-traits")]
480            impl Hash for $name {
481                fn hash<H: Hasher>(&self, _state: &mut H) {}
482            }
483
484            impl $name {
485                #[cfg(feature = "printing")]
486                pub fn surround<F>(&self, tokens: &mut TokenStream, f: F)
487                where
488                    F: FnOnce(&mut TokenStream),
489                {
490                    printing::delim($token, self.span, tokens, f);
491                }
492            }
493
494            #[cfg(feature = "parsing")]
495            impl private::Sealed for $name {}
496        )*
497    };
498}
499
500define_punctuation_structs! {
501    "_" pub struct Underscore/1 /// `_`
502}
503
504#[cfg(feature = "printing")]
505impl ToTokens for Underscore {
506    fn to_tokens(&self, tokens: &mut TokenStream) {
507        tokens.append(Ident::new("_", self.span));
508    }
509}
510
511#[cfg(feature = "parsing")]
512impl Parse for Underscore {
513    fn parse(input: ParseStream) -> Result<Self> {
514        input.step(|cursor| {
515            if let Some((ident, rest)) = cursor.ident() {
516                if ident == "_" {
517                    return Ok((Underscore(ident.span()), rest));
518                }
519            }
520            if let Some((punct, rest)) = cursor.punct() {
521                if punct.as_char() == '_' {
522                    return Ok((Underscore(punct.span()), rest));
523                }
524            }
525            Err(cursor.error("expected `_`"))
526        })
527    }
528}
529
530#[cfg(feature = "parsing")]
531impl Token for Underscore {
532    fn peek(cursor: Cursor) -> bool {
533        if let Some((ident, _rest)) = cursor.ident() {
534            return ident == "_";
535        }
536        if let Some((punct, _rest)) = cursor.punct() {
537            return punct.as_char() == '_';
538        }
539        false
540    }
541
542    fn display() -> &'static str {
543        "`_`"
544    }
545}
546
547#[cfg(feature = "parsing")]
548impl private::Sealed for Underscore {}
549
550#[cfg(feature = "parsing")]
551impl Token for Paren {
552    fn peek(cursor: Cursor) -> bool {
553        lookahead::is_delimiter(cursor, Delimiter::Parenthesis)
554    }
555
556    fn display() -> &'static str {
557        "parentheses"
558    }
559}
560
561#[cfg(feature = "parsing")]
562impl Token for Brace {
563    fn peek(cursor: Cursor) -> bool {
564        lookahead::is_delimiter(cursor, Delimiter::Brace)
565    }
566
567    fn display() -> &'static str {
568        "curly braces"
569    }
570}
571
572#[cfg(feature = "parsing")]
573impl Token for Bracket {
574    fn peek(cursor: Cursor) -> bool {
575        lookahead::is_delimiter(cursor, Delimiter::Bracket)
576    }
577
578    fn display() -> &'static str {
579        "square brackets"
580    }
581}
582
583#[cfg(feature = "parsing")]
584impl Token for Group {
585    fn peek(cursor: Cursor) -> bool {
586        lookahead::is_delimiter(cursor, Delimiter::None)
587    }
588
589    fn display() -> &'static str {
590        "invisible group"
591    }
592}
593
594define_keywords! {
595    "abstract"    pub struct Abstract     /// `abstract`
596    "as"          pub struct As           /// `as`
597    "async"       pub struct Async        /// `async`
598    "auto"        pub struct Auto         /// `auto`
599    "await"       pub struct Await        /// `await`
600    "become"      pub struct Become       /// `become`
601    "box"         pub struct Box          /// `box`
602    "break"       pub struct Break        /// `break`
603    "const"       pub struct Const        /// `const`
604    "continue"    pub struct Continue     /// `continue`
605    "crate"       pub struct Crate        /// `crate`
606    "default"     pub struct Default      /// `default`
607    "do"          pub struct Do           /// `do`
608    "dyn"         pub struct Dyn          /// `dyn`
609    "else"        pub struct Else         /// `else`
610    "enum"        pub struct Enum         /// `enum`
611    "extern"      pub struct Extern       /// `extern`
612    "final"       pub struct Final        /// `final`
613    "fn"          pub struct Fn           /// `fn`
614    "for"         pub struct For          /// `for`
615    "if"          pub struct If           /// `if`
616    "impl"        pub struct Impl         /// `impl`
617    "in"          pub struct In           /// `in`
618    "let"         pub struct Let          /// `let`
619    "loop"        pub struct Loop         /// `loop`
620    "macro"       pub struct Macro        /// `macro`
621    "match"       pub struct Match        /// `match`
622    "mod"         pub struct Mod          /// `mod`
623    "move"        pub struct Move         /// `move`
624    "mut"         pub struct Mut          /// `mut`
625    "override"    pub struct Override     /// `override`
626    "priv"        pub struct Priv         /// `priv`
627    "pub"         pub struct Pub          /// `pub`
628    "ref"         pub struct Ref          /// `ref`
629    "return"      pub struct Return       /// `return`
630    "Self"        pub struct SelfType     /// `Self`
631    "self"        pub struct SelfValue    /// `self`
632    "static"      pub struct Static       /// `static`
633    "struct"      pub struct Struct       /// `struct`
634    "super"       pub struct Super        /// `super`
635    "trait"       pub struct Trait        /// `trait`
636    "try"         pub struct Try          /// `try`
637    "type"        pub struct Type         /// `type`
638    "typeof"      pub struct Typeof       /// `typeof`
639    "union"       pub struct Union        /// `union`
640    "unsafe"      pub struct Unsafe       /// `unsafe`
641    "unsized"     pub struct Unsized      /// `unsized`
642    "use"         pub struct Use          /// `use`
643    "virtual"     pub struct Virtual      /// `virtual`
644    "where"       pub struct Where        /// `where`
645    "while"       pub struct While        /// `while`
646    "yield"       pub struct Yield        /// `yield`
647}
648
649define_punctuation! {
650    "+"           pub struct Add/1        /// `+`
651    "+="          pub struct AddEq/2      /// `+=`
652    "&"           pub struct And/1        /// `&`
653    "&&"          pub struct AndAnd/2     /// `&&`
654    "&="          pub struct AndEq/2      /// `&=`
655    "@"           pub struct At/1         /// `@`
656    "!"           pub struct Bang/1       /// `!`
657    "^"           pub struct Caret/1      /// `^`
658    "^="          pub struct CaretEq/2    /// `^=`
659    ":"           pub struct Colon/1      /// `:`
660    "::"          pub struct Colon2/2     /// `::`
661    ","           pub struct Comma/1      /// `,`
662    "/"           pub struct Div/1        /// `/`
663    "/="          pub struct DivEq/2      /// `/=`
664    "$"           pub struct Dollar/1     /// `$`
665    "."           pub struct Dot/1        /// `.`
666    ".."          pub struct Dot2/2       /// `..`
667    "..."         pub struct Dot3/3       /// `...`
668    "..="         pub struct DotDotEq/3   /// `..=`
669    "="           pub struct Eq/1         /// `=`
670    "=="          pub struct EqEq/2       /// `==`
671    ">="          pub struct Ge/2         /// `>=`
672    ">"           pub struct Gt/1         /// `>`
673    "<="          pub struct Le/2         /// `<=`
674    "<"           pub struct Lt/1         /// `<`
675    "*="          pub struct MulEq/2      /// `*=`
676    "!="          pub struct Ne/2         /// `!=`
677    "|"           pub struct Or/1         /// `|`
678    "|="          pub struct OrEq/2       /// `|=`
679    "||"          pub struct OrOr/2       /// `||`
680    "#"           pub struct Pound/1      /// `#`
681    "?"           pub struct Question/1   /// `?`
682    "->"          pub struct RArrow/2     /// `->`
683    "<-"          pub struct LArrow/2     /// `<-`
684    "%"           pub struct Rem/1        /// `%`
685    "%="          pub struct RemEq/2      /// `%=`
686    "=>"          pub struct FatArrow/2   /// `=>`
687    ";"           pub struct Semi/1       /// `;`
688    "<<"          pub struct Shl/2        /// `<<`
689    "<<="         pub struct ShlEq/3      /// `<<=`
690    ">>"          pub struct Shr/2        /// `>>`
691    ">>="         pub struct ShrEq/3      /// `>>=`
692    "*"           pub struct Star/1       /// `*`
693    "-"           pub struct Sub/1        /// `-`
694    "-="          pub struct SubEq/2      /// `-=`
695    "~"           pub struct Tilde/1      /// `~`
696}
697
698define_delimiters! {
699    "{"           pub struct Brace        /// `{...}`
700    "["           pub struct Bracket      /// `[...]`
701    "("           pub struct Paren        /// `(...)`
702    " "           pub struct Group        /// None-delimited group
703}
704
705macro_rules! export_token_macro {
706    ($($await_rule:tt)*) => {
707        /// A type-macro that expands to the name of the Rust type representation of a
708        /// given token.
709        ///
710        /// See the [token module] documentation for details and examples.
711        ///
712        /// [token module]: crate::token
713        // Unfortunate duplication due to a rustdoc bug.
714        // https://github.com/rust-lang/rust/issues/45939
715        #[macro_export]
716        macro_rules! Token {
717            (abstract)    => { $crate::token::Abstract };
718            (as)          => { $crate::token::As };
719            (async)       => { $crate::token::Async };
720            (auto)        => { $crate::token::Auto };
721            $($await_rule => { $crate::token::Await };)*
722            (become)      => { $crate::token::Become };
723            (box)         => { $crate::token::Box };
724            (break)       => { $crate::token::Break };
725            (const)       => { $crate::token::Const };
726            (continue)    => { $crate::token::Continue };
727            (crate)       => { $crate::token::Crate };
728            (default)     => { $crate::token::Default };
729            (do)          => { $crate::token::Do };
730            (dyn)         => { $crate::token::Dyn };
731            (else)        => { $crate::token::Else };
732            (enum)        => { $crate::token::Enum };
733            (extern)      => { $crate::token::Extern };
734            (final)       => { $crate::token::Final };
735            (fn)          => { $crate::token::Fn };
736            (for)         => { $crate::token::For };
737            (if)          => { $crate::token::If };
738            (impl)        => { $crate::token::Impl };
739            (in)          => { $crate::token::In };
740            (let)         => { $crate::token::Let };
741            (loop)        => { $crate::token::Loop };
742            (macro)       => { $crate::token::Macro };
743            (match)       => { $crate::token::Match };
744            (mod)         => { $crate::token::Mod };
745            (move)        => { $crate::token::Move };
746            (mut)         => { $crate::token::Mut };
747            (override)    => { $crate::token::Override };
748            (priv)        => { $crate::token::Priv };
749            (pub)         => { $crate::token::Pub };
750            (ref)         => { $crate::token::Ref };
751            (return)      => { $crate::token::Return };
752            (Self)        => { $crate::token::SelfType };
753            (self)        => { $crate::token::SelfValue };
754            (static)      => { $crate::token::Static };
755            (struct)      => { $crate::token::Struct };
756            (super)       => { $crate::token::Super };
757            (trait)       => { $crate::token::Trait };
758            (try)         => { $crate::token::Try };
759            (type)        => { $crate::token::Type };
760            (typeof)      => { $crate::token::Typeof };
761            (union)       => { $crate::token::Union };
762            (unsafe)      => { $crate::token::Unsafe };
763            (unsized)     => { $crate::token::Unsized };
764            (use)         => { $crate::token::Use };
765            (virtual)     => { $crate::token::Virtual };
766            (where)       => { $crate::token::Where };
767            (while)       => { $crate::token::While };
768            (yield)       => { $crate::token::Yield };
769            (+)           => { $crate::token::Add };
770            (+=)          => { $crate::token::AddEq };
771            (&)           => { $crate::token::And };
772            (&&)          => { $crate::token::AndAnd };
773            (&=)          => { $crate::token::AndEq };
774            (@)           => { $crate::token::At };
775            (!)           => { $crate::token::Bang };
776            (^)           => { $crate::token::Caret };
777            (^=)          => { $crate::token::CaretEq };
778            (:)           => { $crate::token::Colon };
779            (::)          => { $crate::token::Colon2 };
780            (,)           => { $crate::token::Comma };
781            (/)           => { $crate::token::Div };
782            (/=)          => { $crate::token::DivEq };
783            ($)           => { $crate::token::Dollar };
784            (.)           => { $crate::token::Dot };
785            (..)          => { $crate::token::Dot2 };
786            (...)         => { $crate::token::Dot3 };
787            (..=)         => { $crate::token::DotDotEq };
788            (=)           => { $crate::token::Eq };
789            (==)          => { $crate::token::EqEq };
790            (>=)          => { $crate::token::Ge };
791            (>)           => { $crate::token::Gt };
792            (<=)          => { $crate::token::Le };
793            (<)           => { $crate::token::Lt };
794            (*=)          => { $crate::token::MulEq };
795            (!=)          => { $crate::token::Ne };
796            (|)           => { $crate::token::Or };
797            (|=)          => { $crate::token::OrEq };
798            (||)          => { $crate::token::OrOr };
799            (#)           => { $crate::token::Pound };
800            (?)           => { $crate::token::Question };
801            (->)          => { $crate::token::RArrow };
802            (<-)          => { $crate::token::LArrow };
803            (%)           => { $crate::token::Rem };
804            (%=)          => { $crate::token::RemEq };
805            (=>)          => { $crate::token::FatArrow };
806            (;)           => { $crate::token::Semi };
807            (<<)          => { $crate::token::Shl };
808            (<<=)         => { $crate::token::ShlEq };
809            (>>)          => { $crate::token::Shr };
810            (>>=)         => { $crate::token::ShrEq };
811            (*)           => { $crate::token::Star };
812            (-)           => { $crate::token::Sub };
813            (-=)          => { $crate::token::SubEq };
814            (~)           => { $crate::token::Tilde };
815            (_)           => { $crate::token::Underscore };
816        }
817    };
818}
819
820// Old rustc does not permit `await` appearing anywhere in the source file.
821// https://github.com/rust-lang/rust/issues/57919
822// We put the Token![await] rule in a place that is not lexed by old rustc.
823#[cfg(not(syn_omit_await_from_token_macro))]
824include!("await.rs"); // export_token_macro![(await)];
825#[cfg(syn_omit_await_from_token_macro)]
826export_token_macro![];
827
828// Not public API.
829#[doc(hidden)]
830#[cfg(feature = "parsing")]
831pub mod parsing {
832    use proc_macro2::{Spacing, Span};
833
834    use crate::buffer::Cursor;
835    use crate::error::{Error, Result};
836    use crate::parse::ParseStream;
837    use crate::span::FromSpans;
838
839    pub fn keyword(input: ParseStream, token: &str) -> Result<Span> {
840        input.step(|cursor| {
841            if let Some((ident, rest)) = cursor.ident() {
842                if ident == token {
843                    return Ok((ident.span(), rest));
844                }
845            }
846            Err(cursor.error(format!("expected `{}`", token)))
847        })
848    }
849
850    pub fn peek_keyword(cursor: Cursor, token: &str) -> bool {
851        if let Some((ident, _rest)) = cursor.ident() {
852            ident == token
853        } else {
854            false
855        }
856    }
857
858    pub fn punct<S: FromSpans>(input: ParseStream, token: &str) -> Result<S> {
859        let mut spans = [input.cursor().span(); 3];
860        punct_helper(input, token, &mut spans)?;
861        Ok(S::from_spans(&spans))
862    }
863
864    fn punct_helper(input: ParseStream, token: &str, spans: &mut [Span; 3]) -> Result<()> {
865        input.step(|cursor| {
866            let mut cursor = *cursor;
867            assert!(token.len() <= spans.len());
868
869            for (i, ch) in token.chars().enumerate() {
870                match cursor.punct() {
871                    Some((punct, rest)) => {
872                        spans[i] = punct.span();
873                        if punct.as_char() != ch {
874                            break;
875                        } else if i == token.len() - 1 {
876                            return Ok(((), rest));
877                        } else if punct.spacing() != Spacing::Joint {
878                            break;
879                        }
880                        cursor = rest;
881                    }
882                    None => break,
883                }
884            }
885
886            Err(Error::new(spans[0], format!("expected `{}`", token)))
887        })
888    }
889
890    pub fn peek_punct(mut cursor: Cursor, token: &str) -> bool {
891        for (i, ch) in token.chars().enumerate() {
892            match cursor.punct() {
893                Some((punct, rest)) => {
894                    if punct.as_char() != ch {
895                        break;
896                    } else if i == token.len() - 1 {
897                        return true;
898                    } else if punct.spacing() != Spacing::Joint {
899                        break;
900                    }
901                    cursor = rest;
902                }
903                None => break,
904            }
905        }
906        false
907    }
908}
909
910// Not public API.
911#[doc(hidden)]
912#[cfg(feature = "printing")]
913pub mod printing {
914    use proc_macro2::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream};
915    use quote::TokenStreamExt;
916
917    pub fn punct(s: &str, spans: &[Span], tokens: &mut TokenStream) {
918        assert_eq!(s.len(), spans.len());
919
920        let mut chars = s.chars();
921        let mut spans = spans.iter();
922        let ch = chars.next_back().unwrap();
923        let span = spans.next_back().unwrap();
924        for (ch, span) in chars.zip(spans) {
925            let mut op = Punct::new(ch, Spacing::Joint);
926            op.set_span(*span);
927            tokens.append(op);
928        }
929
930        let mut op = Punct::new(ch, Spacing::Alone);
931        op.set_span(*span);
932        tokens.append(op);
933    }
934
935    pub fn keyword(s: &str, span: Span, tokens: &mut TokenStream) {
936        tokens.append(Ident::new(s, span));
937    }
938
939    pub fn delim<F>(s: &str, span: Span, tokens: &mut TokenStream, f: F)
940    where
941        F: FnOnce(&mut TokenStream),
942    {
943        let delim = match s {
944            "(" => Delimiter::Parenthesis,
945            "[" => Delimiter::Bracket,
946            "{" => Delimiter::Brace,
947            " " => Delimiter::None,
948            _ => panic!("unknown delimiter: {}", s),
949        };
950        let mut inner = TokenStream::new();
951        f(&mut inner);
952        let mut g = Group::new(delim, inner);
953        g.set_span(span);
954        tokens.append(g);
955    }
956}