syn/parse.rs
1//! Parsing interface for parsing a token stream into a syntax tree node.
2//!
3//! Parsing in Syn is built on parser functions that take in a [`ParseStream`]
4//! and produce a [`Result<T>`] where `T` is some syntax tree node. Underlying
5//! these parser functions is a lower level mechanism built around the
6//! [`Cursor`] type. `Cursor` is a cheaply copyable cursor over a range of
7//! tokens in a token stream.
8//!
9//! [`ParseStream`]: type.ParseStream.html
10//! [`Result<T>`]: type.Result.html
11//! [`Cursor`]: ../buffer/index.html
12//!
13//! # Example
14//!
15//! Here is a snippet of parsing code to get a feel for the style of the
16//! library. We define data structures for a subset of Rust syntax including
17//! enums (not shown) and structs, then provide implementations of the [`Parse`]
18//! trait to parse these syntax tree data structures from a token stream.
19//!
20//! Once `Parse` impls have been defined, they can be called conveniently from a
21//! procedural macro through [`parse_macro_input!`] as shown at the bottom of
22//! the snippet. If the caller provides syntactically invalid input to the
23//! procedural macro, they will receive a helpful compiler error message
24//! pointing out the exact token that triggered the failure to parse.
25//!
26//! [`parse_macro_input!`]: ../macro.parse_macro_input.html
27//!
28//! ```
29//! extern crate proc_macro;
30//!
31//! use proc_macro::TokenStream;
32//! use syn::{braced, parse_macro_input, token, Field, Ident, Result, Token};
33//! use syn::parse::{Parse, ParseStream};
34//! use syn::punctuated::Punctuated;
35//!
36//! enum Item {
37//! Struct(ItemStruct),
38//! Enum(ItemEnum),
39//! }
40//!
41//! struct ItemStruct {
42//! struct_token: Token![struct],
43//! ident: Ident,
44//! brace_token: token::Brace,
45//! fields: Punctuated<Field, Token![,]>,
46//! }
47//! #
48//! # enum ItemEnum {}
49//!
50//! impl Parse for Item {
51//! fn parse(input: ParseStream) -> Result<Self> {
52//! let lookahead = input.lookahead1();
53//! if lookahead.peek(Token![struct]) {
54//! input.parse().map(Item::Struct)
55//! } else if lookahead.peek(Token![enum]) {
56//! input.parse().map(Item::Enum)
57//! } else {
58//! Err(lookahead.error())
59//! }
60//! }
61//! }
62//!
63//! impl Parse for ItemStruct {
64//! fn parse(input: ParseStream) -> Result<Self> {
65//! let content;
66//! Ok(ItemStruct {
67//! struct_token: input.parse()?,
68//! ident: input.parse()?,
69//! brace_token: braced!(content in input),
70//! fields: content.parse_terminated(Field::parse_named)?,
71//! })
72//! }
73//! }
74//! #
75//! # impl Parse for ItemEnum {
76//! # fn parse(input: ParseStream) -> Result<Self> {
77//! # unimplemented!()
78//! # }
79//! # }
80//!
81//! # const IGNORE: &str = stringify! {
82//! #[proc_macro]
83//! # };
84//! pub fn my_macro(tokens: TokenStream) -> TokenStream {
85//! let input = parse_macro_input!(tokens as Item);
86//!
87//! /* ... */
88//! # "".parse().unwrap()
89//! }
90//! ```
91//!
92//! # The `syn::parse*` functions
93//!
94//! The [`syn::parse`], [`syn::parse2`], and [`syn::parse_str`] functions serve
95//! as an entry point for parsing syntax tree nodes that can be parsed in an
96//! obvious default way. These functions can return any syntax tree node that
97//! implements the [`Parse`] trait, which includes most types in Syn.
98//!
99//! [`syn::parse`]: ../fn.parse.html
100//! [`syn::parse2`]: ../fn.parse2.html
101//! [`syn::parse_str`]: ../fn.parse_str.html
102//! [`Parse`]: trait.Parse.html
103//!
104//! ```
105//! use syn::Type;
106//!
107//! # fn run_parser() -> syn::Result<()> {
108//! let t: Type = syn::parse_str("std::collections::HashMap<String, Value>")?;
109//! # Ok(())
110//! # }
111//! #
112//! # run_parser().unwrap();
113//! ```
114//!
115//! The [`parse_quote!`] macro also uses this approach.
116//!
117//! [`parse_quote!`]: ../macro.parse_quote.html
118//!
119//! # The `Parser` trait
120//!
121//! Some types can be parsed in several ways depending on context. For example
122//! an [`Attribute`] can be either "outer" like `#[...]` or "inner" like
123//! `#![...]` and parsing the wrong one would be a bug. Similarly [`Punctuated`]
124//! may or may not allow trailing punctuation, and parsing it the wrong way
125//! would either reject valid input or accept invalid input.
126//!
127//! [`Attribute`]: ../struct.Attribute.html
128//! [`Punctuated`]: ../punctuated/index.html
129//!
130//! The `Parse` trait is not implemented in these cases because there is no good
131//! behavior to consider the default.
132//!
133//! ```compile_fail
134//! # extern crate proc_macro;
135//! #
136//! # use syn::punctuated::Punctuated;
137//! # use syn::{PathSegment, Result, Token};
138//! #
139//! # fn f(tokens: proc_macro::TokenStream) -> Result<()> {
140//! #
141//! // Can't parse `Punctuated` without knowing whether trailing punctuation
142//! // should be allowed in this context.
143//! let path: Punctuated<PathSegment, Token![::]> = syn::parse(tokens)?;
144//! #
145//! # Ok(())
146//! # }
147//! ```
148//!
149//! In these cases the types provide a choice of parser functions rather than a
150//! single `Parse` implementation, and those parser functions can be invoked
151//! through the [`Parser`] trait.
152//!
153//! [`Parser`]: trait.Parser.html
154//!
155//! ```
156//! extern crate proc_macro;
157//!
158//! use proc_macro::TokenStream;
159//! use syn::parse::Parser;
160//! use syn::punctuated::Punctuated;
161//! use syn::{Attribute, Expr, PathSegment, Result, Token};
162//!
163//! fn call_some_parser_methods(input: TokenStream) -> Result<()> {
164//! // Parse a nonempty sequence of path segments separated by `::` punctuation
165//! // with no trailing punctuation.
166//! let tokens = input.clone();
167//! let parser = Punctuated::<PathSegment, Token![::]>::parse_separated_nonempty;
168//! let _path = parser.parse(tokens)?;
169//!
170//! // Parse a possibly empty sequence of expressions terminated by commas with
171//! // an optional trailing punctuation.
172//! let tokens = input.clone();
173//! let parser = Punctuated::<Expr, Token![,]>::parse_terminated;
174//! let _args = parser.parse(tokens)?;
175//!
176//! // Parse zero or more outer attributes but not inner attributes.
177//! let tokens = input.clone();
178//! let parser = Attribute::parse_outer;
179//! let _attrs = parser.parse(tokens)?;
180//!
181//! Ok(())
182//! }
183//! ```
184//!
185//! ---
186//!
187//! *This module is available if Syn is built with the `"parsing"` feature.*
188
189#[path = "discouraged.rs"]
190pub mod discouraged;
191
192use std::cell::Cell;
193use std::fmt::{self, Debug, Display};
194use std::marker::PhantomData;
195use std::mem;
196use std::ops::Deref;
197use std::rc::Rc;
198use std::str::FromStr;
199
200#[cfg(all(
201 not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))),
202 feature = "proc-macro"
203))]
204use crate::proc_macro;
205use proc_macro2::{self, Delimiter, Group, Literal, Punct, Span, TokenStream, TokenTree};
206
207use crate::buffer::{Cursor, TokenBuffer};
208use crate::error;
209use crate::lookahead;
210use crate::punctuated::Punctuated;
211use crate::token::Token;
212
213pub use crate::error::{Error, Result};
214pub use crate::lookahead::{Lookahead1, Peek};
215
216/// Parsing interface implemented by all types that can be parsed in a default
217/// way from a token stream.
218pub trait Parse: Sized {
219 fn parse(input: ParseStream) -> Result<Self>;
220}
221
222/// Input to a Syn parser function.
223///
224/// See the methods of this type under the documentation of [`ParseBuffer`]. For
225/// an overview of parsing in Syn, refer to the [module documentation].
226///
227/// [module documentation]: self
228pub type ParseStream<'a> = &'a ParseBuffer<'a>;
229
230/// Cursor position within a buffered token stream.
231///
232/// This type is more commonly used through the type alias [`ParseStream`] which
233/// is an alias for `&ParseBuffer`.
234///
235/// `ParseStream` is the input type for all parser functions in Syn. They have
236/// the signature `fn(ParseStream) -> Result<T>`.
237///
238/// ## Calling a parser function
239///
240/// There is no public way to construct a `ParseBuffer`. Instead, if you are
241/// looking to invoke a parser function that requires `ParseStream` as input,
242/// you will need to go through one of the public parsing entry points.
243///
244/// - The [`parse_macro_input!`] macro if parsing input of a procedural macro;
245/// - One of [the `syn::parse*` functions][syn-parse]; or
246/// - A method of the [`Parser`] trait.
247///
248/// [syn-parse]: index.html#the-synparse-functions
249pub struct ParseBuffer<'a> {
250 scope: Span,
251 // Instead of Cell<Cursor<'a>> so that ParseBuffer<'a> is covariant in 'a.
252 // The rest of the code in this module needs to be careful that only a
253 // cursor derived from this `cell` is ever assigned to this `cell`.
254 //
255 // Cell<Cursor<'a>> cannot be covariant in 'a because then we could take a
256 // ParseBuffer<'a>, upcast to ParseBuffer<'short> for some lifetime shorter
257 // than 'a, and then assign a Cursor<'short> into the Cell.
258 //
259 // By extension, it would not be safe to expose an API that accepts a
260 // Cursor<'a> and trusts that it lives as long as the cursor currently in
261 // the cell.
262 cell: Cell<Cursor<'static>>,
263 marker: PhantomData<Cursor<'a>>,
264 unexpected: Cell<Option<Rc<Cell<Unexpected>>>>,
265}
266
267impl<'a> Drop for ParseBuffer<'a> {
268 fn drop(&mut self) {
269 if !self.is_empty() {
270 let (inner, old_span) = inner_unexpected(self);
271 if old_span.is_none() {
272 inner.set(Unexpected::Some(self.cursor().span()));
273 }
274 }
275 }
276}
277
278impl<'a> Display for ParseBuffer<'a> {
279 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
280 Display::fmt(&self.cursor().token_stream(), f)
281 }
282}
283
284impl<'a> Debug for ParseBuffer<'a> {
285 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
286 Debug::fmt(&self.cursor().token_stream(), f)
287 }
288}
289
290/// Cursor state associated with speculative parsing.
291///
292/// This type is the input of the closure provided to [`ParseStream::step`].
293///
294/// [`ParseStream::step`]: ParseBuffer::step
295///
296/// # Example
297///
298/// ```
299/// use proc_macro2::TokenTree;
300/// use syn::Result;
301/// use syn::parse::ParseStream;
302///
303/// // This function advances the stream past the next occurrence of `@`. If
304/// // no `@` is present in the stream, the stream position is unchanged and
305/// // an error is returned.
306/// fn skip_past_next_at(input: ParseStream) -> Result<()> {
307/// input.step(|cursor| {
308/// let mut rest = *cursor;
309/// while let Some((tt, next)) = rest.token_tree() {
310/// match &tt {
311/// TokenTree::Punct(punct) if punct.as_char() == '@' => {
312/// return Ok(((), next));
313/// }
314/// _ => rest = next,
315/// }
316/// }
317/// Err(cursor.error("no `@` was found after this point"))
318/// })
319/// }
320/// #
321/// # fn remainder_after_skipping_past_next_at(
322/// # input: ParseStream,
323/// # ) -> Result<proc_macro2::TokenStream> {
324/// # skip_past_next_at(input)?;
325/// # input.parse()
326/// # }
327/// #
328/// # use syn::parse::Parser;
329/// # let remainder = remainder_after_skipping_past_next_at
330/// # .parse_str("a @ b c")
331/// # .unwrap();
332/// # assert_eq!(remainder.to_string(), "b c");
333/// ```
334#[derive(Copy, Clone)]
335pub struct StepCursor<'c, 'a> {
336 scope: Span,
337 // This field is covariant in 'c.
338 cursor: Cursor<'c>,
339 // This field is contravariant in 'c. Together these make StepCursor
340 // invariant in 'c. Also covariant in 'a. The user cannot cast 'c to a
341 // different lifetime but can upcast into a StepCursor with a shorter
342 // lifetime 'a.
343 //
344 // As long as we only ever construct a StepCursor for which 'c outlives 'a,
345 // this means if ever a StepCursor<'c, 'a> exists we are guaranteed that 'c
346 // outlives 'a.
347 marker: PhantomData<fn(Cursor<'c>) -> Cursor<'a>>,
348}
349
350impl<'c, 'a> Deref for StepCursor<'c, 'a> {
351 type Target = Cursor<'c>;
352
353 fn deref(&self) -> &Self::Target {
354 &self.cursor
355 }
356}
357
358impl<'c, 'a> StepCursor<'c, 'a> {
359 /// Triggers an error at the current position of the parse stream.
360 ///
361 /// The `ParseStream::step` invocation will return this same error without
362 /// advancing the stream state.
363 pub fn error<T: Display>(self, message: T) -> Error {
364 error::new_at(self.scope, self.cursor, message)
365 }
366}
367
368pub(crate) fn advance_step_cursor<'c, 'a>(proof: StepCursor<'c, 'a>, to: Cursor<'c>) -> Cursor<'a> {
369 // Refer to the comments within the StepCursor definition. We use the
370 // fact that a StepCursor<'c, 'a> exists as proof that 'c outlives 'a.
371 // Cursor is covariant in its lifetime parameter so we can cast a
372 // Cursor<'c> to one with the shorter lifetime Cursor<'a>.
373 let _ = proof;
374 unsafe { mem::transmute::<Cursor<'c>, Cursor<'a>>(to) }
375}
376
377fn skip(input: ParseStream) -> bool {
378 input
379 .step(|cursor| {
380 if let Some((_lifetime, rest)) = cursor.lifetime() {
381 Ok((true, rest))
382 } else if let Some((_token, rest)) = cursor.token_tree() {
383 Ok((true, rest))
384 } else {
385 Ok((false, *cursor))
386 }
387 })
388 .unwrap()
389}
390
391pub(crate) fn new_parse_buffer(
392 scope: Span,
393 cursor: Cursor,
394 unexpected: Rc<Cell<Unexpected>>,
395) -> ParseBuffer {
396 ParseBuffer {
397 scope,
398 // See comment on `cell` in the struct definition.
399 cell: Cell::new(unsafe { mem::transmute::<Cursor, Cursor<'static>>(cursor) }),
400 marker: PhantomData,
401 unexpected: Cell::new(Some(unexpected)),
402 }
403}
404
405#[derive(Clone)]
406pub(crate) enum Unexpected {
407 None,
408 Some(Span),
409 Chain(Rc<Cell<Unexpected>>),
410}
411
412impl Default for Unexpected {
413 fn default() -> Self {
414 Unexpected::None
415 }
416}
417
418// We call this on Cell<Unexpected> and Cell<Option<T>> where temporarily
419// swapping in a None is cheap.
420fn cell_clone<T: Default + Clone>(cell: &Cell<T>) -> T {
421 let prev = cell.take();
422 let ret = prev.clone();
423 cell.set(prev);
424 ret
425}
426
427fn inner_unexpected(buffer: &ParseBuffer) -> (Rc<Cell<Unexpected>>, Option<Span>) {
428 let mut unexpected = get_unexpected(buffer);
429 loop {
430 match cell_clone(&unexpected) {
431 Unexpected::None => return (unexpected, None),
432 Unexpected::Some(span) => return (unexpected, Some(span)),
433 Unexpected::Chain(next) => unexpected = next,
434 }
435 }
436}
437
438pub(crate) fn get_unexpected(buffer: &ParseBuffer) -> Rc<Cell<Unexpected>> {
439 cell_clone(&buffer.unexpected).unwrap()
440}
441
442impl<'a> ParseBuffer<'a> {
443 /// Parses a syntax tree node of type `T`, advancing the position of our
444 /// parse stream past it.
445 pub fn parse<T: Parse>(&self) -> Result<T> {
446 T::parse(self)
447 }
448
449 /// Calls the given parser function to parse a syntax tree node of type `T`
450 /// from this stream.
451 ///
452 /// # Example
453 ///
454 /// The parser below invokes [`Attribute::parse_outer`] to parse a vector of
455 /// zero or more outer attributes.
456 ///
457 /// [`Attribute::parse_outer`]: crate::Attribute::parse_outer
458 ///
459 /// ```
460 /// use syn::{Attribute, Ident, Result, Token};
461 /// use syn::parse::{Parse, ParseStream};
462 ///
463 /// // Parses a unit struct with attributes.
464 /// //
465 /// // #[path = "s.tmpl"]
466 /// // struct S;
467 /// struct UnitStruct {
468 /// attrs: Vec<Attribute>,
469 /// struct_token: Token![struct],
470 /// name: Ident,
471 /// semi_token: Token![;],
472 /// }
473 ///
474 /// impl Parse for UnitStruct {
475 /// fn parse(input: ParseStream) -> Result<Self> {
476 /// Ok(UnitStruct {
477 /// attrs: input.call(Attribute::parse_outer)?,
478 /// struct_token: input.parse()?,
479 /// name: input.parse()?,
480 /// semi_token: input.parse()?,
481 /// })
482 /// }
483 /// }
484 /// ```
485 pub fn call<T>(&self, function: fn(ParseStream) -> Result<T>) -> Result<T> {
486 function(self)
487 }
488
489 /// Looks at the next token in the parse stream to determine whether it
490 /// matches the requested type of token.
491 ///
492 /// Does not advance the position of the parse stream.
493 ///
494 /// # Syntax
495 ///
496 /// Note that this method does not use turbofish syntax. Pass the peek type
497 /// inside of parentheses.
498 ///
499 /// - `input.peek(Token![struct])`
500 /// - `input.peek(Token![==])`
501 /// - `input.peek(Ident)` *(does not accept keywords)*
502 /// - `input.peek(Ident::peek_any)`
503 /// - `input.peek(Lifetime)`
504 /// - `input.peek(token::Brace)`
505 ///
506 /// # Example
507 ///
508 /// In this example we finish parsing the list of supertraits when the next
509 /// token in the input is either `where` or an opening curly brace.
510 ///
511 /// ```
512 /// use syn::{braced, token, Generics, Ident, Result, Token, TypeParamBound};
513 /// use syn::parse::{Parse, ParseStream};
514 /// use syn::punctuated::Punctuated;
515 ///
516 /// // Parses a trait definition containing no associated items.
517 /// //
518 /// // trait Marker<'de, T>: A + B<'de> where Box<T>: Clone {}
519 /// struct MarkerTrait {
520 /// trait_token: Token![trait],
521 /// ident: Ident,
522 /// generics: Generics,
523 /// colon_token: Option<Token![:]>,
524 /// supertraits: Punctuated<TypeParamBound, Token![+]>,
525 /// brace_token: token::Brace,
526 /// }
527 ///
528 /// impl Parse for MarkerTrait {
529 /// fn parse(input: ParseStream) -> Result<Self> {
530 /// let trait_token: Token![trait] = input.parse()?;
531 /// let ident: Ident = input.parse()?;
532 /// let mut generics: Generics = input.parse()?;
533 /// let colon_token: Option<Token![:]> = input.parse()?;
534 ///
535 /// let mut supertraits = Punctuated::new();
536 /// if colon_token.is_some() {
537 /// loop {
538 /// supertraits.push_value(input.parse()?);
539 /// if input.peek(Token![where]) || input.peek(token::Brace) {
540 /// break;
541 /// }
542 /// supertraits.push_punct(input.parse()?);
543 /// }
544 /// }
545 ///
546 /// generics.where_clause = input.parse()?;
547 /// let content;
548 /// let empty_brace_token = braced!(content in input);
549 ///
550 /// Ok(MarkerTrait {
551 /// trait_token,
552 /// ident,
553 /// generics,
554 /// colon_token,
555 /// supertraits,
556 /// brace_token: empty_brace_token,
557 /// })
558 /// }
559 /// }
560 /// ```
561 pub fn peek<T: Peek>(&self, token: T) -> bool {
562 let _ = token;
563 T::Token::peek(self.cursor())
564 }
565
566 /// Looks at the second-next token in the parse stream.
567 ///
568 /// This is commonly useful as a way to implement contextual keywords.
569 ///
570 /// # Example
571 ///
572 /// This example needs to use `peek2` because the symbol `union` is not a
573 /// keyword in Rust. We can't use just `peek` and decide to parse a union if
574 /// the very next token is `union`, because someone is free to write a `mod
575 /// union` and a macro invocation that looks like `union::some_macro! { ...
576 /// }`. In other words `union` is a contextual keyword.
577 ///
578 /// ```
579 /// use syn::{Ident, ItemUnion, Macro, Result, Token};
580 /// use syn::parse::{Parse, ParseStream};
581 ///
582 /// // Parses either a union or a macro invocation.
583 /// enum UnionOrMacro {
584 /// // union MaybeUninit<T> { uninit: (), value: T }
585 /// Union(ItemUnion),
586 /// // lazy_static! { ... }
587 /// Macro(Macro),
588 /// }
589 ///
590 /// impl Parse for UnionOrMacro {
591 /// fn parse(input: ParseStream) -> Result<Self> {
592 /// if input.peek(Token![union]) && input.peek2(Ident) {
593 /// input.parse().map(UnionOrMacro::Union)
594 /// } else {
595 /// input.parse().map(UnionOrMacro::Macro)
596 /// }
597 /// }
598 /// }
599 /// ```
600 pub fn peek2<T: Peek>(&self, token: T) -> bool {
601 let ahead = self.fork();
602 skip(&ahead) && ahead.peek(token)
603 }
604
605 /// Looks at the third-next token in the parse stream.
606 pub fn peek3<T: Peek>(&self, token: T) -> bool {
607 let ahead = self.fork();
608 skip(&ahead) && skip(&ahead) && ahead.peek(token)
609 }
610
611 /// Parses zero or more occurrences of `T` separated by punctuation of type
612 /// `P`, with optional trailing punctuation.
613 ///
614 /// Parsing continues until the end of this parse stream. The entire content
615 /// of this parse stream must consist of `T` and `P`.
616 ///
617 /// # Example
618 ///
619 /// ```
620 /// # use quote::quote;
621 /// #
622 /// use syn::{parenthesized, token, Ident, Result, Token, Type};
623 /// use syn::parse::{Parse, ParseStream};
624 /// use syn::punctuated::Punctuated;
625 ///
626 /// // Parse a simplified tuple struct syntax like:
627 /// //
628 /// // struct S(A, B);
629 /// struct TupleStruct {
630 /// struct_token: Token![struct],
631 /// ident: Ident,
632 /// paren_token: token::Paren,
633 /// fields: Punctuated<Type, Token![,]>,
634 /// semi_token: Token![;],
635 /// }
636 ///
637 /// impl Parse for TupleStruct {
638 /// fn parse(input: ParseStream) -> Result<Self> {
639 /// let content;
640 /// Ok(TupleStruct {
641 /// struct_token: input.parse()?,
642 /// ident: input.parse()?,
643 /// paren_token: parenthesized!(content in input),
644 /// fields: content.parse_terminated(Type::parse)?,
645 /// semi_token: input.parse()?,
646 /// })
647 /// }
648 /// }
649 /// #
650 /// # let input = quote! {
651 /// # struct S(A, B);
652 /// # };
653 /// # syn::parse2::<TupleStruct>(input).unwrap();
654 /// ```
655 pub fn parse_terminated<T, P: Parse>(
656 &self,
657 parser: fn(ParseStream) -> Result<T>,
658 ) -> Result<Punctuated<T, P>> {
659 Punctuated::parse_terminated_with(self, parser)
660 }
661
662 /// Returns whether there are tokens remaining in this stream.
663 ///
664 /// This method returns true at the end of the content of a set of
665 /// delimiters, as well as at the very end of the complete macro input.
666 ///
667 /// # Example
668 ///
669 /// ```
670 /// use syn::{braced, token, Ident, Item, Result, Token};
671 /// use syn::parse::{Parse, ParseStream};
672 ///
673 /// // Parses a Rust `mod m { ... }` containing zero or more items.
674 /// struct Mod {
675 /// mod_token: Token![mod],
676 /// name: Ident,
677 /// brace_token: token::Brace,
678 /// items: Vec<Item>,
679 /// }
680 ///
681 /// impl Parse for Mod {
682 /// fn parse(input: ParseStream) -> Result<Self> {
683 /// let content;
684 /// Ok(Mod {
685 /// mod_token: input.parse()?,
686 /// name: input.parse()?,
687 /// brace_token: braced!(content in input),
688 /// items: {
689 /// let mut items = Vec::new();
690 /// while !content.is_empty() {
691 /// items.push(content.parse()?);
692 /// }
693 /// items
694 /// },
695 /// })
696 /// }
697 /// }
698 /// ```
699 pub fn is_empty(&self) -> bool {
700 self.cursor().eof()
701 }
702
703 /// Constructs a helper for peeking at the next token in this stream and
704 /// building an error message if it is not one of a set of expected tokens.
705 ///
706 /// # Example
707 ///
708 /// ```
709 /// use syn::{ConstParam, Ident, Lifetime, LifetimeDef, Result, Token, TypeParam};
710 /// use syn::parse::{Parse, ParseStream};
711 ///
712 /// // A generic parameter, a single one of the comma-separated elements inside
713 /// // angle brackets in:
714 /// //
715 /// // fn f<T: Clone, 'a, 'b: 'a, const N: usize>() { ... }
716 /// //
717 /// // On invalid input, lookahead gives us a reasonable error message.
718 /// //
719 /// // error: expected one of: identifier, lifetime, `const`
720 /// // |
721 /// // 5 | fn f<!Sized>() {}
722 /// // | ^
723 /// enum GenericParam {
724 /// Type(TypeParam),
725 /// Lifetime(LifetimeDef),
726 /// Const(ConstParam),
727 /// }
728 ///
729 /// impl Parse for GenericParam {
730 /// fn parse(input: ParseStream) -> Result<Self> {
731 /// let lookahead = input.lookahead1();
732 /// if lookahead.peek(Ident) {
733 /// input.parse().map(GenericParam::Type)
734 /// } else if lookahead.peek(Lifetime) {
735 /// input.parse().map(GenericParam::Lifetime)
736 /// } else if lookahead.peek(Token![const]) {
737 /// input.parse().map(GenericParam::Const)
738 /// } else {
739 /// Err(lookahead.error())
740 /// }
741 /// }
742 /// }
743 /// ```
744 pub fn lookahead1(&self) -> Lookahead1<'a> {
745 lookahead::new(self.scope, self.cursor())
746 }
747
748 /// Forks a parse stream so that parsing tokens out of either the original
749 /// or the fork does not advance the position of the other.
750 ///
751 /// # Performance
752 ///
753 /// Forking a parse stream is a cheap fixed amount of work and does not
754 /// involve copying token buffers. Where you might hit performance problems
755 /// is if your macro ends up parsing a large amount of content more than
756 /// once.
757 ///
758 /// ```
759 /// # use syn::{Expr, Result};
760 /// # use syn::parse::ParseStream;
761 /// #
762 /// # fn bad(input: ParseStream) -> Result<Expr> {
763 /// // Do not do this.
764 /// if input.fork().parse::<Expr>().is_ok() {
765 /// return input.parse::<Expr>();
766 /// }
767 /// # unimplemented!()
768 /// # }
769 /// ```
770 ///
771 /// As a rule, avoid parsing an unbounded amount of tokens out of a forked
772 /// parse stream. Only use a fork when the amount of work performed against
773 /// the fork is small and bounded.
774 ///
775 /// When complex speculative parsing against the forked stream is
776 /// unavoidable, use [`parse::discouraged::Speculative`] to advance the
777 /// original stream once the fork's parse is determined to have been
778 /// successful.
779 ///
780 /// For a lower level way to perform speculative parsing at the token level,
781 /// consider using [`ParseStream::step`] instead.
782 ///
783 /// [`parse::discouraged::Speculative`]: discouraged::Speculative
784 /// [`ParseStream::step`]: ParseBuffer::step
785 ///
786 /// # Example
787 ///
788 /// The parse implementation shown here parses possibly restricted `pub`
789 /// visibilities.
790 ///
791 /// - `pub`
792 /// - `pub(crate)`
793 /// - `pub(self)`
794 /// - `pub(super)`
795 /// - `pub(in some::path)`
796 ///
797 /// To handle the case of visibilities inside of tuple structs, the parser
798 /// needs to distinguish parentheses that specify visibility restrictions
799 /// from parentheses that form part of a tuple type.
800 ///
801 /// ```
802 /// # struct A;
803 /// # struct B;
804 /// # struct C;
805 /// #
806 /// struct S(pub(crate) A, pub (B, C));
807 /// ```
808 ///
809 /// In this example input the first tuple struct element of `S` has
810 /// `pub(crate)` visibility while the second tuple struct element has `pub`
811 /// visibility; the parentheses around `(B, C)` are part of the type rather
812 /// than part of a visibility restriction.
813 ///
814 /// The parser uses a forked parse stream to check the first token inside of
815 /// parentheses after the `pub` keyword. This is a small bounded amount of
816 /// work performed against the forked parse stream.
817 ///
818 /// ```
819 /// use syn::{parenthesized, token, Ident, Path, Result, Token};
820 /// use syn::ext::IdentExt;
821 /// use syn::parse::{Parse, ParseStream};
822 ///
823 /// struct PubVisibility {
824 /// pub_token: Token![pub],
825 /// restricted: Option<Restricted>,
826 /// }
827 ///
828 /// struct Restricted {
829 /// paren_token: token::Paren,
830 /// in_token: Option<Token![in]>,
831 /// path: Path,
832 /// }
833 ///
834 /// impl Parse for PubVisibility {
835 /// fn parse(input: ParseStream) -> Result<Self> {
836 /// let pub_token: Token![pub] = input.parse()?;
837 ///
838 /// if input.peek(token::Paren) {
839 /// let ahead = input.fork();
840 /// let mut content;
841 /// parenthesized!(content in ahead);
842 ///
843 /// if content.peek(Token![crate])
844 /// || content.peek(Token![self])
845 /// || content.peek(Token![super])
846 /// {
847 /// return Ok(PubVisibility {
848 /// pub_token,
849 /// restricted: Some(Restricted {
850 /// paren_token: parenthesized!(content in input),
851 /// in_token: None,
852 /// path: Path::from(content.call(Ident::parse_any)?),
853 /// }),
854 /// });
855 /// } else if content.peek(Token![in]) {
856 /// return Ok(PubVisibility {
857 /// pub_token,
858 /// restricted: Some(Restricted {
859 /// paren_token: parenthesized!(content in input),
860 /// in_token: Some(content.parse()?),
861 /// path: content.call(Path::parse_mod_style)?,
862 /// }),
863 /// });
864 /// }
865 /// }
866 ///
867 /// Ok(PubVisibility {
868 /// pub_token,
869 /// restricted: None,
870 /// })
871 /// }
872 /// }
873 /// ```
874 pub fn fork(&self) -> Self {
875 ParseBuffer {
876 scope: self.scope,
877 cell: self.cell.clone(),
878 marker: PhantomData,
879 // Not the parent's unexpected. Nothing cares whether the clone
880 // parses all the way unless we `advance_to`.
881 unexpected: Cell::new(Some(Rc::new(Cell::new(Unexpected::None)))),
882 }
883 }
884
885 /// Triggers an error at the current position of the parse stream.
886 ///
887 /// # Example
888 ///
889 /// ```
890 /// use syn::{Expr, Result, Token};
891 /// use syn::parse::{Parse, ParseStream};
892 ///
893 /// // Some kind of loop: `while` or `for` or `loop`.
894 /// struct Loop {
895 /// expr: Expr,
896 /// }
897 ///
898 /// impl Parse for Loop {
899 /// fn parse(input: ParseStream) -> Result<Self> {
900 /// if input.peek(Token![while])
901 /// || input.peek(Token![for])
902 /// || input.peek(Token![loop])
903 /// {
904 /// Ok(Loop {
905 /// expr: input.parse()?,
906 /// })
907 /// } else {
908 /// Err(input.error("expected some kind of loop"))
909 /// }
910 /// }
911 /// }
912 /// ```
913 pub fn error<T: Display>(&self, message: T) -> Error {
914 error::new_at(self.scope, self.cursor(), message)
915 }
916
917 /// Speculatively parses tokens from this parse stream, advancing the
918 /// position of this stream only if parsing succeeds.
919 ///
920 /// This is a powerful low-level API used for defining the `Parse` impls of
921 /// the basic built-in token types. It is not something that will be used
922 /// widely outside of the Syn codebase.
923 ///
924 /// # Example
925 ///
926 /// ```
927 /// use proc_macro2::TokenTree;
928 /// use syn::Result;
929 /// use syn::parse::ParseStream;
930 ///
931 /// // This function advances the stream past the next occurrence of `@`. If
932 /// // no `@` is present in the stream, the stream position is unchanged and
933 /// // an error is returned.
934 /// fn skip_past_next_at(input: ParseStream) -> Result<()> {
935 /// input.step(|cursor| {
936 /// let mut rest = *cursor;
937 /// while let Some((tt, next)) = rest.token_tree() {
938 /// match &tt {
939 /// TokenTree::Punct(punct) if punct.as_char() == '@' => {
940 /// return Ok(((), next));
941 /// }
942 /// _ => rest = next,
943 /// }
944 /// }
945 /// Err(cursor.error("no `@` was found after this point"))
946 /// })
947 /// }
948 /// #
949 /// # fn remainder_after_skipping_past_next_at(
950 /// # input: ParseStream,
951 /// # ) -> Result<proc_macro2::TokenStream> {
952 /// # skip_past_next_at(input)?;
953 /// # input.parse()
954 /// # }
955 /// #
956 /// # use syn::parse::Parser;
957 /// # let remainder = remainder_after_skipping_past_next_at
958 /// # .parse_str("a @ b c")
959 /// # .unwrap();
960 /// # assert_eq!(remainder.to_string(), "b c");
961 /// ```
962 pub fn step<F, R>(&self, function: F) -> Result<R>
963 where
964 F: for<'c> FnOnce(StepCursor<'c, 'a>) -> Result<(R, Cursor<'c>)>,
965 {
966 // Since the user's function is required to work for any 'c, we know
967 // that the Cursor<'c> they return is either derived from the input
968 // StepCursor<'c, 'a> or from a Cursor<'static>.
969 //
970 // It would not be legal to write this function without the invariant
971 // lifetime 'c in StepCursor<'c, 'a>. If this function were written only
972 // in terms of 'a, the user could take our ParseBuffer<'a>, upcast it to
973 // a ParseBuffer<'short> which some shorter lifetime than 'a, invoke
974 // `step` on their ParseBuffer<'short> with a closure that returns
975 // Cursor<'short>, and we would wrongly write that Cursor<'short> into
976 // the Cell intended to hold Cursor<'a>.
977 //
978 // In some cases it may be necessary for R to contain a Cursor<'a>.
979 // Within Syn we solve this using `advance_step_cursor` which uses the
980 // existence of a StepCursor<'c, 'a> as proof that it is safe to cast
981 // from Cursor<'c> to Cursor<'a>. If needed outside of Syn, it would be
982 // safe to expose that API as a method on StepCursor.
983 let (node, rest) = function(StepCursor {
984 scope: self.scope,
985 cursor: self.cell.get(),
986 marker: PhantomData,
987 })?;
988 self.cell.set(rest);
989 Ok(node)
990 }
991
992 /// Provides low-level access to the token representation underlying this
993 /// parse stream.
994 ///
995 /// Cursors are immutable so no operations you perform against the cursor
996 /// will affect the state of this parse stream.
997 pub fn cursor(&self) -> Cursor<'a> {
998 self.cell.get()
999 }
1000
1001 fn check_unexpected(&self) -> Result<()> {
1002 match inner_unexpected(self).1 {
1003 Some(span) => Err(Error::new(span, "unexpected token")),
1004 None => Ok(()),
1005 }
1006 }
1007}
1008
1009impl<T: Parse> Parse for Box<T> {
1010 fn parse(input: ParseStream) -> Result<Self> {
1011 input.parse().map(Box::new)
1012 }
1013}
1014
1015impl<T: Parse + Token> Parse for Option<T> {
1016 fn parse(input: ParseStream) -> Result<Self> {
1017 if T::peek(input.cursor()) {
1018 Ok(Some(input.parse()?))
1019 } else {
1020 Ok(None)
1021 }
1022 }
1023}
1024
1025impl Parse for TokenStream {
1026 fn parse(input: ParseStream) -> Result<Self> {
1027 input.step(|cursor| Ok((cursor.token_stream(), Cursor::empty())))
1028 }
1029}
1030
1031impl Parse for TokenTree {
1032 fn parse(input: ParseStream) -> Result<Self> {
1033 input.step(|cursor| match cursor.token_tree() {
1034 Some((tt, rest)) => Ok((tt, rest)),
1035 None => Err(cursor.error("expected token tree")),
1036 })
1037 }
1038}
1039
1040impl Parse for Group {
1041 fn parse(input: ParseStream) -> Result<Self> {
1042 input.step(|cursor| {
1043 for delim in &[Delimiter::Parenthesis, Delimiter::Brace, Delimiter::Bracket] {
1044 if let Some((inside, span, rest)) = cursor.group(*delim) {
1045 let mut group = Group::new(*delim, inside.token_stream());
1046 group.set_span(span);
1047 return Ok((group, rest));
1048 }
1049 }
1050 Err(cursor.error("expected group token"))
1051 })
1052 }
1053}
1054
1055impl Parse for Punct {
1056 fn parse(input: ParseStream) -> Result<Self> {
1057 input.step(|cursor| match cursor.punct() {
1058 Some((punct, rest)) => Ok((punct, rest)),
1059 None => Err(cursor.error("expected punctuation token")),
1060 })
1061 }
1062}
1063
1064impl Parse for Literal {
1065 fn parse(input: ParseStream) -> Result<Self> {
1066 input.step(|cursor| match cursor.literal() {
1067 Some((literal, rest)) => Ok((literal, rest)),
1068 None => Err(cursor.error("expected literal token")),
1069 })
1070 }
1071}
1072
1073/// Parser that can parse Rust tokens into a particular syntax tree node.
1074///
1075/// Refer to the [module documentation] for details about parsing in Syn.
1076///
1077/// [module documentation]: self
1078///
1079/// *This trait is available if Syn is built with the `"parsing"` feature.*
1080pub trait Parser: Sized {
1081 type Output;
1082
1083 /// Parse a proc-macro2 token stream into the chosen syntax tree node.
1084 ///
1085 /// This function will check that the input is fully parsed. If there are
1086 /// any unparsed tokens at the end of the stream, an error is returned.
1087 fn parse2(self, tokens: TokenStream) -> Result<Self::Output>;
1088
1089 /// Parse tokens of source code into the chosen syntax tree node.
1090 ///
1091 /// This function will check that the input is fully parsed. If there are
1092 /// any unparsed tokens at the end of the stream, an error is returned.
1093 ///
1094 /// *This method is available if Syn is built with both the `"parsing"` and
1095 /// `"proc-macro"` features.*
1096 #[cfg(all(
1097 not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))),
1098 feature = "proc-macro"
1099 ))]
1100 fn parse(self, tokens: proc_macro::TokenStream) -> Result<Self::Output> {
1101 self.parse2(proc_macro2::TokenStream::from(tokens))
1102 }
1103
1104 /// Parse a string of Rust code into the chosen syntax tree node.
1105 ///
1106 /// This function will check that the input is fully parsed. If there are
1107 /// any unparsed tokens at the end of the string, an error is returned.
1108 ///
1109 /// # Hygiene
1110 ///
1111 /// Every span in the resulting syntax tree will be set to resolve at the
1112 /// macro call site.
1113 fn parse_str(self, s: &str) -> Result<Self::Output> {
1114 self.parse2(proc_macro2::TokenStream::from_str(s)?)
1115 }
1116
1117 // Not public API.
1118 #[doc(hidden)]
1119 fn __parse_scoped(self, scope: Span, tokens: TokenStream) -> Result<Self::Output> {
1120 let _ = scope;
1121 self.parse2(tokens)
1122 }
1123
1124 // Not public API.
1125 #[doc(hidden)]
1126 fn __parse_stream(self, input: ParseStream) -> Result<Self::Output> {
1127 input.parse().and_then(|tokens| self.parse2(tokens))
1128 }
1129}
1130
1131fn tokens_to_parse_buffer(tokens: &TokenBuffer) -> ParseBuffer {
1132 let scope = Span::call_site();
1133 let cursor = tokens.begin();
1134 let unexpected = Rc::new(Cell::new(Unexpected::None));
1135 new_parse_buffer(scope, cursor, unexpected)
1136}
1137
1138impl<F, T> Parser for F
1139where
1140 F: FnOnce(ParseStream) -> Result<T>,
1141{
1142 type Output = T;
1143
1144 fn parse2(self, tokens: TokenStream) -> Result<T> {
1145 let buf = TokenBuffer::new2(tokens);
1146 let state = tokens_to_parse_buffer(&buf);
1147 let node = self(&state)?;
1148 state.check_unexpected()?;
1149 if state.is_empty() {
1150 Ok(node)
1151 } else {
1152 Err(state.error("unexpected token"))
1153 }
1154 }
1155
1156 #[doc(hidden)]
1157 fn __parse_scoped(self, scope: Span, tokens: TokenStream) -> Result<Self::Output> {
1158 let buf = TokenBuffer::new2(tokens);
1159 let cursor = buf.begin();
1160 let unexpected = Rc::new(Cell::new(Unexpected::None));
1161 let state = new_parse_buffer(scope, cursor, unexpected);
1162 let node = self(&state)?;
1163 state.check_unexpected()?;
1164 if state.is_empty() {
1165 Ok(node)
1166 } else {
1167 Err(state.error("unexpected token"))
1168 }
1169 }
1170
1171 #[doc(hidden)]
1172 fn __parse_stream(self, input: ParseStream) -> Result<Self::Output> {
1173 self(input)
1174 }
1175}
1176
1177pub(crate) fn parse_scoped<F: Parser>(f: F, scope: Span, tokens: TokenStream) -> Result<F::Output> {
1178 f.__parse_scoped(scope, tokens)
1179}
1180
1181pub(crate) fn parse_stream<F: Parser>(f: F, input: ParseStream) -> Result<F::Output> {
1182 f.__parse_stream(input)
1183}
1184
1185/// An empty syntax tree node that consumes no tokens when parsed.
1186///
1187/// This is useful for attribute macros that want to ensure they are not
1188/// provided any attribute args.
1189///
1190/// ```
1191/// extern crate proc_macro;
1192///
1193/// use proc_macro::TokenStream;
1194/// use syn::parse_macro_input;
1195/// use syn::parse::Nothing;
1196///
1197/// # const IGNORE: &str = stringify! {
1198/// #[proc_macro_attribute]
1199/// # };
1200/// pub fn my_attr(args: TokenStream, input: TokenStream) -> TokenStream {
1201/// parse_macro_input!(args as Nothing);
1202///
1203/// /* ... */
1204/// # "".parse().unwrap()
1205/// }
1206/// ```
1207///
1208/// ```text
1209/// error: unexpected token
1210/// --> src/main.rs:3:19
1211/// |
1212/// 3 | #[my_attr(asdf)]
1213/// | ^^^^
1214/// ```
1215pub struct Nothing;
1216
1217impl Parse for Nothing {
1218 fn parse(_input: ParseStream) -> Result<Self> {
1219 Ok(Nothing)
1220 }
1221}