path: root/rust/syn/token.rs

// SPDX-License-Identifier: Apache-2.0 OR MIT

//! Tokens representing Rust punctuation, keywords, and delimiters.
//!
//! The type names in this module can be difficult to keep straight, so we
//! prefer to use the [`Token!`] macro instead. This is a type-macro that
//! expands to the token type of the given token.
//!
//! [`Token!`]: crate::Token
//!
//! # Example
//!
//! The [`ItemStatic`] syntax tree node is defined like this.
//!
//! [`ItemStatic`]: crate::ItemStatic
//!
//! ```
//! # use syn::{Attribute, Expr, Ident, Token, Type, Visibility};
//! #
//! pub struct ItemStatic {
//!     pub attrs: Vec<Attribute>,
//!     pub vis: Visibility,
//!     pub static_token: Token![static],
//!     pub mutability: Option<Token![mut]>,
//!     pub ident: Ident,
//!     pub colon_token: Token![:],
//!     pub ty: Box<Type>,
//!     pub eq_token: Token![=],
//!     pub expr: Box<Expr>,
//!     pub semi_token: Token![;],
//! }
//! ```
//!
//! # Parsing
//!
//! Keywords and punctuation can be parsed through the [`ParseStream::parse`]
//! method. Delimiter tokens are parsed using the [`parenthesized!`],
//! [`bracketed!`] and [`braced!`] macros.
//!
//! [`ParseStream::parse`]: crate::parse::ParseBuffer::parse()
//! [`parenthesized!`]: crate::parenthesized!
//! [`bracketed!`]: crate::bracketed!
//! [`braced!`]: crate::braced!
//!
//! ```
//! use syn::{Attribute, Result};
//! use syn::parse::{Parse, ParseStream};
//! #
//! # enum ItemStatic {}
//!
//! // Parse the ItemStatic struct shown above.
//! impl Parse for ItemStatic {
//!     fn parse(input: ParseStream) -> Result<Self> {
//!         # use syn::ItemStatic;
//!         # fn parse(input: ParseStream) -> Result<ItemStatic> {
//!         Ok(ItemStatic {
//!             attrs: input.call(Attribute::parse_outer)?,
//!             vis: input.parse()?,
//!             static_token: input.parse()?,
//!             mutability: input.parse()?,
//!             ident: input.parse()?,
//!             colon_token: input.parse()?,
//!             ty: input.parse()?,
//!             eq_token: input.parse()?,
//!             expr: input.parse()?,
//!             semi_token: input.parse()?,
//!         })
//!         # }
//!         # unimplemented!()
//!     }
//! }
//! ```
//!
//! # Other operations
//!
//! Every keyword and punctuation token supports the following operations.
//!
//! - [Peeking] — `input.peek(Token![...])`
//!
//! - [Parsing] — `input.parse::<Token![...]>()?`
//!
//! - [Printing] — `quote!( ... #the_token ... )`
//!
//! - Construction from a [`Span`] — `let the_token = Token![...](sp)`
//!
//! - Field access to its span — `let sp = the_token.span`
//!
//! [Peeking]: crate::parse::ParseBuffer::peek()
//! [Parsing]: crate::parse::ParseBuffer::parse()
//! [Printing]: https://docs.rs/quote/1.0/quote/trait.ToTokens.html
//! [`Span`]: https://docs.rs/proc-macro2/1.0/proc_macro2/struct.Span.html

#[cfg(feature = "parsing")]
pub(crate) use self::private::CustomToken;
use self::private::WithSpan;
#[cfg(feature = "parsing")]
use crate::buffer::Cursor;
#[cfg(feature = "parsing")]
use crate::error::Result;
#[cfg(feature = "parsing")]
use crate::lifetime::Lifetime;
#[cfg(feature = "parsing")]
use crate::parse::{Parse, ParseStream};
use crate::span::IntoSpans;
use proc_macro2::extra::DelimSpan;
use proc_macro2::Span;
#[cfg(feature = "printing")]
use proc_macro2::TokenStream;
#[cfg(any(feature = "parsing", feature = "printing"))]
use proc_macro2::{Delimiter, Ident};
#[cfg(feature = "parsing")]
use proc_macro2::{Literal, Punct, TokenTree};
#[cfg(feature = "printing")]
use quote::{ToTokens, TokenStreamExt};
#[cfg(feature = "extra-traits")]
use std::cmp;
#[cfg(feature = "extra-traits")]
use std::fmt::{self, Debug};
#[cfg(feature = "extra-traits")]
use std::hash::{Hash, Hasher};
use std::ops::{Deref, DerefMut};

/// Marker trait for types that represent single tokens.
///
/// This trait is sealed and cannot be implemented for types outside of Syn.
#[cfg(feature = "parsing")]
pub trait Token: private::Sealed {
    // Not public API.
    #[doc(hidden)]
    fn peek(cursor: Cursor) -> bool;

    // Not public API.
    #[doc(hidden)]
    fn display() -> &'static str;
}

pub(crate) mod private {
    #[cfg(feature = "parsing")]
    use crate::buffer::Cursor;
    use proc_macro2::Span;

    #[cfg(feature = "parsing")]
    pub trait Sealed {}

    /// Support writing `token.span` rather than `token.spans[0]` on tokens that
    /// hold a single span.
    #[repr(transparent)]
    #[allow(unknown_lints, repr_transparent_external_private_fields)] // False positive: https://github.com/rust-lang/rust/issues/78586#issuecomment-1722680482
    pub struct WithSpan {
        pub span: Span,
    }

    // Not public API.
    #[doc(hidden)]
    #[cfg(feature = "parsing")]
    pub trait CustomToken {
        fn peek(cursor: Cursor) -> bool;
        fn display() -> &'static str;
    }
}

#[cfg(feature = "parsing")]
impl private::Sealed for Ident {}

macro_rules! impl_low_level_token {
    ($display:literal $($path:ident)::+ $get:ident) => {
        #[cfg(feature = "parsing")]
        impl Token for $($path)::+ {
            fn peek(cursor: Cursor) -> bool {
                cursor.$get().is_some()
            }

            fn display() -> &'static str {
                $display
            }
        }

        #[cfg(feature = "parsing")]
        impl private::Sealed