{-# LANGUAGE CPP, DeriveDataTypeable #-}
-----------------------------------------------------------------------------
-- |
-- Module      : Language.Python.Common.Token 
-- Copyright   : (c) 2009 Bernie Pope 
-- License     : BSD-style
-- Maintainer  : bjpop@csse.unimelb.edu.au
-- Stability   : experimental
-- Portability : ghc
--
-- Lexical tokens for the Python lexer. Contains the superset of tokens from
-- version 2 and version 3 of Python (they are mostly the same).
-----------------------------------------------------------------------------

module Language.Python.Common.Token ( 
   -- * The tokens 
   Token (..),
   -- * String conversion
   debugTokenString,
   tokenString,
   -- * Classification
   hasLiteral,
   TokenClass (..),
   classifyToken
   ) where

import Language.Python.Common.Pretty
import Language.Python.Common.SrcLocation (SrcSpan (..), Span(getSpan))
import Data.Data

-- | Lexical tokens.
data Token 
   -- Whitespace
   = IndentToken { Token -> SrcSpan
token_span :: !SrcSpan }                       -- ^ Indentation: increase.
   | DedentToken { token_span :: !SrcSpan }                       -- ^ Indentation: decrease.
   | NewlineToken { token_span :: !SrcSpan }                      -- ^ Newline.
   | LineJoinToken { token_span :: !SrcSpan }                     -- ^ Line join (backslash at end of line).

   -- Comment
   | CommentToken { token_span :: !SrcSpan, Token -> String
token_literal :: !String } -- ^ Single line comment.

   -- Identifiers 
   | IdentifierToken { token_span :: !SrcSpan, token_literal :: !String }            -- ^ Identifier.

   -- Literals
   | StringToken { token_span :: !SrcSpan, token_literal :: !String }                   -- ^ Literal: string.
   | ByteStringToken { token_span :: !SrcSpan, token_literal :: !String }    -- ^ Literal: byte string.
   | UnicodeStringToken { token_span :: !SrcSpan, token_literal :: !String } -- ^ Literal: unicode string, version 2 only.
   | IntegerToken { token_span :: !SrcSpan, token_literal :: !String, Token -> Integer
token_integer :: !Integer }                 -- ^ Literal: integer.
   | LongIntegerToken { token_span :: !SrcSpan, token_literal :: !String, token_integer :: !Integer }             -- ^ Literal: long integer. /Version 2 only/.
   | FloatToken { token_span :: !SrcSpan, token_literal :: !String, Token -> Double
token_double :: !Double }                     -- ^ Literal: floating point.
   | ImaginaryToken { token_span :: !SrcSpan, token_literal :: !String, token_double :: !Double }                 -- ^ Literal: imaginary number.

   -- Keywords
   | DefToken { token_span :: !SrcSpan }                          -- ^ Keyword: \'def\'. 
   | WhileToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'while\'.
   | IfToken { token_span :: !SrcSpan }                           -- ^ Keyword: \'if\'.
   | TrueToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'True\'.
   | FalseToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'False\'.
   | ReturnToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'Return\'.
   | TryToken { token_span :: !SrcSpan }                          -- ^ Keyword: \'try\'.
   | ExceptToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'except\'.
   | RaiseToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'raise\'.
   | InToken { token_span :: !SrcSpan }                           -- ^ Keyword: \'in\'.
   | IsToken { token_span :: !SrcSpan }                           -- ^ Keyword: \'is\'.
   | LambdaToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'lambda\'.
   | ClassToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'class\'.
   | FinallyToken { token_span :: !SrcSpan }                      -- ^ Keyword: \'finally\'.
   | NoneToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'None\'. 
   | ForToken { token_span :: !SrcSpan }                          -- ^ Keyword: \'for\'.
   | FromToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'from\'.
   | GlobalToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'global\'.
   | WithToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'with\'.
   | AsToken { token_span :: !SrcSpan }                           -- ^ Keyword: \'as\'.
   | ElifToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'elif\'.
   | YieldToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'yield\'.
   | AssertToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'assert\'.
   | ImportToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'import\'.
   | PassToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'pass\'.
   | BreakToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'break\'.
   | ContinueToken { token_span :: !SrcSpan }                     -- ^ Keyword: \'continue\'.
   | DeleteToken { token_span :: !SrcSpan }                       -- ^ Keyword: \'del\'.
   | ElseToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'else\'.
   | NotToken { token_span :: !SrcSpan }                          -- ^ Keyword: \'not\'.
   | AndToken { token_span :: !SrcSpan }                          -- ^ Keyword: boolean conjunction \'and\'.
   | OrToken { token_span :: !SrcSpan }                           -- ^ Keyword: boolean disjunction \'or\'.
   -- Version 3.x only:
   | NonLocalToken { token_span :: !SrcSpan }                     -- ^ Keyword: \'nonlocal\' (Python 3.x only)
   | AsyncToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'async\' (Python 3.x only)
   | AwaitToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'await\' (Python 3.x only)
   -- Version 2.x only:
   | PrintToken { token_span :: !SrcSpan }                        -- ^ Keyword: \'print\'. (Python 2.x only)
   | ExecToken { token_span :: !SrcSpan }                         -- ^ Keyword: \'exec\'. (Python 2.x only)

   -- Delimiters
   | AtToken { token_span :: !SrcSpan }                           -- ^ Delimiter: at sign \'\@\'. 
   | LeftRoundBracketToken { token_span :: !SrcSpan }             -- ^ Delimiter: left round bracket \'(\'.
   | RightRoundBracketToken { token_span :: !SrcSpan }            -- ^ Delimiter: right round bracket \')\'.
   | LeftSquareBracketToken { token_span :: !SrcSpan }            -- ^ Delimiter: left square bracket \'[\'.
   | RightSquareBracketToken { token_span :: !SrcSpan }           -- ^ Delimiter: right square bracket \']\'.
   | LeftBraceToken { token_span :: !SrcSpan }                    -- ^ Delimiter: left curly bracket \'{\'.
   | RightBraceToken { token_span :: !SrcSpan }                   -- ^ Delimiter: right curly bracket \'}\'.
   | DotToken { token_span :: !SrcSpan }                          -- ^ Delimiter: dot (full stop) \'.\'.
   | CommaToken { token_span :: !SrcSpan }                        -- ^ Delimiter: comma \',\'.
   | SemiColonToken { token_span :: !SrcSpan }                    -- ^ Delimiter: semicolon \';\'.
   | ColonToken { token_span :: !SrcSpan }                        -- ^ Delimiter: colon \':\'.
   | EllipsisToken { token_span :: !SrcSpan }                     -- ^ Delimiter: ellipses (three dots) \'...\'.
   | RightArrowToken { token_span :: !SrcSpan }                   -- ^ Delimiter: right facing arrow \'->\'.
   | AssignToken { token_span :: !SrcSpan }                       -- ^ Delimiter: assignment \'=\'.
   | PlusAssignToken { token_span :: !SrcSpan }                   -- ^ Delimiter: plus assignment \'+=\'.
   | MinusAssignToken { token_span :: !SrcSpan }                  -- ^ Delimiter: minus assignment \'-=\'.
   | MultAssignToken { token_span :: !SrcSpan }                   -- ^ Delimiter: multiply assignment \'*=\'
   | DivAssignToken { token_span :: !SrcSpan }                    -- ^ Delimiter: divide assignment \'/=\'.
   | ModAssignToken { token_span :: !SrcSpan }                    -- ^ Delimiter: modulus assignment \'%=\'.
   | PowAssignToken { token_span :: !SrcSpan }                    -- ^ Delimiter: power assignment \'**=\'.
   | BinAndAssignToken { token_span :: !SrcSpan }                 -- ^ Delimiter: binary-and assignment \'&=\'.
   | BinOrAssignToken { token_span :: !SrcSpan }                  -- ^ Delimiter: binary-or assignment \'|=\'.
   | BinXorAssignToken { token_span :: !SrcSpan }                 -- ^ Delimiter: binary-xor assignment \'^=\'.
   | LeftShiftAssignToken { token_span :: !SrcSpan }              -- ^ Delimiter: binary-left-shift assignment \'<<=\'.
   | RightShiftAssignToken { token_span :: !SrcSpan }             -- ^ Delimiter: binary-right-shift assignment \'>>=\'.
   | FloorDivAssignToken { token_span :: !SrcSpan }               -- ^ Delimiter: floor-divide assignment \'//=\'.
   | MatrixMultAssignToken { token_span :: !SrcSpan }             -- ^ Delimiter: matrix multiplication assignment \'@=\'.
   | BackQuoteToken { token_span :: !SrcSpan }                    -- ^ Delimiter: back quote character \'`\'.

   -- Operators
   | PlusToken { token_span :: !SrcSpan }                         -- ^ Operator: plus \'+\'.
   | MinusToken { token_span :: !SrcSpan }                        -- ^ Operator: minus: \'-\'.
   | MultToken { token_span :: !SrcSpan }                         -- ^ Operator: multiply \'*\'.
   | DivToken { token_span :: !SrcSpan }                          -- ^ Operator: divide \'/\'.
   | GreaterThanToken { token_span :: !SrcSpan }                  -- ^ Operator: greater-than \'>\'.
   | LessThanToken { token_span :: !SrcSpan }                     -- ^ Operator: less-than \'<\'.
   | EqualityToken { token_span :: !SrcSpan }                     -- ^ Operator: equals \'==\'.
   | GreaterThanEqualsToken { token_span :: !SrcSpan }            -- ^ Operator: greater-than-or-equals \'>=\'.
   | LessThanEqualsToken { token_span :: !SrcSpan }               -- ^ Operator: less-than-or-equals \'<=\'.
   | ExponentToken { token_span :: !SrcSpan }                     -- ^ Operator: exponential \'**\'.
   | BinaryOrToken { token_span :: !SrcSpan }                     -- ^ Operator: binary-or \'|\'.
   | XorToken { token_span :: !SrcSpan }                          -- ^ Operator: binary-xor \'^\'.
   | BinaryAndToken { token_span :: !SrcSpan }                    -- ^ Operator: binary-and \'&\'.
   | ShiftLeftToken { token_span :: !SrcSpan }                    -- ^ Operator: binary-shift-left \'<<\'.
   | ShiftRightToken { token_span :: !SrcSpan }                   -- ^ Operator: binary-shift-right \'>>\'.
   | ModuloToken { token_span :: !SrcSpan }                       -- ^ Operator: modulus \'%\'.
   | FloorDivToken { token_span :: !SrcSpan }                     -- ^ Operator: floor-divide \'//\'.
   | TildeToken { token_span :: !SrcSpan }                        -- ^ Operator: tilde \'~\'.
   | NotEqualsToken { token_span :: !SrcSpan }                    -- ^ Operator: not-equals \'!=\'.
   | NotEqualsV2Token { token_span :: !SrcSpan }                  -- ^ Operator: not-equals \'<>\'. Version 2 only.

   -- Special cases
   | EOFToken { token_span :: !SrcSpan }                          -- ^ End of file 
   deriving (Token -> Token -> Bool
(Token -> Token -> Bool) -> (Token -> Token -> Bool) -> Eq Token
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: Token -> Token -> Bool
== :: Token -> Token -> Bool
$c/= :: Token -> Token -> Bool
/= :: Token -> Token -> Bool
Eq,Eq Token
Eq Token =>
(Token -> Token -> Ordering)
-> (Token -> Token -> Bool)
-> (Token -> Token -> Bool)
-> (Token -> Token -> Bool)
-> (Token -> Token -> Bool)
-> (Token -> Token -> Token)
-> (Token -> Token -> Token)
-> Ord Token
Token -> Token -> Bool
Token -> Token -> Ordering
Token -> Token -> Token
forall a.
Eq a =>
(a -> a -> Ordering)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> a)
-> (a -> a -> a)
-> Ord a
$ccompare :: Token -> Token -> Ordering
compare :: Token -> Token -> Ordering
$c< :: Token -> Token -> Bool
< :: Token -> Token -> Bool
$c<= :: Token -> Token -> Bool
<= :: Token -> Token -> Bool
$c> :: Token -> Token -> Bool
> :: Token -> Token -> Bool
$c>= :: Token -> Token -> Bool
>= :: Token -> Token -> Bool
$cmax :: Token -> Token -> Token
max :: Token -> Token -> Token
$cmin :: Token -> Token -> Token
min :: Token -> Token -> Token
Ord,Int -> Token -> ShowS
[Token] -> ShowS
Token -> String
(Int -> Token -> ShowS)
-> (Token -> String) -> ([Token] -> ShowS) -> Show Token
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> Token -> ShowS
showsPrec :: Int -> Token -> ShowS
$cshow :: Token -> String
show :: Token -> String
$cshowList :: [Token] -> ShowS
showList :: [Token] -> ShowS
Show,Typeable,Typeable Token
Typeable Token =>
(forall (c :: * -> *).
 (forall d b. Data d => c (d -> b) -> d -> c b)
 -> (forall g. g -> c g) -> Token -> c Token)
-> (forall (c :: * -> *).
    (forall b r. Data b => c (b -> r) -> c r)
    -> (forall r. r -> c r) -> Constr -> c Token)
-> (Token -> Constr)
-> (Token -> DataType)
-> (forall (t :: * -> *) (c :: * -> *).
    Typeable t =>
    (forall d. Data d => c (t d)) -> Maybe (c Token))
-> (forall (t :: * -> * -> *) (c :: * -> *).
    Typeable t =>
    (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Token))
-> ((forall b. Data b => b -> b) -> Token -> Token)
-> (forall r r'.
    (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r)
-> (forall r r'.
    (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r)
-> (forall u. (forall d. Data d => d -> u) -> Token -> [u])
-> (forall u. Int -> (forall d. Data d => d -> u) -> Token -> u)
-> (forall (m :: * -> *).
    Monad m =>
    (forall d. Data d => d -> m d) -> Token -> m Token)
-> (forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> Token -> m Token)
-> (forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> Token -> m Token)
-> Data Token
Token -> Constr
Token -> DataType
(forall b. Data b => b -> b) -> Token -> Token
forall a.
Typeable a =>
(forall (c :: * -> *).
 (forall d b. Data d => c (d -> b) -> d -> c b)
 -> (forall g. g -> c g) -> a -> c a)
-> (forall (c :: * -> *).
    (forall b r. Data b => c (b -> r) -> c r)
    -> (forall r. r -> c r) -> Constr -> c a)
-> (a -> Constr)
-> (a -> DataType)
-> (forall (t :: * -> *) (c :: * -> *).
    Typeable t =>
    (forall d. Data d => c (t d)) -> Maybe (c a))
-> (forall (t :: * -> * -> *) (c :: * -> *).
    Typeable t =>
    (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c a))
-> ((forall b. Data b => b -> b) -> a -> a)
-> (forall r r'.
    (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> a -> r)
-> (forall r r'.
    (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> a -> r)
-> (forall u. (forall d. Data d => d -> u) -> a -> [u])
-> (forall u. Int -> (forall d. Data d => d -> u) -> a -> u)
-> (forall (m :: * -> *).
    Monad m =>
    (forall d. Data d => d -> m d) -> a -> m a)
-> (forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> a -> m a)
-> (forall (m :: * -> *).
    MonadPlus m =>
    (forall d. Data d => d -> m d) -> a -> m a)
-> Data a
forall u. Int -> (forall d. Data d => d -> u) -> Token -> u
forall u. (forall d. Data d => d -> u) -> Token -> [u]
forall r r'.
(r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r
forall r r'.
(r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r
forall (m :: * -> *).
Monad m =>
(forall d. Data d => d -> m d) -> Token -> m Token
forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> Token -> m Token
forall (c :: * -> *).
(forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c Token
forall (c :: * -> *).
(forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> Token -> c Token
forall (t :: * -> *) (c :: * -> *).
Typeable t =>
(forall d. Data d => c (t d)) -> Maybe (c Token)
forall (t :: * -> * -> *) (c :: * -> *).
Typeable t =>
(forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Token)
$cgfoldl :: forall (c :: * -> *).
(forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> Token -> c Token
gfoldl :: forall (c :: * -> *).
(forall d b. Data d => c (d -> b) -> d -> c b)
-> (forall g. g -> c g) -> Token -> c Token
$cgunfold :: forall (c :: * -> *).
(forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c Token
gunfold :: forall (c :: * -> *).
(forall b r. Data b => c (b -> r) -> c r)
-> (forall r. r -> c r) -> Constr -> c Token
$ctoConstr :: Token -> Constr
toConstr :: Token -> Constr
$cdataTypeOf :: Token -> DataType
dataTypeOf :: Token -> DataType
$cdataCast1 :: forall (t :: * -> *) (c :: * -> *).
Typeable t =>
(forall d. Data d => c (t d)) -> Maybe (c Token)
dataCast1 :: forall (t :: * -> *) (c :: * -> *).
Typeable t =>
(forall d. Data d => c (t d)) -> Maybe (c Token)
$cdataCast2 :: forall (t :: * -> * -> *) (c :: * -> *).
Typeable t =>
(forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Token)
dataCast2 :: forall (t :: * -> * -> *) (c :: * -> *).
Typeable t =>
(forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Token)
$cgmapT :: (forall b. Data b => b -> b) -> Token -> Token
gmapT :: (forall b. Data b => b -> b) -> Token -> Token
$cgmapQl :: forall r r'.
(r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r
gmapQl :: forall r r'.
(r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r
$cgmapQr :: forall r r'.
(r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r
gmapQr :: forall r r'.
(r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r
$cgmapQ :: forall u. (forall d. Data d => d -> u) -> Token -> [u]
gmapQ :: forall u. (forall d. Data d => d -> u) -> Token -> [u]
$cgmapQi :: forall u. Int -> (forall d. Data d => d -> u) -> Token -> u
gmapQi :: forall u. Int -> (forall d. Data d => d -> u) -> Token -> u
$cgmapM :: forall (m :: * -> *).
Monad m =>
(forall d. Data d => d -> m d) -> Token -> m Token
gmapM :: forall (m :: * -> *).
Monad m =>
(forall d. Data d => d -> m d) -> Token -> m Token
$cgmapMp :: forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> Token -> m Token
gmapMp :: forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> Token -> m Token
$cgmapMo :: forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> Token -> m Token
gmapMo :: forall (m :: * -> *).
MonadPlus m =>
(forall d. Data d => d -> m d) -> Token -> m Token
Data)

instance Span Token where
  getSpan :: Token -> SrcSpan
getSpan = Token -> SrcSpan
token_span 
   
-- | Produce a string from a token containing detailed information. Mainly intended for debugging. 
debugTokenString :: Token -> String
debugTokenString :: Token -> String
debugTokenString Token
token =
   Doc -> String
render (String -> Doc
text (Constr -> String
forall a. Show a => a -> String
show (Constr -> String) -> Constr -> String
forall a b. (a -> b) -> a -> b
$ Token -> Constr
forall a. Data a => a -> Constr
toConstr Token
token) Doc -> Doc -> Doc
<+> SrcSpan -> Doc
forall a. Pretty a => a -> Doc
pretty (Token -> SrcSpan
token_span Token
token) Doc -> Doc -> Doc
<+>
          if Token -> Bool
hasLiteral Token
token then String -> Doc
text (Token -> String
token_literal Token
token) else Doc
empty)

-- | Test if a token contains its literal source text.
hasLiteral :: Token -> Bool
hasLiteral :: Token -> Bool
hasLiteral Token
token =
   case Token
token of
      CommentToken {}       -> Bool
True 
      IdentifierToken {}    -> Bool
True 
      StringToken {}        -> Bool
True 
      ByteStringToken {}    -> Bool
True
      UnicodeStringToken {} -> Bool
True
      IntegerToken {}       -> Bool
True 
      LongIntegerToken {}   -> Bool
True 
      FloatToken {}         -> Bool
True 
      ImaginaryToken  {}    -> Bool
True 
      Token
other                 -> Bool
False

-- | Classification of tokens
data TokenClass
   = Comment
   | Number
   | Identifier
   | Punctuation
   | Bracket
   | Layout 
   | Keyword
   | String
   | Operator
   | Assignment
   deriving (Int -> TokenClass -> ShowS
[TokenClass] -> ShowS
TokenClass -> String
(Int -> TokenClass -> ShowS)
-> (TokenClass -> String)
-> ([TokenClass] -> ShowS)
-> Show TokenClass
forall a.
(Int -> a -> ShowS) -> (a -> String) -> ([a] -> ShowS) -> Show a
$cshowsPrec :: Int -> TokenClass -> ShowS
showsPrec :: Int -> TokenClass -> ShowS
$cshow :: TokenClass -> String
show :: TokenClass -> String
$cshowList :: [TokenClass] -> ShowS
showList :: [TokenClass] -> ShowS
Show, TokenClass -> TokenClass -> Bool
(TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> Bool) -> Eq TokenClass
forall a. (a -> a -> Bool) -> (a -> a -> Bool) -> Eq a
$c== :: TokenClass -> TokenClass -> Bool
== :: TokenClass -> TokenClass -> Bool
$c/= :: TokenClass -> TokenClass -> Bool
/= :: TokenClass -> TokenClass -> Bool
Eq, Eq TokenClass
Eq TokenClass =>
(TokenClass -> TokenClass -> Ordering)
-> (TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> Bool)
-> (TokenClass -> TokenClass -> TokenClass)
-> (TokenClass -> TokenClass -> TokenClass)
-> Ord TokenClass
TokenClass -> TokenClass -> Bool
TokenClass -> TokenClass -> Ordering
TokenClass -> TokenClass -> TokenClass
forall a.
Eq a =>
(a -> a -> Ordering)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> Bool)
-> (a -> a -> a)
-> (a -> a -> a)
-> Ord a
$ccompare :: TokenClass -> TokenClass -> Ordering
compare :: TokenClass -> TokenClass -> Ordering
$c< :: TokenClass -> TokenClass -> Bool
< :: TokenClass -> TokenClass -> Bool
$c<= :: TokenClass -> TokenClass -> Bool
<= :: TokenClass -> TokenClass -> Bool
$c> :: TokenClass -> TokenClass -> Bool
> :: TokenClass -> TokenClass -> Bool
$c>= :: TokenClass -> TokenClass -> Bool
>= :: TokenClass -> TokenClass -> Bool
$cmax :: TokenClass -> TokenClass -> TokenClass
max :: TokenClass -> TokenClass -> TokenClass
$cmin :: TokenClass -> TokenClass -> TokenClass
min :: TokenClass -> TokenClass -> TokenClass
Ord)

classifyToken :: Token -> TokenClass 
classifyToken :: Token -> TokenClass
classifyToken Token
token = 
   case Token
token of
      IndentToken {} -> TokenClass
Layout 
      DedentToken {} -> TokenClass
Layout 
      NewlineToken {} -> TokenClass
Layout 
      CommentToken {} -> TokenClass
Comment 
      IdentifierToken {} -> TokenClass
Identifier 
      StringToken {} -> TokenClass
String 
      ByteStringToken {} -> TokenClass
String
      UnicodeStringToken {} -> TokenClass
String
      IntegerToken {} -> TokenClass
Number 
      LongIntegerToken {} -> TokenClass
Number 
      FloatToken {} -> TokenClass
Number 
      ImaginaryToken {} -> TokenClass
Number 
      DefToken {} -> TokenClass
Keyword 
      WhileToken {} -> TokenClass
Keyword 
      IfToken {} ->  TokenClass
Keyword
      TrueToken {} -> TokenClass
Keyword 
      FalseToken {} -> TokenClass
Keyword 
      ReturnToken {} -> TokenClass
Keyword 
      TryToken {} -> TokenClass
Keyword 
      ExceptToken {} -> TokenClass
Keyword 
      RaiseToken {} -> TokenClass
Keyword 
      InToken {} -> TokenClass
Keyword 
      IsToken {} -> TokenClass
Keyword 
      LambdaToken {} -> TokenClass
Keyword 
      ClassToken {} -> TokenClass
Keyword 
      FinallyToken {} -> TokenClass
Keyword 
      NoneToken {} -> TokenClass
Keyword 
      ForToken {} -> TokenClass
Keyword 
      FromToken {} -> TokenClass
Keyword 
      GlobalToken {} -> TokenClass
Keyword 
      WithToken {} -> TokenClass
Keyword 
      AsToken {} -> TokenClass
Keyword 
      ElifToken {} -> TokenClass
Keyword 
      YieldToken {} -> TokenClass
Keyword 
      AsyncToken {} -> TokenClass
Keyword
      AwaitToken {} -> TokenClass
Keyword
      AssertToken {} -> TokenClass
Keyword 
      ImportToken {} -> TokenClass
Keyword 
      PassToken {} -> TokenClass
Keyword 
      BreakToken {} -> TokenClass
Keyword 
      ContinueToken {} -> TokenClass
Keyword 
      DeleteToken {} -> TokenClass
Keyword
      ElseToken {} -> TokenClass
Keyword 
      NotToken {} -> TokenClass
Keyword 
      AndToken {} -> TokenClass
Keyword 
      OrToken {} -> TokenClass
Keyword 
      NonLocalToken {} -> TokenClass
Keyword 
      PrintToken {} -> TokenClass
Keyword 
      ExecToken {} -> TokenClass
Keyword 
      AtToken {} -> TokenClass
Keyword 
      LeftRoundBracketToken {} -> TokenClass
Bracket 
      RightRoundBracketToken {} -> TokenClass
Bracket 
      LeftSquareBracketToken {} -> TokenClass
Bracket 
      RightSquareBracketToken {} -> TokenClass
Bracket 
      LeftBraceToken {} -> TokenClass
Bracket 
      RightBraceToken {} -> TokenClass
Bracket 
      DotToken {} -> TokenClass
Operator 
      CommaToken {} -> TokenClass
Punctuation 
      SemiColonToken {} -> TokenClass
Punctuation 
      ColonToken {} -> TokenClass
Punctuation 
      EllipsisToken {} -> TokenClass
Keyword  -- What kind of thing is an ellipsis?
      RightArrowToken {} -> TokenClass
Punctuation 
      AssignToken {} -> TokenClass
Assignment 
      PlusAssignToken {} -> TokenClass
Assignment 
      MinusAssignToken {} -> TokenClass
Assignment  
      MultAssignToken {} -> TokenClass
Assignment 
      DivAssignToken {} -> TokenClass
Assignment 
      ModAssignToken {} -> TokenClass
Assignment  
      PowAssignToken {} -> TokenClass
Assignment 
      BinAndAssignToken {} -> TokenClass
Assignment 
      BinOrAssignToken {} -> TokenClass
Assignment 
      BinXorAssignToken {} -> TokenClass
Assignment 
      LeftShiftAssignToken {} -> TokenClass
Assignment 
      RightShiftAssignToken {} -> TokenClass
Assignment 
      FloorDivAssignToken {} -> TokenClass
Assignment 
      MatrixMultAssignToken {} -> TokenClass
Assignment
      BackQuoteToken {} -> TokenClass
Punctuation 
      PlusToken {} -> TokenClass
Operator 
      MinusToken {} -> TokenClass
Operator 
      MultToken {} -> TokenClass
Operator 
      DivToken {} -> TokenClass
Operator 
      GreaterThanToken {} -> TokenClass
Operator 
      LessThanToken {} -> TokenClass
Operator 
      EqualityToken {} -> TokenClass
Operator 
      GreaterThanEqualsToken {} -> TokenClass
Operator 
      LessThanEqualsToken {} -> TokenClass
Operator 
      ExponentToken {} -> TokenClass
Operator 
      BinaryOrToken {} -> TokenClass
Operator 
      XorToken {} -> TokenClass
Operator 
      BinaryAndToken {} -> TokenClass
Operator 
      ShiftLeftToken {} -> TokenClass
Operator 
      ShiftRightToken {} -> TokenClass
Operator 
      ModuloToken {} -> TokenClass
Operator 
      FloorDivToken {} -> TokenClass
Operator 
      TildeToken {} -> TokenClass
Operator 
      NotEqualsToken {} -> TokenClass
Operator 
      NotEqualsV2Token {} -> TokenClass
Operator 
      LineJoinToken {} -> TokenClass
Layout 
      EOFToken {} -> TokenClass
Layout  -- maybe a spurious classification.

-- | Produce a string from a token which is suitable for printing as Python concrete syntax.
-- /Invisible/ tokens yield an empty string.
tokenString :: Token -> String
tokenString :: Token -> String
tokenString Token
token = 
   case Token
token of
      IndentToken {} -> String
"" 
      DedentToken {} -> String
""
      NewlineToken {} -> String
"" 
      CommentToken {} -> Token -> String
token_literal Token
token
      IdentifierToken {} -> Token -> String
token_literal Token
token
      StringToken {} -> Token -> String
token_literal Token
token
      ByteStringToken {} -> Token -> String
token_literal Token
token
      UnicodeStringToken {} -> Token -> String
token_literal Token
token
      IntegerToken {} -> Token -> String
token_literal Token
token
      LongIntegerToken {} -> Token -> String
token_literal Token
token
      FloatToken {} -> Token -> String
token_literal Token
token
      ImaginaryToken {} -> Token -> String
token_literal Token
token
      DefToken {} -> String
"def"
      WhileToken {} -> String
"while"
      IfToken {} -> String
"if"
      TrueToken {} -> String
"True"
      FalseToken {} -> String
"False"
      ReturnToken {} -> String
"return"
      TryToken {} -> String
"try"
      ExceptToken {} -> String
"except"
      RaiseToken {} -> String
"raise"
      InToken {} -> String
"in"
      IsToken {} -> String
"is"
      LambdaToken {} -> String
"lambda"
      ClassToken {} -> String
"class"
      FinallyToken {} -> String
"finally"
      NoneToken {} -> String
"None"
      ForToken {} -> String
"for"
      FromToken {} -> String
"from"
      GlobalToken {} -> String
"global"
      WithToken {} -> String
"with"
      AsToken {} -> String
"as"
      ElifToken {} -> String
"elif" 
      YieldToken {} -> String
"yield"
      AsyncToken {} -> String
"async"
      AwaitToken {} -> String
"await"
      AssertToken {} -> String
"assert" 
      ImportToken {} -> String
"import"
      PassToken {} -> String
"pass" 
      BreakToken {} -> String
"break" 
      ContinueToken {} -> String
"continue"
      DeleteToken {} -> String
"delete"
      ElseToken {} -> String
"else"
      NotToken {} -> String
"not"
      AndToken {} -> String
"and"
      OrToken {} -> String
"or"
      NonLocalToken {} -> String
"nonlocal"
      PrintToken {} -> String
"print"
      ExecToken {} -> String
"exec"
      AtToken {} -> String
"at"
      LeftRoundBracketToken {} -> String
"("
      RightRoundBracketToken {} -> String
")"
      LeftSquareBracketToken {} -> String
"["
      RightSquareBracketToken {} -> String
"]"
      LeftBraceToken {} -> String
"{"
      RightBraceToken {} -> String
"}"
      DotToken {} -> String
"."
      CommaToken {} -> String
","
      SemiColonToken {} -> String
";"
      ColonToken {} -> String
":"
      EllipsisToken {} -> String
"..."
      RightArrowToken {} -> String
"->"
      AssignToken {} -> String
"="
      PlusAssignToken {} -> String
"+="
      MinusAssignToken {} -> String
"-="
      MultAssignToken {} -> String
"*="
      DivAssignToken {} -> String
"/="
      ModAssignToken {} -> String
"%="
      PowAssignToken {} -> String
"**="
      BinAndAssignToken {} -> String
"&="
      BinOrAssignToken {} -> String
"|="
      BinXorAssignToken {} -> String
"^="
      LeftShiftAssignToken {} -> String
"<<="
      RightShiftAssignToken {} -> String
">>="
      FloorDivAssignToken {} -> String
"//=" 
      MatrixMultAssignToken {} -> String
"@="
      BackQuoteToken {} -> String
"`"
      PlusToken {} -> String
"+"
      MinusToken {} -> String
"-"
      MultToken {} -> String
"*"
      DivToken {} -> String
"/"
      GreaterThanToken {} -> String
">"
      LessThanToken {} -> String
"<"
      EqualityToken {} -> String
"=="
      GreaterThanEqualsToken {} -> String
">="
      LessThanEqualsToken {} -> String
"<="
      ExponentToken {} -> String
"**"
      BinaryOrToken {} -> String
"|"
      XorToken {} -> String
"^"
      BinaryAndToken {} -> String
"&"
      ShiftLeftToken {} -> String
"<<"
      ShiftRightToken {} -> String
">>"
      ModuloToken {} -> String
"%"
      FloorDivToken {} -> String
"//"
      TildeToken {} -> String
"~"
      NotEqualsToken {} -> String
"!="
      NotEqualsV2Token {} -> String
"<>"
      LineJoinToken {} -> String
"\\"
      EOFToken {} -> String
""