Language.Python.Common.Token

language-python-0.5.4: Parsing and pretty printing of Python code.

Copyright | (c) 2009 Bernie Pope
License | BSD-style
Maintainer | bjpop@csse.unimelb.edu.au
Stability | experimental
Portability | ghc
Safe Haskell | Safe
Language | Haskell98

Language.Python.Common.Token

Contents

- The tokens
- String conversion
- Classification

Description

Lexical tokens for the Python lexer. Contains the superset of tokens from version 2 and version 3 of Python (they are mostly the same).

Synopsis

- data Token
  - = IndentToken {
    - token_span :: !SrcSpan
    }
  - | DedentToken {
    - token_span :: !SrcSpan
    }
  - | NewlineToken {
    - token_span :: !SrcSpan
    }
  - | LineJoinToken {
    - token_span :: !SrcSpan
    }
  - | CommentToken {
    - token_span :: !SrcSpan
    - token_literal :: !String
    }
  - | IdentifierToken {
    - token_span :: !SrcSpan
    - token_literal :: !String
    }
  - | StringToken {
    - token_span :: !SrcSpan
    - token_literal :: !String
    }
  - | ByteStringToken {
    - token_span :: !SrcSpan
    - token_literal :: !String
    }
  - | UnicodeStringToken {
    - token_span :: !SrcSpan
    - token_literal :: !String
    }
  - | IntegerToken {
    - token_span :: !SrcSpan
    - token_literal :: !String
    - token_integer :: !Integer
    }
  - | LongIntegerToken {
    - token_span :: !SrcSpan
    - token_literal :: !String
    - token_integer :: !Integer
    }
  - | FloatToken {
    - token_span :: !SrcSpan
    - token_literal :: !String
    - token_double :: !Double
    }
  - | ImaginaryToken {
    - token_span :: !SrcSpan
    - token_literal :: !String
    - token_double :: !Double
    }
  - | DefToken {
    - token_span :: !SrcSpan
    }
  - | WhileToken {
    - token_span :: !SrcSpan
    }
  - | IfToken {
    - token_span :: !SrcSpan
    }
  - | TrueToken {
    - token_span :: !SrcSpan
    }
  - | FalseToken {
    - token_span :: !SrcSpan
    }
  - | ReturnToken {
    - token_span :: !SrcSpan
    }
  - | TryToken {
    - token_span :: !SrcSpan
    }
  - | ExceptToken {
    - token_span :: !SrcSpan
    }
  - | RaiseToken {
    - token_span :: !SrcSpan
    }
  - | InToken {
    - token_span :: !SrcSpan
    }
  - | IsToken {
    - token_span :: !SrcSpan
    }
  - | LambdaToken {
    - token_span :: !SrcSpan
    }
  - | ClassToken {
    - token_span :: !SrcSpan
    }
  - | FinallyToken {
    - token_span :: !SrcSpan
    }
  - | NoneToken {
    - token_span :: !SrcSpan
    }
  - | ForToken {
    - token_span :: !SrcSpan
    }
  - | FromToken {
    - token_span :: !SrcSpan
    }
  - | GlobalToken {
    - token_span :: !SrcSpan
    }
  - | WithToken {
    - token_span :: !SrcSpan
    }
  - | AsToken {
    - token_span :: !SrcSpan
    }
  - | ElifToken {
    - token_span :: !SrcSpan
    }
  - | YieldToken {
    - token_span :: !SrcSpan
    }
  - | AssertToken {
    - token_span :: !SrcSpan
    }
  - | ImportToken {
    - token_span :: !SrcSpan
    }
  - | PassToken {
    - token_span :: !SrcSpan
    }
  - | BreakToken {
    - token_span :: !SrcSpan
    }
  - | ContinueToken {
    - token_span :: !SrcSpan
    }
  - | DeleteToken {
    - token_span :: !SrcSpan
    }
  - | ElseToken {
    - token_span :: !SrcSpan
    }
  - | NotToken {
    - token_span :: !SrcSpan
    }
  - | AndToken {
    - token_span :: !SrcSpan
    }
  - | OrToken {
    - token_span :: !SrcSpan
    }
  - | NonLocalToken {
    - token_span :: !SrcSpan
    }
  - | PrintToken {
    - token_span :: !SrcSpan
    }
  - | ExecToken {
    - token_span :: !SrcSpan
    }
  - | AtToken {
    - token_span :: !SrcSpan
    }
  - | LeftRoundBracketToken {
    - token_span :: !SrcSpan
    }
  - | RightRoundBracketToken {
    - token_span :: !SrcSpan
    }
  - | LeftSquareBracketToken {
    - token_span :: !SrcSpan
    }
  - | RightSquareBracketToken {
    - token_span :: !SrcSpan
    }
  - | LeftBraceToken {
    - token_span :: !SrcSpan
    }
  - | RightBraceToken {
    - token_span :: !SrcSpan
    }
  - | DotToken {
    - token_span :: !SrcSpan
    }
  - | CommaToken {
    - token_span :: !SrcSpan
    }
  - | SemiColonToken {
    - token_span :: !SrcSpan
    }
  - | ColonToken {
    - token_span :: !SrcSpan
    }
  - | EllipsisToken {
    - token_span :: !SrcSpan
    }
  - | RightArrowToken {
    - token_span :: !SrcSpan
    }
  - | AssignToken {
    - token_span :: !SrcSpan
    }
  - | PlusAssignToken {
    - token_span :: !SrcSpan
    }
  - | MinusAssignToken {
    - token_span :: !SrcSpan
    }
  - | MultAssignToken {
    - token_span :: !SrcSpan
    }
  - | DivAssignToken {
    - token_span :: !SrcSpan
    }
  - | ModAssignToken {
    - token_span :: !SrcSpan
    }
  - | PowAssignToken {
    - token_span :: !SrcSpan
    }
  - | BinAndAssignToken {
    - token_span :: !SrcSpan
    }
  - | BinOrAssignToken {
    - token_span :: !SrcSpan
    }
  - | BinXorAssignToken {
    - token_span :: !SrcSpan
    }
  - | LeftShiftAssignToken {
    - token_span :: !SrcSpan
    }
  - | RightShiftAssignToken {
    - token_span :: !SrcSpan
    }
  - | FloorDivAssignToken {
    - token_span :: !SrcSpan
    }
  - | BackQuoteToken {
    - token_span :: !SrcSpan
    }
  - | PlusToken {
    - token_span :: !SrcSpan
    }
  - | MinusToken {
    - token_span :: !SrcSpan
    }
  - | MultToken {
    - token_span :: !SrcSpan
    }
  - | DivToken {
    - token_span :: !SrcSpan
    }
  - | GreaterThanToken {
    - token_span :: !SrcSpan
    }
  - | LessThanToken {
    - token_span :: !SrcSpan
    }
  - | EqualityToken {
    - token_span :: !SrcSpan
    }
  - | GreaterThanEqualsToken {
    - token_span :: !SrcSpan
    }
  - | LessThanEqualsToken {
    - token_span :: !SrcSpan
    }
  - | ExponentToken {
    - token_span :: !SrcSpan
    }
  - | BinaryOrToken {
    - token_span :: !SrcSpan
    }
  - | XorToken {
    - token_span :: !SrcSpan
    }
  - | BinaryAndToken {
    - token_span :: !SrcSpan
    }
  - | ShiftLeftToken {
    - token_span :: !SrcSpan
    }
  - | ShiftRightToken {
    - token_span :: !SrcSpan
    }
  - | ModuloToken {
    - token_span :: !SrcSpan
    }
  - | FloorDivToken {
    - token_span :: !SrcSpan
    }
  - | TildeToken {
    - token_span :: !SrcSpan
    }
  - | NotEqualsToken {
    - token_span :: !SrcSpan
    }
  - | NotEqualsV2Token {
    - token_span :: !SrcSpan
    }
  - | EOFToken {
    - token_span :: !SrcSpan
    }
- debugTokenString :: Token -> String
- tokenString :: Token -> String
- hasLiteral :: Token -> Bool
- data TokenClass
  - = Comment
  - | Number
  - | Identifier
  - | Punctuation
  - | Bracket
  - | Layout
  - | Keyword
  - | String
  - | Operator
  - | Assignment
- classifyToken :: Token -> TokenClass

# The tokens

data Token #

Lexical tokens.

Constructors

IndentToken | Indentation: increase.
Fields token_span :: !SrcSpan
DedentToken | Indentation: decrease.
Fields token_span :: !SrcSpan
NewlineToken | Newline.
Fields token_span :: !SrcSpan
LineJoinToken | Line join (backslash at end of line).
Fields token_span :: !SrcSpan
CommentToken | Single line comment.
Fields token_span :: !SrcSpan token_literal :: !String
IdentifierToken | Identifier.
Fields token_span :: !SrcSpan token_literal :: !String
StringToken | Literal: string.
Fields token_span :: !SrcSpan token_literal :: !String
ByteStringToken | Literal: byte string.
Fields token_span :: !SrcSpan token_literal :: !String
UnicodeStringToken | Literal: unicode string, version 2 only.
Fields token_span :: !SrcSpan token_literal :: !String
IntegerToken | Literal: integer.
Fields token_span :: !SrcSpan token_literal :: !String token_integer :: !Integer
LongIntegerToken | Literal: long integer. Version 2 only.
Fields token_span :: !SrcSpan token_literal :: !String token_integer :: !Integer
FloatToken | Literal: floating point.
Fields token_span :: !SrcSpan token_literal :: !String token_double :: !Double
ImaginaryToken | Literal: imaginary number.
Fields token_span :: !SrcSpan token_literal :: !String token_double :: !Double
DefToken | Keyword: 'def'.
Fields token_span :: !SrcSpan
WhileToken | Keyword: 'while'.
Fields token_span :: !SrcSpan
IfToken | Keyword: 'if'.
Fields token_span :: !SrcSpan
TrueToken | Keyword: 'True'.
Fields token_span :: !SrcSpan
FalseToken | Keyword: 'False'.
Fields token_span :: !SrcSpan
ReturnToken | Keyword: 'Return'.
Fields token_span :: !SrcSpan
TryToken | Keyword: 'try'.
Fields token_span :: !SrcSpan
ExceptToken | Keyword: 'except'.
Fields token_span :: !SrcSpan
RaiseToken | Keyword: 'raise'.
Fields token_span :: !SrcSpan
InToken | Keyword: 'in'.
Fields token_span :: !SrcSpan
IsToken | Keyword: 'is'.
Fields token_span :: !SrcSpan
LambdaToken | Keyword: 'lambda'.
Fields token_span :: !SrcSpan
ClassToken | Keyword: 'class'.
Fields token_span :: !SrcSpan
FinallyToken | Keyword: 'finally'.
Fields token_span :: !SrcSpan
NoneToken | Keyword: 'None'.
Fields token_span :: !SrcSpan
ForToken | Keyword: 'for'.
Fields token_span :: !SrcSpan
FromToken | Keyword: 'from'.
Fields token_span :: !SrcSpan
GlobalToken | Keyword: 'global'.
Fields token_span :: !SrcSpan
WithToken | Keyword: 'with'.
Fields token_span :: !SrcSpan
AsToken | Keyword: 'as'.
Fields token_span :: !SrcSpan
ElifToken | Keyword: 'elif'.
Fields token_span :: !SrcSpan
YieldToken | Keyword: 'yield'.
Fields token_span :: !SrcSpan
AssertToken | Keyword: 'assert'.
Fields token_span :: !SrcSpan
ImportToken | Keyword: 'import'.
Fields token_span :: !SrcSpan
PassToken | Keyword: 'pass'.
Fields token_span :: !SrcSpan
BreakToken | Keyword: 'break'.
Fields token_span :: !SrcSpan
ContinueToken | Keyword: 'continue'.
Fields token_span :: !SrcSpan
DeleteToken | Keyword: 'del'.
Fields token_span :: !SrcSpan
ElseToken | Keyword: 'else'.
Fields token_span :: !SrcSpan
NotToken | Keyword: 'not'.
Fields token_span :: !SrcSpan
AndToken | Keyword: boolean conjunction 'and'.
Fields token_span :: !SrcSpan
OrToken | Keyword: boolean disjunction 'or'. Version 3.x only:
Fields token_span :: !SrcSpan
NonLocalToken | Keyword: 'nonlocal' (Python 3.x only) Version 2.x only:
Fields token_span :: !SrcSpan
PrintToken | Keyword: 'print'. (Python 2.x only)
Fields token_span :: !SrcSpan
ExecToken | Keyword: 'exec'. (Python 2.x only)
Fields token_span :: !SrcSpan
AtToken | Delimiter: at sign '@'.
Fields token_span :: !SrcSpan
LeftRoundBracketToken | Delimiter: left round bracket '('.
Fields token_span :: !SrcSpan
RightRoundBracketToken | Delimiter: right round bracket ')'.
Fields token_span :: !SrcSpan
LeftSquareBracketToken | Delimiter: left square bracket '['.
Fields token_span :: !SrcSpan
RightSquareBracketToken | Delimiter: right square bracket ']'.
Fields token_span :: !SrcSpan
LeftBraceToken | Delimiter: left curly bracket '{'.
Fields token_span :: !SrcSpan
RightBraceToken | Delimiter: right curly bracket '}'.
Fields token_span :: !SrcSpan
DotToken | Delimiter: dot (full stop) '.'.
Fields token_span :: !SrcSpan
CommaToken | Delimiter: comma ','.
Fields token_span :: !SrcSpan
SemiColonToken | Delimiter: semicolon ';'.
Fields token_span :: !SrcSpan
ColonToken | Delimiter: colon ':'.
Fields token_span :: !SrcSpan
EllipsisToken | Delimiter: ellipses (three dots) '...'.
Fields token_span :: !SrcSpan
RightArrowToken | Delimiter: right facing arrow '->'.
Fields token_span :: !SrcSpan
AssignToken | Delimiter: assignment '='.
Fields token_span :: !SrcSpan
PlusAssignToken | Delimiter: plus assignment '+='.
Fields token_span :: !SrcSpan
MinusAssignToken | Delimiter: minus assignment '-='.
Fields token_span :: !SrcSpan
MultAssignToken | Delimiter: multiply assignment '*='
Fields token_span :: !SrcSpan
DivAssignToken | Delimiter: divide assignment '/='.
Fields token_span :: !SrcSpan
ModAssignToken | Delimiter: modulus assignment '%='.
Fields token_span :: !SrcSpan
PowAssignToken | Delimiter: power assignment '**='.
Fields token_span :: !SrcSpan
BinAndAssignToken | Delimiter: binary-and assignment '&='.
Fields token_span :: !SrcSpan
BinOrAssignToken | Delimiter: binary-or assignment '|='.
Fields token_span :: !SrcSpan
BinXorAssignToken | Delimiter: binary-xor assignment '^='.
Fields token_span :: !SrcSpan
LeftShiftAssignToken | Delimiter: binary-left-shift assignment '<<='.
Fields token_span :: !SrcSpan
RightShiftAssignToken | Delimiter: binary-right-shift assignment '>>='.
Fields token_span :: !SrcSpan
FloorDivAssignToken | Delimiter: floor-divide assignment '//='.
Fields token_span :: !SrcSpan
BackQuoteToken | Delimiter: back quote character '`\'.
Fields token_span :: !SrcSpan
PlusToken | Operator: plus '+'.
Fields token_span :: !SrcSpan
MinusToken | Operator: minus: '-'.
Fields token_span :: !SrcSpan
MultToken | Operator: multiply '*'.
Fields token_span :: !SrcSpan
DivToken | Operator: divide '/'.
Fields token_span :: !SrcSpan
GreaterThanToken | Operator: greater-than '>'.
Fields token_span :: !SrcSpan
LessThanToken | Operator: less-than '<'.
Fields token_span :: !SrcSpan
EqualityToken | Operator: equals '=='.
Fields token_span :: !SrcSpan
GreaterThanEqualsToken | Operator: greater-than-or-equals '>='.
Fields token_span :: !SrcSpan
LessThanEqualsToken | Operator: less-than-or-equals '<='.
Fields token_span :: !SrcSpan
ExponentToken | Operator: exponential '**'.
Fields token_span :: !SrcSpan
BinaryOrToken | Operator: binary-or '|'.
Fields token_span :: !SrcSpan
XorToken | Operator: binary-xor '^'.
Fields token_span :: !SrcSpan
BinaryAndToken | Operator: binary-and '&'.
Fields token_span :: !SrcSpan
ShiftLeftToken | Operator: binary-shift-left '<<'.
Fields token_span :: !SrcSpan
ShiftRightToken | Operator: binary-shift-right '>>'.
Fields token_span :: !SrcSpan
ModuloToken | Operator: modulus '%'.
Fields token_span :: !SrcSpan
FloorDivToken | Operator: floor-divide '//'.
Fields token_span :: !SrcSpan
TildeToken | Operator: tilde '~'.
Fields token_span :: !SrcSpan
NotEqualsToken | Operator: not-equals '!='.
Fields token_span :: !SrcSpan
NotEqualsV2Token | Operator: not-equals '<>'. Version 2 only.
Fields token_span :: !SrcSpan
EOFToken | End of file
Fields token_span :: !SrcSpan

Instances

Eq Token # |
Methods (==) :: Token -> Token -> Bool # (/=) :: Token -> Token -> Bool #
Data Token # |
Methods gfoldl :: (forall d b. Data d => c (d -> b) -> d -> c b) -> (forall g. g -> c g) -> Token -> c Token # gunfold :: (forall b r. Data b => c (b -> r) -> c r) -> (forall r. r -> c r) -> Constr -> c Token # toConstr :: Token -> Constr # dataTypeOf :: Token -> DataType # dataCast1 :: Typeable (* -> *) t => (forall d. Data d => c (t d)) -> Maybe (c Token) # dataCast2 :: Typeable (* -> * -> *) t => (forall d e. (Data d, Data e) => c (t d e)) -> Maybe (c Token) # gmapT :: (forall b. Data b => b -> b) -> Token -> Token # gmapQl :: (r -> r' -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r # gmapQr :: (r' -> r -> r) -> r -> (forall d. Data d => d -> r') -> Token -> r # gmapQ :: (forall d. Data d => d -> u) -> Token -> [u] # gmapQi :: Int -> (forall d. Data d => d -> u) -> Token -> u # gmapM :: Monad m => (forall d. Data d => d -> m d) -> Token -> m Token # gmapMp :: MonadPlus m => (forall d. Data d => d -> m d) -> Token -> m Token # gmapMo :: MonadPlus m => (forall d. Data d => d -> m d) -> Token -> m Token #
Ord Token # |
Methods compare :: Token -> Token -> Ordering # (<) :: Token -> Token -> Bool # (<=) :: Token -> Token -> Bool # (>) :: Token -> Token -> Bool # (>=) :: Token -> Token -> Bool # max :: Token -> Token -> Token # min :: Token -> Token -> Token #
Show Token # |
Methods showsPrec :: Int -> Token -> ShowS # show :: Token -> String # showList :: [Token] -> ShowS #
Span Token # |
Methods getSpan :: Token -> SrcSpan #

# String conversion

debugTokenString :: Token -> String #

Produce a string from a token containing detailed information. Mainly intended for debugging.

tokenString :: Token -> String #

Produce a string from a token which is suitable for printing as Python concrete syntax. Invisible tokens yield an empty string.

# Classification

hasLiteral :: Token -> Bool #

Test if a token contains its literal source text.

data TokenClass #

Classification of tokens

Constructors

Comment |
Number |
Identifier |
Punctuation |
Bracket |
Layout |
Keyword |
String |
Operator |
Assignment |

Instances

Eq TokenClass # |
Methods (==) :: TokenClass -> TokenClass -> Bool # (/=) :: TokenClass -> TokenClass -> Bool #
Ord TokenClass # |
Methods compare :: TokenClass -> TokenClass -> Ordering # (<) :: TokenClass -> TokenClass -> Bool # (<=) :: TokenClass -> TokenClass -> Bool # (>) :: TokenClass -> TokenClass -> Bool # (>=) :: TokenClass -> TokenClass -> Bool # max :: TokenClass -> TokenClass -> TokenClass # min :: TokenClass -> TokenClass -> TokenClass #
Show TokenClass # |
Methods showsPrec :: Int -> TokenClass -> ShowS # show :: TokenClass -> String # showList :: [TokenClass] -> ShowS #

classifyToken :: Token -> TokenClass #