1 #! /usr/bin/env python

     2 

     3 """Token constants (from "token.h")."""

     4 

     5 #  Taken from Python (r53757) and modified to include some tokens

     6 #   originally monkeypatched in by pgen2.tokenize

     7 

     8 #--start constants--

     9 ENDMARKER = 0

    10 NAME = 1

    11 NUMBER = 2

    12 STRING = 3

    13 NEWLINE = 4

    14 INDENT = 5

    15 DEDENT = 6

    16 LPAR = 7

    17 RPAR = 8

    18 LSQB = 9

    19 RSQB = 10

    20 COLON = 11

    21 COMMA = 12

    22 SEMI = 13

    23 PLUS = 14

    24 MINUS = 15

    25 STAR = 16

    26 SLASH = 17

    27 VBAR = 18

    28 AMPER = 19

    29 LESS = 20

    30 GREATER = 21

    31 EQUAL = 22

    32 DOT = 23

    33 PERCENT = 24

    34 BACKQUOTE = 25

    35 LBRACE = 26

    36 RBRACE = 27

    37 EQEQUAL = 28

    38 NOTEQUAL = 29

    39 LESSEQUAL = 30

    40 GREATEREQUAL = 31

    41 TILDE = 32

    42 CIRCUMFLEX = 33

    43 LEFTSHIFT = 34

    44 RIGHTSHIFT = 35

    45 DOUBLESTAR = 36

    46 PLUSEQUAL = 37

    47 MINEQUAL = 38

    48 STAREQUAL = 39

    49 SLASHEQUAL = 40

    50 PERCENTEQUAL = 41

    51 AMPEREQUAL = 42

    52 VBAREQUAL = 43

    53 CIRCUMFLEXEQUAL = 44

    54 LEFTSHIFTEQUAL = 45

    55 RIGHTSHIFTEQUAL = 46

    56 DOUBLESTAREQUAL = 47

    57 DOUBLESLASH = 48

    58 DOUBLESLASHEQUAL = 49

    59 AT = 50

    60 OP = 51

    61 COMMENT = 52

    62 NL = 53

    63 RARROW = 54

    64 ERRORTOKEN = 55

    65 N_TOKENS = 56

    66 NT_OFFSET = 256

    67 #--end constants--

    68 

    69 tok_name = {}

    70 for _name, _value in globals().items():

    71     if type(_value) is type(0):

    72         tok_name[_value] = _name

    73 

    74 

    75 def ISTERMINAL(x):

    76     return x < NT_OFFSET

    77 

    78 def ISNONTERMINAL(x):

    79     return x >= NT_OFFSET

    80 

    81 def ISEOF(x):

    82     return x == ENDMARKER