# Parent Module win32com.axdebug
# Functions def group(*choices) def printtoken(type, string, linenum, line, start, end) def tokenize(readline, tokeneater = printtoken)
# Variables int AMPER = 19 int BACKQUOTE = 25 string Bracket = '[][(){}]' int CIRCUMFLEX = 33 int COLON = 11 int COMMA = 12 int DEDENT = 6 int DOT = 23 int DOUBLESTAR = 36 string Decnumber = '[1-9][0-9]*[lL]?' string Double = '\\(^"\\|[^\\]"\\)' int ENDMARKER = 0 int EQEQUAL = 28 int EQUAL = 22 int ERRORTOKEN = 38 string Expfloat = '[0-9]+[eE][-+]?[0-9]+' string Exponent = '[eE][-+]?[0-9]+' string Floatnumber = '\\(\\([0-9]+\\.[0-9]*\\|\ ... string Funny = '\\(\\(\\+\\|\\-\\|\\*\\*\ ... int GREATER = 21 int GREATEREQUAL = 31 string Hexnumber = '0[xX][0-9a-fA-F]*[lL]?' int INDENT = 5 string Ignore = '[ \014\011]*\\([\\]\015?\ ... string Intnumber = '\\(0[xX][0-9a-fA-F]*[lL]? ... int LBRACE = 26 int LEFTSHIFT = 34 int LESS = 20 int LESSEQUAL = 30 int LPAR = 7 int LSQB = 9 int MINUS = 15 int NAME = 1 int NEWLINE = 4 int NOTEQUAL = 29 int NT_OFFSET = 256 int NUMBER = 2 int N_TOKENS = 39 string Name = '[a-zA-Z_][a-zA-Z0-9_]*' string Number = '\\(\\(\\([0-9]+\\.[0-9]*\ ... int OP = 37 string Octnumber = '0[0-7]*[lL]?' string Operator = '\\(\\+\\|\\-\\|\\*\\*\\|\ ... int PERCENT = 24 int PLUS = 14 string PlainToken = '\\([a-zA-Z_][a-zA-Z0-9_]* ... string Pointfloat = '\\([0-9]+\\.[0-9]*\\|\\.[ ... int RBRACE = 27 int RIGHTSHIFT = 35 int RPAR = 8 int RSQB = 10 int SEMI = 13 int SLASH = 17 int STAR = 16 int STRING = 3 string Single = "\\(^'\\|[^\\]'\\)" string Special = '\\([\\]?\015?\012\\|[:;., ... string String = '\\(\'\\([\\].\\|[^\'\\]\\ ... int TILDE = 32 string Tdouble = '\\(^"""\\|[^\\]"""\\)' string Token = '[ \014\011]*\\([\\]\015?\ ... string TokenError = 'TokenError' string Triple = '\\(\'\'\'\\|"""\\)' string Tsingle = "\\(^'''\\|[^\\]'''\\)" int VBAR = 18 dictionary endprogs = {"'''": <regex object at 1 ... string name = '__name__' list names = ['AMPER', 'BACKQUOTE', 'CI ... string number = 'token_v1_4' int ENDMARKER = 0 int RPAR = 8 dictionary tok_name = {0: 'ENDMARKER', 1: 'NAME' ... regex tokenprog = ...
This module compiles a regular expression that recognizes Python tokens
in individual lines of text. The regular expression handles everything
except indentation, continuations, and triple-quoted strings. The function
tokenize.tokenize()
takes care of these things for streams of text. It
accepts a file-like object and a function, uses the readline() method to scan
the file, and calls the function called once for each token found passing its
type, a string containing the token, the line number, the line, and the
starting and ending positions of the token within the line. It is designed to
match the working of the Python tokenizer exactly.
Copyright © Greg Stein, Mark Hammond, Curt Hagenlocher and a few other contributors
All Rights Reserved
Comments to author:
Generated: Sun Apr 20 1997