Tokenizer
Tokenizer.tokens
Tokenizer.MAX_ENTITY_LENGTH
Tokenizer.tokenize()
Tokenizer.tokenize_iter()
Tokenizer.return_reset_buffer()
Tokenizer.advance()
Tokenizer.is_at_end()
Tokenizer.peek_is()
Tokenizer.peek()
Tokenizer.peek_two_is()
Tokenizer.peek_two()
Bases: object
object