from codecs import lookup, BOM_UTF8

import collections

from io import TextIOWrapper

from itertools import chain

import itertools as _itertools

import re

import sys

startline = token[0] in (NEWLINE, NL)

prevstring = False

for tok in chain([token], iterable):

for tok in _itertools.chain([token], iterable):

toknum, tokval = tok[:2]

if toknum == ENCODING:

self.encoding = tokval

The first token sequence will always be an ENCODING token

which tells you which encoding was used to decode the bytes stream.

"""

# This import is here to avoid problems when the itertools module is not

# built yet and tokenize is imported.

from itertools import chain, repeat

encoding, consumed = detect_encoding(readline)

rl_gen = iter(readline, b"")

empty = repeat(b"")

return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding)

empty = _itertools.repeat(b"")

rl_gen = _itertools.chain(consumed, iter(readline, b""), empty)

return _tokenize(rl_gen.__next__, encoding)

def _tokenize(readline, encoding):

# BOM will already have been stripped.

encoding = "utf-8"

yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')

while True: # loop over lines in stream

while True: # loop over lines in stream

try:

line = readline()

except StopIteration:

continue

token, initial = line[start:end], line[start]

if (initial in numchars or # ordinary number

if (initial in numchars or # ordinary number

(initial == '.' and token != '.' and token != '...')):

yield TokenInfo(NUMBER, token, spos, epos, line)

elif initial in '\r\n':

# Helper error handling routines

def perror(message):

print(message, file=sys.stderr)

sys.stderr.write(message)

sys.stderr.write('\n')

def error(message, filename=None, location=None):

if location: