logo

oasis-root

Compiled tree of Oasis Linux based on own branch at <https://hacktivis.me/git/oasis/> git clone https://anongit.hacktivis.me/git/oasis-root.git

tokenize.py (25921B)


  1. """Tokenization help for Python programs.
  2. tokenize(readline) is a generator that breaks a stream of bytes into
  3. Python tokens. It decodes the bytes according to PEP-0263 for
  4. determining source file encoding.
  5. It accepts a readline-like method which is called repeatedly to get the
  6. next line of input (or b"" for EOF). It generates 5-tuples with these
  7. members:
  8. the token type (see token.py)
  9. the token (a string)
  10. the starting (row, column) indices of the token (a 2-tuple of ints)
  11. the ending (row, column) indices of the token (a 2-tuple of ints)
  12. the original line (string)
  13. It is designed to match the working of the Python tokenizer exactly, except
  14. that it produces COMMENT tokens for comments and gives type OP for all
  15. operators. Additionally, all token lists start with an ENCODING token
  16. which tells you which encoding was used to decode the bytes stream.
  17. """
  18. __author__ = 'Ka-Ping Yee <ping@lfw.org>'
  19. __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
  20. 'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
  21. 'Michael Foord')
  22. from builtins import open as _builtin_open
  23. from codecs import lookup, BOM_UTF8
  24. import collections
  25. import functools
  26. from io import TextIOWrapper
  27. import itertools as _itertools
  28. import re
  29. import sys
  30. from token import *
  31. from token import EXACT_TOKEN_TYPES
  32. cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII)
  33. blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII)
  34. import token
  35. __all__ = token.__all__ + ["tokenize", "generate_tokens", "detect_encoding",
  36. "untokenize", "TokenInfo"]
  37. del token
  38. class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
  39. def __repr__(self):
  40. annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
  41. return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
  42. self._replace(type=annotated_type))
  43. @property
  44. def exact_type(self):
  45. if self.type == OP and self.string in EXACT_TOKEN_TYPES:
  46. return EXACT_TOKEN_TYPES[self.string]
  47. else:
  48. return self.type
  49. def group(*choices): return '(' + '|'.join(choices) + ')'
  50. def any(*choices): return group(*choices) + '*'
  51. def maybe(*choices): return group(*choices) + '?'
  52. # Note: we use unicode matching for names ("\w") but ascii matching for
  53. # number literals.
  54. Whitespace = r'[ \f\t]*'
  55. Comment = r'#[^\r\n]*'
  56. Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
  57. Name = r'\w+'
  58. Hexnumber = r'0[xX](?:_?[0-9a-fA-F])+'
  59. Binnumber = r'0[bB](?:_?[01])+'
  60. Octnumber = r'0[oO](?:_?[0-7])+'
  61. Decnumber = r'(?:0(?:_?0)*|[1-9](?:_?[0-9])*)'
  62. Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
  63. Exponent = r'[eE][-+]?[0-9](?:_?[0-9])*'
  64. Pointfloat = group(r'[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?',
  65. r'\.[0-9](?:_?[0-9])*') + maybe(Exponent)
  66. Expfloat = r'[0-9](?:_?[0-9])*' + Exponent
  67. Floatnumber = group(Pointfloat, Expfloat)
  68. Imagnumber = group(r'[0-9](?:_?[0-9])*[jJ]', Floatnumber + r'[jJ]')
  69. Number = group(Imagnumber, Floatnumber, Intnumber)
  70. # Return the empty string, plus all of the valid string prefixes.
  71. def _all_string_prefixes():
  72. # The valid string prefixes. Only contain the lower case versions,
  73. # and don't contain any permutations (include 'fr', but not
  74. # 'rf'). The various permutations will be generated.
  75. _valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr']
  76. # if we add binary f-strings, add: ['fb', 'fbr']
  77. result = {''}
  78. for prefix in _valid_string_prefixes:
  79. for t in _itertools.permutations(prefix):
  80. # create a list with upper and lower versions of each
  81. # character
  82. for u in _itertools.product(*[(c, c.upper()) for c in t]):
  83. result.add(''.join(u))
  84. return result
  85. @functools.lru_cache
  86. def _compile(expr):
  87. return re.compile(expr, re.UNICODE)
  88. # Note that since _all_string_prefixes includes the empty string,
  89. # StringPrefix can be the empty string (making it optional).
  90. StringPrefix = group(*_all_string_prefixes())
  91. # Tail end of ' string.
  92. Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
  93. # Tail end of " string.
  94. Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
  95. # Tail end of ''' string.
  96. Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
  97. # Tail end of """ string.
  98. Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
  99. Triple = group(StringPrefix + "'''", StringPrefix + '"""')
  100. # Single-line ' or " string.
  101. String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
  102. StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
  103. # Sorting in reverse order puts the long operators before their prefixes.
  104. # Otherwise if = came before ==, == would get recognized as two instances
  105. # of =.
  106. Special = group(*map(re.escape, sorted(EXACT_TOKEN_TYPES, reverse=True)))
  107. Funny = group(r'\r?\n', Special)
  108. PlainToken = group(Number, Funny, String, Name)
  109. Token = Ignore + PlainToken
  110. # First (or only) line of ' or " string.
  111. ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
  112. group("'", r'\\\r?\n'),
  113. StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
  114. group('"', r'\\\r?\n'))
  115. PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
  116. PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
  117. # For a given string prefix plus quotes, endpats maps it to a regex
  118. # to match the remainder of that string. _prefix can be empty, for
  119. # a normal single or triple quoted string (with no prefix).
  120. endpats = {}
  121. for _prefix in _all_string_prefixes():
  122. endpats[_prefix + "'"] = Single
  123. endpats[_prefix + '"'] = Double
  124. endpats[_prefix + "'''"] = Single3
  125. endpats[_prefix + '"""'] = Double3
  126. # A set of all of the single and triple quoted string prefixes,
  127. # including the opening quotes.
  128. single_quoted = set()
  129. triple_quoted = set()
  130. for t in _all_string_prefixes():
  131. for u in (t + '"', t + "'"):
  132. single_quoted.add(u)
  133. for u in (t + '"""', t + "'''"):
  134. triple_quoted.add(u)
  135. tabsize = 8
  136. class TokenError(Exception): pass
  137. class StopTokenizing(Exception): pass
  138. class Untokenizer:
  139. def __init__(self):
  140. self.tokens = []
  141. self.prev_row = 1
  142. self.prev_col = 0
  143. self.encoding = None
  144. def add_whitespace(self, start):
  145. row, col = start
  146. if row < self.prev_row or row == self.prev_row and col < self.prev_col:
  147. raise ValueError("start ({},{}) precedes previous end ({},{})"
  148. .format(row, col, self.prev_row, self.prev_col))
  149. row_offset = row - self.prev_row
  150. if row_offset:
  151. self.tokens.append("\\\n" * row_offset)
  152. self.prev_col = 0
  153. col_offset = col - self.prev_col
  154. if col_offset:
  155. self.tokens.append(" " * col_offset)
  156. def untokenize(self, iterable):
  157. it = iter(iterable)
  158. indents = []
  159. startline = False
  160. for t in it:
  161. if len(t) == 2:
  162. self.compat(t, it)
  163. break
  164. tok_type, token, start, end, line = t
  165. if tok_type == ENCODING:
  166. self.encoding = token
  167. continue
  168. if tok_type == ENDMARKER:
  169. break
  170. if tok_type == INDENT:
  171. indents.append(token)
  172. continue
  173. elif tok_type == DEDENT:
  174. indents.pop()
  175. self.prev_row, self.prev_col = end
  176. continue
  177. elif tok_type in (NEWLINE, NL):
  178. startline = True
  179. elif startline and indents:
  180. indent = indents[-1]
  181. if start[1] >= len(indent):
  182. self.tokens.append(indent)
  183. self.prev_col = len(indent)
  184. startline = False
  185. self.add_whitespace(start)
  186. self.tokens.append(token)
  187. self.prev_row, self.prev_col = end
  188. if tok_type in (NEWLINE, NL):
  189. self.prev_row += 1
  190. self.prev_col = 0
  191. return "".join(self.tokens)
  192. def compat(self, token, iterable):
  193. indents = []
  194. toks_append = self.tokens.append
  195. startline = token[0] in (NEWLINE, NL)
  196. prevstring = False
  197. for tok in _itertools.chain([token], iterable):
  198. toknum, tokval = tok[:2]
  199. if toknum == ENCODING:
  200. self.encoding = tokval
  201. continue
  202. if toknum in (NAME, NUMBER):
  203. tokval += ' '
  204. # Insert a space between two consecutive strings
  205. if toknum == STRING:
  206. if prevstring:
  207. tokval = ' ' + tokval
  208. prevstring = True
  209. else:
  210. prevstring = False
  211. if toknum == INDENT:
  212. indents.append(tokval)
  213. continue
  214. elif toknum == DEDENT:
  215. indents.pop()
  216. continue
  217. elif toknum in (NEWLINE, NL):
  218. startline = True
  219. elif startline and indents:
  220. toks_append(indents[-1])
  221. startline = False
  222. toks_append(tokval)
  223. def untokenize(iterable):
  224. """Transform tokens back into Python source code.
  225. It returns a bytes object, encoded using the ENCODING
  226. token, which is the first token sequence output by tokenize.
  227. Each element returned by the iterable must be a token sequence
  228. with at least two elements, a token number and token value. If
  229. only two tokens are passed, the resulting output is poor.
  230. Round-trip invariant for full input:
  231. Untokenized source will match input source exactly
  232. Round-trip invariant for limited input:
  233. # Output bytes will tokenize back to the input
  234. t1 = [tok[:2] for tok in tokenize(f.readline)]
  235. newcode = untokenize(t1)
  236. readline = BytesIO(newcode).readline
  237. t2 = [tok[:2] for tok in tokenize(readline)]
  238. assert t1 == t2
  239. """
  240. ut = Untokenizer()
  241. out = ut.untokenize(iterable)
  242. if ut.encoding is not None:
  243. out = out.encode(ut.encoding)
  244. return out
  245. def _get_normal_name(orig_enc):
  246. """Imitates get_normal_name in tokenizer.c."""
  247. # Only care about the first 12 characters.
  248. enc = orig_enc[:12].lower().replace("_", "-")
  249. if enc == "utf-8" or enc.startswith("utf-8-"):
  250. return "utf-8"
  251. if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
  252. enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
  253. return "iso-8859-1"
  254. return orig_enc
  255. def detect_encoding(readline):
  256. """
  257. The detect_encoding() function is used to detect the encoding that should
  258. be used to decode a Python source file. It requires one argument, readline,
  259. in the same way as the tokenize() generator.
  260. It will call readline a maximum of twice, and return the encoding used
  261. (as a string) and a list of any lines (left as bytes) it has read in.
  262. It detects the encoding from the presence of a utf-8 bom or an encoding
  263. cookie as specified in pep-0263. If both a bom and a cookie are present,
  264. but disagree, a SyntaxError will be raised. If the encoding cookie is an
  265. invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found,
  266. 'utf-8-sig' is returned.
  267. If no encoding is specified, then the default of 'utf-8' will be returned.
  268. """
  269. try:
  270. filename = readline.__self__.name
  271. except AttributeError:
  272. filename = None
  273. bom_found = False
  274. encoding = None
  275. default = 'utf-8'
  276. def read_or_stop():
  277. try:
  278. return readline()
  279. except StopIteration:
  280. return b''
  281. def find_cookie(line):
  282. try:
  283. # Decode as UTF-8. Either the line is an encoding declaration,
  284. # in which case it should be pure ASCII, or it must be UTF-8
  285. # per default encoding.
  286. line_string = line.decode('utf-8')
  287. except UnicodeDecodeError:
  288. msg = "invalid or missing encoding declaration"
  289. if filename is not None:
  290. msg = '{} for {!r}'.format(msg, filename)
  291. raise SyntaxError(msg)
  292. match = cookie_re.match(line_string)
  293. if not match:
  294. return None
  295. encoding = _get_normal_name(match.group(1))
  296. try:
  297. codec = lookup(encoding)
  298. except LookupError:
  299. # This behaviour mimics the Python interpreter
  300. if filename is None:
  301. msg = "unknown encoding: " + encoding
  302. else:
  303. msg = "unknown encoding for {!r}: {}".format(filename,
  304. encoding)
  305. raise SyntaxError(msg)
  306. if bom_found:
  307. if encoding != 'utf-8':
  308. # This behaviour mimics the Python interpreter
  309. if filename is None:
  310. msg = 'encoding problem: utf-8'
  311. else:
  312. msg = 'encoding problem for {!r}: utf-8'.format(filename)
  313. raise SyntaxError(msg)
  314. encoding += '-sig'
  315. return encoding
  316. first = read_or_stop()
  317. if first.startswith(BOM_UTF8):
  318. bom_found = True
  319. first = first[3:]
  320. default = 'utf-8-sig'
  321. if not first:
  322. return default, []
  323. encoding = find_cookie(first)
  324. if encoding:
  325. return encoding, [first]
  326. if not blank_re.match(first):
  327. return default, [first]
  328. second = read_or_stop()
  329. if not second:
  330. return default, [first]
  331. encoding = find_cookie(second)
  332. if encoding:
  333. return encoding, [first, second]
  334. return default, [first, second]
  335. def open(filename):
  336. """Open a file in read only mode using the encoding detected by
  337. detect_encoding().
  338. """
  339. buffer = _builtin_open(filename, 'rb')
  340. try:
  341. encoding, lines = detect_encoding(buffer.readline)
  342. buffer.seek(0)
  343. text = TextIOWrapper(buffer, encoding, line_buffering=True)
  344. text.mode = 'r'
  345. return text
  346. except:
  347. buffer.close()
  348. raise
  349. def tokenize(readline):
  350. """
  351. The tokenize() generator requires one argument, readline, which
  352. must be a callable object which provides the same interface as the
  353. readline() method of built-in file objects. Each call to the function
  354. should return one line of input as bytes. Alternatively, readline
  355. can be a callable function terminating with StopIteration:
  356. readline = open(myfile, 'rb').__next__ # Example of alternate readline
  357. The generator produces 5-tuples with these members: the token type; the
  358. token string; a 2-tuple (srow, scol) of ints specifying the row and
  359. column where the token begins in the source; a 2-tuple (erow, ecol) of
  360. ints specifying the row and column where the token ends in the source;
  361. and the line on which the token was found. The line passed is the
  362. physical line.
  363. The first token sequence will always be an ENCODING token
  364. which tells you which encoding was used to decode the bytes stream.
  365. """
  366. encoding, consumed = detect_encoding(readline)
  367. empty = _itertools.repeat(b"")
  368. rl_gen = _itertools.chain(consumed, iter(readline, b""), empty)
  369. return _tokenize(rl_gen.__next__, encoding)
  370. def _tokenize(readline, encoding):
  371. lnum = parenlev = continued = 0
  372. numchars = '0123456789'
  373. contstr, needcont = '', 0
  374. contline = None
  375. indents = [0]
  376. if encoding is not None:
  377. if encoding == "utf-8-sig":
  378. # BOM will already have been stripped.
  379. encoding = "utf-8"
  380. yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
  381. last_line = b''
  382. line = b''
  383. while True: # loop over lines in stream
  384. try:
  385. # We capture the value of the line variable here because
  386. # readline uses the empty string '' to signal end of input,
  387. # hence `line` itself will always be overwritten at the end
  388. # of this loop.
  389. last_line = line
  390. line = readline()
  391. except StopIteration:
  392. line = b''
  393. if encoding is not None:
  394. line = line.decode(encoding)
  395. lnum += 1
  396. pos, max = 0, len(line)
  397. if contstr: # continued string
  398. if not line:
  399. raise TokenError("EOF in multi-line string", strstart)
  400. endmatch = endprog.match(line)
  401. if endmatch:
  402. pos = end = endmatch.end(0)
  403. yield TokenInfo(STRING, contstr + line[:end],
  404. strstart, (lnum, end), contline + line)
  405. contstr, needcont = '', 0
  406. contline = None
  407. elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
  408. yield TokenInfo(ERRORTOKEN, contstr + line,
  409. strstart, (lnum, len(line)), contline)
  410. contstr = ''
  411. contline = None
  412. continue
  413. else:
  414. contstr = contstr + line
  415. contline = contline + line
  416. continue
  417. elif parenlev == 0 and not continued: # new statement
  418. if not line: break
  419. column = 0
  420. while pos < max: # measure leading whitespace
  421. if line[pos] == ' ':
  422. column += 1
  423. elif line[pos] == '\t':
  424. column = (column//tabsize + 1)*tabsize
  425. elif line[pos] == '\f':
  426. column = 0
  427. else:
  428. break
  429. pos += 1
  430. if pos == max:
  431. break
  432. if line[pos] in '#\r\n': # skip comments or blank lines
  433. if line[pos] == '#':
  434. comment_token = line[pos:].rstrip('\r\n')
  435. yield TokenInfo(COMMENT, comment_token,
  436. (lnum, pos), (lnum, pos + len(comment_token)), line)
  437. pos += len(comment_token)
  438. yield TokenInfo(NL, line[pos:],
  439. (lnum, pos), (lnum, len(line)), line)
  440. continue
  441. if column > indents[-1]: # count indents or dedents
  442. indents.append(column)
  443. yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
  444. while column < indents[-1]:
  445. if column not in indents:
  446. raise IndentationError(
  447. "unindent does not match any outer indentation level",
  448. ("<tokenize>", lnum, pos, line))
  449. indents = indents[:-1]
  450. yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
  451. else: # continued statement
  452. if not line:
  453. raise TokenError("EOF in multi-line statement", (lnum, 0))
  454. continued = 0
  455. while pos < max:
  456. pseudomatch = _compile(PseudoToken).match(line, pos)
  457. if pseudomatch: # scan for tokens
  458. start, end = pseudomatch.span(1)
  459. spos, epos, pos = (lnum, start), (lnum, end), end
  460. if start == end:
  461. continue
  462. token, initial = line[start:end], line[start]
  463. if (initial in numchars or # ordinary number
  464. (initial == '.' and token != '.' and token != '...')):
  465. yield TokenInfo(NUMBER, token, spos, epos, line)
  466. elif initial in '\r\n':
  467. if parenlev > 0:
  468. yield TokenInfo(NL, token, spos, epos, line)
  469. else:
  470. yield TokenInfo(NEWLINE, token, spos, epos, line)
  471. elif initial == '#':
  472. assert not token.endswith("\n")
  473. yield TokenInfo(COMMENT, token, spos, epos, line)
  474. elif token in triple_quoted:
  475. endprog = _compile(endpats[token])
  476. endmatch = endprog.match(line, pos)
  477. if endmatch: # all on one line
  478. pos = endmatch.end(0)
  479. token = line[start:pos]
  480. yield TokenInfo(STRING, token, spos, (lnum, pos), line)
  481. else:
  482. strstart = (lnum, start) # multiple lines
  483. contstr = line[start:]
  484. contline = line
  485. break
  486. # Check up to the first 3 chars of the token to see if
  487. # they're in the single_quoted set. If so, they start
  488. # a string.
  489. # We're using the first 3, because we're looking for
  490. # "rb'" (for example) at the start of the token. If
  491. # we switch to longer prefixes, this needs to be
  492. # adjusted.
  493. # Note that initial == token[:1].
  494. # Also note that single quote checking must come after
  495. # triple quote checking (above).
  496. elif (initial in single_quoted or
  497. token[:2] in single_quoted or
  498. token[:3] in single_quoted):
  499. if token[-1] == '\n': # continued string
  500. strstart = (lnum, start)
  501. # Again, using the first 3 chars of the
  502. # token. This is looking for the matching end
  503. # regex for the correct type of quote
  504. # character. So it's really looking for
  505. # endpats["'"] or endpats['"'], by trying to
  506. # skip string prefix characters, if any.
  507. endprog = _compile(endpats.get(initial) or
  508. endpats.get(token[1]) or
  509. endpats.get(token[2]))
  510. contstr, needcont = line[start:], 1
  511. contline = line
  512. break
  513. else: # ordinary string
  514. yield TokenInfo(STRING, token, spos, epos, line)
  515. elif initial.isidentifier(): # ordinary name
  516. yield TokenInfo(NAME, token, spos, epos, line)
  517. elif initial == '\\': # continued stmt
  518. continued = 1
  519. else:
  520. if initial in '([{':
  521. parenlev += 1
  522. elif initial in ')]}':
  523. parenlev -= 1
  524. yield TokenInfo(OP, token, spos, epos, line)
  525. else:
  526. yield TokenInfo(ERRORTOKEN, line[pos],
  527. (lnum, pos), (lnum, pos+1), line)
  528. pos += 1
  529. # Add an implicit NEWLINE if the input doesn't end in one
  530. if last_line and last_line[-1] not in '\r\n' and not last_line.strip().startswith("#"):
  531. yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), '')
  532. for indent in indents[1:]: # pop remaining indent levels
  533. yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
  534. yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')
  535. def generate_tokens(readline):
  536. """Tokenize a source reading Python code as unicode strings.
  537. This has the same API as tokenize(), except that it expects the *readline*
  538. callable to return str objects instead of bytes.
  539. """
  540. return _tokenize(readline, None)
  541. def main():
  542. import argparse
  543. # Helper error handling routines
  544. def perror(message):
  545. sys.stderr.write(message)
  546. sys.stderr.write('\n')
  547. def error(message, filename=None, location=None):
  548. if location:
  549. args = (filename,) + location + (message,)
  550. perror("%s:%d:%d: error: %s" % args)
  551. elif filename:
  552. perror("%s: error: %s" % (filename, message))
  553. else:
  554. perror("error: %s" % message)
  555. sys.exit(1)
  556. # Parse the arguments and options
  557. parser = argparse.ArgumentParser(prog='python -m tokenize')
  558. parser.add_argument(dest='filename', nargs='?',
  559. metavar='filename.py',
  560. help='the file to tokenize; defaults to stdin')
  561. parser.add_argument('-e', '--exact', dest='exact', action='store_true',
  562. help='display token names using the exact type')
  563. args = parser.parse_args()
  564. try:
  565. # Tokenize the input
  566. if args.filename:
  567. filename = args.filename
  568. with _builtin_open(filename, 'rb') as f:
  569. tokens = list(tokenize(f.readline))
  570. else:
  571. filename = "<stdin>"
  572. tokens = _tokenize(sys.stdin.readline, None)
  573. # Output the tokenization
  574. for token in tokens:
  575. token_type = token.type
  576. if args.exact:
  577. token_type = token.exact_type
  578. token_range = "%d,%d-%d,%d:" % (token.start + token.end)
  579. print("%-20s%-15s%-15r" %
  580. (token_range, tok_name[token_type], token.string))
  581. except IndentationError as err:
  582. line, column = err.args[1][1:3]
  583. error(err.args[0], filename, (line, column))
  584. except TokenError as err:
  585. line, column = err.args[1]
  586. error(err.args[0], filename, (line, column))
  587. except SyntaxError as err:
  588. error(err, filename)
  589. except OSError as err:
  590. error(err)
  591. except KeyboardInterrupt:
  592. print("interrupted\n")
  593. except Exception as err:
  594. perror("unexpected error: %s" % err)
  595. raise
  596. if __name__ == "__main__":
  597. main()