logo

oasis-root

Compiled tree of Oasis Linux based on own branch at <https://hacktivis.me/git/oasis/> git clone https://anongit.hacktivis.me/git/oasis-root.git

pyparse.py (19947B)


  1. """Define partial Python code Parser used by editor and hyperparser.
  2. Instances of ParseMap are used with str.translate.
  3. The following bound search and match functions are defined:
  4. _synchre - start of popular statement;
  5. _junkre - whitespace or comment line;
  6. _match_stringre: string, possibly without closer;
  7. _itemre - line that may have bracket structure start;
  8. _closere - line that must be followed by dedent.
  9. _chew_ordinaryre - non-special characters.
  10. """
  11. import re
  12. # Reason last statement is continued (or C_NONE if it's not).
  13. (C_NONE, C_BACKSLASH, C_STRING_FIRST_LINE,
  14. C_STRING_NEXT_LINES, C_BRACKET) = range(5)
  15. # Find what looks like the start of a popular statement.
  16. _synchre = re.compile(r"""
  17. ^
  18. [ \t]*
  19. (?: while
  20. | else
  21. | def
  22. | return
  23. | assert
  24. | break
  25. | class
  26. | continue
  27. | elif
  28. | try
  29. | except
  30. | raise
  31. | import
  32. | yield
  33. )
  34. \b
  35. """, re.VERBOSE | re.MULTILINE).search
  36. # Match blank line or non-indenting comment line.
  37. _junkre = re.compile(r"""
  38. [ \t]*
  39. (?: \# \S .* )?
  40. \n
  41. """, re.VERBOSE).match
  42. # Match any flavor of string; the terminating quote is optional
  43. # so that we're robust in the face of incomplete program text.
  44. _match_stringre = re.compile(r"""
  45. \""" [^"\\]* (?:
  46. (?: \\. | "(?!"") )
  47. [^"\\]*
  48. )*
  49. (?: \""" )?
  50. | " [^"\\\n]* (?: \\. [^"\\\n]* )* "?
  51. | ''' [^'\\]* (?:
  52. (?: \\. | '(?!'') )
  53. [^'\\]*
  54. )*
  55. (?: ''' )?
  56. | ' [^'\\\n]* (?: \\. [^'\\\n]* )* '?
  57. """, re.VERBOSE | re.DOTALL).match
  58. # Match a line that starts with something interesting;
  59. # used to find the first item of a bracket structure.
  60. _itemre = re.compile(r"""
  61. [ \t]*
  62. [^\s#\\] # if we match, m.end()-1 is the interesting char
  63. """, re.VERBOSE).match
  64. # Match start of statements that should be followed by a dedent.
  65. _closere = re.compile(r"""
  66. \s*
  67. (?: return
  68. | break
  69. | continue
  70. | raise
  71. | pass
  72. )
  73. \b
  74. """, re.VERBOSE).match
  75. # Chew up non-special chars as quickly as possible. If match is
  76. # successful, m.end() less 1 is the index of the last boring char
  77. # matched. If match is unsuccessful, the string starts with an
  78. # interesting char.
  79. _chew_ordinaryre = re.compile(r"""
  80. [^[\](){}#'"\\]+
  81. """, re.VERBOSE).match
  82. class ParseMap(dict):
  83. r"""Dict subclass that maps anything not in dict to 'x'.
  84. This is designed to be used with str.translate in study1.
  85. Anything not specifically mapped otherwise becomes 'x'.
  86. Example: replace everything except whitespace with 'x'.
  87. >>> keepwhite = ParseMap((ord(c), ord(c)) for c in ' \t\n\r')
  88. >>> "a + b\tc\nd".translate(keepwhite)
  89. 'x x x\tx\nx'
  90. """
  91. # Calling this triples access time; see bpo-32940
  92. def __missing__(self, key):
  93. return 120 # ord('x')
  94. # Map all ascii to 120 to avoid __missing__ call, then replace some.
  95. trans = ParseMap.fromkeys(range(128), 120)
  96. trans.update((ord(c), ord('(')) for c in "({[") # open brackets => '(';
  97. trans.update((ord(c), ord(')')) for c in ")}]") # close brackets => ')'.
  98. trans.update((ord(c), ord(c)) for c in "\"'\\\n#") # Keep these.
  99. class Parser:
  100. def __init__(self, indentwidth, tabwidth):
  101. self.indentwidth = indentwidth
  102. self.tabwidth = tabwidth
  103. def set_code(self, s):
  104. assert len(s) == 0 or s[-1] == '\n'
  105. self.code = s
  106. self.study_level = 0
  107. def find_good_parse_start(self, is_char_in_string):
  108. """
  109. Return index of a good place to begin parsing, as close to the
  110. end of the string as possible. This will be the start of some
  111. popular stmt like "if" or "def". Return None if none found:
  112. the caller should pass more prior context then, if possible, or
  113. if not (the entire program text up until the point of interest
  114. has already been tried) pass 0 to set_lo().
  115. This will be reliable iff given a reliable is_char_in_string()
  116. function, meaning that when it says "no", it's absolutely
  117. guaranteed that the char is not in a string.
  118. """
  119. code, pos = self.code, None
  120. # Peek back from the end for a good place to start,
  121. # but don't try too often; pos will be left None, or
  122. # bumped to a legitimate synch point.
  123. limit = len(code)
  124. for tries in range(5):
  125. i = code.rfind(":\n", 0, limit)
  126. if i < 0:
  127. break
  128. i = code.rfind('\n', 0, i) + 1 # start of colon line (-1+1=0)
  129. m = _synchre(code, i, limit)
  130. if m and not is_char_in_string(m.start()):
  131. pos = m.start()
  132. break
  133. limit = i
  134. if pos is None:
  135. # Nothing looks like a block-opener, or stuff does
  136. # but is_char_in_string keeps returning true; most likely
  137. # we're in or near a giant string, the colorizer hasn't
  138. # caught up enough to be helpful, or there simply *aren't*
  139. # any interesting stmts. In any of these cases we're
  140. # going to have to parse the whole thing to be sure, so
  141. # give it one last try from the start, but stop wasting
  142. # time here regardless of the outcome.
  143. m = _synchre(code)
  144. if m and not is_char_in_string(m.start()):
  145. pos = m.start()
  146. return pos
  147. # Peeking back worked; look forward until _synchre no longer
  148. # matches.
  149. i = pos + 1
  150. while 1:
  151. m = _synchre(code, i)
  152. if m:
  153. s, i = m.span()
  154. if not is_char_in_string(s):
  155. pos = s
  156. else:
  157. break
  158. return pos
  159. def set_lo(self, lo):
  160. """ Throw away the start of the string.
  161. Intended to be called with the result of find_good_parse_start().
  162. """
  163. assert lo == 0 or self.code[lo-1] == '\n'
  164. if lo > 0:
  165. self.code = self.code[lo:]
  166. def _study1(self):
  167. """Find the line numbers of non-continuation lines.
  168. As quickly as humanly possible <wink>, find the line numbers (0-
  169. based) of the non-continuation lines.
  170. Creates self.{goodlines, continuation}.
  171. """
  172. if self.study_level >= 1:
  173. return
  174. self.study_level = 1
  175. # Map all uninteresting characters to "x", all open brackets
  176. # to "(", all close brackets to ")", then collapse runs of
  177. # uninteresting characters. This can cut the number of chars
  178. # by a factor of 10-40, and so greatly speed the following loop.
  179. code = self.code
  180. code = code.translate(trans)
  181. code = code.replace('xxxxxxxx', 'x')
  182. code = code.replace('xxxx', 'x')
  183. code = code.replace('xx', 'x')
  184. code = code.replace('xx', 'x')
  185. code = code.replace('\nx', '\n')
  186. # Replacing x\n with \n would be incorrect because
  187. # x may be preceded by a backslash.
  188. # March over the squashed version of the program, accumulating
  189. # the line numbers of non-continued stmts, and determining
  190. # whether & why the last stmt is a continuation.
  191. continuation = C_NONE
  192. level = lno = 0 # level is nesting level; lno is line number
  193. self.goodlines = goodlines = [0]
  194. push_good = goodlines.append
  195. i, n = 0, len(code)
  196. while i < n:
  197. ch = code[i]
  198. i = i+1
  199. # cases are checked in decreasing order of frequency
  200. if ch == 'x':
  201. continue
  202. if ch == '\n':
  203. lno = lno + 1
  204. if level == 0:
  205. push_good(lno)
  206. # else we're in an unclosed bracket structure
  207. continue
  208. if ch == '(':
  209. level = level + 1
  210. continue
  211. if ch == ')':
  212. if level:
  213. level = level - 1
  214. # else the program is invalid, but we can't complain
  215. continue
  216. if ch == '"' or ch == "'":
  217. # consume the string
  218. quote = ch
  219. if code[i-1:i+2] == quote * 3:
  220. quote = quote * 3
  221. firstlno = lno
  222. w = len(quote) - 1
  223. i = i+w
  224. while i < n:
  225. ch = code[i]
  226. i = i+1
  227. if ch == 'x':
  228. continue
  229. if code[i-1:i+w] == quote:
  230. i = i+w
  231. break
  232. if ch == '\n':
  233. lno = lno + 1
  234. if w == 0:
  235. # unterminated single-quoted string
  236. if level == 0:
  237. push_good(lno)
  238. break
  239. continue
  240. if ch == '\\':
  241. assert i < n
  242. if code[i] == '\n':
  243. lno = lno + 1
  244. i = i+1
  245. continue
  246. # else comment char or paren inside string
  247. else:
  248. # didn't break out of the loop, so we're still
  249. # inside a string
  250. if (lno - 1) == firstlno:
  251. # before the previous \n in code, we were in the first
  252. # line of the string
  253. continuation = C_STRING_FIRST_LINE
  254. else:
  255. continuation = C_STRING_NEXT_LINES
  256. continue # with outer loop
  257. if ch == '#':
  258. # consume the comment
  259. i = code.find('\n', i)
  260. assert i >= 0
  261. continue
  262. assert ch == '\\'
  263. assert i < n
  264. if code[i] == '\n':
  265. lno = lno + 1
  266. if i+1 == n:
  267. continuation = C_BACKSLASH
  268. i = i+1
  269. # The last stmt may be continued for all 3 reasons.
  270. # String continuation takes precedence over bracket
  271. # continuation, which beats backslash continuation.
  272. if (continuation != C_STRING_FIRST_LINE
  273. and continuation != C_STRING_NEXT_LINES and level > 0):
  274. continuation = C_BRACKET
  275. self.continuation = continuation
  276. # Push the final line number as a sentinel value, regardless of
  277. # whether it's continued.
  278. assert (continuation == C_NONE) == (goodlines[-1] == lno)
  279. if goodlines[-1] != lno:
  280. push_good(lno)
  281. def get_continuation_type(self):
  282. self._study1()
  283. return self.continuation
  284. def _study2(self):
  285. """
  286. study1 was sufficient to determine the continuation status,
  287. but doing more requires looking at every character. study2
  288. does this for the last interesting statement in the block.
  289. Creates:
  290. self.stmt_start, stmt_end
  291. slice indices of last interesting stmt
  292. self.stmt_bracketing
  293. the bracketing structure of the last interesting stmt; for
  294. example, for the statement "say(boo) or die",
  295. stmt_bracketing will be ((0, 0), (0, 1), (2, 0), (2, 1),
  296. (4, 0)). Strings and comments are treated as brackets, for
  297. the matter.
  298. self.lastch
  299. last interesting character before optional trailing comment
  300. self.lastopenbracketpos
  301. if continuation is C_BRACKET, index of last open bracket
  302. """
  303. if self.study_level >= 2:
  304. return
  305. self._study1()
  306. self.study_level = 2
  307. # Set p and q to slice indices of last interesting stmt.
  308. code, goodlines = self.code, self.goodlines
  309. i = len(goodlines) - 1 # Index of newest line.
  310. p = len(code) # End of goodlines[i]
  311. while i:
  312. assert p
  313. # Make p be the index of the stmt at line number goodlines[i].
  314. # Move p back to the stmt at line number goodlines[i-1].
  315. q = p
  316. for nothing in range(goodlines[i-1], goodlines[i]):
  317. # tricky: sets p to 0 if no preceding newline
  318. p = code.rfind('\n', 0, p-1) + 1
  319. # The stmt code[p:q] isn't a continuation, but may be blank
  320. # or a non-indenting comment line.
  321. if _junkre(code, p):
  322. i = i-1
  323. else:
  324. break
  325. if i == 0:
  326. # nothing but junk!
  327. assert p == 0
  328. q = p
  329. self.stmt_start, self.stmt_end = p, q
  330. # Analyze this stmt, to find the last open bracket (if any)
  331. # and last interesting character (if any).
  332. lastch = ""
  333. stack = [] # stack of open bracket indices
  334. push_stack = stack.append
  335. bracketing = [(p, 0)]
  336. while p < q:
  337. # suck up all except ()[]{}'"#\\
  338. m = _chew_ordinaryre(code, p, q)
  339. if m:
  340. # we skipped at least one boring char
  341. newp = m.end()
  342. # back up over totally boring whitespace
  343. i = newp - 1 # index of last boring char
  344. while i >= p and code[i] in " \t\n":
  345. i = i-1
  346. if i >= p:
  347. lastch = code[i]
  348. p = newp
  349. if p >= q:
  350. break
  351. ch = code[p]
  352. if ch in "([{":
  353. push_stack(p)
  354. bracketing.append((p, len(stack)))
  355. lastch = ch
  356. p = p+1
  357. continue
  358. if ch in ")]}":
  359. if stack:
  360. del stack[-1]
  361. lastch = ch
  362. p = p+1
  363. bracketing.append((p, len(stack)))
  364. continue
  365. if ch == '"' or ch == "'":
  366. # consume string
  367. # Note that study1 did this with a Python loop, but
  368. # we use a regexp here; the reason is speed in both
  369. # cases; the string may be huge, but study1 pre-squashed
  370. # strings to a couple of characters per line. study1
  371. # also needed to keep track of newlines, and we don't
  372. # have to.
  373. bracketing.append((p, len(stack)+1))
  374. lastch = ch
  375. p = _match_stringre(code, p, q).end()
  376. bracketing.append((p, len(stack)))
  377. continue
  378. if ch == '#':
  379. # consume comment and trailing newline
  380. bracketing.append((p, len(stack)+1))
  381. p = code.find('\n', p, q) + 1
  382. assert p > 0
  383. bracketing.append((p, len(stack)))
  384. continue
  385. assert ch == '\\'
  386. p = p+1 # beyond backslash
  387. assert p < q
  388. if code[p] != '\n':
  389. # the program is invalid, but can't complain
  390. lastch = ch + code[p]
  391. p = p+1 # beyond escaped char
  392. # end while p < q:
  393. self.lastch = lastch
  394. self.lastopenbracketpos = stack[-1] if stack else None
  395. self.stmt_bracketing = tuple(bracketing)
  396. def compute_bracket_indent(self):
  397. """Return number of spaces the next line should be indented.
  398. Line continuation must be C_BRACKET.
  399. """
  400. self._study2()
  401. assert self.continuation == C_BRACKET
  402. j = self.lastopenbracketpos
  403. code = self.code
  404. n = len(code)
  405. origi = i = code.rfind('\n', 0, j) + 1
  406. j = j+1 # one beyond open bracket
  407. # find first list item; set i to start of its line
  408. while j < n:
  409. m = _itemre(code, j)
  410. if m:
  411. j = m.end() - 1 # index of first interesting char
  412. extra = 0
  413. break
  414. else:
  415. # this line is junk; advance to next line
  416. i = j = code.find('\n', j) + 1
  417. else:
  418. # nothing interesting follows the bracket;
  419. # reproduce the bracket line's indentation + a level
  420. j = i = origi
  421. while code[j] in " \t":
  422. j = j+1
  423. extra = self.indentwidth
  424. return len(code[i:j].expandtabs(self.tabwidth)) + extra
  425. def get_num_lines_in_stmt(self):
  426. """Return number of physical lines in last stmt.
  427. The statement doesn't have to be an interesting statement. This is
  428. intended to be called when continuation is C_BACKSLASH.
  429. """
  430. self._study1()
  431. goodlines = self.goodlines
  432. return goodlines[-1] - goodlines[-2]
  433. def compute_backslash_indent(self):
  434. """Return number of spaces the next line should be indented.
  435. Line continuation must be C_BACKSLASH. Also assume that the new
  436. line is the first one following the initial line of the stmt.
  437. """
  438. self._study2()
  439. assert self.continuation == C_BACKSLASH
  440. code = self.code
  441. i = self.stmt_start
  442. while code[i] in " \t":
  443. i = i+1
  444. startpos = i
  445. # See whether the initial line starts an assignment stmt; i.e.,
  446. # look for an = operator
  447. endpos = code.find('\n', startpos) + 1
  448. found = level = 0
  449. while i < endpos:
  450. ch = code[i]
  451. if ch in "([{":
  452. level = level + 1
  453. i = i+1
  454. elif ch in ")]}":
  455. if level:
  456. level = level - 1
  457. i = i+1
  458. elif ch == '"' or ch == "'":
  459. i = _match_stringre(code, i, endpos).end()
  460. elif ch == '#':
  461. # This line is unreachable because the # makes a comment of
  462. # everything after it.
  463. break
  464. elif level == 0 and ch == '=' and \
  465. (i == 0 or code[i-1] not in "=<>!") and \
  466. code[i+1] != '=':
  467. found = 1
  468. break
  469. else:
  470. i = i+1
  471. if found:
  472. # found a legit =, but it may be the last interesting
  473. # thing on the line
  474. i = i+1 # move beyond the =
  475. found = re.match(r"\s*\\", code[i:endpos]) is None
  476. if not found:
  477. # oh well ... settle for moving beyond the first chunk
  478. # of non-whitespace chars
  479. i = startpos
  480. while code[i] not in " \t\n":
  481. i = i+1
  482. return len(code[self.stmt_start:i].expandtabs(\
  483. self.tabwidth)) + 1
  484. def get_base_indent_string(self):
  485. """Return the leading whitespace on the initial line of the last
  486. interesting stmt.
  487. """
  488. self._study2()
  489. i, n = self.stmt_start, self.stmt_end
  490. j = i
  491. code = self.code
  492. while j < n and code[j] in " \t":
  493. j = j + 1
  494. return code[i:j]
  495. def is_block_opener(self):
  496. "Return True if the last interesting statement opens a block."
  497. self._study2()
  498. return self.lastch == ':'
  499. def is_block_closer(self):
  500. "Return True if the last interesting statement closes a block."
  501. self._study2()
  502. return _closere(self.code, self.stmt_start) is not None
  503. def get_last_stmt_bracketing(self):
  504. """Return bracketing structure of the last interesting statement.
  505. The returned tuple is in the format defined in _study2().
  506. """
  507. self._study2()
  508. return self.stmt_bracketing
  509. if __name__ == '__main__':
  510. from unittest import main
  511. main('idlelib.idle_test.test_pyparse', verbosity=2)