logo

oasis-root

Compiled tree of Oasis Linux based on own branch at <https://hacktivis.me/git/oasis/> git clone https://anongit.hacktivis.me/git/oasis-root.git

compileall.py (20242B)


  1. """Module/script to byte-compile all .py files to .pyc files.
  2. When called as a script with arguments, this compiles the directories
  3. given as arguments recursively; the -l option prevents it from
  4. recursing into directories.
  5. Without arguments, if compiles all modules on sys.path, without
  6. recursing into subdirectories. (Even though it should do so for
  7. packages -- for now, you'll have to deal with packages separately.)
  8. See module py_compile for details of the actual byte-compilation.
  9. """
  10. import os
  11. import sys
  12. import importlib.util
  13. import py_compile
  14. import struct
  15. import filecmp
  16. from functools import partial
  17. from pathlib import Path
  18. __all__ = ["compile_dir","compile_file","compile_path"]
  19. def _walk_dir(dir, maxlevels, quiet=0):
  20. if quiet < 2 and isinstance(dir, os.PathLike):
  21. dir = os.fspath(dir)
  22. if not quiet:
  23. print('Listing {!r}...'.format(dir))
  24. try:
  25. names = os.listdir(dir)
  26. except OSError:
  27. if quiet < 2:
  28. print("Can't list {!r}".format(dir))
  29. names = []
  30. names.sort()
  31. for name in names:
  32. if name == '__pycache__':
  33. continue
  34. fullname = os.path.join(dir, name)
  35. if not os.path.isdir(fullname):
  36. yield fullname
  37. elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
  38. os.path.isdir(fullname) and not os.path.islink(fullname)):
  39. yield from _walk_dir(fullname, maxlevels=maxlevels - 1,
  40. quiet=quiet)
  41. def compile_dir(dir, maxlevels=None, ddir=None, force=False,
  42. rx=None, quiet=0, legacy=False, optimize=-1, workers=1,
  43. invalidation_mode=None, *, stripdir=None,
  44. prependdir=None, limit_sl_dest=None, hardlink_dupes=False):
  45. """Byte-compile all modules in the given directory tree.
  46. Arguments (only dir is required):
  47. dir: the directory to byte-compile
  48. maxlevels: maximum recursion level (default `sys.getrecursionlimit()`)
  49. ddir: the directory that will be prepended to the path to the
  50. file as it is compiled into each byte-code file.
  51. force: if True, force compilation, even if timestamps are up-to-date
  52. quiet: full output with False or 0, errors only with 1,
  53. no output with 2
  54. legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
  55. optimize: int or list of optimization levels or -1 for level of
  56. the interpreter. Multiple levels leads to multiple compiled
  57. files each with one optimization level.
  58. workers: maximum number of parallel workers
  59. invalidation_mode: how the up-to-dateness of the pyc will be checked
  60. stripdir: part of path to left-strip from source file path
  61. prependdir: path to prepend to beginning of original file path, applied
  62. after stripdir
  63. limit_sl_dest: ignore symlinks if they are pointing outside of
  64. the defined path
  65. hardlink_dupes: hardlink duplicated pyc files
  66. """
  67. ProcessPoolExecutor = None
  68. if ddir is not None and (stripdir is not None or prependdir is not None):
  69. raise ValueError(("Destination dir (ddir) cannot be used "
  70. "in combination with stripdir or prependdir"))
  71. if ddir is not None:
  72. stripdir = dir
  73. prependdir = ddir
  74. ddir = None
  75. if workers < 0:
  76. raise ValueError('workers must be greater or equal to 0')
  77. if workers != 1:
  78. # Check if this is a system where ProcessPoolExecutor can function.
  79. from concurrent.futures.process import _check_system_limits
  80. try:
  81. _check_system_limits()
  82. except NotImplementedError:
  83. workers = 1
  84. else:
  85. from concurrent.futures import ProcessPoolExecutor
  86. if maxlevels is None:
  87. maxlevels = sys.getrecursionlimit()
  88. files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels)
  89. success = True
  90. if workers != 1 and ProcessPoolExecutor is not None:
  91. # If workers == 0, let ProcessPoolExecutor choose
  92. workers = workers or None
  93. with ProcessPoolExecutor(max_workers=workers) as executor:
  94. results = executor.map(partial(compile_file,
  95. ddir=ddir, force=force,
  96. rx=rx, quiet=quiet,
  97. legacy=legacy,
  98. optimize=optimize,
  99. invalidation_mode=invalidation_mode,
  100. stripdir=stripdir,
  101. prependdir=prependdir,
  102. limit_sl_dest=limit_sl_dest,
  103. hardlink_dupes=hardlink_dupes),
  104. files)
  105. success = min(results, default=True)
  106. else:
  107. for file in files:
  108. if not compile_file(file, ddir, force, rx, quiet,
  109. legacy, optimize, invalidation_mode,
  110. stripdir=stripdir, prependdir=prependdir,
  111. limit_sl_dest=limit_sl_dest,
  112. hardlink_dupes=hardlink_dupes):
  113. success = False
  114. return success
  115. def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
  116. legacy=False, optimize=-1,
  117. invalidation_mode=None, *, stripdir=None, prependdir=None,
  118. limit_sl_dest=None, hardlink_dupes=False):
  119. """Byte-compile one file.
  120. Arguments (only fullname is required):
  121. fullname: the file to byte-compile
  122. ddir: if given, the directory name compiled in to the
  123. byte-code file.
  124. force: if True, force compilation, even if timestamps are up-to-date
  125. quiet: full output with False or 0, errors only with 1,
  126. no output with 2
  127. legacy: if True, produce legacy pyc paths instead of PEP 3147 paths
  128. optimize: int or list of optimization levels or -1 for level of
  129. the interpreter. Multiple levels leads to multiple compiled
  130. files each with one optimization level.
  131. invalidation_mode: how the up-to-dateness of the pyc will be checked
  132. stripdir: part of path to left-strip from source file path
  133. prependdir: path to prepend to beginning of original file path, applied
  134. after stripdir
  135. limit_sl_dest: ignore symlinks if they are pointing outside of
  136. the defined path.
  137. hardlink_dupes: hardlink duplicated pyc files
  138. """
  139. if ddir is not None and (stripdir is not None or prependdir is not None):
  140. raise ValueError(("Destination dir (ddir) cannot be used "
  141. "in combination with stripdir or prependdir"))
  142. success = True
  143. if quiet < 2 and isinstance(fullname, os.PathLike):
  144. fullname = os.fspath(fullname)
  145. name = os.path.basename(fullname)
  146. dfile = None
  147. if ddir is not None:
  148. dfile = os.path.join(ddir, name)
  149. if stripdir is not None:
  150. fullname_parts = fullname.split(os.path.sep)
  151. stripdir_parts = stripdir.split(os.path.sep)
  152. ddir_parts = list(fullname_parts)
  153. for spart, opart in zip(stripdir_parts, fullname_parts):
  154. if spart == opart:
  155. ddir_parts.remove(spart)
  156. dfile = os.path.join(*ddir_parts)
  157. if prependdir is not None:
  158. if dfile is None:
  159. dfile = os.path.join(prependdir, fullname)
  160. else:
  161. dfile = os.path.join(prependdir, dfile)
  162. if isinstance(optimize, int):
  163. optimize = [optimize]
  164. # Use set() to remove duplicates.
  165. # Use sorted() to create pyc files in a deterministic order.
  166. optimize = sorted(set(optimize))
  167. if hardlink_dupes and len(optimize) < 2:
  168. raise ValueError("Hardlinking of duplicated bytecode makes sense "
  169. "only for more than one optimization level")
  170. if rx is not None:
  171. mo = rx.search(fullname)
  172. if mo:
  173. return success
  174. if limit_sl_dest is not None and os.path.islink(fullname):
  175. if Path(limit_sl_dest).resolve() not in Path(fullname).resolve().parents:
  176. return success
  177. opt_cfiles = {}
  178. if os.path.isfile(fullname):
  179. for opt_level in optimize:
  180. if legacy:
  181. opt_cfiles[opt_level] = fullname + 'c'
  182. else:
  183. if opt_level >= 0:
  184. opt = opt_level if opt_level >= 1 else ''
  185. cfile = (importlib.util.cache_from_source(
  186. fullname, optimization=opt))
  187. opt_cfiles[opt_level] = cfile
  188. else:
  189. cfile = importlib.util.cache_from_source(fullname)
  190. opt_cfiles[opt_level] = cfile
  191. head, tail = name[:-3], name[-3:]
  192. if tail == '.py':
  193. if not force:
  194. try:
  195. mtime = int(os.stat(fullname).st_mtime)
  196. expect = struct.pack('<4sLL', importlib.util.MAGIC_NUMBER,
  197. 0, mtime & 0xFFFF_FFFF)
  198. for cfile in opt_cfiles.values():
  199. with open(cfile, 'rb') as chandle:
  200. actual = chandle.read(12)
  201. if expect != actual:
  202. break
  203. else:
  204. return success
  205. except OSError:
  206. pass
  207. if not quiet:
  208. print('Compiling {!r}...'.format(fullname))
  209. try:
  210. for index, opt_level in enumerate(optimize):
  211. cfile = opt_cfiles[opt_level]
  212. ok = py_compile.compile(fullname, cfile, dfile, True,
  213. optimize=opt_level,
  214. invalidation_mode=invalidation_mode)
  215. if index > 0 and hardlink_dupes:
  216. previous_cfile = opt_cfiles[optimize[index - 1]]
  217. if filecmp.cmp(cfile, previous_cfile, shallow=False):
  218. os.unlink(cfile)
  219. os.link(previous_cfile, cfile)
  220. except py_compile.PyCompileError as err:
  221. success = False
  222. if quiet >= 2:
  223. return success
  224. elif quiet:
  225. print('*** Error compiling {!r}...'.format(fullname))
  226. else:
  227. print('*** ', end='')
  228. # escape non-printable characters in msg
  229. encoding = sys.stdout.encoding or sys.getdefaultencoding()
  230. msg = err.msg.encode(encoding, errors='backslashreplace').decode(encoding)
  231. print(msg)
  232. except (SyntaxError, UnicodeError, OSError) as e:
  233. success = False
  234. if quiet >= 2:
  235. return success
  236. elif quiet:
  237. print('*** Error compiling {!r}...'.format(fullname))
  238. else:
  239. print('*** ', end='')
  240. print(e.__class__.__name__ + ':', e)
  241. else:
  242. if ok == 0:
  243. success = False
  244. return success
  245. def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0,
  246. legacy=False, optimize=-1,
  247. invalidation_mode=None):
  248. """Byte-compile all module on sys.path.
  249. Arguments (all optional):
  250. skip_curdir: if true, skip current directory (default True)
  251. maxlevels: max recursion level (default 0)
  252. force: as for compile_dir() (default False)
  253. quiet: as for compile_dir() (default 0)
  254. legacy: as for compile_dir() (default False)
  255. optimize: as for compile_dir() (default -1)
  256. invalidation_mode: as for compiler_dir()
  257. """
  258. success = True
  259. for dir in sys.path:
  260. if (not dir or dir == os.curdir) and skip_curdir:
  261. if quiet < 2:
  262. print('Skipping current directory')
  263. else:
  264. success = success and compile_dir(
  265. dir,
  266. maxlevels,
  267. None,
  268. force,
  269. quiet=quiet,
  270. legacy=legacy,
  271. optimize=optimize,
  272. invalidation_mode=invalidation_mode,
  273. )
  274. return success
  275. def main():
  276. """Script main program."""
  277. import argparse
  278. parser = argparse.ArgumentParser(
  279. description='Utilities to support installing Python libraries.')
  280. parser.add_argument('-l', action='store_const', const=0,
  281. default=None, dest='maxlevels',
  282. help="don't recurse into subdirectories")
  283. parser.add_argument('-r', type=int, dest='recursion',
  284. help=('control the maximum recursion level. '
  285. 'if `-l` and `-r` options are specified, '
  286. 'then `-r` takes precedence.'))
  287. parser.add_argument('-f', action='store_true', dest='force',
  288. help='force rebuild even if timestamps are up to date')
  289. parser.add_argument('-q', action='count', dest='quiet', default=0,
  290. help='output only error messages; -qq will suppress '
  291. 'the error messages as well.')
  292. parser.add_argument('-b', action='store_true', dest='legacy',
  293. help='use legacy (pre-PEP3147) compiled file locations')
  294. parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None,
  295. help=('directory to prepend to file paths for use in '
  296. 'compile-time tracebacks and in runtime '
  297. 'tracebacks in cases where the source file is '
  298. 'unavailable'))
  299. parser.add_argument('-s', metavar='STRIPDIR', dest='stripdir',
  300. default=None,
  301. help=('part of path to left-strip from path '
  302. 'to source file - for example buildroot. '
  303. '`-d` and `-s` options cannot be '
  304. 'specified together.'))
  305. parser.add_argument('-p', metavar='PREPENDDIR', dest='prependdir',
  306. default=None,
  307. help=('path to add as prefix to path '
  308. 'to source file - for example / to make '
  309. 'it absolute when some part is removed '
  310. 'by `-s` option. '
  311. '`-d` and `-p` options cannot be '
  312. 'specified together.'))
  313. parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None,
  314. help=('skip files matching the regular expression; '
  315. 'the regexp is searched for in the full path '
  316. 'of each file considered for compilation'))
  317. parser.add_argument('-i', metavar='FILE', dest='flist',
  318. help=('add all the files and directories listed in '
  319. 'FILE to the list considered for compilation; '
  320. 'if "-", names are read from stdin'))
  321. parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*',
  322. help=('zero or more file and directory names '
  323. 'to compile; if no arguments given, defaults '
  324. 'to the equivalent of -l sys.path'))
  325. parser.add_argument('-j', '--workers', default=1,
  326. type=int, help='Run compileall concurrently')
  327. invalidation_modes = [mode.name.lower().replace('_', '-')
  328. for mode in py_compile.PycInvalidationMode]
  329. parser.add_argument('--invalidation-mode',
  330. choices=sorted(invalidation_modes),
  331. help=('set .pyc invalidation mode; defaults to '
  332. '"checked-hash" if the SOURCE_DATE_EPOCH '
  333. 'environment variable is set, and '
  334. '"timestamp" otherwise.'))
  335. parser.add_argument('-o', action='append', type=int, dest='opt_levels',
  336. help=('Optimization levels to run compilation with.'
  337. 'Default is -1 which uses optimization level of'
  338. 'Python interpreter itself (specified by -O).'))
  339. parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest',
  340. help='Ignore symlinks pointing outsite of the DIR')
  341. parser.add_argument('--hardlink-dupes', action='store_true',
  342. dest='hardlink_dupes',
  343. help='Hardlink duplicated pyc files')
  344. args = parser.parse_args()
  345. compile_dests = args.compile_dest
  346. if args.rx:
  347. import re
  348. args.rx = re.compile(args.rx)
  349. if args.limit_sl_dest == "":
  350. args.limit_sl_dest = None
  351. if args.recursion is not None:
  352. maxlevels = args.recursion
  353. else:
  354. maxlevels = args.maxlevels
  355. if args.opt_levels is None:
  356. args.opt_levels = [-1]
  357. if len(args.opt_levels) == 1 and args.hardlink_dupes:
  358. parser.error(("Hardlinking of duplicated bytecode makes sense "
  359. "only for more than one optimization level."))
  360. if args.ddir is not None and (
  361. args.stripdir is not None or args.prependdir is not None
  362. ):
  363. parser.error("-d cannot be used in combination with -s or -p")
  364. # if flist is provided then load it
  365. if args.flist:
  366. try:
  367. with (sys.stdin if args.flist=='-' else
  368. open(args.flist, encoding="utf-8")) as f:
  369. for line in f:
  370. compile_dests.append(line.strip())
  371. except OSError:
  372. if args.quiet < 2:
  373. print("Error reading file list {}".format(args.flist))
  374. return False
  375. if args.invalidation_mode:
  376. ivl_mode = args.invalidation_mode.replace('-', '_').upper()
  377. invalidation_mode = py_compile.PycInvalidationMode[ivl_mode]
  378. else:
  379. invalidation_mode = None
  380. success = True
  381. try:
  382. if compile_dests:
  383. for dest in compile_dests:
  384. if os.path.isfile(dest):
  385. if not compile_file(dest, args.ddir, args.force, args.rx,
  386. args.quiet, args.legacy,
  387. invalidation_mode=invalidation_mode,
  388. stripdir=args.stripdir,
  389. prependdir=args.prependdir,
  390. optimize=args.opt_levels,
  391. limit_sl_dest=args.limit_sl_dest,
  392. hardlink_dupes=args.hardlink_dupes):
  393. success = False
  394. else:
  395. if not compile_dir(dest, maxlevels, args.ddir,
  396. args.force, args.rx, args.quiet,
  397. args.legacy, workers=args.workers,
  398. invalidation_mode=invalidation_mode,
  399. stripdir=args.stripdir,
  400. prependdir=args.prependdir,
  401. optimize=args.opt_levels,
  402. limit_sl_dest=args.limit_sl_dest,
  403. hardlink_dupes=args.hardlink_dupes):
  404. success = False
  405. return success
  406. else:
  407. return compile_path(legacy=args.legacy, force=args.force,
  408. quiet=args.quiet,
  409. invalidation_mode=invalidation_mode)
  410. except KeyboardInterrupt:
  411. if args.quiet < 2:
  412. print("\n[interrupted]")
  413. return False
  414. return True
  415. if __name__ == '__main__':
  416. exit_status = int(not main())
  417. sys.exit(exit_status)