logo

oasis-root

Compiled tree of Oasis Linux based on own branch at <https://hacktivis.me/git/oasis/> git clone https://anongit.hacktivis.me/git/oasis-root.git

pickle.py (64947B)


  1. """Create portable serialized representations of Python objects.
  2. See module copyreg for a mechanism for registering custom picklers.
  3. See module pickletools source for extensive comments.
  4. Classes:
  5. Pickler
  6. Unpickler
  7. Functions:
  8. dump(object, file)
  9. dumps(object) -> string
  10. load(file) -> object
  11. loads(bytes) -> object
  12. Misc variables:
  13. __version__
  14. format_version
  15. compatible_formats
  16. """
  17. from types import FunctionType
  18. from copyreg import dispatch_table
  19. from copyreg import _extension_registry, _inverted_registry, _extension_cache
  20. from itertools import islice
  21. from functools import partial
  22. import sys
  23. from sys import maxsize
  24. from struct import pack, unpack
  25. import re
  26. import io
  27. import codecs
  28. import _compat_pickle
  29. __all__ = ["PickleError", "PicklingError", "UnpicklingError", "Pickler",
  30. "Unpickler", "dump", "dumps", "load", "loads"]
  31. try:
  32. from _pickle import PickleBuffer
  33. __all__.append("PickleBuffer")
  34. _HAVE_PICKLE_BUFFER = True
  35. except ImportError:
  36. _HAVE_PICKLE_BUFFER = False
  37. # Shortcut for use in isinstance testing
  38. bytes_types = (bytes, bytearray)
  39. # These are purely informational; no code uses these.
  40. format_version = "4.0" # File format version we write
  41. compatible_formats = ["1.0", # Original protocol 0
  42. "1.1", # Protocol 0 with INST added
  43. "1.2", # Original protocol 1
  44. "1.3", # Protocol 1 with BINFLOAT added
  45. "2.0", # Protocol 2
  46. "3.0", # Protocol 3
  47. "4.0", # Protocol 4
  48. "5.0", # Protocol 5
  49. ] # Old format versions we can read
  50. # This is the highest protocol number we know how to read.
  51. HIGHEST_PROTOCOL = 5
  52. # The protocol we write by default. May be less than HIGHEST_PROTOCOL.
  53. # Only bump this if the oldest still supported version of Python already
  54. # includes it.
  55. DEFAULT_PROTOCOL = 4
  56. class PickleError(Exception):
  57. """A common base class for the other pickling exceptions."""
  58. pass
  59. class PicklingError(PickleError):
  60. """This exception is raised when an unpicklable object is passed to the
  61. dump() method.
  62. """
  63. pass
  64. class UnpicklingError(PickleError):
  65. """This exception is raised when there is a problem unpickling an object,
  66. such as a security violation.
  67. Note that other exceptions may also be raised during unpickling, including
  68. (but not necessarily limited to) AttributeError, EOFError, ImportError,
  69. and IndexError.
  70. """
  71. pass
  72. # An instance of _Stop is raised by Unpickler.load_stop() in response to
  73. # the STOP opcode, passing the object that is the result of unpickling.
  74. class _Stop(Exception):
  75. def __init__(self, value):
  76. self.value = value
  77. # Jython has PyStringMap; it's a dict subclass with string keys
  78. try:
  79. from org.python.core import PyStringMap
  80. except ImportError:
  81. PyStringMap = None
  82. # Pickle opcodes. See pickletools.py for extensive docs. The listing
  83. # here is in kind-of alphabetical order of 1-character pickle code.
  84. # pickletools groups them by purpose.
  85. MARK = b'(' # push special markobject on stack
  86. STOP = b'.' # every pickle ends with STOP
  87. POP = b'0' # discard topmost stack item
  88. POP_MARK = b'1' # discard stack top through topmost markobject
  89. DUP = b'2' # duplicate top stack item
  90. FLOAT = b'F' # push float object; decimal string argument
  91. INT = b'I' # push integer or bool; decimal string argument
  92. BININT = b'J' # push four-byte signed int
  93. BININT1 = b'K' # push 1-byte unsigned int
  94. LONG = b'L' # push long; decimal string argument
  95. BININT2 = b'M' # push 2-byte unsigned int
  96. NONE = b'N' # push None
  97. PERSID = b'P' # push persistent object; id is taken from string arg
  98. BINPERSID = b'Q' # " " " ; " " " " stack
  99. REDUCE = b'R' # apply callable to argtuple, both on stack
  100. STRING = b'S' # push string; NL-terminated string argument
  101. BINSTRING = b'T' # push string; counted binary string argument
  102. SHORT_BINSTRING= b'U' # " " ; " " " " < 256 bytes
  103. UNICODE = b'V' # push Unicode string; raw-unicode-escaped'd argument
  104. BINUNICODE = b'X' # " " " ; counted UTF-8 string argument
  105. APPEND = b'a' # append stack top to list below it
  106. BUILD = b'b' # call __setstate__ or __dict__.update()
  107. GLOBAL = b'c' # push self.find_class(modname, name); 2 string args
  108. DICT = b'd' # build a dict from stack items
  109. EMPTY_DICT = b'}' # push empty dict
  110. APPENDS = b'e' # extend list on stack by topmost stack slice
  111. GET = b'g' # push item from memo on stack; index is string arg
  112. BINGET = b'h' # " " " " " " ; " " 1-byte arg
  113. INST = b'i' # build & push class instance
  114. LONG_BINGET = b'j' # push item from memo on stack; index is 4-byte arg
  115. LIST = b'l' # build list from topmost stack items
  116. EMPTY_LIST = b']' # push empty list
  117. OBJ = b'o' # build & push class instance
  118. PUT = b'p' # store stack top in memo; index is string arg
  119. BINPUT = b'q' # " " " " " ; " " 1-byte arg
  120. LONG_BINPUT = b'r' # " " " " " ; " " 4-byte arg
  121. SETITEM = b's' # add key+value pair to dict
  122. TUPLE = b't' # build tuple from topmost stack items
  123. EMPTY_TUPLE = b')' # push empty tuple
  124. SETITEMS = b'u' # modify dict by adding topmost key+value pairs
  125. BINFLOAT = b'G' # push float; arg is 8-byte float encoding
  126. TRUE = b'I01\n' # not an opcode; see INT docs in pickletools.py
  127. FALSE = b'I00\n' # not an opcode; see INT docs in pickletools.py
  128. # Protocol 2
  129. PROTO = b'\x80' # identify pickle protocol
  130. NEWOBJ = b'\x81' # build object by applying cls.__new__ to argtuple
  131. EXT1 = b'\x82' # push object from extension registry; 1-byte index
  132. EXT2 = b'\x83' # ditto, but 2-byte index
  133. EXT4 = b'\x84' # ditto, but 4-byte index
  134. TUPLE1 = b'\x85' # build 1-tuple from stack top
  135. TUPLE2 = b'\x86' # build 2-tuple from two topmost stack items
  136. TUPLE3 = b'\x87' # build 3-tuple from three topmost stack items
  137. NEWTRUE = b'\x88' # push True
  138. NEWFALSE = b'\x89' # push False
  139. LONG1 = b'\x8a' # push long from < 256 bytes
  140. LONG4 = b'\x8b' # push really big long
  141. _tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3]
  142. # Protocol 3 (Python 3.x)
  143. BINBYTES = b'B' # push bytes; counted binary string argument
  144. SHORT_BINBYTES = b'C' # " " ; " " " " < 256 bytes
  145. # Protocol 4
  146. SHORT_BINUNICODE = b'\x8c' # push short string; UTF-8 length < 256 bytes
  147. BINUNICODE8 = b'\x8d' # push very long string
  148. BINBYTES8 = b'\x8e' # push very long bytes string
  149. EMPTY_SET = b'\x8f' # push empty set on the stack
  150. ADDITEMS = b'\x90' # modify set by adding topmost stack items
  151. FROZENSET = b'\x91' # build frozenset from topmost stack items
  152. NEWOBJ_EX = b'\x92' # like NEWOBJ but work with keyword only arguments
  153. STACK_GLOBAL = b'\x93' # same as GLOBAL but using names on the stacks
  154. MEMOIZE = b'\x94' # store top of the stack in memo
  155. FRAME = b'\x95' # indicate the beginning of a new frame
  156. # Protocol 5
  157. BYTEARRAY8 = b'\x96' # push bytearray
  158. NEXT_BUFFER = b'\x97' # push next out-of-band buffer
  159. READONLY_BUFFER = b'\x98' # make top of stack readonly
  160. __all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$", x)])
  161. class _Framer:
  162. _FRAME_SIZE_MIN = 4
  163. _FRAME_SIZE_TARGET = 64 * 1024
  164. def __init__(self, file_write):
  165. self.file_write = file_write
  166. self.current_frame = None
  167. def start_framing(self):
  168. self.current_frame = io.BytesIO()
  169. def end_framing(self):
  170. if self.current_frame and self.current_frame.tell() > 0:
  171. self.commit_frame(force=True)
  172. self.current_frame = None
  173. def commit_frame(self, force=False):
  174. if self.current_frame:
  175. f = self.current_frame
  176. if f.tell() >= self._FRAME_SIZE_TARGET or force:
  177. data = f.getbuffer()
  178. write = self.file_write
  179. if len(data) >= self._FRAME_SIZE_MIN:
  180. # Issue a single call to the write method of the underlying
  181. # file object for the frame opcode with the size of the
  182. # frame. The concatenation is expected to be less expensive
  183. # than issuing an additional call to write.
  184. write(FRAME + pack("<Q", len(data)))
  185. # Issue a separate call to write to append the frame
  186. # contents without concatenation to the above to avoid a
  187. # memory copy.
  188. write(data)
  189. # Start the new frame with a new io.BytesIO instance so that
  190. # the file object can have delayed access to the previous frame
  191. # contents via an unreleased memoryview of the previous
  192. # io.BytesIO instance.
  193. self.current_frame = io.BytesIO()
  194. def write(self, data):
  195. if self.current_frame:
  196. return self.current_frame.write(data)
  197. else:
  198. return self.file_write(data)
  199. def write_large_bytes(self, header, payload):
  200. write = self.file_write
  201. if self.current_frame:
  202. # Terminate the current frame and flush it to the file.
  203. self.commit_frame(force=True)
  204. # Perform direct write of the header and payload of the large binary
  205. # object. Be careful not to concatenate the header and the payload
  206. # prior to calling 'write' as we do not want to allocate a large
  207. # temporary bytes object.
  208. # We intentionally do not insert a protocol 4 frame opcode to make
  209. # it possible to optimize file.read calls in the loader.
  210. write(header)
  211. write(payload)
  212. class _Unframer:
  213. def __init__(self, file_read, file_readline, file_tell=None):
  214. self.file_read = file_read
  215. self.file_readline = file_readline
  216. self.current_frame = None
  217. def readinto(self, buf):
  218. if self.current_frame:
  219. n = self.current_frame.readinto(buf)
  220. if n == 0 and len(buf) != 0:
  221. self.current_frame = None
  222. n = len(buf)
  223. buf[:] = self.file_read(n)
  224. return n
  225. if n < len(buf):
  226. raise UnpicklingError(
  227. "pickle exhausted before end of frame")
  228. return n
  229. else:
  230. n = len(buf)
  231. buf[:] = self.file_read(n)
  232. return n
  233. def read(self, n):
  234. if self.current_frame:
  235. data = self.current_frame.read(n)
  236. if not data and n != 0:
  237. self.current_frame = None
  238. return self.file_read(n)
  239. if len(data) < n:
  240. raise UnpicklingError(
  241. "pickle exhausted before end of frame")
  242. return data
  243. else:
  244. return self.file_read(n)
  245. def readline(self):
  246. if self.current_frame:
  247. data = self.current_frame.readline()
  248. if not data:
  249. self.current_frame = None
  250. return self.file_readline()
  251. if data[-1] != b'\n'[0]:
  252. raise UnpicklingError(
  253. "pickle exhausted before end of frame")
  254. return data
  255. else:
  256. return self.file_readline()
  257. def load_frame(self, frame_size):
  258. if self.current_frame and self.current_frame.read() != b'':
  259. raise UnpicklingError(
  260. "beginning of a new frame before end of current frame")
  261. self.current_frame = io.BytesIO(self.file_read(frame_size))
  262. # Tools used for pickling.
  263. def _getattribute(obj, name):
  264. for subpath in name.split('.'):
  265. if subpath == '<locals>':
  266. raise AttributeError("Can't get local attribute {!r} on {!r}"
  267. .format(name, obj))
  268. try:
  269. parent = obj
  270. obj = getattr(obj, subpath)
  271. except AttributeError:
  272. raise AttributeError("Can't get attribute {!r} on {!r}"
  273. .format(name, obj)) from None
  274. return obj, parent
  275. def whichmodule(obj, name):
  276. """Find the module an object belong to."""
  277. module_name = getattr(obj, '__module__', None)
  278. if module_name is not None:
  279. return module_name
  280. # Protect the iteration by using a list copy of sys.modules against dynamic
  281. # modules that trigger imports of other modules upon calls to getattr.
  282. for module_name, module in sys.modules.copy().items():
  283. if (module_name == '__main__'
  284. or module_name == '__mp_main__' # bpo-42406
  285. or module is None):
  286. continue
  287. try:
  288. if _getattribute(module, name)[0] is obj:
  289. return module_name
  290. except AttributeError:
  291. pass
  292. return '__main__'
  293. def encode_long(x):
  294. r"""Encode a long to a two's complement little-endian binary string.
  295. Note that 0 is a special case, returning an empty string, to save a
  296. byte in the LONG1 pickling context.
  297. >>> encode_long(0)
  298. b''
  299. >>> encode_long(255)
  300. b'\xff\x00'
  301. >>> encode_long(32767)
  302. b'\xff\x7f'
  303. >>> encode_long(-256)
  304. b'\x00\xff'
  305. >>> encode_long(-32768)
  306. b'\x00\x80'
  307. >>> encode_long(-128)
  308. b'\x80'
  309. >>> encode_long(127)
  310. b'\x7f'
  311. >>>
  312. """
  313. if x == 0:
  314. return b''
  315. nbytes = (x.bit_length() >> 3) + 1
  316. result = x.to_bytes(nbytes, byteorder='little', signed=True)
  317. if x < 0 and nbytes > 1:
  318. if result[-1] == 0xff and (result[-2] & 0x80) != 0:
  319. result = result[:-1]
  320. return result
  321. def decode_long(data):
  322. r"""Decode a long from a two's complement little-endian binary string.
  323. >>> decode_long(b'')
  324. 0
  325. >>> decode_long(b"\xff\x00")
  326. 255
  327. >>> decode_long(b"\xff\x7f")
  328. 32767
  329. >>> decode_long(b"\x00\xff")
  330. -256
  331. >>> decode_long(b"\x00\x80")
  332. -32768
  333. >>> decode_long(b"\x80")
  334. -128
  335. >>> decode_long(b"\x7f")
  336. 127
  337. """
  338. return int.from_bytes(data, byteorder='little', signed=True)
  339. # Pickling machinery
  340. class _Pickler:
  341. def __init__(self, file, protocol=None, *, fix_imports=True,
  342. buffer_callback=None):
  343. """This takes a binary file for writing a pickle data stream.
  344. The optional *protocol* argument tells the pickler to use the
  345. given protocol; supported protocols are 0, 1, 2, 3, 4 and 5.
  346. The default protocol is 4. It was introduced in Python 3.4, and
  347. is incompatible with previous versions.
  348. Specifying a negative protocol version selects the highest
  349. protocol version supported. The higher the protocol used, the
  350. more recent the version of Python needed to read the pickle
  351. produced.
  352. The *file* argument must have a write() method that accepts a
  353. single bytes argument. It can thus be a file object opened for
  354. binary writing, an io.BytesIO instance, or any other custom
  355. object that meets this interface.
  356. If *fix_imports* is True and *protocol* is less than 3, pickle
  357. will try to map the new Python 3 names to the old module names
  358. used in Python 2, so that the pickle data stream is readable
  359. with Python 2.
  360. If *buffer_callback* is None (the default), buffer views are
  361. serialized into *file* as part of the pickle stream.
  362. If *buffer_callback* is not None, then it can be called any number
  363. of times with a buffer view. If the callback returns a false value
  364. (such as None), the given buffer is out-of-band; otherwise the
  365. buffer is serialized in-band, i.e. inside the pickle stream.
  366. It is an error if *buffer_callback* is not None and *protocol*
  367. is None or smaller than 5.
  368. """
  369. if protocol is None:
  370. protocol = DEFAULT_PROTOCOL
  371. if protocol < 0:
  372. protocol = HIGHEST_PROTOCOL
  373. elif not 0 <= protocol <= HIGHEST_PROTOCOL:
  374. raise ValueError("pickle protocol must be <= %d" % HIGHEST_PROTOCOL)
  375. if buffer_callback is not None and protocol < 5:
  376. raise ValueError("buffer_callback needs protocol >= 5")
  377. self._buffer_callback = buffer_callback
  378. try:
  379. self._file_write = file.write
  380. except AttributeError:
  381. raise TypeError("file must have a 'write' attribute")
  382. self.framer = _Framer(self._file_write)
  383. self.write = self.framer.write
  384. self._write_large_bytes = self.framer.write_large_bytes
  385. self.memo = {}
  386. self.proto = int(protocol)
  387. self.bin = protocol >= 1
  388. self.fast = 0
  389. self.fix_imports = fix_imports and protocol < 3
  390. def clear_memo(self):
  391. """Clears the pickler's "memo".
  392. The memo is the data structure that remembers which objects the
  393. pickler has already seen, so that shared or recursive objects
  394. are pickled by reference and not by value. This method is
  395. useful when re-using picklers.
  396. """
  397. self.memo.clear()
  398. def dump(self, obj):
  399. """Write a pickled representation of obj to the open file."""
  400. # Check whether Pickler was initialized correctly. This is
  401. # only needed to mimic the behavior of _pickle.Pickler.dump().
  402. if not hasattr(self, "_file_write"):
  403. raise PicklingError("Pickler.__init__() was not called by "
  404. "%s.__init__()" % (self.__class__.__name__,))
  405. if self.proto >= 2:
  406. self.write(PROTO + pack("<B", self.proto))
  407. if self.proto >= 4:
  408. self.framer.start_framing()
  409. self.save(obj)
  410. self.write(STOP)
  411. self.framer.end_framing()
  412. def memoize(self, obj):
  413. """Store an object in the memo."""
  414. # The Pickler memo is a dictionary mapping object ids to 2-tuples
  415. # that contain the Unpickler memo key and the object being memoized.
  416. # The memo key is written to the pickle and will become
  417. # the key in the Unpickler's memo. The object is stored in the
  418. # Pickler memo so that transient objects are kept alive during
  419. # pickling.
  420. # The use of the Unpickler memo length as the memo key is just a
  421. # convention. The only requirement is that the memo values be unique.
  422. # But there appears no advantage to any other scheme, and this
  423. # scheme allows the Unpickler memo to be implemented as a plain (but
  424. # growable) array, indexed by memo key.
  425. if self.fast:
  426. return
  427. assert id(obj) not in self.memo
  428. idx = len(self.memo)
  429. self.write(self.put(idx))
  430. self.memo[id(obj)] = idx, obj
  431. # Return a PUT (BINPUT, LONG_BINPUT) opcode string, with argument i.
  432. def put(self, idx):
  433. if self.proto >= 4:
  434. return MEMOIZE
  435. elif self.bin:
  436. if idx < 256:
  437. return BINPUT + pack("<B", idx)
  438. else:
  439. return LONG_BINPUT + pack("<I", idx)
  440. else:
  441. return PUT + repr(idx).encode("ascii") + b'\n'
  442. # Return a GET (BINGET, LONG_BINGET) opcode string, with argument i.
  443. def get(self, i):
  444. if self.bin:
  445. if i < 256:
  446. return BINGET + pack("<B", i)
  447. else:
  448. return LONG_BINGET + pack("<I", i)
  449. return GET + repr(i).encode("ascii") + b'\n'
  450. def save(self, obj, save_persistent_id=True):
  451. self.framer.commit_frame()
  452. # Check for persistent id (defined by a subclass)
  453. pid = self.persistent_id(obj)
  454. if pid is not None and save_persistent_id:
  455. self.save_pers(pid)
  456. return
  457. # Check the memo
  458. x = self.memo.get(id(obj))
  459. if x is not None:
  460. self.write(self.get(x[0]))
  461. return
  462. rv = NotImplemented
  463. reduce = getattr(self, "reducer_override", None)
  464. if reduce is not None:
  465. rv = reduce(obj)
  466. if rv is NotImplemented:
  467. # Check the type dispatch table
  468. t = type(obj)
  469. f = self.dispatch.get(t)
  470. if f is not None:
  471. f(self, obj) # Call unbound method with explicit self
  472. return
  473. # Check private dispatch table if any, or else
  474. # copyreg.dispatch_table
  475. reduce = getattr(self, 'dispatch_table', dispatch_table).get(t)
  476. if reduce is not None:
  477. rv = reduce(obj)
  478. else:
  479. # Check for a class with a custom metaclass; treat as regular
  480. # class
  481. if issubclass(t, type):
  482. self.save_global(obj)
  483. return
  484. # Check for a __reduce_ex__ method, fall back to __reduce__
  485. reduce = getattr(obj, "__reduce_ex__", None)
  486. if reduce is not None:
  487. rv = reduce(self.proto)
  488. else:
  489. reduce = getattr(obj, "__reduce__", None)
  490. if reduce is not None:
  491. rv = reduce()
  492. else:
  493. raise PicklingError("Can't pickle %r object: %r" %
  494. (t.__name__, obj))
  495. # Check for string returned by reduce(), meaning "save as global"
  496. if isinstance(rv, str):
  497. self.save_global(obj, rv)
  498. return
  499. # Assert that reduce() returned a tuple
  500. if not isinstance(rv, tuple):
  501. raise PicklingError("%s must return string or tuple" % reduce)
  502. # Assert that it returned an appropriately sized tuple
  503. l = len(rv)
  504. if not (2 <= l <= 6):
  505. raise PicklingError("Tuple returned by %s must have "
  506. "two to six elements" % reduce)
  507. # Save the reduce() output and finally memoize the object
  508. self.save_reduce(obj=obj, *rv)
  509. def persistent_id(self, obj):
  510. # This exists so a subclass can override it
  511. return None
  512. def save_pers(self, pid):
  513. # Save a persistent id reference
  514. if self.bin:
  515. self.save(pid, save_persistent_id=False)
  516. self.write(BINPERSID)
  517. else:
  518. try:
  519. self.write(PERSID + str(pid).encode("ascii") + b'\n')
  520. except UnicodeEncodeError:
  521. raise PicklingError(
  522. "persistent IDs in protocol 0 must be ASCII strings")
  523. def save_reduce(self, func, args, state=None, listitems=None,
  524. dictitems=None, state_setter=None, obj=None):
  525. # This API is called by some subclasses
  526. if not isinstance(args, tuple):
  527. raise PicklingError("args from save_reduce() must be a tuple")
  528. if not callable(func):
  529. raise PicklingError("func from save_reduce() must be callable")
  530. save = self.save
  531. write = self.write
  532. func_name = getattr(func, "__name__", "")
  533. if self.proto >= 2 and func_name == "__newobj_ex__":
  534. cls, args, kwargs = args
  535. if not hasattr(cls, "__new__"):
  536. raise PicklingError("args[0] from {} args has no __new__"
  537. .format(func_name))
  538. if obj is not None and cls is not obj.__class__:
  539. raise PicklingError("args[0] from {} args has the wrong class"
  540. .format(func_name))
  541. if self.proto >= 4:
  542. save(cls)
  543. save(args)
  544. save(kwargs)
  545. write(NEWOBJ_EX)
  546. else:
  547. func = partial(cls.__new__, cls, *args, **kwargs)
  548. save(func)
  549. save(())
  550. write(REDUCE)
  551. elif self.proto >= 2 and func_name == "__newobj__":
  552. # A __reduce__ implementation can direct protocol 2 or newer to
  553. # use the more efficient NEWOBJ opcode, while still
  554. # allowing protocol 0 and 1 to work normally. For this to
  555. # work, the function returned by __reduce__ should be
  556. # called __newobj__, and its first argument should be a
  557. # class. The implementation for __newobj__
  558. # should be as follows, although pickle has no way to
  559. # verify this:
  560. #
  561. # def __newobj__(cls, *args):
  562. # return cls.__new__(cls, *args)
  563. #
  564. # Protocols 0 and 1 will pickle a reference to __newobj__,
  565. # while protocol 2 (and above) will pickle a reference to
  566. # cls, the remaining args tuple, and the NEWOBJ code,
  567. # which calls cls.__new__(cls, *args) at unpickling time
  568. # (see load_newobj below). If __reduce__ returns a
  569. # three-tuple, the state from the third tuple item will be
  570. # pickled regardless of the protocol, calling __setstate__
  571. # at unpickling time (see load_build below).
  572. #
  573. # Note that no standard __newobj__ implementation exists;
  574. # you have to provide your own. This is to enforce
  575. # compatibility with Python 2.2 (pickles written using
  576. # protocol 0 or 1 in Python 2.3 should be unpicklable by
  577. # Python 2.2).
  578. cls = args[0]
  579. if not hasattr(cls, "__new__"):
  580. raise PicklingError(
  581. "args[0] from __newobj__ args has no __new__")
  582. if obj is not None and cls is not obj.__class__:
  583. raise PicklingError(
  584. "args[0] from __newobj__ args has the wrong class")
  585. args = args[1:]
  586. save(cls)
  587. save(args)
  588. write(NEWOBJ)
  589. else:
  590. save(func)
  591. save(args)
  592. write(REDUCE)
  593. if obj is not None:
  594. # If the object is already in the memo, this means it is
  595. # recursive. In this case, throw away everything we put on the
  596. # stack, and fetch the object back from the memo.
  597. if id(obj) in self.memo:
  598. write(POP + self.get(self.memo[id(obj)][0]))
  599. else:
  600. self.memoize(obj)
  601. # More new special cases (that work with older protocols as
  602. # well): when __reduce__ returns a tuple with 4 or 5 items,
  603. # the 4th and 5th item should be iterators that provide list
  604. # items and dict items (as (key, value) tuples), or None.
  605. if listitems is not None:
  606. self._batch_appends(listitems)
  607. if dictitems is not None:
  608. self._batch_setitems(dictitems)
  609. if state is not None:
  610. if state_setter is None:
  611. save(state)
  612. write(BUILD)
  613. else:
  614. # If a state_setter is specified, call it instead of load_build
  615. # to update obj's with its previous state.
  616. # First, push state_setter and its tuple of expected arguments
  617. # (obj, state) onto the stack.
  618. save(state_setter)
  619. save(obj) # simple BINGET opcode as obj is already memoized.
  620. save(state)
  621. write(TUPLE2)
  622. # Trigger a state_setter(obj, state) function call.
  623. write(REDUCE)
  624. # The purpose of state_setter is to carry-out an
  625. # inplace modification of obj. We do not care about what the
  626. # method might return, so its output is eventually removed from
  627. # the stack.
  628. write(POP)
  629. # Methods below this point are dispatched through the dispatch table
  630. dispatch = {}
  631. def save_none(self, obj):
  632. self.write(NONE)
  633. dispatch[type(None)] = save_none
  634. def save_bool(self, obj):
  635. if self.proto >= 2:
  636. self.write(NEWTRUE if obj else NEWFALSE)
  637. else:
  638. self.write(TRUE if obj else FALSE)
  639. dispatch[bool] = save_bool
  640. def save_long(self, obj):
  641. if self.bin:
  642. # If the int is small enough to fit in a signed 4-byte 2's-comp
  643. # format, we can store it more efficiently than the general
  644. # case.
  645. # First one- and two-byte unsigned ints:
  646. if obj >= 0:
  647. if obj <= 0xff:
  648. self.write(BININT1 + pack("<B", obj))
  649. return
  650. if obj <= 0xffff:
  651. self.write(BININT2 + pack("<H", obj))
  652. return
  653. # Next check for 4-byte signed ints:
  654. if -0x80000000 <= obj <= 0x7fffffff:
  655. self.write(BININT + pack("<i", obj))
  656. return
  657. if self.proto >= 2:
  658. encoded = encode_long(obj)
  659. n = len(encoded)
  660. if n < 256:
  661. self.write(LONG1 + pack("<B", n) + encoded)
  662. else:
  663. self.write(LONG4 + pack("<i", n) + encoded)
  664. return
  665. if -0x80000000 <= obj <= 0x7fffffff:
  666. self.write(INT + repr(obj).encode("ascii") + b'\n')
  667. else:
  668. self.write(LONG + repr(obj).encode("ascii") + b'L\n')
  669. dispatch[int] = save_long
  670. def save_float(self, obj):
  671. if self.bin:
  672. self.write(BINFLOAT + pack('>d', obj))
  673. else:
  674. self.write(FLOAT + repr(obj).encode("ascii") + b'\n')
  675. dispatch[float] = save_float
  676. def save_bytes(self, obj):
  677. if self.proto < 3:
  678. if not obj: # bytes object is empty
  679. self.save_reduce(bytes, (), obj=obj)
  680. else:
  681. self.save_reduce(codecs.encode,
  682. (str(obj, 'latin1'), 'latin1'), obj=obj)
  683. return
  684. n = len(obj)
  685. if n <= 0xff:
  686. self.write(SHORT_BINBYTES + pack("<B", n) + obj)
  687. elif n > 0xffffffff and self.proto >= 4:
  688. self._write_large_bytes(BINBYTES8 + pack("<Q", n), obj)
  689. elif n >= self.framer._FRAME_SIZE_TARGET:
  690. self._write_large_bytes(BINBYTES + pack("<I", n), obj)
  691. else:
  692. self.write(BINBYTES + pack("<I", n) + obj)
  693. self.memoize(obj)
  694. dispatch[bytes] = save_bytes
  695. def save_bytearray(self, obj):
  696. if self.proto < 5:
  697. if not obj: # bytearray is empty
  698. self.save_reduce(bytearray, (), obj=obj)
  699. else:
  700. self.save_reduce(bytearray, (bytes(obj),), obj=obj)
  701. return
  702. n = len(obj)
  703. if n >= self.framer._FRAME_SIZE_TARGET:
  704. self._write_large_bytes(BYTEARRAY8 + pack("<Q", n), obj)
  705. else:
  706. self.write(BYTEARRAY8 + pack("<Q", n) + obj)
  707. self.memoize(obj)
  708. dispatch[bytearray] = save_bytearray
  709. if _HAVE_PICKLE_BUFFER:
  710. def save_picklebuffer(self, obj):
  711. if self.proto < 5:
  712. raise PicklingError("PickleBuffer can only pickled with "
  713. "protocol >= 5")
  714. with obj.raw() as m:
  715. if not m.contiguous:
  716. raise PicklingError("PickleBuffer can not be pickled when "
  717. "pointing to a non-contiguous buffer")
  718. in_band = True
  719. if self._buffer_callback is not None:
  720. in_band = bool(self._buffer_callback(obj))
  721. if in_band:
  722. # Write data in-band
  723. # XXX The C implementation avoids a copy here
  724. if m.readonly:
  725. self.save_bytes(m.tobytes())
  726. else:
  727. self.save_bytearray(m.tobytes())
  728. else:
  729. # Write data out-of-band
  730. self.write(NEXT_BUFFER)
  731. if m.readonly:
  732. self.write(READONLY_BUFFER)
  733. dispatch[PickleBuffer] = save_picklebuffer
  734. def save_str(self, obj):
  735. if self.bin:
  736. encoded = obj.encode('utf-8', 'surrogatepass')
  737. n = len(encoded)
  738. if n <= 0xff and self.proto >= 4:
  739. self.write(SHORT_BINUNICODE + pack("<B", n) + encoded)
  740. elif n > 0xffffffff and self.proto >= 4:
  741. self._write_large_bytes(BINUNICODE8 + pack("<Q", n), encoded)
  742. elif n >= self.framer._FRAME_SIZE_TARGET:
  743. self._write_large_bytes(BINUNICODE + pack("<I", n), encoded)
  744. else:
  745. self.write(BINUNICODE + pack("<I", n) + encoded)
  746. else:
  747. obj = obj.replace("\\", "\\u005c")
  748. obj = obj.replace("\0", "\\u0000")
  749. obj = obj.replace("\n", "\\u000a")
  750. obj = obj.replace("\r", "\\u000d")
  751. obj = obj.replace("\x1a", "\\u001a") # EOF on DOS
  752. self.write(UNICODE + obj.encode('raw-unicode-escape') +
  753. b'\n')
  754. self.memoize(obj)
  755. dispatch[str] = save_str
  756. def save_tuple(self, obj):
  757. if not obj: # tuple is empty
  758. if self.bin:
  759. self.write(EMPTY_TUPLE)
  760. else:
  761. self.write(MARK + TUPLE)
  762. return
  763. n = len(obj)
  764. save = self.save
  765. memo = self.memo
  766. if n <= 3 and self.proto >= 2:
  767. for element in obj:
  768. save(element)
  769. # Subtle. Same as in the big comment below.
  770. if id(obj) in memo:
  771. get = self.get(memo[id(obj)][0])
  772. self.write(POP * n + get)
  773. else:
  774. self.write(_tuplesize2code[n])
  775. self.memoize(obj)
  776. return
  777. # proto 0 or proto 1 and tuple isn't empty, or proto > 1 and tuple
  778. # has more than 3 elements.
  779. write = self.write
  780. write(MARK)
  781. for element in obj:
  782. save(element)
  783. if id(obj) in memo:
  784. # Subtle. d was not in memo when we entered save_tuple(), so
  785. # the process of saving the tuple's elements must have saved
  786. # the tuple itself: the tuple is recursive. The proper action
  787. # now is to throw away everything we put on the stack, and
  788. # simply GET the tuple (it's already constructed). This check
  789. # could have been done in the "for element" loop instead, but
  790. # recursive tuples are a rare thing.
  791. get = self.get(memo[id(obj)][0])
  792. if self.bin:
  793. write(POP_MARK + get)
  794. else: # proto 0 -- POP_MARK not available
  795. write(POP * (n+1) + get)
  796. return
  797. # No recursion.
  798. write(TUPLE)
  799. self.memoize(obj)
  800. dispatch[tuple] = save_tuple
  801. def save_list(self, obj):
  802. if self.bin:
  803. self.write(EMPTY_LIST)
  804. else: # proto 0 -- can't use EMPTY_LIST
  805. self.write(MARK + LIST)
  806. self.memoize(obj)
  807. self._batch_appends(obj)
  808. dispatch[list] = save_list
  809. _BATCHSIZE = 1000
  810. def _batch_appends(self, items):
  811. # Helper to batch up APPENDS sequences
  812. save = self.save
  813. write = self.write
  814. if not self.bin:
  815. for x in items:
  816. save(x)
  817. write(APPEND)
  818. return
  819. it = iter(items)
  820. while True:
  821. tmp = list(islice(it, self._BATCHSIZE))
  822. n = len(tmp)
  823. if n > 1:
  824. write(MARK)
  825. for x in tmp:
  826. save(x)
  827. write(APPENDS)
  828. elif n:
  829. save(tmp[0])
  830. write(APPEND)
  831. # else tmp is empty, and we're done
  832. if n < self._BATCHSIZE:
  833. return
  834. def save_dict(self, obj):
  835. if self.bin:
  836. self.write(EMPTY_DICT)
  837. else: # proto 0 -- can't use EMPTY_DICT
  838. self.write(MARK + DICT)
  839. self.memoize(obj)
  840. self._batch_setitems(obj.items())
  841. dispatch[dict] = save_dict
  842. if PyStringMap is not None:
  843. dispatch[PyStringMap] = save_dict
  844. def _batch_setitems(self, items):
  845. # Helper to batch up SETITEMS sequences; proto >= 1 only
  846. save = self.save
  847. write = self.write
  848. if not self.bin:
  849. for k, v in items:
  850. save(k)
  851. save(v)
  852. write(SETITEM)
  853. return
  854. it = iter(items)
  855. while True:
  856. tmp = list(islice(it, self._BATCHSIZE))
  857. n = len(tmp)
  858. if n > 1:
  859. write(MARK)
  860. for k, v in tmp:
  861. save(k)
  862. save(v)
  863. write(SETITEMS)
  864. elif n:
  865. k, v = tmp[0]
  866. save(k)
  867. save(v)
  868. write(SETITEM)
  869. # else tmp is empty, and we're done
  870. if n < self._BATCHSIZE:
  871. return
  872. def save_set(self, obj):
  873. save = self.save
  874. write = self.write
  875. if self.proto < 4:
  876. self.save_reduce(set, (list(obj),), obj=obj)
  877. return
  878. write(EMPTY_SET)
  879. self.memoize(obj)
  880. it = iter(obj)
  881. while True:
  882. batch = list(islice(it, self._BATCHSIZE))
  883. n = len(batch)
  884. if n > 0:
  885. write(MARK)
  886. for item in batch:
  887. save(item)
  888. write(ADDITEMS)
  889. if n < self._BATCHSIZE:
  890. return
  891. dispatch[set] = save_set
  892. def save_frozenset(self, obj):
  893. save = self.save
  894. write = self.write
  895. if self.proto < 4:
  896. self.save_reduce(frozenset, (list(obj),), obj=obj)
  897. return
  898. write(MARK)
  899. for item in obj:
  900. save(item)
  901. if id(obj) in self.memo:
  902. # If the object is already in the memo, this means it is
  903. # recursive. In this case, throw away everything we put on the
  904. # stack, and fetch the object back from the memo.
  905. write(POP_MARK + self.get(self.memo[id(obj)][0]))
  906. return
  907. write(FROZENSET)
  908. self.memoize(obj)
  909. dispatch[frozenset] = save_frozenset
  910. def save_global(self, obj, name=None):
  911. write = self.write
  912. memo = self.memo
  913. if name is None:
  914. name = getattr(obj, '__qualname__', None)
  915. if name is None:
  916. name = obj.__name__
  917. module_name = whichmodule(obj, name)
  918. try:
  919. __import__(module_name, level=0)
  920. module = sys.modules[module_name]
  921. obj2, parent = _getattribute(module, name)
  922. except (ImportError, KeyError, AttributeError):
  923. raise PicklingError(
  924. "Can't pickle %r: it's not found as %s.%s" %
  925. (obj, module_name, name)) from None
  926. else:
  927. if obj2 is not obj:
  928. raise PicklingError(
  929. "Can't pickle %r: it's not the same object as %s.%s" %
  930. (obj, module_name, name))
  931. if self.proto >= 2:
  932. code = _extension_registry.get((module_name, name))
  933. if code:
  934. assert code > 0
  935. if code <= 0xff:
  936. write(EXT1 + pack("<B", code))
  937. elif code <= 0xffff:
  938. write(EXT2 + pack("<H", code))
  939. else:
  940. write(EXT4 + pack("<i", code))
  941. return
  942. lastname = name.rpartition('.')[2]
  943. if parent is module:
  944. name = lastname
  945. # Non-ASCII identifiers are supported only with protocols >= 3.
  946. if self.proto >= 4:
  947. self.save(module_name)
  948. self.save(name)
  949. write(STACK_GLOBAL)
  950. elif parent is not module:
  951. self.save_reduce(getattr, (parent, lastname))
  952. elif self.proto >= 3:
  953. write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
  954. bytes(name, "utf-8") + b'\n')
  955. else:
  956. if self.fix_imports:
  957. r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
  958. r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
  959. if (module_name, name) in r_name_mapping:
  960. module_name, name = r_name_mapping[(module_name, name)]
  961. elif module_name in r_import_mapping:
  962. module_name = r_import_mapping[module_name]
  963. try:
  964. write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
  965. bytes(name, "ascii") + b'\n')
  966. except UnicodeEncodeError:
  967. raise PicklingError(
  968. "can't pickle global identifier '%s.%s' using "
  969. "pickle protocol %i" % (module, name, self.proto)) from None
  970. self.memoize(obj)
  971. def save_type(self, obj):
  972. if obj is type(None):
  973. return self.save_reduce(type, (None,), obj=obj)
  974. elif obj is type(NotImplemented):
  975. return self.save_reduce(type, (NotImplemented,), obj=obj)
  976. elif obj is type(...):
  977. return self.save_reduce(type, (...,), obj=obj)
  978. return self.save_global(obj)
  979. dispatch[FunctionType] = save_global
  980. dispatch[type] = save_type
  981. # Unpickling machinery
  982. class _Unpickler:
  983. def __init__(self, file, *, fix_imports=True,
  984. encoding="ASCII", errors="strict", buffers=None):
  985. """This takes a binary file for reading a pickle data stream.
  986. The protocol version of the pickle is detected automatically, so
  987. no proto argument is needed.
  988. The argument *file* must have two methods, a read() method that
  989. takes an integer argument, and a readline() method that requires
  990. no arguments. Both methods should return bytes. Thus *file*
  991. can be a binary file object opened for reading, an io.BytesIO
  992. object, or any other custom object that meets this interface.
  993. The file-like object must have two methods, a read() method
  994. that takes an integer argument, and a readline() method that
  995. requires no arguments. Both methods should return bytes.
  996. Thus file-like object can be a binary file object opened for
  997. reading, a BytesIO object, or any other custom object that
  998. meets this interface.
  999. If *buffers* is not None, it should be an iterable of buffer-enabled
  1000. objects that is consumed each time the pickle stream references
  1001. an out-of-band buffer view. Such buffers have been given in order
  1002. to the *buffer_callback* of a Pickler object.
  1003. If *buffers* is None (the default), then the buffers are taken
  1004. from the pickle stream, assuming they are serialized there.
  1005. It is an error for *buffers* to be None if the pickle stream
  1006. was produced with a non-None *buffer_callback*.
  1007. Other optional arguments are *fix_imports*, *encoding* and
  1008. *errors*, which are used to control compatibility support for
  1009. pickle stream generated by Python 2. If *fix_imports* is True,
  1010. pickle will try to map the old Python 2 names to the new names
  1011. used in Python 3. The *encoding* and *errors* tell pickle how
  1012. to decode 8-bit string instances pickled by Python 2; these
  1013. default to 'ASCII' and 'strict', respectively. *encoding* can be
  1014. 'bytes' to read theses 8-bit string instances as bytes objects.
  1015. """
  1016. self._buffers = iter(buffers) if buffers is not None else None
  1017. self._file_readline = file.readline
  1018. self._file_read = file.read
  1019. self.memo = {}
  1020. self.encoding = encoding
  1021. self.errors = errors
  1022. self.proto = 0
  1023. self.fix_imports = fix_imports
  1024. def load(self):
  1025. """Read a pickled object representation from the open file.
  1026. Return the reconstituted object hierarchy specified in the file.
  1027. """
  1028. # Check whether Unpickler was initialized correctly. This is
  1029. # only needed to mimic the behavior of _pickle.Unpickler.dump().
  1030. if not hasattr(self, "_file_read"):
  1031. raise UnpicklingError("Unpickler.__init__() was not called by "
  1032. "%s.__init__()" % (self.__class__.__name__,))
  1033. self._unframer = _Unframer(self._file_read, self._file_readline)
  1034. self.read = self._unframer.read
  1035. self.readinto = self._unframer.readinto
  1036. self.readline = self._unframer.readline
  1037. self.metastack = []
  1038. self.stack = []
  1039. self.append = self.stack.append
  1040. self.proto = 0
  1041. read = self.read
  1042. dispatch = self.dispatch
  1043. try:
  1044. while True:
  1045. key = read(1)
  1046. if not key:
  1047. raise EOFError
  1048. assert isinstance(key, bytes_types)
  1049. dispatch[key[0]](self)
  1050. except _Stop as stopinst:
  1051. return stopinst.value
  1052. # Return a list of items pushed in the stack after last MARK instruction.
  1053. def pop_mark(self):
  1054. items = self.stack
  1055. self.stack = self.metastack.pop()
  1056. self.append = self.stack.append
  1057. return items
  1058. def persistent_load(self, pid):
  1059. raise UnpicklingError("unsupported persistent id encountered")
  1060. dispatch = {}
  1061. def load_proto(self):
  1062. proto = self.read(1)[0]
  1063. if not 0 <= proto <= HIGHEST_PROTOCOL:
  1064. raise ValueError("unsupported pickle protocol: %d" % proto)
  1065. self.proto = proto
  1066. dispatch[PROTO[0]] = load_proto
  1067. def load_frame(self):
  1068. frame_size, = unpack('<Q', self.read(8))
  1069. if frame_size > sys.maxsize:
  1070. raise ValueError("frame size > sys.maxsize: %d" % frame_size)
  1071. self._unframer.load_frame(frame_size)
  1072. dispatch[FRAME[0]] = load_frame
  1073. def load_persid(self):
  1074. try:
  1075. pid = self.readline()[:-1].decode("ascii")
  1076. except UnicodeDecodeError:
  1077. raise UnpicklingError(
  1078. "persistent IDs in protocol 0 must be ASCII strings")
  1079. self.append(self.persistent_load(pid))
  1080. dispatch[PERSID[0]] = load_persid
  1081. def load_binpersid(self):
  1082. pid = self.stack.pop()
  1083. self.append(self.persistent_load(pid))
  1084. dispatch[BINPERSID[0]] = load_binpersid
  1085. def load_none(self):
  1086. self.append(None)
  1087. dispatch[NONE[0]] = load_none
  1088. def load_false(self):
  1089. self.append(False)
  1090. dispatch[NEWFALSE[0]] = load_false
  1091. def load_true(self):
  1092. self.append(True)
  1093. dispatch[NEWTRUE[0]] = load_true
  1094. def load_int(self):
  1095. data = self.readline()
  1096. if data == FALSE[1:]:
  1097. val = False
  1098. elif data == TRUE[1:]:
  1099. val = True
  1100. else:
  1101. val = int(data, 0)
  1102. self.append(val)
  1103. dispatch[INT[0]] = load_int
  1104. def load_binint(self):
  1105. self.append(unpack('<i', self.read(4))[0])
  1106. dispatch[BININT[0]] = load_binint
  1107. def load_binint1(self):
  1108. self.append(self.read(1)[0])
  1109. dispatch[BININT1[0]] = load_binint1
  1110. def load_binint2(self):
  1111. self.append(unpack('<H', self.read(2))[0])
  1112. dispatch[BININT2[0]] = load_binint2
  1113. def load_long(self):
  1114. val = self.readline()[:-1]
  1115. if val and val[-1] == b'L'[0]:
  1116. val = val[:-1]
  1117. self.append(int(val, 0))
  1118. dispatch[LONG[0]] = load_long
  1119. def load_long1(self):
  1120. n = self.read(1)[0]
  1121. data = self.read(n)
  1122. self.append(decode_long(data))
  1123. dispatch[LONG1[0]] = load_long1
  1124. def load_long4(self):
  1125. n, = unpack('<i', self.read(4))
  1126. if n < 0:
  1127. # Corrupt or hostile pickle -- we never write one like this
  1128. raise UnpicklingError("LONG pickle has negative byte count")
  1129. data = self.read(n)
  1130. self.append(decode_long(data))
  1131. dispatch[LONG4[0]] = load_long4
  1132. def load_float(self):
  1133. self.append(float(self.readline()[:-1]))
  1134. dispatch[FLOAT[0]] = load_float
  1135. def load_binfloat(self):
  1136. self.append(unpack('>d', self.read(8))[0])
  1137. dispatch[BINFLOAT[0]] = load_binfloat
  1138. def _decode_string(self, value):
  1139. # Used to allow strings from Python 2 to be decoded either as
  1140. # bytes or Unicode strings. This should be used only with the
  1141. # STRING, BINSTRING and SHORT_BINSTRING opcodes.
  1142. if self.encoding == "bytes":
  1143. return value
  1144. else:
  1145. return value.decode(self.encoding, self.errors)
  1146. def load_string(self):
  1147. data = self.readline()[:-1]
  1148. # Strip outermost quotes
  1149. if len(data) >= 2 and data[0] == data[-1] and data[0] in b'"\'':
  1150. data = data[1:-1]
  1151. else:
  1152. raise UnpicklingError("the STRING opcode argument must be quoted")
  1153. self.append(self._decode_string(codecs.escape_decode(data)[0]))
  1154. dispatch[STRING[0]] = load_string
  1155. def load_binstring(self):
  1156. # Deprecated BINSTRING uses signed 32-bit length
  1157. len, = unpack('<i', self.read(4))
  1158. if len < 0:
  1159. raise UnpicklingError("BINSTRING pickle has negative byte count")
  1160. data = self.read(len)
  1161. self.append(self._decode_string(data))
  1162. dispatch[BINSTRING[0]] = load_binstring
  1163. def load_binbytes(self):
  1164. len, = unpack('<I', self.read(4))
  1165. if len > maxsize:
  1166. raise UnpicklingError("BINBYTES exceeds system's maximum size "
  1167. "of %d bytes" % maxsize)
  1168. self.append(self.read(len))
  1169. dispatch[BINBYTES[0]] = load_binbytes
  1170. def load_unicode(self):
  1171. self.append(str(self.readline()[:-1], 'raw-unicode-escape'))
  1172. dispatch[UNICODE[0]] = load_unicode
  1173. def load_binunicode(self):
  1174. len, = unpack('<I', self.read(4))
  1175. if len > maxsize:
  1176. raise UnpicklingError("BINUNICODE exceeds system's maximum size "
  1177. "of %d bytes" % maxsize)
  1178. self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
  1179. dispatch[BINUNICODE[0]] = load_binunicode
  1180. def load_binunicode8(self):
  1181. len, = unpack('<Q', self.read(8))
  1182. if len > maxsize:
  1183. raise UnpicklingError("BINUNICODE8 exceeds system's maximum size "
  1184. "of %d bytes" % maxsize)
  1185. self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
  1186. dispatch[BINUNICODE8[0]] = load_binunicode8
  1187. def load_binbytes8(self):
  1188. len, = unpack('<Q', self.read(8))
  1189. if len > maxsize:
  1190. raise UnpicklingError("BINBYTES8 exceeds system's maximum size "
  1191. "of %d bytes" % maxsize)
  1192. self.append(self.read(len))
  1193. dispatch[BINBYTES8[0]] = load_binbytes8
  1194. def load_bytearray8(self):
  1195. len, = unpack('<Q', self.read(8))
  1196. if len > maxsize:
  1197. raise UnpicklingError("BYTEARRAY8 exceeds system's maximum size "
  1198. "of %d bytes" % maxsize)
  1199. b = bytearray(len)
  1200. self.readinto(b)
  1201. self.append(b)
  1202. dispatch[BYTEARRAY8[0]] = load_bytearray8
  1203. def load_next_buffer(self):
  1204. if self._buffers is None:
  1205. raise UnpicklingError("pickle stream refers to out-of-band data "
  1206. "but no *buffers* argument was given")
  1207. try:
  1208. buf = next(self._buffers)
  1209. except StopIteration:
  1210. raise UnpicklingError("not enough out-of-band buffers")
  1211. self.append(buf)
  1212. dispatch[NEXT_BUFFER[0]] = load_next_buffer
  1213. def load_readonly_buffer(self):
  1214. buf = self.stack[-1]
  1215. with memoryview(buf) as m:
  1216. if not m.readonly:
  1217. self.stack[-1] = m.toreadonly()
  1218. dispatch[READONLY_BUFFER[0]] = load_readonly_buffer
  1219. def load_short_binstring(self):
  1220. len = self.read(1)[0]
  1221. data = self.read(len)
  1222. self.append(self._decode_string(data))
  1223. dispatch[SHORT_BINSTRING[0]] = load_short_binstring
  1224. def load_short_binbytes(self):
  1225. len = self.read(1)[0]
  1226. self.append(self.read(len))
  1227. dispatch[SHORT_BINBYTES[0]] = load_short_binbytes
  1228. def load_short_binunicode(self):
  1229. len = self.read(1)[0]
  1230. self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
  1231. dispatch[SHORT_BINUNICODE[0]] = load_short_binunicode
  1232. def load_tuple(self):
  1233. items = self.pop_mark()
  1234. self.append(tuple(items))
  1235. dispatch[TUPLE[0]] = load_tuple
  1236. def load_empty_tuple(self):
  1237. self.append(())
  1238. dispatch[EMPTY_TUPLE[0]] = load_empty_tuple
  1239. def load_tuple1(self):
  1240. self.stack[-1] = (self.stack[-1],)
  1241. dispatch[TUPLE1[0]] = load_tuple1
  1242. def load_tuple2(self):
  1243. self.stack[-2:] = [(self.stack[-2], self.stack[-1])]
  1244. dispatch[TUPLE2[0]] = load_tuple2
  1245. def load_tuple3(self):
  1246. self.stack[-3:] = [(self.stack[-3], self.stack[-2], self.stack[-1])]
  1247. dispatch[TUPLE3[0]] = load_tuple3
  1248. def load_empty_list(self):
  1249. self.append([])
  1250. dispatch[EMPTY_LIST[0]] = load_empty_list
  1251. def load_empty_dictionary(self):
  1252. self.append({})
  1253. dispatch[EMPTY_DICT[0]] = load_empty_dictionary
  1254. def load_empty_set(self):
  1255. self.append(set())
  1256. dispatch[EMPTY_SET[0]] = load_empty_set
  1257. def load_frozenset(self):
  1258. items = self.pop_mark()
  1259. self.append(frozenset(items))
  1260. dispatch[FROZENSET[0]] = load_frozenset
  1261. def load_list(self):
  1262. items = self.pop_mark()
  1263. self.append(items)
  1264. dispatch[LIST[0]] = load_list
  1265. def load_dict(self):
  1266. items = self.pop_mark()
  1267. d = {items[i]: items[i+1]
  1268. for i in range(0, len(items), 2)}
  1269. self.append(d)
  1270. dispatch[DICT[0]] = load_dict
  1271. # INST and OBJ differ only in how they get a class object. It's not
  1272. # only sensible to do the rest in a common routine, the two routines
  1273. # previously diverged and grew different bugs.
  1274. # klass is the class to instantiate, and k points to the topmost mark
  1275. # object, following which are the arguments for klass.__init__.
  1276. def _instantiate(self, klass, args):
  1277. if (args or not isinstance(klass, type) or
  1278. hasattr(klass, "__getinitargs__")):
  1279. try:
  1280. value = klass(*args)
  1281. except TypeError as err:
  1282. raise TypeError("in constructor for %s: %s" %
  1283. (klass.__name__, str(err)), sys.exc_info()[2])
  1284. else:
  1285. value = klass.__new__(klass)
  1286. self.append(value)
  1287. def load_inst(self):
  1288. module = self.readline()[:-1].decode("ascii")
  1289. name = self.readline()[:-1].decode("ascii")
  1290. klass = self.find_class(module, name)
  1291. self._instantiate(klass, self.pop_mark())
  1292. dispatch[INST[0]] = load_inst
  1293. def load_obj(self):
  1294. # Stack is ... markobject classobject arg1 arg2 ...
  1295. args = self.pop_mark()
  1296. cls = args.pop(0)
  1297. self._instantiate(cls, args)
  1298. dispatch[OBJ[0]] = load_obj
  1299. def load_newobj(self):
  1300. args = self.stack.pop()
  1301. cls = self.stack.pop()
  1302. obj = cls.__new__(cls, *args)
  1303. self.append(obj)
  1304. dispatch[NEWOBJ[0]] = load_newobj
  1305. def load_newobj_ex(self):
  1306. kwargs = self.stack.pop()
  1307. args = self.stack.pop()
  1308. cls = self.stack.pop()
  1309. obj = cls.__new__(cls, *args, **kwargs)
  1310. self.append(obj)
  1311. dispatch[NEWOBJ_EX[0]] = load_newobj_ex
  1312. def load_global(self):
  1313. module = self.readline()[:-1].decode("utf-8")
  1314. name = self.readline()[:-1].decode("utf-8")
  1315. klass = self.find_class(module, name)
  1316. self.append(klass)
  1317. dispatch[GLOBAL[0]] = load_global
  1318. def load_stack_global(self):
  1319. name = self.stack.pop()
  1320. module = self.stack.pop()
  1321. if type(name) is not str or type(module) is not str:
  1322. raise UnpicklingError("STACK_GLOBAL requires str")
  1323. self.append(self.find_class(module, name))
  1324. dispatch[STACK_GLOBAL[0]] = load_stack_global
  1325. def load_ext1(self):
  1326. code = self.read(1)[0]
  1327. self.get_extension(code)
  1328. dispatch[EXT1[0]] = load_ext1
  1329. def load_ext2(self):
  1330. code, = unpack('<H', self.read(2))
  1331. self.get_extension(code)
  1332. dispatch[EXT2[0]] = load_ext2
  1333. def load_ext4(self):
  1334. code, = unpack('<i', self.read(4))
  1335. self.get_extension(code)
  1336. dispatch[EXT4[0]] = load_ext4
  1337. def get_extension(self, code):
  1338. nil = []
  1339. obj = _extension_cache.get(code, nil)
  1340. if obj is not nil:
  1341. self.append(obj)
  1342. return
  1343. key = _inverted_registry.get(code)
  1344. if not key:
  1345. if code <= 0: # note that 0 is forbidden
  1346. # Corrupt or hostile pickle.
  1347. raise UnpicklingError("EXT specifies code <= 0")
  1348. raise ValueError("unregistered extension code %d" % code)
  1349. obj = self.find_class(*key)
  1350. _extension_cache[code] = obj
  1351. self.append(obj)
  1352. def find_class(self, module, name):
  1353. # Subclasses may override this.
  1354. sys.audit('pickle.find_class', module, name)
  1355. if self.proto < 3 and self.fix_imports:
  1356. if (module, name) in _compat_pickle.NAME_MAPPING:
  1357. module, name = _compat_pickle.NAME_MAPPING[(module, name)]
  1358. elif module in _compat_pickle.IMPORT_MAPPING:
  1359. module = _compat_pickle.IMPORT_MAPPING[module]
  1360. __import__(module, level=0)
  1361. if self.proto >= 4:
  1362. return _getattribute(sys.modules[module], name)[0]
  1363. else:
  1364. return getattr(sys.modules[module], name)
  1365. def load_reduce(self):
  1366. stack = self.stack
  1367. args = stack.pop()
  1368. func = stack[-1]
  1369. stack[-1] = func(*args)
  1370. dispatch[REDUCE[0]] = load_reduce
  1371. def load_pop(self):
  1372. if self.stack:
  1373. del self.stack[-1]
  1374. else:
  1375. self.pop_mark()
  1376. dispatch[POP[0]] = load_pop
  1377. def load_pop_mark(self):
  1378. self.pop_mark()
  1379. dispatch[POP_MARK[0]] = load_pop_mark
  1380. def load_dup(self):
  1381. self.append(self.stack[-1])
  1382. dispatch[DUP[0]] = load_dup
  1383. def load_get(self):
  1384. i = int(self.readline()[:-1])
  1385. try:
  1386. self.append(self.memo[i])
  1387. except KeyError:
  1388. msg = f'Memo value not found at index {i}'
  1389. raise UnpicklingError(msg) from None
  1390. dispatch[GET[0]] = load_get
  1391. def load_binget(self):
  1392. i = self.read(1)[0]
  1393. try:
  1394. self.append(self.memo[i])
  1395. except KeyError as exc:
  1396. msg = f'Memo value not found at index {i}'
  1397. raise UnpicklingError(msg) from None
  1398. dispatch[BINGET[0]] = load_binget
  1399. def load_long_binget(self):
  1400. i, = unpack('<I', self.read(4))
  1401. try:
  1402. self.append(self.memo[i])
  1403. except KeyError as exc:
  1404. msg = f'Memo value not found at index {i}'
  1405. raise UnpicklingError(msg) from None
  1406. dispatch[LONG_BINGET[0]] = load_long_binget
  1407. def load_put(self):
  1408. i = int(self.readline()[:-1])
  1409. if i < 0:
  1410. raise ValueError("negative PUT argument")
  1411. self.memo[i] = self.stack[-1]
  1412. dispatch[PUT[0]] = load_put
  1413. def load_binput(self):
  1414. i = self.read(1)[0]
  1415. if i < 0:
  1416. raise ValueError("negative BINPUT argument")
  1417. self.memo[i] = self.stack[-1]
  1418. dispatch[BINPUT[0]] = load_binput
  1419. def load_long_binput(self):
  1420. i, = unpack('<I', self.read(4))
  1421. if i > maxsize:
  1422. raise ValueError("negative LONG_BINPUT argument")
  1423. self.memo[i] = self.stack[-1]
  1424. dispatch[LONG_BINPUT[0]] = load_long_binput
  1425. def load_memoize(self):
  1426. memo = self.memo
  1427. memo[len(memo)] = self.stack[-1]
  1428. dispatch[MEMOIZE[0]] = load_memoize
  1429. def load_append(self):
  1430. stack = self.stack
  1431. value = stack.pop()
  1432. list = stack[-1]
  1433. list.append(value)
  1434. dispatch[APPEND[0]] = load_append
  1435. def load_appends(self):
  1436. items = self.pop_mark()
  1437. list_obj = self.stack[-1]
  1438. try:
  1439. extend = list_obj.extend
  1440. except AttributeError:
  1441. pass
  1442. else:
  1443. extend(items)
  1444. return
  1445. # Even if the PEP 307 requires extend() and append() methods,
  1446. # fall back on append() if the object has no extend() method
  1447. # for backward compatibility.
  1448. append = list_obj.append
  1449. for item in items:
  1450. append(item)
  1451. dispatch[APPENDS[0]] = load_appends
  1452. def load_setitem(self):
  1453. stack = self.stack
  1454. value = stack.pop()
  1455. key = stack.pop()
  1456. dict = stack[-1]
  1457. dict[key] = value
  1458. dispatch[SETITEM[0]] = load_setitem
  1459. def load_setitems(self):
  1460. items = self.pop_mark()
  1461. dict = self.stack[-1]
  1462. for i in range(0, len(items), 2):
  1463. dict[items[i]] = items[i + 1]
  1464. dispatch[SETITEMS[0]] = load_setitems
  1465. def load_additems(self):
  1466. items = self.pop_mark()
  1467. set_obj = self.stack[-1]
  1468. if isinstance(set_obj, set):
  1469. set_obj.update(items)
  1470. else:
  1471. add = set_obj.add
  1472. for item in items:
  1473. add(item)
  1474. dispatch[ADDITEMS[0]] = load_additems
  1475. def load_build(self):
  1476. stack = self.stack
  1477. state = stack.pop()
  1478. inst = stack[-1]
  1479. setstate = getattr(inst, "__setstate__", None)
  1480. if setstate is not None:
  1481. setstate(state)
  1482. return
  1483. slotstate = None
  1484. if isinstance(state, tuple) and len(state) == 2:
  1485. state, slotstate = state
  1486. if state:
  1487. inst_dict = inst.__dict__
  1488. intern = sys.intern
  1489. for k, v in state.items():
  1490. if type(k) is str:
  1491. inst_dict[intern(k)] = v
  1492. else:
  1493. inst_dict[k] = v
  1494. if slotstate:
  1495. for k, v in slotstate.items():
  1496. setattr(inst, k, v)
  1497. dispatch[BUILD[0]] = load_build
  1498. def load_mark(self):
  1499. self.metastack.append(self.stack)
  1500. self.stack = []
  1501. self.append = self.stack.append
  1502. dispatch[MARK[0]] = load_mark
  1503. def load_stop(self):
  1504. value = self.stack.pop()
  1505. raise _Stop(value)
  1506. dispatch[STOP[0]] = load_stop
  1507. # Shorthands
  1508. def _dump(obj, file, protocol=None, *, fix_imports=True, buffer_callback=None):
  1509. _Pickler(file, protocol, fix_imports=fix_imports,
  1510. buffer_callback=buffer_callback).dump(obj)
  1511. def _dumps(obj, protocol=None, *, fix_imports=True, buffer_callback=None):
  1512. f = io.BytesIO()
  1513. _Pickler(f, protocol, fix_imports=fix_imports,
  1514. buffer_callback=buffer_callback).dump(obj)
  1515. res = f.getvalue()
  1516. assert isinstance(res, bytes_types)
  1517. return res
  1518. def _load(file, *, fix_imports=True, encoding="ASCII", errors="strict",
  1519. buffers=None):
  1520. return _Unpickler(file, fix_imports=fix_imports, buffers=buffers,
  1521. encoding=encoding, errors=errors).load()
  1522. def _loads(s, /, *, fix_imports=True, encoding="ASCII", errors="strict",
  1523. buffers=None):
  1524. if isinstance(s, str):
  1525. raise TypeError("Can't load pickle from unicode string")
  1526. file = io.BytesIO(s)
  1527. return _Unpickler(file, fix_imports=fix_imports, buffers=buffers,
  1528. encoding=encoding, errors=errors).load()
  1529. # Use the faster _pickle if possible
  1530. try:
  1531. from _pickle import (
  1532. PickleError,
  1533. PicklingError,
  1534. UnpicklingError,
  1535. Pickler,
  1536. Unpickler,
  1537. dump,
  1538. dumps,
  1539. load,
  1540. loads
  1541. )
  1542. except ImportError:
  1543. Pickler, Unpickler = _Pickler, _Unpickler
  1544. dump, dumps, load, loads = _dump, _dumps, _load, _loads
  1545. # Doctest
  1546. def _test():
  1547. import doctest
  1548. return doctest.testmod()
  1549. if __name__ == "__main__":
  1550. import argparse
  1551. parser = argparse.ArgumentParser(
  1552. description='display contents of the pickle files')
  1553. parser.add_argument(
  1554. 'pickle_file', type=argparse.FileType('br'),
  1555. nargs='*', help='the pickle file')
  1556. parser.add_argument(
  1557. '-t', '--test', action='store_true',
  1558. help='run self-test suite')
  1559. parser.add_argument(
  1560. '-v', action='store_true',
  1561. help='run verbosely; only affects self-test run')
  1562. args = parser.parse_args()
  1563. if args.test:
  1564. _test()
  1565. else:
  1566. if not args.pickle_file:
  1567. parser.print_help()
  1568. else:
  1569. import pprint
  1570. for f in args.pickle_file:
  1571. obj = load(f)
  1572. pprint.pprint(obj)