logo

oasis-root

Compiled tree of Oasis Linux based on own branch at <https://hacktivis.me/git/oasis/> git clone https://anongit.hacktivis.me/git/oasis-root.git

shared_memory.py (18396B)


  1. """Provides shared memory for direct access across processes.
  2. The API of this package is currently provisional. Refer to the
  3. documentation for details.
  4. """
  5. __all__ = [ 'SharedMemory', 'ShareableList' ]
  6. from functools import partial
  7. import mmap
  8. import os
  9. import errno
  10. import struct
  11. import secrets
  12. import types
  13. if os.name == "nt":
  14. import _winapi
  15. _USE_POSIX = False
  16. else:
  17. import _posixshmem
  18. _USE_POSIX = True
  19. _O_CREX = os.O_CREAT | os.O_EXCL
  20. # FreeBSD (and perhaps other BSDs) limit names to 14 characters.
  21. _SHM_SAFE_NAME_LENGTH = 14
  22. # Shared memory block name prefix
  23. if _USE_POSIX:
  24. _SHM_NAME_PREFIX = '/psm_'
  25. else:
  26. _SHM_NAME_PREFIX = 'wnsm_'
  27. def _make_filename():
  28. "Create a random filename for the shared memory object."
  29. # number of random bytes to use for name
  30. nbytes = (_SHM_SAFE_NAME_LENGTH - len(_SHM_NAME_PREFIX)) // 2
  31. assert nbytes >= 2, '_SHM_NAME_PREFIX too long'
  32. name = _SHM_NAME_PREFIX + secrets.token_hex(nbytes)
  33. assert len(name) <= _SHM_SAFE_NAME_LENGTH
  34. return name
  35. class SharedMemory:
  36. """Creates a new shared memory block or attaches to an existing
  37. shared memory block.
  38. Every shared memory block is assigned a unique name. This enables
  39. one process to create a shared memory block with a particular name
  40. so that a different process can attach to that same shared memory
  41. block using that same name.
  42. As a resource for sharing data across processes, shared memory blocks
  43. may outlive the original process that created them. When one process
  44. no longer needs access to a shared memory block that might still be
  45. needed by other processes, the close() method should be called.
  46. When a shared memory block is no longer needed by any process, the
  47. unlink() method should be called to ensure proper cleanup."""
  48. # Defaults; enables close() and unlink() to run without errors.
  49. _name = None
  50. _fd = -1
  51. _mmap = None
  52. _buf = None
  53. _flags = os.O_RDWR
  54. _mode = 0o600
  55. _prepend_leading_slash = True if _USE_POSIX else False
  56. def __init__(self, name=None, create=False, size=0):
  57. if not size >= 0:
  58. raise ValueError("'size' must be a positive integer")
  59. if create:
  60. self._flags = _O_CREX | os.O_RDWR
  61. if size == 0:
  62. raise ValueError("'size' must be a positive number different from zero")
  63. if name is None and not self._flags & os.O_EXCL:
  64. raise ValueError("'name' can only be None if create=True")
  65. if _USE_POSIX:
  66. # POSIX Shared Memory
  67. if name is None:
  68. while True:
  69. name = _make_filename()
  70. try:
  71. self._fd = _posixshmem.shm_open(
  72. name,
  73. self._flags,
  74. mode=self._mode
  75. )
  76. except FileExistsError:
  77. continue
  78. self._name = name
  79. break
  80. else:
  81. name = "/" + name if self._prepend_leading_slash else name
  82. self._fd = _posixshmem.shm_open(
  83. name,
  84. self._flags,
  85. mode=self._mode
  86. )
  87. self._name = name
  88. try:
  89. if create and size:
  90. os.ftruncate(self._fd, size)
  91. stats = os.fstat(self._fd)
  92. size = stats.st_size
  93. self._mmap = mmap.mmap(self._fd, size)
  94. except OSError:
  95. self.unlink()
  96. raise
  97. from .resource_tracker import register
  98. register(self._name, "shared_memory")
  99. else:
  100. # Windows Named Shared Memory
  101. if create:
  102. while True:
  103. temp_name = _make_filename() if name is None else name
  104. # Create and reserve shared memory block with this name
  105. # until it can be attached to by mmap.
  106. h_map = _winapi.CreateFileMapping(
  107. _winapi.INVALID_HANDLE_VALUE,
  108. _winapi.NULL,
  109. _winapi.PAGE_READWRITE,
  110. (size >> 32) & 0xFFFFFFFF,
  111. size & 0xFFFFFFFF,
  112. temp_name
  113. )
  114. try:
  115. last_error_code = _winapi.GetLastError()
  116. if last_error_code == _winapi.ERROR_ALREADY_EXISTS:
  117. if name is not None:
  118. raise FileExistsError(
  119. errno.EEXIST,
  120. os.strerror(errno.EEXIST),
  121. name,
  122. _winapi.ERROR_ALREADY_EXISTS
  123. )
  124. else:
  125. continue
  126. self._mmap = mmap.mmap(-1, size, tagname=temp_name)
  127. finally:
  128. _winapi.CloseHandle(h_map)
  129. self._name = temp_name
  130. break
  131. else:
  132. self._name = name
  133. # Dynamically determine the existing named shared memory
  134. # block's size which is likely a multiple of mmap.PAGESIZE.
  135. h_map = _winapi.OpenFileMapping(
  136. _winapi.FILE_MAP_READ,
  137. False,
  138. name
  139. )
  140. try:
  141. p_buf = _winapi.MapViewOfFile(
  142. h_map,
  143. _winapi.FILE_MAP_READ,
  144. 0,
  145. 0,
  146. 0
  147. )
  148. finally:
  149. _winapi.CloseHandle(h_map)
  150. size = _winapi.VirtualQuerySize(p_buf)
  151. self._mmap = mmap.mmap(-1, size, tagname=name)
  152. self._size = size
  153. self._buf = memoryview(self._mmap)
  154. def __del__(self):
  155. try:
  156. self.close()
  157. except OSError:
  158. pass
  159. def __reduce__(self):
  160. return (
  161. self.__class__,
  162. (
  163. self.name,
  164. False,
  165. self.size,
  166. ),
  167. )
  168. def __repr__(self):
  169. return f'{self.__class__.__name__}({self.name!r}, size={self.size})'
  170. @property
  171. def buf(self):
  172. "A memoryview of contents of the shared memory block."
  173. return self._buf
  174. @property
  175. def name(self):
  176. "Unique name that identifies the shared memory block."
  177. reported_name = self._name
  178. if _USE_POSIX and self._prepend_leading_slash:
  179. if self._name.startswith("/"):
  180. reported_name = self._name[1:]
  181. return reported_name
  182. @property
  183. def size(self):
  184. "Size in bytes."
  185. return self._size
  186. def close(self):
  187. """Closes access to the shared memory from this instance but does
  188. not destroy the shared memory block."""
  189. if self._buf is not None:
  190. self._buf.release()
  191. self._buf = None
  192. if self._mmap is not None:
  193. self._mmap.close()
  194. self._mmap = None
  195. if _USE_POSIX and self._fd >= 0:
  196. os.close(self._fd)
  197. self._fd = -1
  198. def unlink(self):
  199. """Requests that the underlying shared memory block be destroyed.
  200. In order to ensure proper cleanup of resources, unlink should be
  201. called once (and only once) across all processes which have access
  202. to the shared memory block."""
  203. if _USE_POSIX and self._name:
  204. from .resource_tracker import unregister
  205. _posixshmem.shm_unlink(self._name)
  206. unregister(self._name, "shared_memory")
  207. _encoding = "utf8"
  208. class ShareableList:
  209. """Pattern for a mutable list-like object shareable via a shared
  210. memory block. It differs from the built-in list type in that these
  211. lists can not change their overall length (i.e. no append, insert,
  212. etc.)
  213. Because values are packed into a memoryview as bytes, the struct
  214. packing format for any storable value must require no more than 8
  215. characters to describe its format."""
  216. # The shared memory area is organized as follows:
  217. # - 8 bytes: number of items (N) as a 64-bit integer
  218. # - (N + 1) * 8 bytes: offsets of each element from the start of the
  219. # data area
  220. # - K bytes: the data area storing item values (with encoding and size
  221. # depending on their respective types)
  222. # - N * 8 bytes: `struct` format string for each element
  223. # - N bytes: index into _back_transforms_mapping for each element
  224. # (for reconstructing the corresponding Python value)
  225. _types_mapping = {
  226. int: "q",
  227. float: "d",
  228. bool: "xxxxxxx?",
  229. str: "%ds",
  230. bytes: "%ds",
  231. None.__class__: "xxxxxx?x",
  232. }
  233. _alignment = 8
  234. _back_transforms_mapping = {
  235. 0: lambda value: value, # int, float, bool
  236. 1: lambda value: value.rstrip(b'\x00').decode(_encoding), # str
  237. 2: lambda value: value.rstrip(b'\x00'), # bytes
  238. 3: lambda _value: None, # None
  239. }
  240. @staticmethod
  241. def _extract_recreation_code(value):
  242. """Used in concert with _back_transforms_mapping to convert values
  243. into the appropriate Python objects when retrieving them from
  244. the list as well as when storing them."""
  245. if not isinstance(value, (str, bytes, None.__class__)):
  246. return 0
  247. elif isinstance(value, str):
  248. return 1
  249. elif isinstance(value, bytes):
  250. return 2
  251. else:
  252. return 3 # NoneType
  253. def __init__(self, sequence=None, *, name=None):
  254. if name is None or sequence is not None:
  255. sequence = sequence or ()
  256. _formats = [
  257. self._types_mapping[type(item)]
  258. if not isinstance(item, (str, bytes))
  259. else self._types_mapping[type(item)] % (
  260. self._alignment * (len(item) // self._alignment + 1),
  261. )
  262. for item in sequence
  263. ]
  264. self._list_len = len(_formats)
  265. assert sum(len(fmt) <= 8 for fmt in _formats) == self._list_len
  266. offset = 0
  267. # The offsets of each list element into the shared memory's
  268. # data area (0 meaning the start of the data area, not the start
  269. # of the shared memory area).
  270. self._allocated_offsets = [0]
  271. for fmt in _formats:
  272. offset += self._alignment if fmt[-1] != "s" else int(fmt[:-1])
  273. self._allocated_offsets.append(offset)
  274. _recreation_codes = [
  275. self._extract_recreation_code(item) for item in sequence
  276. ]
  277. requested_size = struct.calcsize(
  278. "q" + self._format_size_metainfo +
  279. "".join(_formats) +
  280. self._format_packing_metainfo +
  281. self._format_back_transform_codes
  282. )
  283. self.shm = SharedMemory(name, create=True, size=requested_size)
  284. else:
  285. self.shm = SharedMemory(name)
  286. if sequence is not None:
  287. _enc = _encoding
  288. struct.pack_into(
  289. "q" + self._format_size_metainfo,
  290. self.shm.buf,
  291. 0,
  292. self._list_len,
  293. *(self._allocated_offsets)
  294. )
  295. struct.pack_into(
  296. "".join(_formats),
  297. self.shm.buf,
  298. self._offset_data_start,
  299. *(v.encode(_enc) if isinstance(v, str) else v for v in sequence)
  300. )
  301. struct.pack_into(
  302. self._format_packing_metainfo,
  303. self.shm.buf,
  304. self._offset_packing_formats,
  305. *(v.encode(_enc) for v in _formats)
  306. )
  307. struct.pack_into(
  308. self._format_back_transform_codes,
  309. self.shm.buf,
  310. self._offset_back_transform_codes,
  311. *(_recreation_codes)
  312. )
  313. else:
  314. self._list_len = len(self) # Obtains size from offset 0 in buffer.
  315. self._allocated_offsets = list(
  316. struct.unpack_from(
  317. self._format_size_metainfo,
  318. self.shm.buf,
  319. 1 * 8
  320. )
  321. )
  322. def _get_packing_format(self, position):
  323. "Gets the packing format for a single value stored in the list."
  324. position = position if position >= 0 else position + self._list_len
  325. if (position >= self._list_len) or (self._list_len < 0):
  326. raise IndexError("Requested position out of range.")
  327. v = struct.unpack_from(
  328. "8s",
  329. self.shm.buf,
  330. self._offset_packing_formats + position * 8
  331. )[0]
  332. fmt = v.rstrip(b'\x00')
  333. fmt_as_str = fmt.decode(_encoding)
  334. return fmt_as_str
  335. def _get_back_transform(self, position):
  336. "Gets the back transformation function for a single value."
  337. if (position >= self._list_len) or (self._list_len < 0):
  338. raise IndexError("Requested position out of range.")
  339. transform_code = struct.unpack_from(
  340. "b",
  341. self.shm.buf,
  342. self._offset_back_transform_codes + position
  343. )[0]
  344. transform_function = self._back_transforms_mapping[transform_code]
  345. return transform_function
  346. def _set_packing_format_and_transform(self, position, fmt_as_str, value):
  347. """Sets the packing format and back transformation code for a
  348. single value in the list at the specified position."""
  349. if (position >= self._list_len) or (self._list_len < 0):
  350. raise IndexError("Requested position out of range.")
  351. struct.pack_into(
  352. "8s",
  353. self.shm.buf,
  354. self._offset_packing_formats + position * 8,
  355. fmt_as_str.encode(_encoding)
  356. )
  357. transform_code = self._extract_recreation_code(value)
  358. struct.pack_into(
  359. "b",
  360. self.shm.buf,
  361. self._offset_back_transform_codes + position,
  362. transform_code
  363. )
  364. def __getitem__(self, position):
  365. position = position if position >= 0 else position + self._list_len
  366. try:
  367. offset = self._offset_data_start + self._allocated_offsets[position]
  368. (v,) = struct.unpack_from(
  369. self._get_packing_format(position),
  370. self.shm.buf,
  371. offset
  372. )
  373. except IndexError:
  374. raise IndexError("index out of range")
  375. back_transform = self._get_back_transform(position)
  376. v = back_transform(v)
  377. return v
  378. def __setitem__(self, position, value):
  379. position = position if position >= 0 else position + self._list_len
  380. try:
  381. item_offset = self._allocated_offsets[position]
  382. offset = self._offset_data_start + item_offset
  383. current_format = self._get_packing_format(position)
  384. except IndexError:
  385. raise IndexError("assignment index out of range")
  386. if not isinstance(value, (str, bytes)):
  387. new_format = self._types_mapping[type(value)]
  388. encoded_value = value
  389. else:
  390. allocated_length = self._allocated_offsets[position + 1] - item_offset
  391. encoded_value = (value.encode(_encoding)
  392. if isinstance(value, str) else value)
  393. if len(encoded_value) > allocated_length:
  394. raise ValueError("bytes/str item exceeds available storage")
  395. if current_format[-1] == "s":
  396. new_format = current_format
  397. else:
  398. new_format = self._types_mapping[str] % (
  399. allocated_length,
  400. )
  401. self._set_packing_format_and_transform(
  402. position,
  403. new_format,
  404. value
  405. )
  406. struct.pack_into(new_format, self.shm.buf, offset, encoded_value)
  407. def __reduce__(self):
  408. return partial(self.__class__, name=self.shm.name), ()
  409. def __len__(self):
  410. return struct.unpack_from("q", self.shm.buf, 0)[0]
  411. def __repr__(self):
  412. return f'{self.__class__.__name__}({list(self)}, name={self.shm.name!r})'
  413. @property
  414. def format(self):
  415. "The struct packing format used by all currently stored items."
  416. return "".join(
  417. self._get_packing_format(i) for i in range(self._list_len)
  418. )
  419. @property
  420. def _format_size_metainfo(self):
  421. "The struct packing format used for the items' storage offsets."
  422. return "q" * (self._list_len + 1)
  423. @property
  424. def _format_packing_metainfo(self):
  425. "The struct packing format used for the items' packing formats."
  426. return "8s" * self._list_len
  427. @property
  428. def _format_back_transform_codes(self):
  429. "The struct packing format used for the items' back transforms."
  430. return "b" * self._list_len
  431. @property
  432. def _offset_data_start(self):
  433. # - 8 bytes for the list length
  434. # - (N + 1) * 8 bytes for the element offsets
  435. return (self._list_len + 2) * 8
  436. @property
  437. def _offset_packing_formats(self):
  438. return self._offset_data_start + self._allocated_offsets[-1]
  439. @property
  440. def _offset_back_transform_codes(self):
  441. return self._offset_packing_formats + self._list_len * 8
  442. def count(self, value):
  443. "L.count(value) -> integer -- return number of occurrences of value."
  444. return sum(value == entry for entry in self)
  445. def index(self, value):
  446. """L.index(value) -> integer -- return first index of value.
  447. Raises ValueError if the value is not present."""
  448. for position, entry in enumerate(self):
  449. if value == entry:
  450. return position
  451. else:
  452. raise ValueError(f"{value!r} not in this container")
  453. __class_getitem__ = classmethod(types.GenericAlias)