Development of an internal social media platform with personalised dashboards for students
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

selectors.py 19KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592
  1. """Selectors module.
  2. This module allows high-level and efficient I/O multiplexing, built upon the
  3. `select` module primitives.
  4. The following code adapted from trollius.selectors.
  5. """
  6. from abc import ABCMeta, abstractmethod
  7. from collections import namedtuple, Mapping
  8. import math
  9. import select
  10. import sys
  11. from gunicorn._compat import wrap_error, InterruptedError
  12. from gunicorn import six
  13. # generic events, that must be mapped to implementation-specific ones
  14. EVENT_READ = (1 << 0)
  15. EVENT_WRITE = (1 << 1)
  16. def _fileobj_to_fd(fileobj):
  17. """Return a file descriptor from a file object.
  18. Parameters:
  19. fileobj -- file object or file descriptor
  20. Returns:
  21. corresponding file descriptor
  22. Raises:
  23. ValueError if the object is invalid
  24. """
  25. if isinstance(fileobj, six.integer_types):
  26. fd = fileobj
  27. else:
  28. try:
  29. fd = int(fileobj.fileno())
  30. except (AttributeError, TypeError, ValueError):
  31. raise ValueError("Invalid file object: "
  32. "{0!r}".format(fileobj))
  33. if fd < 0:
  34. raise ValueError("Invalid file descriptor: {0}".format(fd))
  35. return fd
  36. SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data'])
  37. """Object used to associate a file object to its backing file descriptor,
  38. selected event mask and attached data."""
  39. class _SelectorMapping(Mapping):
  40. """Mapping of file objects to selector keys."""
  41. def __init__(self, selector):
  42. self._selector = selector
  43. def __len__(self):
  44. return len(self._selector._fd_to_key)
  45. def __getitem__(self, fileobj):
  46. try:
  47. fd = self._selector._fileobj_lookup(fileobj)
  48. return self._selector._fd_to_key[fd]
  49. except KeyError:
  50. raise KeyError("{0!r} is not registered".format(fileobj))
  51. def __iter__(self):
  52. return iter(self._selector._fd_to_key)
  53. class BaseSelector(six.with_metaclass(ABCMeta)):
  54. """Selector abstract base class.
  55. A selector supports registering file objects to be monitored for specific
  56. I/O events.
  57. A file object is a file descriptor or any object with a `fileno()` method.
  58. An arbitrary object can be attached to the file object, which can be used
  59. for example to store context information, a callback, etc.
  60. A selector can use various implementations (select(), poll(), epoll()...)
  61. depending on the platform. The default `Selector` class uses the most
  62. efficient implementation on the current platform.
  63. """
  64. @abstractmethod
  65. def register(self, fileobj, events, data=None):
  66. """Register a file object.
  67. Parameters:
  68. fileobj -- file object or file descriptor
  69. events -- events to monitor (bitwise mask of EVENT_READ|EVENT_WRITE)
  70. data -- attached data
  71. Returns:
  72. SelectorKey instance
  73. Raises:
  74. ValueError if events is invalid
  75. KeyError if fileobj is already registered
  76. OSError if fileobj is closed or otherwise is unacceptable to
  77. the underlying system call (if a system call is made)
  78. Note:
  79. OSError may or may not be raised
  80. """
  81. raise NotImplementedError
  82. @abstractmethod
  83. def unregister(self, fileobj):
  84. """Unregister a file object.
  85. Parameters:
  86. fileobj -- file object or file descriptor
  87. Returns:
  88. SelectorKey instance
  89. Raises:
  90. KeyError if fileobj is not registered
  91. Note:
  92. If fileobj is registered but has since been closed this does
  93. *not* raise OSError (even if the wrapped syscall does)
  94. """
  95. raise NotImplementedError
  96. def modify(self, fileobj, events, data=None):
  97. """Change a registered file object monitored events or attached data.
  98. Parameters:
  99. fileobj -- file object or file descriptor
  100. events -- events to monitor (bitwise mask of EVENT_READ|EVENT_WRITE)
  101. data -- attached data
  102. Returns:
  103. SelectorKey instance
  104. Raises:
  105. Anything that unregister() or register() raises
  106. """
  107. self.unregister(fileobj)
  108. return self.register(fileobj, events, data)
  109. @abstractmethod
  110. def select(self, timeout=None):
  111. """Perform the actual selection, until some monitored file objects are
  112. ready or a timeout expires.
  113. Parameters:
  114. timeout -- if timeout > 0, this specifies the maximum wait time, in
  115. seconds
  116. if timeout <= 0, the select() call won't block, and will
  117. report the currently ready file objects
  118. if timeout is None, select() will block until a monitored
  119. file object becomes ready
  120. Returns:
  121. list of (key, events) for ready file objects
  122. `events` is a bitwise mask of EVENT_READ|EVENT_WRITE
  123. """
  124. raise NotImplementedError
  125. def close(self):
  126. """Close the selector.
  127. This must be called to make sure that any underlying resource is freed.
  128. """
  129. pass
  130. def get_key(self, fileobj):
  131. """Return the key associated to a registered file object.
  132. Returns:
  133. SelectorKey for this file object
  134. """
  135. mapping = self.get_map()
  136. try:
  137. return mapping[fileobj]
  138. except KeyError:
  139. raise KeyError("{0!r} is not registered".format(fileobj))
  140. @abstractmethod
  141. def get_map(self):
  142. """Return a mapping of file objects to selector keys."""
  143. raise NotImplementedError
  144. def __enter__(self):
  145. return self
  146. def __exit__(self, *args):
  147. self.close()
  148. class _BaseSelectorImpl(BaseSelector):
  149. """Base selector implementation."""
  150. def __init__(self):
  151. # this maps file descriptors to keys
  152. self._fd_to_key = {}
  153. # read-only mapping returned by get_map()
  154. self._map = _SelectorMapping(self)
  155. def _fileobj_lookup(self, fileobj):
  156. """Return a file descriptor from a file object.
  157. This wraps _fileobj_to_fd() to do an exhaustive search in case
  158. the object is invalid but we still have it in our map. This
  159. is used by unregister() so we can unregister an object that
  160. was previously registered even if it is closed. It is also
  161. used by _SelectorMapping.
  162. """
  163. try:
  164. return _fileobj_to_fd(fileobj)
  165. except ValueError:
  166. # Do an exhaustive search.
  167. for key in self._fd_to_key.values():
  168. if key.fileobj is fileobj:
  169. return key.fd
  170. # Raise ValueError after all.
  171. raise
  172. def register(self, fileobj, events, data=None):
  173. if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)):
  174. raise ValueError("Invalid events: {0!r}".format(events))
  175. key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data)
  176. if key.fd in self._fd_to_key:
  177. raise KeyError("{0!r} (FD {1}) is already registered"
  178. .format(fileobj, key.fd))
  179. self._fd_to_key[key.fd] = key
  180. return key
  181. def unregister(self, fileobj):
  182. try:
  183. key = self._fd_to_key.pop(self._fileobj_lookup(fileobj))
  184. except KeyError:
  185. raise KeyError("{0!r} is not registered".format(fileobj))
  186. return key
  187. def modify(self, fileobj, events, data=None):
  188. # TODO: Subclasses can probably optimize this even further.
  189. try:
  190. key = self._fd_to_key[self._fileobj_lookup(fileobj)]
  191. except KeyError:
  192. raise KeyError("{0!r} is not registered".format(fileobj))
  193. if events != key.events:
  194. self.unregister(fileobj)
  195. key = self.register(fileobj, events, data)
  196. elif data != key.data:
  197. # Use a shortcut to update the data.
  198. key = key._replace(data=data)
  199. self._fd_to_key[key.fd] = key
  200. return key
  201. def close(self):
  202. self._fd_to_key.clear()
  203. def get_map(self):
  204. return self._map
  205. def _key_from_fd(self, fd):
  206. """Return the key associated to a given file descriptor.
  207. Parameters:
  208. fd -- file descriptor
  209. Returns:
  210. corresponding key, or None if not found
  211. """
  212. try:
  213. return self._fd_to_key[fd]
  214. except KeyError:
  215. return None
  216. class SelectSelector(_BaseSelectorImpl):
  217. """Select-based selector."""
  218. def __init__(self):
  219. super(SelectSelector, self).__init__()
  220. self._readers = set()
  221. self._writers = set()
  222. def register(self, fileobj, events, data=None):
  223. key = super(SelectSelector, self).register(fileobj, events, data)
  224. if events & EVENT_READ:
  225. self._readers.add(key.fd)
  226. if events & EVENT_WRITE:
  227. self._writers.add(key.fd)
  228. return key
  229. def unregister(self, fileobj):
  230. key = super(SelectSelector, self).unregister(fileobj)
  231. self._readers.discard(key.fd)
  232. self._writers.discard(key.fd)
  233. return key
  234. if sys.platform == 'win32':
  235. def _select(self, r, w, _, timeout=None):
  236. r, w, x = select.select(r, w, w, timeout)
  237. return r, w + x, []
  238. else:
  239. _select = select.select
  240. def select(self, timeout=None):
  241. timeout = None if timeout is None else max(timeout, 0)
  242. ready = []
  243. try:
  244. r, w, _ = wrap_error(self._select,
  245. self._readers, self._writers, [], timeout)
  246. except InterruptedError:
  247. return ready
  248. r = set(r)
  249. w = set(w)
  250. for fd in r | w:
  251. events = 0
  252. if fd in r:
  253. events |= EVENT_READ
  254. if fd in w:
  255. events |= EVENT_WRITE
  256. key = self._key_from_fd(fd)
  257. if key:
  258. ready.append((key, events & key.events))
  259. return ready
  260. if hasattr(select, 'poll'):
  261. class PollSelector(_BaseSelectorImpl):
  262. """Poll-based selector."""
  263. def __init__(self):
  264. super(PollSelector, self).__init__()
  265. self._poll = select.poll()
  266. def register(self, fileobj, events, data=None):
  267. key = super(PollSelector, self).register(fileobj, events, data)
  268. poll_events = 0
  269. if events & EVENT_READ:
  270. poll_events |= select.POLLIN
  271. if events & EVENT_WRITE:
  272. poll_events |= select.POLLOUT
  273. self._poll.register(key.fd, poll_events)
  274. return key
  275. def unregister(self, fileobj):
  276. key = super(PollSelector, self).unregister(fileobj)
  277. self._poll.unregister(key.fd)
  278. return key
  279. def select(self, timeout=None):
  280. if timeout is None:
  281. timeout = None
  282. elif timeout <= 0:
  283. timeout = 0
  284. else:
  285. # poll() has a resolution of 1 millisecond, round away from
  286. # zero to wait *at least* timeout seconds.
  287. timeout = int(math.ceil(timeout * 1e3))
  288. ready = []
  289. try:
  290. fd_event_list = wrap_error(self._poll.poll, timeout)
  291. except InterruptedError:
  292. return ready
  293. for fd, event in fd_event_list:
  294. events = 0
  295. if event & ~select.POLLIN:
  296. events |= EVENT_WRITE
  297. if event & ~select.POLLOUT:
  298. events |= EVENT_READ
  299. key = self._key_from_fd(fd)
  300. if key:
  301. ready.append((key, events & key.events))
  302. return ready
  303. if hasattr(select, 'epoll'):
  304. class EpollSelector(_BaseSelectorImpl):
  305. """Epoll-based selector."""
  306. def __init__(self):
  307. super(EpollSelector, self).__init__()
  308. self._epoll = select.epoll()
  309. def fileno(self):
  310. return self._epoll.fileno()
  311. def register(self, fileobj, events, data=None):
  312. key = super(EpollSelector, self).register(fileobj, events, data)
  313. epoll_events = 0
  314. if events & EVENT_READ:
  315. epoll_events |= select.EPOLLIN
  316. if events & EVENT_WRITE:
  317. epoll_events |= select.EPOLLOUT
  318. self._epoll.register(key.fd, epoll_events)
  319. return key
  320. def unregister(self, fileobj):
  321. key = super(EpollSelector, self).unregister(fileobj)
  322. try:
  323. self._epoll.unregister(key.fd)
  324. except OSError:
  325. # This can happen if the FD was closed since it
  326. # was registered.
  327. pass
  328. return key
  329. def select(self, timeout=None):
  330. if timeout is None:
  331. timeout = -1
  332. elif timeout <= 0:
  333. timeout = 0
  334. else:
  335. # epoll_wait() has a resolution of 1 millisecond, round away
  336. # from zero to wait *at least* timeout seconds.
  337. timeout = math.ceil(timeout * 1e3) * 1e-3
  338. max_ev = len(self._fd_to_key)
  339. ready = []
  340. try:
  341. fd_event_list = wrap_error(self._epoll.poll, timeout, max_ev)
  342. except InterruptedError:
  343. return ready
  344. for fd, event in fd_event_list:
  345. events = 0
  346. if event & ~select.EPOLLIN:
  347. events |= EVENT_WRITE
  348. if event & ~select.EPOLLOUT:
  349. events |= EVENT_READ
  350. key = self._key_from_fd(fd)
  351. if key:
  352. ready.append((key, events & key.events))
  353. return ready
  354. def close(self):
  355. self._epoll.close()
  356. super(EpollSelector, self).close()
  357. if hasattr(select, 'devpoll'):
  358. class DevpollSelector(_BaseSelectorImpl):
  359. """Solaris /dev/poll selector."""
  360. def __init__(self):
  361. super(DevpollSelector, self).__init__()
  362. self._devpoll = select.devpoll()
  363. def fileno(self):
  364. return self._devpoll.fileno()
  365. def register(self, fileobj, events, data=None):
  366. key = super(DevpollSelector, self).register(fileobj, events, data)
  367. poll_events = 0
  368. if events & EVENT_READ:
  369. poll_events |= select.POLLIN
  370. if events & EVENT_WRITE:
  371. poll_events |= select.POLLOUT
  372. self._devpoll.register(key.fd, poll_events)
  373. return key
  374. def unregister(self, fileobj):
  375. key = super(DevpollSelector, self).unregister(fileobj)
  376. self._devpoll.unregister(key.fd)
  377. return key
  378. def select(self, timeout=None):
  379. if timeout is None:
  380. timeout = None
  381. elif timeout <= 0:
  382. timeout = 0
  383. else:
  384. # devpoll() has a resolution of 1 millisecond, round away from
  385. # zero to wait *at least* timeout seconds.
  386. timeout = math.ceil(timeout * 1e3)
  387. ready = []
  388. try:
  389. fd_event_list = self._devpoll.poll(timeout)
  390. except InterruptedError:
  391. return ready
  392. for fd, event in fd_event_list:
  393. events = 0
  394. if event & ~select.POLLIN:
  395. events |= EVENT_WRITE
  396. if event & ~select.POLLOUT:
  397. events |= EVENT_READ
  398. key = self._key_from_fd(fd)
  399. if key:
  400. ready.append((key, events & key.events))
  401. return ready
  402. def close(self):
  403. self._devpoll.close()
  404. super(DevpollSelector, self).close()
  405. if hasattr(select, 'kqueue'):
  406. class KqueueSelector(_BaseSelectorImpl):
  407. """Kqueue-based selector."""
  408. def __init__(self):
  409. super(KqueueSelector, self).__init__()
  410. self._kqueue = select.kqueue()
  411. def fileno(self):
  412. return self._kqueue.fileno()
  413. def register(self, fileobj, events, data=None):
  414. key = super(KqueueSelector, self).register(fileobj, events, data)
  415. if events & EVENT_READ:
  416. kev = select.kevent(key.fd, select.KQ_FILTER_READ,
  417. select.KQ_EV_ADD)
  418. self._kqueue.control([kev], 0, 0)
  419. if events & EVENT_WRITE:
  420. kev = select.kevent(key.fd, select.KQ_FILTER_WRITE,
  421. select.KQ_EV_ADD)
  422. self._kqueue.control([kev], 0, 0)
  423. return key
  424. def unregister(self, fileobj):
  425. key = super(KqueueSelector, self).unregister(fileobj)
  426. if key.events & EVENT_READ:
  427. kev = select.kevent(key.fd, select.KQ_FILTER_READ,
  428. select.KQ_EV_DELETE)
  429. try:
  430. self._kqueue.control([kev], 0, 0)
  431. except OSError:
  432. # This can happen if the FD was closed since it
  433. # was registered.
  434. pass
  435. if key.events & EVENT_WRITE:
  436. kev = select.kevent(key.fd, select.KQ_FILTER_WRITE,
  437. select.KQ_EV_DELETE)
  438. try:
  439. self._kqueue.control([kev], 0, 0)
  440. except OSError:
  441. # See comment above.
  442. pass
  443. return key
  444. def select(self, timeout=None):
  445. timeout = None if timeout is None else max(timeout, 0)
  446. max_ev = len(self._fd_to_key)
  447. ready = []
  448. try:
  449. kev_list = wrap_error(self._kqueue.control,
  450. None, max_ev, timeout)
  451. except InterruptedError:
  452. return ready
  453. for kev in kev_list:
  454. fd = kev.ident
  455. flag = kev.filter
  456. events = 0
  457. if flag == select.KQ_FILTER_READ:
  458. events |= EVENT_READ
  459. if flag == select.KQ_FILTER_WRITE:
  460. events |= EVENT_WRITE
  461. key = self._key_from_fd(fd)
  462. if key:
  463. ready.append((key, events & key.events))
  464. return ready
  465. def close(self):
  466. self._kqueue.close()
  467. super(KqueueSelector, self).close()
  468. # Choose the best implementation: roughly, epoll|kqueue|devpoll > poll > select.
  469. # select() also can't accept a FD > FD_SETSIZE (usually around 1024)
  470. if 'KqueueSelector' in globals():
  471. DefaultSelector = KqueueSelector
  472. elif 'EpollSelector' in globals():
  473. DefaultSelector = EpollSelector
  474. elif 'DevpollSelector' in globals():
  475. DefaultSelector = DevpollSelector
  476. elif 'PollSelector' in globals():
  477. DefaultSelector = PollSelector
  478. else:
  479. DefaultSelector = SelectSelector