Development of an internal social media platform with personalised dashboards for students
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

connection.py 38KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074
  1. """
  2. kombu.connection
  3. ================
  4. Broker connection and pools.
  5. """
  6. from __future__ import absolute_import
  7. import os
  8. import socket
  9. from contextlib import contextmanager
  10. from itertools import count, cycle
  11. from operator import itemgetter
  12. # jython breaks on relative import for .exceptions for some reason
  13. # (Issue #112)
  14. from kombu import exceptions
  15. from .five import Empty, range, string_t, text_t, LifoQueue as _LifoQueue
  16. from .log import get_logger
  17. from .transport import get_transport_cls, supports_librabbitmq
  18. from .utils import cached_property, retry_over_time, shufflecycle, HashedSeq
  19. from .utils.compat import OrderedDict
  20. from .utils.functional import lazy
  21. from .utils.url import as_url, parse_url, quote, urlparse
  22. __all__ = ['Connection', 'ConnectionPool', 'ChannelPool']
  23. RESOLVE_ALIASES = {'pyamqp': 'amqp',
  24. 'librabbitmq': 'amqp'}
  25. _LOG_CONNECTION = os.environ.get('KOMBU_LOG_CONNECTION', False)
  26. _LOG_CHANNEL = os.environ.get('KOMBU_LOG_CHANNEL', False)
  27. logger = get_logger(__name__)
  28. roundrobin_failover = cycle
  29. failover_strategies = {
  30. 'round-robin': roundrobin_failover,
  31. 'shuffle': shufflecycle,
  32. }
  33. class Connection(object):
  34. """A connection to the broker.
  35. :param URL: Broker URL, or a list of URLs, e.g.
  36. .. code-block:: python
  37. Connection('amqp://guest:guest@localhost:5672//')
  38. Connection('amqp://foo;amqp://bar', failover_strategy='round-robin')
  39. Connection('redis://', transport_options={
  40. 'visibility_timeout': 3000,
  41. })
  42. import ssl
  43. Connection('amqp://', login_method='EXTERNAL', ssl={
  44. 'ca_certs': '/etc/pki/tls/certs/something.crt',
  45. 'keyfile': '/etc/something/system.key',
  46. 'certfile': '/etc/something/system.cert',
  47. 'cert_reqs': ssl.CERT_REQUIRED,
  48. })
  49. .. admonition:: SSL compatibility
  50. SSL currently only works with the py-amqp, amqplib, and qpid
  51. transports. For other transports you can use stunnel.
  52. :keyword ssl: Use SSL to connect to the server. Default is ``False``.
  53. May not be supported by the specified transport.
  54. :keyword transport: Default transport if not specified in the URL.
  55. :keyword connect_timeout: Timeout in seconds for connecting to the
  56. server. May not be supported by the specified transport.
  57. :keyword transport_options: A dict of additional connection arguments to
  58. pass to alternate kombu channel implementations. Consult the transport
  59. documentation for available options.
  60. :keyword heartbeat: Heartbeat interval in int/float seconds.
  61. Note that if heartbeats are enabled then the :meth:`heartbeat_check`
  62. method must be called regularly, around once per second.
  63. .. note::
  64. The connection is established lazily when needed. If you need the
  65. connection to be established, then force it by calling
  66. :meth:`connect`::
  67. >>> conn = Connection('amqp://')
  68. >>> conn.connect()
  69. and always remember to close the connection::
  70. >>> conn.release()
  71. *Legacy options*
  72. These options have been replaced by the URL argument, but are still
  73. supported for backwards compatibility:
  74. :keyword hostname: Host name/address.
  75. NOTE: You cannot specify both the URL argument and use the hostname
  76. keyword argument at the same time.
  77. :keyword userid: Default user name if not provided in the URL.
  78. :keyword password: Default password if not provided in the URL.
  79. :keyword virtual_host: Default virtual host if not provided in the URL.
  80. :keyword port: Default port if not provided in the URL.
  81. """
  82. port = None
  83. virtual_host = '/'
  84. connect_timeout = 5
  85. _closed = None
  86. _connection = None
  87. _default_channel = None
  88. _transport = None
  89. _logger = False
  90. uri_prefix = None
  91. #: The cache of declared entities is per connection,
  92. #: in case the server loses data.
  93. declared_entities = None
  94. #: Iterator returning the next broker URL to try in the event
  95. #: of connection failure (initialized by :attr:`failover_strategy`).
  96. cycle = None
  97. #: Additional transport specific options,
  98. #: passed on to the transport instance.
  99. transport_options = None
  100. #: Strategy used to select new hosts when reconnecting after connection
  101. #: failure. One of "round-robin", "shuffle" or any custom iterator
  102. #: constantly yielding new URLs to try.
  103. failover_strategy = 'round-robin'
  104. #: Map of failover strategy name to Callable
  105. failover_strategies = failover_strategies
  106. #: Heartbeat value, currently only supported by the py-amqp transport.
  107. heartbeat = None
  108. hostname = userid = password = ssl = login_method = None
  109. def __init__(self, hostname='localhost', userid=None,
  110. password=None, virtual_host=None, port=None, insist=False,
  111. ssl=False, transport=None, connect_timeout=5,
  112. transport_options=None, login_method=None, uri_prefix=None,
  113. heartbeat=0, failover_strategy='round-robin',
  114. alternates=None, **kwargs):
  115. alt = [] if alternates is None else alternates
  116. # have to spell the args out, just to get nice docstrings :(
  117. params = self._initial_params = {
  118. 'hostname': hostname, 'userid': userid,
  119. 'password': password, 'virtual_host': virtual_host,
  120. 'port': port, 'insist': insist, 'ssl': ssl,
  121. 'transport': transport, 'connect_timeout': connect_timeout,
  122. 'login_method': login_method, 'heartbeat': heartbeat
  123. }
  124. if hostname and not isinstance(hostname, string_t):
  125. alt.extend(hostname)
  126. hostname = alt[0]
  127. if hostname and '://' in hostname:
  128. if ';' in hostname:
  129. alt.extend(hostname.split(';'))
  130. hostname = alt[0]
  131. if '+' in hostname[:hostname.index('://')]:
  132. # e.g. sqla+mysql://root:masterkey@localhost/
  133. params['transport'], params['hostname'] = \
  134. hostname.split('+', 1)
  135. transport = self.uri_prefix = params['transport']
  136. else:
  137. transport = transport or urlparse(hostname).scheme
  138. if not get_transport_cls(transport).can_parse_url:
  139. # we must parse the URL
  140. params.update(parse_url(hostname))
  141. params['transport'] = transport
  142. self._init_params(**params)
  143. # fallback hosts
  144. self.alt = alt
  145. self._failover_strategy_arg = failover_strategy or 'round-robin'
  146. self.failover_strategy = self.failover_strategies.get(
  147. self._failover_strategy_arg) or failover_strategy
  148. if self.alt:
  149. self.cycle = self.failover_strategy(self.alt)
  150. next(self.cycle) # skip first entry
  151. if transport_options is None:
  152. transport_options = {}
  153. self.transport_options = transport_options
  154. if _LOG_CONNECTION: # pragma: no cover
  155. self._logger = True
  156. if uri_prefix:
  157. self.uri_prefix = uri_prefix
  158. self.declared_entities = set()
  159. def switch(self, url):
  160. """Switch connection parameters to use a new URL (does not
  161. reconnect)"""
  162. self.close()
  163. self.declared_entities.clear()
  164. self._closed = False
  165. self._init_params(**dict(self._initial_params, **parse_url(url)))
  166. def maybe_switch_next(self):
  167. """Switch to next URL given by the current failover strategy (if
  168. any)."""
  169. if self.cycle:
  170. self.switch(next(self.cycle))
  171. def _init_params(self, hostname, userid, password, virtual_host, port,
  172. insist, ssl, transport, connect_timeout,
  173. login_method, heartbeat):
  174. transport = transport or 'amqp'
  175. if transport == 'amqp' and supports_librabbitmq():
  176. transport = 'librabbitmq'
  177. self.hostname = hostname
  178. self.userid = userid
  179. self.password = password
  180. self.login_method = login_method
  181. self.virtual_host = virtual_host or self.virtual_host
  182. self.port = port or self.port
  183. self.insist = insist
  184. self.connect_timeout = connect_timeout
  185. self.ssl = ssl
  186. self.transport_cls = transport
  187. self.heartbeat = heartbeat and float(heartbeat)
  188. def register_with_event_loop(self, loop):
  189. self.transport.register_with_event_loop(self.connection, loop)
  190. def _debug(self, msg, *args, **kwargs):
  191. if self._logger: # pragma: no cover
  192. fmt = '[Kombu connection:0x{id:x}] {msg}'
  193. logger.debug(fmt.format(id=id(self), msg=text_t(msg)),
  194. *args, **kwargs)
  195. def connect(self):
  196. """Establish connection to server immediately."""
  197. self._closed = False
  198. return self.connection
  199. def channel(self):
  200. """Create and return a new channel."""
  201. self._debug('create channel')
  202. chan = self.transport.create_channel(self.connection)
  203. if _LOG_CHANNEL: # pragma: no cover
  204. from .utils.debug import Logwrapped
  205. return Logwrapped(chan, 'kombu.channel',
  206. '[Kombu channel:{0.channel_id}] ')
  207. return chan
  208. def heartbeat_check(self, rate=2):
  209. """Allow the transport to perform any periodic tasks
  210. required to make heartbeats work. This should be called
  211. approximately every second.
  212. If the current transport does not support heartbeats then
  213. this is a noop operation.
  214. :keyword rate: Rate is how often the tick is called
  215. compared to the actual heartbeat value. E.g. if
  216. the heartbeat is set to 3 seconds, and the tick
  217. is called every 3 / 2 seconds, then the rate is 2.
  218. This value is currently unused by any transports.
  219. """
  220. return self.transport.heartbeat_check(self.connection, rate=rate)
  221. def drain_events(self, **kwargs):
  222. """Wait for a single event from the server.
  223. :keyword timeout: Timeout in seconds before we give up.
  224. :raises :exc:`socket.timeout`: if the timeout is exceeded.
  225. """
  226. return self.transport.drain_events(self.connection, **kwargs)
  227. def maybe_close_channel(self, channel):
  228. """Close given channel, but ignore connection and channel errors."""
  229. try:
  230. channel.close()
  231. except (self.connection_errors + self.channel_errors):
  232. pass
  233. def _do_close_self(self):
  234. # Close only connection and channel(s), but not transport.
  235. self.declared_entities.clear()
  236. if self._default_channel:
  237. self.maybe_close_channel(self._default_channel)
  238. if self._connection:
  239. try:
  240. self.transport.close_connection(self._connection)
  241. except self.connection_errors + (AttributeError, socket.error):
  242. pass
  243. self._connection = None
  244. def _close(self):
  245. """Really close connection, even if part of a connection pool."""
  246. self._do_close_self()
  247. if self._transport:
  248. self._transport.client = None
  249. self._transport = None
  250. self._debug('closed')
  251. self._closed = True
  252. def collect(self, socket_timeout=None):
  253. # amqp requires communication to close, we don't need that just
  254. # to clear out references, Transport._collect can also be implemented
  255. # by other transports that want fast after fork
  256. try:
  257. gc_transport = self._transport._collect
  258. except AttributeError:
  259. _timeo = socket.getdefaulttimeout()
  260. socket.setdefaulttimeout(socket_timeout)
  261. try:
  262. self._close()
  263. except socket.timeout:
  264. pass
  265. finally:
  266. socket.setdefaulttimeout(_timeo)
  267. else:
  268. gc_transport(self._connection)
  269. if self._transport:
  270. self._transport.client = None
  271. self._transport = None
  272. self.declared_entities.clear()
  273. self._connection = None
  274. def release(self):
  275. """Close the connection (if open)."""
  276. self._close()
  277. close = release
  278. def ensure_connection(self, errback=None, max_retries=None,
  279. interval_start=2, interval_step=2, interval_max=30,
  280. callback=None):
  281. """Ensure we have a connection to the server.
  282. If not retry establishing the connection with the settings
  283. specified.
  284. :keyword errback: Optional callback called each time the connection
  285. can't be established. Arguments provided are the exception
  286. raised and the interval that will be slept ``(exc, interval)``.
  287. :keyword max_retries: Maximum number of times to retry.
  288. If this limit is exceeded the connection error will be re-raised.
  289. :keyword interval_start: The number of seconds we start sleeping for.
  290. :keyword interval_step: How many seconds added to the interval
  291. for each retry.
  292. :keyword interval_max: Maximum number of seconds to sleep between
  293. each retry.
  294. :keyword callback: Optional callback that is called for every
  295. internal iteration (1 s)
  296. """
  297. def on_error(exc, intervals, retries, interval=0):
  298. round = self.completes_cycle(retries)
  299. if round:
  300. interval = next(intervals)
  301. if errback:
  302. errback(exc, interval)
  303. self.maybe_switch_next() # select next host
  304. return interval if round else 0
  305. retry_over_time(self.connect, self.recoverable_connection_errors,
  306. (), {}, on_error, max_retries,
  307. interval_start, interval_step, interval_max, callback)
  308. return self
  309. def completes_cycle(self, retries):
  310. """Return true if the cycle is complete after number of `retries`."""
  311. return not (retries + 1) % len(self.alt) if self.alt else True
  312. def revive(self, new_channel):
  313. """Revive connection after connection re-established."""
  314. if self._default_channel:
  315. self.maybe_close_channel(self._default_channel)
  316. self._default_channel = None
  317. def _default_ensure_callback(self, exc, interval):
  318. logger.error("Ensure: Operation error: %r. Retry in %ss",
  319. exc, interval, exc_info=True)
  320. def ensure(self, obj, fun, errback=None, max_retries=None,
  321. interval_start=1, interval_step=1, interval_max=1,
  322. on_revive=None):
  323. """Ensure operation completes, regardless of any channel/connection
  324. errors occurring.
  325. Will retry by establishing the connection, and reapplying
  326. the function.
  327. :param fun: Method to apply.
  328. :keyword errback: Optional callback called each time the connection
  329. can't be established. Arguments provided are the exception
  330. raised and the interval that will be slept ``(exc, interval)``.
  331. :keyword max_retries: Maximum number of times to retry.
  332. If this limit is exceeded the connection error will be re-raised.
  333. :keyword interval_start: The number of seconds we start sleeping for.
  334. :keyword interval_step: How many seconds added to the interval
  335. for each retry.
  336. :keyword interval_max: Maximum number of seconds to sleep between
  337. each retry.
  338. **Example**
  339. This is an example ensuring a publish operation::
  340. >>> from kombu import Connection, Producer
  341. >>> conn = Connection('amqp://')
  342. >>> producer = Producer(conn)
  343. >>> def errback(exc, interval):
  344. ... logger.error('Error: %r', exc, exc_info=1)
  345. ... logger.info('Retry in %s seconds.', interval)
  346. >>> publish = conn.ensure(producer, producer.publish,
  347. ... errback=errback, max_retries=3)
  348. >>> publish({'hello': 'world'}, routing_key='dest')
  349. """
  350. def _ensured(*args, **kwargs):
  351. got_connection = 0
  352. conn_errors = self.recoverable_connection_errors
  353. chan_errors = self.recoverable_channel_errors
  354. has_modern_errors = hasattr(
  355. self.transport, 'recoverable_connection_errors',
  356. )
  357. for retries in count(0): # for infinity
  358. try:
  359. return fun(*args, **kwargs)
  360. except conn_errors as exc:
  361. if got_connection and not has_modern_errors:
  362. # transport can not distinguish between
  363. # recoverable/irrecoverable errors, so we propagate
  364. # the error if it persists after a new connection was
  365. # successfully established.
  366. raise
  367. if max_retries is not None and retries > max_retries:
  368. raise
  369. self._debug('ensure connection error: %r', exc, exc_info=1)
  370. self._connection = None
  371. self._do_close_self()
  372. errback and errback(exc, 0)
  373. remaining_retries = None
  374. if max_retries is not None:
  375. remaining_retries = max(max_retries - retries, 1)
  376. self.ensure_connection(errback,
  377. remaining_retries,
  378. interval_start,
  379. interval_step,
  380. interval_max)
  381. new_channel = self.channel()
  382. self.revive(new_channel)
  383. obj.revive(new_channel)
  384. if on_revive:
  385. on_revive(new_channel)
  386. got_connection += 1
  387. except chan_errors as exc:
  388. if max_retries is not None and retries > max_retries:
  389. raise
  390. self._debug('ensure channel error: %r', exc, exc_info=1)
  391. errback and errback(exc, 0)
  392. _ensured.__name__ = "%s(ensured)" % fun.__name__
  393. _ensured.__doc__ = fun.__doc__
  394. _ensured.__module__ = fun.__module__
  395. return _ensured
  396. def autoretry(self, fun, channel=None, **ensure_options):
  397. """Decorator for functions supporting a ``channel`` keyword argument.
  398. The resulting callable will retry calling the function if
  399. it raises connection or channel related errors.
  400. The return value will be a tuple of ``(retval, last_created_channel)``.
  401. If a ``channel`` is not provided, then one will be automatically
  402. acquired (remember to close it afterwards).
  403. See :meth:`ensure` for the full list of supported keyword arguments.
  404. Example usage::
  405. channel = connection.channel()
  406. try:
  407. ret, channel = connection.autoretry(publish_messages, channel)
  408. finally:
  409. channel.close()
  410. """
  411. channels = [channel]
  412. create_channel = self.channel
  413. class Revival(object):
  414. __name__ = getattr(fun, '__name__', None)
  415. __module__ = getattr(fun, '__module__', None)
  416. __doc__ = getattr(fun, '__doc__', None)
  417. def revive(self, channel):
  418. channels[0] = channel
  419. def __call__(self, *args, **kwargs):
  420. if channels[0] is None:
  421. self.revive(create_channel())
  422. return fun(*args, channel=channels[0], **kwargs), channels[0]
  423. revive = Revival()
  424. return self.ensure(revive, revive, **ensure_options)
  425. def create_transport(self):
  426. return self.get_transport_cls()(client=self)
  427. def get_transport_cls(self):
  428. """Get the currently used transport class."""
  429. transport_cls = self.transport_cls
  430. if not transport_cls or isinstance(transport_cls, string_t):
  431. transport_cls = get_transport_cls(transport_cls)
  432. return transport_cls
  433. def clone(self, **kwargs):
  434. """Create a copy of the connection with the same connection
  435. settings."""
  436. return self.__class__(**dict(self._info(resolve=False), **kwargs))
  437. def get_heartbeat_interval(self):
  438. return self.transport.get_heartbeat_interval(self.connection)
  439. def _info(self, resolve=True):
  440. transport_cls = self.transport_cls
  441. if resolve:
  442. transport_cls = RESOLVE_ALIASES.get(transport_cls, transport_cls)
  443. D = self.transport.default_connection_params
  444. hostname = self.hostname or D.get('hostname')
  445. if self.uri_prefix:
  446. hostname = '%s+%s' % (self.uri_prefix, hostname)
  447. info = (
  448. ('hostname', hostname),
  449. ('userid', self.userid or D.get('userid')),
  450. ('password', self.password or D.get('password')),
  451. ('virtual_host', self.virtual_host or D.get('virtual_host')),
  452. ('port', self.port or D.get('port')),
  453. ('insist', self.insist),
  454. ('ssl', self.ssl),
  455. ('transport', transport_cls),
  456. ('connect_timeout', self.connect_timeout),
  457. ('transport_options', self.transport_options),
  458. ('login_method', self.login_method or D.get('login_method')),
  459. ('uri_prefix', self.uri_prefix),
  460. ('heartbeat', self.heartbeat),
  461. ('failover_strategy', self._failover_strategy_arg),
  462. ('alternates', self.alt),
  463. )
  464. return info
  465. def info(self):
  466. """Get connection info."""
  467. return OrderedDict(self._info())
  468. def __eqhash__(self):
  469. return HashedSeq(self.transport_cls, self.hostname, self.userid,
  470. self.password, self.virtual_host, self.port,
  471. repr(self.transport_options))
  472. def as_uri(self, include_password=False, mask='**',
  473. getfields=itemgetter('port', 'userid', 'password',
  474. 'virtual_host', 'transport')):
  475. """Convert connection parameters to URL form."""
  476. hostname = self.hostname or 'localhost'
  477. if self.transport.can_parse_url:
  478. if self.uri_prefix:
  479. return '%s+%s' % (self.uri_prefix, hostname)
  480. return self.hostname
  481. if self.uri_prefix:
  482. return '%s+%s' % (self.uri_prefix, hostname)
  483. fields = self.info()
  484. port, userid, password, vhost, transport = getfields(fields)
  485. return as_url(
  486. transport, hostname, port, userid, password, quote(vhost),
  487. sanitize=not include_password, mask=mask,
  488. )
  489. def Pool(self, limit=None, preload=None):
  490. """Pool of connections.
  491. See :class:`ConnectionPool`.
  492. :keyword limit: Maximum number of active connections.
  493. Default is no limit.
  494. :keyword preload: Number of connections to preload
  495. when the pool is created. Default is 0.
  496. *Example usage*::
  497. >>> connection = Connection('amqp://')
  498. >>> pool = connection.Pool(2)
  499. >>> c1 = pool.acquire()
  500. >>> c2 = pool.acquire()
  501. >>> c3 = pool.acquire()
  502. Traceback (most recent call last):
  503. File "<stdin>", line 1, in <module>
  504. File "kombu/connection.py", line 354, in acquire
  505. raise ConnectionLimitExceeded(self.limit)
  506. kombu.exceptions.ConnectionLimitExceeded: 2
  507. >>> c1.release()
  508. >>> c3 = pool.acquire()
  509. """
  510. return ConnectionPool(self, limit, preload)
  511. def ChannelPool(self, limit=None, preload=None):
  512. """Pool of channels.
  513. See :class:`ChannelPool`.
  514. :keyword limit: Maximum number of active channels.
  515. Default is no limit.
  516. :keyword preload: Number of channels to preload
  517. when the pool is created. Default is 0.
  518. *Example usage*::
  519. >>> connection = Connection('amqp://')
  520. >>> pool = connection.ChannelPool(2)
  521. >>> c1 = pool.acquire()
  522. >>> c2 = pool.acquire()
  523. >>> c3 = pool.acquire()
  524. Traceback (most recent call last):
  525. File "<stdin>", line 1, in <module>
  526. File "kombu/connection.py", line 354, in acquire
  527. raise ChannelLimitExceeded(self.limit)
  528. kombu.connection.ChannelLimitExceeded: 2
  529. >>> c1.release()
  530. >>> c3 = pool.acquire()
  531. """
  532. return ChannelPool(self, limit, preload)
  533. def Producer(self, channel=None, *args, **kwargs):
  534. """Create new :class:`kombu.Producer` instance using this
  535. connection."""
  536. from .messaging import Producer
  537. return Producer(channel or self, *args, **kwargs)
  538. def Consumer(self, queues=None, channel=None, *args, **kwargs):
  539. """Create new :class:`kombu.Consumer` instance using this
  540. connection."""
  541. from .messaging import Consumer
  542. return Consumer(channel or self, queues, *args, **kwargs)
  543. def SimpleQueue(self, name, no_ack=None, queue_opts=None,
  544. exchange_opts=None, channel=None, **kwargs):
  545. """Create new :class:`~kombu.simple.SimpleQueue`, using a channel
  546. from this connection.
  547. If ``name`` is a string, a queue and exchange will be automatically
  548. created using that name as the name of the queue and exchange,
  549. also it will be used as the default routing key.
  550. :param name: Name of the queue/or a :class:`~kombu.Queue`.
  551. :keyword no_ack: Disable acknowledgements. Default is false.
  552. :keyword queue_opts: Additional keyword arguments passed to the
  553. constructor of the automatically created
  554. :class:`~kombu.Queue`.
  555. :keyword exchange_opts: Additional keyword arguments passed to the
  556. constructor of the automatically created
  557. :class:`~kombu.Exchange`.
  558. :keyword channel: Custom channel to use. If not specified the
  559. connection default channel is used.
  560. """
  561. from .simple import SimpleQueue
  562. return SimpleQueue(channel or self, name, no_ack, queue_opts,
  563. exchange_opts, **kwargs)
  564. def SimpleBuffer(self, name, no_ack=None, queue_opts=None,
  565. exchange_opts=None, channel=None, **kwargs):
  566. """Create new :class:`~kombu.simple.SimpleQueue` using a channel
  567. from this connection.
  568. Same as :meth:`SimpleQueue`, but configured with buffering
  569. semantics. The resulting queue and exchange will not be durable, also
  570. auto delete is enabled. Messages will be transient (not persistent),
  571. and acknowledgements are disabled (``no_ack``).
  572. """
  573. from .simple import SimpleBuffer
  574. return SimpleBuffer(channel or self, name, no_ack, queue_opts,
  575. exchange_opts, **kwargs)
  576. def _establish_connection(self):
  577. self._debug('establishing connection...')
  578. conn = self.transport.establish_connection()
  579. self._debug('connection established: %r', conn)
  580. return conn
  581. def __repr__(self):
  582. """``x.__repr__() <==> repr(x)``"""
  583. return '<Connection: {0} at 0x{1:x}>'.format(self.as_uri(), id(self))
  584. def __copy__(self):
  585. """``x.__copy__() <==> copy(x)``"""
  586. return self.clone()
  587. def __reduce__(self):
  588. return self.__class__, tuple(self.info().values()), None
  589. def __enter__(self):
  590. return self
  591. def __exit__(self, *args):
  592. self.release()
  593. @property
  594. def qos_semantics_matches_spec(self):
  595. return self.transport.qos_semantics_matches_spec(self.connection)
  596. @property
  597. def connected(self):
  598. """Return true if the connection has been established."""
  599. return (not self._closed and
  600. self._connection is not None and
  601. self.transport.verify_connection(self._connection))
  602. @property
  603. def connection(self):
  604. """The underlying connection object.
  605. .. warning::
  606. This instance is transport specific, so do not
  607. depend on the interface of this object.
  608. """
  609. if not self._closed:
  610. if not self.connected:
  611. self.declared_entities.clear()
  612. self._default_channel = None
  613. self._connection = self._establish_connection()
  614. self._closed = False
  615. return self._connection
  616. @property
  617. def default_channel(self):
  618. """Default channel, created upon access and closed when the connection
  619. is closed.
  620. Can be used for automatic channel handling when you only need one
  621. channel, and also it is the channel implicitly used if a connection
  622. is passed instead of a channel, to functions that require a channel.
  623. """
  624. # make sure we're still connected, and if not refresh.
  625. self.connection
  626. if self._default_channel is None:
  627. self._default_channel = self.channel()
  628. return self._default_channel
  629. @property
  630. def host(self):
  631. """The host as a host name/port pair separated by colon."""
  632. return ':'.join([self.hostname, str(self.port)])
  633. @property
  634. def transport(self):
  635. if self._transport is None:
  636. self._transport = self.create_transport()
  637. return self._transport
  638. @cached_property
  639. def manager(self):
  640. """Experimental manager that can be used to manage/monitor the broker
  641. instance. Not available for all transports."""
  642. return self.transport.manager
  643. def get_manager(self, *args, **kwargs):
  644. return self.transport.get_manager(*args, **kwargs)
  645. @cached_property
  646. def recoverable_connection_errors(self):
  647. """List of connection related exceptions that can be recovered from,
  648. but where the connection must be closed and re-established first."""
  649. try:
  650. return self.transport.recoverable_connection_errors
  651. except AttributeError:
  652. # There were no such classification before,
  653. # and all errors were assumed to be recoverable,
  654. # so this is a fallback for transports that do
  655. # not support the new recoverable/irrecoverable classes.
  656. return self.connection_errors + self.channel_errors
  657. @cached_property
  658. def recoverable_channel_errors(self):
  659. """List of channel related exceptions that can be automatically
  660. recovered from without re-establishing the connection."""
  661. try:
  662. return self.transport.recoverable_channel_errors
  663. except AttributeError:
  664. return ()
  665. @cached_property
  666. def connection_errors(self):
  667. """List of exceptions that may be raised by the connection."""
  668. return self.transport.connection_errors
  669. @cached_property
  670. def channel_errors(self):
  671. """List of exceptions that may be raised by the channel."""
  672. return self.transport.channel_errors
  673. @property
  674. def supports_heartbeats(self):
  675. return self.transport.supports_heartbeats
  676. @property
  677. def is_evented(self):
  678. return self.transport.supports_ev
  679. BrokerConnection = Connection
  680. class Resource(object):
  681. LimitExceeded = exceptions.LimitExceeded
  682. def __init__(self, limit=None, preload=None):
  683. self.limit = limit
  684. self.preload = preload or 0
  685. self._closed = False
  686. self._resource = _LifoQueue()
  687. self._dirty = set()
  688. self.setup()
  689. def setup(self):
  690. raise NotImplementedError('subclass responsibility')
  691. def _add_when_empty(self):
  692. if self.limit and len(self._dirty) >= self.limit:
  693. raise self.LimitExceeded(self.limit)
  694. # All taken, put new on the queue and
  695. # try get again, this way the first in line
  696. # will get the resource.
  697. self._resource.put_nowait(self.new())
  698. def acquire(self, block=False, timeout=None):
  699. """Acquire resource.
  700. :keyword block: If the limit is exceeded,
  701. block until there is an available item.
  702. :keyword timeout: Timeout to wait
  703. if ``block`` is true. Default is :const:`None` (forever).
  704. :raises LimitExceeded: if block is false
  705. and the limit has been exceeded.
  706. """
  707. if self._closed:
  708. raise RuntimeError('Acquire on closed pool')
  709. if self.limit:
  710. while 1:
  711. try:
  712. R = self._resource.get(block=block, timeout=timeout)
  713. except Empty:
  714. self._add_when_empty()
  715. else:
  716. try:
  717. R = self.prepare(R)
  718. except BaseException:
  719. if isinstance(R, lazy):
  720. # no evaluated yet, just put it back
  721. self._resource.put_nowait(R)
  722. else:
  723. # evaluted so must try to release/close first.
  724. self.release(R)
  725. raise
  726. self._dirty.add(R)
  727. break
  728. else:
  729. R = self.prepare(self.new())
  730. def release():
  731. """Release resource so it can be used by another thread.
  732. The caller is responsible for discarding the object,
  733. and to never use the resource again. A new resource must
  734. be acquired if so needed.
  735. """
  736. self.release(R)
  737. R.release = release
  738. return R
  739. def prepare(self, resource):
  740. return resource
  741. def close_resource(self, resource):
  742. resource.close()
  743. def release_resource(self, resource):
  744. pass
  745. def replace(self, resource):
  746. """Replace resource with a new instance. This can be used in case
  747. of defective resources."""
  748. if self.limit:
  749. self._dirty.discard(resource)
  750. self.close_resource(resource)
  751. def release(self, resource):
  752. if self.limit:
  753. self._dirty.discard(resource)
  754. self._resource.put_nowait(resource)
  755. self.release_resource(resource)
  756. else:
  757. self.close_resource(resource)
  758. def collect_resource(self, resource):
  759. pass
  760. def force_close_all(self):
  761. """Close and remove all resources in the pool (also those in use).
  762. Can be used to close resources from parent processes
  763. after fork (e.g. sockets/connections).
  764. """
  765. self._closed = True
  766. dirty = self._dirty
  767. resource = self._resource
  768. while 1: # - acquired
  769. try:
  770. dres = dirty.pop()
  771. except KeyError:
  772. break
  773. try:
  774. self.collect_resource(dres)
  775. except AttributeError: # Issue #78
  776. pass
  777. while 1: # - available
  778. # deque supports '.clear', but lists do not, so for that
  779. # reason we use pop here, so that the underlying object can
  780. # be any object supporting '.pop' and '.append'.
  781. try:
  782. res = resource.queue.pop()
  783. except IndexError:
  784. break
  785. try:
  786. self.collect_resource(res)
  787. except AttributeError:
  788. pass # Issue #78
  789. if os.environ.get('KOMBU_DEBUG_POOL'): # pragma: no cover
  790. _orig_acquire = acquire
  791. _orig_release = release
  792. _next_resource_id = 0
  793. def acquire(self, *args, **kwargs): # noqa
  794. import traceback
  795. id = self._next_resource_id = self._next_resource_id + 1
  796. print('+{0} ACQUIRE {1}'.format(id, self.__class__.__name__))
  797. r = self._orig_acquire(*args, **kwargs)
  798. r._resource_id = id
  799. print('-{0} ACQUIRE {1}'.format(id, self.__class__.__name__))
  800. if not hasattr(r, 'acquired_by'):
  801. r.acquired_by = []
  802. r.acquired_by.append(traceback.format_stack())
  803. return r
  804. def release(self, resource): # noqa
  805. id = resource._resource_id
  806. print('+{0} RELEASE {1}'.format(id, self.__class__.__name__))
  807. r = self._orig_release(resource)
  808. print('-{0} RELEASE {1}'.format(id, self.__class__.__name__))
  809. self._next_resource_id -= 1
  810. return r
  811. class ConnectionPool(Resource):
  812. LimitExceeded = exceptions.ConnectionLimitExceeded
  813. def __init__(self, connection, limit=None, preload=None):
  814. self.connection = connection
  815. super(ConnectionPool, self).__init__(limit=limit,
  816. preload=preload)
  817. def new(self):
  818. return self.connection.clone()
  819. def release_resource(self, resource):
  820. try:
  821. resource._debug('released')
  822. except AttributeError:
  823. pass
  824. def close_resource(self, resource):
  825. resource._close()
  826. def collect_resource(self, resource, socket_timeout=0.1):
  827. return resource.collect(socket_timeout)
  828. @contextmanager
  829. def acquire_channel(self, block=False):
  830. with self.acquire(block=block) as connection:
  831. yield connection, connection.default_channel
  832. def setup(self):
  833. if self.limit:
  834. for i in range(self.limit):
  835. if i < self.preload:
  836. conn = self.new()
  837. conn.connect()
  838. else:
  839. conn = lazy(self.new)
  840. self._resource.put_nowait(conn)
  841. def prepare(self, resource):
  842. if callable(resource):
  843. resource = resource()
  844. resource._debug('acquired')
  845. return resource
  846. class ChannelPool(Resource):
  847. LimitExceeded = exceptions.ChannelLimitExceeded
  848. def __init__(self, connection, limit=None, preload=None):
  849. self.connection = connection
  850. super(ChannelPool, self).__init__(limit=limit,
  851. preload=preload)
  852. def new(self):
  853. return lazy(self.connection.channel)
  854. def setup(self):
  855. channel = self.new()
  856. if self.limit:
  857. for i in range(self.limit):
  858. self._resource.put_nowait(
  859. i < self.preload and channel() or lazy(channel))
  860. def prepare(self, channel):
  861. if callable(channel):
  862. channel = channel()
  863. return channel
  864. def maybe_channel(channel):
  865. """Return the default channel if argument is a connection instance,
  866. otherwise just return the channel given."""
  867. if isinstance(channel, Connection):
  868. return channel.default_channel
  869. return channel
  870. def is_connection(obj):
  871. return isinstance(obj, Connection)