Development of an internal social media platform with personalised dashboards for students
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

SLMQ.py 5.9KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186
  1. """
  2. kombu.transport.SLMQ
  3. ====================
  4. SoftLayer Message Queue transport.
  5. """
  6. from __future__ import absolute_import
  7. import socket
  8. import string
  9. from anyjson import loads, dumps
  10. import os
  11. from kombu.five import Empty, text_t
  12. from kombu.utils import cached_property # , uuid
  13. from kombu.utils.encoding import bytes_to_str, safe_str
  14. from . import virtual
  15. try:
  16. from softlayer_messaging import get_client
  17. from softlayer_messaging.errors import ResponseError
  18. except ImportError: # pragma: no cover
  19. get_client = ResponseError = None # noqa
  20. # dots are replaced by dash, all other punctuation replaced by underscore.
  21. CHARS_REPLACE_TABLE = dict(
  22. (ord(c), 0x5f) for c in string.punctuation if c not in '_')
  23. class Channel(virtual.Channel):
  24. default_visibility_timeout = 1800 # 30 minutes.
  25. domain_format = 'kombu%(vhost)s'
  26. _slmq = None
  27. _queue_cache = {}
  28. _noack_queues = set()
  29. def __init__(self, *args, **kwargs):
  30. if get_client is None:
  31. raise ImportError(
  32. 'SLMQ transport requires the softlayer_messaging library',
  33. )
  34. super(Channel, self).__init__(*args, **kwargs)
  35. queues = self.slmq.queues()
  36. for queue in queues:
  37. self._queue_cache[queue] = queue
  38. def basic_consume(self, queue, no_ack, *args, **kwargs):
  39. if no_ack:
  40. self._noack_queues.add(queue)
  41. return super(Channel, self).basic_consume(queue, no_ack,
  42. *args, **kwargs)
  43. def basic_cancel(self, consumer_tag):
  44. if consumer_tag in self._consumers:
  45. queue = self._tag_to_queue[consumer_tag]
  46. self._noack_queues.discard(queue)
  47. return super(Channel, self).basic_cancel(consumer_tag)
  48. def entity_name(self, name, table=CHARS_REPLACE_TABLE):
  49. """Format AMQP queue name into a valid SLQS queue name."""
  50. return text_t(safe_str(name)).translate(table)
  51. def _new_queue(self, queue, **kwargs):
  52. """Ensures a queue exists in SLQS."""
  53. queue = self.entity_name(self.queue_name_prefix + queue)
  54. try:
  55. return self._queue_cache[queue]
  56. except KeyError:
  57. try:
  58. self.slmq.create_queue(
  59. queue, visibility_timeout=self.visibility_timeout)
  60. except ResponseError:
  61. pass
  62. q = self._queue_cache[queue] = self.slmq.queue(queue)
  63. return q
  64. def _delete(self, queue, *args):
  65. """delete queue by name."""
  66. queue_name = self.entity_name(queue)
  67. self._queue_cache.pop(queue_name, None)
  68. self.slmq.queue(queue_name).delete(force=True)
  69. super(Channel, self)._delete(queue_name)
  70. def _put(self, queue, message, **kwargs):
  71. """Put message onto queue."""
  72. q = self._new_queue(queue)
  73. q.push(dumps(message))
  74. def _get(self, queue):
  75. """Try to retrieve a single message off ``queue``."""
  76. q = self._new_queue(queue)
  77. rs = q.pop(1)
  78. if rs['items']:
  79. m = rs['items'][0]
  80. payload = loads(bytes_to_str(m['body']))
  81. if queue in self._noack_queues:
  82. q.message(m['id']).delete()
  83. else:
  84. payload['properties']['delivery_info'].update({
  85. 'slmq_message_id': m['id'], 'slmq_queue_name': q.name})
  86. return payload
  87. raise Empty()
  88. def basic_ack(self, delivery_tag):
  89. delivery_info = self.qos.get(delivery_tag).delivery_info
  90. try:
  91. queue = delivery_info['slmq_queue_name']
  92. except KeyError:
  93. pass
  94. else:
  95. self.delete_message(queue, delivery_info['slmq_message_id'])
  96. super(Channel, self).basic_ack(delivery_tag)
  97. def _size(self, queue):
  98. """Return the number of messages in a queue."""
  99. return self._new_queue(queue).detail()['message_count']
  100. def _purge(self, queue):
  101. """Delete all current messages in a queue."""
  102. q = self._new_queue(queue)
  103. n = 0
  104. l = q.pop(10)
  105. while l['items']:
  106. for m in l['items']:
  107. self.delete_message(queue, m['id'])
  108. n += 1
  109. l = q.pop(10)
  110. return n
  111. def delete_message(self, queue, message_id):
  112. q = self.slmq.queue(self.entity_name(queue))
  113. return q.message(message_id).delete()
  114. @property
  115. def slmq(self):
  116. if self._slmq is None:
  117. conninfo = self.conninfo
  118. account = os.environ.get('SLMQ_ACCOUNT', conninfo.virtual_host)
  119. user = os.environ.get('SL_USERNAME', conninfo.userid)
  120. api_key = os.environ.get('SL_API_KEY', conninfo.password)
  121. host = os.environ.get('SLMQ_HOST', conninfo.hostname)
  122. port = os.environ.get('SLMQ_PORT', conninfo.port)
  123. secure = bool(os.environ.get(
  124. 'SLMQ_SECURE', self.transport_options.get('secure')) or True,
  125. )
  126. endpoint = '{0}://{1}{2}'.format(
  127. 'https' if secure else 'http', host,
  128. ':{0}'.format(port) if port else '',
  129. )
  130. self._slmq = get_client(account, endpoint=endpoint)
  131. self._slmq.authenticate(user, api_key)
  132. return self._slmq
  133. @property
  134. def conninfo(self):
  135. return self.connection.client
  136. @property
  137. def transport_options(self):
  138. return self.connection.client.transport_options
  139. @cached_property
  140. def visibility_timeout(self):
  141. return (self.transport_options.get('visibility_timeout') or
  142. self.default_visibility_timeout)
  143. @cached_property
  144. def queue_name_prefix(self):
  145. return self.transport_options.get('queue_name_prefix', '')
  146. class Transport(virtual.Transport):
  147. Channel = Channel
  148. polling_interval = 1
  149. default_port = None
  150. connection_errors = (
  151. virtual.Transport.connection_errors + (
  152. ResponseError, socket.error
  153. )
  154. )