You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

db.py 11KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284
  1. "Database cache backend."
  2. import base64
  3. import pickle
  4. from datetime import datetime
  5. from django.conf import settings
  6. from django.core.cache.backends.base import DEFAULT_TIMEOUT, BaseCache
  7. from django.db import DatabaseError, connections, models, router, transaction
  8. from django.utils import timezone
  9. from django.utils.inspect import func_supports_parameter
  10. class Options:
  11. """A class that will quack like a Django model _meta class.
  12. This allows cache operations to be controlled by the router
  13. """
  14. def __init__(self, table):
  15. self.db_table = table
  16. self.app_label = 'django_cache'
  17. self.model_name = 'cacheentry'
  18. self.verbose_name = 'cache entry'
  19. self.verbose_name_plural = 'cache entries'
  20. self.object_name = 'CacheEntry'
  21. self.abstract = False
  22. self.managed = True
  23. self.proxy = False
  24. self.swapped = False
  25. class BaseDatabaseCache(BaseCache):
  26. def __init__(self, table, params):
  27. super().__init__(params)
  28. self._table = table
  29. class CacheEntry:
  30. _meta = Options(table)
  31. self.cache_model_class = CacheEntry
  32. class DatabaseCache(BaseDatabaseCache):
  33. # This class uses cursors provided by the database connection. This means
  34. # it reads expiration values as aware or naive datetimes, depending on the
  35. # value of USE_TZ and whether the database supports time zones. The ORM's
  36. # conversion and adaptation infrastructure is then used to avoid comparing
  37. # aware and naive datetimes accidentally.
  38. pickle_protocol = pickle.HIGHEST_PROTOCOL
  39. def get(self, key, default=None, version=None):
  40. return self.get_many([key], version).get(key, default)
  41. def get_many(self, keys, version=None):
  42. if not keys:
  43. return {}
  44. key_map = {}
  45. for key in keys:
  46. self.validate_key(key)
  47. key_map[self.make_key(key, version)] = key
  48. db = router.db_for_read(self.cache_model_class)
  49. connection = connections[db]
  50. quote_name = connection.ops.quote_name
  51. table = quote_name(self._table)
  52. with connection.cursor() as cursor:
  53. cursor.execute(
  54. 'SELECT %s, %s, %s FROM %s WHERE %s IN (%s)' % (
  55. quote_name('cache_key'),
  56. quote_name('value'),
  57. quote_name('expires'),
  58. table,
  59. quote_name('cache_key'),
  60. ', '.join(['%s'] * len(key_map)),
  61. ),
  62. list(key_map),
  63. )
  64. rows = cursor.fetchall()
  65. result = {}
  66. expired_keys = []
  67. expression = models.Expression(output_field=models.DateTimeField())
  68. converters = (connection.ops.get_db_converters(expression) + expression.get_db_converters(connection))
  69. for key, value, expires in rows:
  70. for converter in converters:
  71. if func_supports_parameter(converter, 'context'): # RemovedInDjango30Warning
  72. expires = converter(expires, expression, connection, {})
  73. else:
  74. expires = converter(expires, expression, connection)
  75. if expires < timezone.now():
  76. expired_keys.append(key)
  77. else:
  78. value = connection.ops.process_clob(value)
  79. value = pickle.loads(base64.b64decode(value.encode()))
  80. result[key_map.get(key)] = value
  81. self._base_delete_many(expired_keys)
  82. return result
  83. def set(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
  84. key = self.make_key(key, version=version)
  85. self.validate_key(key)
  86. self._base_set('set', key, value, timeout)
  87. def add(self, key, value, timeout=DEFAULT_TIMEOUT, version=None):
  88. key = self.make_key(key, version=version)
  89. self.validate_key(key)
  90. return self._base_set('add', key, value, timeout)
  91. def touch(self, key, timeout=DEFAULT_TIMEOUT, version=None):
  92. key = self.make_key(key, version=version)
  93. self.validate_key(key)
  94. return self._base_set('touch', key, None, timeout)
  95. def _base_set(self, mode, key, value, timeout=DEFAULT_TIMEOUT):
  96. timeout = self.get_backend_timeout(timeout)
  97. db = router.db_for_write(self.cache_model_class)
  98. connection = connections[db]
  99. quote_name = connection.ops.quote_name
  100. table = quote_name(self._table)
  101. with connection.cursor() as cursor:
  102. cursor.execute("SELECT COUNT(*) FROM %s" % table)
  103. num = cursor.fetchone()[0]
  104. now = timezone.now()
  105. now = now.replace(microsecond=0)
  106. if timeout is None:
  107. exp = datetime.max
  108. elif settings.USE_TZ:
  109. exp = datetime.utcfromtimestamp(timeout)
  110. else:
  111. exp = datetime.fromtimestamp(timeout)
  112. exp = exp.replace(microsecond=0)
  113. if num > self._max_entries:
  114. self._cull(db, cursor, now)
  115. pickled = pickle.dumps(value, self.pickle_protocol)
  116. # The DB column is expecting a string, so make sure the value is a
  117. # string, not bytes. Refs #19274.
  118. b64encoded = base64.b64encode(pickled).decode('latin1')
  119. try:
  120. # Note: typecasting for datetimes is needed by some 3rd party
  121. # database backends. All core backends work without typecasting,
  122. # so be careful about changes here - test suite will NOT pick
  123. # regressions.
  124. with transaction.atomic(using=db):
  125. cursor.execute(
  126. 'SELECT %s, %s FROM %s WHERE %s = %%s' % (
  127. quote_name('cache_key'),
  128. quote_name('expires'),
  129. table,
  130. quote_name('cache_key'),
  131. ),
  132. [key]
  133. )
  134. result = cursor.fetchone()
  135. if result:
  136. current_expires = result[1]
  137. expression = models.Expression(output_field=models.DateTimeField())
  138. for converter in (connection.ops.get_db_converters(expression) +
  139. expression.get_db_converters(connection)):
  140. if func_supports_parameter(converter, 'context'): # RemovedInDjango30Warning
  141. current_expires = converter(current_expires, expression, connection, {})
  142. else:
  143. current_expires = converter(current_expires, expression, connection)
  144. exp = connection.ops.adapt_datetimefield_value(exp)
  145. if result and mode == 'touch':
  146. cursor.execute(
  147. 'UPDATE %s SET %s = %%s WHERE %s = %%s' % (
  148. table,
  149. quote_name('expires'),
  150. quote_name('cache_key')
  151. ),
  152. [exp, key]
  153. )
  154. elif result and (mode == 'set' or (mode == 'add' and current_expires < now)):
  155. cursor.execute(
  156. 'UPDATE %s SET %s = %%s, %s = %%s WHERE %s = %%s' % (
  157. table,
  158. quote_name('value'),
  159. quote_name('expires'),
  160. quote_name('cache_key'),
  161. ),
  162. [b64encoded, exp, key]
  163. )
  164. elif mode != 'touch':
  165. cursor.execute(
  166. 'INSERT INTO %s (%s, %s, %s) VALUES (%%s, %%s, %%s)' % (
  167. table,
  168. quote_name('cache_key'),
  169. quote_name('value'),
  170. quote_name('expires'),
  171. ),
  172. [key, b64encoded, exp]
  173. )
  174. else:
  175. return False # touch failed.
  176. except DatabaseError:
  177. # To be threadsafe, updates/inserts are allowed to fail silently
  178. return False
  179. else:
  180. return True
  181. def delete(self, key, version=None):
  182. self.delete_many([key], version)
  183. def delete_many(self, keys, version=None):
  184. key_list = []
  185. for key in keys:
  186. self.validate_key(key)
  187. key_list.append(self.make_key(key, version))
  188. self._base_delete_many(key_list)
  189. def _base_delete_many(self, keys):
  190. if not keys:
  191. return
  192. db = router.db_for_write(self.cache_model_class)
  193. connection = connections[db]
  194. quote_name = connection.ops.quote_name
  195. table = quote_name(self._table)
  196. with connection.cursor() as cursor:
  197. cursor.execute(
  198. 'DELETE FROM %s WHERE %s IN (%s)' % (
  199. table,
  200. quote_name('cache_key'),
  201. ', '.join(['%s'] * len(keys)),
  202. ),
  203. keys,
  204. )
  205. def has_key(self, key, version=None):
  206. key = self.make_key(key, version=version)
  207. self.validate_key(key)
  208. db = router.db_for_read(self.cache_model_class)
  209. connection = connections[db]
  210. quote_name = connection.ops.quote_name
  211. if settings.USE_TZ:
  212. now = datetime.utcnow()
  213. else:
  214. now = datetime.now()
  215. now = now.replace(microsecond=0)
  216. with connection.cursor() as cursor:
  217. cursor.execute(
  218. 'SELECT %s FROM %s WHERE %s = %%s and expires > %%s' % (
  219. quote_name('cache_key'),
  220. quote_name(self._table),
  221. quote_name('cache_key'),
  222. ),
  223. [key, connection.ops.adapt_datetimefield_value(now)]
  224. )
  225. return cursor.fetchone() is not None
  226. def _cull(self, db, cursor, now):
  227. if self._cull_frequency == 0:
  228. self.clear()
  229. else:
  230. connection = connections[db]
  231. table = connection.ops.quote_name(self._table)
  232. cursor.execute("DELETE FROM %s WHERE expires < %%s" % table,
  233. [connection.ops.adapt_datetimefield_value(now)])
  234. cursor.execute("SELECT COUNT(*) FROM %s" % table)
  235. num = cursor.fetchone()[0]
  236. if num > self._max_entries:
  237. cull_num = num // self._cull_frequency
  238. cursor.execute(
  239. connection.ops.cache_key_culling_sql() % table,
  240. [cull_num])
  241. cursor.execute("DELETE FROM %s "
  242. "WHERE cache_key < %%s" % table,
  243. [cursor.fetchone()[0]])
  244. def clear(self):
  245. db = router.db_for_write(self.cache_model_class)
  246. connection = connections[db]
  247. table = connection.ops.quote_name(self._table)
  248. with connection.cursor() as cursor:
  249. cursor.execute('DELETE FROM %s' % table)