Development of an internal social media platform with personalised dashboards for students
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

base.py 24KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658
  1. import copy
  2. import time
  3. import warnings
  4. from collections import deque
  5. from contextlib import contextmanager
  6. import _thread
  7. import pytz
  8. from django.conf import settings
  9. from django.core.exceptions import ImproperlyConfigured
  10. from django.db import DEFAULT_DB_ALIAS
  11. from django.db.backends import utils
  12. from django.db.backends.base.validation import BaseDatabaseValidation
  13. from django.db.backends.signals import connection_created
  14. from django.db.transaction import TransactionManagementError
  15. from django.db.utils import DatabaseError, DatabaseErrorWrapper
  16. from django.utils import timezone
  17. from django.utils.functional import cached_property
  18. NO_DB_ALIAS = '__no_db__'
  19. class BaseDatabaseWrapper:
  20. """Represent a database connection."""
  21. # Mapping of Field objects to their column types.
  22. data_types = {}
  23. # Mapping of Field objects to their SQL suffix such as AUTOINCREMENT.
  24. data_types_suffix = {}
  25. # Mapping of Field objects to their SQL for CHECK constraints.
  26. data_type_check_constraints = {}
  27. ops = None
  28. vendor = 'unknown'
  29. display_name = 'unknown'
  30. SchemaEditorClass = None
  31. # Classes instantiated in __init__().
  32. client_class = None
  33. creation_class = None
  34. features_class = None
  35. introspection_class = None
  36. ops_class = None
  37. validation_class = BaseDatabaseValidation
  38. queries_limit = 9000
  39. def __init__(self, settings_dict, alias=DEFAULT_DB_ALIAS,
  40. allow_thread_sharing=False):
  41. # Connection related attributes.
  42. # The underlying database connection.
  43. self.connection = None
  44. # `settings_dict` should be a dictionary containing keys such as
  45. # NAME, USER, etc. It's called `settings_dict` instead of `settings`
  46. # to disambiguate it from Django settings modules.
  47. self.settings_dict = settings_dict
  48. self.alias = alias
  49. # Query logging in debug mode or when explicitly enabled.
  50. self.queries_log = deque(maxlen=self.queries_limit)
  51. self.force_debug_cursor = False
  52. # Transaction related attributes.
  53. # Tracks if the connection is in autocommit mode. Per PEP 249, by
  54. # default, it isn't.
  55. self.autocommit = False
  56. # Tracks if the connection is in a transaction managed by 'atomic'.
  57. self.in_atomic_block = False
  58. # Increment to generate unique savepoint ids.
  59. self.savepoint_state = 0
  60. # List of savepoints created by 'atomic'.
  61. self.savepoint_ids = []
  62. # Tracks if the outermost 'atomic' block should commit on exit,
  63. # ie. if autocommit was active on entry.
  64. self.commit_on_exit = True
  65. # Tracks if the transaction should be rolled back to the next
  66. # available savepoint because of an exception in an inner block.
  67. self.needs_rollback = False
  68. # Connection termination related attributes.
  69. self.close_at = None
  70. self.closed_in_transaction = False
  71. self.errors_occurred = False
  72. # Thread-safety related attributes.
  73. self.allow_thread_sharing = allow_thread_sharing
  74. self._thread_ident = _thread.get_ident()
  75. # A list of no-argument functions to run when the transaction commits.
  76. # Each entry is an (sids, func) tuple, where sids is a set of the
  77. # active savepoint IDs when this function was registered.
  78. self.run_on_commit = []
  79. # Should we run the on-commit hooks the next time set_autocommit(True)
  80. # is called?
  81. self.run_commit_hooks_on_set_autocommit_on = False
  82. # A stack of wrappers to be invoked around execute()/executemany()
  83. # calls. Each entry is a function taking five arguments: execute, sql,
  84. # params, many, and context. It's the function's responsibility to
  85. # call execute(sql, params, many, context).
  86. self.execute_wrappers = []
  87. self.client = self.client_class(self)
  88. self.creation = self.creation_class(self)
  89. self.features = self.features_class(self)
  90. self.introspection = self.introspection_class(self)
  91. self.ops = self.ops_class(self)
  92. self.validation = self.validation_class(self)
  93. def ensure_timezone(self):
  94. """
  95. Ensure the connection's timezone is set to `self.timezone_name` and
  96. return whether it changed or not.
  97. """
  98. return False
  99. @cached_property
  100. def timezone(self):
  101. """
  102. Time zone for datetimes stored as naive values in the database.
  103. Return a tzinfo object or None.
  104. This is only needed when time zone support is enabled and the database
  105. doesn't support time zones. (When the database supports time zones,
  106. the adapter handles aware datetimes so Django doesn't need to.)
  107. """
  108. if not settings.USE_TZ:
  109. return None
  110. elif self.features.supports_timezones:
  111. return None
  112. elif self.settings_dict['TIME_ZONE'] is None:
  113. return timezone.utc
  114. else:
  115. return pytz.timezone(self.settings_dict['TIME_ZONE'])
  116. @cached_property
  117. def timezone_name(self):
  118. """
  119. Name of the time zone of the database connection.
  120. """
  121. if not settings.USE_TZ:
  122. return settings.TIME_ZONE
  123. elif self.settings_dict['TIME_ZONE'] is None:
  124. return 'UTC'
  125. else:
  126. return self.settings_dict['TIME_ZONE']
  127. @property
  128. def queries_logged(self):
  129. return self.force_debug_cursor or settings.DEBUG
  130. @property
  131. def queries(self):
  132. if len(self.queries_log) == self.queries_log.maxlen:
  133. warnings.warn(
  134. "Limit for query logging exceeded, only the last {} queries "
  135. "will be returned.".format(self.queries_log.maxlen))
  136. return list(self.queries_log)
  137. # ##### Backend-specific methods for creating connections and cursors #####
  138. def get_connection_params(self):
  139. """Return a dict of parameters suitable for get_new_connection."""
  140. raise NotImplementedError('subclasses of BaseDatabaseWrapper may require a get_connection_params() method')
  141. def get_new_connection(self, conn_params):
  142. """Open a connection to the database."""
  143. raise NotImplementedError('subclasses of BaseDatabaseWrapper may require a get_new_connection() method')
  144. def init_connection_state(self):
  145. """Initialize the database connection settings."""
  146. raise NotImplementedError('subclasses of BaseDatabaseWrapper may require an init_connection_state() method')
  147. def create_cursor(self, name=None):
  148. """Create a cursor. Assume that a connection is established."""
  149. raise NotImplementedError('subclasses of BaseDatabaseWrapper may require a create_cursor() method')
  150. # ##### Backend-specific methods for creating connections #####
  151. def connect(self):
  152. """Connect to the database. Assume that the connection is closed."""
  153. # Check for invalid configurations.
  154. self.check_settings()
  155. # In case the previous connection was closed while in an atomic block
  156. self.in_atomic_block = False
  157. self.savepoint_ids = []
  158. self.needs_rollback = False
  159. # Reset parameters defining when to close the connection
  160. max_age = self.settings_dict['CONN_MAX_AGE']
  161. self.close_at = None if max_age is None else time.time() + max_age
  162. self.closed_in_transaction = False
  163. self.errors_occurred = False
  164. # Establish the connection
  165. conn_params = self.get_connection_params()
  166. self.connection = self.get_new_connection(conn_params)
  167. self.set_autocommit(self.settings_dict['AUTOCOMMIT'])
  168. self.init_connection_state()
  169. connection_created.send(sender=self.__class__, connection=self)
  170. self.run_on_commit = []
  171. def check_settings(self):
  172. if self.settings_dict['TIME_ZONE'] is not None:
  173. if not settings.USE_TZ:
  174. raise ImproperlyConfigured(
  175. "Connection '%s' cannot set TIME_ZONE because USE_TZ is "
  176. "False." % self.alias)
  177. elif self.features.supports_timezones:
  178. raise ImproperlyConfigured(
  179. "Connection '%s' cannot set TIME_ZONE because its engine "
  180. "handles time zones conversions natively." % self.alias)
  181. def ensure_connection(self):
  182. """Guarantee that a connection to the database is established."""
  183. if self.connection is None:
  184. with self.wrap_database_errors:
  185. self.connect()
  186. # ##### Backend-specific wrappers for PEP-249 connection methods #####
  187. def _prepare_cursor(self, cursor):
  188. """
  189. Validate the connection is usable and perform database cursor wrapping.
  190. """
  191. self.validate_thread_sharing()
  192. if self.queries_logged:
  193. wrapped_cursor = self.make_debug_cursor(cursor)
  194. else:
  195. wrapped_cursor = self.make_cursor(cursor)
  196. return wrapped_cursor
  197. def _cursor(self, name=None):
  198. self.ensure_connection()
  199. with self.wrap_database_errors:
  200. return self._prepare_cursor(self.create_cursor(name))
  201. def _commit(self):
  202. if self.connection is not None:
  203. with self.wrap_database_errors:
  204. return self.connection.commit()
  205. def _rollback(self):
  206. if self.connection is not None:
  207. with self.wrap_database_errors:
  208. return self.connection.rollback()
  209. def _close(self):
  210. if self.connection is not None:
  211. with self.wrap_database_errors:
  212. return self.connection.close()
  213. # ##### Generic wrappers for PEP-249 connection methods #####
  214. def cursor(self):
  215. """Create a cursor, opening a connection if necessary."""
  216. return self._cursor()
  217. def commit(self):
  218. """Commit a transaction and reset the dirty flag."""
  219. self.validate_thread_sharing()
  220. self.validate_no_atomic_block()
  221. self._commit()
  222. # A successful commit means that the database connection works.
  223. self.errors_occurred = False
  224. self.run_commit_hooks_on_set_autocommit_on = True
  225. def rollback(self):
  226. """Roll back a transaction and reset the dirty flag."""
  227. self.validate_thread_sharing()
  228. self.validate_no_atomic_block()
  229. self._rollback()
  230. # A successful rollback means that the database connection works.
  231. self.errors_occurred = False
  232. self.needs_rollback = False
  233. self.run_on_commit = []
  234. def close(self):
  235. """Close the connection to the database."""
  236. self.validate_thread_sharing()
  237. self.run_on_commit = []
  238. # Don't call validate_no_atomic_block() to avoid making it difficult
  239. # to get rid of a connection in an invalid state. The next connect()
  240. # will reset the transaction state anyway.
  241. if self.closed_in_transaction or self.connection is None:
  242. return
  243. try:
  244. self._close()
  245. finally:
  246. if self.in_atomic_block:
  247. self.closed_in_transaction = True
  248. self.needs_rollback = True
  249. else:
  250. self.connection = None
  251. # ##### Backend-specific savepoint management methods #####
  252. def _savepoint(self, sid):
  253. with self.cursor() as cursor:
  254. cursor.execute(self.ops.savepoint_create_sql(sid))
  255. def _savepoint_rollback(self, sid):
  256. with self.cursor() as cursor:
  257. cursor.execute(self.ops.savepoint_rollback_sql(sid))
  258. def _savepoint_commit(self, sid):
  259. with self.cursor() as cursor:
  260. cursor.execute(self.ops.savepoint_commit_sql(sid))
  261. def _savepoint_allowed(self):
  262. # Savepoints cannot be created outside a transaction
  263. return self.features.uses_savepoints and not self.get_autocommit()
  264. # ##### Generic savepoint management methods #####
  265. def savepoint(self):
  266. """
  267. Create a savepoint inside the current transaction. Return an
  268. identifier for the savepoint that will be used for the subsequent
  269. rollback or commit. Do nothing if savepoints are not supported.
  270. """
  271. if not self._savepoint_allowed():
  272. return
  273. thread_ident = _thread.get_ident()
  274. tid = str(thread_ident).replace('-', '')
  275. self.savepoint_state += 1
  276. sid = "s%s_x%d" % (tid, self.savepoint_state)
  277. self.validate_thread_sharing()
  278. self._savepoint(sid)
  279. return sid
  280. def savepoint_rollback(self, sid):
  281. """
  282. Roll back to a savepoint. Do nothing if savepoints are not supported.
  283. """
  284. if not self._savepoint_allowed():
  285. return
  286. self.validate_thread_sharing()
  287. self._savepoint_rollback(sid)
  288. # Remove any callbacks registered while this savepoint was active.
  289. self.run_on_commit = [
  290. (sids, func) for (sids, func) in self.run_on_commit if sid not in sids
  291. ]
  292. def savepoint_commit(self, sid):
  293. """
  294. Release a savepoint. Do nothing if savepoints are not supported.
  295. """
  296. if not self._savepoint_allowed():
  297. return
  298. self.validate_thread_sharing()
  299. self._savepoint_commit(sid)
  300. def clean_savepoints(self):
  301. """
  302. Reset the counter used to generate unique savepoint ids in this thread.
  303. """
  304. self.savepoint_state = 0
  305. # ##### Backend-specific transaction management methods #####
  306. def _set_autocommit(self, autocommit):
  307. """
  308. Backend-specific implementation to enable or disable autocommit.
  309. """
  310. raise NotImplementedError('subclasses of BaseDatabaseWrapper may require a _set_autocommit() method')
  311. # ##### Generic transaction management methods #####
  312. def get_autocommit(self):
  313. """Get the autocommit state."""
  314. self.ensure_connection()
  315. return self.autocommit
  316. def set_autocommit(self, autocommit, force_begin_transaction_with_broken_autocommit=False):
  317. """
  318. Enable or disable autocommit.
  319. The usual way to start a transaction is to turn autocommit off.
  320. SQLite does not properly start a transaction when disabling
  321. autocommit. To avoid this buggy behavior and to actually enter a new
  322. transaction, an explcit BEGIN is required. Using
  323. force_begin_transaction_with_broken_autocommit=True will issue an
  324. explicit BEGIN with SQLite. This option will be ignored for other
  325. backends.
  326. """
  327. self.validate_no_atomic_block()
  328. self.ensure_connection()
  329. start_transaction_under_autocommit = (
  330. force_begin_transaction_with_broken_autocommit and not autocommit and
  331. self.features.autocommits_when_autocommit_is_off
  332. )
  333. if start_transaction_under_autocommit:
  334. self._start_transaction_under_autocommit()
  335. else:
  336. self._set_autocommit(autocommit)
  337. self.autocommit = autocommit
  338. if autocommit and self.run_commit_hooks_on_set_autocommit_on:
  339. self.run_and_clear_commit_hooks()
  340. self.run_commit_hooks_on_set_autocommit_on = False
  341. def get_rollback(self):
  342. """Get the "needs rollback" flag -- for *advanced use* only."""
  343. if not self.in_atomic_block:
  344. raise TransactionManagementError(
  345. "The rollback flag doesn't work outside of an 'atomic' block.")
  346. return self.needs_rollback
  347. def set_rollback(self, rollback):
  348. """
  349. Set or unset the "needs rollback" flag -- for *advanced use* only.
  350. """
  351. if not self.in_atomic_block:
  352. raise TransactionManagementError(
  353. "The rollback flag doesn't work outside of an 'atomic' block.")
  354. self.needs_rollback = rollback
  355. def validate_no_atomic_block(self):
  356. """Raise an error if an atomic block is active."""
  357. if self.in_atomic_block:
  358. raise TransactionManagementError(
  359. "This is forbidden when an 'atomic' block is active.")
  360. def validate_no_broken_transaction(self):
  361. if self.needs_rollback:
  362. raise TransactionManagementError(
  363. "An error occurred in the current transaction. You can't "
  364. "execute queries until the end of the 'atomic' block.")
  365. # ##### Foreign key constraints checks handling #####
  366. @contextmanager
  367. def constraint_checks_disabled(self):
  368. """
  369. Disable foreign key constraint checking.
  370. """
  371. disabled = self.disable_constraint_checking()
  372. try:
  373. yield
  374. finally:
  375. if disabled:
  376. self.enable_constraint_checking()
  377. def disable_constraint_checking(self):
  378. """
  379. Backends can implement as needed to temporarily disable foreign key
  380. constraint checking. Should return True if the constraints were
  381. disabled and will need to be reenabled.
  382. """
  383. return False
  384. def enable_constraint_checking(self):
  385. """
  386. Backends can implement as needed to re-enable foreign key constraint
  387. checking.
  388. """
  389. pass
  390. def check_constraints(self, table_names=None):
  391. """
  392. Backends can override this method if they can apply constraint
  393. checking (e.g. via "SET CONSTRAINTS ALL IMMEDIATE"). Should raise an
  394. IntegrityError if any invalid foreign key references are encountered.
  395. """
  396. pass
  397. # ##### Connection termination handling #####
  398. def is_usable(self):
  399. """
  400. Test if the database connection is usable.
  401. This method may assume that self.connection is not None.
  402. Actual implementations should take care not to raise exceptions
  403. as that may prevent Django from recycling unusable connections.
  404. """
  405. raise NotImplementedError(
  406. "subclasses of BaseDatabaseWrapper may require an is_usable() method")
  407. def close_if_unusable_or_obsolete(self):
  408. """
  409. Close the current connection if unrecoverable errors have occurred
  410. or if it outlived its maximum age.
  411. """
  412. if self.connection is not None:
  413. # If the application didn't restore the original autocommit setting,
  414. # don't take chances, drop the connection.
  415. if self.get_autocommit() != self.settings_dict['AUTOCOMMIT']:
  416. self.close()
  417. return
  418. # If an exception other than DataError or IntegrityError occurred
  419. # since the last commit / rollback, check if the connection works.
  420. if self.errors_occurred:
  421. if self.is_usable():
  422. self.errors_occurred = False
  423. else:
  424. self.close()
  425. return
  426. if self.close_at is not None and time.time() >= self.close_at:
  427. self.close()
  428. return
  429. # ##### Thread safety handling #####
  430. def validate_thread_sharing(self):
  431. """
  432. Validate that the connection isn't accessed by another thread than the
  433. one which originally created it, unless the connection was explicitly
  434. authorized to be shared between threads (via the `allow_thread_sharing`
  435. property). Raise an exception if the validation fails.
  436. """
  437. if not (self.allow_thread_sharing or self._thread_ident == _thread.get_ident()):
  438. raise DatabaseError(
  439. "DatabaseWrapper objects created in a "
  440. "thread can only be used in that same thread. The object "
  441. "with alias '%s' was created in thread id %s and this is "
  442. "thread id %s."
  443. % (self.alias, self._thread_ident, _thread.get_ident())
  444. )
  445. # ##### Miscellaneous #####
  446. def prepare_database(self):
  447. """
  448. Hook to do any database check or preparation, generally called before
  449. migrating a project or an app.
  450. """
  451. pass
  452. @cached_property
  453. def wrap_database_errors(self):
  454. """
  455. Context manager and decorator that re-throws backend-specific database
  456. exceptions using Django's common wrappers.
  457. """
  458. return DatabaseErrorWrapper(self)
  459. def chunked_cursor(self):
  460. """
  461. Return a cursor that tries to avoid caching in the database (if
  462. supported by the database), otherwise return a regular cursor.
  463. """
  464. return self.cursor()
  465. def make_debug_cursor(self, cursor):
  466. """Create a cursor that logs all queries in self.queries_log."""
  467. return utils.CursorDebugWrapper(cursor, self)
  468. def make_cursor(self, cursor):
  469. """Create a cursor without debug logging."""
  470. return utils.CursorWrapper(cursor, self)
  471. @contextmanager
  472. def temporary_connection(self):
  473. """
  474. Context manager that ensures that a connection is established, and
  475. if it opened one, closes it to avoid leaving a dangling connection.
  476. This is useful for operations outside of the request-response cycle.
  477. Provide a cursor: with self.temporary_connection() as cursor: ...
  478. """
  479. must_close = self.connection is None
  480. try:
  481. with self.cursor() as cursor:
  482. yield cursor
  483. finally:
  484. if must_close:
  485. self.close()
  486. @property
  487. def _nodb_connection(self):
  488. """
  489. Return an alternative connection to be used when there is no need to
  490. access the main database, specifically for test db creation/deletion.
  491. This also prevents the production database from being exposed to
  492. potential child threads while (or after) the test database is destroyed.
  493. Refs #10868, #17786, #16969.
  494. """
  495. return self.__class__(
  496. {**self.settings_dict, 'NAME': None},
  497. alias=NO_DB_ALIAS,
  498. allow_thread_sharing=False,
  499. )
  500. def _start_transaction_under_autocommit(self):
  501. """
  502. Only required when autocommits_when_autocommit_is_off = True.
  503. """
  504. raise NotImplementedError(
  505. 'subclasses of BaseDatabaseWrapper may require a '
  506. '_start_transaction_under_autocommit() method'
  507. )
  508. def schema_editor(self, *args, **kwargs):
  509. """
  510. Return a new instance of this backend's SchemaEditor.
  511. """
  512. if self.SchemaEditorClass is None:
  513. raise NotImplementedError(
  514. 'The SchemaEditorClass attribute of this database wrapper is still None')
  515. return self.SchemaEditorClass(self, *args, **kwargs)
  516. def on_commit(self, func):
  517. if self.in_atomic_block:
  518. # Transaction in progress; save for execution on commit.
  519. self.run_on_commit.append((set(self.savepoint_ids), func))
  520. elif not self.get_autocommit():
  521. raise TransactionManagementError('on_commit() cannot be used in manual transaction management')
  522. else:
  523. # No transaction in progress and in autocommit mode; execute
  524. # immediately.
  525. func()
  526. def run_and_clear_commit_hooks(self):
  527. self.validate_no_atomic_block()
  528. current_run_on_commit = self.run_on_commit
  529. self.run_on_commit = []
  530. while current_run_on_commit:
  531. sids, func = current_run_on_commit.pop(0)
  532. func()
  533. @contextmanager
  534. def execute_wrapper(self, wrapper):
  535. """
  536. Return a context manager under which the wrapper is applied to suitable
  537. database query executions.
  538. """
  539. self.execute_wrappers.append(wrapper)
  540. try:
  541. yield
  542. finally:
  543. self.execute_wrappers.pop()
  544. def copy(self, alias=None, allow_thread_sharing=None):
  545. """
  546. Return a copy of this connection.
  547. For tests that require two connections to the same database.
  548. """
  549. settings_dict = copy.deepcopy(self.settings_dict)
  550. if alias is None:
  551. alias = self.alias
  552. if allow_thread_sharing is None:
  553. allow_thread_sharing = self.allow_thread_sharing
  554. return type(self)(settings_dict, alias, allow_thread_sharing)