You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

schema.py 19KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411
  1. import copy
  2. from decimal import Decimal
  3. from django.apps.registry import Apps
  4. from django.db.backends.base.schema import BaseDatabaseSchemaEditor
  5. from django.db.backends.ddl_references import Statement
  6. from django.db.models import UniqueConstraint
  7. from django.db.transaction import atomic
  8. from django.db.utils import NotSupportedError
  9. class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):
  10. sql_delete_table = "DROP TABLE %(table)s"
  11. sql_create_fk = None
  12. sql_create_inline_fk = "REFERENCES %(to_table)s (%(to_column)s) DEFERRABLE INITIALLY DEFERRED"
  13. sql_create_unique = "CREATE UNIQUE INDEX %(name)s ON %(table)s (%(columns)s)"
  14. sql_delete_unique = "DROP INDEX %(name)s"
  15. def __enter__(self):
  16. # Some SQLite schema alterations need foreign key constraints to be
  17. # disabled. Enforce it here for the duration of the schema edition.
  18. if not self.connection.disable_constraint_checking():
  19. raise NotSupportedError(
  20. 'SQLite schema editor cannot be used while foreign key '
  21. 'constraint checks are enabled. Make sure to disable them '
  22. 'before entering a transaction.atomic() context because '
  23. 'SQLite does not support disabling them in the middle of '
  24. 'a multi-statement transaction.'
  25. )
  26. return super().__enter__()
  27. def __exit__(self, exc_type, exc_value, traceback):
  28. self.connection.check_constraints()
  29. super().__exit__(exc_type, exc_value, traceback)
  30. self.connection.enable_constraint_checking()
  31. def quote_value(self, value):
  32. # The backend "mostly works" without this function and there are use
  33. # cases for compiling Python without the sqlite3 libraries (e.g.
  34. # security hardening).
  35. try:
  36. import sqlite3
  37. value = sqlite3.adapt(value)
  38. except ImportError:
  39. pass
  40. except sqlite3.ProgrammingError:
  41. pass
  42. # Manual emulation of SQLite parameter quoting
  43. if isinstance(value, bool):
  44. return str(int(value))
  45. elif isinstance(value, (Decimal, float, int)):
  46. return str(value)
  47. elif isinstance(value, str):
  48. return "'%s'" % value.replace("\'", "\'\'")
  49. elif value is None:
  50. return "NULL"
  51. elif isinstance(value, (bytes, bytearray, memoryview)):
  52. # Bytes are only allowed for BLOB fields, encoded as string
  53. # literals containing hexadecimal data and preceded by a single "X"
  54. # character.
  55. return "X'%s'" % value.hex()
  56. else:
  57. raise ValueError("Cannot quote parameter value %r of type %s" % (value, type(value)))
  58. def _is_referenced_by_fk_constraint(self, table_name, column_name=None, ignore_self=False):
  59. """
  60. Return whether or not the provided table name is referenced by another
  61. one. If `column_name` is specified, only references pointing to that
  62. column are considered. If `ignore_self` is True, self-referential
  63. constraints are ignored.
  64. """
  65. with self.connection.cursor() as cursor:
  66. for other_table in self.connection.introspection.get_table_list(cursor):
  67. if ignore_self and other_table.name == table_name:
  68. continue
  69. constraints = self.connection.introspection._get_foreign_key_constraints(cursor, other_table.name)
  70. for constraint in constraints.values():
  71. constraint_table, constraint_column = constraint['foreign_key']
  72. if (constraint_table == table_name and
  73. (column_name is None or constraint_column == column_name)):
  74. return True
  75. return False
  76. def alter_db_table(self, model, old_db_table, new_db_table, disable_constraints=True):
  77. if (not self.connection.features.supports_atomic_references_rename and
  78. disable_constraints and self._is_referenced_by_fk_constraint(old_db_table)):
  79. if self.connection.in_atomic_block:
  80. raise NotSupportedError((
  81. 'Renaming the %r table while in a transaction is not '
  82. 'supported on SQLite < 3.26 because it would break referential '
  83. 'integrity. Try adding `atomic = False` to the Migration class.'
  84. ) % old_db_table)
  85. self.connection.enable_constraint_checking()
  86. super().alter_db_table(model, old_db_table, new_db_table)
  87. self.connection.disable_constraint_checking()
  88. else:
  89. super().alter_db_table(model, old_db_table, new_db_table)
  90. def alter_field(self, model, old_field, new_field, strict=False):
  91. old_field_name = old_field.name
  92. table_name = model._meta.db_table
  93. _, old_column_name = old_field.get_attname_column()
  94. if (new_field.name != old_field_name and
  95. not self.connection.features.supports_atomic_references_rename and
  96. self._is_referenced_by_fk_constraint(table_name, old_column_name, ignore_self=True)):
  97. if self.connection.in_atomic_block:
  98. raise NotSupportedError((
  99. 'Renaming the %r.%r column while in a transaction is not '
  100. 'supported on SQLite < 3.26 because it would break referential '
  101. 'integrity. Try adding `atomic = False` to the Migration class.'
  102. ) % (model._meta.db_table, old_field_name))
  103. with atomic(self.connection.alias):
  104. super().alter_field(model, old_field, new_field, strict=strict)
  105. # Follow SQLite's documented procedure for performing changes
  106. # that don't affect the on-disk content.
  107. # https://sqlite.org/lang_altertable.html#otheralter
  108. with self.connection.cursor() as cursor:
  109. schema_version = cursor.execute('PRAGMA schema_version').fetchone()[0]
  110. cursor.execute('PRAGMA writable_schema = 1')
  111. references_template = ' REFERENCES "%s" ("%%s") ' % table_name
  112. new_column_name = new_field.get_attname_column()[1]
  113. search = references_template % old_column_name
  114. replacement = references_template % new_column_name
  115. cursor.execute('UPDATE sqlite_master SET sql = replace(sql, %s, %s)', (search, replacement))
  116. cursor.execute('PRAGMA schema_version = %d' % (schema_version + 1))
  117. cursor.execute('PRAGMA writable_schema = 0')
  118. # The integrity check will raise an exception and rollback
  119. # the transaction if the sqlite_master updates corrupt the
  120. # database.
  121. cursor.execute('PRAGMA integrity_check')
  122. # Perform a VACUUM to refresh the database representation from
  123. # the sqlite_master table.
  124. with self.connection.cursor() as cursor:
  125. cursor.execute('VACUUM')
  126. else:
  127. super().alter_field(model, old_field, new_field, strict=strict)
  128. def _remake_table(self, model, create_field=None, delete_field=None, alter_field=None):
  129. """
  130. Shortcut to transform a model from old_model into new_model
  131. This follows the correct procedure to perform non-rename or column
  132. addition operations based on SQLite's documentation
  133. https://www.sqlite.org/lang_altertable.html#caution
  134. The essential steps are:
  135. 1. Create a table with the updated definition called "new__app_model"
  136. 2. Copy the data from the existing "app_model" table to the new table
  137. 3. Drop the "app_model" table
  138. 4. Rename the "new__app_model" table to "app_model"
  139. 5. Restore any index of the previous "app_model" table.
  140. """
  141. # Self-referential fields must be recreated rather than copied from
  142. # the old model to ensure their remote_field.field_name doesn't refer
  143. # to an altered field.
  144. def is_self_referential(f):
  145. return f.is_relation and f.remote_field.model is model
  146. # Work out the new fields dict / mapping
  147. body = {
  148. f.name: f.clone() if is_self_referential(f) else f
  149. for f in model._meta.local_concrete_fields
  150. }
  151. # Since mapping might mix column names and default values,
  152. # its values must be already quoted.
  153. mapping = {f.column: self.quote_name(f.column) for f in model._meta.local_concrete_fields}
  154. # This maps field names (not columns) for things like unique_together
  155. rename_mapping = {}
  156. # If any of the new or altered fields is introducing a new PK,
  157. # remove the old one
  158. restore_pk_field = None
  159. if getattr(create_field, 'primary_key', False) or (
  160. alter_field and getattr(alter_field[1], 'primary_key', False)):
  161. for name, field in list(body.items()):
  162. if field.primary_key:
  163. field.primary_key = False
  164. restore_pk_field = field
  165. if field.auto_created:
  166. del body[name]
  167. del mapping[field.column]
  168. # Add in any created fields
  169. if create_field:
  170. body[create_field.name] = create_field
  171. # Choose a default and insert it into the copy map
  172. if not create_field.many_to_many and create_field.concrete:
  173. mapping[create_field.column] = self.quote_value(
  174. self.effective_default(create_field)
  175. )
  176. # Add in any altered fields
  177. if alter_field:
  178. old_field, new_field = alter_field
  179. body.pop(old_field.name, None)
  180. mapping.pop(old_field.column, None)
  181. body[new_field.name] = new_field
  182. if old_field.null and not new_field.null:
  183. case_sql = "coalesce(%(col)s, %(default)s)" % {
  184. 'col': self.quote_name(old_field.column),
  185. 'default': self.quote_value(self.effective_default(new_field))
  186. }
  187. mapping[new_field.column] = case_sql
  188. else:
  189. mapping[new_field.column] = self.quote_name(old_field.column)
  190. rename_mapping[old_field.name] = new_field.name
  191. # Remove any deleted fields
  192. if delete_field:
  193. del body[delete_field.name]
  194. del mapping[delete_field.column]
  195. # Remove any implicit M2M tables
  196. if delete_field.many_to_many and delete_field.remote_field.through._meta.auto_created:
  197. return self.delete_model(delete_field.remote_field.through)
  198. # Work inside a new app registry
  199. apps = Apps()
  200. # Work out the new value of unique_together, taking renames into
  201. # account
  202. unique_together = [
  203. [rename_mapping.get(n, n) for n in unique]
  204. for unique in model._meta.unique_together
  205. ]
  206. # Work out the new value for index_together, taking renames into
  207. # account
  208. index_together = [
  209. [rename_mapping.get(n, n) for n in index]
  210. for index in model._meta.index_together
  211. ]
  212. indexes = model._meta.indexes
  213. if delete_field:
  214. indexes = [
  215. index for index in indexes
  216. if delete_field.name not in index.fields
  217. ]
  218. constraints = list(model._meta.constraints)
  219. # Provide isolated instances of the fields to the new model body so
  220. # that the existing model's internals aren't interfered with when
  221. # the dummy model is constructed.
  222. body_copy = copy.deepcopy(body)
  223. # Construct a new model with the new fields to allow self referential
  224. # primary key to resolve to. This model won't ever be materialized as a
  225. # table and solely exists for foreign key reference resolution purposes.
  226. # This wouldn't be required if the schema editor was operating on model
  227. # states instead of rendered models.
  228. meta_contents = {
  229. 'app_label': model._meta.app_label,
  230. 'db_table': model._meta.db_table,
  231. 'unique_together': unique_together,
  232. 'index_together': index_together,
  233. 'indexes': indexes,
  234. 'constraints': constraints,
  235. 'apps': apps,
  236. }
  237. meta = type("Meta", (), meta_contents)
  238. body_copy['Meta'] = meta
  239. body_copy['__module__'] = model.__module__
  240. type(model._meta.object_name, model.__bases__, body_copy)
  241. # Construct a model with a renamed table name.
  242. body_copy = copy.deepcopy(body)
  243. meta_contents = {
  244. 'app_label': model._meta.app_label,
  245. 'db_table': 'new__%s' % model._meta.db_table,
  246. 'unique_together': unique_together,
  247. 'index_together': index_together,
  248. 'indexes': indexes,
  249. 'constraints': constraints,
  250. 'apps': apps,
  251. }
  252. meta = type("Meta", (), meta_contents)
  253. body_copy['Meta'] = meta
  254. body_copy['__module__'] = model.__module__
  255. new_model = type('New%s' % model._meta.object_name, model.__bases__, body_copy)
  256. # Create a new table with the updated schema.
  257. self.create_model(new_model)
  258. # Copy data from the old table into the new table
  259. self.execute("INSERT INTO %s (%s) SELECT %s FROM %s" % (
  260. self.quote_name(new_model._meta.db_table),
  261. ', '.join(self.quote_name(x) for x in mapping),
  262. ', '.join(mapping.values()),
  263. self.quote_name(model._meta.db_table),
  264. ))
  265. # Delete the old table to make way for the new
  266. self.delete_model(model, handle_autom2m=False)
  267. # Rename the new table to take way for the old
  268. self.alter_db_table(
  269. new_model, new_model._meta.db_table, model._meta.db_table,
  270. disable_constraints=False,
  271. )
  272. # Run deferred SQL on correct table
  273. for sql in self.deferred_sql:
  274. self.execute(sql)
  275. self.deferred_sql = []
  276. # Fix any PK-removed field
  277. if restore_pk_field:
  278. restore_pk_field.primary_key = True
  279. def delete_model(self, model, handle_autom2m=True):
  280. if handle_autom2m:
  281. super().delete_model(model)
  282. else:
  283. # Delete the table (and only that)
  284. self.execute(self.sql_delete_table % {
  285. "table": self.quote_name(model._meta.db_table),
  286. })
  287. # Remove all deferred statements referencing the deleted table.
  288. for sql in list(self.deferred_sql):
  289. if isinstance(sql, Statement) and sql.references_table(model._meta.db_table):
  290. self.deferred_sql.remove(sql)
  291. def add_field(self, model, field):
  292. """
  293. Create a field on a model. Usually involves adding a column, but may
  294. involve adding a table instead (for M2M fields).
  295. """
  296. # Special-case implicit M2M tables
  297. if field.many_to_many and field.remote_field.through._meta.auto_created:
  298. return self.create_model(field.remote_field.through)
  299. self._remake_table(model, create_field=field)
  300. def remove_field(self, model, field):
  301. """
  302. Remove a field from a model. Usually involves deleting a column,
  303. but for M2Ms may involve deleting a table.
  304. """
  305. # M2M fields are a special case
  306. if field.many_to_many:
  307. # For implicit M2M tables, delete the auto-created table
  308. if field.remote_field.through._meta.auto_created:
  309. self.delete_model(field.remote_field.through)
  310. # For explicit "through" M2M fields, do nothing
  311. # For everything else, remake.
  312. else:
  313. # It might not actually have a column behind it
  314. if field.db_parameters(connection=self.connection)['type'] is None:
  315. return
  316. self._remake_table(model, delete_field=field)
  317. def _alter_field(self, model, old_field, new_field, old_type, new_type,
  318. old_db_params, new_db_params, strict=False):
  319. """Perform a "physical" (non-ManyToMany) field update."""
  320. # Use "ALTER TABLE ... RENAME COLUMN" if only the column name
  321. # changed and there aren't any constraints.
  322. if (self.connection.features.can_alter_table_rename_column and
  323. old_field.column != new_field.column and
  324. self.column_sql(model, old_field) == self.column_sql(model, new_field) and
  325. not (old_field.remote_field and old_field.db_constraint or
  326. new_field.remote_field and new_field.db_constraint)):
  327. return self.execute(self._rename_field_sql(model._meta.db_table, old_field, new_field, new_type))
  328. # Alter by remaking table
  329. self._remake_table(model, alter_field=(old_field, new_field))
  330. # Rebuild tables with FKs pointing to this field if the PK type changed.
  331. if old_field.primary_key and new_field.primary_key and old_type != new_type:
  332. for rel in new_field.model._meta.related_objects:
  333. if not rel.many_to_many:
  334. self._remake_table(rel.related_model)
  335. def _alter_many_to_many(self, model, old_field, new_field, strict):
  336. """Alter M2Ms to repoint their to= endpoints."""
  337. if old_field.remote_field.through._meta.db_table == new_field.remote_field.through._meta.db_table:
  338. # The field name didn't change, but some options did; we have to propagate this altering.
  339. self._remake_table(
  340. old_field.remote_field.through,
  341. alter_field=(
  342. # We need the field that points to the target model, so we can tell alter_field to change it -
  343. # this is m2m_reverse_field_name() (as opposed to m2m_field_name, which points to our model)
  344. old_field.remote_field.through._meta.get_field(old_field.m2m_reverse_field_name()),
  345. new_field.remote_field.through._meta.get_field(new_field.m2m_reverse_field_name()),
  346. ),
  347. )
  348. return
  349. # Make a new through table
  350. self.create_model(new_field.remote_field.through)
  351. # Copy the data across
  352. self.execute("INSERT INTO %s (%s) SELECT %s FROM %s" % (
  353. self.quote_name(new_field.remote_field.through._meta.db_table),
  354. ', '.join([
  355. "id",
  356. new_field.m2m_column_name(),
  357. new_field.m2m_reverse_name(),
  358. ]),
  359. ', '.join([
  360. "id",
  361. old_field.m2m_column_name(),
  362. old_field.m2m_reverse_name(),
  363. ]),
  364. self.quote_name(old_field.remote_field.through._meta.db_table),
  365. ))
  366. # Delete the old through table
  367. self.delete_model(old_field.remote_field.through)
  368. def add_constraint(self, model, constraint):
  369. if isinstance(constraint, UniqueConstraint) and constraint.condition:
  370. super().add_constraint(model, constraint)
  371. else:
  372. self._remake_table(model)
  373. def remove_constraint(self, model, constraint):
  374. if isinstance(constraint, UniqueConstraint) and constraint.condition:
  375. super().remove_constraint(model, constraint)
  376. else:
  377. self._remake_table(model)