You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

5 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612
  1. import copy
  2. from collections import OrderedDict
  3. from contextlib import contextmanager
  4. from django.apps import AppConfig
  5. from django.apps.registry import Apps, apps as global_apps
  6. from django.conf import settings
  7. from django.db import models
  8. from django.db.models.fields.proxy import OrderWrt
  9. from django.db.models.fields.related import RECURSIVE_RELATIONSHIP_CONSTANT
  10. from django.db.models.options import DEFAULT_NAMES, normalize_together
  11. from django.db.models.utils import make_model_tuple
  12. from django.utils.functional import cached_property
  13. from django.utils.module_loading import import_string
  14. from django.utils.version import get_docs_version
  15. from .exceptions import InvalidBasesError
  16. def _get_app_label_and_model_name(model, app_label=''):
  17. if isinstance(model, str):
  18. split = model.split('.', 1)
  19. return tuple(split) if len(split) == 2 else (app_label, split[0])
  20. else:
  21. return model._meta.app_label, model._meta.model_name
  22. def _get_related_models(m):
  23. """Return all models that have a direct relationship to the given model."""
  24. related_models = [
  25. subclass for subclass in m.__subclasses__()
  26. if issubclass(subclass, models.Model)
  27. ]
  28. related_fields_models = set()
  29. for f in m._meta.get_fields(include_parents=True, include_hidden=True):
  30. if f.is_relation and f.related_model is not None and not isinstance(f.related_model, str):
  31. related_fields_models.add(f.model)
  32. related_models.append(f.related_model)
  33. # Reverse accessors of foreign keys to proxy models are attached to their
  34. # concrete proxied model.
  35. opts = m._meta
  36. if opts.proxy and m in related_fields_models:
  37. related_models.append(opts.concrete_model)
  38. return related_models
  39. def get_related_models_tuples(model):
  40. """
  41. Return a list of typical (app_label, model_name) tuples for all related
  42. models for the given model.
  43. """
  44. return {
  45. (rel_mod._meta.app_label, rel_mod._meta.model_name)
  46. for rel_mod in _get_related_models(model)
  47. }
  48. def get_related_models_recursive(model):
  49. """
  50. Return all models that have a direct or indirect relationship
  51. to the given model.
  52. Relationships are either defined by explicit relational fields, like
  53. ForeignKey, ManyToManyField or OneToOneField, or by inheriting from another
  54. model (a superclass is related to its subclasses, but not vice versa). Note,
  55. however, that a model inheriting from a concrete model is also related to
  56. its superclass through the implicit *_ptr OneToOneField on the subclass.
  57. """
  58. seen = set()
  59. queue = _get_related_models(model)
  60. for rel_mod in queue:
  61. rel_app_label, rel_model_name = rel_mod._meta.app_label, rel_mod._meta.model_name
  62. if (rel_app_label, rel_model_name) in seen:
  63. continue
  64. seen.add((rel_app_label, rel_model_name))
  65. queue.extend(_get_related_models(rel_mod))
  66. return seen - {(model._meta.app_label, model._meta.model_name)}
  67. class ProjectState:
  68. """
  69. Represent the entire project's overall state. This is the item that is
  70. passed around - do it here rather than at the app level so that cross-app
  71. FKs/etc. resolve properly.
  72. """
  73. def __init__(self, models=None, real_apps=None):
  74. self.models = models or {}
  75. # Apps to include from main registry, usually unmigrated ones
  76. self.real_apps = real_apps or []
  77. self.is_delayed = False
  78. def add_model(self, model_state):
  79. app_label, model_name = model_state.app_label, model_state.name_lower
  80. self.models[(app_label, model_name)] = model_state
  81. if 'apps' in self.__dict__: # hasattr would cache the property
  82. self.reload_model(app_label, model_name)
  83. def remove_model(self, app_label, model_name):
  84. del self.models[app_label, model_name]
  85. if 'apps' in self.__dict__: # hasattr would cache the property
  86. self.apps.unregister_model(app_label, model_name)
  87. # Need to do this explicitly since unregister_model() doesn't clear
  88. # the cache automatically (#24513)
  89. self.apps.clear_cache()
  90. def _find_reload_model(self, app_label, model_name, delay=False):
  91. if delay:
  92. self.is_delayed = True
  93. related_models = set()
  94. try:
  95. old_model = self.apps.get_model(app_label, model_name)
  96. except LookupError:
  97. pass
  98. else:
  99. # Get all relations to and from the old model before reloading,
  100. # as _meta.apps may change
  101. if delay:
  102. related_models = get_related_models_tuples(old_model)
  103. else:
  104. related_models = get_related_models_recursive(old_model)
  105. # Get all outgoing references from the model to be rendered
  106. model_state = self.models[(app_label, model_name)]
  107. # Directly related models are the models pointed to by ForeignKeys,
  108. # OneToOneFields, and ManyToManyFields.
  109. direct_related_models = set()
  110. for name, field in model_state.fields:
  111. if field.is_relation:
  112. if field.remote_field.model == RECURSIVE_RELATIONSHIP_CONSTANT:
  113. continue
  114. rel_app_label, rel_model_name = _get_app_label_and_model_name(field.related_model, app_label)
  115. direct_related_models.add((rel_app_label, rel_model_name.lower()))
  116. # For all direct related models recursively get all related models.
  117. related_models.update(direct_related_models)
  118. for rel_app_label, rel_model_name in direct_related_models:
  119. try:
  120. rel_model = self.apps.get_model(rel_app_label, rel_model_name)
  121. except LookupError:
  122. pass
  123. else:
  124. if delay:
  125. related_models.update(get_related_models_tuples(rel_model))
  126. else:
  127. related_models.update(get_related_models_recursive(rel_model))
  128. # Include the model itself
  129. related_models.add((app_label, model_name))
  130. return related_models
  131. def reload_model(self, app_label, model_name, delay=False):
  132. if 'apps' in self.__dict__: # hasattr would cache the property
  133. related_models = self._find_reload_model(app_label, model_name, delay)
  134. self._reload(related_models)
  135. def reload_models(self, models, delay=True):
  136. if 'apps' in self.__dict__: # hasattr would cache the property
  137. related_models = set()
  138. for app_label, model_name in models:
  139. related_models.update(self._find_reload_model(app_label, model_name, delay))
  140. self._reload(related_models)
  141. def _reload(self, related_models):
  142. # Unregister all related models
  143. with self.apps.bulk_update():
  144. for rel_app_label, rel_model_name in related_models:
  145. self.apps.unregister_model(rel_app_label, rel_model_name)
  146. states_to_be_rendered = []
  147. # Gather all models states of those models that will be rerendered.
  148. # This includes:
  149. # 1. All related models of unmigrated apps
  150. for model_state in self.apps.real_models:
  151. if (model_state.app_label, model_state.name_lower) in related_models:
  152. states_to_be_rendered.append(model_state)
  153. # 2. All related models of migrated apps
  154. for rel_app_label, rel_model_name in related_models:
  155. try:
  156. model_state = self.models[rel_app_label, rel_model_name]
  157. except KeyError:
  158. pass
  159. else:
  160. states_to_be_rendered.append(model_state)
  161. # Render all models
  162. self.apps.render_multiple(states_to_be_rendered)
  163. def clone(self):
  164. """Return an exact copy of this ProjectState."""
  165. new_state = ProjectState(
  166. models={k: v.clone() for k, v in self.models.items()},
  167. real_apps=self.real_apps,
  168. )
  169. if 'apps' in self.__dict__:
  170. new_state.apps = self.apps.clone()
  171. new_state.is_delayed = self.is_delayed
  172. return new_state
  173. def clear_delayed_apps_cache(self):
  174. if self.is_delayed and 'apps' in self.__dict__:
  175. del self.__dict__['apps']
  176. @cached_property
  177. def apps(self):
  178. return StateApps(self.real_apps, self.models)
  179. @property
  180. def concrete_apps(self):
  181. self.apps = StateApps(self.real_apps, self.models, ignore_swappable=True)
  182. return self.apps
  183. @classmethod
  184. def from_apps(cls, apps):
  185. """Take an Apps and return a ProjectState matching it."""
  186. app_models = {}
  187. for model in apps.get_models(include_swapped=True):
  188. model_state = ModelState.from_model(model)
  189. app_models[(model_state.app_label, model_state.name_lower)] = model_state
  190. return cls(app_models)
  191. def __eq__(self, other):
  192. return self.models == other.models and set(self.real_apps) == set(other.real_apps)
  193. class AppConfigStub(AppConfig):
  194. """Stub of an AppConfig. Only provides a label and a dict of models."""
  195. # Not used, but required by AppConfig.__init__
  196. path = ''
  197. def __init__(self, label):
  198. self.label = label
  199. # App-label and app-name are not the same thing, so technically passing
  200. # in the label here is wrong. In practice, migrations don't care about
  201. # the app name, but we need something unique, and the label works fine.
  202. super().__init__(label, None)
  203. def import_models(self):
  204. self.models = self.apps.all_models[self.label]
  205. class StateApps(Apps):
  206. """
  207. Subclass of the global Apps registry class to better handle dynamic model
  208. additions and removals.
  209. """
  210. def __init__(self, real_apps, models, ignore_swappable=False):
  211. # Any apps in self.real_apps should have all their models included
  212. # in the render. We don't use the original model instances as there
  213. # are some variables that refer to the Apps object.
  214. # FKs/M2Ms from real apps are also not included as they just
  215. # mess things up with partial states (due to lack of dependencies)
  216. self.real_models = []
  217. for app_label in real_apps:
  218. app = global_apps.get_app_config(app_label)
  219. for model in app.get_models():
  220. self.real_models.append(ModelState.from_model(model, exclude_rels=True))
  221. # Populate the app registry with a stub for each application.
  222. app_labels = {model_state.app_label for model_state in models.values()}
  223. app_configs = [AppConfigStub(label) for label in sorted([*real_apps, *app_labels])]
  224. super().__init__(app_configs)
  225. # These locks get in the way of copying as implemented in clone(),
  226. # which is called whenever Django duplicates a StateApps before
  227. # updating it.
  228. self._lock = None
  229. self.ready_event = None
  230. self.render_multiple([*models.values(), *self.real_models])
  231. # There shouldn't be any operations pending at this point.
  232. from django.core.checks.model_checks import _check_lazy_references
  233. ignore = {make_model_tuple(settings.AUTH_USER_MODEL)} if ignore_swappable else set()
  234. errors = _check_lazy_references(self, ignore=ignore)
  235. if errors:
  236. raise ValueError("\n".join(error.msg for error in errors))
  237. @contextmanager
  238. def bulk_update(self):
  239. # Avoid clearing each model's cache for each change. Instead, clear
  240. # all caches when we're finished updating the model instances.
  241. ready = self.ready
  242. self.ready = False
  243. try:
  244. yield
  245. finally:
  246. self.ready = ready
  247. self.clear_cache()
  248. def render_multiple(self, model_states):
  249. # We keep trying to render the models in a loop, ignoring invalid
  250. # base errors, until the size of the unrendered models doesn't
  251. # decrease by at least one, meaning there's a base dependency loop/
  252. # missing base.
  253. if not model_states:
  254. return
  255. # Prevent that all model caches are expired for each render.
  256. with self.bulk_update():
  257. unrendered_models = model_states
  258. while unrendered_models:
  259. new_unrendered_models = []
  260. for model in unrendered_models:
  261. try:
  262. model.render(self)
  263. except InvalidBasesError:
  264. new_unrendered_models.append(model)
  265. if len(new_unrendered_models) == len(unrendered_models):
  266. raise InvalidBasesError(
  267. "Cannot resolve bases for %r\nThis can happen if you are inheriting models from an "
  268. "app with migrations (e.g. contrib.auth)\n in an app with no migrations; see "
  269. "https://docs.djangoproject.com/en/%s/topics/migrations/#dependencies "
  270. "for more" % (new_unrendered_models, get_docs_version())
  271. )
  272. unrendered_models = new_unrendered_models
  273. def clone(self):
  274. """Return a clone of this registry."""
  275. clone = StateApps([], {})
  276. clone.all_models = copy.deepcopy(self.all_models)
  277. clone.app_configs = copy.deepcopy(self.app_configs)
  278. # Set the pointer to the correct app registry.
  279. for app_config in clone.app_configs.values():
  280. app_config.apps = clone
  281. # No need to actually clone them, they'll never change
  282. clone.real_models = self.real_models
  283. return clone
  284. def register_model(self, app_label, model):
  285. self.all_models[app_label][model._meta.model_name] = model
  286. if app_label not in self.app_configs:
  287. self.app_configs[app_label] = AppConfigStub(app_label)
  288. self.app_configs[app_label].apps = self
  289. self.app_configs[app_label].models = OrderedDict()
  290. self.app_configs[app_label].models[model._meta.model_name] = model
  291. self.do_pending_operations(model)
  292. self.clear_cache()
  293. def unregister_model(self, app_label, model_name):
  294. try:
  295. del self.all_models[app_label][model_name]
  296. del self.app_configs[app_label].models[model_name]
  297. except KeyError:
  298. pass
  299. class ModelState:
  300. """
  301. Represent a Django Model. Don't use the actual Model class as it's not
  302. designed to have its options changed - instead, mutate this one and then
  303. render it into a Model as required.
  304. Note that while you are allowed to mutate .fields, you are not allowed
  305. to mutate the Field instances inside there themselves - you must instead
  306. assign new ones, as these are not detached during a clone.
  307. """
  308. def __init__(self, app_label, name, fields, options=None, bases=None, managers=None):
  309. self.app_label = app_label
  310. self.name = name
  311. self.fields = fields
  312. self.options = options or {}
  313. self.options.setdefault('indexes', [])
  314. self.options.setdefault('constraints', [])
  315. self.bases = bases or (models.Model,)
  316. self.managers = managers or []
  317. # Sanity-check that fields is NOT a dict. It must be ordered.
  318. if isinstance(self.fields, dict):
  319. raise ValueError("ModelState.fields cannot be a dict - it must be a list of 2-tuples.")
  320. for name, field in fields:
  321. # Sanity-check that fields are NOT already bound to a model.
  322. if hasattr(field, 'model'):
  323. raise ValueError(
  324. 'ModelState.fields cannot be bound to a model - "%s" is.' % name
  325. )
  326. # Sanity-check that relation fields are NOT referring to a model class.
  327. if field.is_relation and hasattr(field.related_model, '_meta'):
  328. raise ValueError(
  329. 'ModelState.fields cannot refer to a model class - "%s.to" does. '
  330. 'Use a string reference instead.' % name
  331. )
  332. if field.many_to_many and hasattr(field.remote_field.through, '_meta'):
  333. raise ValueError(
  334. 'ModelState.fields cannot refer to a model class - "%s.through" does. '
  335. 'Use a string reference instead.' % name
  336. )
  337. # Sanity-check that indexes have their name set.
  338. for index in self.options['indexes']:
  339. if not index.name:
  340. raise ValueError(
  341. "Indexes passed to ModelState require a name attribute. "
  342. "%r doesn't have one." % index
  343. )
  344. @cached_property
  345. def name_lower(self):
  346. return self.name.lower()
  347. @classmethod
  348. def from_model(cls, model, exclude_rels=False):
  349. """Given a model, return a ModelState representing it."""
  350. # Deconstruct the fields
  351. fields = []
  352. for field in model._meta.local_fields:
  353. if getattr(field, "remote_field", None) and exclude_rels:
  354. continue
  355. if isinstance(field, OrderWrt):
  356. continue
  357. name = field.name
  358. try:
  359. fields.append((name, field.clone()))
  360. except TypeError as e:
  361. raise TypeError("Couldn't reconstruct field %s on %s: %s" % (
  362. name,
  363. model._meta.label,
  364. e,
  365. ))
  366. if not exclude_rels:
  367. for field in model._meta.local_many_to_many:
  368. name = field.name
  369. try:
  370. fields.append((name, field.clone()))
  371. except TypeError as e:
  372. raise TypeError("Couldn't reconstruct m2m field %s on %s: %s" % (
  373. name,
  374. model._meta.object_name,
  375. e,
  376. ))
  377. # Extract the options
  378. options = {}
  379. for name in DEFAULT_NAMES:
  380. # Ignore some special options
  381. if name in ["apps", "app_label"]:
  382. continue
  383. elif name in model._meta.original_attrs:
  384. if name == "unique_together":
  385. ut = model._meta.original_attrs["unique_together"]
  386. options[name] = set(normalize_together(ut))
  387. elif name == "index_together":
  388. it = model._meta.original_attrs["index_together"]
  389. options[name] = set(normalize_together(it))
  390. elif name == "indexes":
  391. indexes = [idx.clone() for idx in model._meta.indexes]
  392. for index in indexes:
  393. if not index.name:
  394. index.set_name_with_model(model)
  395. options['indexes'] = indexes
  396. elif name == 'constraints':
  397. options['constraints'] = [con.clone() for con in model._meta.constraints]
  398. else:
  399. options[name] = model._meta.original_attrs[name]
  400. # If we're ignoring relationships, remove all field-listing model
  401. # options (that option basically just means "make a stub model")
  402. if exclude_rels:
  403. for key in ["unique_together", "index_together", "order_with_respect_to"]:
  404. if key in options:
  405. del options[key]
  406. # Private fields are ignored, so remove options that refer to them.
  407. elif options.get('order_with_respect_to') in {field.name for field in model._meta.private_fields}:
  408. del options['order_with_respect_to']
  409. def flatten_bases(model):
  410. bases = []
  411. for base in model.__bases__:
  412. if hasattr(base, "_meta") and base._meta.abstract:
  413. bases.extend(flatten_bases(base))
  414. else:
  415. bases.append(base)
  416. return bases
  417. # We can't rely on __mro__ directly because we only want to flatten
  418. # abstract models and not the whole tree. However by recursing on
  419. # __bases__ we may end up with duplicates and ordering issues, we
  420. # therefore discard any duplicates and reorder the bases according
  421. # to their index in the MRO.
  422. flattened_bases = sorted(set(flatten_bases(model)), key=lambda x: model.__mro__.index(x))
  423. # Make our record
  424. bases = tuple(
  425. (
  426. base._meta.label_lower
  427. if hasattr(base, "_meta") else
  428. base
  429. )
  430. for base in flattened_bases
  431. )
  432. # Ensure at least one base inherits from models.Model
  433. if not any((isinstance(base, str) or issubclass(base, models.Model)) for base in bases):
  434. bases = (models.Model,)
  435. managers = []
  436. manager_names = set()
  437. default_manager_shim = None
  438. for manager in model._meta.managers:
  439. if manager.name in manager_names:
  440. # Skip overridden managers.
  441. continue
  442. elif manager.use_in_migrations:
  443. # Copy managers usable in migrations.
  444. new_manager = copy.copy(manager)
  445. new_manager._set_creation_counter()
  446. elif manager is model._base_manager or manager is model._default_manager:
  447. # Shim custom managers used as default and base managers.
  448. new_manager = models.Manager()
  449. new_manager.model = manager.model
  450. new_manager.name = manager.name
  451. if manager is model._default_manager:
  452. default_manager_shim = new_manager
  453. else:
  454. continue
  455. manager_names.add(manager.name)
  456. managers.append((manager.name, new_manager))
  457. # Ignore a shimmed default manager called objects if it's the only one.
  458. if managers == [('objects', default_manager_shim)]:
  459. managers = []
  460. # Construct the new ModelState
  461. return cls(
  462. model._meta.app_label,
  463. model._meta.object_name,
  464. fields,
  465. options,
  466. bases,
  467. managers,
  468. )
  469. def construct_managers(self):
  470. """Deep-clone the managers using deconstruction."""
  471. # Sort all managers by their creation counter
  472. sorted_managers = sorted(self.managers, key=lambda v: v[1].creation_counter)
  473. for mgr_name, manager in sorted_managers:
  474. as_manager, manager_path, qs_path, args, kwargs = manager.deconstruct()
  475. if as_manager:
  476. qs_class = import_string(qs_path)
  477. yield mgr_name, qs_class.as_manager()
  478. else:
  479. manager_class = import_string(manager_path)
  480. yield mgr_name, manager_class(*args, **kwargs)
  481. def clone(self):
  482. """Return an exact copy of this ModelState."""
  483. return self.__class__(
  484. app_label=self.app_label,
  485. name=self.name,
  486. fields=list(self.fields),
  487. # Since options are shallow-copied here, operations such as
  488. # AddIndex must replace their option (e.g 'indexes') rather
  489. # than mutating it.
  490. options=dict(self.options),
  491. bases=self.bases,
  492. managers=list(self.managers),
  493. )
  494. def render(self, apps):
  495. """Create a Model object from our current state into the given apps."""
  496. # First, make a Meta object
  497. meta_contents = {'app_label': self.app_label, 'apps': apps, **self.options}
  498. meta = type("Meta", (), meta_contents)
  499. # Then, work out our bases
  500. try:
  501. bases = tuple(
  502. (apps.get_model(base) if isinstance(base, str) else base)
  503. for base in self.bases
  504. )
  505. except LookupError:
  506. raise InvalidBasesError("Cannot resolve one or more bases from %r" % (self.bases,))
  507. # Turn fields into a dict for the body, add other bits
  508. body = {name: field.clone() for name, field in self.fields}
  509. body['Meta'] = meta
  510. body['__module__'] = "__fake__"
  511. # Restore managers
  512. body.update(self.construct_managers())
  513. # Then, make a Model object (apps.register_model is called in __new__)
  514. return type(self.name, bases, body)
  515. def get_field_by_name(self, name):
  516. for fname, field in self.fields:
  517. if fname == name:
  518. return field
  519. raise ValueError("No field called %s on model %s" % (name, self.name))
  520. def get_index_by_name(self, name):
  521. for index in self.options['indexes']:
  522. if index.name == name:
  523. return index
  524. raise ValueError("No index named %s on model %s" % (name, self.name))
  525. def get_constraint_by_name(self, name):
  526. for constraint in self.options['constraints']:
  527. if constraint.name == name:
  528. return constraint
  529. raise ValueError('No constraint named %s on model %s' % (name, self.name))
  530. def __repr__(self):
  531. return "<%s: '%s.%s'>" % (self.__class__.__name__, self.app_label, self.name)
  532. def __eq__(self, other):
  533. return (
  534. (self.app_label == other.app_label) and
  535. (self.name == other.name) and
  536. (len(self.fields) == len(other.fields)) and
  537. all((k1 == k2 and (f1.deconstruct()[1:] == f2.deconstruct()[1:]))
  538. for (k1, f1), (k2, f2) in zip(self.fields, other.fields)) and
  539. (self.options == other.options) and
  540. (self.bases == other.bases) and
  541. (self.managers == other.managers)
  542. )