You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

loader.py 15KB

5 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324
  1. import pkgutil
  2. import sys
  3. from importlib import import_module, reload
  4. from django.apps import apps
  5. from django.conf import settings
  6. from django.db.migrations.graph import MigrationGraph
  7. from django.db.migrations.recorder import MigrationRecorder
  8. from .exceptions import (
  9. AmbiguityError, BadMigrationError, InconsistentMigrationHistory,
  10. NodeNotFoundError,
  11. )
  12. MIGRATIONS_MODULE_NAME = 'migrations'
  13. class MigrationLoader:
  14. """
  15. Load migration files from disk and their status from the database.
  16. Migration files are expected to live in the "migrations" directory of
  17. an app. Their names are entirely unimportant from a code perspective,
  18. but will probably follow the 1234_name.py convention.
  19. On initialization, this class will scan those directories, and open and
  20. read the Python files, looking for a class called Migration, which should
  21. inherit from django.db.migrations.Migration. See
  22. django.db.migrations.migration for what that looks like.
  23. Some migrations will be marked as "replacing" another set of migrations.
  24. These are loaded into a separate set of migrations away from the main ones.
  25. If all the migrations they replace are either unapplied or missing from
  26. disk, then they are injected into the main set, replacing the named migrations.
  27. Any dependency pointers to the replaced migrations are re-pointed to the
  28. new migration.
  29. This does mean that this class MUST also talk to the database as well as
  30. to disk, but this is probably fine. We're already not just operating
  31. in memory.
  32. """
  33. def __init__(self, connection, load=True, ignore_no_migrations=False):
  34. self.connection = connection
  35. self.disk_migrations = None
  36. self.applied_migrations = None
  37. self.ignore_no_migrations = ignore_no_migrations
  38. if load:
  39. self.build_graph()
  40. @classmethod
  41. def migrations_module(cls, app_label):
  42. """
  43. Return the path to the migrations module for the specified app_label
  44. and a boolean indicating if the module is specified in
  45. settings.MIGRATION_MODULE.
  46. """
  47. if app_label in settings.MIGRATION_MODULES:
  48. return settings.MIGRATION_MODULES[app_label], True
  49. else:
  50. app_package_name = apps.get_app_config(app_label).name
  51. return '%s.%s' % (app_package_name, MIGRATIONS_MODULE_NAME), False
  52. def load_disk(self):
  53. """Load the migrations from all INSTALLED_APPS from disk."""
  54. self.disk_migrations = {}
  55. self.unmigrated_apps = set()
  56. self.migrated_apps = set()
  57. for app_config in apps.get_app_configs():
  58. # Get the migrations module directory
  59. module_name, explicit = self.migrations_module(app_config.label)
  60. if module_name is None:
  61. self.unmigrated_apps.add(app_config.label)
  62. continue
  63. was_loaded = module_name in sys.modules
  64. try:
  65. module = import_module(module_name)
  66. except ImportError as e:
  67. # I hate doing this, but I don't want to squash other import errors.
  68. # Might be better to try a directory check directly.
  69. if ((explicit and self.ignore_no_migrations) or (
  70. not explicit and "No module named" in str(e) and MIGRATIONS_MODULE_NAME in str(e))):
  71. self.unmigrated_apps.add(app_config.label)
  72. continue
  73. raise
  74. else:
  75. # Empty directories are namespaces.
  76. # getattr() needed on PY36 and older (replace w/attribute access).
  77. if getattr(module, '__file__', None) is None:
  78. self.unmigrated_apps.add(app_config.label)
  79. continue
  80. # Module is not a package (e.g. migrations.py).
  81. if not hasattr(module, '__path__'):
  82. self.unmigrated_apps.add(app_config.label)
  83. continue
  84. # Force a reload if it's already loaded (tests need this)
  85. if was_loaded:
  86. reload(module)
  87. self.migrated_apps.add(app_config.label)
  88. migration_names = {
  89. name for _, name, is_pkg in pkgutil.iter_modules(module.__path__)
  90. if not is_pkg and name[0] not in '_~'
  91. }
  92. # Load migrations
  93. for migration_name in migration_names:
  94. migration_path = '%s.%s' % (module_name, migration_name)
  95. try:
  96. migration_module = import_module(migration_path)
  97. except ImportError as e:
  98. if 'bad magic number' in str(e):
  99. raise ImportError(
  100. "Couldn't import %r as it appears to be a stale "
  101. ".pyc file." % migration_path
  102. ) from e
  103. else:
  104. raise
  105. if not hasattr(migration_module, "Migration"):
  106. raise BadMigrationError(
  107. "Migration %s in app %s has no Migration class" % (migration_name, app_config.label)
  108. )
  109. self.disk_migrations[app_config.label, migration_name] = migration_module.Migration(
  110. migration_name,
  111. app_config.label,
  112. )
  113. def get_migration(self, app_label, name_prefix):
  114. """Return the named migration or raise NodeNotFoundError."""
  115. return self.graph.nodes[app_label, name_prefix]
  116. def get_migration_by_prefix(self, app_label, name_prefix):
  117. """
  118. Return the migration(s) which match the given app label and name_prefix.
  119. """
  120. # Do the search
  121. results = []
  122. for migration_app_label, migration_name in self.disk_migrations:
  123. if migration_app_label == app_label and migration_name.startswith(name_prefix):
  124. results.append((migration_app_label, migration_name))
  125. if len(results) > 1:
  126. raise AmbiguityError(
  127. "There is more than one migration for '%s' with the prefix '%s'" % (app_label, name_prefix)
  128. )
  129. elif not results:
  130. raise KeyError("There no migrations for '%s' with the prefix '%s'" % (app_label, name_prefix))
  131. else:
  132. return self.disk_migrations[results[0]]
  133. def check_key(self, key, current_app):
  134. if (key[1] != "__first__" and key[1] != "__latest__") or key in self.graph:
  135. return key
  136. # Special-case __first__, which means "the first migration" for
  137. # migrated apps, and is ignored for unmigrated apps. It allows
  138. # makemigrations to declare dependencies on apps before they even have
  139. # migrations.
  140. if key[0] == current_app:
  141. # Ignore __first__ references to the same app (#22325)
  142. return
  143. if key[0] in self.unmigrated_apps:
  144. # This app isn't migrated, but something depends on it.
  145. # The models will get auto-added into the state, though
  146. # so we're fine.
  147. return
  148. if key[0] in self.migrated_apps:
  149. try:
  150. if key[1] == "__first__":
  151. return self.graph.root_nodes(key[0])[0]
  152. else: # "__latest__"
  153. return self.graph.leaf_nodes(key[0])[0]
  154. except IndexError:
  155. if self.ignore_no_migrations:
  156. return None
  157. else:
  158. raise ValueError("Dependency on app with no migrations: %s" % key[0])
  159. raise ValueError("Dependency on unknown app: %s" % key[0])
  160. def add_internal_dependencies(self, key, migration):
  161. """
  162. Internal dependencies need to be added first to ensure `__first__`
  163. dependencies find the correct root node.
  164. """
  165. for parent in migration.dependencies:
  166. # Ignore __first__ references to the same app.
  167. if parent[0] == key[0] and parent[1] != '__first__':
  168. self.graph.add_dependency(migration, key, parent, skip_validation=True)
  169. def add_external_dependencies(self, key, migration):
  170. for parent in migration.dependencies:
  171. # Skip internal dependencies
  172. if key[0] == parent[0]:
  173. continue
  174. parent = self.check_key(parent, key[0])
  175. if parent is not None:
  176. self.graph.add_dependency(migration, key, parent, skip_validation=True)
  177. for child in migration.run_before:
  178. child = self.check_key(child, key[0])
  179. if child is not None:
  180. self.graph.add_dependency(migration, child, key, skip_validation=True)
  181. def build_graph(self):
  182. """
  183. Build a migration dependency graph using both the disk and database.
  184. You'll need to rebuild the graph if you apply migrations. This isn't
  185. usually a problem as generally migration stuff runs in a one-shot process.
  186. """
  187. # Load disk data
  188. self.load_disk()
  189. # Load database data
  190. if self.connection is None:
  191. self.applied_migrations = set()
  192. else:
  193. recorder = MigrationRecorder(self.connection)
  194. self.applied_migrations = recorder.applied_migrations()
  195. # To start, populate the migration graph with nodes for ALL migrations
  196. # and their dependencies. Also make note of replacing migrations at this step.
  197. self.graph = MigrationGraph()
  198. self.replacements = {}
  199. for key, migration in self.disk_migrations.items():
  200. self.graph.add_node(key, migration)
  201. # Replacing migrations.
  202. if migration.replaces:
  203. self.replacements[key] = migration
  204. for key, migration in self.disk_migrations.items():
  205. # Internal (same app) dependencies.
  206. self.add_internal_dependencies(key, migration)
  207. # Add external dependencies now that the internal ones have been resolved.
  208. for key, migration in self.disk_migrations.items():
  209. self.add_external_dependencies(key, migration)
  210. # Carry out replacements where possible.
  211. for key, migration in self.replacements.items():
  212. # Get applied status of each of this migration's replacement targets.
  213. applied_statuses = [(target in self.applied_migrations) for target in migration.replaces]
  214. # Ensure the replacing migration is only marked as applied if all of
  215. # its replacement targets are.
  216. if all(applied_statuses):
  217. self.applied_migrations.add(key)
  218. else:
  219. self.applied_migrations.discard(key)
  220. # A replacing migration can be used if either all or none of its
  221. # replacement targets have been applied.
  222. if all(applied_statuses) or (not any(applied_statuses)):
  223. self.graph.remove_replaced_nodes(key, migration.replaces)
  224. else:
  225. # This replacing migration cannot be used because it is partially applied.
  226. # Remove it from the graph and remap dependencies to it (#25945).
  227. self.graph.remove_replacement_node(key, migration.replaces)
  228. # Ensure the graph is consistent.
  229. try:
  230. self.graph.validate_consistency()
  231. except NodeNotFoundError as exc:
  232. # Check if the missing node could have been replaced by any squash
  233. # migration but wasn't because the squash migration was partially
  234. # applied before. In that case raise a more understandable exception
  235. # (#23556).
  236. # Get reverse replacements.
  237. reverse_replacements = {}
  238. for key, migration in self.replacements.items():
  239. for replaced in migration.replaces:
  240. reverse_replacements.setdefault(replaced, set()).add(key)
  241. # Try to reraise exception with more detail.
  242. if exc.node in reverse_replacements:
  243. candidates = reverse_replacements.get(exc.node, set())
  244. is_replaced = any(candidate in self.graph.nodes for candidate in candidates)
  245. if not is_replaced:
  246. tries = ', '.join('%s.%s' % c for c in candidates)
  247. raise NodeNotFoundError(
  248. "Migration {0} depends on nonexistent node ('{1}', '{2}'). "
  249. "Django tried to replace migration {1}.{2} with any of [{3}] "
  250. "but wasn't able to because some of the replaced migrations "
  251. "are already applied.".format(
  252. exc.origin, exc.node[0], exc.node[1], tries
  253. ),
  254. exc.node
  255. ) from exc
  256. raise exc
  257. self.graph.ensure_not_cyclic()
  258. def check_consistent_history(self, connection):
  259. """
  260. Raise InconsistentMigrationHistory if any applied migrations have
  261. unapplied dependencies.
  262. """
  263. recorder = MigrationRecorder(connection)
  264. applied = recorder.applied_migrations()
  265. for migration in applied:
  266. # If the migration is unknown, skip it.
  267. if migration not in self.graph.nodes:
  268. continue
  269. for parent in self.graph.node_map[migration].parents:
  270. if parent not in applied:
  271. # Skip unapplied squashed migrations that have all of their
  272. # `replaces` applied.
  273. if parent in self.replacements:
  274. if all(m in applied for m in self.replacements[parent].replaces):
  275. continue
  276. raise InconsistentMigrationHistory(
  277. "Migration {}.{} is applied before its dependency "
  278. "{}.{} on database '{}'.".format(
  279. migration[0], migration[1], parent[0], parent[1],
  280. connection.alias,
  281. )
  282. )
  283. def detect_conflicts(self):
  284. """
  285. Look through the loaded graph and detect any conflicts - apps
  286. with more than one leaf migration. Return a dict of the app labels
  287. that conflict with the migration names that conflict.
  288. """
  289. seen_apps = {}
  290. conflicting_apps = set()
  291. for app_label, migration_name in self.graph.leaf_nodes():
  292. if app_label in seen_apps:
  293. conflicting_apps.add(app_label)
  294. seen_apps.setdefault(app_label, set()).add(migration_name)
  295. return {app_label: seen_apps[app_label] for app_label in conflicting_apps}
  296. def project_state(self, nodes=None, at_end=True):
  297. """
  298. Return a ProjectState object representing the most recent state
  299. that the loaded migrations represent.
  300. See graph.make_state() for the meaning of "nodes" and "at_end".
  301. """
  302. return self.graph.make_state(nodes=nodes, at_end=at_end, real_apps=list(self.unmigrated_apps))