You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

base.py 11KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318
  1. """
  2. Module for abstract serializer/unserializer base classes.
  3. """
  4. from io import StringIO
  5. from django.core.exceptions import ObjectDoesNotExist
  6. from django.db import models
  7. DEFER_FIELD = object()
  8. class SerializerDoesNotExist(KeyError):
  9. """The requested serializer was not found."""
  10. pass
  11. class SerializationError(Exception):
  12. """Something bad happened during serialization."""
  13. pass
  14. class DeserializationError(Exception):
  15. """Something bad happened during deserialization."""
  16. @classmethod
  17. def WithData(cls, original_exc, model, fk, field_value):
  18. """
  19. Factory method for creating a deserialization error which has a more
  20. explanatory message.
  21. """
  22. return cls("%s: (%s:pk=%s) field_value was '%s'" % (original_exc, model, fk, field_value))
  23. class M2MDeserializationError(Exception):
  24. """Something bad happened during deserialization of a ManyToManyField."""
  25. def __init__(self, original_exc, pk):
  26. self.original_exc = original_exc
  27. self.pk = pk
  28. class ProgressBar:
  29. progress_width = 75
  30. def __init__(self, output, total_count):
  31. self.output = output
  32. self.total_count = total_count
  33. self.prev_done = 0
  34. def update(self, count):
  35. if not self.output:
  36. return
  37. perc = count * 100 // self.total_count
  38. done = perc * self.progress_width // 100
  39. if self.prev_done >= done:
  40. return
  41. self.prev_done = done
  42. cr = '' if self.total_count == 1 else '\r'
  43. self.output.write(cr + '[' + '.' * done + ' ' * (self.progress_width - done) + ']')
  44. if done == self.progress_width:
  45. self.output.write('\n')
  46. self.output.flush()
  47. class Serializer:
  48. """
  49. Abstract serializer base class.
  50. """
  51. # Indicates if the implemented serializer is only available for
  52. # internal Django use.
  53. internal_use_only = False
  54. progress_class = ProgressBar
  55. stream_class = StringIO
  56. def serialize(self, queryset, *, stream=None, fields=None, use_natural_foreign_keys=False,
  57. use_natural_primary_keys=False, progress_output=None, object_count=0, **options):
  58. """
  59. Serialize a queryset.
  60. """
  61. self.options = options
  62. self.stream = stream if stream is not None else self.stream_class()
  63. self.selected_fields = fields
  64. self.use_natural_foreign_keys = use_natural_foreign_keys
  65. self.use_natural_primary_keys = use_natural_primary_keys
  66. progress_bar = self.progress_class(progress_output, object_count)
  67. self.start_serialization()
  68. self.first = True
  69. for count, obj in enumerate(queryset, start=1):
  70. self.start_object(obj)
  71. # Use the concrete parent class' _meta instead of the object's _meta
  72. # This is to avoid local_fields problems for proxy models. Refs #17717.
  73. concrete_model = obj._meta.concrete_model
  74. # When using natural primary keys, retrieve the pk field of the
  75. # parent for multi-table inheritance child models. That field must
  76. # be serialized, otherwise deserialization isn't possible.
  77. if self.use_natural_primary_keys:
  78. pk = concrete_model._meta.pk
  79. pk_parent = pk if pk.remote_field and pk.remote_field.parent_link else None
  80. else:
  81. pk_parent = None
  82. for field in concrete_model._meta.local_fields:
  83. if field.serialize or field is pk_parent:
  84. if field.remote_field is None:
  85. if self.selected_fields is None or field.attname in self.selected_fields:
  86. self.handle_field(obj, field)
  87. else:
  88. if self.selected_fields is None or field.attname[:-3] in self.selected_fields:
  89. self.handle_fk_field(obj, field)
  90. for field in concrete_model._meta.many_to_many:
  91. if field.serialize:
  92. if self.selected_fields is None or field.attname in self.selected_fields:
  93. self.handle_m2m_field(obj, field)
  94. self.end_object(obj)
  95. progress_bar.update(count)
  96. self.first = self.first and False
  97. self.end_serialization()
  98. return self.getvalue()
  99. def start_serialization(self):
  100. """
  101. Called when serializing of the queryset starts.
  102. """
  103. raise NotImplementedError('subclasses of Serializer must provide a start_serialization() method')
  104. def end_serialization(self):
  105. """
  106. Called when serializing of the queryset ends.
  107. """
  108. pass
  109. def start_object(self, obj):
  110. """
  111. Called when serializing of an object starts.
  112. """
  113. raise NotImplementedError('subclasses of Serializer must provide a start_object() method')
  114. def end_object(self, obj):
  115. """
  116. Called when serializing of an object ends.
  117. """
  118. pass
  119. def handle_field(self, obj, field):
  120. """
  121. Called to handle each individual (non-relational) field on an object.
  122. """
  123. raise NotImplementedError('subclasses of Serializer must provide an handle_field() method')
  124. def handle_fk_field(self, obj, field):
  125. """
  126. Called to handle a ForeignKey field.
  127. """
  128. raise NotImplementedError('subclasses of Serializer must provide an handle_fk_field() method')
  129. def handle_m2m_field(self, obj, field):
  130. """
  131. Called to handle a ManyToManyField.
  132. """
  133. raise NotImplementedError('subclasses of Serializer must provide an handle_m2m_field() method')
  134. def getvalue(self):
  135. """
  136. Return the fully serialized queryset (or None if the output stream is
  137. not seekable).
  138. """
  139. if callable(getattr(self.stream, 'getvalue', None)):
  140. return self.stream.getvalue()
  141. class Deserializer:
  142. """
  143. Abstract base deserializer class.
  144. """
  145. def __init__(self, stream_or_string, **options):
  146. """
  147. Init this serializer given a stream or a string
  148. """
  149. self.options = options
  150. if isinstance(stream_or_string, str):
  151. self.stream = StringIO(stream_or_string)
  152. else:
  153. self.stream = stream_or_string
  154. def __iter__(self):
  155. return self
  156. def __next__(self):
  157. """Iteration interface -- return the next item in the stream"""
  158. raise NotImplementedError('subclasses of Deserializer must provide a __next__() method')
  159. class DeserializedObject:
  160. """
  161. A deserialized model.
  162. Basically a container for holding the pre-saved deserialized data along
  163. with the many-to-many data saved with the object.
  164. Call ``save()`` to save the object (with the many-to-many data) to the
  165. database; call ``save(save_m2m=False)`` to save just the object fields
  166. (and not touch the many-to-many stuff.)
  167. """
  168. def __init__(self, obj, m2m_data=None, deferred_fields=None):
  169. self.object = obj
  170. self.m2m_data = m2m_data
  171. self.deferred_fields = deferred_fields
  172. def __repr__(self):
  173. return "<%s: %s(pk=%s)>" % (
  174. self.__class__.__name__,
  175. self.object._meta.label,
  176. self.object.pk,
  177. )
  178. def save(self, save_m2m=True, using=None, **kwargs):
  179. # Call save on the Model baseclass directly. This bypasses any
  180. # model-defined save. The save is also forced to be raw.
  181. # raw=True is passed to any pre/post_save signals.
  182. models.Model.save_base(self.object, using=using, raw=True, **kwargs)
  183. if self.m2m_data and save_m2m:
  184. for accessor_name, object_list in self.m2m_data.items():
  185. getattr(self.object, accessor_name).set(object_list)
  186. # prevent a second (possibly accidental) call to save() from saving
  187. # the m2m data twice.
  188. self.m2m_data = None
  189. def save_deferred_fields(self, using=None):
  190. self.m2m_data = {}
  191. for field, field_value in self.deferred_fields.items():
  192. opts = self.object._meta
  193. label = opts.app_label + '.' + opts.model_name
  194. if isinstance(field.remote_field, models.ManyToManyRel):
  195. try:
  196. values = deserialize_m2m_values(field, field_value, using, handle_forward_references=False)
  197. except M2MDeserializationError as e:
  198. raise DeserializationError.WithData(e.original_exc, label, self.object.pk, e.pk)
  199. self.m2m_data[field.name] = values
  200. elif isinstance(field.remote_field, models.ManyToOneRel):
  201. try:
  202. value = deserialize_fk_value(field, field_value, using, handle_forward_references=False)
  203. except Exception as e:
  204. raise DeserializationError.WithData(e, label, self.object.pk, field_value)
  205. setattr(self.object, field.attname, value)
  206. self.save()
  207. def build_instance(Model, data, db):
  208. """
  209. Build a model instance.
  210. If the model instance doesn't have a primary key and the model supports
  211. natural keys, try to retrieve it from the database.
  212. """
  213. default_manager = Model._meta.default_manager
  214. pk = data.get(Model._meta.pk.name)
  215. if (pk is None and hasattr(default_manager, 'get_by_natural_key') and
  216. hasattr(Model, 'natural_key')):
  217. natural_key = Model(**data).natural_key()
  218. try:
  219. data[Model._meta.pk.attname] = Model._meta.pk.to_python(
  220. default_manager.db_manager(db).get_by_natural_key(*natural_key).pk
  221. )
  222. except Model.DoesNotExist:
  223. pass
  224. return Model(**data)
  225. def deserialize_m2m_values(field, field_value, using, handle_forward_references):
  226. model = field.remote_field.model
  227. if hasattr(model._default_manager, 'get_by_natural_key'):
  228. def m2m_convert(value):
  229. if hasattr(value, '__iter__') and not isinstance(value, str):
  230. return model._default_manager.db_manager(using).get_by_natural_key(*value).pk
  231. else:
  232. return model._meta.pk.to_python(value)
  233. else:
  234. def m2m_convert(v):
  235. return model._meta.pk.to_python(v)
  236. try:
  237. values = []
  238. for pk in field_value:
  239. values.append(m2m_convert(pk))
  240. return values
  241. except Exception as e:
  242. if isinstance(e, ObjectDoesNotExist) and handle_forward_references:
  243. return DEFER_FIELD
  244. else:
  245. raise M2MDeserializationError(e, pk)
  246. def deserialize_fk_value(field, field_value, using, handle_forward_references):
  247. if field_value is None:
  248. return None
  249. model = field.remote_field.model
  250. default_manager = model._default_manager
  251. field_name = field.remote_field.field_name
  252. if (hasattr(default_manager, 'get_by_natural_key') and
  253. hasattr(field_value, '__iter__') and not isinstance(field_value, str)):
  254. try:
  255. obj = default_manager.db_manager(using).get_by_natural_key(*field_value)
  256. except ObjectDoesNotExist:
  257. if handle_forward_references:
  258. return DEFER_FIELD
  259. else:
  260. raise
  261. value = getattr(obj, field_name)
  262. # If this is a natural foreign key to an object that has a FK/O2O as
  263. # the foreign key, use the FK value.
  264. if model._meta.pk.remote_field:
  265. value = value.pk
  266. return value
  267. return model._meta.get_field(field_name).to_python(field_value)