Funktionierender Prototyp des Serious Games zur Vermittlung von Wissen zu Software-Engineering-Arbeitsmodellen.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

encoding.py 8.6KB

1 year ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264
  1. import codecs
  2. import datetime
  3. import locale
  4. from decimal import Decimal
  5. from urllib.parse import quote
  6. from django.utils.functional import Promise
  7. class DjangoUnicodeDecodeError(UnicodeDecodeError):
  8. def __init__(self, obj, *args):
  9. self.obj = obj
  10. super().__init__(*args)
  11. def __str__(self):
  12. return "%s. You passed in %r (%s)" % (
  13. super().__str__(),
  14. self.obj,
  15. type(self.obj),
  16. )
  17. def smart_str(s, encoding="utf-8", strings_only=False, errors="strict"):
  18. """
  19. Return a string representing 's'. Treat bytestrings using the 'encoding'
  20. codec.
  21. If strings_only is True, don't convert (some) non-string-like objects.
  22. """
  23. if isinstance(s, Promise):
  24. # The input is the result of a gettext_lazy() call.
  25. return s
  26. return force_str(s, encoding, strings_only, errors)
  27. _PROTECTED_TYPES = (
  28. type(None),
  29. int,
  30. float,
  31. Decimal,
  32. datetime.datetime,
  33. datetime.date,
  34. datetime.time,
  35. )
  36. def is_protected_type(obj):
  37. """Determine if the object instance is of a protected type.
  38. Objects of protected types are preserved as-is when passed to
  39. force_str(strings_only=True).
  40. """
  41. return isinstance(obj, _PROTECTED_TYPES)
  42. def force_str(s, encoding="utf-8", strings_only=False, errors="strict"):
  43. """
  44. Similar to smart_str(), except that lazy instances are resolved to
  45. strings, rather than kept as lazy objects.
  46. If strings_only is True, don't convert (some) non-string-like objects.
  47. """
  48. # Handle the common case first for performance reasons.
  49. if issubclass(type(s), str):
  50. return s
  51. if strings_only and is_protected_type(s):
  52. return s
  53. try:
  54. if isinstance(s, bytes):
  55. s = str(s, encoding, errors)
  56. else:
  57. s = str(s)
  58. except UnicodeDecodeError as e:
  59. raise DjangoUnicodeDecodeError(s, *e.args)
  60. return s
  61. def smart_bytes(s, encoding="utf-8", strings_only=False, errors="strict"):
  62. """
  63. Return a bytestring version of 's', encoded as specified in 'encoding'.
  64. If strings_only is True, don't convert (some) non-string-like objects.
  65. """
  66. if isinstance(s, Promise):
  67. # The input is the result of a gettext_lazy() call.
  68. return s
  69. return force_bytes(s, encoding, strings_only, errors)
  70. def force_bytes(s, encoding="utf-8", strings_only=False, errors="strict"):
  71. """
  72. Similar to smart_bytes, except that lazy instances are resolved to
  73. strings, rather than kept as lazy objects.
  74. If strings_only is True, don't convert (some) non-string-like objects.
  75. """
  76. # Handle the common case first for performance reasons.
  77. if isinstance(s, bytes):
  78. if encoding == "utf-8":
  79. return s
  80. else:
  81. return s.decode("utf-8", errors).encode(encoding, errors)
  82. if strings_only and is_protected_type(s):
  83. return s
  84. if isinstance(s, memoryview):
  85. return bytes(s)
  86. return str(s).encode(encoding, errors)
  87. def iri_to_uri(iri):
  88. """
  89. Convert an Internationalized Resource Identifier (IRI) portion to a URI
  90. portion that is suitable for inclusion in a URL.
  91. This is the algorithm from section 3.1 of RFC 3987, slightly simplified
  92. since the input is assumed to be a string rather than an arbitrary byte
  93. stream.
  94. Take an IRI (string or UTF-8 bytes, e.g. '/I ♥ Django/' or
  95. b'/I \xe2\x99\xa5 Django/') and return a string containing the encoded
  96. result with ASCII chars only (e.g. '/I%20%E2%99%A5%20Django/').
  97. """
  98. # The list of safe characters here is constructed from the "reserved" and
  99. # "unreserved" characters specified in sections 2.2 and 2.3 of RFC 3986:
  100. # reserved = gen-delims / sub-delims
  101. # gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@"
  102. # sub-delims = "!" / "$" / "&" / "'" / "(" / ")"
  103. # / "*" / "+" / "," / ";" / "="
  104. # unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
  105. # Of the unreserved characters, urllib.parse.quote() already considers all
  106. # but the ~ safe.
  107. # The % character is also added to the list of safe characters here, as the
  108. # end of section 3.1 of RFC 3987 specifically mentions that % must not be
  109. # converted.
  110. if iri is None:
  111. return iri
  112. elif isinstance(iri, Promise):
  113. iri = str(iri)
  114. return quote(iri, safe="/#%[]=:;$&()+,!?*@'~")
  115. # List of byte values that uri_to_iri() decodes from percent encoding.
  116. # First, the unreserved characters from RFC 3986:
  117. _ascii_ranges = [[45, 46, 95, 126], range(65, 91), range(97, 123)]
  118. _hextobyte = {
  119. (fmt % char).encode(): bytes((char,))
  120. for ascii_range in _ascii_ranges
  121. for char in ascii_range
  122. for fmt in ["%02x", "%02X"]
  123. }
  124. # And then everything above 128, because bytes ≥ 128 are part of multibyte
  125. # Unicode characters.
  126. _hexdig = "0123456789ABCDEFabcdef"
  127. _hextobyte.update(
  128. {(a + b).encode(): bytes.fromhex(a + b) for a in _hexdig[8:] for b in _hexdig}
  129. )
  130. def uri_to_iri(uri):
  131. """
  132. Convert a Uniform Resource Identifier(URI) into an Internationalized
  133. Resource Identifier(IRI).
  134. This is the algorithm from section 3.2 of RFC 3987, excluding step 4.
  135. Take an URI in ASCII bytes (e.g. '/I%20%E2%99%A5%20Django/') and return
  136. a string containing the encoded result (e.g. '/I%20♥%20Django/').
  137. """
  138. if uri is None:
  139. return uri
  140. uri = force_bytes(uri)
  141. # Fast selective unquote: First, split on '%' and then starting with the
  142. # second block, decode the first 2 bytes if they represent a hex code to
  143. # decode. The rest of the block is the part after '%AB', not containing
  144. # any '%'. Add that to the output without further processing.
  145. bits = uri.split(b"%")
  146. if len(bits) == 1:
  147. iri = uri
  148. else:
  149. parts = [bits[0]]
  150. append = parts.append
  151. hextobyte = _hextobyte
  152. for item in bits[1:]:
  153. hex = item[:2]
  154. if hex in hextobyte:
  155. append(hextobyte[item[:2]])
  156. append(item[2:])
  157. else:
  158. append(b"%")
  159. append(item)
  160. iri = b"".join(parts)
  161. return repercent_broken_unicode(iri).decode()
  162. def escape_uri_path(path):
  163. """
  164. Escape the unsafe characters from the path portion of a Uniform Resource
  165. Identifier (URI).
  166. """
  167. # These are the "reserved" and "unreserved" characters specified in
  168. # sections 2.2 and 2.3 of RFC 2396:
  169. # reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | "$" | ","
  170. # unreserved = alphanum | mark
  171. # mark = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")"
  172. # The list of safe characters here is constructed subtracting ";", "=",
  173. # and "?" according to section 3.3 of RFC 2396.
  174. # The reason for not subtracting and escaping "/" is that we are escaping
  175. # the entire path, not a path segment.
  176. return quote(path, safe="/:@&+$,-_.!~*'()")
  177. def punycode(domain):
  178. """Return the Punycode of the given domain if it's non-ASCII."""
  179. return domain.encode("idna").decode("ascii")
  180. def repercent_broken_unicode(path):
  181. """
  182. As per section 3.2 of RFC 3987, step three of converting a URI into an IRI,
  183. repercent-encode any octet produced that is not part of a strictly legal
  184. UTF-8 octet sequence.
  185. """
  186. while True:
  187. try:
  188. path.decode()
  189. except UnicodeDecodeError as e:
  190. # CVE-2019-14235: A recursion shouldn't be used since the exception
  191. # handling uses massive amounts of memory
  192. repercent = quote(path[e.start : e.end], safe=b"/#%[]=:;$&()+,!?*@'~")
  193. path = path[: e.start] + repercent.encode() + path[e.end :]
  194. else:
  195. return path
  196. def filepath_to_uri(path):
  197. """Convert a file system path to a URI portion that is suitable for
  198. inclusion in a URL.
  199. Encode certain chars that would normally be recognized as special chars
  200. for URIs. Do not encode the ' character, as it is a valid character
  201. within URIs. See the encodeURIComponent() JavaScript function for details.
  202. """
  203. if path is None:
  204. return path
  205. # I know about `os.sep` and `os.altsep` but I want to leave
  206. # some flexibility for hardcoding separators.
  207. return quote(str(path).replace("\\", "/"), safe="/~!*()'")
  208. def get_system_encoding():
  209. """
  210. The encoding for the character type functions. Fallback to 'ascii' if the
  211. #encoding is unsupported by Python or could not be determined. See tickets
  212. #10335 and #5846.
  213. """
  214. try:
  215. encoding = locale.getlocale()[1] or "ascii"
  216. codecs.lookup(encoding)
  217. except Exception:
  218. encoding = "ascii"
  219. return encoding
  220. DEFAULT_LOCALE_ENCODING = get_system_encoding()