You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

unicode_utils.py 996B

1234567891011121314151617181920212223242526272829303132333435363738394041424344
  1. import unicodedata
  2. import sys
  3. from setuptools.extern import six
  4. # HFS Plus uses decomposed UTF-8
  5. def decompose(path):
  6. if isinstance(path, six.text_type):
  7. return unicodedata.normalize('NFD', path)
  8. try:
  9. path = path.decode('utf-8')
  10. path = unicodedata.normalize('NFD', path)
  11. path = path.encode('utf-8')
  12. except UnicodeError:
  13. pass # Not UTF-8
  14. return path
  15. def filesys_decode(path):
  16. """
  17. Ensure that the given path is decoded,
  18. NONE when no expected encoding works
  19. """
  20. if isinstance(path, six.text_type):
  21. return path
  22. fs_enc = sys.getfilesystemencoding() or 'utf-8'
  23. candidates = fs_enc, 'utf-8'
  24. for enc in candidates:
  25. try:
  26. return path.decode(enc)
  27. except UnicodeDecodeError:
  28. continue
  29. def try_encode(string, enc):
  30. "turn unicode encoding into a functional routine"
  31. try:
  32. return string.encode(enc)
  33. except UnicodeEncodeError:
  34. return None