You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

file_cache.py 4.1KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133
  1. import hashlib
  2. import os
  3. from textwrap import dedent
  4. from ..cache import BaseCache
  5. from ..controller import CacheController
  6. try:
  7. FileNotFoundError
  8. except NameError:
  9. # py2.X
  10. FileNotFoundError = OSError
  11. def _secure_open_write(filename, fmode):
  12. # We only want to write to this file, so open it in write only mode
  13. flags = os.O_WRONLY
  14. # os.O_CREAT | os.O_EXCL will fail if the file already exists, so we only
  15. # will open *new* files.
  16. # We specify this because we want to ensure that the mode we pass is the
  17. # mode of the file.
  18. flags |= os.O_CREAT | os.O_EXCL
  19. # Do not follow symlinks to prevent someone from making a symlink that
  20. # we follow and insecurely open a cache file.
  21. if hasattr(os, "O_NOFOLLOW"):
  22. flags |= os.O_NOFOLLOW
  23. # On Windows we'll mark this file as binary
  24. if hasattr(os, "O_BINARY"):
  25. flags |= os.O_BINARY
  26. # Before we open our file, we want to delete any existing file that is
  27. # there
  28. try:
  29. os.remove(filename)
  30. except (IOError, OSError):
  31. # The file must not exist already, so we can just skip ahead to opening
  32. pass
  33. # Open our file, the use of os.O_CREAT | os.O_EXCL will ensure that if a
  34. # race condition happens between the os.remove and this line, that an
  35. # error will be raised. Because we utilize a lockfile this should only
  36. # happen if someone is attempting to attack us.
  37. fd = os.open(filename, flags, fmode)
  38. try:
  39. return os.fdopen(fd, "wb")
  40. except:
  41. # An error occurred wrapping our FD in a file object
  42. os.close(fd)
  43. raise
  44. class FileCache(BaseCache):
  45. def __init__(self, directory, forever=False, filemode=0o0600,
  46. dirmode=0o0700, use_dir_lock=None, lock_class=None):
  47. if use_dir_lock is not None and lock_class is not None:
  48. raise ValueError("Cannot use use_dir_lock and lock_class together")
  49. try:
  50. from pip._vendor.lockfile import LockFile
  51. from pip._vendor.lockfile.mkdirlockfile import MkdirLockFile
  52. except ImportError:
  53. notice = dedent("""
  54. NOTE: In order to use the FileCache you must have
  55. lockfile installed. You can install it via pip:
  56. pip install lockfile
  57. """)
  58. raise ImportError(notice)
  59. else:
  60. if use_dir_lock:
  61. lock_class = MkdirLockFile
  62. elif lock_class is None:
  63. lock_class = LockFile
  64. self.directory = directory
  65. self.forever = forever
  66. self.filemode = filemode
  67. self.dirmode = dirmode
  68. self.lock_class = lock_class
  69. @staticmethod
  70. def encode(x):
  71. return hashlib.sha224(x.encode()).hexdigest()
  72. def _fn(self, name):
  73. # NOTE: This method should not change as some may depend on it.
  74. # See: https://github.com/ionrock/cachecontrol/issues/63
  75. hashed = self.encode(name)
  76. parts = list(hashed[:5]) + [hashed]
  77. return os.path.join(self.directory, *parts)
  78. def get(self, key):
  79. name = self._fn(key)
  80. if not os.path.exists(name):
  81. return None
  82. with open(name, 'rb') as fh:
  83. return fh.read()
  84. def set(self, key, value):
  85. name = self._fn(key)
  86. # Make sure the directory exists
  87. try:
  88. os.makedirs(os.path.dirname(name), self.dirmode)
  89. except (IOError, OSError):
  90. pass
  91. with self.lock_class(name) as lock:
  92. # Write our actual file
  93. with _secure_open_write(lock.path, self.filemode) as fh:
  94. fh.write(value)
  95. def delete(self, key):
  96. name = self._fn(key)
  97. if not self.forever:
  98. try:
  99. os.remove(name)
  100. except FileNotFoundError:
  101. pass
  102. def url_to_file_path(url, filecache):
  103. """Return the file cache path based on the URL.
  104. This does not ensure the file exists!
  105. """
  106. key = CacheController.cache_url(url)
  107. return filecache._fn(key)