123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361 |
- # -*- test-case-name: twisted.test.test_dirdbm -*-
- #
- # Copyright (c) Twisted Matrix Laboratories.
- # See LICENSE for details.
-
-
- """
- DBM-style interface to a directory.
-
- Each key is stored as a single file. This is not expected to be very fast or
- efficient, but it's good for easy debugging.
-
- DirDBMs are *not* thread-safe, they should only be accessed by one thread at
- a time.
-
- No files should be placed in the working directory of a DirDBM save those
- created by the DirDBM itself!
-
- Maintainer: Itamar Shtull-Trauring
- """
-
-
- import base64
- import glob
- import os
- import pickle
-
- from twisted.python.filepath import FilePath
-
- try:
- _open # type: ignore[has-type]
- except NameError:
- _open = open
-
-
- class DirDBM:
- """
- A directory with a DBM interface.
-
- This class presents a hash-like interface to a directory of small,
- flat files. It can only use strings as keys or values.
- """
-
- def __init__(self, name):
- """
- @type name: str
- @param name: Base path to use for the directory storage.
- """
- self.dname = os.path.abspath(name)
- self._dnamePath = FilePath(name)
- if not self._dnamePath.isdir():
- self._dnamePath.createDirectory()
- else:
- # Run recovery, in case we crashed. we delete all files ending
- # with ".new". Then we find all files who end with ".rpl". If a
- # corresponding file exists without ".rpl", we assume the write
- # failed and delete the ".rpl" file. If only a ".rpl" exist we
- # assume the program crashed right after deleting the old entry
- # but before renaming the replacement entry.
- #
- # NOTE: '.' is NOT in the base64 alphabet!
- for f in glob.glob(self._dnamePath.child("*.new").path):
- os.remove(f)
- replacements = glob.glob(self._dnamePath.child("*.rpl").path)
- for f in replacements:
- old = f[:-4]
- if os.path.exists(old):
- os.remove(f)
- else:
- os.rename(f, old)
-
- def _encode(self, k):
- """
- Encode a key so it can be used as a filename.
- """
- # NOTE: '_' is NOT in the base64 alphabet!
- return base64.encodebytes(k).replace(b"\n", b"_").replace(b"/", b"-")
-
- def _decode(self, k):
- """
- Decode a filename to get the key.
- """
- return base64.decodebytes(k.replace(b"_", b"\n").replace(b"-", b"/"))
-
- def _readFile(self, path):
- """
- Read in the contents of a file.
-
- Override in subclasses to e.g. provide transparently encrypted dirdbm.
- """
- with _open(path.path, "rb") as f:
- s = f.read()
- return s
-
- def _writeFile(self, path, data):
- """
- Write data to a file.
-
- Override in subclasses to e.g. provide transparently encrypted dirdbm.
- """
- with _open(path.path, "wb") as f:
- f.write(data)
- f.flush()
-
- def __len__(self):
- """
- @return: The number of key/value pairs in this Shelf
- """
- return len(self._dnamePath.listdir())
-
- def __setitem__(self, k, v):
- """
- C{dirdbm[k] = v}
- Create or modify a textfile in this directory
-
- @type k: bytes
- @param k: key to set
-
- @type v: bytes
- @param v: value to associate with C{k}
- """
- if not type(k) == bytes:
- raise TypeError("DirDBM key must be bytes")
- if not type(v) == bytes:
- raise TypeError("DirDBM value must be bytes")
- k = self._encode(k)
-
- # We create a new file with extension .new, write the data to it, and
- # if the write succeeds delete the old file and rename the new one.
- old = self._dnamePath.child(k)
- if old.exists():
- new = old.siblingExtension(".rpl") # Replacement entry
- else:
- new = old.siblingExtension(".new") # New entry
- try:
- self._writeFile(new, v)
- except BaseException:
- new.remove()
- raise
- else:
- if old.exists():
- old.remove()
- new.moveTo(old)
-
- def __getitem__(self, k):
- """
- C{dirdbm[k]}
- Get the contents of a file in this directory as a string.
-
- @type k: bytes
- @param k: key to lookup
-
- @return: The value associated with C{k}
- @raise KeyError: Raised when there is no such key
- """
- if not type(k) == bytes:
- raise TypeError("DirDBM key must be bytes")
- path = self._dnamePath.child(self._encode(k))
- try:
- return self._readFile(path)
- except (OSError):
- raise KeyError(k)
-
- def __delitem__(self, k):
- """
- C{del dirdbm[foo]}
- Delete a file in this directory.
-
- @type k: bytes
- @param k: key to delete
-
- @raise KeyError: Raised when there is no such key
- """
- if not type(k) == bytes:
- raise TypeError("DirDBM key must be bytes")
- k = self._encode(k)
- try:
- self._dnamePath.child(k).remove()
- except (OSError):
- raise KeyError(self._decode(k))
-
- def keys(self):
- """
- @return: a L{list} of filenames (keys).
- """
- return list(map(self._decode, self._dnamePath.asBytesMode().listdir()))
-
- def values(self):
- """
- @return: a L{list} of file-contents (values).
- """
- vals = []
- keys = self.keys()
- for key in keys:
- vals.append(self[key])
- return vals
-
- def items(self):
- """
- @return: a L{list} of 2-tuples containing key/value pairs.
- """
- items = []
- keys = self.keys()
- for key in keys:
- items.append((key, self[key]))
- return items
-
- def has_key(self, key):
- """
- @type key: bytes
- @param key: The key to test
-
- @return: A true value if this dirdbm has the specified key, a false
- value otherwise.
- """
- if not type(key) == bytes:
- raise TypeError("DirDBM key must be bytes")
- key = self._encode(key)
- return self._dnamePath.child(key).isfile()
-
- def setdefault(self, key, value):
- """
- @type key: bytes
- @param key: The key to lookup
-
- @param value: The value to associate with key if key is not already
- associated with a value.
- """
- if key not in self:
- self[key] = value
- return value
- return self[key]
-
- def get(self, key, default=None):
- """
- @type key: bytes
- @param key: The key to lookup
-
- @param default: The value to return if the given key does not exist
-
- @return: The value associated with C{key} or C{default} if not
- L{DirDBM.has_key(key)}
- """
- if key in self:
- return self[key]
- else:
- return default
-
- def __contains__(self, key):
- """
- @see: L{DirDBM.has_key}
- """
- return self.has_key(key)
-
- def update(self, dict):
- """
- Add all the key/value pairs in L{dict} to this dirdbm. Any conflicting
- keys will be overwritten with the values from L{dict}.
-
- @type dict: mapping
- @param dict: A mapping of key/value pairs to add to this dirdbm.
- """
- for key, val in dict.items():
- self[key] = val
-
- def copyTo(self, path):
- """
- Copy the contents of this dirdbm to the dirdbm at C{path}.
-
- @type path: L{str}
- @param path: The path of the dirdbm to copy to. If a dirdbm
- exists at the destination path, it is cleared first.
-
- @rtype: C{DirDBM}
- @return: The dirdbm this dirdbm was copied to.
- """
- path = FilePath(path)
- assert path != self._dnamePath
-
- d = self.__class__(path.path)
- d.clear()
- for k in self.keys():
- d[k] = self[k]
- return d
-
- def clear(self):
- """
- Delete all key/value pairs in this dirdbm.
- """
- for k in self.keys():
- del self[k]
-
- def close(self):
- """
- Close this dbm: no-op, for dbm-style interface compliance.
- """
-
- def getModificationTime(self, key):
- """
- Returns modification time of an entry.
-
- @return: Last modification date (seconds since epoch) of entry C{key}
- @raise KeyError: Raised when there is no such key
- """
- if not type(key) == bytes:
- raise TypeError("DirDBM key must be bytes")
- path = self._dnamePath.child(self._encode(key))
- if path.isfile():
- return path.getModificationTime()
- else:
- raise KeyError(key)
-
-
- class Shelf(DirDBM):
- """
- A directory with a DBM shelf interface.
-
- This class presents a hash-like interface to a directory of small,
- flat files. Keys must be strings, but values can be any given object.
- """
-
- def __setitem__(self, k, v):
- """
- C{shelf[foo] = bar}
- Create or modify a textfile in this directory.
-
- @type k: str
- @param k: The key to set
-
- @param v: The value to associate with C{key}
- """
- v = pickle.dumps(v)
- DirDBM.__setitem__(self, k, v)
-
- def __getitem__(self, k):
- """
- C{dirdbm[foo]}
- Get and unpickle the contents of a file in this directory.
-
- @type k: bytes
- @param k: The key to lookup
-
- @return: The value associated with the given key
- @raise KeyError: Raised if the given key does not exist
- """
- return pickle.loads(DirDBM.__getitem__(self, k))
-
-
- def open(file, flag=None, mode=None):
- """
- This is for 'anydbm' compatibility.
-
- @param file: The parameter to pass to the DirDBM constructor.
-
- @param flag: ignored
- @param mode: ignored
- """
- return DirDBM(file)
-
-
- __all__ = ["open", "DirDBM", "Shelf"]
|