0001 """Manage shelves of pickled objects. 0002 0003 A "shelf" is a persistent, dictionary-like object. The difference 0004 with dbm databases is that the values (not the keys!) in a shelf can 0005 be essentially arbitrary Python objects -- anything that the "pickle" 0006 module can handle. This includes most class instances, recursive data 0007 types, and objects containing lots of shared sub-objects. The keys 0008 are ordinary strings. 0009 0010 To summarize the interface (key is a string, data is an arbitrary 0011 object): 0012 0013 import shelve 0014 d = shelve.open(filename) # open, with (g)dbm filename -- no suffix 0015 0016 d[key] = data # store data at key (overwrites old data if 0017 # using an existing key) 0018 data = d[key] # retrieve a COPY of the data at key (raise 0019 # KeyError if no such key) -- NOTE that this 0020 # access returns a *copy* of the entry! 0021 del d[key] # delete data stored at key (raises KeyError 0022 # if no such key) 0023 flag = d.has_key(key) # true if the key exists; same as "key in d" 0024 list = d.keys() # a list of all existing keys (slow!) 0025 0026 d.close() # close it 0027 0028 Dependent on the implementation, closing a persistent dictionary may 0029 or may not be necessary to flush changes to disk. 0030 0031 Normally, d[key] returns a COPY of the entry. This needs care when 0032 mutable entries are mutated: for example, if d[key] is a list, 0033 d[key].append(anitem) 0034 does NOT modify the entry d[key] itself, as stored in the persistent 0035 mapping -- it only modifies the copy, which is then immediately 0036 discarded, so that the append has NO effect whatsoever. To append an 0037 item to d[key] in a way that will affect the persistent mapping, use: 0038 data = d[key] 0039 data.append(anitem) 0040 d[key] = data 0041 0042 To avoid the problem with mutable entries, you may pass the keyword 0043 argument writeback=True in the call to shelve.open. When you use: 0044 d = shelve.open(filename, writeback=True) 0045 then d keeps a cache of all entries you access, and writes them all back 0046 to the persistent mapping when you call d.close(). This ensures that 0047 such usage as d[key].append(anitem) works as intended. 0048 0049 However, using keyword argument writeback=True may consume vast amount 0050 of memory for the cache, and it may make d.close() very slow, if you 0051 access many of d's entries after opening it in this way: d has no way to 0052 check which of the entries you access are mutable and/or which ones you 0053 actually mutate, so it must cache, and write back at close, all of the 0054 entries that you access. You can call d.sync() to write back all the 0055 entries in the cache, and empty the cache (d.sync() also synchronizes 0056 the persistent dictionary on disk, if feasible). 0057 """ 0058 0059 # Try using cPickle and cStringIO if available. 0060 0061 try: 0062 from cPickle import Pickler, Unpickler 0063 except ImportError: 0064 from pickle import Pickler, Unpickler 0065 0066 try: 0067 from cStringIO import StringIO 0068 except ImportError: 0069 from StringIO import StringIO 0070 0071 import UserDict 0072 import warnings 0073 0074 __all__ = ["Shelf","BsdDbShelf","DbfilenameShelf","open"] 0075 0076 class Shelf(UserDict.DictMixin): 0077 """Base class for shelf implementations. 0078 0079 This is initialized with a dictionary-like object. 0080 See the module's __doc__ string for an overview of the interface. 0081 """ 0082 0083 def __init__(self, dict, protocol=None, writeback=False, binary=None): 0084 self.dict = dict 0085 if protocol is not None and binary is not None: 0086 raise ValueError, "can't specify both 'protocol' and 'binary'" 0087 if binary is not None: 0088 warnings.warn("The 'binary' argument to Shelf() is deprecated", 0089 PendingDeprecationWarning) 0090 protocol = int(binary) 0091 if protocol is None: 0092 protocol = 0 0093 self._protocol = protocol 0094 self.writeback = writeback 0095 self.cache = {} 0096 0097 def keys(self): 0098 return self.dict.keys() 0099 0100 def __len__(self): 0101 return len(self.dict) 0102 0103 def has_key(self, key): 0104 return self.dict.has_key(key) 0105 0106 def __contains__(self, key): 0107 return self.dict.has_key(key) 0108 0109 def get(self, key, default=None): 0110 if self.dict.has_key(key): 0111 return self[key] 0112 return default 0113 0114 def __getitem__(self, key): 0115 try: 0116 value = self.cache[key] 0117 except KeyError: 0118 f = StringIO(self.dict[key]) 0119 value = Unpickler(f).load() 0120 if self.writeback: 0121 self.cache[key] = value 0122 return value 0123 0124 def __setitem__(self, key, value): 0125 if self.writeback: 0126 self.cache[key] = value 0127 f = StringIO() 0128 p = Pickler(f, self._protocol) 0129 p.dump(value) 0130 self.dict[key] = f.getvalue() 0131 0132 def __delitem__(self, key): 0133 del self.dict[key] 0134 try: 0135 del self.cache[key] 0136 except KeyError: 0137 pass 0138 0139 def close(self): 0140 self.sync() 0141 try: 0142 self.dict.close() 0143 except AttributeError: 0144 pass 0145 self.dict = 0 0146 0147 def __del__(self): 0148 self.close() 0149 0150 def sync(self): 0151 if self.writeback and self.cache: 0152 self.writeback = False 0153 for key, entry in self.cache.iteritems(): 0154 self[key] = entry 0155 self.writeback = True 0156 self.cache = {} 0157 if hasattr(self.dict, 'sync'): 0158 self.dict.sync() 0159 0160 0161 class BsdDbShelf(Shelf): 0162 """Shelf implementation using the "BSD" db interface. 0163 0164 This adds methods first(), next(), previous(), last() and 0165 set_location() that have no counterpart in [g]dbm databases. 0166 0167 The actual database must be opened using one of the "bsddb" 0168 modules "open" routines (i.e. bsddb.hashopen, bsddb.btopen or 0169 bsddb.rnopen) and passed to the constructor. 0170 0171 See the module's __doc__ string for an overview of the interface. 0172 """ 0173 0174 def __init__(self, dict, protocol=None, writeback=False, binary=None): 0175 Shelf.__init__(self, dict, protocol, writeback, binary) 0176 0177 def set_location(self, key): 0178 (key, value) = self.dict.set_location(key) 0179 f = StringIO(value) 0180 return (key, Unpickler(f).load()) 0181 0182 def next(self): 0183 (key, value) = self.dict.next() 0184 f = StringIO(value) 0185 return (key, Unpickler(f).load()) 0186 0187 def previous(self): 0188 (key, value) = self.dict.previous() 0189 f = StringIO(value) 0190 return (key, Unpickler(f).load()) 0191 0192 def first(self): 0193 (key, value) = self.dict.first() 0194 f = StringIO(value) 0195 return (key, Unpickler(f).load()) 0196 0197 def last(self): 0198 (key, value) = self.dict.last() 0199 f = StringIO(value) 0200 return (key, Unpickler(f).load()) 0201 0202 0203 class DbfilenameShelf(Shelf): 0204 """Shelf implementation using the "anydbm" generic dbm interface. 0205 0206 This is initialized with the filename for the dbm database. 0207 See the module's __doc__ string for an overview of the interface. 0208 """ 0209 0210 def __init__(self, filename, flag='c', protocol=None, writeback=False, binary=None): 0211 import anydbm 0212 Shelf.__init__(self, anydbm.open(filename, flag), protocol, writeback, binary) 0213 0214 0215 def open(filename, flag='c', protocol=None, writeback=False, binary=None): 0216 """Open a persistent dictionary for reading and writing. 0217 0218 The filename parameter is the base filename for the underlying 0219 database. As a side-effect, an extension may be added to the 0220 filename and more than one file may be created. The optional flag 0221 parameter has the same interpretation as the flag parameter of 0222 anydbm.open(). The optional protocol parameter specifies the 0223 version of the pickle protocol (0, 1, or 2). 0224 0225 The optional binary parameter is deprecated and may be set to True 0226 to force the use of binary pickles for serializing data values. 0227 0228 See the module's __doc__ string for an overview of the interface. 0229 """ 0230 0231 return DbfilenameShelf(filename, flag, protocol, writeback, binary) 0232
Generated by PyXR 0.9.4