1 #----------------------------------------------------------------------

     2 #  Copyright (c) 1999-2001, Digital Creations, Fredericksburg, VA, USA

     3 #  and Andrew Kuchling. All rights reserved.

     4 #

     5 #  Redistribution and use in source and binary forms, with or without

     6 #  modification, are permitted provided that the following conditions are

     7 #  met:

     8 #

     9 #    o Redistributions of source code must retain the above copyright

    10 #      notice, this list of conditions, and the disclaimer that follows.

    11 #

    12 #    o Redistributions in binary form must reproduce the above copyright

    13 #      notice, this list of conditions, and the following disclaimer in

    14 #      the documentation and/or other materials provided with the

    15 #      distribution.

    16 #

    17 #    o Neither the name of Digital Creations nor the names of its

    18 #      contributors may be used to endorse or promote products derived

    19 #      from this software without specific prior written permission.

    20 #

    21 #  THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS AND CONTRIBUTORS *AS

    22 #  IS* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED

    23 #  TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A

    24 #  PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DIGITAL

    25 #  CREATIONS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,

    26 #  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,

    27 #  BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS

    28 #  OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND

    29 #  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR

    30 #  TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE

    31 #  USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH

    32 #  DAMAGE.

    33 #----------------------------------------------------------------------

    34 

    35 

    36 """Support for Berkeley DB 4.0 through 4.7 with a simple interface.

    37 

    38 For the full featured object oriented interface use the bsddb.db module

    39 instead.  It mirrors the Oracle Berkeley DB C API.

    40 """

    41 

    42 import sys

    43 absolute_import = (sys.version_info[0] >= 3)

    44 

    45 if sys.py3kwarning:

    46     import warnings

    47     warnings.warnpy3k("in 3.x, bsddb has been removed; "

    48                       "please use the pybsddb project instead",

    49                       DeprecationWarning, 2)

    50 

    51 try:

    52     if __name__ == 'bsddb3':

    53         # import _pybsddb binary as it should be the more recent version from

    54         # a standalone pybsddb addon package than the version included with

    55         # python as bsddb._bsddb.

    56         if absolute_import :

    57             # Because this syntaxis is not valid before Python 2.5

    58             exec("from . import _pybsddb")

    59         else :

    60             import _pybsddb

    61         _bsddb = _pybsddb

    62         from bsddb3.dbutils import DeadlockWrap as _DeadlockWrap

    63     else:

    64         import _bsddb

    65         from bsddb.dbutils import DeadlockWrap as _DeadlockWrap

    66 except ImportError:

    67     # Remove ourselves from sys.modules

    68     import sys

    69     del sys.modules[__name__]

    70     raise

    71 

    72 # bsddb3 calls it db, but provide _db for backwards compatibility

    73 db = _db = _bsddb

    74 __version__ = db.__version__

    75 

    76 error = db.DBError  # So bsddb.error will mean something...

    77 

    78 #----------------------------------------------------------------------

    79 

    80 import sys, os

    81 

    82 from weakref import ref

    83 

    84 if sys.version_info[0:2] <= (2, 5) :

    85     import UserDict

    86     MutableMapping = UserDict.DictMixin

    87 else :

    88     import collections

    89     MutableMapping = collections.MutableMapping

    90 

    91 class _iter_mixin(MutableMapping):

    92     def _make_iter_cursor(self):

    93         cur = _DeadlockWrap(self.db.cursor)

    94         key = id(cur)

    95         self._cursor_refs[key] = ref(cur, self._gen_cref_cleaner(key))

    96         return cur

    97 

    98     def _gen_cref_cleaner(self, key):

    99         # use generate the function for the weakref callback here

   100         # to ensure that we do not hold a strict reference to cur

   101         # in the callback.

   102         return lambda ref: self._cursor_refs.pop(key, None)

   103 

   104     def __iter__(self):

   105         self._kill_iteration = False

   106         self._in_iter += 1

   107         try:

   108             try:

   109                 cur = self._make_iter_cursor()

   110 

   111                 # FIXME-20031102-greg: race condition.  cursor could

   112                 # be closed by another thread before this call.

   113 

   114                 # since we're only returning keys, we call the cursor

   115                 # methods with flags=0, dlen=0, dofs=0

   116                 key = _DeadlockWrap(cur.first, 0,0,0)[0]

   117                 yield key

   118 

   119                 next = getattr(cur, "next")

   120                 while 1:

   121                     try:

   122                         key = _DeadlockWrap(next, 0,0,0)[0]

   123                         yield key

   124                     except _bsddb.DBCursorClosedError:

   125                         if self._kill_iteration:

   126                             raise RuntimeError('Database changed size '

   127                                                'during iteration.')

   128                         cur = self._make_iter_cursor()

   129                         # FIXME-20031101-greg: race condition.  cursor could

   130                         # be closed by another thread before this call.

   131                         _DeadlockWrap(cur.set, key,0,0,0)

   132                         next = getattr(cur, "next")

   133             except _bsddb.DBNotFoundError:

   134                 pass

   135             except _bsddb.DBCursorClosedError:

   136                 # the database was modified during iteration.  abort.

   137                 pass

   138 # When Python 2.3 not supported in bsddb3, we can change this to "finally"

   139         except :

   140             self._in_iter -= 1

   141             raise

   142 

   143         self._in_iter -= 1

   144 

   145     def iteritems(self):

   146         if not self.db:

   147             return

   148         self._kill_iteration = False

   149         self._in_iter += 1

   150         try:

   151             try:

   152                 cur = self._make_iter_cursor()

   153 

   154                 # FIXME-20031102-greg: race condition.  cursor could

   155                 # be closed by another thread before this call.

   156 

   157                 kv = _DeadlockWrap(cur.first)

   158                 key = kv[0]

   159                 yield kv

   160 

   161                 next = getattr(cur, "next")

   162                 while 1:

   163                     try:

   164                         kv = _DeadlockWrap(next)

   165                         key = kv[0]

   166                         yield kv

   167                     except _bsddb.DBCursorClosedError:

   168                         if self._kill_iteration:

   169                             raise RuntimeError('Database changed size '

   170                                                'during iteration.')

   171                         cur = self._make_iter_cursor()

   172                         # FIXME-20031101-greg: race condition.  cursor could

   173                         # be closed by another thread before this call.

   174                         _DeadlockWrap(cur.set, key,0,0,0)

   175                         next = getattr(cur, "next")

   176             except _bsddb.DBNotFoundError:

   177                 pass

   178             except _bsddb.DBCursorClosedError:

   179                 # the database was modified during iteration.  abort.

   180                 pass

   181 # When Python 2.3 not supported in bsddb3, we can change this to "finally"

   182         except :

   183             self._in_iter -= 1

   184             raise

   185 

   186         self._in_iter -= 1

   187 

   188 

   189 class _DBWithCursor(_iter_mixin):

   190     """

   191     A simple wrapper around DB that makes it look like the bsddbobject in

   192     the old module.  It uses a cursor as needed to provide DB traversal.

   193     """

   194     def __init__(self, db):

   195         self.db = db

   196         self.db.set_get_returns_none(0)

   197 

   198         # FIXME-20031101-greg: I believe there is still the potential

   199         # for deadlocks in a multithreaded environment if someone

   200         # attempts to use the any of the cursor interfaces in one

   201         # thread while doing a put or delete in another thread.  The

   202         # reason is that _checkCursor and _closeCursors are not atomic

   203         # operations.  Doing our own locking around self.dbc,

   204         # self.saved_dbc_key and self._cursor_refs could prevent this.

   205         # TODO: A test case demonstrating the problem needs to be written.

   206 

   207         # self.dbc is a DBCursor object used to implement the

   208         # first/next/previous/last/set_location methods.

   209         self.dbc = None

   210         self.saved_dbc_key = None

   211 

   212         # a collection of all DBCursor objects currently allocated

   213         # by the _iter_mixin interface.

   214         self._cursor_refs = {}

   215         self._in_iter = 0

   216         self._kill_iteration = False

   217 

   218     def __del__(self):

   219         self.close()

   220 

   221     def _checkCursor(self):

   222         if self.dbc is None:

   223             self.dbc = _DeadlockWrap(self.db.cursor)

   224             if self.saved_dbc_key is not None:

   225                 _DeadlockWrap(self.dbc.set, self.saved_dbc_key)

   226                 self.saved_dbc_key = None

   227 

   228     # This method is needed for all non-cursor DB calls to avoid

   229     # Berkeley DB deadlocks (due to being opened with DB_INIT_LOCK

   230     # and DB_THREAD to be thread safe) when intermixing database

   231     # operations that use the cursor internally with those that don't.

   232     def _closeCursors(self, save=1):

   233         if self.dbc:

   234             c = self.dbc

   235             self.dbc = None

   236             if save:

   237                 try:

   238                     self.saved_dbc_key = _DeadlockWrap(c.current, 0,0,0)[0]

   239                 except db.DBError:

   240                     pass

   241             _DeadlockWrap(c.close)

   242             del c

   243         for cref in self._cursor_refs.values():

   244             c = cref()

   245             if c is not None:

   246                 _DeadlockWrap(c.close)

   247 

   248     def _checkOpen(self):

   249         if self.db is None:

   250             raise error, "BSDDB object has already been closed"

   251 

   252     def isOpen(self):

   253         return self.db is not None

   254 

   255     def __len__(self):

   256         self._checkOpen()

   257         return _DeadlockWrap(lambda: len(self.db))  # len(self.db)

   258 

   259     if sys.version_info[0:2] >= (2, 6) :

   260         def __repr__(self) :

   261             if self.isOpen() :

   262                 return repr(dict(_DeadlockWrap(self.db.items)))

   263             return repr(dict())

   264 

   265     def __getitem__(self, key):

   266         self._checkOpen()

   267         return _DeadlockWrap(lambda: self.db[key])  # self.db[key]

   268 

   269     def __setitem__(self, key, value):

   270         self._checkOpen()

   271         self._closeCursors()

   272         if self._in_iter and key not in self:

   273             self._kill_iteration = True

   274         def wrapF():

   275             self.db[key] = value

   276         _DeadlockWrap(wrapF)  # self.db[key] = value

   277 

   278     def __delitem__(self, key):

   279         self._checkOpen()

   280         self._closeCursors()

   281         if self._in_iter and key in self:

   282             self._kill_iteration = True

   283         def wrapF():

   284             del self.db[key]

   285         _DeadlockWrap(wrapF)  # del self.db[key]

   286 

   287     def close(self):

   288         self._closeCursors(save=0)

   289         if self.dbc is not None:

   290             _DeadlockWrap(self.dbc.close)

   291         v = 0

   292         if self.db is not None:

   293             v = _DeadlockWrap(self.db.close)

   294         self.dbc = None

   295         self.db = None

   296         return v

   297 

   298     def keys(self):

   299         self._checkOpen()

   300         return _DeadlockWrap(self.db.keys)

   301 

   302     def has_key(self, key):

   303         self._checkOpen()

   304         return _DeadlockWrap(self.db.has_key, key)

   305 

   306     def set_location(self, key):

   307         self._checkOpen()

   308         self._checkCursor()

   309         return _DeadlockWrap(self.dbc.set_range, key)

   310 

   311     def next(self):  # Renamed by "2to3"

   312         self._checkOpen()

   313         self._checkCursor()

   314         rv = _DeadlockWrap(getattr(self.dbc, "next"))

   315         return rv

   316 

   317     if sys.version_info[0] >= 3 :  # For "2to3" conversion

   318         next = __next__

   319 

   320     def previous(self):

   321         self._checkOpen()

   322         self._checkCursor()

   323         rv = _DeadlockWrap(self.dbc.prev)

   324         return rv

   325 

   326     def first(self):

   327         self._checkOpen()

   328         # fix 1725856: don't needlessly try to restore our cursor position

   329         self.saved_dbc_key = None

   330         self._checkCursor()

   331         rv = _DeadlockWrap(self.dbc.first)

   332         return rv

   333 

   334     def last(self):

   335         self._checkOpen()

   336         # fix 1725856: don't needlessly try to restore our cursor position

   337         self.saved_dbc_key = None

   338         self._checkCursor()

   339         rv = _DeadlockWrap(self.dbc.last)

   340         return rv

   341 

   342     def sync(self):

   343         self._checkOpen()

   344         return _DeadlockWrap(self.db.sync)

   345 

   346 

   347 #----------------------------------------------------------------------

   348 # Compatibility object factory functions

   349 

   350 def hashopen(file, flag='c', mode=0666, pgsize=None, ffactor=None, nelem=None,

   351             cachesize=None, lorder=None, hflags=0):

   352 

   353     flags = _checkflag(flag, file)

   354     e = _openDBEnv(cachesize)

   355     d = db.DB(e)

   356     d.set_flags(hflags)

   357     if pgsize is not None:    d.set_pagesize(pgsize)

   358     if lorder is not None:    d.set_lorder(lorder)

   359     if ffactor is not None:   d.set_h_ffactor(ffactor)

   360     if nelem is not None:     d.set_h_nelem(nelem)

   361     d.open(file, db.DB_HASH, flags, mode)

   362     return _DBWithCursor(d)

   363 

   364 #----------------------------------------------------------------------

   365 

   366 def btopen(file, flag='c', mode=0666,

   367             btflags=0, cachesize=None, maxkeypage=None, minkeypage=None,

   368             pgsize=None, lorder=None):

   369 

   370     flags = _checkflag(flag, file)

   371     e = _openDBEnv(cachesize)

   372     d = db.DB(e)

   373     if pgsize is not None: d.set_pagesize(pgsize)

   374     if lorder is not None: d.set_lorder(lorder)

   375     d.set_flags(btflags)

   376     if minkeypage is not None: d.set_bt_minkey(minkeypage)

   377     if maxkeypage is not None: d.set_bt_maxkey(maxkeypage)

   378     d.open(file, db.DB_BTREE, flags, mode)

   379     return _DBWithCursor(d)

   380 

   381 #----------------------------------------------------------------------

   382 

   383 

   384 def rnopen(file, flag='c', mode=0666,

   385             rnflags=0, cachesize=None, pgsize=None, lorder=None,

   386             rlen=None, delim=None, source=None, pad=None):

   387 

   388     flags = _checkflag(flag, file)

   389     e = _openDBEnv(cachesize)

   390     d = db.DB(e)

   391     if pgsize is not None: d.set_pagesize(pgsize)

   392     if lorder is not None: d.set_lorder(lorder)

   393     d.set_flags(rnflags)

   394     if delim is not None: d.set_re_delim(delim)

   395     if rlen is not None: d.set_re_len(rlen)

   396     if source is not None: d.set_re_source(source)

   397     if pad is not None: d.set_re_pad(pad)

   398     d.open(file, db.DB_RECNO, flags, mode)

   399     return _DBWithCursor(d)

   400 

   401 #----------------------------------------------------------------------

   402 

   403 def _openDBEnv(cachesize):

   404     e = db.DBEnv()

   405     if cachesize is not None:

   406         if cachesize >= 20480:

   407             e.set_cachesize(0, cachesize)

   408         else:

   409             raise error, "cachesize must be >= 20480"

   410     e.set_lk_detect(db.DB_LOCK_DEFAULT)

   411     e.open('.', db.DB_PRIVATE | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_LOCK | db.DB_INIT_MPOOL)

   412     return e

   413 

   414 def _checkflag(flag, file):

   415     if flag == 'r':

   416         flags = db.DB_RDONLY

   417     elif flag == 'rw':

   418         flags = 0

   419     elif flag == 'w':

   420         flags =  db.DB_CREATE

   421     elif flag == 'c':

   422         flags =  db.DB_CREATE

   423     elif flag == 'n':

   424         flags = db.DB_CREATE

   425         #flags = db.DB_CREATE | db.DB_TRUNCATE

   426         # we used db.DB_TRUNCATE flag for this before but Berkeley DB

   427         # 4.2.52 changed to disallowed truncate with txn environments.

   428         if file is not None and os.path.isfile(file):

   429             os.unlink(file)

   430     else:

   431         raise error, "flags should be one of 'r', 'w', 'c' or 'n'"

   432     return flags | db.DB_THREAD

   433 

   434 #----------------------------------------------------------------------

   435 

   436 

   437 # This is a silly little hack that allows apps to continue to use the

   438 # DB_THREAD flag even on systems without threads without freaking out

   439 # Berkeley DB.

   440 #

   441 # This assumes that if Python was built with thread support then

   442 # Berkeley DB was too.

   443 

   444 try:

   445     import thread

   446     del thread

   447 except ImportError:

   448     db.DB_THREAD = 0

   449 

   450 #----------------------------------------------------------------------