[Zodb-checkins] CVS: Packages/bsddb3Storage - Packless.py:1.5 PacklessReadme.txt:1.3 base.py:1.5
chrism@serenade.digicool.com
chrism@serenade.digicool.com
Tue, 27 Mar 2001 16:26:17 -0500
Update of /cvs-repository/Packages/bsddb3Storage
In directory serenade.digicool.com:/home/chrism/bsddb3Storage
Modified Files:
Packless.py PacklessReadme.txt base.py
Log Message:
Merge from PacklessBerkeley3Storage branch.
--- Updated File Packless.py in package Packages/bsddb3Storage --
--- Packless.py 2001/02/10 03:20:58 1.4
+++ Packless.py 2001/03/27 21:26:16 1.5
@@ -90,7 +90,7 @@
__version__ ='$Revision$'[11:-2]
-from base import Base
+from base import Base, DBError
from base import BerkeleyDBError
from bsddb3 import db
from struct import pack, unpack
@@ -129,6 +129,9 @@
"""
return 'current', 'referenceCount', 'oreferences', 'opickle'
+ def _abort(self):
+ pass
+
def load(self, oid, version):
self._lock_acquire()
try:
@@ -136,7 +139,7 @@
s=self._index[oid]
p=self._opickle[oid]
return p, s # pickle, serial
- except db.error, msg:
+ except DBError, msg:
raise BerkeleyDBError, (
"%s (%s)" % (BerkeleyDBError.__doc__, msg)
)
@@ -200,8 +203,8 @@
referenced=references.has_key
# Create refcnt
- if not referenceCount_get(oid, txn):
- referenceCount_put(oid, '\0'*intlen, txn)
+ if not referenceCount_get(oid, txn=txn):
+ referenceCount_put(oid, '\0'*intlen, txn=txn)
# zeros[oid]=1
# ^^^^^^^^^^^^
# this should happen when ZODB import is fixed
@@ -228,7 +231,7 @@
c.delete()
# decrement refcnt:
rc=unpack(">i",
- referenceCount_get(roid, txn))[0]
+ referenceCount_get(roid,txn=txn))[0]
rc=rc-1
if rc < 0:
# This should never happen
@@ -248,7 +251,7 @@
oreferences_put(oid, roid, txn)
# Create/update refcnt
- rcs=referenceCount_get(roid, txn)
+ rcs=referenceCount_get(roid, txn=txn)
if rcs:
rc=unpack(">i", rcs)[0]
if rc==0:
@@ -275,7 +278,7 @@
tmp.seek(0)
if fsize > MAXTEMPFSIZE: tmp.truncate()
- except db.error, msg:
+ except DBError, msg:
try:
txn.abort()
except db.error, msg:
@@ -310,7 +313,7 @@
roid=roid[1]
# decrement refcnt:
- rc=referenceCount_get(roid, txn)
+ rc=referenceCount_get(roid, txn=txn)
if rc:
rc=unpack(">i", rc)[0]-1
if rc < 0:
--- Updated File PacklessReadme.txt in package Packages/bsddb3Storage --
--- PacklessReadme.txt 2001/02/10 03:21:36 1.2
+++ PacklessReadme.txt 2001/03/27 21:26:16 1.3
@@ -1,4 +1,4 @@
-Packless bsddb3 (Berkeley) Storage
+Packless bsddb3 (Berkeley) Storage 1.0b1
Introduction
@@ -31,11 +31,11 @@
Obtain the latest source release of Sleepycat's BerkeleyDB 3.X
database product (http://www.sleepycat.com). As of this writing,
- the latest BerkeleyDB release is 3.1.17. Before using BerkeleyDB,
+ the latest BerkeleyDB release is 3.2.9. Before using BerkeleyDB,
make sure that you comply with its licensing requirements
(http://www.sleepycat.com/licensing.html).
- Obtain the latest source release of pybsddb
+ Obtain the latest source release of Robin Dunn's pybsddb module
(http://pybsddb.sourceforge.net).
Install both BerkeleyDB and pybsddb as per the instructions which
@@ -48,11 +48,44 @@
When you can run the tests which ship with pybsddb, you'll know
you've been successful at both BerkeleyDB and pybsddb installation.
+ A common installation of both BerkeleyDB and pybsddb on Red Hat Linux
+ would go something like:
+
+ Download BerkeleyDB 3.2.9 from
+ http://www.sleepycat.com/update/3.2.9/db-3.2.9.tar.gz
+
+ Untar and ungzip db-3.2.9.tar.gz
+
+ cd to the db-3.2.9/build_unix directory
+
+ type "../dist/configure"
+
+ type "make; make_install"
+
+ Download Python Distutils 1.0.1 (rpm version) from
+ http://www.python.org/sigs/distutils-sig/download.html (use the
+ source version for other Linux/Unix distributions)
+
+ type "rpm -Uvh Distutils-1.0.1-1.noarch.rpm"
+
+ Download pybsddb 3.0 from
+ http://ftp1.sourceforge.net/pybsddb/bsddb3-3.0.tar.gz
+
+ Untar and ungzip bsddb3-3.0.tar.gz
+
+ cd to the bsddb3-3.0 directory
+
+ type "python setup.py install"
+
+ To test the bsddb module, cd to the "test" directory inside
+ bsddb3-3.0 directory and type "python test_all.py" to test the
+ installation (make sure all tests pass).
+
Installing The Storage Product
- After you've got BerkeleyDB and pybsddb installed, you'll want to
- untar the PacklessBerkeleyStorage distribution into your Zope's
- "Products" directory. Generally, this is
+ After you've got BerkeleyDB and pybsddb installed and working,
+ you'll want to untar the PacklessBerkeleyStorage distribution into
+ your Zope's "Products" directory. Generally, this is
"$SOFTWARE_HOME/lib/python/Products". For example, if your Zope is
installed into "/home/chrism/Zope2", you'll want to untar the
distribution file while you're in the
@@ -63,7 +96,7 @@
example.
Once you've "untarred" the product, you should have a subdirectory
- named "bsddb3Storage" with this file and a number of
+ named "PacklessBerkeley3Storage" with this file and a number of
other Python and documentation files in it.
Creating a custom_zodb.py
@@ -145,6 +178,42 @@
advantageous to put your log files and your database files on
separate devices, see
http://www.sleepycat.com/docs/ref/transapp/reclimit.html.
+
+Setting BerkeleyDB Maximum Locks
+
+ ZODB transactions can be of almost arbitrary sizes (actually, they
+ "top out" at a total size of 2GB). BerkeleyDB is configured to use
+ 500 locks by default. Larger transactions in BerkeleyDB require
+ more locks. Thus, Packless ships with the default number of
+ Berkeley locks set to 10,000. This should allow almost any Zope
+ transaction to commit at the expense of increased RAM consumption.
+ Utilizing 10,000 locks requires (at least on Linux systems)
+ approximately 3MB of RAM overhead, perhaps little of which is being
+ actually used in environments which do not commit large
+ transactions. You can reduce RAM consumption by manually sizing
+ BerkeleyDB locking.
+
+ To manually size locking, create (or edit) the file 'DB_CONFIG'
+ within the BerkeleyStorage "environment" directory you've chosen in
+ 'custom_zodb.py', adding the following directives to the 'DB_CONFIG'
+ file::
+
+ set_lk_max_locks 500
+ set_lk_max_objects 500
+ set_lk_max_lockers 3
+
+ Change the integers as necessary. When Packless starts up, the
+ Berkeley directives supplied in DB_CONFIG will override the defaults.
+
+ Precision-sizing BerkeleyDB locking is a site-dependent task.
+ Sleepycat recommends that you run the "db_stat -c" command against
+ the database environment to see what the "max number of locks, lock
+ objects and lockers so far" numbers are during highly stressful
+ operations, multiply each of those numbers by 2, and provide the
+ multiplied-by-2 numbers as arguments to set_lk_max_locks,
+ set_lk_max_objects, and set_lk_max_lockers respectively in
+ DB_CONFIG. For detailed BerkeleyDB locking sizing strategy, see
+ http://www.sleepycat.com/docs/ref/lock/max.html.
Archival and Maintenance
--- Updated File base.py in package Packages/bsddb3Storage --
--- base.py 2001/02/10 03:21:52 1.4
+++ base.py 2001/03/27 21:26:16 1.5
@@ -90,13 +90,23 @@
from ZODB import POSException
from bsddb3 import db
import os, tempfile
-
+try:
+ DB = db.DB
+ DBEnv = db.DBEnv
+ DBError = db.DBError
+except:
+ DB = db.Db
+ DBEnv = db.DbEnv
+ DBError = db.error
+
class BerkeleyDBError(POSException.POSError):
""" A BerkeleyDB exception occurred. This probably indicates that
there is a low memory condition, a tempfile space shortage, or
a space shortage in the directory which houses the BerkeleyDB log
files. Check available tempfile space, logfile space, and RAM and
- restart the server process."""
+ restart the server process. This error could have additionally
+ been caused by too few locks available to BerkeleyDB for the
+ transaction size you were attempting to commit."""
class Base(BaseStorage):
@@ -118,7 +128,7 @@
def _setupDB(self, name, flags=0):
"""Open an individual database and assign to an "_" attribute.
"""
- d=db.Db(self._env)
+ d=DB(self._env)
if flags: d.set_flags(flags)
d.open(self._prefix+name, db.DB_BTREE, db.DB_CREATE)
setattr(self, '_'+name, d)
@@ -152,12 +162,6 @@
# TBD
return 0
- def _finish(self, tid, user, desc, ext):
- self._txn.commit()
-
- def _abort(self, tid, user, desc, ext):
- self._txn.abort()
-
def _clear_temp(self):
self._tmp.seek(0)
@@ -182,12 +186,13 @@
if not os.path.exists(name): os.mkdir(name)
except:
raise "Error creating BerkeleyDB environment dir: %s" % name
- e=db.DbEnv()
+ e=DBEnv()
+ e.set_lk_max(10000) # this can be overridden in the DB_CONFIG file
try:
e.open(name,
db.DB_CREATE | db.DB_RECOVER
| db.DB_INIT_MPOOL | db.DB_INIT_LOCK | db.DB_INIT_TXN
)
- except db.error, msg:
+ except DBError, msg:
raise BerkeleyDBError, "%s (%s)" % (BerkeleyDBError.__doc__, msg)
return e