[Zodb-checkins]
SVN: ZODB/branches/hannosch-pickle-protocol-version2/src/
Turned references to the pickle protocol version into a
simple constant for now,
to easily set it to version 2. With the constant set to 1 all tests
pass,
with 2 we have five test failures. Most of those failures look like
tests for explicit sizes which might have changed without
leading to any real problems.
Hanno Schlichting
plone at hannosch.info
Wed Dec 12 21:37:39 EST 2007
Log message for revision 82272:
Turned references to the pickle protocol version into a simple constant for now, to easily set it to version 2. With the constant set to 1 all tests pass, with 2 we have five test failures. Most of those failures look like tests for explicit sizes which might have changed without leading to any real problems.
Changed:
U ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/ClientStorage.py
U ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/CommitLog.py
U ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/TransactionBuffer.py
U ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/scripts/zeoqueue.py
U ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/zrpc/marshal.py
U ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/BaseStorage.py
U ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/ConflictResolution.py
U ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/DB.py
U ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/ExportImport.py
U ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/FileStorage/FileStorage.py
U ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/broken.py
U ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/config.py
U ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/serialize.py
U ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/tests/PackableStorage.py
U ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/tests/StorageTestBase.py
U ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/tests/testFileStorage.py
U ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/tests/testSerialize.py
U ZODB/branches/hannosch-pickle-protocol-version2/src/persistent/tests/testPersistent.py
-=-
Modified: ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/ClientStorage.py
===================================================================
--- ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/ClientStorage.py 2007-12-13 02:34:39 UTC (rev 82271)
+++ ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/ClientStorage.py 2007-12-13 02:37:39 UTC (rev 82272)
@@ -40,6 +40,7 @@
import ZODB.lock_file
from ZODB import POSException
from ZODB import utils
+from ZODB.config import PICKLE_PROTOCOL_VERSION
from ZODB.loglevels import BLATHER
from ZODB.interfaces import IBlobStorage
from ZODB.blob import rename_or_copy_blob
@@ -598,7 +599,7 @@
log2("Verifying cache")
# setup tempfile to hold zeoVerify results
self._tfile = tempfile.TemporaryFile(suffix=".inv")
- self._pickler = cPickle.Pickler(self._tfile, 1)
+ self._pickler = cPickle.Pickler(self._tfile, PICKLE_PROTOCOL_VERSION)
self._pickler.fast = 1 # Don't use the memo
# TODO: should batch these operations for efficiency; would need
Modified: ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/CommitLog.py
===================================================================
--- ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/CommitLog.py 2007-12-13 02:34:39 UTC (rev 82271)
+++ ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/CommitLog.py 2007-12-13 02:37:39 UTC (rev 82272)
@@ -22,11 +22,13 @@
import cPickle
import tempfile
+from ZODB.config import PICKLE_PROTOCOL_VERSION
+
class CommitLog:
def __init__(self):
self.file = tempfile.TemporaryFile(suffix=".log")
- self.pickler = cPickle.Pickler(self.file, 1)
+ self.pickler = cPickle.Pickler(self.file, PICKLE_PROTOCOL_VERSION)
self.pickler.fast = 1
self.stores = 0
self.read = 0
Modified: ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/TransactionBuffer.py
===================================================================
--- ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/TransactionBuffer.py 2007-12-13 02:34:39 UTC (rev 82271)
+++ ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/TransactionBuffer.py 2007-12-13 02:37:39 UTC (rev 82272)
@@ -25,6 +25,8 @@
import tempfile
from threading import Lock
+from ZODB.config import PICKLE_PROTOCOL_VERSION
+
class TransactionBuffer:
# Valid call sequences:
@@ -62,7 +64,7 @@
self.blobs = []
# It's safe to use a fast pickler because the only objects
# stored are builtin types -- strings or None.
- self.pickler = cPickle.Pickler(self.file, 1)
+ self.pickler = cPickle.Pickler(self.file, PICKLE_PROTOCOL_VERSION)
self.pickler.fast = 1
def close(self):
Modified: ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/scripts/zeoqueue.py
===================================================================
--- ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/scripts/zeoqueue.py 2007-12-13 02:34:39 UTC (rev 82271)
+++ ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/scripts/zeoqueue.py 2007-12-13 02:37:39 UTC (rev 82272)
@@ -31,6 +31,8 @@
import getopt
import cPickle as pickle
+from ZODB.config import PICKLE_PROTOCOL_VERSION
+
COMMASPACE = ', '
STATEFILE = 'zeoqueue.pck'
PROGRAM = sys.argv[0]
@@ -389,7 +391,7 @@
fp.close()
# Save state
statefp = open(file, 'wb')
- pickle.dump(status, statefp, 1)
+ pickle.dump(status, statefp, PICKLE_PROTOCOL_VERSION)
statefp.close()
# Print the report and return the number of blocked clients in the exit
# status code.
Modified: ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/zrpc/marshal.py
===================================================================
--- ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/zrpc/marshal.py 2007-12-13 02:34:39 UTC (rev 82271)
+++ ZODB/branches/hannosch-pickle-protocol-version2/src/ZEO/zrpc/marshal.py 2007-12-13 02:37:39 UTC (rev 82272)
@@ -15,6 +15,8 @@
from cStringIO import StringIO
import logging
+from ZODB.config import PICKLE_PROTOCOL_VERSION
+
from ZEO.zrpc.error import ZRPCError
from ZEO.zrpc.log import log, short_repr
@@ -31,7 +33,7 @@
# being represented by \xij escapes in proto 0).
# Undocumented: cPickle.Pickler accepts a lone protocol argument;
# pickle.py does not.
- pickler = cPickle.Pickler(1)
+ pickler = cPickle.Pickler(PICKLE_PROTOCOL_VERSION)
pickler.fast = 1
# Undocumented: pickler.dump(), for a cPickle.Pickler, takes
Modified: ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/BaseStorage.py
===================================================================
--- ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/BaseStorage.py 2007-12-13 02:34:39 UTC (rev 82271)
+++ ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/BaseStorage.py 2007-12-13 02:37:39 UTC (rev 82272)
@@ -24,6 +24,7 @@
from persistent.TimeStamp import TimeStamp
from ZODB import POSException
+from ZODB.config import PICKLE_PROTOCOL_VERSION
from ZODB.utils import z64, oid_repr
from ZODB.UndoLogCompatible import UndoLogCompatible
@@ -185,7 +186,7 @@
desc = transaction.description
ext = transaction._extension
if ext:
- ext = cPickle.dumps(ext, 1)
+ ext = cPickle.dumps(ext, PICKLE_PROTOCOL_VERSION)
else:
ext = ""
self._ude = user, desc, ext
Modified: ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/ConflictResolution.py
===================================================================
--- ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/ConflictResolution.py 2007-12-13 02:34:39 UTC (rev 82271)
+++ ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/ConflictResolution.py 2007-12-13 02:37:39 UTC (rev 82272)
@@ -19,6 +19,7 @@
import zope.interface
+from ZODB.config import PICKLE_PROTOCOL_VERSION
from ZODB.POSException import ConflictError
from ZODB.loglevels import BLATHER
@@ -207,7 +208,7 @@
resolved = resolve(old, committed, newstate)
file = StringIO()
- pickler = Pickler(file,1)
+ pickler = Pickler(file, PICKLE_PROTOCOL_VERSION)
pickler.persistent_id = persistent_id
pickler.dump(meta)
pickler.dump(resolved)
Modified: ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/DB.py
===================================================================
--- ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/DB.py 2007-12-13 02:34:39 UTC (rev 82271)
+++ ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/DB.py 2007-12-13 02:37:39 UTC (rev 82272)
@@ -25,6 +25,7 @@
import calendar
from ZODB.broken import find_global
+from ZODB.config import PICKLE_PROTOCOL_VERSION
from ZODB.utils import z64
from ZODB.Connection import Connection
import ZODB.serialize
@@ -454,7 +455,7 @@
# Manually create a pickle for the root to put in the storage.
# The pickle must be in the special ZODB format.
file = cStringIO.StringIO()
- p = cPickle.Pickler(file, 1)
+ p = cPickle.Pickler(file, PICKLE_PROTOCOL_VERSION)
p.dump((root.__class__, None))
p.dump(root.__getstate__())
t = transaction.Transaction()
Modified: ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/ExportImport.py
===================================================================
--- ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/ExportImport.py 2007-12-13 02:34:39 UTC (rev 82271)
+++ ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/ExportImport.py 2007-12-13 02:37:39 UTC (rev 82272)
@@ -21,6 +21,7 @@
import logging
from ZODB.blob import Blob
+from ZODB.config import PICKLE_PROTOCOL_VERSION
from ZODB.interfaces import IBlobStorage
from ZODB.POSException import ExportError, POSKeyError
from ZODB.serialize import referencesf
@@ -169,7 +170,7 @@
unpickler.persistent_load = persistent_load
newp = StringIO()
- pickler = Pickler(newp, 1)
+ pickler = Pickler(newp, PICKLE_PROTOCOL_VERSION)
pickler.persistent_id = persistent_id
pickler.dump(unpickler.load())
Modified: ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/FileStorage/FileStorage.py
===================================================================
--- ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/FileStorage/FileStorage.py 2007-12-13 02:34:39 UTC (rev 82271)
+++ ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/FileStorage/FileStorage.py 2007-12-13 02:37:39 UTC (rev 82272)
@@ -33,6 +33,7 @@
from ZODB.POSException import UndoError, POSKeyError, MultipleUndoErrors
from ZODB.POSException import VersionLockError
from persistent.TimeStamp import TimeStamp
+from ZODB.config import PICKLE_PROTOCOL_VERSION
from ZODB.lock_file import LockFile
from ZODB.utils import p64, u64, cp, z64
from ZODB.FileStorage.fspack import FileStoragePacker
@@ -242,7 +243,7 @@
tmp_name = index_name + '.index_tmp'
f=open(tmp_name,'wb')
- p=Pickler(f,1)
+ p=Pickler(f, PICKLE_PROTOCOL_VERSION)
# Note: starting with ZODB 3.2.6, the 'oid' value stored is ignored
# by the code that reads the index. We still write it, so that
@@ -382,7 +383,7 @@
if not self._is_read_only:
# Save the converted index.
f = open(index_name, 'wb')
- p = Pickler(f, 1)
+ p = Pickler(f, PICKLE_PROTOCOL_VERSION)
info['index'] = index
p.dump(info)
f.close()
Modified: ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/broken.py
===================================================================
--- ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/broken.py 2007-12-13 02:34:39 UTC (rev 82271)
+++ ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/broken.py 2007-12-13 02:37:39 UTC (rev 82272)
@@ -87,6 +87,16 @@
>>> a2.__Broken_state__
{'x': 1}
+ >>> import cPickle
+ >>> a2 = cPickle.loads(cPickle.dumps(a, 2))
+ >>> a2
+ <broken not.there.Atall instance>
+ >>> a2.__Broken_newargs__
+ (1, 2)
+ >>> a2.__Broken_initargs__
+ >>> a2.__Broken_state__
+ {'x': 1}
+
Cleanup::
>>> broken_cache.clear()
Modified: ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/config.py
===================================================================
--- ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/config.py 2007-12-13 02:34:39 UTC (rev 82271)
+++ ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/config.py 2007-12-13 02:37:39 UTC (rev 82272)
@@ -28,6 +28,11 @@
s_schema_path = os.path.join(ZODB.__path__[0], "storage.xml")
_s_schema = None
+# TODO: Turn this into a configurable option on a per storage basis
+# Does is make sense to use a different protocol version for ZEO transport?
+PICKLE_PROTOCOL_VERSION = 2
+
+
def getDbSchema():
global _db_schema
if _db_schema is None:
Modified: ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/serialize.py
===================================================================
--- ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/serialize.py 2007-12-13 02:34:39 UTC (rev 82271)
+++ ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/serialize.py 2007-12-13 02:37:39 UTC (rev 82272)
@@ -142,6 +142,7 @@
from persistent.wref import WeakRefMarker, WeakRef
from ZODB import broken
from ZODB.broken import Broken
+from ZODB.config import PICKLE_PROTOCOL_VERSION
from ZODB.POSException import InvalidObjectReference
_oidtypes = str, type(None)
@@ -172,7 +173,7 @@
def __init__(self, obj=None):
self._file = cStringIO.StringIO()
- self._p = cPickle.Pickler(self._file, 1)
+ self._p = cPickle.Pickler(self._file, PICKLE_PROTOCOL_VERSION)
self._p.persistent_id = self.persistent_id
self._stack = []
if obj is not None:
Modified: ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/tests/PackableStorage.py
===================================================================
--- ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/tests/PackableStorage.py 2007-12-13 02:34:39 UTC (rev 82271)
+++ ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/tests/PackableStorage.py 2007-12-13 02:37:39 UTC (rev 82272)
@@ -31,6 +31,7 @@
from persistent.mapping import PersistentMapping
import transaction
from ZODB import DB
+from ZODB.config import PICKLE_PROTOCOL_VERSION
from ZODB.serialize import referencesf
from ZODB.tests.MinPO import MinPO
from ZODB.tests.StorageTestBase import snooze
@@ -83,6 +84,7 @@
return obj.getoid()
return None
s = StringIO()
+ # TODO: Pickle protocol 0?
p = pickle.Pickler(s)
p.persistent_id = getpersid
p.dump(obj)
@@ -91,6 +93,7 @@
def pdumps(obj):
s = StringIO()
+ # TODO: Pickle protocol 0?
p = pickle.Pickler(s)
p.dump(obj)
p.dump(None)
@@ -137,7 +140,7 @@
except KeyError:
from transaction import Transaction
file = StringIO()
- p = cPickle.Pickler(file, 1)
+ p = cPickle.Pickler(file, PICKLE_PROTOCOL_VERSION)
p.dump((PersistentMapping, None))
p.dump({'_container': {}})
t=Transaction()
Modified: ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/tests/StorageTestBase.py
===================================================================
--- ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/tests/StorageTestBase.py 2007-12-13 02:34:39 UTC (rev 82271)
+++ ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/tests/StorageTestBase.py 2007-12-13 02:37:39 UTC (rev 82272)
@@ -28,6 +28,7 @@
import transaction
+from ZODB.config import PICKLE_PROTOCOL_VERSION
from ZODB.utils import u64
from ZODB.tests.MinPO import MinPO
@@ -52,7 +53,7 @@
def zodb_pickle(obj):
"""Create a pickle in the format expected by ZODB."""
f = StringIO()
- p = Pickler(f, 1)
+ p = Pickler(f, PICKLE_PROTOCOL_VERSION)
p.persistent_id = _persistent_id
klass = obj.__class__
assert not hasattr(obj, '__getinitargs__'), "not ready for constructors"
Modified: ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/tests/testFileStorage.py
===================================================================
--- ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/tests/testFileStorage.py 2007-12-13 02:34:39 UTC (rev 82271)
+++ ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/tests/testFileStorage.py 2007-12-13 02:37:39 UTC (rev 82272)
@@ -17,6 +17,7 @@
import ZODB.tests.util
from ZODB import POSException
from ZODB import DB
+from ZODB.config import PICKLE_PROTOCOL_VERSION
from ZODB.tests import StorageTestBase, BasicStorage, TransactionalUndoStorage
from ZODB.tests import VersionStorage, TransactionalUndoVersionStorage
@@ -96,7 +97,7 @@
f.seek(0)
f.truncate()
- p = pickle.Pickler(f, 1)
+ p = pickle.Pickler(f, PICKLE_PROTOCOL_VERSION)
p.dump(data)
f.close()
return index
@@ -212,7 +213,7 @@
data['oid'] = z64
f.seek(0)
f.truncate()
- p = pickle.Pickler(f, 1)
+ p = pickle.Pickler(f, PICKLE_PROTOCOL_VERSION)
p.dump(data)
f.close()
Modified: ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/tests/testSerialize.py
===================================================================
--- ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/tests/testSerialize.py 2007-12-13 02:34:39 UTC (rev 82271)
+++ ZODB/branches/hannosch-pickle-protocol-version2/src/ZODB/tests/testSerialize.py 2007-12-13 02:37:39 UTC (rev 82272)
@@ -19,6 +19,7 @@
import unittest
from ZODB import serialize
+from ZODB.config import PICKLE_PROTOCOL_VERSION
class ClassWithNewargs(int):
@@ -34,7 +35,7 @@
def make_pickle(ob):
sio = StringIO.StringIO()
- p = cPickle.Pickler(sio, 1)
+ p = cPickle.Pickler(sio, PICKLE_PROTOCOL_VERSION)
p.dump(ob)
return sio.getvalue()
Modified: ZODB/branches/hannosch-pickle-protocol-version2/src/persistent/tests/testPersistent.py
===================================================================
--- ZODB/branches/hannosch-pickle-protocol-version2/src/persistent/tests/testPersistent.py 2007-12-13 02:34:39 UTC (rev 82271)
+++ ZODB/branches/hannosch-pickle-protocol-version2/src/persistent/tests/testPersistent.py 2007-12-13 02:37:39 UTC (rev 82272)
@@ -215,6 +215,14 @@
obj2 = pickle.loads(s)
self.assertEqual(obj.attr, obj2.attr)
+ s = pickle.dumps(obj, 1)
+ obj2 = pickle.loads(s)
+ self.assertEqual(obj.attr, obj2.attr)
+
+ s = pickle.dumps(obj, 2)
+ obj2 = pickle.loads(s)
+ self.assertEqual(obj.attr, obj2.attr)
+
def testGetattr(self):
obj = H1()
self.assertEqual(obj.larry, 1)
More information about the Zodb-checkins
mailing list