[Zope-CVS] CVS: Products/AdaptableStorage/zodb - RemainingState.py:1.1 ASConnection.py:1.8 public.py:1.2

Shane Hathaway shane@zope.com
Fri, 13 Dec 2002 15:42:04 -0500


Update of /cvs-repository/Products/AdaptableStorage/zodb
In directory cvs.zope.org:/tmp/cvs-serv32288/zodb

Modified Files:
	ASConnection.py public.py 
Added Files:
	RemainingState.py 
Log Message:
Hopefully solved the problem of unmanaged persistent objects.

Because AdaptableStorage lets the mappers choose their own persistence
boundaries, some objects that inherit from Persistent may be unknown
to ZODB, even though they are in the database.  I call these unmanaged
persistent objects.

The problem with unmanaged persistent objects surfaces when you change
them without changing their container.  For example, if you add an
item to a BTree, the BTree object (which is managed) does not get
changed, but rather a contained Bucket object (which is often
unmanaged).  We really need the BTree to be notified when the Buckets
change.

To solve this, certain aspect serializers (currently only
RemainderSerializer) now detect unmanaged persistent objects and addq
them to a list.  ZODB looks over this list and assigns them a one-off
"_p_jar" with a register() method.  This special register() method
just sets the _p_changed attribute of the managed object, notifying
ZODB that it must be saved.

I think this is the best solution, even though it's awfully complex to
explain.  The change involved moving the RemainingState aspect to the
"zodb" subpackage, since it already depended on the particulars of the
Persistent base class anyway.  It also required changing
ObjectSerializer so that the event object gets constructed by the
caller, which is appropriate, I think.

Also made some minor changes.


=== Added File Products/AdaptableStorage/zodb/RemainingState.py ===
##############################################################################
#
# Copyright (c) 2002 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Aspect for (de)serializing the remaining state of a Persistent object

$Id: RemainingState.py,v 1.1 2002/12/13 20:42:03 shane Exp $
"""

from cStringIO import StringIO
from cPickle import Pickler, Unpickler
from types import DictType

from ZODB import Persistent

from serial_public import \
     IAspectSerializer, FieldSchema, \
     IFullSerializationEvent, IFullDeserializationEvent


class RemainingState:
    __implements__ = IAspectSerializer

    schema = FieldSchema('data', 'string')

    def getSchema(self):
        return self.schema


    def serialize(self, object, event):
        assert IFullSerializationEvent.isImplementedBy(event)
        assert isinstance(object, Persistent)
        state = object.__dict__.copy()
        for key in state.keys():
            if key.startswith('_v_'):
                del state[key]
        for attrname in event.getSerializedAttributeNames():
            if state.has_key(attrname):
                del state[attrname]
        if not state:
            # No data needs to be stored
            return ''

        outfile = StringIO()
        p = Pickler(outfile)
        unmanaged = []

        def persistent_id(ob, getInternalRef=event.getInternalRef,
                          unmanaged=unmanaged):
            ref = getInternalRef(ob)
            if ref is None:
                if isinstance(ob, Persistent):
                    # Persistent objects that end up in the remainder
                    # are unmanaged.  Tell ZODB about them so that
                    # ZODB can deal with them specially.
                    unmanaged.append(ob)
            return ref

        p.persistent_id = persistent_id
        p.dump(state)
        p.dump(unmanaged)
        s = outfile.getvalue()
        event.addUnmanagedPersistentObjects(unmanaged)
        return s


    def deserialize(self, object, event, state):
        assert IFullDeserializationEvent.isImplementedBy(event)
        assert isinstance(object, Persistent)
        if state:
            infile = StringIO(state)
            u = Unpickler(infile)
            u.persistent_load = event.loadInternalRef
            s = u.load()
            object.__dict__.update(s)
            try:
                unmanaged = u.load()
            except EOFError:
                # old pickle
                pass
            else:
                event.addUnmanagedPersistentObjects(unmanaged)



=== Products/AdaptableStorage/zodb/ASConnection.py 1.7 => 1.8 ===
--- Products/AdaptableStorage/zodb/ASConnection.py:1.7	Mon Dec  9 17:11:08 2002
+++ Products/AdaptableStorage/zodb/ASConnection.py	Fri Dec 13 15:42:03 2002
@@ -20,12 +20,13 @@
 from time import time
 from types import StringType, TupleType
 
-import ZODB
+from ZODB import Persistent
 from ZODB.Connection import Connection, StringIO, Unpickler, Pickler, \
      ConflictError, ReadConflictError, ExtensionKlass, LOG, ERROR
 
 from consts import SERIAL0, DEBUG
-from serial_public import IKeyedObjectSystem
+from serial_public import IKeyedObjectSystem, SerializationEvent, \
+     DeserializationEvent
 
 
 class ASConnection (Connection):
@@ -247,7 +248,9 @@
                 ser = mapper.getSerializer()
                 if DEBUG:
                     print 'serializing', repr(oid), repr(serial)
-                state, ext_refs = ser.serialize(self, mapper, keychain, object)
+                event = SerializationEvent(self, mapper, keychain, object)
+                state = ser.serialize(object, event)
+                ext_refs = event.getExternalRefs()
                 if ext_refs:
                     for (ext_keychain, ext_ref) in ext_refs:
                         if (not ext_ref._p_serial
@@ -265,6 +268,10 @@
                                 ext_ref._p_oid = ext_oid
                             stack.append(ext_ref)
 
+                unmanaged = event.getUnmanagedPersistentObjects()
+                if unmanaged:
+                    self.handleUnmanaged(object, unmanaged)
+
             seek(0)
             clear_memo()
             dump((classification, mapper_names))
@@ -339,7 +346,12 @@
             ser = mapper.getSerializer()
             if DEBUG:
                 print 'deserializing', repr(oid), repr(serial)
-            ser.deserialize(self, mapper, keychain, object, state)
+            event = DeserializationEvent(self, mapper, keychain, object)
+            ser.deserialize(object, event, state)
+
+            unmanaged = event.getUnmanagedPersistentObjects()
+            if unmanaged:
+                self.handleUnmanaged(object, unmanaged)
 
             if mapper.isVolatile():
                 v = self._volatile
@@ -374,6 +386,17 @@
         return '<%s at %08x%s>' % (self.__class__.__name__, id(self), ver)
 
 
+    def handleUnmanaged(self, object, unmanaged):
+        for o in unmanaged:
+            if isinstance(o, Persistent):
+                if o._p_jar is None:
+                    o._p_oid = 'unmanaged'
+                    o._p_jar = UnmanagedJar(self, object._p_oid)
+                else:
+                    # Turn off the "changed" flag
+                    o._p_changed = 0
+
+
     # IKeyedObjectSystem implementation
 
     def loadStub(self, keychain, mapper_names=None):
@@ -419,4 +442,26 @@
     def sync(self):
         self.invalidateVolatileObjects()
         Connection.sync(self)
+
+
+
+class UnmanagedJar:
+    """Special jar for unmanaged persistent objects.
+
+    There is one such jar for each unmanaged persistent object.  All
+    it does is notify the managed persistent object of changes.
+
+    Note that unmanaged persistent objects should never be ghosted!
+    Instead, when the managed persistent object gets ghosted, it
+    usually removes the last reference to the unmanaged object, which
+    is then deallocated.
+    """
+
+    def __init__(self, real_jar, real_oid):
+        self.real_jar = real_jar
+        self.real_oid = real_oid
+
+    def register(self, ob):
+        o = self.real_jar[self.real_oid]
+        o._p_changed = 1
 


=== Products/AdaptableStorage/zodb/public.py 1.1 => 1.2 ===
--- Products/AdaptableStorage/zodb/public.py:1.1	Tue Dec  3 18:10:55 2002
+++ Products/AdaptableStorage/zodb/public.py	Fri Dec 13 15:42:03 2002
@@ -22,3 +22,4 @@
 from ASStorage import ASStorage
 from OIDEncoder import OIDEncoder
 from StaticResource import StaticResource
+from RemainingState import RemainingState