[Zope3-checkins] SVN: Zope3/branches/srichter-twisted-integration/
Fixed the recorder to work with Twsited's servers. The
wrapping code in
Stephan Richter
srichter at cosmos.phy.tufts.edu
Tue Apr 19 15:33:37 EDT 2005
Log message for revision 30049:
Fixed the recorder to work with Twsited's servers. The wrapping code in
twsited is really nice and worked great!
Changed:
U Zope3/branches/srichter-twisted-integration/package-includes/zope.app.recorder-configure.zcml
U Zope3/branches/srichter-twisted-integration/src/zope/app/recorder/__init__.py
U Zope3/branches/srichter-twisted-integration/src/zope/app/recorder/configure.zcml
U Zope3/branches/srichter-twisted-integration/src/zope/app/recorder/tests.py
-=-
Modified: Zope3/branches/srichter-twisted-integration/package-includes/zope.app.recorder-configure.zcml
===================================================================
--- Zope3/branches/srichter-twisted-integration/package-includes/zope.app.recorder-configure.zcml 2005-04-19 19:32:37 UTC (rev 30048)
+++ Zope3/branches/srichter-twisted-integration/package-includes/zope.app.recorder-configure.zcml 2005-04-19 19:33:37 UTC (rev 30049)
@@ -1,3 +1 @@
-<!-- XXX: This code relies on ZServer, which is not more! -->
-<!-- include package="zope.app.recorder" / -->
-<configure />
+<include package="zope.app.recorder" />
\ No newline at end of file
Modified: Zope3/branches/srichter-twisted-integration/src/zope/app/recorder/__init__.py
===================================================================
--- Zope3/branches/srichter-twisted-integration/src/zope/app/recorder/__init__.py 2005-04-19 19:32:37 UTC (rev 30048)
+++ Zope3/branches/srichter-twisted-integration/src/zope/app/recorder/__init__.py 2005-04-19 19:33:37 UTC (rev 30049)
@@ -11,115 +11,85 @@
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
-"""
-HTTP session recorder.
+"""HTTP session recorder.
$Id$
"""
__docformat__ = 'restructuredtext'
+import time
import thread
import threading
+import cStringIO
+
+import twisted.web2.wsgi
+from twisted.protocols import policies
+
import transaction
import ZODB.MappingStorage
from ZODB.POSException import ConflictError
from BTrees.IOBTree import IOBTree
-from zope.app.publication.httpfactory import HTTPPublicationRequestFactory
-from zope.app.server.servertype import ServerType
-from zope.server.http.commonaccesslogger import CommonAccessLogger
-from zope.server.http.publisherhttpserver import PublisherHTTPServer
-from zope.server.http.httpserverchannel import HTTPServerChannel
-from zope.server.http.httprequestparser import HTTPRequestParser
-from zope.server.http.httptask import HTTPTask
-from zope.publisher.publish import publish
+from zope.app import wsgi
+from zope.app.server.server import ServerType
+class RecordingProtocol(policies.ProtocolWrapper):
+ """A special protocol that keeps track of all input and output of an HTTP
+ connection.
-class RecordingHTTPTask(HTTPTask):
- """An HTTPTask that remembers the response as a string."""
+ The data is recorded for later analysis, such as generation of doc tests.
+ """
- def __init__(self, *args, **kw):
- self._response_data = []
- HTTPTask.__init__(self, *args, **kw)
+ def __init__(self, factory, wrappedProtocol):
+ policies.ProtocolWrapper.__init__(self, factory, wrappedProtocol)
+ self.input = cStringIO.StringIO()
+ self.output = cStringIO.StringIO()
+ self.chanRequest = None
+ def dataReceived(self, data):
+ self.input.write(data)
+ policies.ProtocolWrapper.dataReceived(self, data)
+
def write(self, data):
- """Send data to the client.
+ if not self.chanRequest:
+ self.chanRequest = self.wrappedProtocol.requests[-1]
+ self.output.write(data)
+ policies.ProtocolWrapper.write(self, data)
- Wraps HTTPTask.write and records the response.
- """
- if not self.wrote_header:
- # HTTPTask.write will call self.buildResponseHeader() and send the
- # result before sending 'data'. This code assumes that
- # buildResponseHeader will return the same string when called the
- # second time.
- self._response_data.append(self.buildResponseHeader())
- HTTPTask.write(self, data)
- self._response_data.append(data)
+ def writeSequence(self, data):
+ for entry in data:
+ self.output.write(entry)
+ policies.ProtocolWrapper.writeSequence(self, data)
- def getRawResponse(self):
- """Return the full HTTP response as a string."""
- return ''.join(self._response_data)
+ def connectionLost(self, reason):
+ policies.ProtocolWrapper.connectionLost(self, reason)
+ if not self.chanRequest:
+ return
+ firstLine = self.output.getvalue().split('\r\n')[0]
+ proto, status, reason = firstLine.split(' ', 2)
+ requestStorage.add(RecordedRequest(
+ time.time(),
+ self.input.getvalue(),
+ self.output.getvalue(),
+ method = self.chanRequest.command.upper(),
+ path = self.chanRequest.path,
+ status = int(status),
+ reason = reason
+ ) )
-class RecordingHTTPRequestParser(HTTPRequestParser):
- """An HTTPRequestParser that remembers the raw request as a string."""
- def __init__(self, *args, **kw):
- self._request_data = []
- HTTPRequestParser.__init__(self, *args, **kw)
+class RecordingFactory(policies.WrappingFactory):
+ """Special server factory that supports recording."""
+ protocol = RecordingProtocol
- def received(self, data):
- """Process data received from the client.
- Wraps HTTPRequestParser.write and records the request.
- """
- consumed = HTTPRequestParser.received(self, data)
- self._request_data.append(data[:consumed])
- return consumed
+def createRecordingHTTPFactory(db):
+ resource = twisted.web2.wsgi.WSGIResource(
+ wsgi.WSGIPublisherApplication(db))
+
+ return RecordingFactory(twisted.web2.server.Site(resource))
- def getRawRequest(self):
- """Return the full HTTP request as a string."""
- return ''.join(self._request_data)
-
-class RecordingHTTPServerChannel(HTTPServerChannel):
- """An HTTPServerChannel that records request and response."""
-
- task_class = RecordingHTTPTask
- parser_class = RecordingHTTPRequestParser
-
-
-class RecordingHTTPServer(PublisherHTTPServer):
- """Zope Publisher-specific HTTP server that can record requests."""
-
- channel_class = RecordingHTTPServerChannel
- num_retries = 10
-
- def executeRequest(self, task):
- """Process a request.
-
- Wraps PublisherHTTPServer.executeRequest().
- """
- PublisherHTTPServer.executeRequest(self, task)
- # PublisherHTTPServer either committed or aborted a transaction,
- # so we need a new one.
- # TODO: Actually, we only need a transaction if we use
- # ZODBBasedRequestStorage, which we don't since it has problems
- # keeping data fresh enough. This loop will go away soon, unless
- # I manage to fix ZODBBasedRequestStorage.
- for n in range(self.num_retries):
- try:
- txn = transaction.begin()
- txn.note("request recorder")
- requestStorage.add(RecordedRequest.fromHTTPTask(task))
- transaction.commit()
- except ConflictError:
- transaction.abort()
- if n == self.num_retries - 1:
- raise
- else:
- break
-
-
class RecordedRequest(object):
"""A single recorded request and response."""
@@ -130,27 +100,13 @@
self.response_string = response_string
# The following attributes could be extracted from request_string and
# response_string, but it is simpler to just take readily-available
- # values from RecordingHTTPTask.
+ # values from RecordingProtocol.
self.method = method
self.path = path
self.status = status
self.reason = reason
- def fromHTTPTask(cls, task):
- """Create a RecordedRequest with data extracted from RecordingHTTPTask.
- """
- rq = cls(timestamp=task.start_time,
- request_string=task.request_data.getRawRequest(),
- response_string=task.getRawResponse(),
- method=task.request_data.command.upper(),
- path=task.request_data.path,
- status=task.status,
- reason=task.reason)
- return rq
- fromHTTPTask = classmethod(fromHTTPTask)
-
-
class RequestStorage(object):
"""A collection of recorded requests.
@@ -194,75 +150,10 @@
self._requests.clear()
-class ZODBBasedRequestStorage(object):
- """A collection of recorded requests.
-
- This class is thread-safe, that is, its methods can be called from multiple
- threads simultaneously.
-
- In addition, it is transactional.
-
- TODO: The simple ID allocation strategy used by RequestStorage.add will
- cause frequent conflict errors. Something should be done about that.
-
- TODO: _getData() tends to return stale data, and you need to refresh the
- ++etc++process/RecordedSessions.html page two or three times until
- it becomes up to date.
-
- TODO: This class is not used because of the previous problem. Either fix
- the problem, or remove this class.
- """
-
- _key = 'RequestStorage'
-
- def __init__(self):
- self._ram_storage = ZODB.MappingStorage.MappingStorage()
- self._ram_db = ZODB.DB(self._ram_storage)
- self._conns = {}
-
- def _getData(self):
- """Get the shared data container from the mapping storage."""
- # This method closely mimics RAMSessionDataContainer._getData
- # from zope.app.session.session
- tid = thread.get_ident()
- if tid not in self._conns:
- self._conns[tid] = self._ram_db.open()
- root = self._conns[tid].root()
- if self._key not in root:
- root[self._key] = IOBTree()
- return root[self._key]
-
- def add(self, rr):
- """Add a RecordedRequest to the list."""
- requests = self._getData()
- rr.id = len(requests) + 1
- requests[rr.id] = rr
-
- def __len__(self):
- """Return the number of recorded requests."""
- return len(self._getData())
-
- def __iter__(self):
- """List all recorded requests."""
- return iter(self._getData().values())
-
- def get(self, id):
- """Return the request with a given id, or None."""
- requests = self._getData()
- return requests.get(id)
-
- def clear(self):
- """Clear all recorded requests."""
- self._getData().clear()
-
-
#
# Globals
#
requestStorage = RequestStorage()
-recordinghttp = ServerType(RecordingHTTPServer,
- HTTPPublicationRequestFactory,
- CommonAccessLogger,
- 8081, True)
+recordinghttp = ServerType(createRecordingHTTPFactory, 8081)
Modified: Zope3/branches/srichter-twisted-integration/src/zope/app/recorder/configure.zcml
===================================================================
--- Zope3/branches/srichter-twisted-integration/src/zope/app/recorder/configure.zcml 2005-04-19 19:32:37 UTC (rev 30048)
+++ Zope3/branches/srichter-twisted-integration/src/zope/app/recorder/configure.zcml 2005-04-19 19:33:37 UTC (rev 30049)
@@ -2,10 +2,11 @@
xmlns:browser="http://namespaces.zope.org/browser"
i18n_domain="zope">
+
<utility
name="RecordingHTTP"
component=".recordinghttp"
- provides="zope.app.server.servertype.IServerType"
+ provides="zope.app.server.interfaces.IServerType"
/>
<browser:page
@@ -33,4 +34,5 @@
attribute="recordedResponse"
/>
+
</configure>
Modified: Zope3/branches/srichter-twisted-integration/src/zope/app/recorder/tests.py
===================================================================
--- Zope3/branches/srichter-twisted-integration/src/zope/app/recorder/tests.py 2005-04-19 19:32:37 UTC (rev 30048)
+++ Zope3/branches/srichter-twisted-integration/src/zope/app/recorder/tests.py 2005-04-19 19:33:37 UTC (rev 30049)
@@ -19,6 +19,7 @@
"""
__docformat__ = 'restructuredtext'
+import time
import unittest
import transaction
from zope.testing import doctest
@@ -27,132 +28,85 @@
from zope.app.publisher.browser import BrowserView
-def doctest_RecordingHTTPServer():
- r"""Unit tests for RecordingHTTPServer.
+def doctest_RecordingProtocol():
+ r"""Unit tests for ``RecordingProtocol``.
- We will use stubs instead of real channel and request parser objects, to
- keep the test fixture small.
+ To create a recording protocol we need a protocol and a factory. To keep
+ the test fixture small, we are using a stub implementation for the
+ protocol
- >>> from zope.app.recorder import RecordingHTTPTask
- >>> channel = ChannelStub()
- >>> request_data = RequestDataStub()
- >>> task = RecordingHTTPTask(channel, request_data)
+ >>> protocol = ProtocolStub()
- RecordingHTTPTask is a thin wrapper around HTTPTask. It records all data
- written through task.write, plus the response header, of course.
+ and the factory is created with a provided function:
- >>> task.write('request body\n')
- >>> task.write('goes in here')
+ >>> from zope.app import recorder
+ >>> db = 'ZODB'
+ >>> factory = recorder.createRecordingHTTPFactory(db)
- We need to strip CR characters, as they confuse doctest.
+ We can now instantiate the recording protcol:
- >>> print task.getRawResponse().replace('\r', '')
- HTTP/1.1 200 Ok
- Connection: close
- Server: Stub Server
- <BLANKLINE>
- request body
- goes in here
+ >>> recording = recorder.RecordingProtocol(factory, protocol)
+ >>> recording.transport = TransportStub()
+ >>> factory.protocols = {recording: 1}
- """
+ When we now send data to the protocol,
+ >>> recording.dataReceived('GET / HTTP/1.1\n\n')
+ >>> recording.dataReceived('hello world!\n')
-def doctest_RecordingHTTPRequestParser():
- r"""Unit tests for RecordingHTTPRequestParser.
+ then the result is immediately available in the ``input`` attribute:
- >>> from zope.app.recorder import RecordingHTTPRequestParser
- >>> from zope.server.adjustments import default_adj
- >>> parser = RecordingHTTPRequestParser(default_adj)
+ >>> print recording.input.getvalue()
+ GET / HTTP/1.1
+ <BLANKLINE>
+ hello world!
+ <BLANKLINE>
- RecordingHTTPRequestParser is a thin wrapper around HTTPRequestParser. It
- records all data consumed by parser.received.
+ Once the request has been processed, the response is written
- >>> parser.received('GET / HTTP/1.1\r\n')
- 16
- >>> parser.received('Content-Length: 3\r\n')
- 19
- >>> parser.received('\r\n')
- 2
- >>> parser.received('abc plus some junk')
- 3
+ >>> recording.writeSequence(('HTTP/1.1 200 Okay.\n',
+ ... 'header1: value1\n',
+ ... 'header2: value2\n'))
+ >>> recording.write('\n')
+ >>> recording.write('This is my answer.')
- We need to strip CR characters, as they confuse doctest.
+ and we can again look at it:
- >>> print parser.getRawRequest().replace('\r', '')
- GET / HTTP/1.1
- Content-Length: 3
- <BLANKLINE>
- abc
+ >>> print recording.output.getvalue()
+ HTTP/1.1 200 Okay.
+ header1: value1
+ header2: value2
+ <BLANKLINE>
+ This is my answer.
- """
+ Once the request is finished and the response is written, the connection
+ is closed and a recorded request obejct is written:
+ >>> recording.connectionLost(None)
-def doctest_RecordingHTTPServer():
- r"""Unit tests for RecordingHTTPServer.
+ Let's now inspect the recorded requets object:
- RecordingHTTPServer is a very thin wrapper over PublisherHTTPServer. First
- we create a custom request:
+ >>> len(recorder.requestStorage)
+ 1
+ >>> rq = iter(recorder.requestStorage).next()
+ >>> rq.timestamp < time.time()
+ True
+ >>> rq.request_string
+ 'GET / HTTP/1.1\n\nhello world!\n'
+ >>> rq.method
+ 'GET'
+ >>> rq.path
+ '/'
+ >>> print rq.response_string.replace('\r', '')
+ HTTP/1.1 200 Okay.
+ header1: value1
+ header2: value2
+ <BLANKLINE>
+ This is my answer.
- >>> class RecorderRequest(TestRequest):
- ... publication = PublicationStub()
-
- Further, to keep things simple, we will override the constructor and
- prevent it from listening on sockets.
-
- >>> from zope.app.recorder import RecordingHTTPServer
- >>> class RecordingHTTPServerForTests(RecordingHTTPServer):
- ... def __init__(self):
- ... self.request_factory = RecorderRequest
- >>> server = RecordingHTTPServerForTests()
-
- We will need a request parser
-
- >>> from zope.app.recorder import RecordingHTTPRequestParser
- >>> from zope.server.adjustments import default_adj
- >>> parser = RecordingHTTPRequestParser(default_adj)
- >>> parser.received('GET / HTTP/1.1\r\n\r\n')
- 18
-
- We will also need a task
-
- >>> from zope.app.recorder import RecordingHTTPTask
- >>> channel = ChannelStub()
- >>> task = RecordingHTTPTask(channel, parser)
- >>> task.start_time = 42
-
- Go!
-
- >>> server.executeRequest(task)
-
- Let's see what we got:
-
- >>> from zope.app import recorder
- >>> len(recorder.requestStorage)
- 1
- >>> rq = iter(recorder.requestStorage).next()
- >>> rq.timestamp
- 42
- >>> rq.request_string
- 'GET / HTTP/1.1\r\n\r\n'
- >>> rq.method
- 'GET'
- >>> rq.path
- '/'
- >>> print rq.response_string.replace('\r', '')
- HTTP/1.1 599 No status set
- Content-Length: 0
- X-Powered-By: Zope (www.zope.org), Python (www.python.org)
- Server: Stub Server
- <BLANKLINE>
- <BLANKLINE>
- >>> rq.status
- 599
- >>> rq.reason
- 'No status set'
-
Clean up:
- >>> recorder.requestStorage.clear()
+ >>> recorder.requestStorage.clear()
"""
@@ -606,33 +560,33 @@
"""
+class ChannelRequestStub(object):
-class ServerStub(object):
- """Stub for HTTPServer."""
+ command = 'GeT'
+ path = '/'
- SERVER_IDENT = 'Stub Server'
- server_name = 'RecordingHTTPServer'
- port = 8081
-
-class ChannelStub(object):
- """Stub for HTTPServerChannel."""
-
- server = ServerStub()
- creation_time = 42
- addr = ('addr', )
-
+class TransportStub(object):
+
def write(self, data):
pass
+
+ def writeSequence(self, data):
+ pass
-class RequestDataStub(object):
- """Stub for HTTPRequestParser."""
+class ProtocolStub(object):
+ """Stub for the HTTP Protocol"""
- version = "1.1"
- headers = {}
+ requests = [ChannelRequestStub()]
+ def dataReceived(self, data):
+ pass
+ def connectionLost(self, reason):
+ pass
+
+
class PublicationStub(object):
"""Stub for Publication."""
More information about the Zope3-Checkins
mailing list