[Zope-CVS] CVS: Products/FileCacheManager/tests - TestFCMRig.py:1.1
utils.py:1.1 stresstest.py:1.7
Jens Vagelpohl
jens at dataflake.org
Sun Aug 29 06:26:54 EDT 2004
Update of /cvs-repository/Products/FileCacheManager/tests
In directory cvs.zope.org:/tmp/cvs-serv1783
Modified Files:
stresstest.py
Added Files:
TestFCMRig.py utils.py
Log Message:
- Adding a test script based on Chris' Sessioning stress test script.
The script needs to have a specific ZODB layout built and runs against
several ZEO clients. Looking at the top of TestFCMRig.py will clarify
what's expected.
=== Added File Products/FileCacheManager/tests/TestFCMRig.py ===
#!/usr/local/bin/python
import urllib
import urllib2
import httplib
import mimetypes
import threading
import time
import random
import sys
import os
from utils import post_multipart
WRITERATIO = 0.0
FILESPATH = '/usr/local/zope/opt/FileCacheManager-HEAD/tests/files'
TESTFILES = { 'small.pdf' : os.path.join(FILESPATH, 'small.pdf')
, 'medium.pdf' : os.path.join(FILESPATH, 'medium.pdf')
, 'large.pdf' : os.path.join(FILESPATH, 'large.pdf')
}
TESTFILESIZES = { 'small.pdf' : os.stat(TESTFILES.get('small.pdf'))[6]
, 'medium.pdf' : os.stat(TESTFILES.get('medium.pdf'))[6]
, 'large.pdf' : os.stat(TESTFILES.get('large.pdf'))[6]
}
SERVERS = ( 'localhost:10080'
, 'localhost:20080'
, 'localhost:30080'
, 'localhost:40080'
, 'localhost:50080'
)
FOLDERS = ('folder1', 'folder2', 'folder3', 'folder4', 'folder5')
FILES = ('file1', 'file2', 'file3')
METHODS = ('index_html', 'manage_edit')
PATHS = []
for server in SERVERS:
for folder in FOLDERS:
for file in FILES:
PATHS.append('http://%s/fcmrig/%s/%s' % (server, folder, file))
def pickUrl():
# Pick whether to read or write based on the WRITERATIO
write_randvalue = random.random()
if write_randvalue <= WRITERATIO:
method = 'manage_edit'
else:
method = 'index_html'
return ( random.choice(PATHS)
, method
, random.choice(TESTFILES.keys())
)
def openUrl(url, method, note, file):
if method == 'index_html':
print '.',
sys.stdout.flush()
request = urllib2.Request(url)
try:
response = urllib2.urlopen(request)
body = response.read()
assert len(body) in TESTFILESIZES.values()
except urllib2.HTTPError, error:
print error
except urllib2.URLError, error:
print error
except AssertionError:
print 'Problem file: %s (%d long)' % (url , len(body))
sys.stdout.flush()
elif method == 'manage_edit':
print ':',
sys.stdout.flush()
# We have to create a special request
# Ugh, have to hack apart the URL to get at its components
host, basefolder, folder, fname = url[7:].split('/')
urlpath1 = '/%s/%s/%s/manage_upload' % (basefolder, folder, fname)
upload_data = open(TESTFILES.get(file), 'rb').read()
httplib_resp = post_multipart( host
, urlpath1
, ()
, ( ('file', file, upload_data), )
)
status = int(httplib_resp.status)
if status == 200:
print '!',
sys.stdout.flush()
else:
print status, httplib_resp.reason, httplib_resp.read()
sys.stdout.flush()
class DetachedRequest(threading.Thread):
def __init__(self, url, note):
self.url = url[0]
self.method = url[1]
self.note = note
self.file = url[2]
self.isDone = 0
threading.Thread.__init__(self)
def run(self):
begin = time.time()
time.sleep(random.uniform(0,0.25))
self.note['detachedSleepfor'] = time.time() - begin
openUrl(self.url, self.method, self.note, self.file)
self.isDone = 1
class Browser(threading.Thread):
def __init__(self, sleepiness, browserThreads, iterations):
self.sleepiness = sleepiness
self.browserThreads = browserThreads
self.iterations = iterations
self.isDone = 0
threading.Thread.__init__(self)
def run(self):
requests = []
for i in range(self.iterations):
# Each loop simulates a browser session - several page requests
url = pickUrl()
sleepfor = 0
note = { 'sleepiness' : self.sleepiness
, 'sleepfor' : sleepfor
}
for reference in range(random.randint(0,self.browserThreads)):
req = DetachedRequest(pickUrl(), note)
requests.append(req)
req.start()
# Sleep for a bit. Remembing just how long we intented to sleep
# for so that we can report this if needed.
begin = time.time()
time.sleep(random.uniform(0, self.sleepiness))
sleepfor = time.time() - begin
url = pickUrl()
while requests:
for request in requests:
if request.isDone:
print ',',
sys.stdout.flush()
requests.remove(request)
self.isDone = 1
def start(browserCount, sleepiness, browserThreads=4, iterations=10):
browsers = []
start_time = time.time()
for browser in range(browserCount):
browser = Browser(sleepiness, browserThreads, iterations)
browsers.append(browser)
browser.start()
while browsers:
for browser in browsers:
if browser.isDone:
browsers.remove(browser)
end_time = time.time()
duration = end_time - start_time
print '\n%d browsers, %d browser threads, %d iterations, %d sleepiness' % (
browserCount, browserThreads, iterations, sleepiness)
print 'Write Ratio:', WRITERATIO
print 'Took %.2f seconds' % duration
# Kick off a mini colony of robots. With a range of sleepiness.
start(5,2,4,10)
=== Added File Products/FileCacheManager/tests/utils.py ===
#####################################################################
#
# The following code was lifted from the "Python Cookbook Recipes"
# site run by ActiveState and submitted by Wade Leftwich. See
# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
#
#####################################################################
import httplib
import mimetypes
def post_multipart(host, selector, fields, files):
"""
Post fields and files to an http host as multipart/form-data.
fields is a sequence of (name, value) elements for regular form fields.
files is a sequence of (name, filename, value) elements for data to be
uploaded as files Return the server's response page.
"""
content_type, body = encode_multipart_formdata(fields, files)
h = httplib.HTTPConnection(host)
h.putrequest('POST', selector)
h.putheader('content-type', content_type)
h.putheader('content-length', str(len(body)))
h.endheaders()
h.send(body)
return h.getresponse()
#errcode, errmsg, headers = h.getreply()
#return h.file.read()
def encode_multipart_formdata(fields, files):
"""
fields is a sequence of (name, value) elements for regular form fields.
files is a sequence of (name, filename, value) elements for data to be
uploaded as files Return (content_type, body) ready for httplib.HTTP
instance
"""
BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
CRLF = '\r\n'
L = []
for (key, value) in fields:
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"' % key)
L.append('')
L.append(value)
for (key, filename, value) in files:
L.append('--' + BOUNDARY)
L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (
key, filename))
L.append('Content-Type: %s' % get_content_type(filename))
L.append('')
L.append(value)
L.append('--' + BOUNDARY + '--')
L.append('')
body = CRLF.join(L)
content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
return content_type, body
def get_content_type(filename):
return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
=== Products/FileCacheManager/tests/stresstest.py 1.6 => 1.7 ===
--- Products/FileCacheManager/tests/stresstest.py:1.6 Wed Aug 18 15:03:21 2004
+++ Products/FileCacheManager/tests/stresstest.py Sun Aug 29 06:26:54 2004
@@ -43,7 +43,7 @@
full_path = os.path.join(IMAGE_DIR, test_file)
if os.path.isfile(full_path):
fh = open(full_path, 'rb')
- IMAGES[test_file] = fh.read()
+ IMAGES[test_file] = (fh.read() * 100)
fh.close()
######################
@@ -53,8 +53,8 @@
PYSCRIPT_PARAMS = ''
PYSCRIPT_BODY = """
"""
-READERS=10
-WRITERS=2
+READERS=1
+WRITERS=0
ITERATIONS = 10
# End tweakable items
######################
@@ -144,7 +144,7 @@
for path in self.paths:
root = makerequest.makerequest(self.root)
ob = root.unrestrictedTraverse(path)
- print '.',
+ #print '.',
sys.stdout.flush()
data = ob.index_html(root.REQUEST, root.REQUEST['RESPONSE'])
original_data = IMAGES.get(ob.getId())
@@ -199,7 +199,7 @@
for path in self.paths:
root = makerequest.makerequest(self.root)
ob = root.unrestrictedTraverse(path)
- print ':',
+ #print ':',
sys.stdout.flush()
original_data = IMAGES.get(ob.getId())
ob.manage_upload(file=original_data)
More information about the Zope-CVS
mailing list