[Zope-Checkins] CVS: Zope2 - loadmail.py:1.1.2.1
Jim Fulton
jim@digicool.com
Tue, 3 Apr 2001 11:40:17 -0400 (EDT)
Update of /cvs-repository/Zope2/lib/python/Products/ZCatalog/tests
In directory korak:/tmp/cvs-serv17992
Added Files:
Tag: zope-2_3-branch
loadmail.py
Log Message:
Handy test module (not unit) for testing catalog indexing performance.
--- Added File loadmail.py in package Zope2 ---
"""Test script for exercising various catalog features under load
Usage:
cd lib/python
python Products/ZCatalog/tests/loadmail.py command args
where each command has it's own command-line arguments that it expects.
Note that all of the commands operate on the Zope database,
typically var/Data.fs.
Note that this script uses the proc file system to get memory size.
Many of the commands output performance statisics on lines that look like::
11.3585170507 0.06 2217781L 7212
where the numbers are:
- clock time in seconds
- cpu time used by the main thread, in seconds,
- Database size growth over the test
- Memory growth during the test (if the proc file system is available).
Commands:
base mbox max
Build a base database by:
- Deleting ../../Data.fs
- Starting Zope
- Adding a top-level folder names 'mail'
- Reading up to max messages from the Unix mailbox file, mbox
and adding them as documents to the mail folder.
index threshold
Index all of the DTML documents in the database, committing
sub-transactions after each threshold objects.
If the threshold is less than the number of messages, then the
size of the temporary sub-transaction commit file is output.
inc mbox start end [threads wait]
Incrementally index messages start to end in unix mailbox mbox.
If the threads argument is supplied, then it specifies the
number of threads to use. For example, with:
python Products/ZCatalog/tests/loadmail.py inc mbox 0 200 2
One thread indexes messages 0 to 99 and another thread indexes messages
100 to 199.
If wait is specified, then after each document is indexed, the
thread sleeps a random number of seconds between 0 and 2*wait.
The default wait is 0.25 seconds.
For each thread, a line that looks like::
3.41 (0, 1)
is output, containing:
- The cpu time
- The number of ZODB trabsaction conflicts detected when reading
- The number of ZODB trabsaction conflicts detected when committing
edit edits deletes inserts threads wait
Incrementally edit edits messages from mail. For each message,
do a random number of word deletes between 0 and deletes * 2
and do a random number of inserts between 0 and inserts * 2.
For each thread, a line that looks like::
3.41 (0, 1)
as described above.
pdebug command args
Run one of the other commands in the Python debugger.
sample suite of tests::
cd lib/python
python Products/ZCatalog/tests/loadmail.py base ~/zope.mbox 1000
python Products/ZCatalog/tests/loadmail.py index 100
python Products/ZCatalog/tests/loadmail.py inc ~/python-dev.mbox 0 10 2
python Products/ZCatalog/tests/loadmail.py edit 10 10 10 2
"""
import mailbox, time, sys, os, string
sys.path.insert(0, '.')
import whrandom
whrandom.seed(1,2,3)
from string import strip, find, split, lower, atoi
from urllib import quote
def do(db, f, args, returnf=None):
"""Do something and measure it's impact"""
size=db.getSize()
mem=VmSize()
t=time.time()
c=time.clock()
r=apply(f, args)
t=time.time() - t
c=time.clock() - c
size=db.getSize()-size
mem=VmSize()-mem
if returnf is not None: returnf(t, c, size, mem, r)
else: return t, c, size, mem, r
def loadmessage(dest, message, i, body=None, headers=None):
if body is None: body=message.fp.read()
if headers is None: headers=message.headers
dest.manage_addDTMLDocument(str(i), file=body)
doc=dest[str(i)]
for h in headers:
h=strip(h)
l=find(h,':')
if l <= 0: continue
name=lower(h[:l])
if name=='subject': name='title'
v=strip(h[l+1:])
type='string'
if 0 and name=='date': type='date'
elif 0:
try: atoi(v)
except: pass
else: type=int
if name=='title':
doc.manage_changeProperties(title=h)
else:
try: doc.manage_addProperty(name, v, type)
except: pass
def loadmail(dest, name, mbox, printstat=0, max=99999999):
try:
import Products.BTreeFolder.BTreeFolder
except:
dest.manage_addFolder(name)
else:
Products.BTreeFolder.BTreeFolder.manage_addBTreeFolder(dest, name)
dest=getattr(dest, name)
f=open(mbox)
mb=mailbox.UnixMailbox(f)
i=0
message=mb.next()
while message:
if i > max: break
if i%100 == 0 and printstat:
sys.stdout.write("\t%s\t%s\t\r" % (i, f.tell()))
sys.stdout.flush()
if i and (i%5000 == 0):
get_transaction().commit()
dest._p_jar._cache.invalidate(None)
dest._p_jar._cache.minimize()
loadmessage(dest, message, i)
i=i+1
message=mb.next()
dest.number_of_messages=i
get_transaction().commit()
def loadinc(name, mb, printstat=0, max=99999999, wait=1):
from ZODB.POSException import ConflictError
from time import sleep
from whrandom import uniform
import Zope, sys
rconflicts=wconflicts=0
i=0
message=mb.next()
body=message.fp.read()
headers=list(message.headers)
while i < max:
# sys.stderr.write("%s " % i)
# sys.stdout.flush()
if wait: sleep(uniform(0,wait))
jar=Zope.DB.open()
app=jar.root()['Application']
mdest=getattr(app, name)
if i%100 == 0 and printstat:
sys.stdout.write("\t%s\t%s\t\r" % (i, f.tell()))
sys.stdout.flush()
did=str(i)
try:
loadmessage(mdest, message, did, body, headers)
doc=mdest[did]
app.cat.catalog_object(doc)
except ConflictError, v:
# print v.args
rconflicts=rconflicts+1
get_transaction().abort()
else:
try:
get_transaction().commit()
i=i+1
message=mb.next()
body=message.fp.read()
headers=list(message.headers)
except ConflictError:
wconflicts=wconflicts+1
get_transaction().abort()
doc=app=mdest=0
jar.close()
if printstat: sys.stdout.write("\t%s\t%s\t\n" % (i, f.tell()))
sys.stdout.flush()
return rconflicts, wconflicts
def buildbase():
try: os.unlink('../../var/Data.fs')
except: pass
import Zope, Products.ZCatalog.ZCatalog
app=Zope.app()
Products.ZCatalog.ZCatalog.manage_addZCatalog(app, 'cat', '')
get_transaction().commit()
return app
def base():
try: os.unlink('../../var/Data.fs')
except: pass
import Zope
app=Zope.app()
max=atoi(sys.argv[3])
print do(Zope.DB, loadmail, (app, 'mail', sys.argv[2], 1, max))
Zope.DB.close()
class RE:
def redirect(*args, **kw): pass
def indexf(app):
r=RE()
r.PARENTS=[0, app.mail]
app.cat.manage_catalogFoundItems(r,r,'','',['DTML Document'])
get_transaction().commit()
def index():
os.environ['STUPID_LOG_FILE']=''
os.environ['STUPID_LOG_SEVERITY']='-111'
import Zope, Products.ZCatalog.ZCatalog
import AccessControl.SecurityManagement, AccessControl.SpecialUsers
app=Zope.app()
Products.ZCatalog.ZCatalog.manage_addZCatalog(app, 'cat', '')
app.cat.threshold=atoi(sys.argv[2])
app.cat._catalog.delIndex('bobobase_modification_time')
get_transaction().commit()
system = AccessControl.SpecialUsers.system
AccessControl.SecurityManagement.newSecurityManager(None, system)
r=RE()
r.PARENTS=[app.cat, app]
print do(Zope.DB, indexf, (app,))
hist(sys.argv[2])
Zope.DB.close()
def initmaili(n):
import Zope
app=Zope.app()
try:
import Products.BTreeFolder.BTreeFolder
except:
app.manage_addFolder(n)
else:
Products.BTreeFolder.BTreeFolder.manage_addBTreeFolder(app, n)
get_transaction().commit()
app._p_jar.close()
def hist(n):
import Zope
app=Zope.app()
import cPickle
pickler=cPickle.Pickler(open("h%s.hist" % n, 'w'))
h=app.cat._catalog.indexes['PrincipiaSearchSource'].histogram()
pickler.dump(list(h.items()))
#h=app.cat._catalog.uids.keys()
#pickler.dump(list(h))
def inc():
import Zope, thread
min, max = atoi(sys.argv[3]), atoi(sys.argv[4])
count = max-min
try: threads=atoi(sys.argv[5])
except:
threads=1
wait=0
else:
try: wait=atof(sys.argv[6])
except: wait=0.25
wait = wait * 2
count = count / threads
max = min + count
omin=min
db=Zope.DB
size=db.getSize()
mem=VmSize()
t=time.time()
c=time.clock()
mbox=sys.argv[2]
argss=[]
for i in range(threads):
amin=min+i*count
dest='maili%s' % amin
initmaili(dest)
mb=mailbox.UnixMailbox(open(mbox))
j=0
while j < amin:
mb.next()
j=j+1
lock=thread.allocate_lock()
lock.acquire()
def returnf(t, c, size, mem, r, lock=lock):
print c, r
lock.release()
argss.append((lock, (dest, mb, 0, count, wait), returnf))
for lock, args, returnf in argss:
thread.start_new_thread(do, (Zope.DB, loadinc, args, returnf))
for lock, args, returnf in argss:
lock.acquire()
t=time.time() - t
c=time.clock() - c
size=db.getSize()-size
mem=VmSize()-mem
print t, c, size, mem
hist("%s-%s-%s" % (omin, count, threads))
Zope.DB.close()
words=['banishment', 'indirectly', 'imprecise', 'peeks',
'opportunely', 'bribe', 'sufficiently', 'Occidentalized', 'elapsing',
'fermenting', 'listen', 'orphanage', 'younger', 'draperies', 'Ida',
'cuttlefish', 'mastermind', 'Michaels', 'populations', 'lent',
'cater', 'attentional', 'hastiness', 'dragnet', 'mangling',
'scabbards', 'princely', 'star', 'repeat', 'deviation', 'agers',
'fix', 'digital', 'ambitious', 'transit', 'jeeps', 'lighted',
'Prussianizations', 'Kickapoo', 'virtual', 'Andrew', 'generally',
'boatsman', 'amounts', 'promulgation', 'Malay', 'savaging',
'courtesan', 'nursed', 'hungered', 'shiningly', 'ship', 'presides',
'Parke', 'moderns', 'Jonas', 'unenlightening', 'dearth', 'deer',
'domesticates', 'recognize', 'gong', 'penetrating', 'dependents',
'unusually', 'complications', 'Dennis', 'imbalances', 'nightgown',
'attached', 'testaments', 'congresswoman', 'circuits', 'bumpers',
'braver', 'Boreas', 'hauled', 'Howe', 'seethed', 'cult', 'numismatic',
'vitality', 'differences', 'collapsed', 'Sandburg', 'inches', 'head',
'rhythmic', 'opponent', 'blanketer', 'attorneys', 'hen', 'spies',
'indispensably', 'clinical', 'redirection', 'submit', 'catalysts',
'councilwoman', 'kills', 'topologies', 'noxious', 'exactions',
'dashers', 'balanced', 'slider', 'cancerous', 'bathtubs', 'legged',
'respectably', 'crochets', 'absenteeism', 'arcsine', 'facility',
'cleaners', 'bobwhite', 'Hawkins', 'stockade', 'provisional',
'tenants', 'forearms', 'Knowlton', 'commit', 'scornful',
'pediatrician', 'greets', 'clenches', 'trowels', 'accepts',
'Carboloy', 'Glenn', 'Leigh', 'enroll', 'Madison', 'Macon', 'oiling',
'entertainingly', 'super', 'propositional', 'pliers', 'beneficiary',
'hospitable', 'emigration', 'sift', 'sensor', 'reserved',
'colonization', 'shrilled', 'momentously', 'stevedore', 'Shanghaiing',
'schoolmasters', 'shaken', 'biology', 'inclination', 'immoderate',
'stem', 'allegory', 'economical', 'daytime', 'Newell', 'Moscow',
'archeology', 'ported', 'scandals', 'Blackfoot', 'leery', 'kilobit',
'empire', 'obliviousness', 'productions', 'sacrificed', 'ideals',
'enrolling', 'certainties', 'Capsicum', 'Brookdale', 'Markism',
'unkind', 'dyers', 'legislates', 'grotesquely', 'megawords',
'arbitrary', 'laughing', 'wildcats', 'thrower', 'sex', 'devils',
'Wehr', 'ablates', 'consume', 'gossips', 'doorways', 'Shari',
'advanced', 'enumerable', 'existentially', 'stunt', 'auctioneers',
'scheduler', 'blanching', 'petulance', 'perceptibly', 'vapors',
'progressed', 'rains', 'intercom', 'emergency', 'increased',
'fluctuating', 'Krishna', 'silken', 'reformed', 'transformation',
'easter', 'fares', 'comprehensible', 'trespasses', 'hallmark',
'tormenter', 'breastworks', 'brassiere', 'bladders', 'civet', 'death',
'transformer', 'tolerably', 'bugle', 'clergy', 'mantels', 'satin',
'Boswellizes', 'Bloomington', 'notifier', 'Filippo', 'circling',
'unassigned', 'dumbness', 'sentries', 'representativeness', 'souped',
'Klux', 'Kingstown', 'gerund', 'Russell', 'splices', 'bellow',
'bandies', 'beefers', 'cameramen', 'appalled', 'Ionian', 'butterball',
'Portland', 'pleaded', 'admiringly', 'pricks', 'hearty', 'corer',
'deliverable', 'accountably', 'mentors', 'accorded',
'acknowledgement', 'Lawrenceville', 'morphology', 'eucalyptus',
'Rena', 'enchanting', 'tighter', 'scholars', 'graduations', 'edges',
'Latinization', 'proficiency', 'monolithic', 'parenthesizing', 'defy',
'shames', 'enjoyment', 'Purdue', 'disagrees', 'barefoot', 'maims',
'flabbergast', 'dishonorable', 'interpolation', 'fanatics', 'dickens',
'abysses', 'adverse', 'components', 'bowl', 'belong', 'Pipestone',
'trainees', 'paw', 'pigtail', 'feed', 'whore', 'conditioner',
'Volstead', 'voices', 'strain', 'inhabits', 'Edwin', 'discourses',
'deigns', 'cruiser', 'biconvex', 'biking', 'depreciation', 'Harrison',
'Persian', 'stunning', 'agar', 'rope', 'wagoner', 'elections',
'reticulately', 'Cruz', 'pulpits', 'wilt', 'peels', 'plants',
'administerings', 'deepen', 'rubs', 'hence', 'dissension', 'implored',
'bereavement', 'abyss', 'Pennsylvania', 'benevolent', 'corresponding',
'Poseidon', 'inactive', 'butchers', 'Mach', 'woke', 'loading',
'utilizing', 'Hoosier', 'undo', 'Semitization', 'trigger', 'Mouthe',
'mark', 'disgracefully', 'copier', 'futility', 'gondola', 'algebraic',
'lecturers', 'sponged', 'instigators', 'looted', 'ether', 'trust',
'feeblest', 'sequencer', 'disjointness', 'congresses', 'Vicksburg',
'incompatibilities', 'commend', 'Luxembourg', 'reticulation',
'instructively', 'reconstructs', 'bricks', 'attache', 'Englishman',
'provocation', 'roughen', 'cynic', 'plugged', 'scrawls', 'antipode',
'injected', 'Daedalus', 'Burnsides', 'asker', 'confronter',
'merriment', 'disdain', 'thicket', 'stinker', 'great', 'tiers',
'oust', 'antipodes', 'Macintosh', 'tented', 'packages',
'Mediterraneanize', 'hurts', 'orthodontist', 'seeder', 'readying',
'babying', 'Florida', 'Sri', 'buckets', 'complementary',
'cartographer', 'chateaus', 'shaves', 'thinkable', 'Tehran',
'Gordian', 'Angles', 'arguable', 'bureau', 'smallest', 'fans',
'navigated', 'dipole', 'bootleg', 'distinctive', 'minimization',
'absorbed', 'surmised', 'Malawi', 'absorbent', 'close', 'conciseness',
'hopefully', 'declares', 'descent', 'trick', 'portend', 'unable',
'mildly', 'Morse', 'reference', 'scours', 'Caribbean', 'battlers',
'astringency', 'likelier', 'Byronizes', 'econometric', 'grad',
'steak', 'Austrian', 'ban', 'voting', 'Darlington', 'bison', 'Cetus',
'proclaim', 'Gilbertson', 'evictions', 'submittal', 'bearings',
'Gothicizer', 'settings', 'McMahon', 'densities', 'determinants',
'period', 'DeKastere', 'swindle', 'promptness', 'enablers', 'wordy',
'during', 'tables', 'responder', 'baffle', 'phosgene', 'muttering',
'limiters', 'custodian', 'prevented', 'Stouffer', 'waltz', 'Videotex',
'brainstorms', 'alcoholism', 'jab', 'shouldering', 'screening',
'explicitly', 'earner', 'commandment', 'French', 'scrutinizing',
'Gemma', 'capacitive', 'sheriff', 'herbivore', 'Betsey', 'Formosa',
'scorcher', 'font', 'damming', 'soldiers', 'flack', 'Marks',
'unlinking', 'serenely', 'rotating', 'converge', 'celebrities',
'unassailable', 'bawling', 'wording', 'silencing', 'scotch',
'coincided', 'masochists', 'graphs', 'pernicious', 'disease',
'depreciates', 'later', 'torus', 'interject', 'mutated', 'causer',
'messy', 'Bechtel', 'redundantly', 'profoundest', 'autopsy',
'philosophic', 'iterate', 'Poisson', 'horridly', 'silversmith',
'millennium', 'plunder', 'salmon', 'missioner', 'advances', 'provers',
'earthliness', 'manor', 'resurrectors', 'Dahl', 'canto', 'gangrene',
'gabler', 'ashore', 'frictionless', 'expansionism', 'emphasis',
'preservations', 'Duane', 'descend', 'isolated', 'firmware',
'dynamites', 'scrawled', 'cavemen', 'ponder', 'prosperity', 'squaw',
'vulnerable', 'opthalmic', 'Simms', 'unite', 'totallers', 'Waring',
'enforced', 'bridge', 'collecting', 'sublime', 'Moore', 'gobble',
'criticizes', 'daydreams', 'sedate', 'apples', 'Concordia',
'subsequence', 'distill', 'Allan', 'seizure', 'Isadore', 'Lancashire',
'spacings', 'corresponded', 'hobble', 'Boonton', 'genuineness',
'artifact', 'gratuities', 'interviewee', 'Vladimir', 'mailable',
'Bini', 'Kowalewski', 'interprets', 'bereave', 'evacuated', 'friend',
'tourists', 'crunched', 'soothsayer', 'fleetly', 'Romanizations',
'Medicaid', 'persevering', 'flimsy', 'doomsday', 'trillion',
'carcasses', 'guess', 'seersucker', 'ripping', 'affliction',
'wildest', 'spokes', 'sheaths', 'procreate', 'rusticates', 'Schapiro',
'thereafter', 'mistakenly', 'shelf', 'ruination', 'bushel',
'assuredly', 'corrupting', 'federation', 'portmanteau', 'wading',
'incendiary', 'thing', 'wanderers', 'messages', 'Paso', 'reexamined',
'freeings', 'denture', 'potting', 'disturber', 'laborer', 'comrade',
'intercommunicating', 'Pelham', 'reproach', 'Fenton', 'Alva', 'oasis',
'attending', 'cockpit', 'scout', 'Jude', 'gagging', 'jailed',
'crustaceans', 'dirt', 'exquisitely', 'Internet', 'blocker', 'smock',
'Troutman', 'neighboring', 'surprise', 'midscale', 'impart',
'badgering', 'fountain', 'Essen', 'societies', 'redresses',
'afterwards', 'puckering', 'silks', 'Blakey', 'sequel', 'greet',
'basements', 'Aubrey', 'helmsman', 'album', 'wheelers', 'easternmost',
'flock', 'ambassadors', 'astatine', 'supplant', 'gird', 'clockwork',
'foxes', 'rerouting', 'divisional', 'bends', 'spacer',
'physiologically', 'exquisite', 'concerts', 'unbridled', 'crossing',
'rock', 'leatherneck', 'Fortescue', 'reloading', 'Laramie', 'Tim',
'forlorn', 'revert', 'scarcer', 'spigot', 'equality', 'paranormal',
'aggrieves', 'pegs', 'committeewomen', 'documented', 'interrupt',
'emerald', 'Battelle', 'reconverted', 'anticipated', 'prejudices',
'drowsiness', 'trivialities', 'food', 'blackberries', 'Cyclades',
'tourist', 'branching', 'nugget', 'Asilomar', 'repairmen', 'Cowan',
'receptacles', 'nobler', 'Nebraskan', 'territorial', 'chickadee',
'bedbug', 'darted', 'vigilance', 'Octavia', 'summands', 'policemen',
'twirls', 'style', 'outlawing', 'specifiable', 'pang', 'Orpheus',
'epigram', 'Babel', 'butyrate', 'wishing', 'fiendish', 'accentuate',
'much', 'pulsed', 'adorned', 'arbiters', 'counted', 'Afrikaner',
'parameterizes', 'agenda', 'Americanism', 'referenda', 'derived',
'liquidity', 'trembling', 'lordly', 'Agway', 'Dillon', 'propellers',
'statement', 'stickiest', 'thankfully', 'autograph', 'parallel',
'impulse', 'Hamey', 'stylistic', 'disproved', 'inquirer', 'hoisting',
'residues', 'variant', 'colonials', 'dequeued', 'especial', 'Samoa',
'Polaris', 'dismisses', 'surpasses', 'prognosis', 'urinates',
'leaguers', 'ostriches', 'calculative', 'digested', 'divided',
'reconfigurer', 'Lakewood', 'illegalities', 'redundancy',
'approachability', 'masterly', 'cookery', 'crystallized', 'Dunham',
'exclaims', 'mainline', 'Australianizes', 'nationhood', 'pusher',
'ushers', 'paranoia', 'workstations', 'radiance', 'impedes',
'Minotaur', 'cataloging', 'bites', 'fashioning', 'Alsop', 'servants',
'Onondaga', 'paragraph', 'leadings', 'clients', 'Latrobe',
'Cornwallis', 'excitingly', 'calorimetric', 'savior', 'tandem',
'antibiotics', 'excuse', 'brushy', 'selfish', 'naive', 'becomes',
'towers', 'popularizes', 'engender', 'introducing', 'possession',
'slaughtered', 'marginally', 'Packards', 'parabola', 'utopia',
'automata', 'deterrent', 'chocolates', 'objectives', 'clannish',
'aspirin', 'ferociousness', 'primarily', 'armpit', 'handfuls',
'dangle', 'Manila', 'enlivened', 'decrease', 'phylum', 'hardy',
'objectively', 'baskets', 'chaired', 'Sepoy', 'deputy', 'blizzard',
'shootings', 'breathtaking', 'sticking', 'initials', 'epitomized',
'Forrest', 'cellular', 'amatory', 'radioed', 'horrified', 'Neva',
'simultaneous', 'delimiter', 'expulsion', 'Himmler', 'contradiction',
'Remus', 'Franklinizations', 'luggage', 'moisture', 'Jews',
'comptroller', 'brevity', 'contradictions', 'Ohio', 'active',
'babysit', 'China', 'youngest', 'superstition', 'clawing', 'raccoons',
'chose', 'shoreline', 'helmets', 'Jeffersonian', 'papered',
'kindergarten', 'reply', 'succinct', 'split', 'wriggle', 'suitcases',
'nonce', 'grinders', 'anthem', 'showcase', 'maimed', 'blue', 'obeys',
'unreported', 'perusing', 'recalculate', 'rancher', 'demonic',
'Lilliputianize', 'approximation', 'repents', 'yellowness',
'irritates', 'Ferber', 'flashlights', 'booty', 'Neanderthal',
'someday', 'foregoes', 'lingering', 'cloudiness', 'guy', 'consumer',
'Berkowitz', 'relics', 'interpolating', 'reappearing', 'advisements',
'Nolan', 'turrets', 'skeletal', 'skills', 'mammas', 'Winsett',
'wheelings', 'stiffen', 'monkeys', 'plainness', 'braziers', 'Leary',
'advisee', 'jack', 'verb', 'reinterpret', 'geometrical', 'trolleys',
'arboreal', 'overpowered', 'Cuzco', 'poetical', 'admirations',
'Hobbes', 'phonemes', 'Newsweek', 'agitator', 'finally', 'prophets',
'environment', 'easterners', 'precomputed', 'faults', 'rankly',
'swallowing', 'crawl', 'trolley', 'spreading', 'resourceful', 'go',
'demandingly', 'broader', 'spiders', 'Marsha', 'debris', 'operates',
'Dundee', 'alleles', 'crunchier', 'quizzical', 'hanging', 'Fisk']
from ZODB.utils import u64
def incedit(edits, wait, ndel=20, nins=20):
import Zope, whrandom, string, time
from ZODB.POSException import ConflictError
rconflicts=wconflicts=0
did=str(edits.pop())
while edits:
if wait: time.sleep(whrandom.uniform(0,wait))
jar=Zope.DB.open()
app=jar.root()['Application']
doc=getattr(app.mail, did)
text=string.split(doc.raw)
n=whrandom.randint(0,ndel*2)
for j in range(n):
if len(text) < 2: break
j=whrandom.randint(0,len(text)-1)
#del text[j]
n=whrandom.randint(0,nins*2)
for j in range(n):
word=whrandom.choice(words)
text.append(word)
doc.raw=string.join(text)
try: app.cat.catalog_object(doc)
except ConflictError, v:
#print v.args
rconflicts=rconflicts+1
get_transaction().abort()
else:
try:
get_transaction().commit()
did=str(edits.pop())
except ConflictError:
wconflicts=wconflicts+1
get_transaction().abort()
doc=app=0
jar.close()
return rconflicts, wconflicts
def edit():
import Zope, thread
nedit, ndel, nins = atoi(sys.argv[2]), atoi(sys.argv[3]), atoi(sys.argv[4])
try: threads=atoi(sys.argv[5])
except:
threads=1
wait=0
else:
try: wait=atof(sys.argv[6])
except: wait=0.25
wait = wait * 2
if threads==1: start_new_thread=apply
else: start_new_thread=thread.start_new_thread
db=Zope.DB
app=Zope.app()
number_of_messages=app.mail.number_of_messages
app._p_jar.close()
size=db.getSize()
mem=VmSize()
t=time.time()
c=time.clock()
alledits={}
argss=[]
for i in range(threads):
lock=thread.allocate_lock()
if threads > 1:
lock.acquire()
def returnf(t, c, size, mem, r, lock=lock):
print c, r
lock.release()
else:
def returnf(t, c, size, mem, r, lock=lock):
print c, r
edits=[0]
while len(edits) <= nedit:
edit=whrandom.randint(0, number_of_messages)
if not alledits.has_key(edit):
alledits[edit]=1
edits.append(edit)
#print edits
argss.append((lock, (edits, wait, ndel, nins), returnf))
for lock, args, returnf in argss:
start_new_thread(do, (Zope.DB, incedit, args, returnf))
for lock, args, returnf in argss:
lock.acquire()
t=time.time() - t
c=time.clock() - c
size=db.getSize()-size
mem=VmSize()-mem
print t, c, size, mem
#hist("e%s" % (threads))
Zope.DB.close()
def VmSize():
try: f=open('/proc/%s/status' % os.getpid())
except: return 0
else:
l=filter(lambda l: l[:7]=='VmSize:', f.readlines())
if l:
l=string.split(string.strip(l[0][7:]))[0]
return string.atoi(l)
return 0
def pdebug():
import pdb
del sys.argv[1]
pdb.run('globals()[sys.argv[1]]()')
if __name__=='__main__':
try: f=globals()[sys.argv[1]]
except:
print __doc__
sys.exit(1)
else: f()