[Zope3-checkins] CVS: Zope3/src/zope/fssync - fsmerger.py:1.1 fsutil.py:1.1 fssync.py:1.16 main.py:1.12 merger.py:1.7
Guido van Rossum
guido@python.org
Wed, 14 May 2003 18:16:40 -0400
Update of /cvs-repository/Zope3/src/zope/fssync
In directory cvs.zope.org:/tmp/cvs-serv8605
Modified Files:
fssync.py main.py merger.py
Added Files:
fsmerger.py fsutil.py
Log Message:
More refactoring.
The new FSMerger class has some unit tests
(though not nearly enough).
=== Added File Zope3/src/zope/fssync/fsmerger.py ===
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Higher-level three-way file and directory merger.
$Id: fsmerger.py,v 1.1 2003/05/14 22:16:09 gvanrossum Exp $
"""
import os
from os.path import exists, isfile, isdir, split, join
from os.path import realpath, normcase, normpath
from zope.fssync.merger import Merger
from zope.fssync import fsutil
class FSMerger(object):
"""Higher-level three-way file and directory merger."""
def __init__(self, metadata, reporter):
"""Constructor.
Arguments are a metadata database and a reporting function.
"""
self.metadata = metadata
self.reporter = reporter
self.merger = Merger(metadata)
def merge(self, local, remote):
"""Merge remote file or directory into local file or directory."""
if ((isfile(local) or not exists(local))
and
(isfile(remote) or not exists(remote))):
self.merge_files(local, remote)
elif ((isdir(local) or not exists(local))
and
(isdir(remote) or not exists(remote))):
self.merge_dirs(local, remote)
else:
# One is a file, the other is a directory
# XXX We should be able to deal with this case, too
self.reporter("XXX %s" % local)
self.merge_extra(local, remote)
self.merge_annotations(local, remote)
def merge_extra(self, local, remote):
lextra = fsutil.getextra(local)
rextra = fsutil.getextra(remote)
self.merge_dirs(lextra, rextra)
def merge_annotations(self, local, remote):
lannotations = fsutil.getannotations(local)
rannotations = fsutil.getannotations(remote)
self.merge_dirs(lannotations, rannotations)
def merge_files(self, local, remote):
"""Merge remote file into local file."""
original = fsutil.getoriginal(local)
action, state = self.merger.classify_files(local, original, remote)
state = self.merger.merge_files(local, original, remote,
action, state) or state
self.reportaction(action, state, local)
def merge_dirs(self, localdir, remotedir):
"""Merge remote directory into local directory."""
lnames = self.metadata.getnames(localdir)
rnames = self.metadata.getnames(remotedir)
lentry = self.metadata.getentry(localdir)
rentry = self.metadata.getentry(remotedir)
if not lnames and not rnames:
if not lentry:
if not rentry:
if exists(localdir):
self.reportdir("?", localdir)
else:
if not exists(localdir):
fsutil.ensuredir(localdir)
self.reportdir("N", localdir)
else:
self.reportdir("*", localdir)
return
if lentry.get("flag") == "added":
if not rentry:
self.reportdir("A", localdir)
else:
self.reportdir("U", localdir)
del lentry["flag"]
return
if lentry.get("flag") == "removed":
if rentry:
self.reportdir("R", localdir)
else:
self.reportdir("D", localdir)
lentry.clear()
return
if not rentry:
try:
os.rmdir(localdir)
except os.error:
pass
self.reportdir("D", localdir)
lentry.clear()
return
if exists(localdir):
self.reportdir("/", localdir)
lnames = dict([(normcase(name), name)
for name in os.listdir(localdir)])
else:
if lentry.get("flag") != "removed" and (rentry or rnames):
fsutil.ensuredir(localdir)
lentry.update(rentry)
self.reportdir("N", localdir)
lnames = {}
if exists(remotedir):
rnames = dict([(normcase(name), name)
for name in os.listdir(remotedir)])
else:
rnames = {}
names = {}
names.update(lnames)
names.update(rnames)
nczope = normcase("@@Zope")
if nczope in names:
del names[nczope]
ncnames = names.keys()
ncnames.sort()
for ncname in ncnames:
name = names[ncname]
self.merge(join(localdir, name), join(remotedir, name))
def reportdir(self, letter, localdir):
"""Helper to report something for a directory.
This adds a separator (e.g. '/') to the end of the pathname to
signal that it is a directory.
"""
self.reporter("%s %s" % (letter, join(localdir, "")))
def reportaction(self, action, state, local):
"""Helper to report an action and a resulting state.
This always results in exactly one line being reported.
Report letters are:
C -- conflicting changes not resolved (not committed)
U -- file brought up to date (possibly created)
M -- modified (not committed)
A -- added (not committed)
R -- removed (not committed)
D -- file deleted
? -- file exists locally but not remotely
* -- nothing happened
"""
assert action in ('Fix', 'Copy', 'Merge', 'Delete', 'Nothing'), action
assert state in ('Conflict', 'Uptodate', 'Modified', 'Spurious',
'Added', 'Removed', 'Nonexistent'), state
letter = "*"
if state == "Conflict":
letter = "C"
elif state == "Uptodate":
if action in ("Copy", "Fix", "Merge"):
letter = "U"
elif state == "Modified":
letter = "M"
elif state == "Added":
letter = "A"
elif state == "Removed":
letter = "R"
elif state == "Spurious":
if not self.ignore(local):
letter = "?"
elif state == "Nonexistent":
if action == "Delete":
letter = "D"
if letter:
self.reporter("%s %s" % (letter, local))
def ignore(self, path):
# XXX This should have a larger set of default patterns to
# ignore, and honor .cvsignore
return path.endswith("~")
=== Added File Zope3/src/zope/fssync/fsutil.py ===
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""A few common items that don't fit elsewhere, it seems.
Classes:
- Error -- an exception
Functions:
- getoriginal(path)
- getextra(path)
- getannotations(path)
- getspecial(path, what)
- split(path)
- ensuredir(dir)
Variables:
- unwanted -- a sequence containing the pseudo path components "", ".", ".."
$Id: fsutil.py,v 1.1 2003/05/14 22:16:09 gvanrossum Exp $
"""
import os
class Error(Exception):
"""User-level error, e.g. non-existent file.
This can be used in several ways:
1) raise Error("message")
2) raise Error("message %r %r" % (arg1, arg2))
3) raise Error("message %r %r", arg1, arg2)
4) raise Error("message", arg1, arg2)
- Forms 2-4 are equivalent.
- Form 4 assumes that "message" contains no % characters.
- When using forms 2 and 3, all % formats are supported.
- Form 2 has the disadvantage that when you specify a single
argument that happens to be a tuple, it may get misinterpreted.
- The message argument is required.
- Any number of arguments after that is allowed.
"""
def __init__(self, msg, *args):
self.msg = msg
self.args = args
def __str__(self):
msg, args = self.msg, self.args
if args:
if "%" in msg:
msg = msg % args
else:
msg += " "
msg += " ".join(map(repr, args))
return str(msg)
def __repr__(self):
return "%s%r" % (self.__class__.__name__, (self.msg,)+self.args)
unwanted = ("", os.curdir, os.pardir)
def getoriginal(path):
"""Return the path of the Original file corresponding to path."""
return getspecial(path, "Original")
def getextra(path):
"""Return the path of the Extra directory corresponding to path."""
return getspecial(path, "Extra")
def getannotations(path):
"""Return the path of the Annotations directory corresponding to path."""
return getspecial(path, "Annotations")
def getspecial(path, what):
"""Helper for getoriginal(), getextra(), getannotations()."""
head, tail = os.path.split(path)
return os.path.join(head, "@@Zope", what, tail)
def split(path):
"""Split a path, making sure that the tail returned is real."""
head, tail = os.path.split(path)
if tail in unwanted:
newpath = os.path.normpath(path)
head, tail = os.path.split(newpath)
if tail in unwanted:
newpath = os.path.realpath(path)
head, tail = os.path.split(newpath)
if head == newpath or tail in unwanted:
raise Error("path '%s' is the filesystem root", path)
if not head:
head = os.curdir
return head, tail
def ensuredir(path):
"""Make sure that the given path is a directory, creating it if necessary.
This may raise OSError if the creation operation fails.
"""
if not os.path.isdir(path):
os.makedirs(path)
=== Zope3/src/zope/fssync/fssync.py 1.15 => 1.16 ===
--- Zope3/src/zope/fssync/fssync.py:1.15 Wed May 14 15:18:15 2003
+++ Zope3/src/zope/fssync/fssync.py Wed May 14 18:16:09 2003
@@ -35,53 +35,10 @@
from os.path import dirname, basename, split, join
from os.path import realpath, normcase, normpath
-from zope.xmlpickle import loads, dumps
-from zope.fssync.compare import classifyContents
from zope.fssync.metadata import Metadata
-from zope.fssync.merger import Merger
-
-unwanted = ("", os.curdir, os.pardir)
-
-class Error(Exception):
- """User-level error, e.g. non-existent file.
-
- This can be used in several ways:
-
- 1) raise Error("message")
- 2) raise Error("message %r %r" % (arg1, arg2))
- 3) raise Error("message %r %r", arg1, arg2)
- 4) raise Error("message", arg1, arg2)
-
- - Forms 2-4 are equivalent.
-
- - Form 4 assumes that "message" contains no % characters.
-
- - When using forms 2 and 3, all % formats are supported.
-
- - Form 2 has the disadvantage that when you specify a single
- argument that happens to be a tuple, it may get misinterpreted.
-
- - The message argument is required.
-
- - Any number of arguments after that is allowed.
- """
-
- def __init__(self, msg, *args):
- self.msg = msg
- self.args = args
-
- def __str__(self):
- msg, args = self.msg, self.args
- if args:
- if "%" in msg:
- msg = msg % args
- else:
- msg += " "
- msg += " ".join(map(repr, args))
- return str(msg)
-
- def __repr__(self):
- return "%s%r" % (self.__class__.__name__, (self.msg,)+self.args)
+from zope.fssync.fsmerger import FSMerger
+from zope.fssync.fsutil import Error
+from zope.fssync import fsutil
class Network(object):
@@ -149,7 +106,7 @@
if data:
return data
head, tail = split(dir)
- if tail in unwanted:
+ if tail in fsutil.unwanted:
break
dir = head
return None
@@ -308,7 +265,7 @@
raise Error("target already registered", target)
if exists(target) and not isdir(target):
raise Error("target should be a directory", target)
- self.ensuredir(target)
+ fsutil.ensuredir(target)
fp, headers = self.network.httpreq(rootpath, "@@toFS.zip")
try:
self.merge_zipfile(fp, target)
@@ -366,7 +323,7 @@
if not entry:
raise Error("nothing known about", target)
self.network.loadrooturl(target)
- head, tail = self.split(target)
+ head, tail = fsutil.split(target)
path = entry["path"]
fp, headers = self.network.httpreq(path, "@@toFS.zip")
try:
@@ -387,7 +344,11 @@
os.mkdir(tmpdir)
cmd = "cd %s; unzip -q %s" % (tmpdir, zipfile)
sts, output = commands.getstatusoutput(cmd)
- self.merge_dirs(localdir, tmpdir)
+ if sts:
+ raise Error("unzip failed:\n%s" % output)
+ m = FSMerger(self.metadata, self.reporter)
+ m.merge(localdir, tmpdir)
+ self.metadata.flush()
print "All done."
finally:
if isdir(tmpdir):
@@ -396,6 +357,10 @@
if isfile(zipfile):
os.remove(zipfile)
+ def reporter(self, msg):
+ if msg[0] not in "/*":
+ print msg
+
def diff(self, target, mode=1, diffopts=""):
assert mode == 1, "modes 2 and 3 are not yet supported"
entry = self.metadata.getentry(target)
@@ -408,7 +373,7 @@
return
if not isfile(target):
raise Error("diff target '%s' is file nor directory", target)
- orig = self.getorig(target)
+ orig = fsutil.getoriginal(target)
if not isfile(orig):
raise Error("can't find original for diff target '%s'", target)
if self.cmp(target, orig):
@@ -433,7 +398,7 @@
entry = self.metadata.getentry(path)
if entry:
raise Error("path '%s' is already registered", path)
- head, tail = self.split(path)
+ head, tail = fsutil.split(path)
pentry = self.metadata.getentry(head)
if not pentry:
raise Error("can't add '%s': its parent is not registered", path)
@@ -468,142 +433,3 @@
else:
entry["flag"] = "removed"
self.metadata.flush()
-
- def merge_dirs(self, localdir, remotedir):
- if not isdir(remotedir):
- return
-
- self.ensuredir(localdir)
-
- ldirs, lnondirs = classifyContents(localdir)
- rdirs, rnondirs = classifyContents(remotedir)
-
- dirs = {}
- dirs.update(ldirs)
- dirs.update(rdirs)
-
- nondirs = {}
- nondirs.update(lnondirs)
- nondirs.update(rnondirs)
-
- def sorted(d): keys = d.keys(); keys.sort(); return keys
-
- merger = Merger(self.metadata)
-
- for x in sorted(dirs):
- local = join(localdir, x)
- if x in nondirs:
- # Too weird to handle
- print "should '%s' be a directory or a file???" % local
- continue
- remote = join(remotedir, x)
- lentry = self.metadata.getentry(local)
- rentry = self.metadata.getentry(remote)
- if lentry or rentry:
- if x not in ldirs:
- os.mkdir(local)
- self.merge_dirs(local, remote)
-
- for x in sorted(nondirs):
- if x in dirs:
- # Error message was already printed by previous loop
- continue
- local = join(localdir, x)
- origdir = join(localdir, "@@Zope", "Original")
- self.ensuredir(origdir)
- orig = join(origdir, x)
- remote = join(remotedir, x)
- action, state = merger.classify_files(local, orig, remote)
- state = merger.merge_files(local, orig, remote, action, state)
- self.report(action, state, local)
- self.merge_extra(local, remote)
- self.merge_annotations(local, remote)
-
- self.merge_extra(localdir, remotedir)
- self.merge_annotations(localdir, remotedir)
-
- lentry = self.metadata.getentry(localdir)
- rentry = self.metadata.getentry(remotedir)
- lentry.update(rentry)
-
- self.metadata.flush()
-
- def merge_extra(self, local, remote):
- lextra = self.getextra(local)
- rextra = self.getextra(remote)
- if isdir(rextra):
- self.merge_dirs(lextra, rextra)
-
- def merge_annotations(self, local, remote):
- lannotations = self.getannotations(local)
- rannotations = self.getannotations(remote)
- if isdir(rannotations):
- self.merge_dirs(lannotations, rannotations)
-
- def report(self, action, state, local):
- letter = None
- if state == "Conflict":
- letter = "C"
- elif state == "Uptodate":
- if action in ("Copy", "Fix", "Merge"):
- letter = "U"
- elif state == "Modified":
- letter = "M"
- entry = self.metadata.getentry(local)
- conflict_mtime = entry.get("conflict")
- if conflict_mtime:
- if conflict_mtime == os.path.getmtime(local):
- letter = "C"
- else:
- del entry["conflict"]
- elif state == "Added":
- letter = "A"
- elif state == "Removed":
- letter = "R"
- elif state == "Spurious":
- if not self.ignore(local):
- letter = "?"
- elif state == "Nonexistent":
- if action == "Delete":
- print "local file '%s' is no longer relevant" % local
- if letter:
- print letter, local
-
- def ignore(self, path):
- # XXX This should have a larger set of default patterns to
- # ignore, and honor .cvsignore
- return path.endswith("~")
-
- def cmp(self, f1, f2):
- try:
- return filecmp.cmp(f1, f2, shallow=False)
- except (os.error, IOError):
- return False
-
- def ensuredir(self, dir):
- if not isdir(dir):
- os.makedirs(dir)
-
- def getextra(self, path):
- return self.getspecial(path, "Extra")
-
- def getannotations(self, path):
- return self.getspecial(path, "Annotations")
-
- def getorig(self, path):
- return self.getspecial(path, "Original")
-
- def getspecial(self, path, what):
- head, tail = self.split(path)
- return join(head, "@@Zope", what, tail)
-
- def split(self, path):
- head, tail = split(path)
- if tail in unwanted:
- newpath = realpath(path)
- head, tail = split(newpath)
- if head == newpath or tail in unwanted:
- raise Error("path '%s' is the filesystem root", path)
- if not head:
- head = os.curdir
- return head, tail
=== Zope3/src/zope/fssync/main.py 1.11 => 1.12 ===
--- Zope3/src/zope/fssync/main.py:1.11 Wed May 14 10:40:50 2003
+++ Zope3/src/zope/fssync/main.py Wed May 14 18:16:09 2003
@@ -54,7 +54,8 @@
srcdir = join(rootdir, "src")
sys.path.append(srcdir)
-from zope.fssync.fssync import Error, FSSync
+from zope.fssync.fsutil import Error
+from zope.fssync.fssync import FSSync
class Usage(Error):
"""Subclass for usage error (command-line syntax).
@@ -123,9 +124,9 @@
print >>sys.stderr, "for help use --help"
return 2
- except Error, msg:
- print >>sys.stderr, msg
- return 1
+## except Error, msg:
+## print >>sys.stderr, msg
+## return 1
else:
return None
=== Zope3/src/zope/fssync/merger.py 1.6 => 1.7 ===
--- Zope3/src/zope/fssync/merger.py:1.6 Wed May 14 15:20:20 2003
+++ Zope3/src/zope/fssync/merger.py Wed May 14 18:16:09 2003
@@ -23,7 +23,8 @@
import filecmp
import commands
-from os.path import exists, isfile
+from os.path import exists, isfile, dirname
+from zope.fssync import fsutil
class Merger(object):
"""Augmented three-way file merges.
@@ -59,10 +60,11 @@
actions are:
Fix -- copy the remote copy to the local original, nothing else
- Copy -- copy the remote copy over the local copy
+ Copy -- copy the remote copy over the local copy and original
Merge -- merge the remote copy into the local copy
- (this may cause merge conflicts when tried)
- Delete -- delete the local copy
+ (this may cause merge conflicts when executed);
+ copy the remote copy to the local original
+ Delete -- delete the local copy and original
Nothing -- do nothing
The original file is made a copy of the remote file for actions
@@ -131,7 +133,7 @@
def merge_files_nothing(self, local, original, remote):
return None
- def merge_files_remove(self, local, original, remote):
+ def merge_files_delete(self, local, original, remote):
if isfile(local):
os.remove(local)
if isfile(original):
@@ -141,6 +143,7 @@
def merge_files_copy(self, local, original, remote):
shutil.copy(remote, local)
+ fsutil.ensuredir(dirname(original))
shutil.copy(remote, original)
self.getentry(local).update(self.getentry(remote))
self.clearflag(local)
@@ -192,8 +195,8 @@
Return a pair of strings (action, state) where action is one
of 'Fix', 'Copy', 'Merge', 'Delete' or 'Nothing', and state is
- one of 'Conflict', 'Uptodate', 'Modified', 'Added', 'Removed'
- or 'Nonexistent'.
+ one of 'Conflict', 'Uptodate', 'Modified', 'Added', 'Removed',
+ 'Spurious' or 'Nonexistent'.
"""
lmeta = self.getentry(local)
@@ -228,7 +231,7 @@
if lmeta.get("flag") == "removed":
if not rmeta:
# Removed remotely too
- return ("Remove", "Nonexistent")
+ return ("Delete", "Nonexistent")
else:
# Removed locally
if self.cmpfile(original, remote):
@@ -239,14 +242,14 @@
if lmeta and not rmeta:
assert lmeta.get("flag") is None
# Removed remotely
- return ("Remove", "Nonexistent")
+ return ("Delete", "Nonexistent")
if lmeta.get("flag") is None and not exists(local):
# Lost locally
if rmeta:
return ("Copy", "Uptodate")
else:
- return ("Remove", "Nonexistent")
+ return ("Delete", "Nonexistent")
# Sort out cases involving simple changes to files