#!/usr/bin/env python
#
# rdiff-backup -- Mirror files while keeping incremental changes
# Version 0.3.0 released August 29, 2001
# Copyright (C) 2001 Ben Escoto <bescoto@stanford.edu>
#
# This program is licensed under the GNU General Public License (GPL).
# See http://www.gnu.org/copyleft/gpl.html for details.
#
# Please send me mail if you find bugs or have any suggestions.

from __future__ import nested_scopes, generators
import os, stat, time, sys, tempfile, getopt, re, cPickle, types, shutil


#######################################################################
#
# globals - aggregate some configuration options
#

class Globals:

	# The current version of rdiff-backup
	version = "0.3.0"
	
	# This determines how many bytes to read at a time when copying
	blocksize = 32768

	# True if script is running as a server
	server = None

	# If true, when copying attributes, also change target's uid/gid
	change_ownership = None

	# If true, change the permissions of unwriteable mirror files
	# (such as directories) so that they can be written, and then
	# change them back.
	change_mirror_perms = 1

	# If true, temporarily change permissions of unreadable files in
	# the source directory to make sure we can read all files.
	change_source_perms = None

	# This is a list of compiled regular expressions.  If one of them
	# matches a file in the source area, do not process that file.
	exclude_regexps = [re.compile("/proc")]

	# Another list of compiled regexps; this time the file is excluded
	# if it matches something in the destination area.
	exclude_mirror_regexps = []

	# This can be set to Rdiff.copy or RPath.copy depending on
	# preferences.  It should be set in the Main() class.
	copy_func = None


#######################################################################
#
# static - MakeStatic and MakeClass
#
# These functions are used to make all the instance methods in a class
# into static or class methods.
#

class StaticMethodsError(Exception):
	pass

def MakeStatic(cls):
	"""turn instance methods into static ones

	The methods (that don't begin with _) of any class that
	subclasses this will be turned into static methods.

	"""
	for name in dir(cls):
		if name[0] != "_":
			cls.__dict__[name] = staticmethod(cls.__dict__[name])


def MakeClass(cls):
	"""Turn instance methods into classmethods.  Ignore _ like above"""
	for name in dir(cls):
		if name[0] != "_":
			cls.__dict__[name] = classmethod(cls.__dict__[name])


#######################################################################
#
# lazy - Define some lazy data structures and functions acting on them
#

class Iter:
	"""Hold static methods for the manipulation of lazy iterators"""

	def filter(predicate, iterator):
		"""Like filter in a lazy functional programming language"""
		for i in iterator:
			if predicate(i): yield i

	def map(function, iterator):
		"""Like map in a lazy functional programming language"""
		for i in iterator: yield function(i)

	def foreach(function, iterator):
		"""Run function on each element in iterator"""
		for i in iterator: function(i)

	def cat(*iters):
		"""Lazily concatenate iterators"""
		for iter in iters:
			for i in iter: yield i

	def cat2(iter_of_iters):
		"""Lazily concatenate iterators, iterated by big iterator"""
		for iter in iter_of_iters:
			for i in iter: yield i

	def empty(iter):
		"""True if iterator has length 0"""
		for i in iter: return None
		return 1

	def equal(iter1, iter2, operator = lambda x, y: x == y):
		"""True if iterator 1 has same elements as iterator 2

		Use equality operator, or == if it is unspecified.

		"""
		for i in iter1:
			try:
				if not operator(i, iter2.next()): return None
			except StopIteration: return None
		try:
			iter2.next()
			return None
		except StopIteration: return 1

	def Or(iter):
		"""True if any element in iterator is true.  Short circuiting"""
		i = None
		for i in iter:
			if i: return i
		return i

	def And(iter):
		"""True if all elements in iterator are true.  Short circuiting"""
		i = 1
		for i in iter:
			if not i: return i
		return i

	def len(iter):
		"""Return length of iterator"""
		i = 0
		while 1:
			try: iter.next()
			except StopIteration: return i
			i = i+1

	def foldr(f, default, iter):
		"""foldr the "fundamental list recursion operator"?"""
		try: next = iter.next()
		except StopIteration: return default
		return f(next, Iter.foldr(f, default, iter))

	def foldl(f, default, iter):
		"""the fundamental list iteration operator.."""
		while 1:
			try: next = iter.next()
			except StopIteration: return default
			default = f(default, next)


MakeStatic(Iter)


class Tree:
	"""Tree data structure designed to represent directories

	Each tree has one label, and an arbitrary number of branches
	(other trees) and leaves.  This class is not meant to be used
	directly, but is just a template to be subclassed.

	"""
	def __init__(self, node, branches = []):
		self._node = node
		self._branches = branches

	def node(self):
		return self._node

	def branches(self):
		for branch in self._branches: yield branch

	def __iter__(self):
		"""Return an iterator which traverses the Tree depth first"""
		for branch in self.branches():
			for i in branch: yield i
		yield self.node()

	def newtree(self, nodefunc, branchesfunc):
		"""Return new tree with the given node/leaves/branches function"""
		ft = self.__class__(None)
		ft.node = nodefunc
		ft.branches = branchesfunc
		return ft

	def for_node(self, command):
		"""Execute command on all nodes of the tree"""
		command(self.node())
		for branch in self.branches(): branch.for_node(command)

	def for_node_after(self, command):
		"""Same as above but process node afterwards"""
		for branch in self.branches(): branch.for_node_after(command)
		command(self.node())

	def for_node_twice(self, command):
		"""Same as above but do node before and afterwards, if any branches"""
		command(self.node())
		empty = 1
		for branch in self.branches():
			branch.for_node_twice(command)
			empty = None
		if not empty: command(self.node())

	def node_filter(self, predicate):
		"""Prune tree by testing predicate on nodes

		Empty tree is None.  Creates a new tree that is missing all
		trees whose nodes don't match predicate

		"""
		if not predicate(self.node()): return None
		return self.newtree(self.node,
							lambda: 
							Iter.map(lambda tree: tree.node_filter(predicate),
									 Iter.filter(lambda b:
												 predicate(b.node()),
												 self.branches())))

	def filter(self, predicate):
		"""This time only remove branches if they fail predicate"""
		if not predicate(self): return None
		return self.newtree(self.node,
							lambda: Iter.filter(lambda x: x,
										Iter.map(lambda b: b.filter(predicate),
												 self.branches())))

	def reduce(self, f):
		"""folding/reduction function for Tree"""
		return f(self.node(), Iter.map(lambda b: b.reduce(f), self.branches()))

	def bare(self):
		"""True if self has no node, leaves, or branches with nodes etc."""
		return self.reduce(lambda node, reduced_branches: node is None and
						   Iter.And(reduced_branches))

	def remove_bare_branches(self):
		"""Return new tree with bare branches removed"""
		return self.filter(lambda tree: not tree.bare())

	def equal(self, tree2):
		return (self.node() == tree2.node() and
				Iter.equal(self.branches(), tree2.branches, Tree.equal))


#######################################################################
#
# log - Manage logging 
#

class Logger:
	"""All functions which deal with logging"""
	def __init__(self):
		self.logfp = self.logrp = None
		self.verbosity = self.term_verbosity = 3
		# termverbset is true if the term_verbosity has been explicity set
		self.termverbset = None

	def setverbosity(self, verbosity):
		"""Set verbosity levels"""
		self.verbosity = verbosity
		if not self.termverbset: self.term_verbosity = verbosity

	def setterm_verbosity(self, termverb):
		"""Set verbosity to terminal"""
		self.term_verbosity = termverb
		self.termverbset = 1

	def open_logfile(self, rpath):
		"""Sets rpath to be the logfile.  May be on remote side."""
		self.logrp = rpath
		self.logfp = rpath.open("a")

	def close_logfile(self):
		if self.logfp: self.logfp.close()

	def format(self, message, verbosity):
		"""Format the message, possibly adding date information"""
		if verbosity < 9: return message + "\n"
		else: return "%s  %s\n" % (time.asctime(time.localtime(time.time())),
								   message)

	def __call__(self, message, verbosity):
		"""Log message that has verbosity importance"""
		if verbosity <= self.verbosity: self.log_to_file(message)
		if verbosity <= self.term_verbosity:
			self.log_to_term(message, verbosity)

	def log_to_file(self, message):
		"""Write the message to the log file, if possible"""
		if self.logfp: self.logfp.write(self.format(message, self.verbosity))

	def log_to_term(self, message, verbosity):
		"""Write message to stdout/stderr"""
		if verbosity <= 2 or Globals.server: termfp = sys.stderr
		else: termfp = sys.stdout
		termfp.write(self.format(message, self.term_verbosity))

	def conn(self, message):
		"""Log a message about connection - only to terminal"""
		if self.term_verbosity >= 9: self.log_to_term(message, 9)

	def FatalError(self, message):
		self("Fatal Error: " + message, 1)
		sys.exit(1)

Log = Logger()


#######################################################################
#
# ttime - Provide Time class, which contains time related functions.
#

class TimeError(Exception): pass

class Time:
	"""Functions which act on the time"""
	def setcurtime(cls):
		"""Sets the current time in curtime and curtimestr"""
		cls.curtime = time.time()
		cls.curtimestr = cls.timetostring(cls.curtime)

	def setprevtime(cls, timeinseconds):
		"""Sets the previous inc time in prevtime and prevtimestr"""
		cls.prevtime = timeinseconds
		cls.prevtimestr = cls.timetostring(timeinseconds)

	def timetostring(cls, timeinseconds):
		"""Return w3 datetime compliant listing of timeinseconds"""
		return time.strftime("%Y-%m-%dT%H:%M:%S",
							 time.localtime(timeinseconds)) + cls.gettzd()

	def stringtotime(cls, timestring):
		"""Return time in seconds from w3 timestring

		If there is an error parsing the string, or it doesn't look
		like a w3 datetime string, return None.

		"""
		try:
			date, daytime = timestring[:19].split("T")
			year, month, day = map(int, date.split("-"))
			hour, minute, second = map(int, daytime.split(":"))
			assert 1900 < year < 2100, year
			assert 1 <= month <= 12
			assert 1 <= day <= 31
			assert 0 <= hour <= 23
			assert 0 <= minute <= 59
			assert 0 <= second <= 61  # leap seconds
			timetuple = (year, month, day, hour, minute, second, -1, -1, -1)
			if time.daylight:
				utc_in_secs = time.mktime(timetuple) - time.altzone
			else: utc_in_secs = time.mktime(timetuple) - time.timezone

			return utc_in_secs + cls.tzdtoseconds(timestring[19:])
		except (TypeError, ValueError, AssertionError): return None

	def gettzd(cls):
		"""Return w3's timezone identification string.

		Expresed as [+/-]hh:mm.  For instance, PST is -08:00.  Zone is
		coincides with what localtime(), etc., use.

		"""
		if time.daylight: offset = -1 * time.altzone/60
		else: offset = -1 * time.timezone/60
		if offset > 0: prefix = "+"
		elif offset < 0: prefix = "-"
		else: return "Z" # time is already in UTC

		hours, minutes = map(abs, divmod(offset, 60))
		assert 0 <= hours <= 23
		assert 0 <= minutes <= 59
		return prefix + "%02d:%02d" % (hours, minutes)

	def tzdtoseconds(cls, tzd):
		"""Given w3 compliant TZD, return how far ahead UTC is"""
		if tzd == "Z": return 0
		assert len(tzd) == 6 # only accept forms like +08:00 for now
		assert (tzd[0] == "-" or tzd[0] == "+") and tzd[3] == ":"
		return -60 * (60 * int(tzd[:3]) + int(tzd[4:]))

	def cmp(cls, time1, time2):
		"""Compare time1 and time2 and return -1, 0, or 1"""
		if type(time1) is types.StringType:
			time1 = cls.stringtotime(time1)
			assert time1 is not None
		if type(time2) is types.StringType:
			time2 = cls.stringtotime(time2)
			assert time2 is not None
		
		if time1 < time2: return -1
		elif time1 == time2: return 0
		else: return 1

MakeClass(Time)


#######################################################################
#
# connection - Code that deals with remote execution
#

class ConnectionError(Exception):
	pass

class ConnectionQuit(Exception):
	pass


class Connection:
	"""Connection class - represent remote execution

	The idea is that, if c is an instance of this class, c.foo will
	return the object on the remote side.  For functions, c.foo will
	return a function that, when called, executes foo on the remote
	side, sending over the arguments and sending back the result.

	"""
	pass


class LocalConnection(Connection):
	"""Local connection

	This is a dummy connection class, so that LC.foo just evaluates to
	foo using global scope.

	"""
	def __getattr__(self, name):
		try: return globals()[name]
		except KeyError:
			builtins = globals()["__builtins__"]
			try:
				if type(builtins) is types.ModuleType:
					return builtins.__dict__[name]
				else: return builtins[name]
			except KeyError: raise NameError, name

	def __setattr__(self, name, value):
		globals()[name] = value

	def __delattr__(self, name):
		del globals()[name]

	def rexec(self, command):
		exec command in globals()

	def quit(self): pass


class LowLevelPipeConnection(Connection):
	"""Routines for just sending objects from one side of pipe to another"""
	def __init__(self, inpipe, outpipe):
		"""inpipe is a file-type open for reading, outpipe for writing"""
		self.inpipe = inpipe
		self.outpipe = outpipe

	def _put(self, obj):
		"""Put an object into the pipe (will send raw if string)"""
		if type(obj) is types.StringType: self._putbuf(obj)
		elif type(obj) is types.FileType or isinstance(obj, VirtualFile):
			self._putfile(obj)
		else: self._putobj(obj)

	def _putobj(self, obj):
		"""Send a python obj down the outpipe"""
		self._write("o", cPickle.dumps(obj, 1))

	def _putbuf(self, buf):
		"""Send buffer buf down the outpipe"""
		self._write("b", buf)

	def _putfile(self, fp):
		"""Send a file to the client using virtual files"""
		assert self.server
		self._write("f", str(VirtualFile.new(fp)))

	def _putquit(self):
		"""Send a string that takes down server"""
		self._write("q", "")

	def _write(self, headerchar, data):
		"""Write header and then data to the pipe"""
		self.outpipe.write(headerchar + self._l2s(len(data)))
		self.outpipe.write(data)
		self.outpipe.flush()

	def _read(self, length):
		"""Read length bytes from inpipe, returning result"""
		return self.inpipe.read(length)

	def _s2l(self, s):
		"""Convert string to long int"""
		assert len(s) == 7
		l = 0L
		for i in range(7): l = l*256 + ord(s[i])
		return l

	def _l2s(self, l):
		"""Convert long int to string"""
		s = ""
		for i in range(7):
			l, remainder = divmod(l, 256)
			s = chr(remainder) + s
		assert remainder == 0
		return s

	def _get(self):
		"""Read an object from the pipe and return value"""
		header_string = self.inpipe.read(8)
		try:
			format_string, length = (header_string[0],
									 self._s2l(header_string[1:]))
		except IndexError: raise ConnectionError()
		if format_string == "o":
			result = cPickle.loads(self._read(length))
			if isinstance(result, Exception): raise result
			else: return result
		elif format_string == "b": return self._read(length)
		elif format_string == "f":
			return VirtualFile(self, int(self._read(length)))
		else:
			assert format_string == "q"
			raise ConnectionQuit("Received quit signal")

	def _close(self):
		"""Close the pipes associated with the connection"""
		self.inpipe.close()
		self.outpipe.close()


class PipeConnection(LowLevelPipeConnection):
	"""Provide server and client functions for a Pipe Connection

	The server enters a loop where it reads a tuple ("string", int),
	where int a the number of arguments.  It then reads int objects
	from the pipe, and evaluates the result of "string" (a callable
	object) on the objects read.

	The client side acts as if is a module that allows for remote
	execution.  For instance, self.conn.pow(2,8) will execute the
	operation on the server side.

	"""
	def Server(self):
		"""Start server's read eval return loop"""
		self.server = 1
		Log("Starting server", 6)
		while 1:
			try: funcstring, numargs = self._get()
			except ConnectionQuit: break
			arguments = []
			for i in range(numargs): arguments.append(self._get())
			try: result = apply(eval(funcstring), arguments)
			except: result = sys.exc_info()[1]
			self._put(result)

	def reval(*args):
		"""Execute command on remote side

		The first argument should be a string that evaluates to a
		function, like "pow", and the remaining are arguments to that
		function.

		"""
		self = args[0]
		self._put((args[1], len(args) - 2))
		for arg in args[2:]: self._put(arg)
		return self._get()

	def quit(self):
		"""Close the associated pipes and tell server side to quit"""
		assert not Globals.server
		self._putquit()
		self._close()

	def __getattr__(self, name):
		"""Intercept attributes to allow for . invocation"""
		return EmulateCallable(self, name)
			

class EmulateCallable:
	"""This is used by PipeConnection in calls like conn.os.chmod(foo)"""
	def __init__(self, connection, name):
		self.connection = connection
		self.name = name
	def __call__(*args):
		self = args[0]
		return apply(self.connection.reval, (self.name,) + args[1:])
	def __getattr__(self, attr_name):
		return EmulateCallable(self.connection,
							   "%s.%s" % (self.name, attr_name))


class VirtualFile:
	"""When the client asks for a file over the connection, it gets this

	The returned instance then forwards requests over the connection.
	The class's dictionary is used by the server to associate each
	with a unique file number.

	"""
	#### The following are used by the server
	vfiles = {}
	counter = 0

	def getbyid(cls, id):
		return cls.vfiles[id]
	getbyid = classmethod(getbyid)

	def readfromid(cls, id, length):
		return cls.vfiles[id].read(length)
	readfromid = classmethod(readfromid)

	def writetoid(cls, id, buffer):
		return cls.vfiles[id].write(buffer)
	writetoid = classmethod(writetoid)

	def closebyid(cls, id):
		fp = cls.vfiles[id]
		del cls.vfiles[id]
		return fp.close()
	closebyid = classmethod(closebyid)

	def new(cls, fileobj):
		"""Associate a new VirtualFile with a read fileobject, return id"""
		count = cls.counter
		cls.vfiles[count] = fileobj
		cls.counter = count + 1
		return count
	new = classmethod(new)


	#### And these are used by the client
	def __init__(self, connection, id):
		self.connection = connection
		self.id = id

	def read(self, length = -1):
		return self.connection.VirtualFile.readfromid(self.id, length)

	def write(self, buf):
		return self.connection.VirtualFile.writetoid(self.id, buf)

	def close(self):
		return self.connection.VirtualFile.closebyid(self.id)


#######################################################################
#
# rpath - Wrapper class around a real path like "/usr/bin/env"
#
# The RPath and associated classes make some function calls more
# convenient (e.g. RPath.getperms()) and also make working with files
# on remote systems transparent.
#

class RPathException(Exception):
	pass


class RPathStatic:
	"""Contains static methods for use with RPaths"""
	def copyfileobj(inputfp, outputfp):
		"""Copies file inputfp to outputfp in blocksize intervals"""
		blocksize = Globals.blocksize
		while 1:
			inbuf = inputfp.read(blocksize)
			outputfp.write(inbuf)
			if len(inbuf) < blocksize: break

	def cmpfileobj(fp1, fp2):
		"""True if file objects fp1 and fp2 contain same data"""
		blocksize = Globals.blocksize
		while 1:
			buf1 = fp1.read(blocksize)
			buf2 = fp2.read(blocksize)
			if buf1 != buf2: return None
			elif not buf1: return 1

	def check_for_files(*rps):
		"""Make sure that all the rps exist, raise error if not"""
		for rp in rps:
			if not rp.lstat():
				raise RPathException("File %s does not exist" % rp.path)

	def copy(rpin, rpout):
		"""Copy RPath rpin to rpout.  Works for symlinks, dirs, etc."""
		Log("Regular copying %s to %s" % (rpin.path, rpout.path), 6)
		if not rpin.lstat():
			raise RPathException, ("File %s does not exist" % rpin.path)

		if rpout.lstat():
			if rpin.isreg() or not RPath.cmp(rpin, rpout):
				rpout.delete()   # easier to write that compare
			else: return
			
		if rpin.isreg(): RPath.copy_reg_file(rpin, rpout)
		elif rpin.isdir(): rpout.mkdir()
		elif rpin.issym(): rpout.symlink(rpin.readlink())
		elif rpin.ischardev():
			major, minor = rpin.getdevnums()
			rpout.makedev("c", major, minor)
		elif rpin.isblkdev():
			major, minor = rpin.getdevnums()
			rpout.makedev("b", major, minor)
		elif rpin.isfifo(): rpout.mkfifo()
		elif rpin.issock(): sys.stderr.write("Found socket %s, ignoring\n" %
											 rpin.path)
		else: raise RPathException("File %s has unknown type" % rpin.path)

	def copy_reg_file(rpin, rpout):
		"""Copy regular file rpin to rpout, possibly avoiding connection"""
		if rpout.conn is rpin.conn:
			rpout.conn.shutil.copyfile(rpin.path, rpout.path)
			rpout.clearstats()
		else: rpout.write_from_fileobj(rpin.open("rb"))

	def cmp(rpin, rpout):
		"""True if rpin has the same data as rpout

		cmp does not compare file ownership, permissions, or times.
		"""
		RPath.check_for_files(rpin, rpout)
		if rpin.isreg():
			if not rpout.isreg(): return None
			fp1, fp2 = rpin.open("rb"), rpout.open("rb")
			result = RPathStatic.cmpfileobj(fp1, fp2)
			fp1.close()
			fp2.close()
			return result
		elif rpin.isdir(): return rpout.isdir()
		elif rpin.issym():
			return rpout.issym() and (rpin.readlink() == rpout.readlink())
		elif rpin.ischardev():
			return rpout.ischardev() and \
				   (rpin.getdevnums() == rpout.getdevnums())
		elif rpin.isblkdev():
			return rpout.isblkdev() and \
				   (rpin.getdevnums() == rpout.getdevnums())
		elif rpin.isfifo(): return rpout.isfifo()
		elif rpin.issock(): return rpout.issock()
		else: raise RPathException("File %s has unknown type" % rpin.path)

	def copy_attribs(rpin, rpout):
		"""Change file attributes of rpout to match rpin

		Only changes the chmoddable bits, uid/gid ownership, and
		timestamps, so both must already exist.

		"""
		RPath.check_for_files(rpin, rpout)
		s = rpin.lstat()
		if rpin.issym(): return # symlinks have no valid attributes
		if Globals.change_ownership: apply(rpout.chown, rpin.getuidgid())
		rpout.chmod(rpin.getperms())
		apply(rpout.settime, rpin.gettime())

	def cmp_attribs(rp1, rp2):
		"""True if rp1 has the same file attributes as rp2

		Does not compare file access times.

		"""
		RPath.check_for_files(rp1, rp2)
		result = ((rp1.getuidgid() == rp2.getuidgid()) and
				  ((rp1.gettime()[1] == rp2.gettime()[1]) or
				   (rp1.issym() and rp2.issym())) and  #timeset on syms broken?
				  rp1.getperms() == rp2.getperms())
		Log("Compare attribs %s and %s: %s" % (rp1.path, rp2.path, result), 7)
		return result

	def copy_with_attribs(rpin, rpout):
		"""Copy file and then copy over attributes"""
		RPath.copy(rpin, rpout)
		RPath.copy_attribs(rpin, rpout)

	def quick_cmp_with_attribs(rp1, rp2):
		"""Quicker version of cmp_with_attribs

		Instead of reading all of each file, assume that regular files
		are the same if the attributes compare.

		"""
		if not RPath.cmp_attribs(rp1, rp2): return None
		if rp1.isreg() and rp2.isreg() and (rp1.getlen() == rp2.getlen()):
			return 1
		return RPath.cmp(rp1, rp2)

	def cmp_with_attribs(rp1, rp2):
		"""Combine cmp and cmp_attribs"""
		return RPath.cmp(rp1, rp2) and RPath.cmp_attribs(rp1, rp2)

	def rename(rp_source, rp_dest):
		"""Rename rp_source to rp_dest"""
		assert rp_source.conn is rp_dest.conn
		rp_source.conn.os.rename(rp_source.path, rp_dest.path)
		rp_source.clearstats()
		rp_dest.clearstats()

MakeStatic(RPathStatic)



class RPath(RPathStatic):
	"""Remote Path class - wrapper around a possibly non-local pathname

	This class caches lstat calls (in self.statblock), the time in
	seconds the filename indicates (self.filetime), and directory
	listings (self.dirlist).

	"""
	normalfile_regexp = re.compile(r"[a-zA-Z0-9\.\-_]+$")

	def __init__(self, connection, pathname):
		"""RPath constructor

		connection = self.conn is the Connection the RPath will use to
		make system calls, and pathname is the string representing the
		file on that connection.

		"""
		self.conn = connection
		self.path = pathname
		self.statted = None
		self.filetime = None

	def lstat(self):
		"""Return output of os.lstat, caching results"""
		if not self.statted:
			try: self.statblock = self.conn.os.lstat(self.path)
			except os.error: self.statblock = None
			self.statted = 1
		return self.statblock

	def clearstats(self):
		"""Clear any cached stats

		This should be called by any methods which change lstat's output.
		"""
		self.statted = None

	def isdir(self):
		"""True if self is a dir"""
		return self.lstat() and stat.S_ISDIR(self.lstat()[stat.ST_MODE])

	def isreg(self):
		"""True if self is a regular file"""
		return self.lstat() and stat.S_ISREG(self.lstat()[stat.ST_MODE])

	def issym(self):
		"""True if path is of a symlink"""
		return self.lstat() and stat.S_ISLNK(self.lstat()[stat.ST_MODE])

	def isfifo(self):
		"""True if path is a fifo"""
		return self.lstat() and stat.S_ISFIFO(self.lstat()[stat.ST_MODE])

	def ischardev(self):
		"""True if path is a character device file"""
		return self.lstat() and stat.S_ISCHR(self.lstat()[stat.ST_MODE])

	def isblkdev(self):
		"""True if path is a block device file"""
		return self.lstat() and stat.S_ISBLK(self.lstat()[stat.ST_MODE])

	def issock(self):
		"""True if path is a socket"""
		return self.lstat() and stat.S_ISSOCK(self.lstat()[stat.ST_MODE])

	def gettype(self):
		"""Return a type like "dir", "reg", etc..."""
		if self.isdir(): return "dir"
		elif self.isreg(): return "reg"
		elif self.issym(): return "sym"
		elif self.isfifo(): return "fifo"
		elif self.ischardev() or self.isblkdev(): return "dev"
		elif self.issock(): return "sock"
		else: raise RPathException("Unknown type for %s" % self.path)

	def getperms(self):
		"""Return permission block of file"""
		statblock = self.lstat()
		if not statblock: raise os.error
		else: return stat.S_IMODE(statblock[stat.ST_MODE])

	def hasfullperms(self):
		"""Return true if owner has full permissions on the file"""
		return self.getperms() % 01000 >= 0700

	def readable(self):
		"""Return true if owner has read permissions on the file"""
		return self.getperms() % 01000 >= 0400

	def executable(self):
		"""Return true if owner has execute permissions"""
		return self.getperms() % 0200 >= 0100

	def make_readable(self, thunk):
		"""Use this when you want to read a possibly unreadable file

		thunk is a function with no arguments.  make_readable will
		save the perms, make it readable, run the thunk, and then
		restore everything.

		"""
		if self.isreg() and not self.readable():
			Log("Making %s temporarily readable" % self.path, 6)
			perms = self.getperms()
			atime, mtime = self.gettime()
			self.chmod(perms + 0400)
			result = thunk()
			self.chmod(perms)
			self.settime(atime, mtime)
			return result
		else: return thunk()

	def getsize(self):
		"""Return length of file in bytes"""
		return self.lstat()[stat.ST_SIZE]

	def getuidgid(self):
		"""Return userid/groupid of file"""
		s = self.lstat()
		return s[stat.ST_UID], s[stat.ST_GID]

	def gettime(self):
		"""Return (access time, mod time) pair"""
		s = self.lstat()
		return int(s[stat.ST_ATIME]), int(s[stat.ST_MTIME])

	def chmod(self, permissions):
		"""Wrapper around os.chmod"""
		self.conn.os.chmod(self.path, permissions)
		self.clearstats()

	def settime(self, accesstime, modtime):
		"""Change file modification times"""
		Log("Setting time of %s to %d" % (self.path, modtime), 7)
		self.conn.os.utime(self.path, (accesstime, modtime))
		self.clearstats()

	def chown(self, uid, gid):
		"""Set file's uid and gid"""
		self.conn.os.chown(self.path, uid, gid)
		self.clearstats()

	def mkdir(self):
		Log("Making directory " + self.path, 6)
		self.conn.os.mkdir(self.path)
		self.clearstats()

	def rmdir(self):
		Log("Removing directory " + self.path, 6)
		self.conn.os.rmdir(self.path)
		self.clearstats()

	def listdir(self):
		"""Return list of string paths returned by os.listdir"""
		return self.conn.os.listdir(self.path)

	def readlink(self):
		"""Wrapper around os.readlink()"""
		return self.conn.os.readlink(self.path)

	def symlink(self, linktext):
		"""Make symlink at self.path pointing to linktext"""
		self.conn.os.symlink(linktext, self.path)
		self.clearstats()

	def mkfifo(self):
		"""Make a fifo at self.path"""
		self.conn.os.mkfifo(self.path)
		self.clearstats()

	def touch(self):
		"""Make sure file at self.path exists"""
		Log("Touching " + self.path, 7)
		self.conn.open(self.path, "w").close()
		self.clearstats()

	def delete(self):
		"""Delete file at self.path

		The destructive stepping allows this function to delete
		directories even if they have files and we lack permissions.

		"""
		if self.isdir():
			def helper(rp):
				if rp.isdir(): rp.rmdir()
				else: rp.delete()
			RPTree(self).destructive_stepping().for_node_after(helper)
		else: self.conn.os.unlink(self.path)
		self.clearstats()

	def quote(self):
		"""Return quoted self.path for use with os.system()"""
		if self.normalfile_regexp.match(self.path): return self.path

		ql = []
		for char in self.path:
			ql.append("\\" + oct(ord(char))[1:])
		return "$'%s'" % "".join(ql)

	def normalize(self):
		"""Return RPath canonical version of self.path

		This just means that redundant /'s will be removed, including
		the trailing one, even for directories.  ".." components will
		be retained.

		"""
		newpath = "/".join(filter(lambda x: x and x != ".",
								  self.path.split("/")))
		if self.path[0] == "/": newpath = "/" + newpath
		elif not newpath: newpath = "."
		return RPath(self.conn, newpath)

	def dirsplit(self):
		"""Returns a tuple of strings (dirname, basename)

		Basename is never '' unless self is root, so it is unlike
		os.path.basename.  If path is just above root (so dirname is
		root), then dirname is ''.  In all other cases dirname is not
		the empty string.  Also, dirsplit depends on the format of
		self, so basename could be ".." and dirname could be a
		subdirectory.  For an atomic relative path, dirname will be
		'.'.

		"""
		normed = self.normalize()
		if normed.path.find("/") == -1: return (".", normed.path)
		comps = normed.path.split("/")
		return "/".join(comps[:-1]), comps[-1]

	def append(self, ext):
		"""Return new RPath with same connection by adjoing ext"""
		return RPath(self.conn, os.path.join(self.path, ext))

	def add_inc_ext(self, typestr):
		"""Return new RPath with same connection and inc extension"""
		return RPath(self.conn, "%s.%s.%s" % (self.path, Time.prevtimestr,
											  typestr))

	def open(self, mode):
		"""Return open file.  Supports modes "w" and "r"."""
		return self.conn.open(self.path, mode)

	def write_from_fileobj(self, fp):
		"""Reads fp and writes to self.path.  Closes both when done"""
		Log("Writing file object to " + self.path, 7)
		assert not self.lstat()
		outfp = self.open("wb")
		RPath.copyfileobj(fp, outfp)
		fp.close()
		outfp.close()
		self.clearstats()

	def isincfile(self):
		"""Return true if path looks like an increment file"""
		dotsplit = self.path.split(".")
		if len(dotsplit) < 3: return None
		timestring, ext = dotsplit[-2:]
		if Time.stringtotime(timestring) is None: return None
		return (ext == "snapshot" or ext == "dir" or
				ext == "missing" or ext == "diff")

	def getinctype(self):
		"""Return type of an increment file"""
		return self.path.split(".")[-1]

	def getinctime(self):
		"""Return timestring of an increment file"""
		return self.path.split(".")[-2]
	
	def getincbase(self):
		"""Return the base filename of an increment file in rp form"""
		return RPath(self.conn, ".".join(self.path.split(".")[:-2]))

	def makedev(self, type, major, minor):
		"""Make a special file with specified type, and major/minor nums"""
		self.conn.os.system("mknod %s %s %d %d" %
							(self.quote(), type, major, minor))
		self.clearstats()

	def getdevnums(self):
		"""Return tuple for special file (type, major, minor)"""
		statpipe = self.conn.os.popen("stat -t %s" % self.quote())
		statoutputlist = statpipe.read().split()
		statpipe.close()
		return tuple(map(lambda x: int(x, 16), statoutputlist[9:11]))

	def getlen(self):
		"""Return the length of regular file in bytes"""
		return self.lstat()[stat.ST_SIZE]


#######################################################################
#
# trees - Apply the Tree class (defined in lazy) to directory trees
#

class RPTree(Tree):
	"""Make a lazy directory tree from a input RPath directory

	The label is the RPath, the branches, if any, correspond to other
	RPath directories, and the leaves are non-dir RPaths.  The
	branches and leaves will be sorted by filename.

	"""
	def __init__(self, rp):
		self.rp = rp

	def node(self): return self.rp

	def branches(self):
		if not self.rp.isdir(): dirlist = []
		else:
			dirlist = self.rp.listdir()
			dirlist.sort()
		for filename in dirlist:
			yield RPTree(self.rp.append(filename))

	def cmp(self, rpt):
		"""True if this tree is equal to rpt"""
		if (self.node().lstat() and not rpt.node().lstat() or
			not self.node().lstat() and rpt.node().lstat()):
			Log("Of %s and %s, one exists and the other doesn't." %
				(self.node().path, rpt.node().path), 7)
			return None
		
		if not RPath.cmp_attribs(self.node(), rpt.node()):
			Log("%s and %s have different attributes" %
				(self.node().path, rpt.node().path), 7)
			return None

		if not self.node().make_readable(lambda:
					 rpt.node().make_readable(lambda:
						 RPath.cmp(self.node(), rpt.node()))):
			Log("data of %s and %s different" %
				(self.node().path, rpt.node().path), 7)
			return None

		if not Iter.equal(self.branches(), rpt.branches(), RPTree.cmp):
			Log("Unequal branches for %s and %s" %
			(self.node().path, rpt.node().path), 7)
			return None
		return 1

	def cmp_with_ds(self, rpt):
		"""Like cmp, but compensate for bad permissions"""
		return self.destructive_stepping().cmp(rpt.destructive_stepping())

	def filter_excludes(self, regexps):
		"""Return RTT filtered by given regexps"""
		def isnotexcluded(rp):
			for regexp in regexps:
				if regexp.match(rp.path): return None
			return 1
		return self.node_filter(isnotexcluded)

	def destructive_stepping(self):
		"""See the version in TripleTree.  This is only here for testing"""
		def new_branches(rp, old_branches):
			if not rp.isdir():
				for branch in old_branches: yield branch
				return

			perms = None
			if not rp.hasfullperms():
				perms, (atime, mtime) = rp.getperms(), rp.gettime()
				rp.chmod(0700)
			for branch in old_branches: yield branch
			if perms is not None:
				rp.chmod(perms)
				rp.settime(atime, mtime)

		def reduction_helper(node, reduced_branches):
			outtree = self.__class__(node)
			outtree.branches = lambda: new_branches(node, reduced_branches)
			return outtree
		return self.reduce(reduction_helper)

class RPTriple:
	"""Three rpaths container used for incrementing"""
	def __init__(self, new, mirror, inc = None):
		"""RPTriple initializer

		As the name implies, each of these is supposed to be a rpath.
		New will point to a file in the area to be backed up, mirror
		in the part which mirrors the new part, and inc is the
		basefilename of the increment file which records old mirrors.
		inc can be omitted for use in mirroring.

		"""
		self.new, self.mirror, self.inc = new, mirror, inc

	def add_ext(self, extension):
		"""Make new RPTriple by adding extension to all rpaths"""
		return RPTriple(self.new.append(extension),
						self.mirror.append(extension),
						self.inc and self.inc.append(extension))

	def tuple(self): return self.new, self.mirror, self.inc

	def make_mirror(self):
		"""Turns self.mirror into a copy of self.new"""
		if not self.new.lstat() and not self.mirror.lstat(): return
		Log("Mirroring %s to %s" % (self.new.path, self.mirror.path), 5)
		if not self.new.lstat():
			Log("Deleting mirror file %s" % self.mirror.path, 5)
			self.mirror.delete()
			return

		# Compensate for unreadable source files
		def helper():
			if Globals.change_source_perms:
				self.new.make_readable(lambda: Globals.copy_func(self.new,
																 self.mirror))
			else: Globals.copy_func(self.new, self.mirror)
		if self.mirror.isreg() and Globals.change_mirror_perms:
			self.mirror.make_readable(helper)
		else: helper()
			
		if self.mirror.lstat():
			RPath.copy_attribs(self.new, self.mirror)

	def needs_updating(self):
		"""True if one is missing or they have different attribs"""
		return not (self.new.lstat() and self.mirror.lstat() and
					RPath.quick_cmp_with_attribs(self.new, self.mirror))
	

class TripleTree(Tree):
	"""Tree for RPTriples

	This tree combines three RPTrees to form a tree of RPTriples.  The
	tree will have a branch or leaf with an ending filename if either
	the newtree or the mirrortree has one.

	"""
	def __init__(self, rptriple):
		self.rptriple = rptriple
		self.dirlist = None

	def node(self): return self.rptriple

	def setdirlist(self):
		"Set dirlisting to be combination of new and mirror dirlistings"
		dirdict = {}
		if self.rptriple.new.isdir():
			for filename in self.rptriple.new.listdir():
				dirdict[filename] = 1
		if self.rptriple.mirror.isdir():
			for filename in self.rptriple.mirror.listdir():
				dirdict[filename] = 1
		self.dirlist = dirdict.keys()
		self.dirlist.sort()

	def branches(self):
		if self.dirlist is None: self.setdirlist()
		for filename in self.dirlist:
			yield TripleTree(self.rptriple.add_ext(filename))

	def filter_excludes(self):
		"""Return TT filtered by exclude regexps"""
		def isnotexcluded(rpt):
			for regexp in Globals.exclude_regexps:
				if regexp.match(rpt.new.path):
					Log("Excluding %s" % rpt.new.path, 5)
					return None
			for regexp in Globals.exclude_mirror_regexps:
				if regexp.match(rpt.mirror.path):
					Log("Excluding %s" % rpt.mirror.path, 5)
					return None
			return 1
		return self.node_filter(isnotexcluded)

	def destructive_stepping(self):
		"""Return a version of self that manages directory permissions

		The problem is that we may be trying to mirror a directory we
		don't have read access to, or to write to one without read
		access.  We need to change the permissions when we go in, and
		restore them coming out.  A function on rptriples can use this
		iterator without worring about this issues.

		"""
		def new_branches(new, mirror, old_branches):
			if not new.isdir() and not mirror.isdir():
				for branch in old_branches: yield branch
				return

			if not mirror.isdir():
				if mirror.lstat(): mirror.delete()
				mirror.mkdir()
			new_perms, mirror_perms = None, None
			if Globals.change_mirror_perms and not mirror.hasfullperms():
				Log("Adjusting permissions of %s" % mirror.path, 6)
				mirror_perms = mirror.getperms()
				mirror_atime, mirror_mtime = mirror.gettime()
				mirror.chmod(0700)
			if (Globals.change_source_perms and new.isdir()
				and not (new.readable() and new.executable())):
				Log("Adjusting permissions of %s" % new.path, 6)
				new_perms = new.getperms()
				new_atime, new_mtime = new.gettime()
				new.chmod(0500)

			for branch in old_branches: yield branch

			if new_perms is not None:
				Log("Restoring permissions of %s" % new.path, 6)
				new.chmod(new_perms)
				new.settime(new_atime, new_mtime)
			if mirror_perms is not None:
				Log("Restoring permissions of %s" % mirror.path, 6)
				mirror.chmod(mirror_perms)
				mirror.settime(mirror_atime, mirror_mtime)

		def reduction_helper(node, reduced_branches):
			outtree = self.__class__(node)
			outtree.branches = lambda: new_branches(node.new, node.mirror,
													reduced_branches)
			return outtree
		return self.reduce(reduction_helper)

	def enumerate_changed(self):
		"""Return iterator that lists rptriples that need changing

		The node of a tree with branches may be listed twice, so that
		the function can fix it afterwards, if relevant mtimes were
		changed by writes into the directory.

		"""
		def reduction_func(rpt, iter_iter_rp):
			if rpt.needs_updating(): yield rpt
			empty = 1
			for iter_rp in iter_iter_rp:
				for rp in iter_rp:
					empty = None
					yield rp
			if not empty: yield rpt
		return self.reduce(reduction_func)


class RestoreStruct:
	"""What each element in a RestoreTree is

	Contains three rpaths (the target, increment basename (used for
	directories), and the mirror file) and a list, which can be empty,
	of related increment files.

	"""
	def __init__(self, mirror, incbase, inclist, target):
		self.mirror = mirror
		self.incbase = incbase
		self.inclist = inclist
		self.target = target

	def add_ext(self, extension, inclist):
		"""Return new rs by appending extension and with new inclist"""
		return RestoreStruct(self.mirror.append(extension),
							 self.incbase.append(extension), inclist,
							 self.target.append(extension))


class RestoreTree(Tree):
	"""This is what the restore operation acts on"""
	def __init__(self, restorestruct):
		self.rs = restorestruct
		self.extlist = None

	def node(self): return self.rs

	def setextlist(self):
		"""self.extlist will be a listing of all relevant extensions

		These are: anything in the mirror area, directories in the
		increment area, and the basenames of any increment files.
		Note that this is part of an algorithm which is quadratic,
		when it should be linear - tell me if it matters.

		"""
		extdict = {}
		if self.rs.mirror.isdir():
			for filename in self.rs.mirror.listdir():
				extdict[filename] = 1
		if self.rs.incbase.isdir():
			self.rpincdirlist = map(lambda fn: self.rs.incbase.append(fn),
									self.rs.incbase.listdir())
			for rp in self.rpincdirlist:
				if rp.isdir(): extdict[filename] = 1
				elif rp.isincfile():
					extdict[rp.getincbase().dirsplit()[1]] = 1
		else: self.rpincdirlist = []

		self.extlist = extdict.keys()
		self.extlist.sort()

	def get_incrps(self, basename):
		"""Return rps of any increment files with base in self.incdirlist"""
		return filter(lambda rp: (rp.isincfile() and
								  rp.getincbase().dirsplit()[1] == basename),
					  self.rpincdirlist)

	def branches(self):
		if not self.extlist: self.setextlist()
		for filename in self.extlist:
			yield RestoreTree(self.rs.add_ext(filename,
											  self.get_incrps(filename)))

	def filter_excludes(self):
		"""Return TT filtered by exclude regexps"""
		def isnotexcluded(rs):
			for regexp in Globals.exclude_mirror_regexps:
				if regexp.match(rs.mirror.path):
					Log("Excluding %s" % rs.mirror.path, 5)
					return None
			return 1
		return self.node_filter(isnotexcluded)

	def destructive_stepping(self):
		"""See destructive stepping in TripleTree for more information

		Note that unlike the TT version, this one assumes that the
		node will be processed before the branches - otherwise it
		won't know whether the target is a directory.

		"""
		def new_branches(mirror, target, old_branches):
			if not mirror.isdir() and not target.isdir():
				for branch in old_branches: yield branch
				return

			mirror_perms, target_perms = None, None
			if (Globals.change_mirror_perms and mirror.isdir() and
				not (mirror.readable() and mirror.executable())):
				Log("Adjusting permissions of %s" % mirror.path, 6)
				mirror_perms = mirror.getperms()
				mirror_atime, mirror_mtime = mirror.gettime()
				mirror.chmod(0500)
			if target.isdir() and not target.hasfullperms():
				Log("Adjusting permissions of %s" % target.path, 6)
				target_perms = target.getperms()
				target_atime, target_mtime = target.gettime()
				target.chmod(0700)

			for branch in old_branches: yield branch

			if target_perms is not None:
				Log("Restoring permissions of %s" % target.path, 6)
				target.chmod(target_perms)
				target.settime(target_atime, target_mtime)
			if mirror_perms is not None:
				Log("Restoring permissions of %s" % mirror.path, 6)
				mirror.chmod(mirror_perms)
				mirror.settime(mirror_atime, mirror_mtime)

		def reduction_helper(node, reduced_branches):
			outtree = self.__class__(node)
			outtree.branches = lambda: new_branches(node.mirror, node.target,
													reduced_branches)
			return outtree
		return self.reduce(reduction_helper)


#######################################################################
#
# rdiff - Invoke rdiff utility to make signatures, deltas, or patch
#

class Rdiff:
	"""Contains static methods for rdiff operations"""
	def get_signature(rp):
		"""Take signature of rpin file and return in file object"""
		Log("Getting signature of %s" % rp.path, 7)
		return rp.conn.os.popen("rdiff signature %s" % rp.quote())

	def get_delta(rp_signature, rp_new):
		"""Take signature rp and new rp, return delta file object"""
		assert rp_signature.conn is rp_new.conn
		Log("Getting delta of %s with signature %s" %
			(rp_new.path, rp_signature.path), 7)
		return rp_new.conn.os.popen("rdiff delta %s %s" %
									(rp_signature.quote(), rp_new.quote()))

	def write_delta(basis, new, delta):
		"""Write deltafile delta which brings basis to new"""
		Log("Writing delta %s from %s -> %s" %
			(basis.path, new.path, delta.path), 7)
		sig = RPath(new.conn, new.conn.tempfile.mktemp())
		sig.write_from_fileobj(Rdiff.get_signature(basis))
		delta.write_from_fileobj(Rdiff.get_delta(sig, new))
		sig.delete()

	def patch(rp_basis, rp_delta, rp_new = None):
		"""Patch rp_basis with rp_delta

		If rp_new is specified, leave rp_basis the way it is,
		otherwise replace rp_basis with rp_new.

		"""
		assert rp_basis.conn is rp_delta.conn
		if rp_new:
			assert rp_new.conn is rp_basis.conn
			out_rp = rp_new
		else: out_rp = RPath(rp_basis.conn, rp_basis.conn.tempfile.mktemp())

		Log("Patching %s using %s to %s" % (rp_basis.path, rp_delta.path,
											out_rp.path), 7)
		rp_basis.conn.os.system("rdiff patch %s %s %s" %
								(rp_basis.quote(), rp_delta.quote(),
								 out_rp.quote()))
		if rp_new: rp_new.clearstats()
		else:
			rp_basis.delete()
			RPath.copy(out_rp, rp_basis)
			out_rp.delete()

	def copy(rpin, rpout):
		"""Use rdiff to copy rpin to rpout, conserving bandwidth"""
		if not rpin.isreg() or not rpout.isreg() or rpin.conn is rpout.conn:
			RPath.copy(rpin, rpout)  # fallback to regular copying
		else:
			Log("Rdiff copying %s to %s" % (rpin.path, rpout.path), 6)
			rp_delta = RPath(rpout.conn, rpout.conn.tempfile.mktemp())
			Rdiff.write_delta(rpout, rpin, rp_delta)
			Rdiff.patch(rpout, rp_delta)
			rp_delta.delete()


MakeStatic(Rdiff)


#######################################################################
#
# increment - Provides Inc class, which writes increment files
#
# This code is what writes files ending in .diff, .snapshot, etc.
#

class Inc:
	"""Class containing increment functions"""
	def Increment(new, mirror, incpref):
		"""Main file incrementing function

		new is the file on the active partition,
		mirror is the mirrored file from the last backup,
		incpref is the prefix of the increment file.

		This function basically moves mirror -> incpref.

		"""
		assert new.lstat() or mirror.lstat()
		Log("Incrementing mirror file " + mirror.path, 5)
		if (new.isdir() or mirror.isdir()) and not incpref.isdir():
			incpref.mkdir()

		if not mirror.lstat(): Inc.makemissing(incpref)
		elif mirror.isdir(): Inc.makedir(mirror, incpref)
		elif new.isreg() and mirror.isreg():
			Inc.makediff(new, mirror, incpref)
		else: Inc.makesnapshot(mirror, incpref)

	def makemissing(incpref):
		"""Signify that mirror file was missing"""
		incpref.add_inc_ext("missing").touch()
		
	def makesnapshot(mirror, incpref):
		"""Copy mirror to incfile, since new is quite different"""
		snapshotrp = incpref.add_inc_ext("snapshot")
		if Globals.change_mirror_perms:
			mirror.make_readable(lambda: RPath.copy(mirror, snapshotrp))
		else: RPath.copy(mirror, snapshotrp)
		RPath.copy_attribs(mirror, snapshotrp)

	def makediff(new, mirror, incpref):
		"""Make incfile which is a diff new -> mirror"""
		diff = incpref.add_inc_ext("diff")
		def helper():
			if Globals.change_source_perms:
				new.make_readable(lambda: Rdiff.write_delta(new, mirror, diff))
			else: Rdiff.write_delta(new, mirror, diff)
		if Globals.change_mirror_perms: mirror.make_readable(helper)
		else: helper()
		RPath.copy_attribs(mirror, diff)

	def makedir(mirrordir, incpref):
		"""Make file indicating directory mirrordir has changed"""
		dirsign = incpref.add_inc_ext("dir")
		dirsign.touch()
		RPath.copy_attribs(mirrordir, dirsign)

	def IncrementTTree(tripletree):
		"""Increment a TripleTree"""
		node = tripletree.node()
		if (node.new.isdir() or node.mirror.isdir()) and not node.inc.isdir():
			node.inc.mkdir()
		for tt in tripletree.branches():
			Inc.IncrementTTree(tt)
		Inc.Increment(node.new, node.mirror, node.inc)

	def IncrementMirrorTTree(tripletree):
		"""Increment and mirror a tripletree

		Returns true if anything needed to be updated, false
		otherwise.  The part about already_inced is a hack - if we
		don't make the mirror dir early, further mirrored files may
		not have a directory to go into.  But if we do make it early,
		the mirrored temp directory will be processed for the
		increment!  Similarly, when deleting a directory we need to
		get the time before everything inside is deleted, but usually
		we want to wait until afterwards, so we can mark it only if
		something has changed.

		"""
		node = tripletree.node()
		updated, already_inced = None, None
		if node.new.isdir() or node.mirror.isdir():
			if node.needs_updating():
				already_inced = 1
				apply(Inc.Increment, node.tuple())
			if not node.mirror.isdir():
				updated = 1
				node.mirror.mkdir()
			if not node.inc.isdir(): node.inc.mkdir()
		for tt in tripletree.branches():
			updated = Inc.IncrementMirrorTTree(tt) or updated
		if updated or node.needs_updating():
			updated = 1
			node.mirror.clearstats()
			node.inc.clearstats()
			if not already_inced: apply(Inc.Increment, node.tuple())
			node.make_mirror()
		return updated


MakeStatic(Inc)


#######################################################################
#
# restore - Read increment files and restore to original
#

class RestoreError(Exception): pass

class Restore:
	def RestoreFile(rest_time, rpbase, inclist, rptarget):
		"""Non-recursive restore function

		rest_time is the time in seconds to restore to,
		rpbase is the base name of the file being restored,
		inclist is a list of rpaths containing all the relevant increments,
		and rptarget is the rpath that will be written with the restored file.

		"""
		inclist = Restore.sortincseq(rest_time, inclist)
		if not inclist and not rpbase.lstat(): return
		Log("Restoring %s with increments %s to %s" %
			(rpbase.path, ", ".join(map(lambda x: x.path, inclist)),
			 rptarget.path), 5)
		if not inclist or inclist[0].getinctype() == "diff":
			if Globals.change_mirror_perms:
				rpbase.make_readable(lambda: RPath.copy(rpbase, rptarget))
			else: RPath.copy(rpbase, rptarget)
			RPath.copy_attribs(rpbase, rptarget)
		for inc in inclist: Restore.applyinc(inc, rptarget)

	def sortincseq(rest_time, inclist):
		"""Sort the inc sequence, and throw away irrelevant increments"""
		incpairs = map(lambda rp: (Time.stringtotime(rp.getinctime()), rp),
					   inclist)
		# Only consider increments at or after the time being restored
		incpairs = filter(lambda pair: pair[0] >= rest_time, incpairs)

		# Now throw away older unnecessary increments
		incpairs.sort()
		i = 0
		while(i < len(incpairs)):
			# Only diff type increments require later versions
			if incpairs[i][1].getinctype() != "diff": break
			i = i+1
		incpairs = incpairs[:i+1]

		# Return increments in reversed order
		incpairs.reverse()
		return map(lambda pair: pair[1], incpairs)

	def applyinc(inc, target):
		"""Apply increment rp inc to targetrp target"""
		Log("Applying increment %s to %s" % (inc.path, target.path), 6)
		inctype = inc.getinctype()
		if inctype == "diff":
			if not target.lstat():
				raise RestoreError("Bad increment sequence at " + inc.path)
			def helper():
				if Globals.change_mirror_perms:
					inc.make_readable(lambda: Rdiff.patch(target, inc))
				else: Rdiff.patch(target, inc)
			target.make_readable(helper)
		elif inctype == "dir":
			if not target.isdir():
				if target.lstat():
					raise RestoreError("File %s already exists" % target.path)
				target.mkdir()
		elif inctype == "missing": return
		elif inctype == "snapshot":
			if Globals.change_mirror_perms:
				inc.make_readable(lambda: RPath.copy(inc, target))
			else: RPath.copy(inc, target)
		else: raise RestoreError("Unknown inctype %s" % inctype)
		RPath.copy_attribs(inc, target)

	def restore_tree(rt, rest_time):
		"""Restores the given RestoreTree up to rest_time

		Note that for_node_twice is used so that directory mtimes will
		be corrected if they are changed by writing inside the
		directory.

		"""
		rt.filter_excludes().destructive_stepping().for_node_twice(lambda rs:
			 Restore.RestoreFile(rest_time, rs.mirror, rs.inclist, rs.target))


MakeStatic(Restore)


#######################################################################
#
# main - Start here: Read arguments, set global settings, etc.
#

class Main:
	def __init__(self):
		self.action = None
		self.remote_cmd = None
		self.force = None

	def parse_cmdlineoptions(self):
		"""Parse argument list and set global preferences"""
		try: optlist, self.args = getopt.getopt(sys.argv[1:], "bmv:Vs",
			 ["backup-mode", "version", "verbosity=", "exclude=",
			  "exclude-mirror=", "server", "test-server", "remote-cmd=",
			  "mirror-only", "no-rdiff-copy", "force",
			  "change-source-perms"])
		except getopt.error:
			self.commandline_error("Unrecognized commandline option")

		Globals.copy_func = staticmethod(Rdiff.copy)
		for opt, arg in optlist:
			if opt == "-b" or opt == "--backup-mode": self.action = "backup"
			elif opt == "-s" or opt == "--server": self.action = "server"
			elif opt == "-m" or opt == "--mirror-only": self.action = "mirror"
			elif opt == "--remote-cmd": self.remote_cmd = arg
			elif opt == "-v" or opt == "--verbosity":
				Log.setverbosity(int(arg))
			elif opt == "--terminal-verbosity":
				Log.setterm_verbosity(int(arg))
			elif opt == "--exclude":
				Globals.exclude_regexps.append(re.compile(arg))
			elif opt == "--exclude-mirror":
				Globals.exclude_mirror_regexps.append(re.compile(arg))
			elif opt == "--force": self.force = 1
			elif opt == "-V" or opt == "--version":
				print "rdiff-backup version " + Globals.version
				sys.exit(0)
			elif opt == "--no-rdiff-copy":
				Globals.copy_func = staticmethod(RPath.copy)
			elif opt == "--test-server": self.action = "test-server"
			elif opt == "--change-source-perms":
				Globals.change_source_perms = 1
			else: Log.FatalError("Unknown option %s" % opt)

	def set_action(self):
		"""Check arguments and try to set self.action"""
		l = len(self.args)
		if not self.action:
			if l == 0: self.commandline_error("No arguments given")
			elif l == 1: self.action = "restore"
			elif l == 2:
				if RPath(None, self.args[0]).isincfile():
					self.action = "restore"
				else: self.action = "backup"
			else: self.commandline_error("Too many arguments given")

		if l == 0 and self.action != "server" and self.action != "test-server":
			self.commandline_error("No arguments given")
		elif l > 0 and (self.action == "server" or
						self.action == "test-server"):
			self.commandline_error("Too many arguments given")
		elif l < 2 and (self.action == "backup" or self.action == "mirror"):
			self.commandline_error("Two arguments are required "
								   "(source, destination).")

	def commandline_error(self, message):
		sys.stderr.write("Error: %s\n" % message)
		sys.stderr.write("See the rdiff-backup manual page for instructions\n")
		sys.exit(1)

	def init_connection(self):
		"""Start the connection if necessary"""
		self.sourceconn = LocalConnection()
		if self.remote_cmd is None:
			self.destconn = self.sourceconn
			return

		stdin, stdout = os.popen2(self.remote_cmd)
		self.destconn = PipeConnection(stdout, stdin)
		self.destconn.Log.setverbosity(Log.verbosity)
		self.destconn.Log.setterm_verbosity(Log.term_verbosity)

	def misc_setup(self):
		"""Set default change ownership flag, umask"""
		Globals.change_ownership = (self.sourceconn.os.getuid() == 0 and
									self.destconn.os.getuid() == 0)
		os.umask(077)

	def take_action(self):
		"""Do whatever self.action says"""
		if self.action == "server":
			PipeConnection(sys.stdin, sys.stdout).Server()
		elif self.action == "backup":
			self.Backup(self.args[0], self.args[1])
		elif self.action == "restore":
			if len(self.args) == 1: self.Restore(self.args[0])
			else: apply(self.Restore, self.args)
		elif self.action == "mirror":
			self.Mirror(self.args[0], self.args[1])
		elif self.action == "test-server":
			self.TestServer()
		else: raise AssertionError("Unknown action " + self.action)

	def cleanup(self):
		"""Do any last minute cleaning before exiting"""
		Log("Cleaning up", 6)
		Log.close_logfile()
		self.destconn.quit()

	def Main(self):
		"""Start everything up!"""
		self.parse_cmdlineoptions()
		self.set_action()
		self.init_connection()
		self.misc_setup()
		self.take_action()
		self.cleanup()


	def TestServer(self):
		"""Run a couple simple tests of the remote connection"""
		if not isinstance(self.destconn, PipeConnection):
			Log.FatalError("No remote server specified "
						   "(the --remote-cmd option is required)")
		try:
			assert self.destconn.pow(2,3) == 8
			assert self.destconn.os.path.join("a", "b") == "a/b"
		except:
			sys.stderr.write(str(sys.exc_info()) + "\n")
			Log.FatalError("Server tests failed")
		print "Server OK"
		

	def Mirror(self, src_path, dest_path):
		"""Turn dest_path into a copy of src_path"""
		Log("Mirroring %s to %s" % (src_path, dest_path), 5)
		rpin, rpout = self.mirror_check_paths(src_path, dest_path)
		self.mirror_rps(rpin, rpout)

	def mirror_check_paths(self, src_path, dest_path):
		"""Check paths and return rpin, rpout"""
		rpin = RPath(self.sourceconn, src_path)
		rpout = RPath(self.destconn, dest_path)
		if not rpin.lstat():
			Log.FatalError("Source directory %s does not exist" % rpin.path)
		if rpout.lstat() and not self.force:
			Log.FatalError(
"""Destination %s exists so continuing could mess it up.  Run
rdiff-backup with the --force option if you want to mirror anyway.""" %
			rpout.path)
		return rpin, rpout

	def mirror_rps(self, rpin, rpout):
		"""Main mirroring function sets rpout = rpin"""
		Iter.foreach(RPTriple.make_mirror,
					 Iter.filter(RPTriple.needs_updating,
      					 TripleTree(RPTriple(rpin, rpout)).filter_excludes()
					                .destructive_stepping()))


	def Backup(self, src_path, dest_path):
		"""Backup, possibly incrementally, src_path to dest_path."""
		rpin = RPath(self.sourceconn, src_path)
		rpout = RPath(self.destconn, dest_path)
		self.backup_init_dirs(rpin, rpout)
		Time.setcurtime()
		if self.prevtime: self.backup_incremental(rpin, rpout)
		else: self.mirror_rps(rpin, rpout)
		self.backup_touch_curmirror(rpin, rpout)

	def backup_init_dirs(self, rpin, rpout):
		"""Make sure rpin and rpout are valid, init data dir and logging"""
		if rpout.lstat() and not rpout.isdir():
			if not self.force:
				Log.FatalError("Destination %s exists and is not a "
							   "directory" % rpout.path)
			else:
				Log("Deleting %s" % rpout.path, 3)
				rpout.delete()
			
		if not rpin.lstat():
			Log.FatalError("Source directory %s does not exist" % rpin.path)
		elif not rpin.isdir():
			Log.FatalError("Source %s is not a directory" % rpin.path)

		self.datadir = rpout.append("rdiff-backup-data")
		self.prevtime = self.backup_get_mirrortime()

		if rpout.lstat() and not self.prevtime and not self.force:
			Log.FatalError(
"""Destination directory %s exists, but does not look like a
rdiff-backup directory.  Running rdiff-backup like this could mess up
what is currently in it.  If you want to overwrite it, run
rdiff-backup with the --force option.""" % rpout.path)

		if not rpout.lstat(): rpout.mkdir()
		if not self.datadir.lstat(): self.datadir.mkdir()
		Globals.exclude_mirror_regexps.append(re.compile(self.datadir.path))
		if Log.verbosity > 0:
			Log.open_logfile(self.datadir.append("backup.log"))

	def backup_get_mirrorrps(self):
		"""Return list of current_mirror rps"""
		if not self.datadir.isdir(): return []
		mirrorfiles = filter(lambda f: f.startswith("current_mirror."),
							 self.datadir.listdir())
		mirrorrps = map(lambda x: self.datadir.append(x), mirrorfiles)
		return filter(lambda rp: rp.isincfile(), mirrorrps)

	def backup_get_mirrortime(self):
		"""Return time in seconds of previous mirror, or None if cannot"""
		mirrorrps = self.backup_get_mirrorrps()
		if not mirrorrps: return None
		if len(mirrorrps) > 1:
			Log(
"""Warning: duplicate current_mirror files found.  Perhaps something
went wrong during your last backup?  Using """ + mirrorrps[-1].path, 2)

		timestr = self.datadir.append(mirrorrps[-1].path).getinctime()
		return Time.stringtotime(timestr)
	
	def backup_incremental(self, rpin, rpout):
		"""Start an increment backup from rpin to rpout"""
		Time.setprevtime(self.prevtime)
		Inc.IncrementMirrorTTree(
			TripleTree(RPTriple(rpin, rpout,
								self.datadir.append("increments")))
                          .filter_excludes().destructive_stepping())

	def backup_touch_curmirror(self, rpin, rpout):
		"""Make a file like current_mirror.time.snapshot to record time

		Then update rpout to leave no trace...
		"""
		map(RPath.delete, self.backup_get_mirrorrps())
		mirrorrp = self.datadir.append("current_mirror.%s.%s" %
										  (Time.curtimestr, "snapshot"))
		Log("Touching mirror marker %s" % mirrorrp.path, 6)
		mirrorrp.touch()
		RPath.copy_attribs(rpin, rpout)


	def Restore(self, src_path, dest_path = None):
		"""Main restoring function - take src_path to dest_path"""
		Log("Starting Restore", 5)
		rpin, rpout = self.restore_check_paths(src_path, dest_path)
		rs = self.restore_getrs(rpin, rpout)
		rtime = Time.stringtotime(rpin.getinctime())
		Log.open_logfile(self.datadir.append("restore.log"))
		Restore.restore_tree(RestoreTree(rs), rtime)

	def restore_check_paths(self, src_path, dest_path):
		"""Check paths and return pair of corresponding rps"""
		rpin = RPath(self.sourceconn, src_path)
		if not rpin.lstat():
			Log.FatalError("Increment file %s does not exist" % src_path)
		if not rpin.isincfile():
			Log.FatalError("""File %s does not look like an increment file.

Try restoring from an increment file (the filenames look like
"foobar.2001-09-01T04:49:04-07:00.diff").""")

		if dest_path: rpout = RPath(self.destconn, dest_path)
		else: rpout = rpin.getincbase()
		if rpout.lstat():
			Log.FatalError("Restore target %s already exists.  "
						   "Will not overwrite." % rpout.path)
		return rpin, rpout

	def restore_getrs(self, rpin, rpout):
		"""Return the initial RestoreStruct"""
		rpin_dir = rpin.dirsplit()[0]
		if not rpin_dir: rpin_dir = "/"
		rpin_dir_rp = RPath(self.sourceconn, rpin_dir)
		incbasename = rpin.getincbase().dirsplit()[1]
		incrps = filter(lambda rp: rp.isincfile() and
						rp.getincbase().dirsplit()[1] == incbasename,
						map(rpin_dir_rp.append, rpin_dir_rp.listdir()))
		return RestoreStruct(self.restore_get_mirror(rpin),
							 rpin.getincbase(), incrps, rpout)

	def restore_get_mirror(self, rpin):
		"""Return mirror file and set the data dir

		The idea here is to keep backing up on the path until we find
		something named "rdiff-backup-data".  Then use that as a
		reference to calculate the oldfile.  This could fail if the
		increment file is pointed to in a funny way, using symlinks or
		somesuch.

		"""
		pathcomps = os.path.join(rpin.conn.os.getcwd(),
								 rpin.getincbase().path).split("/")
		for i in range(1, len(pathcomps)):
			datadirrp = RPath(rpin.conn, "/".join(pathcomps[:i+1]))
			if pathcomps[i] == "rdiff-backup-data" and datadirrp.isdir():
				break
		else: Log.FatalError("Unable to find rdiff-backup-data dir")

		self.datadir = datadirrp
		Globals.exclude_mirror_regexps.append(re.compile(self.datadir.path))
		rootrp = RPath(rpin.conn, "/".join(pathcomps[:i]))
		if not rootrp.lstat():
			Log.FatalError("Root of mirror area %s does not exist" %
						   rootrp.path)
		else: Log("Using root mirror %s" % rootrp.path, 6)

		from_datadir = pathcomps[i+1:]
		if len(from_datadir) == 1: result = rootrp
		elif len(from_datadir) > 1:
			result = rootrp.append(apply(os.path.join, from_datadir[1:]))
		else: raise RestoreError("Problem finding mirror file")

		Log("Using mirror file %s" % result.path, 6)
		return result
		

if __name__ == "__main__": Main().Main()


