#!/usr/bin/env python
#
# rdiff-backup -- Mirror files while keeping incremental changes
# Version 0.8.0 released June 14, 2002
# Copyright (C) 2001, 2002 Ben Escoto <bescoto@stanford.edu>
#
# This program is licensed under the GNU General Public License (GPL).
# Distributions of rdiff-backup usually include a copy of the GPL in a
# file called COPYING.  The GPL is also available online at
# http://www.gnu.org/copyleft/gpl.html.
#
# See http://www.stanford.edu/~bescoto/rdiff-backup for more
# information.  Please send mail to me or the mailing list if you find
# bugs or have any suggestions.

from __future__ import nested_scopes, generators
import os, stat, time, sys, getopt, re, cPickle, types, shutil, sha, marshal, traceback, popen2, tempfile, gzip, UserList, errno, signal


#######################################################################
#
# globals - aggregate some configuration options
#

class Globals:

	# The current version of rdiff-backup
	version = "0.8.0"
	
	# If this is set, use this value in seconds as the current time
	# instead of reading it from the clock.
	current_time = None

	# This determines how many bytes to read at a time when copying
	blocksize = 32768

	# This is used by the BufferedRead class to determine how many
	# bytes to request from the underlying file per read().  Larger
	# values may save on connection overhead and latency.
	conn_bufsize = 98304

	# True if script is running as a server
	server = None

	# uid and gid of the owner of the rdiff-backup process.  This can
	# vary depending on the connection.
	process_uid = os.getuid()
	process_gid = os.getgid()

	# If true, when copying attributes, also change target's uid/gid
	change_ownership = None

	# If true, change the permissions of unwriteable mirror files
	# (such as directories) so that they can be written, and then
	# change them back.  This defaults to 1 just in case the process
	# is not running as root (root doesn't need to change
	# permissions).
	change_mirror_perms = (process_uid != 0)

	# If true, temporarily change permissions of unreadable files in
	# the source directory to make sure we can read all files.
	change_source_perms = None

	# If true, try to reset the atimes of the source partition.
	preserve_atime = None

	# This will be set as soon as the LocalConnection class loads
	local_connection = None

	# All connections should be added to the following list, so
	# further global changes can be propagated to the remote systems.
	# The first element should be Globals.local_connection.  For a
	# server, the second is the connection to the client.
	connections = []

	# Each process should have a connection number unique to the
	# session.  The client has connection number 0.
	connection_number = 0

	# Dictionary pairing connection numbers with connections.  Set in
	# SetConnections for all connections.
	connection_dict = {}

	# True if the script is the end that reads the source directory
	# for backups.  It is true for purely local sessions.
	isbackup_reader = None

	# Connection of the real backup reader (for which isbackup_reader
	# is true)
	backup_reader = None

	# True if the script is the end that writes to the increment and
	# mirror directories.  True for purely local sessions.
	isbackup_writer = None

	# Connection of the backup writer
	backup_writer = None

	# True if this process is the client invoked by the user
	isclient = None

	# Connection of the client
	client_conn = None

	# This list is used by the set function below.  When a new
	# connection is created with init_connection, its Globals class
	# will match this one for all the variables mentioned in this
	# list.
	changed_settings = []

	# rdiff-backup will try to checkpoint its state every
	# checkpoint_interval seconds.  Then when resuming, at most this
	# amount of time is lost.
	checkpoint_interval = 20

	# The RPath of the rdiff-backup-data directory.
	rbdir = None

	# Indicates if a resume or a lack of resume is forced.  This
	# should be None for the default.  0 means don't resume, and 1
	# means resume.
	resume = None

	# If there has been an aborted backup fewer than this many seconds
	# ago, attempt to resume it where it left off instead of starting
	# a new one.
	resume_window = 7200

	# This string is used when recognizing and creating time strings.
	# If the time_separator is ":", then W3 datetime strings like
	# 2001-12-07T04:22:01-07:00 are produced.  It can be set to "_" to
	# make filenames that don't contain colons, which aren't allowed
	# under MS windows NT.
	time_separator = ":"

	# quoting_enabled is true if we should quote certain characters in
	# filenames on the source side (see FilenameMapping for more
	# info).  chars_to_quote is a string whose characters should be
	# quoted, and quoting_char is the character to quote with.
	quoting_enabled = None
	chars_to_quote = ""
	quoting_char = ';'

	# If true, emit output intended to be easily readable by a
	# computer.  False means output is intended for humans.
	parsable_output = None

	# If true, then hardlinks will be preserved to mirror and recorded
	# in the increments directory.  There is also a difference here
	# between None and 0.  When restoring, None or 1 means to preserve
	# hardlinks iff can find a hardlink dictionary.  0 means ignore
	# hardlink information regardless.
	preserve_hardlinks = 1

	# If this is false, then rdiff-backup will not compress any
	# increments.  Default is to compress based on regexp below.
	compression = 1

	# Increments based on files whose names match this
	# case-insensitive regular expression won't be compressed (applies
	# to .snapshots and .diffs).  The second below will be the
	# compiled version of the first.
	no_compression_regexp_string = "(?i).*\\.(gz|z|bz|bz2|tgz|zip|rpm|deb|" \
							"jpg|gif|png|jp2|mp3|ogg|avi|wmv|mpeg|mpg|rm|mov)$"
	no_compression_regexp = None

	# If true, filelists and directory statistics will be split on
	# nulls instead of newlines.
	null_separator = None

	# Determines whether or not ssh will be run with the -C switch
	ssh_compression = 1

	# If true, print statistics after successful backup
	print_statistics = None

	# On the reader and writer connections, the following will be
	# replaced by the source and mirror Select objects respectively.
	select_source, select_mirror = None, None

	# On the backup writer connection, holds the main incrementing
	# function.  Access is provided to increment error counts.
	ITR = None

	def get(cls, name):
		"""Return the value of something in this class"""
		return cls.__dict__[name]
	get = classmethod(get)

	def is_not_None(cls, name):
		"""Returns true if value is not None"""
		return cls.__dict__[name] is not None
	is_not_None = classmethod(is_not_None)

	def set(cls, name, val):
		"""Set the value of something in this class

		Use this instead of writing the values directly if the setting
		matters to remote sides.  This function updates the
		changed_settings list, so other connections know to copy the
		changes.

		"""
		cls.changed_settings.append(name)
		cls.__dict__[name] = val
	set = classmethod(set)

	def set_integer(cls, name, val):
		"""Like set, but make sure val is an integer"""
		try: intval = int(val)
		except ValueError:
			Log.FatalError("Variable %s must be set to an integer -\n"
						   "received %s instead." % (name, val))
		cls.set(name, intval)
	set_integer = classmethod(set_integer)

	def get_dict_val(cls, name, key):
		"""Return val from dictionary in this class"""
		return cls.__dict__[name][key]
	get_dict_val = classmethod(get_dict_val)

	def set_dict_val(cls, name, key, val):
		"""Set value for dictionary in this class"""
		cls.__dict__[name][key] = val
	set_dict_val = classmethod(set_dict_val)

	def postset_regexp(cls, name, re_string, flags = None):
		"""Compile re_string on all existing connections, set to name"""
		for conn in Globals.connections:
			conn.Globals.postset_regexp_local(name, re_string, flags)
	postset_regexp = classmethod(postset_regexp)

	def postset_regexp_local(cls, name, re_string, flags):
		"""Set name to compiled re_string locally"""
		if flags: cls.__dict__[name] = re.compile(re_string, flags)
		else: cls.__dict__[name] = re.compile(re_string)
	postset_regexp_local = classmethod(postset_regexp_local)

	def set_select(cls, dsrpath, tuplelist, quote_mode, *filelists):
		"""Initialize select object using tuplelist

		Note that each list in filelists must each be passed as
		separate arguments, so each is recognized as a file by the
		connection.  Otherwise we will get an error because a list
		containing files can't be pickled.

		"""
		if dsrpath.source:
			cls.select_source = Select(dsrpath, quote_mode)
			cls.select_source.ParseArgs(tuplelist, filelists)
		else:
			cls.select_mirror = Select(dsrpath, quote_mode)
			cls.select_mirror.ParseArgs(tuplelist, filelists)
	set_select = classmethod(set_select)


#######################################################################
#
# static - MakeStatic and MakeClass
#
# These functions are used to make all the instance methods in a class
# into static or class methods.
#

class StaticMethodsError(Exception):
	pass

def MakeStatic(cls):
	"""turn instance methods into static ones

	The methods (that don't begin with _) of any class that
	subclasses this will be turned into static methods.

	"""
	for name in dir(cls):
		if name[0] != "_":
			cls.__dict__[name] = staticmethod(cls.__dict__[name])


def MakeClass(cls):
	"""Turn instance methods into classmethods.  Ignore _ like above"""
	for name in dir(cls):
		if name[0] != "_":
			cls.__dict__[name] = classmethod(cls.__dict__[name])


#######################################################################
#
# lazy - Define some lazy data structures and functions acting on them
#

class Iter:
	"""Hold static methods for the manipulation of lazy iterators"""

	def filter(predicate, iterator):
		"""Like filter in a lazy functional programming language"""
		for i in iterator:
			if predicate(i): yield i

	def map(function, iterator):
		"""Like map in a lazy functional programming language"""
		for i in iterator: yield function(i)

	def foreach(function, iterator):
		"""Run function on each element in iterator"""
		for i in iterator: function(i)

	def cat(*iters):
		"""Lazily concatenate iterators"""
		for iter in iters:
			for i in iter: yield i

	def cat2(iter_of_iters):
		"""Lazily concatenate iterators, iterated by big iterator"""
		for iter in iter_of_iters:
			for i in iter: yield i

	def empty(iter):
		"""True if iterator has length 0"""
		for i in iter: return None
		return 1

	def equal(iter1, iter2, verbose = None, operator = lambda x, y: x == y):
		"""True if iterator 1 has same elements as iterator 2

		Use equality operator, or == if it is unspecified.

		"""
		for i1 in iter1:
			try: i2 = iter2.next()
			except StopIteration:
				if verbose: print "End when i1 = %s" % i1
				return None
			if not operator(i1, i2):
				if verbose: print "%s not equal to %s" % (i1, i2)
				return None
		try: i2 = iter2.next()
		except StopIteration: return 1
		if verbose: print "End when i2 = %s" % (i2,)
		return None

	def Or(iter):
		"""True if any element in iterator is true.  Short circuiting"""
		i = None
		for i in iter:
			if i: return i
		return i

	def And(iter):
		"""True if all elements in iterator are true.  Short circuiting"""
		i = 1
		for i in iter:
			if not i: return i
		return i

	def len(iter):
		"""Return length of iterator"""
		i = 0
		while 1:
			try: iter.next()
			except StopIteration: return i
			i = i+1

	def foldr(f, default, iter):
		"""foldr the "fundamental list recursion operator"?"""
		try: next = iter.next()
		except StopIteration: return default
		return f(next, Iter.foldr(f, default, iter))

	def foldl(f, default, iter):
		"""the fundamental list iteration operator.."""
		while 1:
			try: next = iter.next()
			except StopIteration: return default
			default = f(default, next)

	def multiplex(iter, num_of_forks, final_func = None, closing_func = None):
		"""Split a single iterater into a number of streams

		The return val will be a list with length num_of_forks, each
		of which will be an iterator like iter.  final_func is the
		function that will be called on each element in iter just as
		it is being removed from the buffer.  closing_func is called
		when all the streams are finished.

		"""
		if num_of_forks == 2 and not final_func and not closing_func:
			im2 = IterMultiplex2(iter)
			return (im2.yielda(), im2.yieldb())
		if not final_func: final_func = lambda i: None
		if not closing_func: closing_func = lambda: None

		# buffer is a list of elements that some iterators need and others
		# don't
		buffer = []

		# buffer[forkposition[i]] is the next element yieled by iterator
		# i.  If it is -1, yield from the original iter
		starting_forkposition = [-1] * num_of_forks
		forkposition = starting_forkposition[:]
		called_closing_func = [None]

		def get_next(fork_num):
			"""Return the next element requested by fork_num"""
			if forkposition[fork_num] == -1:
				try:  buffer.insert(0, iter.next())
				except StopIteration:
					# call closing_func if necessary
					if (forkposition == starting_forkposition and
						not called_closing_func[0]):
						closing_func()
						called_closing_func[0] = None
					raise StopIteration
				for i in range(num_of_forks): forkposition[i] += 1

			return_val = buffer[forkposition[fork_num]]
			forkposition[fork_num] -= 1

			blen = len(buffer)
			if not (blen-1) in forkposition:
				# Last position in buffer no longer needed
				assert forkposition[fork_num] == blen-2
				final_func(buffer[blen-1])
				del buffer[blen-1]
			return return_val

		def make_iterator(fork_num):
			while(1): yield get_next(fork_num)

		return tuple(map(make_iterator, range(num_of_forks)))

MakeStatic(Iter)


class IterMultiplex2:
	"""Multiplex an iterator into 2 parts

	This is a special optimized case of the Iter.multiplex function,
	used when there is no closing_func or final_func, and we only want
	to split it into 2.  By profiling, this is a time sensitive class.

	"""
	def __init__(self, iter):
		self.a_leading_by = 0 # How many places a is ahead of b
		self.buffer = []
		self.iter = iter

	def yielda(self):
		"""Return first iterator"""
		buf, iter = self.buffer, self.iter
		while(1):
			if self.a_leading_by >= 0: # a is in front, add new element
				elem = iter.next() # exception will be passed
				buf.append(elem)
			else: elem = buf.pop(0) # b is in front, subtract an element
			self.a_leading_by += 1
			yield elem

	def yieldb(self):
		"""Return second iterator"""
		buf, iter = self.buffer, self.iter
		while(1):
			if self.a_leading_by <= 0: # b is in front, add new element
				elem = iter.next() # exception will be passed
				buf.append(elem)
			else: elem = buf.pop(0) # a is in front, subtract an element
			self.a_leading_by -= 1
			yield elem


class IterTreeReducer:
	"""Tree style reducer object for iterator

	The indicies of a RORPIter form a tree type structure.  This class
	can be used on each element of an iter in sequence and the result
	will be as if the corresponding tree was reduced.  This tries to
	bridge the gap between the tree nature of directories, and the
	iterator nature of the connection between hosts and the temporal
	order in which the files are processed.

	There are four stub functions below: start_process, end_process,
	branch_process, and check_for_errors.  A class that subclasses
	this one will probably fill in these functions to do more.

	It is important that this class be pickable, so keep that in mind
	when subclassing (this is used to resume failed sessions).

	"""
	def __init__(self, *args):
		"""ITR initializer"""
		self.init_args = args
		self.index = None
		self.subinstance = None
		self.finished = None
		self.caught_exception, self.start_successful = None, None

	def intree(self, index):
		"""Return true if index is still in current tree"""
		return self.base_index == index[:len(self.base_index)]

	def set_subinstance(self):
		"""Return subinstance of same type as self"""
		self.subinstance = self.__class__(*self.init_args)

	def process_w_subinstance(self, args):
		"""Give object to subinstance, if necessary update branch_val"""
		if not self.subinstance: self.set_subinstance()
		if not self.subinstance(*args):
			self.branch_process(self.subinstance)
			self.set_subinstance()
			assert self.subinstance(*args)

	def start_process(self, *args):
		"""Do some initial processing (stub)"""
		pass

	def end_process(self):
		"""Do any final processing before leaving branch (stub)"""
		pass

	def branch_process(self, subinstance):
		"""Process a branch right after it is finished (stub)"""
		pass

	def check_for_errors(self, function, *args):
		"""start/end_process is called by this function

		Usually it will distinguish between two types of errors.  Some
		are serious and will be reraised, others are caught and simply
		invalidate the current instance by setting
		self.caught_exception.

		"""
		try: return apply(function, args)
		except: raise

	def Finish(self):
		"""Call at end of sequence to tie everything up"""
		if not self.start_successful or self.finished:
			self.caught_exception = 1
		if self.caught_exception: self.log_prev_error(self.index)
		else:
			if self.subinstance:
				self.subinstance.Finish()
				self.branch_process(self.subinstance)
			self.check_for_errors(self.end_process)
			self.finished = 1

	def log_prev_error(self, index):
		"""Call function if no pending exception"""
		Log("Skipping %s because of previous error" % os.path.join(*index), 2)

	def __call__(self, *args):
		"""Process args, where args[0] is current position in iterator

		Returns true if args successfully processed, false if index is
		not in the current tree and thus the final result is
		available.

		Also note below we set self.index after doing the necessary
		start processing, in case there is a crash in the middle.

		"""
		index = args[0]
		assert type(index) is types.TupleType, type(index)

		if self.index is None:
			self.check_for_errors(self.start_process, *args)
			self.start_successful = 1
			self.index = self.base_index = index
			return 1

		if index <= self.index:
			Log("Warning: oldindex %s >= newindex %s" % (self.index, index), 2)
			return 1

		if not self.intree(index):
			self.Finish()
			return None

		if self.caught_exception: self.log_prev_error(index)
		else: self.process_w_subinstance(args)
		self.index = index
		return 1


class ErrorITR(IterTreeReducer):
	"""Adds some error handling to above ITR, if ITR processes files"""
	def on_error(self, exc, *args):
		"""This is run on any exception in start/end-process"""
		self.caught_exception = 1
		if args and isinstance(args[0], tuple):
			filename = os.path.join(*args[0])
		elif self.index: filename = os.path.join(*self.index)
		else: filename = "."
		Log("Error '%s' processing %s" % (exc, filename), 2)

	def check_for_errors(self, function, *args):
		"""Catch some non-fatal errors"""
		return Robust.check_common_error(self.on_error, function, *args)


#######################################################################
#
# log - Manage logging 
#

class LoggerError(Exception): pass

class Logger:
	"""All functions which deal with logging"""
	def __init__(self):
		self.log_file_open = None
		self.log_file_local = None
		self.verbosity = self.term_verbosity = 3
		# termverbset is true if the term_verbosity has been explicity set
		self.termverbset = None

	def setverbosity(self, verbosity_string):
		"""Set verbosity levels.  Takes a number string"""
		try: self.verbosity = int(verbosity_string)
		except ValueError:
			Log.FatalError("Verbosity must be a number, received '%s' "
						   "instead." % verbosity_string)
		if not self.termverbset: self.term_verbosity = self.verbosity

	def setterm_verbosity(self, termverb_string):
		"""Set verbosity to terminal.  Takes a number string"""
		try: self.term_verbosity = int(termverb_string)
		except ValueError:
			Log.FatalError("Terminal verbosity must be a number, received "
						   "'%s' insteaxd." % termverb_string)
		self.termverbset = 1

	def open_logfile(self, rpath):
		"""Inform all connections of an open logfile.

		rpath.conn will write to the file, and the others will pass
		write commands off to it.

		"""
		assert not self.log_file_open
		for conn in Globals.connections:
			conn.Log.open_logfile_allconn(rpath.conn)
		rpath.conn.Log.open_logfile_local(rpath)

	def open_logfile_allconn(self, log_file_conn):
		"""Run on all connections to signal log file is open"""
		self.log_file_open = 1
		self.log_file_conn = log_file_conn

	def open_logfile_local(self, rpath):
		"""Open logfile locally - should only be run on one connection"""
		assert self.log_file_conn is Globals.local_connection
		self.log_file_local = 1
		self.logrp = rpath
		self.logfp = rpath.open("a")

	def close_logfile(self):
		"""Close logfile and inform all connections"""
		if self.log_file_open:
			for conn in Globals.connections:
				conn.Log.close_logfile_allconn()
			self.log_file_conn.Log.close_logfile_local()

	def close_logfile_allconn(self):
		"""Run on every connection"""
		self.log_file_open = None

	def close_logfile_local(self):
		"""Run by logging connection - close logfile"""
		assert self.log_file_conn is Globals.local_connection
		assert not self.logfp.close()
		self.log_file_local = None

	def format(self, message, verbosity):
		"""Format the message, possibly adding date information"""
		if verbosity < 9: return message + "\n"
		else: return "%s  %s\n" % (time.asctime(time.localtime(time.time())),
								   message)

	def __call__(self, message, verbosity):
		"""Log message that has verbosity importance"""
		if verbosity <= self.verbosity: self.log_to_file(message)
		if verbosity <= self.term_verbosity:
			self.log_to_term(message, verbosity)

	def log_to_file(self, message):
		"""Write the message to the log file, if possible"""
		if self.log_file_open:
			if self.log_file_local:
				self.logfp.write(self.format(message, self.verbosity))
			else: self.log_file_conn.Log.log_to_file(message)

	def log_to_term(self, message, verbosity):
		"""Write message to stdout/stderr"""
		if verbosity <= 2 or Globals.server: termfp = sys.stderr
		else: termfp = sys.stdout
		termfp.write(self.format(message, self.term_verbosity))

	def conn(self, direction, result, req_num):
		"""Log some data on the connection

		The main worry with this function is that something in here
		will create more network traffic, which will spiral to
		infinite regress.  So, for instance, logging must only be done
		to the terminal, because otherwise the log file may be remote.

		"""
		if self.term_verbosity < 9: return
		if type(result) is types.StringType: result_repr = repr(result)
		else: result_repr = str(result)
		if Globals.server: conn_str = "Server"
		else: conn_str = "Client"
		self.log_to_term("%s %s (%d): %s" %
						 (conn_str, direction, req_num, result_repr), 9)

	def FatalError(self, message):
		self("Fatal Error: " + message, 1)
		Globals.Main.cleanup()
		sys.exit(1)

	def exception_to_string(self):
		"""Return string version of current exception"""
		type, value, tb = sys.exc_info()
		return ("Exception '%s' raised of class '%s':\n%s" %
				(value, type, "".join(traceback.format_tb(tb))))

	def exception(self, only_terminal = 0, verbosity = 5):
		"""Log an exception and traceback

		If only_terminal is None, log normally.  If it is 1, then only
		log to disk if log file is local (self.log_file_open = 1).  If
		it is 2, don't log to disk at all.

		"""
		assert only_terminal in (0, 1, 2)
		if (only_terminal == 0 or
			(only_terminal == 1 and self.log_file_open)):
			logging_func = self.__call__
		else: logging_func = self.log_to_term

		logging_func(self.exception_to_string(), verbosity)

Log = Logger()


#######################################################################
#
# ttime - Provide Time class, which contains time related functions.
#

class TimeException(Exception): pass

class Time:
	"""Functions which act on the time"""
	_interval_conv_dict = {"s": 1, "m": 60, "h": 3600, "D": 86400,
						   "W": 7*86400, "M": 30*86400, "Y": 365*86400}
	_integer_regexp = re.compile("^[0-9]+$")
	_interval_regexp = re.compile("^([0-9]+)([smhDWMY])")
	_genstr_date_regexp1 = re.compile("^(?P<year>[0-9]{4})[-/]"
						   "(?P<month>[0-9]{1,2})[-/](?P<day>[0-9]{1,2})$")
	_genstr_date_regexp2 = re.compile("^(?P<month>[0-9]{1,2})[-/]"
						   "(?P<day>[0-9]{1,2})[-/](?P<year>[0-9]{4})$")

	def setcurtime(cls, curtime = None):
		"""Sets the current time in curtime and curtimestr on all systems"""
		t = curtime or time.time()
		for conn in Globals.connections:
			conn.Time.setcurtime_local(t, cls.timetostring(t))

	def setcurtime_local(cls, timeinseconds, timestr):
		"""Only set the current time locally"""
		cls.curtime = timeinseconds
		cls.curtimestr = timestr

	def setprevtime(cls, timeinseconds):
		"""Sets the previous inc time in prevtime and prevtimestr"""
		assert timeinseconds > 0, timeinseconds
		timestr = cls.timetostring(timeinseconds)
		for conn in Globals.connections:
			conn.Time.setprevtime_local(timeinseconds, timestr)

	def setprevtime_local(cls, timeinseconds, timestr):
		"""Like setprevtime but only set the local version"""
		cls.prevtime = timeinseconds
		cls.prevtimestr = timestr

	def timetostring(cls, timeinseconds):
		"""Return w3 datetime compliant listing of timeinseconds"""
		return time.strftime("%Y-%m-%dT%H" + Globals.time_separator +
							 "%M" + Globals.time_separator + "%S",
							 time.localtime(timeinseconds)) + cls.gettzd()

	def stringtotime(cls, timestring):
		"""Return time in seconds from w3 timestring

		If there is an error parsing the string, or it doesn't look
		like a w3 datetime string, return None.

		"""
		try:
			date, daytime = timestring[:19].split("T")
			year, month, day = map(int, date.split("-"))
			hour, minute, second = map(int,
									   daytime.split(Globals.time_separator))
			assert 1900 < year < 2100, year
			assert 1 <= month <= 12
			assert 1 <= day <= 31
			assert 0 <= hour <= 23
			assert 0 <= minute <= 59
			assert 0 <= second <= 61  # leap seconds
			timetuple = (year, month, day, hour, minute, second, -1, -1, -1)
			if time.daylight:
				utc_in_secs = time.mktime(timetuple) - time.altzone
			else: utc_in_secs = time.mktime(timetuple) - time.timezone

			return long(utc_in_secs) + cls.tzdtoseconds(timestring[19:])
		except (TypeError, ValueError, AssertionError): return None

	def timetopretty(cls, timeinseconds):
		"""Return pretty version of time"""
		return time.asctime(time.localtime(timeinseconds))

	def stringtopretty(cls, timestring):
		"""Return pretty version of time given w3 time string"""
		return cls.timetopretty(cls.stringtotime(timestring))

	def inttopretty(cls, seconds):
		"""Convert num of seconds to readable string like "2 hours"."""
		partlist = []
		hours, seconds = divmod(seconds, 3600)
		if hours > 1: partlist.append("%d hours" % hours)
		elif hours == 1: partlist.append("1 hour")

		minutes, seconds = divmod(seconds, 60)
		if minutes > 1: partlist.append("%d minutes" % minutes)
		elif minutes == 1: partlist.append("1 minute")

		if seconds == 1: partlist.append("1 second")
		elif not partlist or seconds > 1:
			if isinstance(seconds, int) or isinstance(seconds, long):
				partlist.append("%s seconds" % seconds)
			else: partlist.append("%.2f seconds" % seconds)
		return " ".join(partlist)

	def intstringtoseconds(cls, interval_string):
		"""Convert a string expressing an interval (e.g. "4D2s") to seconds"""
		def error():
			raise TimeException("""Bad interval string "%s"

Intervals are specified like 2Y (2 years) or 2h30m (2.5 hours).  The
allowed special characters are s, m, h, D, W, M, and Y.  See the man
page for more information.
""" % interval_string)
		if len(interval_string) < 2: error()
		
		total = 0
		while interval_string:
			match = cls._interval_regexp.match(interval_string)
			if not match: error()
			num, ext = int(match.group(1)), match.group(2)
			if not ext in cls._interval_conv_dict or num < 0: error()
			total += num*cls._interval_conv_dict[ext]
			interval_string = interval_string[match.end(0):]
		return total

	def gettzd(cls):
		"""Return w3's timezone identification string.

		Expresed as [+/-]hh:mm.  For instance, PST is -08:00.  Zone is
		coincides with what localtime(), etc., use.

		"""
		if time.daylight: offset = -1 * time.altzone/60
		else: offset = -1 * time.timezone/60
		if offset > 0: prefix = "+"
		elif offset < 0: prefix = "-"
		else: return "Z" # time is already in UTC

		hours, minutes = map(abs, divmod(offset, 60))
		assert 0 <= hours <= 23
		assert 0 <= minutes <= 59
		return "%s%02d%s%02d" % (prefix, hours,
								 Globals.time_separator, minutes)

	def tzdtoseconds(cls, tzd):
		"""Given w3 compliant TZD, return how far ahead UTC is"""
		if tzd == "Z": return 0
		assert len(tzd) == 6 # only accept forms like +08:00 for now
		assert (tzd[0] == "-" or tzd[0] == "+") and \
			   tzd[3] == Globals.time_separator
		return -60 * (60 * int(tzd[:3]) + int(tzd[4:]))

	def cmp(cls, time1, time2):
		"""Compare time1 and time2 and return -1, 0, or 1"""
		if type(time1) is types.StringType:
			time1 = cls.stringtotime(time1)
			assert time1 is not None
		if type(time2) is types.StringType:
			time2 = cls.stringtotime(time2)
			assert time2 is not None
		
		if time1 < time2: return -1
		elif time1 == time2: return 0
		else: return 1

	def genstrtotime(cls, timestr, curtime = None):
		"""Convert a generic time string to a time in seconds"""
		if curtime is None: curtime = cls.curtime
		if timestr == "now": return curtime

		def error():
			raise TimeException("""Bad time string "%s"

The acceptible time strings are intervals (like "3D64s"), w3-datetime
strings, like "2002-04-26T04:22:01-07:00" (strings like
"2002-04-26T04:22:01" are also acceptable - rdiff-backup will use the
current time zone), or ordinary dates like 2/4/1997 or 2001-04-23
(various combinations are acceptable, but the month always precedes
the day).""" % timestr)

		# Test for straight integer
		if cls._integer_regexp.search(timestr): return int(timestr)

		# Test for w3-datetime format, possibly missing tzd
		t = cls.stringtotime(timestr) or cls.stringtotime(timestr+cls.gettzd())
		if t: return t

		try: # test for an interval, like "2 days ago"
			return curtime - cls.intstringtoseconds(timestr)
		except TimeException: pass

		# Now check for dates like 2001/3/23
		match = cls._genstr_date_regexp1.search(timestr) or \
				cls._genstr_date_regexp2.search(timestr)
		if not match: error()
		timestr = "%s-%02d-%02dT00:00:00%s" % \
				  (match.group('year'), int(match.group('month')),
				   int(match.group('day')), cls.gettzd())
		t = cls.stringtotime(timestr)
		if t: return t
		else: error()

MakeClass(Time)


#######################################################################
#
# iterfile - Convert an iterator to a file object and vice-versa
#

class IterFileException(Exception): pass

class UnwrapFile:
	"""Contains some basic methods for parsing a file containing an iter"""
	def __init__(self, file):
		self.file = file

	def _s2l(self, s):
		"""Convert string to long int"""
		assert len(s) == 7
		l = 0L
		for i in range(7): l = l*256 + ord(s[i])
		return l

	def _get(self):
		"""Return pair (type, data) next in line on the file

		type is a single character which is either "o" for object, "f"
		for file, "c" for a continution of a file, or None if no more
		data can be read.  Data is either the file's data, if type is
		"c" or "f", or the actual object if the type is "o".

		"""
		header = self.file.read(8)
		if not header: return None, None
		assert len(header) == 8, "Header is only %d bytes" % len(header)
		type, length = header[0], self._s2l(header[1:])
		buf = self.file.read(length)
		if type == "o": return type, cPickle.loads(buf)
		else: return type, buf


class IterWrappingFile(UnwrapFile):
	"""An iterator generated from a file.

	Initialize with a file type object, and then it will return the
	elements of the file in order.

	"""
	def __init__(self, file):
		UnwrapFile.__init__(self, file)
		self.currently_in_file = None

	def __iter__(self): return self

	def next(self):
		if self.currently_in_file:
			self.currently_in_file.close() # no error checking by this point
		type, data = self._get()
		if not type: raise StopIteration
		if type == "o": return data
		elif type == "f":
			file = IterVirtualFile(self, data)
			if data: self.currently_in_file = file
			else: self.currently_in_file = None
			return file
		else: raise IterFileException("Bad file type %s" % type)


class IterVirtualFile(UnwrapFile):
	"""Another version of a pretend file

	This is returned by IterWrappingFile when a file is embedded in
	the main file that the IterWrappingFile is based around.

	"""
	def __init__(self, iwf, initial_data):
		"""Initializer

		initial_data is the data from the first block of the file.
		iwf is the iter wrapping file that spawned this
		IterVirtualFile.

		"""
		UnwrapFile.__init__(self, iwf.file)
		self.iwf = iwf
		self.bufferlist = [initial_data]
		self.bufferlen = len(initial_data)
		self.closed = None

	def check_consistency(self):
		l = len("".join(self.bufferlist))
		assert l == self.bufferlen, \
			   "Length of IVF bufferlist doesn't match (%s, %s)" % \
			   (l, self.bufferlen)

	def read(self, length):
		assert not self.closed
		if self.iwf.currently_in_file:
			while length >= self.bufferlen:
				if not self.addtobuffer(): break

		real_len = min(length, self.bufferlen)
		combined_buffer = "".join(self.bufferlist)
		assert len(combined_buffer) == self.bufferlen, \
			   (len(combined_buffer), self.bufferlen)
		self.bufferlist = [combined_buffer[real_len:]]
		self.bufferlen = self.bufferlen - real_len
		return combined_buffer[:real_len]
			
	def addtobuffer(self):
		"""Read a chunk from the file and add it to the buffer"""
		assert self.iwf.currently_in_file
		type, data = self._get()
		assert type == "c", "Type is %s instead of c" % type
		if data:
			self.bufferlen = self.bufferlen + len(data)
			self.bufferlist.append(data)
			return 1
		else:
			self.iwf.currently_in_file = None
			return None

	def close(self):
		"""Currently just reads whats left and discards it"""
		while self.iwf.currently_in_file:
			self.addtobuffer()
			self.bufferlist = []
			self.bufferlen = 0
		self.closed = 1


class FileWrappingIter:
	"""A file interface wrapping around an iterator

	This is initialized with an iterator, and then converts it into a
	stream of characters.  The object will evaluate as little of the
	iterator as is necessary to provide the requested bytes.

	The actual file is a sequence of marshaled objects, each preceded
	by 8 bytes which identifies the following the type of object, and
	specifies its length.  File objects are not marshalled, but the
	data is written in chunks of Globals.blocksize, and the following
	blocks can identify themselves as continuations.

	"""
	def __init__(self, iter):
		"""Initialize with iter"""
		self.iter = iter
		self.bufferlist = []
		self.bufferlen = 0L
		self.currently_in_file = None
		self.closed = None

	def read(self, length):
		"""Return next length bytes in file"""
		assert not self.closed
		while self.bufferlen < length:
			if not self.addtobuffer(): break

		combined_buffer = "".join(self.bufferlist)
		assert len(combined_buffer) == self.bufferlen
		real_len = min(self.bufferlen, length)
		self.bufferlen = self.bufferlen - real_len
		self.bufferlist = [combined_buffer[real_len:]]
		return combined_buffer[:real_len]

	def addtobuffer(self):
		"""Updates self.bufferlist and self.bufferlen, adding on a chunk

		Returns None if we have reached the end of the iterator,
		otherwise return true.

		"""
		if self.currently_in_file:
			buf = "c" + self.addfromfile()
		else:
			try: currentobj = self.iter.next()
			except StopIteration: return None
			if hasattr(currentobj, "read") and hasattr(currentobj, "close"):
				self.currently_in_file = currentobj
				buf = "f" + self.addfromfile()
			else:
				pickle = cPickle.dumps(currentobj, 1)
				buf = "o" + self._l2s(len(pickle)) + pickle
				
		self.bufferlist.append(buf)
		self.bufferlen = self.bufferlen + len(buf)
		return 1

	def addfromfile(self):
		"""Read a chunk from the current file and return it"""
		buf = self.currently_in_file.read(Globals.blocksize)
		if not buf:
			assert not self.currently_in_file.close()
			self.currently_in_file = None
		return self._l2s(len(buf)) + buf

	def _l2s(self, l):
		"""Convert long int to string of 7 characters"""
		s = ""
		for i in range(7):
			l, remainder = divmod(l, 256)
			s = chr(remainder) + s
		assert remainder == 0
		return s

	def close(self): self.closed = 1


class BufferedRead:
	"""Buffer the .read() calls to the given file

	This is used to lessen overhead and latency when a file is sent
	over a connection.

	"""
	def __init__(self, file):
		self.file = file
		self.buffer = ""
		self.bufsize = Globals.conn_bufsize

	def read(self, l = -1):
		if l < 0: # Read as much as possible
			result = self.buffer + self.file.read()
			self.buffer = ""
			return result

		if len(self.buffer) < l: # Try to make buffer as long as l
			self.buffer += self.file.read(max(self.bufsize,
											  l - len(self.buffer)))
		actual_size = min(l, len(self.buffer))
		result = self.buffer[:actual_size]
		self.buffer = self.buffer[actual_size:]
		return result

	def close(self): return self.file.close()


#######################################################################
#
# rdiff - Invoke rdiff utility to make signatures, deltas, or patch
#

class RdiffException(Exception): pass

class Rdiff:
	"""Contains static methods for rdiff operations

	All these operations should be done in a relatively safe manner
	using RobustAction and the like.

	"""
	def get_signature(rp):
		"""Take signature of rpin file and return in file object"""
		Log("Getting signature of %s" % rp.path, 7)
		return rp.conn.RdiffPopen(['rdiff', 'signature', rp.path])

	def get_delta_sigfileobj(sig_fileobj, rp_new):
		"""Like get_delta but signature is in a file object"""
		sig_tf = TempFileManager.new(rp_new, None)
		sig_tf.write_from_fileobj(sig_fileobj)
		rdiff_popen_obj = Rdiff.get_delta_sigrp(sig_tf, rp_new)
		rdiff_popen_obj.set_thunk(sig_tf.delete)
		return rdiff_popen_obj

	def get_delta_sigrp(rp_signature, rp_new):
		"""Take signature rp and new rp, return delta file object"""
		assert rp_signature.conn is rp_new.conn
		Log("Getting delta of %s with signature %s" %
			(rp_new.path, rp_signature.path), 7)
		return rp_new.conn.RdiffPopen(['rdiff', 'delta',
									   rp_signature.path, rp_new.path])

	def write_delta_action(basis, new, delta, compress = None):
		"""Return action writing delta which brings basis to new

		If compress is true, the output of rdiff will be gzipped
		before written to delta.

		"""
		sig_tf = TempFileManager.new(new, None)
		delta_tf = TempFileManager.new(delta)
		def init(): Rdiff.write_delta(basis, new, delta_tf, compress, sig_tf)
		return Robust.make_tf_robustaction(init, (sig_tf, delta_tf),
										   (None, delta))

	def write_delta(basis, new, delta, compress = None, sig_tf = None):
		"""Write rdiff delta which brings basis to new"""
		Log("Writing delta %s from %s -> %s" %
			(basis.path, new.path, delta.path), 7)
		if not sig_tf: sig_tf = TempFileManager.new(new, None)
		sig_tf.write_from_fileobj(Rdiff.get_signature(basis))
		delta.write_from_fileobj(Rdiff.get_delta_sigrp(sig_tf, new), compress)
		sig_tf.delete()

	def patch_action(rp_basis, rp_delta, rp_out = None,
					 out_tf = None, delta_compressed = None):
		"""Return RobustAction which patches rp_basis with rp_delta

		If rp_out is None, put output in rp_basis.  Will use TempFile
		out_tf it is specified.  If delta_compressed is true, the
		delta file will be decompressed before processing with rdiff.

		"""
		if not rp_out: rp_out = rp_basis
		else: assert rp_out.conn is rp_basis.conn
		if (delta_compressed or
			not (isinstance(rp_delta, RPath) and isinstance(rp_basis, RPath)
				 and rp_basis.conn is rp_delta.conn)):
			if delta_compressed:
				assert isinstance(rp_delta, RPath)
				return Rdiff.patch_fileobj_action(rp_basis,
												  rp_delta.open('rb', 1),
												  rp_out, out_tf)
			else: return Rdiff.patch_fileobj_action(rp_basis,
													rp_delta.open('rb'),
													rp_out, out_tf)

		# Files are uncompressed on same connection, run rdiff
		if out_tf is None: out_tf = TempFileManager.new(rp_out)
		def init():
			Log("Patching %s using %s to %s via %s" %
				(rp_basis.path, rp_delta.path, rp_out.path, out_tf.path), 7)
			cmdlist = ["rdiff", "patch", rp_basis.path,
					   rp_delta.path, out_tf.path]
			return_val = rp_basis.conn.os.spawnvp(os.P_WAIT, 'rdiff', cmdlist)
			out_tf.setdata()
			if return_val != 0 or not out_tf.lstat():
				RdiffException("Error running %s" % cmdlist)
		return Robust.make_tf_robustaction(init, (out_tf,), (rp_out,))

	def patch_fileobj_action(rp_basis, delta_fileobj, rp_out = None,
							 out_tf = None, delta_compressed = None):
		"""Like patch_action but diff is given in fileobj form

		Nest a writing of a tempfile with the actual patching to
		create a new action.  We have to nest so that the tempfile
		will be around until the patching finishes.

		"""
		if not rp_out: rp_out = rp_basis
		delta_tf = TempFileManager.new(rp_out, None)
		def init(): delta_tf.write_from_fileobj(delta_fileobj)
		def final(init_val): delta_tf.delete()
		def error(exc, ran_init, init_val): delta_tf.delete()
		write_delta_action = RobustAction(init, final, error)
		return Robust.chain(write_delta_action,
							Rdiff.patch_action(rp_basis, delta_tf,
											   rp_out, out_tf))

	def patch_with_attribs_action(rp_basis, rp_delta, rp_out = None):
		"""Like patch_action, but also transfers attributs from rp_delta"""
		if not rp_out: rp_out = rp_basis
		tf = TempFileManager.new(rp_out)
		return Robust.chain_nested(
			Rdiff.patch_action(rp_basis, rp_delta, rp_out, tf),
			Robust.copy_attribs_action(rp_delta, tf))

	def copy_action(rpin, rpout):
		"""Use rdiff to copy rpin to rpout, conserving bandwidth"""
		if not rpin.isreg() or not rpout.isreg() or rpin.conn is rpout.conn:
			# rdiff not applicable, fallback to regular copying
			return Robust.copy_action(rpin, rpout)

		Log("Rdiff copying %s to %s" % (rpin.path, rpout.path), 6)		
		delta_tf = TempFileManager.new(rpout, None)
		return Robust.chain(Rdiff.write_delta_action(rpout, rpin, delta_tf),
							Rdiff.patch_action(rpout, delta_tf),
							RobustAction(lambda: None, delta_tf.delete,
										 lambda exc: delta_tf.delete))

MakeStatic(Rdiff)


class RdiffPopen:
	"""Spawn process and treat stdout as file object

	Instead of using popen, which evaluates arguments with the shell
	and thus may lead to security holes (thanks to Jamie Heilman for
	this point), use the popen2 class and discard stdin.

	When closed, this object checks to make sure the process exited
	cleanly, and executes closing_thunk.

	"""
	def __init__(self, cmdlist, closing_thunk = None):
		"""RdiffFilehook initializer

		fileobj is the file we are emulating
		thunk is called with no parameters right after the file is closed

		"""
		assert type(cmdlist) is types.ListType
		self.p3obj = popen2.Popen3(cmdlist)
		self.fileobj = self.p3obj.fromchild
		self.closing_thunk = closing_thunk
		self.cmdlist = cmdlist

	def set_thunk(self, closing_thunk):
		"""Set closing_thunk if not already"""
		assert not self.closing_thunk
		self.closing_thunk = closing_thunk

	def read(self, length = -1): return self.fileobj.read(length)

	def close(self):
		closeval = self.fileobj.close()
		if self.closing_thunk: self.closing_thunk()
		exitval = self.p3obj.poll()
		if exitval == 0: return closeval
		elif exitval == 256:
			Log("Failure probably because %s couldn't be found in PATH."
				% self.cmdlist[0], 2)
			assert 0, "rdiff not found"
		elif exitval == -1:
			# There may a race condition where a process closes
			# but doesn't provide its exitval fast enough.
			Log("Waiting for process to close", 8)
			time.sleep(0.2)
			exitval = self.p3obj.poll()
			if exitval == 0: return closeval
		raise RdiffException("%s exited with non-zero value %d" %
							 (self.cmdlist, exitval))


#######################################################################
#
# connection - Code that deals with remote execution
#

class ConnectionError(Exception): pass
class ConnectionReadError(ConnectionError): pass

class ConnectionQuit(Exception): pass


class Connection:
	"""Connection class - represent remote execution

	The idea is that, if c is an instance of this class, c.foo will
	return the object on the remote side.  For functions, c.foo will
	return a function that, when called, executes foo on the remote
	side, sending over the arguments and sending back the result.

	"""
	def __repr__(self): return self.__str__()
	def __str__(self): return "Simple Connection" # override later

class LocalConnection(Connection):
	"""Local connection

	This is a dummy connection class, so that LC.foo just evaluates to
	foo using global scope.

	"""
	def __init__(self):
		"""This prevents two instances of LocalConnection"""
		assert not Globals.local_connection
		self.conn_number = 0 # changed by SetConnections for server

	def __getattr__(self, name):
		try: return globals()[name]
		except KeyError:
			builtins = globals()["__builtins__"]
			try:
				if type(builtins) is types.ModuleType:
					return builtins.__dict__[name]
				else: return builtins[name]
			except KeyError: raise NameError, name

	def __setattr__(self, name, value):
		globals()[name] = value

	def __delattr__(self, name):
		del globals()[name]

	def __str__(self): return "LocalConnection"

	def reval(self, function_string, *args):
		return apply(eval(function_string), args)

	def quit(self): pass

Globals.local_connection = LocalConnection()
Globals.connections.append(Globals.local_connection)
# Following changed by server in SetConnections
Globals.connection_dict[0] = Globals.local_connection


class ConnectionRequest:
	"""Simple wrapper around a PipeConnection request"""
	def __init__(self, function_string, num_args):
		self.function_string = function_string
		self.num_args = num_args

	def __str__(self):
		return "ConnectionRequest: %s with %d arguments" % \
			   (self.function_string, self.num_args)


class LowLevelPipeConnection(Connection):
	"""Routines for just sending objects from one side of pipe to another

	Each thing sent down the pipe is paired with a request number,
	currently limited to be between 0 and 255.  The size of each thing
	should be less than 2^56.

	Each thing also has a type, indicated by one of the following
	characters:

	o - generic object
	i - iterator/generator of RORPs
	f - file object
	b - string
	q - quit signal
	t - TempFile
	d - DSRPath
	R - RPath
	r - RORPath only
	c - PipeConnection object

	"""
	def __init__(self, inpipe, outpipe):
		"""inpipe is a file-type open for reading, outpipe for writing"""
		self.inpipe = inpipe
		self.outpipe = outpipe

	def __str__(self):
		"""Return string version

		This is actually an important function, because otherwise
		requests to represent this object would result in "__str__"
		being executed on the other side of the connection.

		"""
		return "LowLevelPipeConnection"

	def _put(self, obj, req_num):
		"""Put an object into the pipe (will send raw if string)"""
		Log.conn("sending", obj, req_num)
		if type(obj) is types.StringType: self._putbuf(obj, req_num)
		elif isinstance(obj, Connection): self._putconn(obj, req_num)
		elif isinstance(obj, TempFile): self._puttempfile(obj, req_num)
		elif isinstance(obj, DSRPath): self._putdsrpath(obj, req_num)
		elif isinstance(obj, RPath): self._putrpath(obj, req_num)
		elif isinstance(obj, RORPath): self._putrorpath(obj, req_num)
		elif ((hasattr(obj, "read") or hasattr(obj, "write"))
			  and hasattr(obj, "close")): self._putfile(obj, req_num)
		elif hasattr(obj, "next"): self._putiter(obj, req_num)
		else: self._putobj(obj, req_num)

	def _putobj(self, obj, req_num):
		"""Send a generic python obj down the outpipe"""
		self._write("o", cPickle.dumps(obj, 1), req_num)

	def _putbuf(self, buf, req_num):
		"""Send buffer buf down the outpipe"""
		self._write("b", buf, req_num)

	def _putfile(self, fp, req_num):
		"""Send a file to the client using virtual files"""
		self._write("f", str(VirtualFile.new(fp)), req_num)

	def _putiter(self, iterator, req_num):
		"""Put an iterator through the pipe"""
		self._write("i", str(VirtualFile.new(RORPIter.ToFile(iterator))),
					req_num)

	def _puttempfile(self, tempfile, req_num):
		"""Put a tempfile into pipe.  See _putrpath"""
		tf_repr = (tempfile.conn.conn_number, tempfile.base,
				   tempfile.index, tempfile.data)
		self._write("t", cPickle.dumps(tf_repr, 1), req_num)

	def _putdsrpath(self, dsrpath, req_num):
		"""Put DSRPath into pipe.  See _putrpath"""
		dsrpath_repr = (dsrpath.conn.conn_number, dsrpath.getstatedict())
		self._write("d", cPickle.dumps(dsrpath_repr, 1), req_num)

	def _putrpath(self, rpath, req_num):
		"""Put an rpath into the pipe

		The rpath's connection will be encoded as its conn_number.  It
		and the other information is put in a tuple.

		"""
		rpath_repr = (rpath.conn.conn_number, rpath.base,
					  rpath.index, rpath.data)
		self._write("R", cPickle.dumps(rpath_repr, 1), req_num)

	def _putrorpath(self, rorpath, req_num):
		"""Put an rorpath into the pipe

		This is only necessary because if there is a .file attached,
		it must be excluded from the pickling

		"""
		rorpath_repr = (rorpath.index, rorpath.data)
		self._write("r", cPickle.dumps(rorpath_repr, 1), req_num)

	def _putconn(self, pipeconn, req_num):
		"""Put a connection into the pipe

		A pipe connection is represented just as the integer (in
		string form) of its connection number it is *connected to*.

		"""
		self._write("c", str(pipeconn.conn_number), req_num)

	def _putquit(self):
		"""Send a string that takes down server"""
		self._write("q", "", 255)

	def _write(self, headerchar, data, req_num):
		"""Write header and then data to the pipe"""
		self.outpipe.write(headerchar + chr(req_num) + self._l2s(len(data)))
		self.outpipe.write(data)
		self.outpipe.flush()

	def _read(self, length):
		"""Read length bytes from inpipe, returning result"""
		return self.inpipe.read(length)

	def _s2l(self, s):
		"""Convert string to long int"""
		assert len(s) == 7
		l = 0L
		for i in range(7): l = l*256 + ord(s[i])
		return l

	def _l2s(self, l):
		"""Convert long int to string"""
		s = ""
		for i in range(7):
			l, remainder = divmod(l, 256)
			s = chr(remainder) + s
		assert remainder == 0
		return s

	def _get(self):
		"""Read an object from the pipe and return (req_num, value)"""
		header_string = self.inpipe.read(9)
		if not len(header_string) == 9:
			raise ConnectionReadError("Truncated header string (problem "
									  "probably originated remotely)")
		try:
			format_string, req_num, length = (header_string[0],
											  ord(header_string[1]),
											  self._s2l(header_string[2:]))
		except IndexError: raise ConnectionError()
		if format_string == "q": raise ConnectionQuit("Received quit signal")

		data = self._read(length)
		if format_string == "o": result = cPickle.loads(data)
		elif format_string == "b": result = data
		elif format_string == "f": result = VirtualFile(self, int(data))
		elif format_string == "i":
			result = RORPIter.FromFile(BufferedRead(VirtualFile(self,
																int(data))))
		elif format_string == "t": result = self._gettempfile(data)
		elif format_string == "r": result = self._getrorpath(data)
		elif format_string == "R": result = self._getrpath(data)
		elif format_string == "d": result = self._getdsrpath(data)
		else:
			assert format_string == "c", header_string
			result = Globals.connection_dict[int(data)]
		Log.conn("received", result, req_num)
		return (req_num, result)

	def _getrorpath(self, raw_rorpath_buf):
		"""Reconstruct RORPath object from raw data"""
		index, data = cPickle.loads(raw_rorpath_buf)
		return RORPath(index, data)

	def _gettempfile(self, raw_tf_buf):
		"""Return TempFile object indicated by raw_tf_buf"""
		conn_number, base, index, data = cPickle.loads(raw_tf_buf)
		return TempFile(Globals.connection_dict[conn_number],
						base, index, data)

	def _getrpath(self, raw_rpath_buf):
		"""Return RPath object indicated by raw_rpath_buf"""
		conn_number, base, index, data = cPickle.loads(raw_rpath_buf)
		return RPath(Globals.connection_dict[conn_number], base, index, data)

	def _getdsrpath(self, raw_dsrpath_buf):
		"""Return DSRPath object indicated by buf"""
		conn_number, state_dict = cPickle.loads(raw_dsrpath_buf)
		empty_dsrp = DSRPath("bypass", Globals.local_connection, None)
		empty_dsrp.__setstate__(state_dict)
		empty_dsrp.conn = Globals.connection_dict[conn_number]
		empty_dsrp.file = None
		return empty_dsrp

	def _close(self):
		"""Close the pipes associated with the connection"""
		self.outpipe.close()
		self.inpipe.close()


class PipeConnection(LowLevelPipeConnection):
	"""Provide server and client functions for a Pipe Connection

	Both sides act as modules that allows for remote execution.  For
	instance, self.conn.pow(2,8) will execute the operation on the
	server side.

	The only difference between the client and server is that the
	client makes the first request, and the server listens first.

	"""
	def __init__(self, inpipe, outpipe, conn_number = 0):
		"""Init PipeConnection

		conn_number should be a unique (to the session) integer to
		identify the connection.  For instance, all connections to the
		client have conn_number 0.  Other connections can use this
		number to route commands to the correct process.

		"""
		LowLevelPipeConnection.__init__(self, inpipe, outpipe)
		self.conn_number = conn_number
		self.unused_request_numbers = {}
		for i in range(256): self.unused_request_numbers[i] = None

	def __str__(self): return "PipeConnection %d" % self.conn_number

	def get_response(self, desired_req_num):
		"""Read from pipe, responding to requests until req_num.

		Sometimes after a request is sent, the other side will make
		another request before responding to the original one.  In
		that case, respond to the request.  But return once the right
		response is given.

		"""
		while 1:
			try: req_num, object = self._get()
			except ConnectionQuit:
				self._put("quitting", self.get_new_req_num())
				return
			if req_num == desired_req_num: return object
			else:
				assert isinstance(object, ConnectionRequest)
				self.answer_request(object, req_num)

	def answer_request(self, request, req_num):
		"""Put the object requested by request down the pipe"""
		del self.unused_request_numbers[req_num]
		argument_list = []
		for i in range(request.num_args):
			arg_req_num, arg = self._get()
			assert arg_req_num == req_num
			argument_list.append(arg)
		try: result = apply(eval(request.function_string), argument_list)
		except: result = self.extract_exception()
		self._put(result, req_num)
		self.unused_request_numbers[req_num] = None

	def extract_exception(self):
		"""Return active exception"""
		if Log.verbosity >= 5 or Log.term_verbosity >= 5:
			Log("Sending back exception %s of type %s: \n%s" %
				(sys.exc_info()[1], sys.exc_info()[0],
				 "".join(traceback.format_tb(sys.exc_info()[2]))), 5)
		return sys.exc_info()[1]

	def Server(self):
		"""Start server's read eval return loop"""
		Globals.server = 1
		Globals.connections.append(self)
		Log("Starting server", 6)
		self.get_response(-1)

	def reval(self, function_string, *args):
		"""Execute command on remote side

		The first argument should be a string that evaluates to a
		function, like "pow", and the remaining are arguments to that
		function.

		"""
		req_num = self.get_new_req_num()
		self._put(ConnectionRequest(function_string, len(args)), req_num)
		for arg in args: self._put(arg, req_num)
		result = self.get_response(req_num)
		self.unused_request_numbers[req_num] = None
		if isinstance(result, Exception): raise result
		else: return result

	def get_new_req_num(self):
		"""Allot a new request number and return it"""
		if not self.unused_request_numbers:
			raise ConnectionError("Exhaused possible connection numbers")
		req_num = self.unused_request_numbers.keys()[0]
		del self.unused_request_numbers[req_num]
		return req_num

	def quit(self):
		"""Close the associated pipes and tell server side to quit"""
		assert not Globals.server
		self._putquit()
		self._get()
		self._close()

	def __getattr__(self, name):
		"""Intercept attributes to allow for . invocation"""
		return EmulateCallable(self, name)


class RedirectedConnection(Connection):
	"""Represent a connection more than one move away

	For instance, suppose things are connected like this: S1---C---S2.
	If Server1 wants something done by Server2, it will have to go
	through the Client.  So on S1's side, S2 will be represented by a
	RedirectedConnection.

	"""
	def __init__(self, conn_number, routing_number = 0):
		"""RedirectedConnection initializer

		Returns a RedirectedConnection object for the given
		conn_number, where commands are routed through the connection
		with the given routing_number.  0 is the client, so the
		default shouldn't have to be changed.

		"""
		self.conn_number = conn_number
		self.routing_number = routing_number
		self.routing_conn = Globals.connection_dict[routing_number]

	def __str__(self):
		return "RedirectedConnection %d,%d" % (self.conn_number,
											   self.routing_number)

	def __getattr__(self, name):
		return EmulateCallable(self.routing_conn,
		   "Globals.get_dict_val('connection_dict', %d).%s"
							   % (self.conn_number, name))


class EmulateCallable:
	"""This is used by PipeConnection in calls like conn.os.chmod(foo)"""
	def __init__(self, connection, name):
		self.connection = connection
		self.name = name
	def __call__(self, *args):
		return apply(self.connection.reval, (self.name,) + args)
	def __getattr__(self, attr_name):
		return EmulateCallable(self.connection,
							   "%s.%s" % (self.name, attr_name))


class VirtualFile:
	"""When the client asks for a file over the connection, it gets this

	The returned instance then forwards requests over the connection.
	The class's dictionary is used by the server to associate each
	with a unique file number.

	"""
	#### The following are used by the server
	vfiles = {}
	counter = 0

	def getbyid(cls, id):
		return cls.vfiles[id]
	getbyid = classmethod(getbyid)

	def readfromid(cls, id, length):
		return cls.vfiles[id].read(length)
	readfromid = classmethod(readfromid)

	def readlinefromid(cls, id):
		return cls.vfiles[id].readline()
	readlinefromid = classmethod(readlinefromid)

	def writetoid(cls, id, buffer):
		return cls.vfiles[id].write(buffer)
	writetoid = classmethod(writetoid)

	def closebyid(cls, id):
		fp = cls.vfiles[id]
		del cls.vfiles[id]
		return fp.close()
	closebyid = classmethod(closebyid)

	def new(cls, fileobj):
		"""Associate a new VirtualFile with a read fileobject, return id"""
		count = cls.counter
		cls.vfiles[count] = fileobj
		cls.counter = count + 1
		return count
	new = classmethod(new)


	#### And these are used by the client
	def __init__(self, connection, id):
		self.connection = connection
		self.id = id

	def read(self, length = -1):
		return self.connection.VirtualFile.readfromid(self.id, length)

	def readline(self):
		return self.connection.VirtualFile.readlinefromid(self.id)

	def write(self, buf):
		return self.connection.VirtualFile.writetoid(self.id, buf)

	def close(self):
		return self.connection.VirtualFile.closebyid(self.id)

	def __iter__(self):
		"""Iterates lines in file, like normal iter(file) behavior"""
		while 1:
			line = self.readline()
			if not line: break
			yield line


#######################################################################
#
# rpath - Wrapper class around a real path like "/usr/bin/env"
#
# The RPath and associated classes make some function calls more
# convenient (e.g. RPath.getperms()) and also make working with files
# on remote systems transparent.
#

class RPathException(Exception): pass

class RPathStatic:
	"""Contains static methods for use with RPaths"""
	def copyfileobj(inputfp, outputfp):
		"""Copies file inputfp to outputfp in blocksize intervals"""
		blocksize = Globals.blocksize
		while 1:
			inbuf = inputfp.read(blocksize)
			if not inbuf: break
			outputfp.write(inbuf)

	def cmpfileobj(fp1, fp2):
		"""True if file objects fp1 and fp2 contain same data"""
		blocksize = Globals.blocksize
		while 1:
			buf1 = fp1.read(blocksize)
			buf2 = fp2.read(blocksize)
			if buf1 != buf2: return None
			elif not buf1: return 1

	def check_for_files(*rps):
		"""Make sure that all the rps exist, raise error if not"""
		for rp in rps:
			if not rp.lstat():
				raise RPathException("File %s does not exist" % rp.path)

	def move(rpin, rpout):
		"""Move rpin to rpout, renaming if possible"""
		try: RPath.rename(rpin, rpout)
		except os.error:
			RPath.copy(rpin, rpout)
			rpin.delete()

	def copy(rpin, rpout):
		"""Copy RPath rpin to rpout.  Works for symlinks, dirs, etc."""
		Log("Regular copying %s to %s" % (rpin.index, rpout.path), 6)
		if not rpin.lstat():
			raise RPathException, ("File %s does not exist" % rpin.index)

		if rpout.lstat():
			if rpin.isreg() or not RPath.cmp(rpin, rpout):
				rpout.delete()   # easier to write that compare
			else: return
			
		if rpin.isreg(): RPath.copy_reg_file(rpin, rpout)
		elif rpin.isdir(): rpout.mkdir()
		elif rpin.issym(): rpout.symlink(rpin.readlink())
		elif rpin.ischardev():
			major, minor = rpin.getdevnums()
			rpout.makedev("c", major, minor)
		elif rpin.isblkdev():
			major, minor = rpin.getdevnums()
			rpout.makedev("b", major, minor)
		elif rpin.isfifo(): rpout.mkfifo()
		elif rpin.issock(): Log("Found socket, ignoring", 1)
		else: raise RPathException("File %s has unknown type" % rpin.path)

	def copy_reg_file(rpin, rpout):
		"""Copy regular file rpin to rpout, possibly avoiding connection"""
		try:
			if rpout.conn is rpin.conn:
				rpout.conn.shutil.copyfile(rpin.path, rpout.path)
				rpout.setdata()
				return
		except AttributeError: pass
		rpout.write_from_fileobj(rpin.open("rb"))

	def cmp(rpin, rpout):
		"""True if rpin has the same data as rpout

		cmp does not compare file ownership, permissions, or times, or
		examine the contents of a directory.

		"""
		RPath.check_for_files(rpin, rpout)
		if rpin.isreg():
			if not rpout.isreg(): return None
			fp1, fp2 = rpin.open("rb"), rpout.open("rb")
			result = RPathStatic.cmpfileobj(fp1, fp2)
			if fp1.close() or fp2.close():
				raise RPathException("Error closing file")
			return result
		elif rpin.isdir(): return rpout.isdir()
		elif rpin.issym():
			return rpout.issym() and (rpin.readlink() == rpout.readlink())
		elif rpin.ischardev():
			return rpout.ischardev() and \
				   (rpin.getdevnums() == rpout.getdevnums())
		elif rpin.isblkdev():
			return rpout.isblkdev() and \
				   (rpin.getdevnums() == rpout.getdevnums())
		elif rpin.isfifo(): return rpout.isfifo()
		elif rpin.issock(): return rpout.issock()
		else: raise RPathException("File %s has unknown type" % rpin.path)

	def copy_attribs(rpin, rpout):
		"""Change file attributes of rpout to match rpin

		Only changes the chmoddable bits, uid/gid ownership, and
		timestamps, so both must already exist.

		"""
		Log("Copying attributes from %s to %s" % (rpin.index, rpout.path), 7)
		RPath.check_for_files(rpin, rpout)
		if rpin.issym(): return # symlinks have no valid attributes
		if Globals.change_ownership: apply(rpout.chown, rpin.getuidgid())
		rpout.chmod(rpin.getperms())
		if not rpin.isdev(): rpout.setmtime(rpin.getmtime())

	def cmp_attribs(rp1, rp2):
		"""True if rp1 has the same file attributes as rp2

		Does not compare file access times.  If not changing
		ownership, do not check user/group id.

		"""
		RPath.check_for_files(rp1, rp2)
		if Globals.change_ownership and rp1.getuidgid() != rp2.getuidgid():
			result = None
		elif rp1.getperms() != rp2.getperms(): result = None
		elif rp1.issym() and rp2.issym(): # Don't check times for some types
			result = 1
		elif rp1.isblkdev() and rp2.isblkdev(): result = 1
		elif rp1.ischardev() and rp2.ischardev(): result = 1
		else: result = (rp1.getmtime() == rp2.getmtime())
		Log("Compare attribs %s and %s: %s" % (rp1.path, rp2.path, result), 7)
		return result

	def copy_with_attribs(rpin, rpout):
		"""Copy file and then copy over attributes"""
		RPath.copy(rpin, rpout)
		RPath.copy_attribs(rpin, rpout)

	def quick_cmp_with_attribs(rp1, rp2):
		"""Quicker version of cmp_with_attribs

		Instead of reading all of each file, assume that regular files
		are the same if the attributes compare.

		"""
		if not RPath.cmp_attribs(rp1, rp2): return None
		if rp1.isreg() and rp2.isreg() and (rp1.getlen() == rp2.getlen()):
			return 1
		return RPath.cmp(rp1, rp2)

	def cmp_with_attribs(rp1, rp2):
		"""Combine cmp and cmp_attribs"""
		return RPath.cmp_attribs(rp1, rp2) and RPath.cmp(rp1, rp2)

	def rename(rp_source, rp_dest):
		"""Rename rp_source to rp_dest"""
		assert rp_source.conn is rp_dest.conn
		Log("Renaming %s to %s" % (rp_source.path, rp_dest.path), 7)
		rp_source.conn.os.rename(rp_source.path, rp_dest.path)
		rp_dest.data = rp_source.data
		rp_source.data = {'type': None}

		# If we are moving to a DSRPath, assume that the current times
		# are the intended ones.  We need to save them now in case
		# they are changed later.
		if isinstance(rp_dest, DSRPath):
			if rp_dest.delay_mtime:
				if 'mtime' in rp_dest.data:
					rp_dest.setmtime(rp_dest.data['mtime'])
			if rp_dest.delay_atime:
				if 'atime' in rp_dest.data:
					rp_dest.setatime(rp_dest.data['atime'])

	def tupled_lstat(filename):
		"""Like os.lstat, but return only a tuple, or None if os.error

		Later versions of os.lstat return a special lstat object,
		which can confuse the pickler and cause errors in remote
		operations.  This has been fixed in Python 2.2.1.

		"""
		try: return tuple(os.lstat(filename))
		except os.error: return None

MakeStatic(RPathStatic)


class RORPath(RPathStatic):
	"""Read Only RPath - carry information about a path

	These contain information about a file, and possible the file's
	data, but do not have a connection and cannot be written to or
	changed.  The advantage of these objects is that they can be
	communicated by encoding their index and data dictionary.

	"""
	def __init__(self, index, data = None):
		self.index = index
		if data: self.data = data
		else: self.data = {'type':None} # signify empty file
		self.file = None

	def __eq__(self, other):
		"""True iff the two rorpaths are equivalent"""
		if self.index != other.index: return None

		for key in self.data.keys(): # compare dicts key by key
			if ((key == 'uid' or key == 'gid') and
				(not Globals.change_ownership or self.issym())):
				# Don't compare gid/uid for symlinks or if not change_ownership
				pass
			elif key == 'devloc' or key == 'inode' or key == 'nlink': pass
			elif (not other.data.has_key(key) or
				  self.data[key] != other.data[key]): return None
		return 1

	def __ne__(self, other): return not self.__eq__(other)

	def __str__(self):
		"""Pretty print file statistics"""
		return "Index: %s\nData: %s" % (self.index, self.data)

	def __getstate__(self):
		"""Return picklable state

		This is necessary in case the RORPath is carrying around a
		file object, which can't/shouldn't be pickled.

		"""
		return (self.index, self.data)

	def __setstate__(self, rorp_state):
		"""Reproduce RORPath from __getstate__ output"""
		self.index, self.data = rorp_state

	def make_placeholder(self):
		"""Make rorp into a placeholder

		This object doesn't contain any information about the file,
		but, when passed along, may show where the previous stages are
		in their processing.  It is the RORPath equivalent of fiber.
		This placeholder size, in conjunction with the placeholder
		threshold in Highlevel .. generate_dissimilar seem to yield an
		OK tradeoff between unnecessary placeholders and lots of
		memory usage, but I'm not sure exactly why.

		"""
		self.data = {'placeholder': " "*500}

	def isplaceholder(self):
		"""True if the object is a placeholder"""
		return self.data.has_key('placeholder')

	def lstat(self):
		"""Returns type of file

		The allowable types are None if the file doesn't exist, 'reg'
		for a regular file, 'dir' for a directory, 'dev' for a device
		file, 'fifo' for a fifo, 'sock' for a socket, and 'sym' for a
		symlink.
		
		"""
		return self.data['type']
	gettype = lstat

	def isdir(self):
		"""True if self is a dir"""
		return self.data['type'] == 'dir'

	def isreg(self):
		"""True if self is a regular file"""
		return self.data['type'] == 'reg'

	def issym(self):
		"""True if path is of a symlink"""
		return self.data['type'] == 'sym'

	def isfifo(self):
		"""True if path is a fifo"""
		return self.data['type'] == 'fifo'

	def ischardev(self):
		"""True if path is a character device file"""
		return self.data['type'] == 'dev' and self.data['devnums'][0] == 'c'

	def isblkdev(self):
		"""True if path is a block device file"""
		return self.data['type'] == 'dev' and self.data['devnums'][0] == 'b'

	def isdev(self):
		"""True if path is a device file"""
		return self.data['type'] == 'dev'

	def issock(self):
		"""True if path is a socket"""
		return self.data['type'] == 'sock'

	def getperms(self):
		"""Return permission block of file"""
		return self.data['perms']

	def getsize(self):
		"""Return length of file in bytes"""
		return self.data['size']

	def getuidgid(self):
		"""Return userid/groupid of file"""
		return self.data['uid'], self.data['gid']

	def getatime(self):
		"""Return access time in seconds"""
		return self.data['atime']

	def getmtime(self):
		"""Return modification time in seconds"""
		return self.data['mtime']
	
	def getinode(self):
		"""Return inode number of file"""
		return self.data['inode']

	def getdevloc(self):
		"""Device number file resides on"""
		return self.data['devloc']

	def getnumlinks(self):
		"""Number of places inode is linked to"""
		return self.data['nlink']

	def readlink(self):
		"""Wrapper around os.readlink()"""
		return self.data['linkname']

	def getdevnums(self):
		"""Return a devices major/minor numbers from dictionary"""
		return self.data['devnums'][1:]

	def setfile(self, file):
		"""Right now just set self.file to be the already opened file"""
		assert file and not self.file
		def closing_hook(): self.file_already_open = None
		self.file = RPathFileHook(file, closing_hook)
		self.file_already_open = None

	def get_attached_filetype(self):
		"""If there is a file attached, say what it is

		Currently the choices are 'snapshot' meaning an exact copy of
		something, and 'diff' for an rdiff style diff.

		"""
		return self.data['filetype']
	
	def set_attached_filetype(self, type):
		"""Set the type of the attached file"""
		self.data['filetype'] = type

	def isflaglinked(self):
		"""True if rorp is a signature/diff for a hardlink file

		This indicates that a file's data need not be transferred
		because it is hardlinked on the remote side.

		"""
		return self.data.has_key('linked')

	def flaglinked(self):
		"""Signal that rorp is a signature/diff for a hardlink file"""
		self.data['linked'] = 1

	def open(self, mode):
		"""Return file type object if any was given using self.setfile"""
		if mode != "rb": raise RPathException("Bad mode %s" % mode)
		if self.file_already_open:
			raise RPathException("Attempt to open same file twice")
		self.file_already_open = 1
		return self.file

	def close_if_necessary(self):
		"""If file is present, discard data and close"""
		if self.file:
			while self.file.read(Globals.blocksize): pass
			assert not self.file.close(), \
			  "Error closing file\ndata = %s\nindex = %s\n" % (self.data,
															   self.index)
			self.file_already_open = None


class RPath(RORPath):
	"""Remote Path class - wrapper around a possibly non-local pathname

	This class contains a dictionary called "data" which should
	contain all the information about the file sufficient for
	identification (i.e. if two files have the the same (==) data
	dictionary, they are the same file).

	"""
	regex_chars_to_quote = re.compile("[\\\\\\\"\\$`]")

	def __init__(self, connection, base, index = (), data = None):
		"""RPath constructor

		connection = self.conn is the Connection the RPath will use to
		make system calls, and index is the name of the rpath used for
		comparison, and should be a tuple consisting of the parts of
		the rpath after the base split up.  For instance ("foo",
		"bar") for "foo/bar" (no base), and ("local", "bin") for
		"/usr/local/bin" if the base is "/usr".

		For the root directory "/", the index is empty and the base is
		"/".

		"""
		self.conn = connection
		self.index = index
		self.base = base
		self.path = apply(os.path.join, (base,) + self.index)
		self.file = None
		if data or base is None: self.data = data
		else: self.setdata()

	def __str__(self):
		return "Path: %s\nIndex: %s\nData: %s" % (self.path, self.index,
												  self.data)

	def __getstate__(self):
		"""Return picklable state

		The connection must be local because we can't pickle a
		connection.  Data and any attached file also won't be saved.

		"""
		assert self.conn is Globals.local_connection
		return (self.index, self.base, self.data)

	def __setstate__(self, rpath_state):
		"""Reproduce RPath from __getstate__ output"""
		self.conn = Globals.local_connection
		self.index, self.base, self.data = rpath_state
		self.path = apply(os.path.join, (self.base,) + self.index)

	def setdata(self):
		"""Create the data dictionary"""
		statblock = self.conn.RPathStatic.tupled_lstat(self.path)
		if statblock is None:
			self.data = {'type':None}
			return
		data = {}
		mode = statblock[stat.ST_MODE]

		if stat.S_ISREG(mode):
			type = 'reg'
		elif stat.S_ISDIR(mode): type = 'dir'
		elif stat.S_ISCHR(mode):
			type = 'dev'
			data['devnums'] = ('c',) + self._getdevnums()
		elif stat.S_ISBLK(mode):
			type = 'dev'
			data['devnums'] = ('b',) + self._getdevnums()
		elif stat.S_ISFIFO(mode): type = 'fifo'
		elif stat.S_ISLNK(mode):
			type = 'sym'
			data['linkname'] = self.conn.os.readlink(self.path)
		elif stat.S_ISSOCK(mode): type = 'sock'
		else: raise RPathException("Unknown type for %s" % self.path)
		data['type'] = type
		data['size'] = statblock[stat.ST_SIZE]
		data['perms'] = stat.S_IMODE(mode)
		data['uid'] = statblock[stat.ST_UID]
		data['gid'] = statblock[stat.ST_GID]
		data['inode'] = statblock[stat.ST_INO]
		data['devloc'] = statblock[stat.ST_DEV]
		data['nlink'] = statblock[stat.ST_NLINK]

		if not (type == 'sym' or type == 'dev'):
			# mtimes on symlinks and dev files don't work consistently
			data['mtime'] = long(statblock[stat.ST_MTIME])

		if Globals.preserve_atime and not type == 'sym':
			data['atime'] = long(statblock[stat.ST_ATIME])
		self.data = data

	def check_consistency(self):
		"""Raise an error if consistency of rp broken

		This is useful for debugging when the cache and disk get out
		of sync and you need to find out where it happened.

		"""
		temptype = self.data['type']
		self.setdata()
		assert temptype == self.data['type'], \
			   "\nName: %s\nOld: %s --> New: %s\n" % \
			   (self.path, temptype, self.data['type'])

	def _getdevnums(self):
		"""Return tuple for special file (major, minor)"""
		s = self.conn.reval("lambda path: os.lstat(path).st_rdev", self.path)
		return (s >> 8, s & 0xff)

	def quote_path(self):
		"""Set path from quoted version of index"""
		quoted_list = [FilenameMapping.quote(path) for path in self.index]
		self.path = apply(os.path.join, [self.base] + quoted_list)
		self.setdata()

	def chmod(self, permissions):
		"""Wrapper around os.chmod"""
		self.conn.os.chmod(self.path, permissions)
		self.data['perms'] = permissions

	def settime(self, accesstime, modtime):
		"""Change file modification times"""
		Log("Setting time of %s to %d" % (self.path, modtime), 7)
		self.conn.os.utime(self.path, (accesstime, modtime))
		self.data['atime'] = accesstime
		self.data['mtime'] = modtime

	def setmtime(self, modtime):
		"""Set only modtime (access time to present)"""
		Log("Setting time of %s to %d" % (self.path, modtime), 7)
		self.conn.os.utime(self.path, (time.time(), modtime))
		self.data['mtime'] = modtime

	def chown(self, uid, gid):
		"""Set file's uid and gid"""
		self.conn.os.chown(self.path, uid, gid)
		self.data['uid'] = uid
		self.data['gid'] = gid

	def mkdir(self):
		Log("Making directory " + self.path, 6)
		self.conn.os.mkdir(self.path)
		self.setdata()

	def rmdir(self):
		Log("Removing directory " + self.path, 6)
		self.conn.os.rmdir(self.path)
		self.data = {'type': None}

	def listdir(self):
		"""Return list of string paths returned by os.listdir"""
		return self.conn.os.listdir(self.path)

	def symlink(self, linktext):
		"""Make symlink at self.path pointing to linktext"""
		self.conn.os.symlink(linktext, self.path)
		self.setdata()
		assert self.issym()

	def hardlink(self, linkpath):
		"""Make self into a hardlink joined to linkpath"""
		self.conn.os.link(linkpath, self.path)
		self.setdata()

	def mkfifo(self):
		"""Make a fifo at self.path"""
		self.conn.os.mkfifo(self.path)
		self.setdata()
		assert self.isfifo()

	def touch(self):
		"""Make sure file at self.path exists"""
		Log("Touching " + self.path, 7)
		self.conn.open(self.path, "w").close()
		self.setdata()
		assert self.isreg()

	def hasfullperms(self):
		"""Return true if current process has full permissions on the file"""
		if self.isowner(): return self.getperms() % 01000 >= 0700
		elif self.isgroup(): return self.getperms() % 0100 >= 070
		else: return self.getperms() % 010 >= 07

	def readable(self):
		"""Return true if current process has read permissions on the file"""
		if self.isowner(): return self.getperms() % 01000 >= 0400
		elif self.isgroup(): return self.getperms() % 0100 >= 040
		else: return self.getperms() % 010 >= 04

	def executable(self):
		"""Return true if current process has execute permissions"""
		if self.isowner(): return self.getperms() % 0200 >= 0100
		elif self.isgroup(): return self.getperms() % 020 >= 010
		else: return self.getperms() % 02 >= 01
		
	def isowner(self):
		"""Return true if current process is owner of rp or root"""
		uid = self.conn.Globals.get('process_uid')
		return uid == 0 or uid == self.data['uid']

	def isgroup(self):
		"""Return true if current process is in group of rp"""
		return self.conn.Globals.get('process_gid') == self.data['gid']

	def delete(self):
		"""Delete file at self.path

		The destructive stepping allows this function to delete
		directories even if they have files and we lack permissions.

		"""
		Log("Deleting %s" % self.path, 7)
		self.setdata()
		if not self.lstat(): return # must have been deleted in meantime
		elif self.isdir():
			itm = RpathDeleter()
			for dsrp in Select(DSRPath(None, self)).set_iter():
				itm(dsrp.index, dsrp)
			itm.Finish()
		else: self.conn.os.unlink(self.path)
		self.setdata()

	def quote(self):
		"""Return quoted self.path for use with os.system()"""
		return '"%s"' % self.regex_chars_to_quote.sub(
			lambda m: "\\"+m.group(0), self.path)

	def normalize(self):
		"""Return RPath canonical version of self.path

		This just means that redundant /'s will be removed, including
		the trailing one, even for directories.  ".." components will
		be retained.

		"""
		newpath = "/".join(filter(lambda x: x and x != ".",
								  self.path.split("/")))
		if self.path[0] == "/": newpath = "/" + newpath
		elif not newpath: newpath = "."
		return self.newpath(newpath)

	def dirsplit(self):
		"""Returns a tuple of strings (dirname, basename)

		Basename is never '' unless self is root, so it is unlike
		os.path.basename.  If path is just above root (so dirname is
		root), then dirname is ''.  In all other cases dirname is not
		the empty string.  Also, dirsplit depends on the format of
		self, so basename could be ".." and dirname could be a
		subdirectory.  For an atomic relative path, dirname will be
		'.'.

		"""
		normed = self.normalize()
		if normed.path.find("/") == -1: return (".", normed.path)
		comps = normed.path.split("/")
		return "/".join(comps[:-1]), comps[-1]

	def newpath(self, newpath, index = ()):
		"""Return new RPath with the same connection but different path"""
		return self.__class__(self.conn, newpath, index)

	def append(self, ext):
		"""Return new RPath with same connection by adjoing ext"""
		return self.__class__(self.conn, self.base, self.index + (ext,))

	def append_path(self, ext, new_index = ()):
		"""Like append, but add ext to path instead of to index"""
		assert not self.index # doesn't make sense if index isn't ()
		return self.__class__(self.conn, os.path.join(self.base, ext),
							  new_index)

	def new_index(self, index):
		"""Return similar RPath but with new index"""
		return self.__class__(self.conn, self.base, index)

	def open(self, mode, compress = None):
		"""Return open file.  Supports modes "w" and "r".

		If compress is true, data written/read will be gzip
		compressed/decompressed on the fly.

		"""
		if compress: return self.conn.gzip.GzipFile(self.path, mode)
		else: return self.conn.open(self.path, mode)

	def write_from_fileobj(self, fp, compress = None):
		"""Reads fp and writes to self.path.  Closes both when done

		If compress is true, fp will be gzip compressed before being
		written to self.

		"""
		Log("Writing file object to " + self.path, 7)
		assert not self.lstat(), "File %s already exists" % self.path
		outfp = self.open("wb", compress = compress)
		RPath.copyfileobj(fp, outfp)
		if fp.close() or outfp.close():
			raise RPathException("Error closing file")
		self.setdata()

	def isincfile(self):
		"""Return true if path looks like an increment file

		Also sets various inc information used by the *inc* functions.

		"""
		if self.index: dotsplit = self.index[-1].split(".")
		else: dotsplit = self.base.split(".")
		if dotsplit[-1] == "gz":
			compressed = 1
			if len(dotsplit) < 4: return None
			timestring, ext = dotsplit[-3:-1]
		else:
			compressed = None
			if len(dotsplit) < 3: return None
			timestring, ext = dotsplit[-2:]
		if Time.stringtotime(timestring) is None: return None
		if not (ext == "snapshot" or ext == "dir" or
				ext == "missing" or ext == "diff" or ext == "data"):
			return None
		self.inc_timestr = timestring
		self.inc_compressed = compressed
		self.inc_type = ext
		if compressed: self.inc_basestr = ".".join(dotsplit[:-3])
		else: self.inc_basestr = ".".join(dotsplit[:-2])
		return 1

	def isinccompressed(self):
		"""Return true if inc file is compressed"""
		return self.inc_compressed

	def getinctype(self):
		"""Return type of an increment file"""
		return self.inc_type

	def getinctime(self):
		"""Return timestring of an increment file"""
		return self.inc_timestr
	
	def getincbase(self):
		"""Return the base filename of an increment file in rp form"""
		if self.index:
			return self.__class__(self.conn, self.base, self.index[:-1] +
								  (self.inc_basestr,))
		else: return self.__class__(self.conn, self.inc_basestr)

	def getincbase_str(self):
		"""Return the base filename string of an increment file"""
		return self.getincbase().dirsplit()[1]

	def makedev(self, type, major, minor):
		"""Make a special file with specified type, and major/minor nums"""
		cmdlist = ['mknod', self.path, type, str(major), str(minor)]
		if self.conn.os.spawnvp(os.P_WAIT, 'mknod', cmdlist) != 0:
			RPathException("Error running %s" % cmdlist)
		if type == 'c': datatype = 'chr'
		elif type == 'b': datatype = 'blk'
		else: raise RPathException
		self.setdata()

	def getRORPath(self, include_contents = None):
		"""Return read only version of self"""
		rorp = RORPath(self.index, self.data)
		if include_contents: rorp.setfile(self.open("rb"))
		return rorp


class RPathFileHook:
	"""Look like a file, but add closing hook"""
	def __init__(self, file, closing_thunk):
		self.file = file
		self.closing_thunk = closing_thunk

	def read(self, length = -1): return self.file.read(length)
	def write(self, buf): return self.file.write(buf)

	def close(self):
		"""Close file and then run closing thunk"""
		result = self.file.close()
		self.closing_thunk()
		return result


class RpathDeleter(IterTreeReducer):
	"""Delete a directory.  Called by RPath.delete()"""
	def start_process(self, index, dsrp):
		self.dsrp = dsrp

	def end_process(self):
		if self.dsrp.isdir(): self.dsrp.rmdir()
		else: self.dsrp.delete()


#######################################################################
#
# hardlink - code for preserving and restoring hardlinks
#
# If the preserve_hardlinks option is selected, linked files in the
# source directory will be linked in the mirror directory.  Linked
# files are treated like any other with respect to incrementing, but a
# database of all links will be recorded at each session, so linked
# files can still be restored from the increments.
#

class Hardlink:
	"""Hardlink class methods and data

	All these functions are meant to be executed on the destination
	side.  The source side should only transmit inode information.

	"""
	# In all of these lists of indicies are the values.  The keys in
	# _inode_ ones are (inode, devloc) pairs.
	_src_inode_indicies = {}
	_dest_inode_indicies = {}

	# The keys for these two are just indicies.  They share values
	# with the earlier dictionaries.
	_src_index_indicies = {}
	_dest_index_indicies = {}

	# When a linked file is restored, its path is added to this dict,
	# so it can be found when later paths being restored are linked to
	# it.
	_restore_index_path = {}

	def get_inode_key(cls, rorp):
		"""Return rorp's key for _inode_ dictionaries"""
		return (rorp.getinode(), rorp.getdevloc())

	def get_indicies(cls, rorp, source):
		"""Return a list of similarly linked indicies, using rorp's index"""
		if source: dict = cls._src_index_indicies
		else: dict = cls._dest_index_indicies
		try: return dict[rorp.index]
		except KeyError: return []

	def add_rorp(cls, rorp, source):
		"""Process new rorp and update hard link dictionaries

		First enter it into src_inode_indicies.  If we have already
		seen all the hard links, then we can delete the entry.
		Everything must stay recorded in src_index_indicies though.

		"""
		if not rorp.isreg() or rorp.getnumlinks() < 2: return

		if source: inode_dict, index_dict = (cls._src_inode_indicies,
											 cls._src_index_indicies)
		else: inode_dict, index_dict = (cls._dest_inode_indicies,
										cls._dest_index_indicies)

		rp_inode_key = cls.get_inode_key(rorp)
		if inode_dict.has_key(rp_inode_key):
			index_list = inode_dict[rp_inode_key]
			index_list.append(rorp.index)
			if len(index_list) == rorp.getnumlinks():
				del inode_dict[rp_inode_key]
		else: # make new entry in both src dicts
			index_list = [rorp.index]
			inode_dict[rp_inode_key] = index_list
		index_dict[rorp.index] = index_list

	def add_rorp_iter(cls, iter, source):
		"""Return new rorp iterator like iter that cls.add_rorp's first"""
		for rorp in iter:
			cls.add_rorp(rorp, source)
			yield rorp

	def rorp_eq(cls, src_rorp, dest_rorp):
		"""Compare hardlinked for equality

		Two files may otherwise seem equal but be hardlinked in
		different ways.  This function considers them equal enough if
		they have been hardlinked correctly to the previously seen
		indicies.

		"""
		assert src_rorp.index == dest_rorp.index
		if (not src_rorp.isreg() or not dest_rorp.isreg() or
			src_rorp.getnumlinks() == dest_rorp.getnumlinks() == 1):
			return 1 # Hard links don't apply

		src_index_list = cls.get_indicies(src_rorp, 1)
		dest_index_list = cls.get_indicies(dest_rorp, None)

		# If a list only has one element, then it is only hardlinked
		# to itself so far, so that is not a genuine difference yet.
		if not src_index_list or len(src_index_list) == 1:
			return not dest_index_list or len(dest_index_list) == 1
		if not dest_index_list or len(dest_index_list) == 1: return None

		# Both index lists exist and are non-empty
		return src_index_list == dest_index_list # they are always sorted

	def islinked(cls, rorp):
		"""True if rorp's index is already linked to something on src side"""
		return len(cls.get_indicies(rorp, 1)) >= 2

	def restore_link(cls, index, rpath):
		"""Restores a linked file by linking it

		When restoring, all the hardlink data is already present, and
		we can only link to something already written.  In either
		case, add to the _restore_index_path dict, so we know later
		that the file is available for hard
		linking.

		Returns true if succeeded in creating rpath, false if must
		restore rpath normally.

		"""
		if index not in cls._src_index_indicies: return None
		for linked_index in cls._src_index_indicies[index]:
			if linked_index in cls._restore_index_path:
				srcpath = cls._restore_index_path[linked_index]
				Log("Restoring %s by hard linking to %s" %
					(rpath.path, srcpath), 6)
				rpath.hardlink(srcpath)
				return 1
		cls._restore_index_path[index] = rpath.path
		return None

	def link_rp(cls, src_rorp, dest_rpath, dest_root = None):
		"""Make dest_rpath into a link analogous to that of src_rorp"""
		if not dest_root: dest_root = dest_rpath # use base of dest_rpath
		dest_link_rpath = RPath(dest_root.conn, dest_root.base,
								cls.get_indicies(src_rorp, 1)[0])
		dest_rpath.hardlink(dest_link_rpath.path)

	def write_linkdict(cls, rpath, dict, compress = None):
		"""Write link data to the rbdata dir

		It is stored as the a big pickled dictionary dated to match
		the current hardlinks.

		"""
		assert (Globals.isbackup_writer and
				rpath.conn is Globals.local_connection)
		tf = TempFileManager.new(rpath)
		def init():
			fp = tf.open("wb", compress)
			cPickle.dump(dict, fp)
			assert not fp.close()
			tf.setdata()
		Robust.make_tf_robustaction(init, (tf,), (rpath,)).execute()

	def get_linkrp(cls, data_rpath, time, prefix):
		"""Return RPath of linkdata, or None if cannot find"""
		for rp in map(data_rpath.append, data_rpath.listdir()):
			if (rp.isincfile() and rp.getincbase_str() == prefix and
				(rp.getinctype() == 'snapshot' or rp.getinctype() == 'data')
				and Time.stringtotime(rp.getinctime()) == time):
				return rp
		return None

	def get_linkdata(cls, data_rpath, time, prefix = 'hardlink_data'):
		"""Return index dictionary written by write_linkdata at time"""
		rp = cls.get_linkrp(data_rpath, time, prefix)
		if not rp: return None
		fp = rp.open("rb", rp.isinccompressed())
		index_dict = cPickle.load(fp)
		assert not fp.close()
		return index_dict

	def final_writedata(cls):
		"""Write final checkpoint data to rbdir after successful backup"""
		if not cls._src_index_indicies: # no hardlinks, so writing unnecessary
			cls.final_inc = None
			return
		Log("Writing hard link data", 6)
		if Globals.compression:
			cls.final_inc = Globals.rbdir.append("hardlink_data.%s.data.gz" %
												 Time.curtimestr)
		else: cls.final_inc = Globals.rbdir.append("hardlink_data.%s.data" %
												   Time.curtimestr)
		cls.write_linkdict(cls.final_inc,
						   cls._src_index_indicies, Globals.compression)

	def retrieve_final(cls, time):
		"""Set source index dictionary from hardlink_data file if avail"""
		hd = cls.get_linkdata(Globals.rbdir, time)
		if hd is None: return None
		cls._src_index_indicies = hd
		return 1

	def final_checkpoint(cls, data_rpath):
		"""Write contents of the four dictionaries to the data dir

		If rdiff-backup receives a fatal error, it may still be able
		to save the contents of the four hard link dictionaries.
		Because these dictionaries may be big, they are not saved
		after every 20 seconds or whatever, but just at the end.

		"""
		Log("Writing intermediate hard link data to disk", 2)
		src_inode_rp = data_rpath.append("hardlink_source_inode_checkpoint."
										 "%s.data" % Time.curtimestr)
		src_index_rp = data_rpath.append("hardlink_source_index_checkpoint."
										 "%s.data" % Time.curtimestr)
		dest_inode_rp = data_rpath.append("hardlink_dest_inode_checkpoint."
										  "%s.data" % Time.curtimestr)
		dest_index_rp = data_rpath.append("hardlink_dest_index_checkpoint."
										  "%s.data" % Time.curtimestr)
		for (rp, dict) in ((src_inode_rp, cls._src_inode_indicies),
						   (src_index_rp, cls._src_index_indicies),
						   (dest_inode_rp, cls._dest_inode_indicies),
						   (dest_index_rp, cls._dest_index_indicies)):
			cls.write_linkdict(rp, dict)

	def retrieve_checkpoint(cls, data_rpath, time):
		"""Retrieve hardlink data from final checkpoint

		Return true if the retrieval worked, false otherwise.

		"""
		try:
			src_inode = cls.get_linkdata(data_rpath, time,
										 "hardlink_source_inode_checkpoint")
			src_index = cls.get_linkdata(data_rpath, time,
										 "hardlink_source_index_checkpoint")
			dest_inode = cls.get_linkdata(data_rpath, time,
										  "hardlink_dest_inode_checkpoint")
			dest_index = cls.get_linkdata(data_rpath, time,
										  "hardlink_dest_index_checkpoint")
		except cPickle.UnpicklingError:
			Log("Unpickling Error", 2)
			return None
		if (src_inode is None or src_index is None or
			dest_inode is None or dest_index is None): return None
		cls._src_inode_indicies = src_inode
		cls._src_index_indicies = src_index
		cls._dest_inode_indicies = dest_inode
		cls._dest_index_indicies = dest_index
		return 1

	def remove_all_checkpoints(cls):
		"""Remove all hardlink checkpoint information from directory"""
		prefix_list = ["hardlink_source_inode_checkpoint",
					   "hardlink_source_index_checkpoint",
					   "hardlink_dest_inode_checkpoint",
					   "hardlink_dest_index_checkpoint"]
		for rp in map(Globals.rbdir.append, Globals.rbdir.listdir()):
			if (rp.isincfile() and rp.getincbase_str() in prefix_list and
				(rp.getinctype() == 'snapshot' or rp.getinctype() == 'data')):
				rp.delete()

MakeClass(Hardlink)


#######################################################################
#
# robust - code which prevents mirror from being corrupted, error-recovery
#
# Ideally no matter an instance of rdiff-backup gets aborted, no
# information should get lost.  The target directory should be left in
# a coherent state, and later instances of rdiff-backup should clean
# things up so there is no sign that anything ever got aborted or
# failed.
#
# Thus, files should be updated in an atomic way as possible.  Each
# file should be updated (and the corresponding diff files written) or
# not, and it should be clear which happened.  In general, I don't
# think this is possible, since the creation of the diff files and the
# changing of updated files cannot be guarateed to happen together.
# It is possible, I think, to record various information to files
# which would allow a later process to figure out what the last
# operation was, but this would add several file operations to the
# processing of each file, and I don't think, would be a good
# tradeoff.
#
# The compromise reached here is that diff files should be created
# just before the mirror files are updated, and each file update
# should be done with a rename operation on a file in the same
# directory.  Furthermore, every once in a while, rdiff-backup will
# record which file it just finished processing.  If any fatal errors
# are caught, it will also record the last processed file.  Future
# instances may not know exactly when the previous instance was
# aborted, but they will be able to narrow down the possibilities.

class RobustAction:
	"""Represents a file operation to be accomplished later"""
	def __init__(self, init_thunk, final_func, error_handler):
		"""RobustAction initializer

		All the thunks are functions whose return value will be
		ignored.  init_thunk should not make any irreversible changes
		but prepare for the writing of the important data. final_func
		should be as short as possible and do the real work.
		error_handler is run if there is an error in init_thunk or
		final_func.  Errors in init_thunk should be corrected by
		error_handler as if nothing had been run in the first place.

		init_thunk takes no arguments.

		final_thunk takes the return value of init_thunk as its
		argument, and its return value is returned by execute().

		error_handler takes three arguments: the exception, a value
		which is true just in case self.init_thunk ran correctly, and
		a value which will be the return value of init_thunk if it ran
		correctly.

		"""
		self.init_thunk = init_thunk or self.default_init_thunk
		self.final_func = final_func or self.default_final_func
		self.error_handler = error_handler or self.default_error_handler

	def execute(self):
		"""Actually run the operation"""
		ran_init_thunk = None
		try:
			init_val = self.init_thunk()
			ran_init_thunk = 1
			return self.final_func(init_val)
		except Exception, exc: # Catch all errors
			Log.exception()
			TracebackArchive.add()
			if ran_init_thunk: self.error_handler(exc, 1, init_val)
			else: self.error_handler(exc, None, None)
			raise exc

	def default_init_thunk(self): return None
	def default_final_func(self, init_val): return init_val
	def default_error_handler(self, exc, ran_init, init_val): pass


class Robust:
	"""Contains various methods designed to make things safer"""
	null_action = RobustAction(None, None, None)
	def chain(*robust_action_list):
		"""Return chain tying together a number of robust actions

		The whole chain will be aborted if some error occurs in
		initialization stage of any of the component actions.

		"""
		ras_with_started_inits, init_return_vals = [], []
		def init():
			for ra in robust_action_list:
				ras_with_started_inits.append(ra)
				init_return_vals.append(ra.init_thunk())
			return init_return_vals
		def final(init_return_vals):
			final_vals = []
			for ra, init_val in zip(robust_action_list, init_return_vals):
				final_vals.append(ra.final_func(init_val))
			return final_vals
		def error(exc, ran_init, init_val):
			for ra, init_val in zip(ras_with_started_inits, init_return_vals):
				ra.error_handler(exc, 1, init_val)
			for ra in ras_with_started_inits[len(init_return_vals):]:
				ra.error_handler(exc, None, None)
		return RobustAction(init, final, error)

	def chain_nested(*robust_action_list):
		"""Like chain but final actions performed in reverse order"""
		ras_with_started_inits, init_vals = [], []
		def init():
			for ra in robust_action_list:
				ras_with_started_inits.append(ra)
				init_vals.append(ra.init_thunk())
			return init_vals
		def final(init_vals):
			ras_and_inits = zip(robust_action_list, init_vals)
			ras_and_inits.reverse()
			final_vals = []
			for ra, init_val in ras_and_inits:
				final_vals.append(ra.final_func(init_val))
			return final_vals
		def error(exc, ran_init, init_val):
			for ra, init_val in zip(ras_with_started_inits, init_vals):
				ra.error_handler(exc, 1, init_val)
			for ra in ras_with_started_inits[len(init_vals):]:
				ra.error_handler(exc, None, None)
		return RobustAction(init, final, error)

	def make_tf_robustaction(init_thunk, tempfiles, final_renames = None):
		"""Shortcut RobustAction creator when only tempfiles involved

		Often the robust action will just consist of some initial
		stage, renaming tempfiles in the final stage, and deleting
		them if there is an error.  This function makes it easier to
		create RobustActions of that type.

		"""
		if isinstance(tempfiles, TempFile): tempfiles = (tempfiles,)
		if isinstance(final_renames, RPath): final_renames = (final_renames,)
		if final_renames is None: final_renames = [None] * len(tempfiles)
		assert len(tempfiles) == len(final_renames)

		def final(init_val): # rename tempfiles to final positions
			for tempfile, destination in zip(tempfiles, final_renames):
				if destination:
					if destination.isdir(): # Cannot rename over directory
						destination.delete()
					tempfile.rename(destination)
			return init_val
		def error(exc, ran_init, init_val):
			for tf in tempfiles: tf.delete()
		return RobustAction(init_thunk, final, error)

	def copy_action(rorpin, rpout):
		"""Return robust action copying rorpin to rpout

		The source can be a rorp or an rpath.  Does not recurse.  If
		directories copied, then just exit (output directory not
		overwritten).
		
		"""
		tfl = [None] # Need some mutable state to hold tf value
		def init(): 
			if not (rorpin.isdir() and rpout.isdir()): # already a dir
				tfl[0] = tf = TempFileManager.new(rpout)
				if rorpin.isreg(): tf.write_from_fileobj(rorpin.open("rb"))
				else: RPath.copy(rorpin, tf)
				return tf
			else: return None
		def final(tf):
			if tf and tf.lstat():
				if rpout.isdir(): rpout.delete()
				tf.rename(rpout)
			return rpout
		def error(exc, ran_init, init_val):
			if tfl[0]: tfl[0].delete()
		return RobustAction(init, final, error)

	def copy_with_attribs_action(rorpin, rpout, compress = None):
		"""Like copy_action but also copy attributes"""
		tfl = [None] # Need some mutable state for error handler
		def init(): 
			if not (rorpin.isdir() and rpout.isdir()): # already a dir
				tfl[0] = tf = TempFileManager.new(rpout)
				if rorpin.isreg():
					tf.write_from_fileobj(rorpin.open("rb"), compress)
				else: RPath.copy(rorpin, tf)
				if tf.lstat(): # Some files, like sockets, won't be created
					RPathStatic.copy_attribs(rorpin, tf)
				return tf
			else: return None
		def final(tf):
			if rorpin.isdir() and rpout.isdir():
				RPath.copy_attribs(rorpin, rpout)
			elif tf and tf.lstat():
				if rpout.isdir(): rpout.delete() # can't rename over dir
				tf.rename(rpout)
			return rpout
		def error(exc, ran_init, init_val):
			if tfl[0]: tfl[0].delete()
		return RobustAction(init, final, error)

	def copy_attribs_action(rorpin, rpout):
		"""Return action which just copies attributes

		Copying attributes is already pretty atomic, so just run
		normal sequence.

		"""
		def final(init_val):
			RPath.copy_attribs(rorpin, rpout)
			return rpout
		return RobustAction(None, final, None)

	def symlink_action(rpath, linktext):
		"""Return symlink action by moving one file over another"""
		tf = TempFileManager.new(rpath)
		def init(): tf.symlink(linktext)
		return Robust.make_tf_robustaction(init, tf, rpath)

	def destructive_write_action(rp, s):
		"""Return action writing string s to rpath rp in robust way

		This will overwrite any data currently in rp.

		"""
		tf = TempFileManager.new(rp)
		def init():
			fp = tf.open("wb")
			fp.write(s)
			fp.close()
			tf.setdata()
		return Robust.make_tf_robustaction(init, tf, rp)

	def check_common_error(error_handler, function, *args):
		"""Apply function to args, if error, run error_handler on exception

		This only catches certain exceptions which seems innocent
		enough.

		"""
		try: return function(*args)
 		except (EnvironmentError, SkipFileException, DSRPPermError,
				RPathException, RdiffException), exc:
			TracebackArchive.add()
			if (not isinstance(exc, EnvironmentError) or
				(errno.errorcode[exc[0]] in
				 ['EPERM', 'ENOENT', 'EACCES', 'EBUSY', 'EEXIST',
				  'ENOTDIR', 'ENAMETOOLONG', 'EINTR', 'ENOTEMPTY',
				  'EIO', 'ETXTBSY', 'ESRCH', 'EINVAL'])):
				Log.exception()
				conn = Globals.backup_writer
				if conn is not None: # increment error count
					ITR_exists = conn.Globals.is_not_None('ITR')
					if ITR_exists: conn.Globals.ITR.increment_stat('Errors')
				if error_handler: return error_handler(exc, *args)
			else:
				Log.exception(1, 2)
				raise

	def listrp(rp):
		"""Like rp.listdir() but return [] if error, and sort results"""
		def error_handler(exc):
			Log("Error listing directory %s" % rp.path, 2)
			return []
		dir_listing = Robust.check_common_error(error_handler, rp.listdir)
		dir_listing.sort()
		return dir_listing

	def signal_handler(signum, frame):
		"""This is called when signal signum is caught"""
		raise SignalException(signum)

	def install_signal_handlers():
		"""Install signal handlers on current connection"""
		for signum in [signal.SIGQUIT, signal.SIGHUP, signal.SIGTERM]:
			signal.signal(signum, Robust.signal_handler)

MakeStatic(Robust)


class SignalException(Exception):
	"""SignalException(signum) means signal signum has been received"""
	pass


class TracebackArchive:
	"""Save last 10 caught exceptions, so they can be printed if fatal"""
	_traceback_strings = []
	def add(cls):
		"""Add most recent exception to archived list"""
		cls._traceback_strings.append(Log.exception_to_string())
		if len(cls._traceback_strings) > 10:
			cls._traceback_strings = cls._traceback_strings[:10]

	def log(cls):
		"""Print all exception information to log file"""
		if cls._traceback_strings:
			Log("------------ Old traceback info -----------\n%s"
				"-------------------------------------------" %
				("\n".join(cls._traceback_strings),), 3)

MakeClass(TracebackArchive)


class TempFileManager:
	"""Manage temp files"""

	# This is a connection-specific list of temp files, to be cleaned
	# up before rdiff-backup exits.
	_tempfiles = []

	# To make collisions less likely, this gets put in the file name
	# and incremented whenever a new file is requested.
	_tfindex = 0

	def new(cls, rp_base, same_dir = 1):
		"""Return new tempfile that isn't in use.

		If same_dir, tempfile will be in same directory as rp_base.
		Otherwise, use tempfile module to get filename.

		"""
		conn = rp_base.conn
		if conn is not Globals.local_connection:
			return conn.TempFileManager.new(rp_base, same_dir)

		def find_unused(conn, dir):
			"""Find an unused tempfile with connection conn in directory dir"""
			while 1:
				if cls._tfindex > 100000000:
					Log("Resetting index", 2)
					cls._tfindex = 0
				tf = TempFile(conn, os.path.join(dir,
								       "rdiff-backup.tmp.%d" % cls._tfindex))
				cls._tfindex = cls._tfindex+1
				if not tf.lstat(): return tf

		if same_dir: tf = find_unused(conn, rp_base.dirsplit()[0])
		else: tf = TempFile(conn, tempfile.mktemp())
		cls._tempfiles.append(tf)
		return tf

	def remove_listing(cls, tempfile):
		"""Remove listing of tempfile"""
		if Globals.local_connection is not tempfile.conn:
			tempfile.conn.TempFileManager.remove_listing(tempfile)
		elif tempfile in cls._tempfiles: cls._tempfiles.remove(tempfile)

	def delete_all(cls):
		"""Delete all remaining tempfiles"""
		for tf in cls._tempfiles[:]: tf.delete()

MakeClass(TempFileManager)


class TempFile(RPath):
	"""Like an RPath, but keep track of which ones are still here"""
	def rename(self, rp_dest):
		"""Rename temp file to permanent location, possibly overwriting"""
		if self.isdir() and not rp_dest.isdir():
			# Cannot move a directory directly over another file
			rp_dest.delete()
			if (isinstance(rp_dest, DSRPath) and rp_dest.delay_perms
				and not self.hasfullperms()):
				# If we are moving to a delayed perm directory, delay
				# permission change on destination.
				rp_dest.chmod(self.getperms())
				self.chmod(0700)
		RPathStatic.rename(self, rp_dest)

		# Sometimes this just seems to fail silently, as in one
		# hardlinked twin is moved over the other.  So check to make
		# sure below.
		self.setdata()
		if self.lstat():
			rp_dest.delete()
			RPathStatic.rename(self, rp_dest)
			self.setdata()
			if self.lstat(): raise OSError("Cannot rename tmp file correctly")
		TempFileManager.remove_listing(self)

	def delete(self):
		RPath.delete(self)
		TempFileManager.remove_listing(self)


class SaveState:
	"""Save state in the middle of backups for resuming later"""
	_last_file_sym = None # RPath of sym pointing to last file processed
	_last_file_definitive_rp = None # Touch this if last file is really last
	_last_checkpoint_time = 0 # time in seconds of last checkpoint
	_checkpoint_rp = None # RPath of checkpoint data pickle
	
	def init_filenames(cls):
		"""Set rpaths of markers.  Assume rbdir already set."""
		if not Globals.isbackup_writer:
			return Globals.backup_writer.SaveState.init_filenames()

		assert Globals.local_connection is Globals.rbdir.conn, \
			   (Globals.rbdir.conn, Globals.backup_writer)

		cls._last_file_sym = Globals.rbdir.append(
			"last-file-incremented.%s.data" % Time.curtimestr)
		cls._checkpoint_rp = Globals.rbdir.append(
			"checkpoint-data.%s.data" % Time.curtimestr)
		cls._last_file_definitive_rp = Globals.rbdir.append(
			"last-file-definitive.%s.data" % Time.curtimestr)

	def touch_last_file(cls):
		"""Touch last file marker, indicating backup has begun"""
		if not cls._last_file_sym.lstat(): cls._last_file_sym.touch()

	def touch_last_file_definitive(cls):
		"""Create last-file-definitive marker

		When a backup gets aborted, there may be time to indicate the
		last file successfully processed, and this should be touched.
		Sometimes when the abort is hard, there may be a last file
		indicated, but further files since then have been processed,
		in which case this shouldn't be touched.

		"""
		cls._last_file_definitive_rp.touch()

	def record_last_file_action(cls, last_file_rorp):
		"""Action recording last file to be processed as symlink in rbdir

		last_file_rorp is None means that no file is known to have
		been processed.

		"""
		if last_file_rorp:
			symtext = apply(os.path.join,
							('increments',) + last_file_rorp.index)
			return Robust.symlink_action(cls._last_file_sym, symtext)
		else: return RobustAction(None, lambda init_val: cls.touch_last_file(),
								  None)

	def checkpoint(cls, ITR, finalizer, last_file_rorp, override = None):
		"""Save states of tree reducer and finalizer during inc backup

		If override is true, checkpoint even if one isn't due.

		"""
		if not override and not cls.checkpoint_needed(): return
		assert cls._checkpoint_rp, "_checkpoint_rp not set yet"

		cls._last_checkpoint_time = time.time()
		Log("Writing checkpoint time %s" % cls._last_checkpoint_time, 7)
		state_string = cPickle.dumps((ITR, finalizer))
		Robust.chain(Robust.destructive_write_action(cls._checkpoint_rp,
													 state_string),
					 cls.record_last_file_action(last_file_rorp)).execute()

	def checkpoint_needed(cls):
		"""Returns true if another checkpoint is called for"""
		return (time.time() > cls._last_checkpoint_time +
				Globals.checkpoint_interval)

	def checkpoint_remove(cls):
		"""Remove all checkpointing data after successful operation"""
		for rp in Resume.get_relevant_rps(): rp.delete()
		if Globals.preserve_hardlinks: Hardlink.remove_all_checkpoints()

MakeClass(SaveState)


class Resume:
	"""Check for old aborted backups and resume if necessary"""
	_session_info_list = None # List of ResumeSessionInfo's, sorted by time
	def FindTime(cls, index, later_than = 0):
		"""For a given index, find the appropriate time to use for inc

		If it is clear which time to use (because it is determined by
		definitive records, or there are no aborted backup, etc.) then
		just return the appropriate time.  Otherwise, if an aborted
		backup was last checkpointed before the index, assume that it
		didn't get there, and go for the older time.  If an inc file
		is already present, the function will be rerun with later time
		specified.

		"""
		assert Globals.isbackup_writer
		if Time.prevtime > later_than: return Time.prevtime # usual case

		for si in cls.get_sis_covering_index(index):
			if si.time > later_than: return si.time
		raise SkipFileException("Index %s already covered, skipping" %
								str(index))

	def get_sis_covering_index(cls, index):
		"""Return sorted list of SessionInfos which may cover index

		Aborted backup may be relevant unless index is lower and we
		are sure that it didn't go further.

		"""
		return filter(lambda session_info:
					  not ((session_info.last_index is None or
							session_info.last_index < index) and
						   session_info.last_definitive), 
				cls._session_info_list)

	def SetSessionInfo(cls):
		"""Read data directory and initialize _session_info"""
		assert Globals.isbackup_writer
		silist = []
		rp_quad_dict = cls.group_rps_by_time(cls.get_relevant_rps())
		times = rp_quad_dict.keys()
		times.sort()
		for time in times:
			silist.append(cls.quad_to_si(time, rp_quad_dict[time]))
		cls._session_info_list = silist

	def get_relevant_rps(cls):
		"""Return list of relevant rpaths in rbdata directory"""
		relevant_bases = ['last-file-incremented', 'last-file-mirrored',
						  'checkpoint-data', 'last-file-definitive']
		rps = map(Globals.rbdir.append, Globals.rbdir.listdir())
		return filter(lambda rp: rp.isincfile()
					  and rp.getincbase_str() in relevant_bases, rps)

	def group_rps_by_time(cls, rplist):
		"""Take list of rps return time dict {time: quadlist}

		Times in seconds are the keys, values are triples of rps
		[last-file-incremented, last-file-mirrored, checkpoint-data,
		 last-is-definitive].

		"""
		result = {}
		for rp in rplist:
			time = Time.stringtotime(rp.getinctime())
			if result.has_key(time): quadlist = result[time]
			else: quadlist = [None, None, None, None]
			base_string = rp.getincbase_str()
			if base_string == 'last-file-incremented': quadlist[0] = rp
			elif base_string == 'last-file-mirrored': quadlist[1] = rp
			elif base_string == 'last-file-definitive': quadlist[3] = 1
			else:
				assert base_string == 'checkpoint-data'
				quadlist[2] = rp
			result[time] = quadlist
		return result

	def quad_to_si(cls, time, quad):
		"""Take time, quadlist, return associated ResumeSessionInfo"""
		increment_sym, mirror_sym, checkpoint_rp, last_definitive = quad
		assert not (increment_sym and mirror_sym) # both shouldn't exist
		ITR, finalizer = None, None
		if increment_sym:
			mirror = None
			last_index = cls.sym_to_index(increment_sym)
			if checkpoint_rp:
				ITR, finalizer = cls.unpickle_checkpoint(checkpoint_rp)
		elif mirror_sym:
			mirror = 1
			last_index = cls.sym_to_index(mirror_sym)
			if checkpoint_rp:
				finalizer = cls.unpickle_checkpoint(checkpoint_rp)
		return ResumeSessionInfo(mirror, time, last_index, last_definitive,
								 finalizer, ITR)

	def sym_to_index(cls, sym_rp):
		"""Read last file sym rp, return last file index

		If sym_rp is not a sym at all, return None, indicating that no
		file index was ever conclusively processed.

		"""
		if not sym_rp.issym(): return None
		link_components = sym_rp.readlink().split("/")
		assert link_components[0] == 'increments'
		return tuple(link_components[1:])

	def unpickle_checkpoint(cls, checkpoint_rp):
		"""Read data from checkpoint_rp and return unpickled data

		Return value is pair (patch increment ITR, finalizer state).

		"""
		fp = checkpoint_rp.open("rb")
		data = fp.read()
		fp.close()
		return cPickle.loads(data)

	def ResumeCheck(cls):
		"""Return relevant ResumeSessionInfo if there's one we should resume

		Also if find RSI to resume, reset current time to old resume
		time.

		"""
		cls.SetSessionInfo()
		if not cls._session_info_list:
			if Globals.resume == 1: 
				Log.FatalError("User specified resume, but no data on "
							   "previous backup found.")
			else: return None
		else:
			si = cls._session_info_list[-1]
			if (Globals.resume == 1 or
				(time.time() <= (si.time + Globals.resume_window) and
				 not Globals.resume == 0)):
				Log("Resuming aborted backup dated %s" %
					Time.timetopretty(si.time), 2)
				Time.setcurtime(si.time)
				if Globals.preserve_hardlinks:
					if (not si.last_definitive or not
						Hardlink.retrieve_checkpoint(Globals.rbdir, si.time)):
						Log("Hardlink information not successfully "
							"recovered.", 2)
				return si
			else:
				Log("Last backup dated %s was aborted, but we aren't "
					"resuming it." % Time.timetopretty(si.time), 2)
				return None
		assert None

MakeClass(Resume)


class ResumeSessionInfo:
	"""Hold information about a previously aborted session"""
	def __init__(self, mirror, time, last_index,
				 last_definitive, finalizer = None, ITR = None):
		"""Class initializer

		time - starting time in seconds of backup
		mirror - true if backup was a mirror, false if increment
		last_index - Last confirmed index processed by backup, or None
		last_definitive - True is we know last_index is really last
		finalizer - the dsrp finalizer if available
		ITR - For increment, ITM reducer (assume mirror if NA)

		"""
		self.time = time
		self.mirror = mirror
		self.last_index = last_index
		self.last_definitive = last_definitive
		self.ITR, self.finalizer, = ITR, finalizer


#######################################################################
#
# rorpiter - Operations on Iterators of Read Only Remote Paths
#

class RORPIterException(Exception): pass

class RORPIter:
	"""Functions relating to iterators of Read Only RPaths

	The main structure will be an iterator that yields RORPaths.
	Every RORPath has a "raw" form that makes it more amenable to
	being turned into a file.  The raw form of the iterator yields
	each RORPath in the form of the tuple (index, data_dictionary,
	files), where files is the number of files attached (usually 1 or
	0).  After that, if a file is attached, it yields that file.

	"""
	def ToRaw(rorp_iter):
		"""Convert a rorp iterator to raw form"""
		for rorp in rorp_iter:
			if rorp.file:
				yield (rorp.index, rorp.data, 1)
				yield rorp.file
			else: yield (rorp.index, rorp.data, 0)

	def FromRaw(raw_iter):
		"""Convert raw rorp iter back to standard form"""
		for index, data, num_files in raw_iter:
			rorp = RORPath(index, data)
			if num_files:
				assert num_files == 1, "Only one file accepted right now"
				rorp.setfile(RORPIter.getnext(raw_iter))
			yield rorp

	def ToFile(rorp_iter):
		"""Return file version of iterator"""
		return FileWrappingIter(RORPIter.ToRaw(rorp_iter))

	def FromFile(fileobj):
		"""Recover rorp iterator from file interface"""
		return RORPIter.FromRaw(IterWrappingFile(fileobj))

	def IterateRPaths(base_rp):
		"""Return an iterator yielding RPaths with given base rp"""
		yield base_rp
		if base_rp.isdir():
			dirlisting = base_rp.listdir()
			dirlisting.sort()
			for filename in dirlisting:
				for rp in RORPIter.IterateRPaths(base_rp.append(filename)):
					yield rp

	def Signatures(rp_iter):
		"""Yield signatures of rpaths in given rp_iter"""
		def error_handler(exc, rp):
			Log("Error generating signature for %s" % rp.path)
			return None

		for rp in rp_iter:
			if rp.isplaceholder(): yield rp
			else:
				rorp = rp.getRORPath()
				if rp.isreg():
					if rp.isflaglinked(): rorp.flaglinked()
					else:
						fp = Robust.check_common_error(
							error_handler, Rdiff.get_signature, rp)
						if fp: rorp.setfile(fp)
						else: continue
				yield rorp

	def GetSignatureIter(base_rp):
		"""Return a signature iterator recurring over the base_rp"""
		return RORPIter.Signatures(RORPIter.IterateRPaths(base_rp))

	def CollateIterators(*rorp_iters):
		"""Collate RORPath iterators by index

		So it takes two or more iterators of rorps and returns an
		iterator yielding tuples like (rorp1, rorp2) with the same
		index.  If one or the other lacks that index, it will be None

		"""
		# overflow[i] means that iter[i] has been exhausted
		# rorps[i] is None means that it is time to replenish it.
		iter_num = len(rorp_iters)
		if iter_num == 2:
			return RORPIter.Collate2Iters(rorp_iters[0], rorp_iters[1])
		overflow = [None] * iter_num
		rorps = overflow[:]

		def setrorps(overflow, rorps):
			"""Set the overflow and rorps list"""
			for i in range(iter_num):
				if not overflow[i] and rorps[i] is None:
					try: rorps[i] = rorp_iters[i].next()
					except StopIteration:
						overflow[i] = 1
						rorps[i] = None

		def getleastindex(rorps):
			"""Return the first index in rorps, assuming rorps isn't empty"""
			return min(map(lambda rorp: rorp.index,
						   filter(lambda x: x, rorps)))

		def yield_tuples(iter_num, overflow, rorps):
			while 1:
				setrorps(overflow, rorps)
				if not None in overflow: break

				index = getleastindex(rorps)
				yieldval = []
				for i in range(iter_num):
					if rorps[i] and rorps[i].index == index:
						yieldval.append(rorps[i])
						rorps[i] = None
					else: yieldval.append(None)
				yield IndexedTuple(index, yieldval)
		return yield_tuples(iter_num, overflow, rorps)

	def Collate2Iters(riter1, riter2):
		"""Special case of CollateIterators with 2 arguments

		This does the same thing but is faster because it doesn't have
		to consider the >2 iterator case.  Profiler says speed is
		important here.
		
		"""
		relem1, relem2 = None, None
		while 1:
			if not relem1:
				try: relem1 = riter1.next()
				except StopIteration:
					if relem2: yield IndexedTuple(index2, (None, relem2))
					for relem2 in riter2:
						yield IndexedTuple(relem2.index, (None, relem2))
					break
				index1 = relem1.index
			if not relem2:
				try: relem2 = riter2.next()
				except StopIteration:
					if relem1: yield IndexedTuple(index1, (relem1, None))
					for relem1 in riter1:
						yield IndexedTuple(relem1.index, (relem1, None))
					break
				index2 = relem2.index

			if index1 < index2:
				yield IndexedTuple(index1, (relem1, None))
				relem1 = None
			elif index1 == index2:
				yield IndexedTuple(index1, (relem1, relem2))
				relem1, relem2 = None, None
			else: # index2 is less
				yield IndexedTuple(index2, (None, relem2))
				relem2 = None

	def getnext(iter):
		"""Return the next element of an iterator, raising error if none"""
		try: next = iter.next()
		except StopIteration: raise RORPIterException("Unexpected end to iter")
		return next

	def GetDiffIter(sig_iter, new_iter):
		"""Return delta iterator from sig_iter to new_iter

		The accompanying file for each will be a delta as produced by
		rdiff, unless the destination file does not exist, in which
		case it will be the file in its entirety.

		sig_iter may be composed of rorps, but new_iter should have
		full RPaths.

		"""
		collated_iter = RORPIter.CollateIterators(sig_iter, new_iter)
		for rorp, rp in collated_iter: yield RORPIter.diffonce(rorp, rp)

	def diffonce(sig_rorp, new_rp):
		"""Return one diff rorp, based from signature rorp and orig rp"""
		if sig_rorp and Globals.preserve_hardlinks and sig_rorp.isflaglinked():
			if new_rp: diff_rorp = new_rp.getRORPath()
			else: diff_rorp = RORPath(sig_rorp.index)
			diff_rorp.flaglinked()
			return diff_rorp
		elif sig_rorp and sig_rorp.isreg() and new_rp and new_rp.isreg():
			diff_rorp = new_rp.getRORPath()
			diff_rorp.setfile(Rdiff.get_delta_sigfileobj(sig_rorp.open("rb"),
														 new_rp))
			diff_rorp.set_attached_filetype('diff')
			return diff_rorp
		else:
			# Just send over originial if diff isn't appropriate
			if sig_rorp: sig_rorp.close_if_necessary()
			if not new_rp: return RORPath(sig_rorp.index)
			elif new_rp.isreg():
				diff_rorp = new_rp.getRORPath(1)
				diff_rorp.set_attached_filetype('snapshot')
				return diff_rorp
			else: return new_rp.getRORPath()

	def PatchIter(base_rp, diff_iter):
		"""Patch the appropriate rps in basis_iter using diff_iter"""
		basis_iter = RORPIter.IterateRPaths(base_rp)
		collated_iter = RORPIter.CollateIterators(basis_iter, diff_iter)
		for basisrp, diff_rorp in collated_iter:
			RORPIter.patchonce_action(base_rp, basisrp, diff_rorp).execute()

	def patchonce_action(base_rp, basisrp, diff_rorp):
		"""Return action patching basisrp using diff_rorp"""
		assert diff_rorp, "Missing diff index %s" % basisrp.index
		if not diff_rorp.lstat():
			return RobustAction(None, lambda init_val: basisrp.delete(), None)

		if Globals.preserve_hardlinks and diff_rorp.isflaglinked():
			if not basisrp: basisrp = base_rp.new_index(diff_rorp.index)
			tf = TempFileManager.new(basisrp)
			def init(): Hardlink.link_rp(diff_rorp, tf, basisrp)
			return Robust.make_tf_robustaction(init, tf, basisrp)
		elif basisrp and basisrp.isreg() and diff_rorp.isreg():
			assert diff_rorp.get_attached_filetype() == 'diff'
			return Rdiff.patch_with_attribs_action(basisrp, diff_rorp)
		else: # Diff contains whole file, just copy it over
			if not basisrp: basisrp = base_rp.new_index(diff_rorp.index)
			return Robust.copy_with_attribs_action(diff_rorp, basisrp)

MakeStatic(RORPIter)



class IndexedTuple(UserList.UserList):
	"""Like a tuple, but has .index

	This is used by CollateIterator above, and can be passed to the
	IterTreeReducer.

	"""
	def __init__(self, index, sequence):
		self.index = index
		self.data = tuple(sequence)

	def __len__(self): return len(self.data)

	def __getitem__(self, key):
		"""This only works for numerical keys (easier this way)"""
		return self.data[key]

	def __lt__(self, other): return self.__cmp__(other) == -1
	def __le__(self, other): return self.__cmp__(other) != 1
	def __ne__(self, other): return not self.__eq__(other)
	def __gt__(self, other): return self.__cmp__(other) == 1
	def __ge__(self, other): return self.__cmp__(other) != -1
	
	def __cmp__(self, other):
		assert isinstance(other, IndexedTuple)
		if self.index < other.index: return -1
		elif self.index == other.index: return 0
		else: return 1

	def __eq__(self, other):
		if isinstance(other, IndexedTuple):
			return self.index == other.index and self.data == other.data
		elif type(other) is types.TupleType:
			return self.data == other
		else: return None

	def __str__(self):
		return  "(%s).%s" % (", ".join(map(str, self.data)), self.index)


#######################################################################
#
# destructive-stepping - Deal with side effects from traversing trees
#

class DSRPPermError(Exception):
	"""Exception used when a DSRPath can't get sufficient permissions"""
	pass

class DSRPath(RPath):
	"""Destructive Stepping RPath

	Sometimes when we traverse the directory tree, even when we just
	want to read files, we have to change things, like the permissions
	of a file or directory in order to read it, or the file's access
	times.  This class is like an RPath, but the permission and time
	modifications are delayed, so that they can be done at the very
	end when they won't be disturbed later.

	Here are the new class variables:
	delay_perms - true iff future perm changes should be delayed
	newperms - holds the perm values while they are delayed
	delay_atime - true iff some atime change are being delayed
	newatime - holds the new atime
	delay_mtime - true if some mtime change is being delayed
	newmtime - holds the new mtime

	"""
	def __init__(self, source, *args):
		"""Initialize DSRP

		Source should be true iff the DSRPath is taken from the
		"source" partition and thus settings like
		Globals.change_source_perms should be paid attention to.

		If args is [rpath], return the dsrpath equivalent of rpath,
		otherwise use the same arguments as the RPath initializer.

		"""
		if len(args) == 1 and isinstance(args[0], RPath):
			rp = args[0]
			RPath.__init__(self, rp.conn, rp.base, rp.index)
		else: RPath.__init__(self, *args)

		if source != "bypass":
			# "bypass" val is used when unpackaging over connection
			assert source is None or source is 1
			self.source = source
			self.set_delays(source)
			self.set_init_perms(source)

	def set_delays(self, source):
		"""Delay writing permissions and times where appropriate"""
		if not source or Globals.change_source_perms:
			self.delay_perms, self.newperms = 1, None
		else: self.delay_perms = None

		if Globals.preserve_atime:
			self.delay_atime = 1
			# Now get atime right away if possible
			if self.data.has_key('atime'): self.newatime = self.data['atime']
			else: self.newatime = None
		else: self.delay_atime = None
		
		if source:
			self.delay_mtime = None # we'll never change mtime of source file
		else:
			self.delay_mtime = 1
			# Save mtime now for a dir, because it might inadvertantly change
			if self.isdir(): self.newmtime = self.data['mtime']
			else: self.newmtime = None

	def set_init_perms(self, source):
		"""If necessary, change permissions to ensure access"""
		if self.isreg() and not self.readable():
			if (source and Globals.change_source_perms or
				not source and Globals.change_mirror_perms):
				self.chmod_bypass(0400)
		elif self.isdir():
			if source and Globals.change_source_perms:
				if not self.readable() or not self.executable():
					self.chmod_bypass(0500)
			elif not source and Globals.change_mirror_perms:
				if not self.hasfullperms(): self.chmod_bypass(0700)

	def warn(self, err):
		Log("Received error '%s' when dealing with file %s, skipping..."
			% (err, self.path), 1)
		raise DSRPPermError(self.path)

	def __getstate__(self):
		"""Return picklable state.  See RPath __getstate__."""
		assert self.conn is Globals.local_connection # Can't pickle a conn
		return self.getstatedict()

	def getstatedict(self):
		"""Return dictionary containing the attributes we can save"""
		pickle_dict = {}
		for attrib in ['index', 'data', 'delay_perms', 'newperms',
					   'delay_atime', 'newatime',
					   'delay_mtime', 'newmtime',
					   'path', 'base', 'source']:
			if self.__dict__.has_key(attrib):
				pickle_dict[attrib] = self.__dict__[attrib]
		return pickle_dict

	def __setstate__(self, pickle_dict):
		"""Set state from object produced by getstate"""
		self.conn = Globals.local_connection
		for attrib in pickle_dict.keys():
			self.__dict__[attrib] = pickle_dict[attrib]

	def chmod(self, permissions):
		"""Change permissions, delaying if self.perms_delayed is set"""
		if self.delay_perms: self.newperms = self.data['perms'] = permissions
		else: RPath.chmod(self, permissions)

	def getperms(self):
		"""Return dsrp's intended permissions"""
		if self.delay_perms and self.newperms is not None:
			return self.newperms
		else: return self.data['perms']

	def chmod_bypass(self, permissions):
		"""Change permissions without updating the data dictionary"""
		self.delay_perms = 1
		if self.newperms is None: self.newperms = self.getperms()
		Log("DSRP: Perm bypass %s to %o" % (self.path, permissions), 8)
		self.conn.os.chmod(self.path, permissions)

	def settime(self, accesstime, modtime):
		"""Change times, delaying if self.times_delayed is set"""
		if self.delay_atime: self.newatime = self.data['atime'] = accesstime
		if self.delay_mtime: self.newmtime = self.data['mtime'] = modtime

		if not self.delay_atime or not self.delay_mtime:
			RPath.settime(self, accesstime, modtime)
		
	def setmtime(self, modtime):
		"""Change mtime, delaying if self.times_delayed is set"""
		if self.delay_mtime: self.newmtime = self.data['mtime'] = modtime
		else: RPath.setmtime(self, modtime)

	def getmtime(self):
		"""Return dsrp's intended modification time"""
		if self.delay_mtime and self.newmtime is not None:
			return self.newmtime
		else: return self.data['mtime']

	def getatime(self):
		"""Return dsrp's intended access time"""
		if self.delay_atime and self.newatime is not None:
			return self.newatime
		else: return self.data['atime']

	def write_changes(self):
		"""Write saved up permission/time changes"""
		if not self.lstat(): return # File has been deleted in meantime

		if self.delay_perms and self.newperms is not None:
			Log("Finalizing permissions of dsrp %s to %s" %
				(self.path, self.newperms), 8)
			RPath.chmod(self, self.newperms)

		do_atime = self.delay_atime and self.newatime is not None
		do_mtime = self.delay_mtime and self.newmtime is not None
		if do_atime and do_mtime:
			RPath.settime(self, self.newatime, self.newmtime)
		elif do_atime and not do_mtime:
			RPath.settime(self, self.newatime, self.getmtime())
		elif not do_atime and do_mtime:
			RPath.setmtime(self, self.newmtime)

	def newpath(self, newpath, index = ()):
		"""Return similar DSRPath but with new path"""
		return self.__class__(self.source, self.conn, newpath, index)

	def append(self, ext):
		"""Return similar DSRPath with new extension"""
		return self.__class__(self.source, self.conn, self.base,
							  self.index + (ext,))

	def new_index(self, index):
		"""Return similar DSRPath with new index"""
		return self.__class__(self.source, self.conn, self.base, index)


class DestructiveSteppingFinalizer(ErrorITR):
		"""Finalizer that can work on an iterator of dsrpaths

		The reason we have to use an IterTreeReducer is that some files
		should be updated immediately, but for directories we sometimes
		need to update all the files in the directory before finally
		coming back to it.

		"""
		dsrpath = None
		def start_process(self, index, dsrpath):
			self.dsrpath = dsrpath

		def end_process(self):
			if self.dsrpath: self.dsrpath.write_changes()


#######################################################################
#
# selection - Provides the iterator-like DSRPIterator class
#
# Parses includes and excludes to yield correct files.  More
# documentation on what this code does can be found on the man page.
#

class SelectError(Exception):
	"""Some error dealing with the Select class"""
	pass

class FilePrefixError(SelectError):
	"""Signals that a specified file doesn't start with correct prefix"""
	pass

class GlobbingError(SelectError):
	"""Something has gone wrong when parsing a glob string"""
	pass


class Select:
	"""Iterate appropriate DSRPaths in given directory

	This class acts as an iterator on account of its next() method.
	Basically, it just goes through all the files in a directory in
	order (depth-first) and subjects each file to a bunch of tests
	(selection functions) in order.  The first test that includes or
	excludes the file means that the file gets included (iterated) or
	excluded.  The default is include, so with no tests we would just
	iterate all the files in the directory in order.

	The one complication to this is that sometimes we don't know
	whether or not to include a directory until we examine its
	contents.  For instance, if we want to include all the **.py
	files.  If /home/ben/foo.py exists, we should also include /home
	and /home/ben, but if these directories contain no **.py files,
	they shouldn't be included.  For this reason, a test may not
	include or exclude a directory, but merely "scan" it.  If later a
	file in the directory gets included, so does the directory.

	As mentioned above, each test takes the form of a selection
	function.  The selection function takes a dsrp, and returns:

	None - means the test has nothing to say about the related file
	0 - the file is excluded by the test
	1 - the file is included
	2 - the test says the file (must be directory) should be scanned

	Also, a selection function f has a variable f.exclude which should
	be true iff f could potentially exclude some file.  This is used
	to signal an error if the last function only includes, which would
	be redundant and presumably isn't what the user intends.

	"""
	# This re should not match normal filenames, but usually just globs
	glob_re = re.compile("(.*[*?[]|ignorecase\\:)", re.I | re.S)

	def __init__(self, dsrpath, quoted_filenames = None):
		"""DSRPIterator initializer.  dsrp is the root directory

		When files have quoted characters in them, quoted_filenames
		should be true.  Then RPath's index will be the unquoted
		version.

		"""
		assert isinstance(dsrpath, DSRPath)
		self.selection_functions = []
		self.dsrpath = dsrpath
		self.prefix = self.dsrpath.path
		self.quoting_on = Globals.quoting_enabled and quoted_filenames

	def set_iter(self, starting_index = None, iterate_parents = None,
				 sel_func = None):
		"""Initialize more variables, get ready to iterate

		Will iterate indicies greater than starting_index.  If
		iterate_parents is true, will also include parents of
		starting_index in iteration.  Selection function sel_func is
		called on each dsrp and is usually self.Select.  Returns self
		just for convenience.

		"""
		if not sel_func: sel_func = self.Select
		self.dsrpath.setdata() # this may have changed since Select init
		if starting_index is not None:
			self.starting_index = starting_index
			self.iter = self.iterate_starting_from(self.dsrpath,
						            self.iterate_starting_from, sel_func)
		else: self.iter = self.Iterate(self.dsrpath, self.Iterate, sel_func)

		# only iterate parents if we are not starting from beginning
		self.iterate_parents = starting_index is not None and iterate_parents
		self.next = self.iter.next
		self.__iter__ = lambda: self
		return self

	def Iterate(self, dsrpath, rec_func, sel_func):
		"""Return iterator yielding dsrps in dsrpath

		rec_func is usually the same as this function and is what
		Iterate uses to find files in subdirectories.  It is used in
		iterate_starting_from.

		sel_func is the selection function to use on the dsrps.  It is
		usually self.Select.

		"""
		s = sel_func(dsrpath)
		if s == 0: return
		elif s == 1: # File is included
			yield dsrpath
			if dsrpath.isdir():
				for dsrp in self.iterate_in_dir(dsrpath, rec_func, sel_func):
					yield dsrp
		elif s == 2 and dsrpath.isdir(): # Directory is merely scanned
			iid = self.iterate_in_dir(dsrpath, rec_func, sel_func)
			try: first = iid.next()
			except StopIteration: return # no files inside; skip dsrp
			yield dsrpath
			yield first
			for dsrp in iid: yield dsrp

	def iterate_in_dir(self, dsrpath, rec_func, sel_func):
		"""Iterate the dsrps in directory dsrpath."""
		def error_handler(exc, filename):
			Log("Error initializing file %s/%s" % (dsrpath.path, filename), 2)
			return None

		if self.quoting_on:
			for subdir in FilenameMapping.get_quoted_dir_children(dsrpath):
				for dsrp in rec_func(subdir, rec_func, sel_func): yield dsrp
		else:
			for filename in Robust.listrp(dsrpath):
				new_dsrp = Robust.check_common_error(
					error_handler, dsrpath.append, filename)
				if new_dsrp:
					for dsrp in rec_func(new_dsrp, rec_func, sel_func):
						yield dsrp

	def iterate_starting_from(self, dsrpath, rec_func, sel_func):
		"""Like Iterate, but only yield indicies > self.starting_index"""
		if dsrpath.index > self.starting_index: # past starting_index
			for dsrp in self.Iterate(dsrpath, self.Iterate, sel_func):
				yield dsrp
		elif (dsrpath.index == self.starting_index[:len(dsrpath.index)]
			  and dsrpath.isdir()):
			# May encounter starting index on this branch
			if self.iterate_parents: yield dsrpath
			for dsrp in self.iterate_in_dir(dsrpath,
											self.iterate_starting_from,
											sel_func): yield dsrp

	def iterate_with_finalizer(self):
		"""Like Iterate, but missing some options, and add finalizer"""
		finalize = DestructiveSteppingFinalizer()
		for dsrp in self:
			yield dsrp
			finalize(dsrp.index, dsrp)
		finalize.Finish()

	def Select(self, dsrp):
		"""Run through the selection functions and return dominant val 0/1/2"""
		for sf in self.selection_functions:
			result = sf(dsrp)
			if result is not None: return result
		return 1

	def ParseArgs(self, argtuples, filelists):
		"""Create selection functions based on list of tuples

		The tuples have the form (option string, additional argument)
		and are created when the initial commandline arguments are
		read.  The reason for the extra level of processing is that
		the filelists may only be openable by the main connection, but
		the selection functions need to be on the backup reader or
		writer side.  When the initial arguments are parsed the right
		information is sent over the link.

		"""
		filelists_index = 0
		try:
			for opt, arg in argtuples:
				if opt == "--exclude":
					self.add_selection_func(self.glob_get_sf(arg, 0))
				elif opt == "--exclude-device-files":
					self.add_selection_func(self.devfiles_get_sf())
				elif opt == "--exclude-filelist":
					self.add_selection_func(self.filelist_get_sf(
						filelists[filelists_index], 0, arg))
					filelists_index += 1
				elif opt == "--exclude-regexp":
					self.add_selection_func(self.regexp_get_sf(arg, 0))
				elif opt == "--include":
					self.add_selection_func(self.glob_get_sf(arg, 1))
				elif opt == "--include-filelist":
					self.add_selection_func(self.filelist_get_sf(
						filelists[filelists_index], 1, arg))
					filelists_index += 1
				elif opt == "--include-regexp":
					self.add_selection_func(self.regexp_get_sf(arg, 1))
				else: assert 0, "Bad selection option %s" % opt
		except SelectError, e: self.parse_catch_error(e)
		assert filelists_index == len(filelists)

		self.parse_last_excludes()
		self.parse_rbdir_exclude()

	def parse_catch_error(self, exc):
		"""Deal with selection error exc"""
		if isinstance(exc, FilePrefixError):
			Log.FatalError(
"""Fatal Error: The file specification
    %s
cannot match any files in the base directory
    %s
Useful file specifications begin with the base directory or some
pattern (such as '**') which matches the base directory.""" %
			(exc, self.prefix))
		elif isinstance(e, GlobbingError):
			Log.FatalError("Fatal Error while processing expression\n"
						   "%s" % exc)
		else: raise

	def parse_rbdir_exclude(self):
		"""Add exclusion of rdiff-backup-data dir to front of list"""
		self.add_selection_func(
			self.glob_get_tuple_sf(("rdiff-backup-data",), 0), 1)

	def parse_last_excludes(self):
		"""Exit with error if last selection function isn't an exclude"""
		if (self.selection_functions and
			not self.selection_functions[-1].exclude):
			Log.FatalError(
"""Last selection expression:
    %s
only specifies that files be included.  Because the default is to
include all files, the expression is redundant.  Exiting because this
probably isn't what you meant.""" %
			(self.selection_functions[-1].name,))

	def add_selection_func(self, sel_func, add_to_start = None):
		"""Add another selection function at the end or beginning"""
		if add_to_start: self.selection_functions.insert(0, sel_func)
		else: self.selection_functions.append(sel_func)

	def filelist_get_sf(self, filelist_fp, inc_default, filelist_name):
		"""Return selection function by reading list of files

		The format of the filelist is documented in the man page.
		filelist_fp should be an (open) file object.
		inc_default should be true if this is an include list,
		false for an exclude list.
		filelist_name is just a string used for logging.

		"""
		Log("Reading filelist %s" % filelist_name, 4)
		tuple_list, something_excluded = \
					self.filelist_read(filelist_fp, inc_default, filelist_name)
		Log("Sorting filelist %s" % filelist_name, 4)
		tuple_list.sort()
		i = [0] # We have to put index in list because of stupid scoping rules

		def selection_function(dsrp):
			while 1:
				if i[0] >= len(tuple_list): return None
				include, move_on = \
						 self.filelist_pair_match(dsrp, tuple_list[i[0]])
				if move_on:
					i[0] += 1
					if include is None: continue # later line may match
				return include

		selection_function.exclude = something_excluded or inc_default == 0
		selection_function.name = "Filelist: " + filelist_name
		return selection_function

	def filelist_read(self, filelist_fp, include, filelist_name):
		"""Read filelist from fp, return (tuplelist, something_excluded)"""
		prefix_warnings = [0]
		def incr_warnings(exc):
			"""Warn if prefix is incorrect"""
			prefix_warnings[0] += 1
			if prefix_warnings[0] < 6:
				Log("Warning: file specification '%s' in filelist %s\n"
					"doesn't start with correct prefix %s.  Ignoring." %
					(exc, filelist_name, self.prefix), 2)
				if prefix_warnings[0] == 5:
					Log("Future prefix errors will not be logged.", 2)

		something_excluded, tuple_list = None, []
		separator = Globals.null_separator and "\0" or "\n"
		for line in filelist_fp.read().split(separator):
			if not line: continue # skip blanks
			try: tuple = self.filelist_parse_line(line, include)
			except FilePrefixError, exc:
				incr_warnings(exc)
				continue
			tuple_list.append(tuple)
			if not tuple[1]: something_excluded = 1
		if filelist_fp.close():
			Log("Error closing filelist %s" % filelist_name, 2)
		return (tuple_list, something_excluded)

	def filelist_parse_line(self, line, include):
		"""Parse a single line of a filelist, returning a pair

		pair will be of form (index, include), where index is another
		tuple, and include is 1 if the line specifies that we are
		including a file.  The default is given as an argument.
		prefix is the string that the index is relative to.

		"""
		line = line.strip()
		if line[:2] == "+ ": # Check for "+ "/"- " syntax
			include = 1
			line = line[2:]
		elif line[:2] == "- ":
			include = 0
			line = line[2:]

		if not line.startswith(self.prefix): raise FilePrefixError(line)
		line = line[len(self.prefix):] # Discard prefix
		index = tuple(filter(lambda x: x, line.split("/"))) # remove empties
		return (index, include)

	def filelist_pair_match(self, dsrp, pair):
		"""Matches a filelist tuple against a dsrp

		Returns a pair (include, move_on).  include is None if the
		tuple doesn't match either way, and 0/1 if the tuple excludes
		or includes the dsrp.

		move_on is true if the tuple cannot match a later index, and
		so we should move on to the next tuple in the index.

		"""
		index, include = pair
		if include == 1:
			if index < dsrp.index: return (None, 1)
			if index == dsrp.index: return (1, 1)
			elif index[:len(dsrp.index)] == dsrp.index:
				return (1, None) # /foo/bar implicitly includes /foo
			else: return (None, None) # dsrp greater, not initial sequence
		elif include == 0:
			if dsrp.index[:len(index)] == index:
				return (0, None) # /foo implicitly excludes /foo/bar
			elif index < dsrp.index: return (None, 1)
			else: return (None, None) # dsrp greater, not initial sequence
		else: assert 0, "Include is %s, should be 0 or 1" % (include,)

	def regexp_get_sf(self, regexp_string, include):
		"""Return selection function given by regexp_string"""
		assert include == 0 or include == 1
		try: regexp = re.compile(regexp_string)
		except:
			Log("Error compiling regular expression %s" % regexp_string, 1)
			raise
		
		def sel_func(dsrp):
			if regexp.search(dsrp.path): return include
			else: return None

		sel_func.exclude = not include
		sel_func.name = "Regular expression: %s" % regexp_string
		return sel_func

	def devfiles_get_sf(self):
		"""Return a selection function to exclude all dev files"""
		if self.selection_functions:
			Log("Warning: exclude-device-files is not the first "
				"selector.\nThis may not be what you intended", 3)
		def sel_func(dsrp):
			if dsrp.isdev(): return 0
			else: return None
		sel_func.exclude = 1
		sel_func.name = "Exclude device files"
		return sel_func

	def glob_get_sf(self, glob_str, include):
		"""Return selection function given by glob string"""
		assert include == 0 or include == 1
		if glob_str == "**": sel_func = lambda dsrp: include
		elif not self.glob_re.match(glob_str): # normal file
			sel_func = self.glob_get_filename_sf(glob_str, include)
		else: sel_func = self.glob_get_normal_sf(glob_str, include)

		sel_func.exclude = not include
		sel_func.name = "Command-line %s glob: %s" % \
						(include and "include" or "exclude", glob_str)
		return sel_func

	def glob_get_filename_sf(self, filename, include):
		"""Get a selection function given a normal filename

		Some of the parsing is better explained in
		filelist_parse_line.  The reason this is split from normal
		globbing is things are a lot less complicated if no special
		globbing characters are used.

		"""
		if not filename.startswith(self.prefix):
			raise FilePrefixError(filename)
		index = tuple(filter(lambda x: x,
							 filename[len(self.prefix):].split("/")))
		return self.glob_get_tuple_sf(index, include)

	def glob_get_tuple_sf(self, tuple, include):
		"""Return selection function based on tuple"""
		def include_sel_func(dsrp):
			if (dsrp.index == tuple[:len(dsrp.index)] or
				dsrp.index[:len(tuple)] == tuple):
				return 1 # /foo/bar implicitly matches /foo, vice-versa
			else: return None

		def exclude_sel_func(dsrp):
			if dsrp.index[:len(tuple)] == tuple:
				return 0 # /foo excludes /foo/bar, not vice-versa
			else: return None

		if include == 1: sel_func = include_sel_func
		elif include == 0: sel_func = exclude_sel_func
		sel_func.exclude = not include
		sel_func.name = "Tuple select %s" % (tuple,)
		return sel_func

	def glob_get_normal_sf(self, glob_str, include):
		"""Return selection function based on glob_str

		The basic idea is to turn glob_str into a regular expression,
		and just use the normal regular expression.  There is a
		complication because the selection function should return '2'
		(scan) for directories which may contain a file which matches
		the glob_str.  So we break up the glob string into parts, and
		any file which matches an initial sequence of glob parts gets
		scanned.

		Thanks to Donovan Baarda who provided some code which did some
		things similar to this.

		"""
		if glob_str.lower().startswith("ignorecase:"):
			re_comp = lambda r: re.compile(r, re.I | re.S)
			glob_str = glob_str[len("ignorecase:"):]
		else: re_comp = lambda r: re.compile(r, re.S)

		# matches what glob matches and any files in directory
		glob_comp_re = re_comp("^%s($|/)" % self.glob_to_re(glob_str))

		if glob_str.find("**") != -1:
			glob_str = glob_str[:glob_str.find("**")+2] # truncate after **

		scan_comp_re = re_comp("^(%s)$" %
							   "|".join(self.glob_get_prefix_res(glob_str)))

		def include_sel_func(dsrp):
			if glob_comp_re.match(dsrp.path): return 1
			elif scan_comp_re.match(dsrp.path): return 2
			else: return None

		def exclude_sel_func(dsrp):
			if glob_comp_re.match(dsrp.path): return 0
			else: return None

		# Check to make sure prefix is ok
		if not include_sel_func(self.dsrpath): raise FilePrefixError(glob_str)
		
		if include: return include_sel_func
		else: return exclude_sel_func

	def glob_get_prefix_res(self, glob_str):
		"""Return list of regexps equivalent to prefixes of glob_str"""
		glob_parts = glob_str.split("/")
		if "" in glob_parts[1:-1]: # "" OK if comes first or last, as in /foo/
			raise GlobbingError("Consecutive '/'s found in globbing string "
								+ glob_str)

		prefixes = map(lambda i: "/".join(glob_parts[:i+1]),
					   range(len(glob_parts)))
		# we must make exception for root "/", only dir to end in slash
		if prefixes[0] == "": prefixes[0] = "/"
		return map(self.glob_to_re, prefixes)

	def glob_to_re(self, pat):
		"""Returned regular expression equivalent to shell glob pat

		Currently only the ?, *, [], and ** expressions are supported.
		Ranges like [a-z] are also currently unsupported.  There is no
		way to quote these special characters.

		This function taken with minor modifications from efnmatch.py
		by Donovan Baarda.

		"""
		i, n, res = 0, len(pat), ''
		while i < n:
			c, s = pat[i], pat[i:i+2]
			i = i+1
			if s == '**':
				res = res + '.*'
				i = i + 1
			elif c == '*': res = res + '[^/]*'
			elif c == '?': res = res + '[^/]'
			elif c == '[':
				j = i
				if j < n and pat[j] in '!^': j = j+1
				if j < n and pat[j] == ']': j = j+1
				while j < n and pat[j] != ']': j = j+1
				if j >= n: res = res + '\\[' # interpret the [ literally
				else: # Deal with inside of [..]
					stuff = pat[i:j].replace('\\','\\\\')
					i = j+1
					if stuff[0] in '!^': stuff = '^' + stuff[1:]
					res = res + '[' + stuff + ']'
			else: res = res + re.escape(c)
		return res


#######################################################################
#
# filename_mapping - used to coordinate related filenames
#
# For instance, some source filenames may contain characters not
# allowed on the mirror end.  Also, if a source filename is very long
# (say 240 characters), the extra characters added to related
# increments may put them over the usual 255 character limit.
#

class FilenameMapping:
	"""Contains class methods which coordinate related filenames"""
	max_filename_length = 255

	# If true, enable character quoting, and set characters making
	# regex-style range.
	chars_to_quote = None

	# These compiled regular expressions are used in quoting and unquoting
	chars_to_quote_regexp = None
	unquoting_regexp = None

	# Use given char to quote.  Default is set in Globals.
	quoting_char = None

	def set_init_quote_vals(cls):
		"""Set quoting value from Globals on all conns"""
		for conn in Globals.connections:
			conn.FilenameMapping.set_init_quote_vals_local()
		
	def set_init_quote_vals_local(cls):
		"""Set value on local connection, initialize regexps"""
		cls.chars_to_quote = Globals.chars_to_quote
		if len(Globals.quoting_char) != 1:
			Log.FatalError("Expected single character for quoting char,"
						   "got '%s' instead" % (Globals.quoting_char,))
		cls.quoting_char = Globals.quoting_char
		cls.init_quoting_regexps()

	def init_quoting_regexps(cls):
		"""Compile quoting regular expressions"""
		try:
			cls.chars_to_quote_regexp = \
			   re.compile("[%s%s]" % (cls.chars_to_quote,
									  cls.quoting_char), re.S)
			cls.unquoting_regexp = \
			   re.compile("%s[0-9]{3}" % cls.quoting_char, re.S)
		except re.error:
			Log.FatalError("Error '%s' when processing char quote list %s" %
						   (re.error, cls.chars_to_quote))

	def quote(cls, path):
		"""Return quoted version of given path

		Any characters quoted will be replaced by the quoting char and
		the ascii number of the character.  For instance, "10:11:12"
		would go to "10;05811;05812" if ":" were quoted and ";" were
		the quoting character.

		"""
		return cls.chars_to_quote_regexp.sub(cls.quote_single, path)

	def quote_single(cls, match):
		"""Return replacement for a single character"""
		return "%s%03d" % (cls.quoting_char, ord(match.group()))

	def unquote(cls, path):
		"""Return original version of quoted filename"""
		return cls.unquoting_regexp.sub(cls.unquote_single, path)

	def unquote_single(cls, match):
		"""Unquote a single quoted character"""
		assert len(match.group()) == 4
		return chr(int(match.group()[1:]))

	def get_quoted_dir_children(cls, rpath):
		"""For rpath directory, return list of quoted children in dir"""
		if not rpath.isdir(): return []
		dir_pairs = [(cls.unquote(filename), filename)
					 for filename in Robust.listrp(rpath)]
		dir_pairs.sort() # sort by real index, not quoted part
		child_list = []
		for unquoted, filename in dir_pairs:
			childrp = rpath.append(unquoted)
			childrp.quote_path()
			child_list.append(childrp)
		return child_list

MakeClass(FilenameMapping)


#######################################################################
#
# statistics - Generate and process aggregated backup information
#

class StatsException(Exception): pass

class StatsObj:
	"""Contains various statistics, provide string conversion functions"""

	stat_file_attrs = ('SourceFiles', 'SourceFileSize',
					   'MirrorFiles', 'MirrorFileSize',
					   'NewFiles', 'NewFileSize',
					   'DeletedFiles', 'DeletedFileSize',
					   'ChangedFiles',
					   'ChangedSourceSize', 'ChangedMirrorSize',
					   'IncrementFiles', 'IncrementFileSize')
	stat_misc_attrs = ('Errors',)
	stat_time_attrs = ('StartTime', 'EndTime', 'ElapsedTime')
	stat_attrs = (('Filename',) + stat_time_attrs +
				  stat_misc_attrs + stat_file_attrs)

	# Below, the second value in each pair is true iff the value
	# indicates a number of bytes
	stat_file_pairs = (('SourceFiles', None), ('SourceFileSize', 1),
					   ('MirrorFiles', None), ('MirrorFileSize', 1),
					   ('NewFiles', None), ('NewFileSize', 1),
					   ('DeletedFiles', None), ('DeletedFileSize', 1),
					   ('ChangedFiles', None),
					   ('ChangedSourceSize', 1), ('ChangedMirrorSize', 1),
					   ('IncrementFiles', None), ('IncrementFileSize', 1))

	# Set all stats to None, indicating info not available
	for attr in stat_attrs: locals()[attr] = None

	# This is used in get_byte_summary_string below
	byte_abbrev_list = ((1024*1024*1024*1024, "TB"),
						(1024*1024*1024, "GB"),
						(1024*1024, "MB"),
						(1024, "KB"))

	def get_stat(self, attribute):
		"""Get a statistic"""
		try: return self.__dict__[attribute]
		except KeyError:
			# this may be a hack, but seems no good way to get attrs in python
			return eval("self.%s" % attribute)

	def set_stat(self, attr, value):
		"""Set attribute to given value"""
		self.__dict__[attr] = value

	def increment_stat(self, attr):
		"""Add 1 to value of attribute"""
		self.__dict__[attr] = self.get_stat(attr) + 1

	def get_stats_line(self, index, use_repr = 1):
		"""Return one line abbreviated version of full stats string"""
		file_attrs = map(lambda attr: str(self.get_stat(attr)),
						 self.stat_file_attrs)
		if not index: filename = "."
		else:
			filename = apply(os.path.join, index)
			if use_repr:
				# use repr to quote newlines in relative filename, then
				# take of leading and trailing quote.
				filename = repr(filename)[1:-1]
		return " ".join([filename,] + file_attrs)

	def set_stats_from_line(self, line):
		"""Set statistics from given line"""
		def error(): raise StatsException("Bad line '%s'" % line)
		if line[-1] == "\n": line = line[:-1]
		lineparts = line.split(" ")
		if len(lineparts) < len(stat_file_attrs): error()
		for attr, val_string in zip(stat_file_attrs,
									lineparts[-len(stat_file_attrs):]):
			try: val = long(val_string)
			except ValueError:
				try: val = float(val_string)
				except ValueError: error()
			self.set_stat(attr, val)
		return self

	def get_stats_string(self):
		"""Return extended string printing out statistics"""
		return self.get_timestats_string() + self.get_filestats_string()

	def get_timestats_string(self):
		"""Return portion of statistics string dealing with time"""
		timelist = []
		if self.StartTime is not None:
			timelist.append("StartTime %.2f (%s)\n" %
						(self.StartTime, Time.timetopretty(self.StartTime)))
		if self.EndTime is not None:
			timelist.append("EndTime %.2f (%s)\n" %
							(self.EndTime, Time.timetopretty(self.EndTime)))
		if self.ElapsedTime or (self.StartTime is not None and
								self.EndTime is not None):
			if self.ElapsedTime is None:
				self.ElapsedTime = self.EndTime - self.StartTime
			timelist.append("ElapsedTime %.2f (%s)\n" %
				   (self.ElapsedTime, Time.inttopretty(self.ElapsedTime)))
		if self.Errors is not None:
			timelist.append("Errors %d\n" % self.Errors)
		return "".join(timelist)

	def get_filestats_string(self):
		"""Return portion of statistics string about files and bytes"""
		def fileline(stat_file_pair):
			"""Return zero or one line of the string"""
			attr, in_bytes = stat_file_pair
			val = self.get_stat(attr)
			if val is None: return ""
			if in_bytes:
				return "%s %s (%s)\n" % (attr, val,
										 self.get_byte_summary_string(val))
			else: return "%s %s\n" % (attr, val)

		return "".join(map(fileline, self.stat_file_pairs))

	def get_byte_summary_string(self, byte_count):
		"""Turn byte count into human readable string like "7.23GB" """
		for abbrev_bytes, abbrev_string in self.byte_abbrev_list:
			if byte_count >= abbrev_bytes:
				# Now get 3 significant figures
				abbrev_count = float(byte_count)/abbrev_bytes
				if abbrev_count >= 100: precision = 0
				elif abbrev_count >= 10: precision = 1
				else: precision = 2
				return "%%.%df %s" % (precision, abbrev_string) \
					   % (abbrev_count,)
		byte_count = round(byte_count)
		if byte_count == 1: return "1 byte"
		else: return "%d bytes" % (byte_count,)

	def get_stats_logstring(self, title):
		"""Like get_stats_string, but add header and footer"""
		header = "--------------[ %s ]--------------" % title
		footer = "-" * len(header)
		return "%s\n%s%s\n" % (header, self.get_stats_string(), footer)

	def set_stats_from_string(self, s):
		"""Initialize attributes from string, return self for convenience"""
		def error(line): raise StatsException("Bad line '%s'" % line)

		for line in s.split("\n"):
			if not line: continue
			line_parts = line.split()
			if len(line_parts) < 2: error(line)
			attr, value_string = line_parts[:2]
			if not attr in self.stat_attrs: error(line)
			try:
				try: val1 = long(value_string)
				except ValueError: val1 = None
				val2 = float(value_string)
				if val1 == val2: self.set_stat(attr, val1) # use integer val
				else: self.set_stat(attr, val2) # use float
			except ValueError: error(line)
		return self

	def write_stats_to_rp(self, rp):
		"""Write statistics string to given rpath"""
		tf = TempFileManager.new(rp)
		def init_thunk():
			fp = tf.open("w")
			fp.write(self.get_stats_string())
			fp.close()
		Robust.make_tf_robustaction(init_thunk, (tf,), (rp,)).execute()

	def read_stats_from_rp(self, rp):
		"""Set statistics from rpath, return self for convenience"""
		fp = rp.open("r")
		self.set_stats_from_string(fp.read())
		fp.close()
		return self

	def stats_equal(self, s):
		"""Return true if s has same statistics as self"""
		assert isinstance(s, StatsObj)
		for attr in self.stat_file_attrs:
			if self.get_stat(attr) != s.get_stat(attr): return None
		return 1

	def set_to_average(self, statobj_list):
		"""Set self's attributes to average of those in statobj_list"""
		for attr in self.stat_attrs: self.set_stat(attr, 0)
		for statobj in statobj_list:
			for attr in self.stat_attrs:
				if statobj.get_stat(attr) is None: self.set_stat(attr, None)
				elif self.get_stat(attr) is not None:
					self.set_stat(attr, statobj.get_stat(attr) +
								  self.get_stat(attr))

		# Don't compute average starting/stopping time		
		self.StartTime = None
		self.EndTime = None

		for attr in self.stat_attrs:
			if self.get_stat(attr) is not None:
				self.set_stat(attr,
							  self.get_stat(attr)/float(len(statobj_list)))
		return self

	def get_statsobj_copy(self):
		"""Return new StatsObj object with same stats as self"""
		s = StatObj()
		for attr in self.stat_attrs: s.set_stat(attr, self.get_stat(attr))
		return s


class StatsITR(IterTreeReducer, StatsObj):
	"""Keep track of per directory statistics

	This is subclassed by the mirroring and incrementing ITRs.

	"""
	# zero out file statistics
	for attr in StatsObj.stat_file_attrs: locals()[attr] = 0

	def start_stats(self, mirror_dsrp):
		"""Record status of mirror dsrp

		This is called before the mirror is processed so we remember
		the old state.

		"""
		if mirror_dsrp.lstat():
			self.mirror_base_exists = 1
			self.mirror_base_size = self.stats_getsize(mirror_dsrp)
		else: self.mirror_base_exists = None

	def stats_getsize(self, rp):
		"""Return size of rp, with error checking"""
		try: return rp.getsize()
		except KeyError: return 0

	def end_stats(self, diff_rorp, mirror_dsrp, inc_rp = None):
		"""Set various statistics after mirror processed"""
		if mirror_dsrp.lstat():
			source_size = self.stats_getsize(mirror_dsrp)
			self.SourceFiles += 1
			self.SourceFileSize += source_size
			if self.mirror_base_exists:
				self.MirrorFiles += 1
				self.MirrorFileSize += self.mirror_base_size
				if diff_rorp: # otherwise no change
					self.ChangedFiles += 1
					self.ChangedSourceSize += source_size
					self.ChangedMirrorSize += self.mirror_base_size
					self.stats_incr_incfiles(inc_rp)
			else: # new file was created
				self.NewFiles += 1
				self.NewFileSize += source_size
				self.stats_incr_incfiles(inc_rp)
		else:
			if self.mirror_base_exists: # file was deleted from mirror
				self.MirrorFiles += 1
				self.MirrorFileSize += self.mirror_base_size
				self.DeletedFiles += 1
				self.DeletedFileSize += self.mirror_base_size
				self.stats_incr_incfiles(inc_rp)

	def stats_incr_incfiles(self, inc_rp):
		"""Increment IncrementFile statistics"""
		if inc_rp:
			self.IncrementFiles += 1
			self.IncrementFileSize += self.stats_getsize(inc_rp)

	def add_file_stats(self, subinstance):
		"""Add all file statistics from subinstance to current totals"""
		for attr in self.stat_file_attrs:
			self.set_stat(attr,
						  self.get_stat(attr) + subinstance.get_stat(attr))


class Stats:
	"""Misc statistics methods, pertaining to dir and session stat files"""
	# This is the RPath of the directory statistics file, and the
	# associated open file.  It will hold a line of statistics for
	# each directory that is backed up.
	_dir_stats_rp = None
	_dir_stats_fp = None

	# This goes at the beginning of the directory statistics file and
	# explains the format.
	_dir_stats_header = """# rdiff-backup directory statistics file
#
# Each line is in the following format:
# RelativeDirName %s
""" % " ".join(StatsObj.stat_file_attrs)

	def open_dir_stats_file(cls):
		"""Open directory statistics file, write header"""
		assert not cls._dir_stats_fp, "Directory file already open"

		if Globals.compression: suffix = "data.gz"
		else: suffix = "data"
		cls._dir_stats_rp = Inc.get_inc(Globals.rbdir.append(
			"directory_statistics"), Time.curtime, suffix)

		if cls._dir_stats_rp.lstat():
			Log("Warning, statistics file %s already exists, appending" %
				cls._dir_stats_rp.path, 2)
			cls._dir_stats_fp = cls._dir_stats_rp.open("ab",
													   Globals.compression)
		else: cls._dir_stats_fp = \
			  cls._dir_stats_rp.open("wb", Globals.compression)
		cls._dir_stats_fp.write(cls._dir_stats_header)

	def write_dir_stats_line(cls, statobj, index):
		"""Write info from statobj about rpath to statistics file"""
		if Globals.null_separator:
			cls._dir_stats_fp.write(statobj.get_stats_line(index, None) + "\0")
		else: cls._dir_stats_fp.write(statobj.get_stats_line(index) + "\n")

	def close_dir_stats_file(cls):
		"""Close directory statistics file if its open"""
		if cls._dir_stats_fp:
			cls._dir_stats_fp.close()
			cls._dir_stats_fp = None

	def write_session_statistics(cls, statobj):
		"""Write session statistics into file, log"""
		stat_inc = Inc.get_inc(Globals.rbdir.append("session_statistics"),
							   Time.curtime, "data")
		statobj.StartTime = Time.curtime
		statobj.EndTime = time.time()

		# include hardlink data and dir stats in size of increments
		if Globals.preserve_hardlinks and Hardlink.final_inc:
			# include hardlink data in size of increments
			statobj.IncrementFiles += 1
			statobj.IncrementFileSize += Hardlink.final_inc.getsize()
		if cls._dir_stats_rp and cls._dir_stats_rp.lstat():
			statobj.IncrementFiles += 1
			statobj.IncrementFileSize += cls._dir_stats_rp.getsize()

		statobj.write_stats_to_rp(stat_inc)
		if Globals.print_statistics:
			message = statobj.get_stats_logstring("Session statistics")
			Log.log_to_file(message)
			Globals.client_conn.sys.stdout.write(message)

MakeClass(Stats)


#######################################################################
#
# increment - Provides Inc class, which writes increment files
#
# This code is what writes files ending in .diff, .snapshot, etc.
#

class Inc:
	"""Class containing increment functions"""
	def Increment_action(new, mirror, incpref):
		"""Main file incrementing function, returns RobustAction

		new is the file on the active partition,
		mirror is the mirrored file from the last backup,
		incpref is the prefix of the increment file.

		This function basically moves the information about the mirror
		file to incpref.

		The returned RobustAction when executed should return the name
		of the incfile, or None if none was created.

		"""
		if not (new and new.lstat() or mirror.lstat()):
			return Robust.null_action # Files deleted in meantime, do nothing

		Log("Incrementing mirror file " + mirror.path, 5)
		if ((new and new.isdir()) or mirror.isdir()) and not incpref.isdir():
			incpref.mkdir()

		if not mirror.lstat(): return Inc.makemissing_action(incpref)
		elif mirror.isdir(): return Inc.makedir_action(mirror, incpref)
		elif new.isreg() and mirror.isreg():
			return Inc.makediff_action(new, mirror, incpref)
		else: return Inc.makesnapshot_action(mirror, incpref)

	def Increment(new, mirror, incpref):
		return Inc.Increment_action(new, mirror, incpref).execute()

	def makemissing_action(incpref):
		"""Signify that mirror file was missing"""
		def final(init_val):
			incrp = Inc.get_inc_ext(incpref, "missing")
			incrp.touch()
			return incrp
		return RobustAction(None, final, None)
		
	def makesnapshot_action(mirror, incpref):
		"""Copy mirror to incfile, since new is quite different"""
		if (mirror.isreg() and Globals.compression and
			not Globals.no_compression_regexp.match(mirror.path)):
			snapshotrp = Inc.get_inc_ext(incpref, "snapshot.gz")
			return Robust.copy_with_attribs_action(mirror, snapshotrp, 1)
		else:
			snapshotrp = Inc.get_inc_ext(incpref, "snapshot")
			return Robust.copy_with_attribs_action(mirror, snapshotrp, None)

	def makediff_action(new, mirror, incpref):
		"""Make incfile which is a diff new -> mirror"""
		if (Globals.compression and
			not Globals.no_compression_regexp.match(mirror.path)):
			diff = Inc.get_inc_ext(incpref, "diff.gz")
			compress = 1
		else: 
			diff = Inc.get_inc_ext(incpref, "diff")
			compress = None
			
		diff_tf = TempFileManager.new(diff)
		sig_tf = TempFileManager.new(mirror, None)
		def init():
			Rdiff.write_delta(new, mirror, diff_tf, compress, sig_tf)
			RPath.copy_attribs(mirror, diff_tf)
			return diff
		return Robust.make_tf_robustaction(init, (diff_tf, sig_tf),
										   (diff, None))

	def makedir_action(mirrordir, incpref):
		"""Make file indicating directory mirrordir has changed"""
		dirsign = Inc.get_inc_ext(incpref, "dir")
		tf = TempFileManager.new(dirsign)
		def init():
			tf.touch()
			RPath.copy_attribs(mirrordir, tf)
			return dirsign
		return Robust.make_tf_robustaction(init, tf, dirsign)

	def get_inc(rp, time, typestr):
		"""Return increment like rp but with time and typestr suffixes"""
		addtostr = lambda s: "%s.%s.%s" % (s, Time.timetostring(time), typestr)
		if rp.index:
			incrp = rp.__class__(rp.conn, rp.base, rp.index[:-1] +
								 (addtostr(rp.index[-1]),))
		else: incrp = rp.__class__(rp.conn, addtostr(rp.base), rp.index)
		if Globals.quoting_enabled: incrp.quote_path()
		return incrp

	def get_inc_ext(rp, typestr):
		"""Return increment with specified type and correct time

		If the file exists, then probably a previous backup has been
		aborted.  We then keep asking FindTime to get a time later
		than the one that already has an inc file.

		"""
		inctime = 0
		while 1:
			inctime = Resume.FindTime(rp.index, inctime)
			incrp = Inc.get_inc(rp, inctime, typestr)
			if not incrp.lstat(): break
		return incrp

MakeStatic(Inc)


class IncrementITR(ErrorITR, StatsITR):
	"""Patch and increment mirror directory

	This has to be an ITR because directories that have files in them
	changed are flagged with an increment marker.  There are four
	possibilities as to the order:

	1.  Normal file -> Normal file:  right away
	2.  Directory -> Directory:  wait until files in the directory
	    are processed, as we won't know whether to add a marker
		until the end.
	3.  Normal file -> Directory:  right away, so later files will
	    have a directory to go into.
	4.  Directory -> Normal file:  Wait until the end, so we can
	    process all the files in the directory.	

	Remember this object needs to be pickable.
	
	"""
	# Iff true, mirror file was a directory
	mirror_isdirectory = None
	# If set, what the directory on the mirror side will be replaced with
	directory_replacement = None
	# True iff there has been some change at this level or lower (used
	# for marking directories to be flagged)
	changed = None
	# Holds the RPath of the created increment file, if any
	incrp = None

	def __init__(self, inc_rpath):
		"""Set inc_rpath, an rpath of the base of the tree"""
		self.inc_rpath = inc_rpath
		StatsITR.__init__(self, inc_rpath)

	def start_process(self, index, diff_rorp, dsrp):
		"""Initial processing of file

		diff_rorp is the RORPath of the diff from the remote side, and
		dsrp is the local file to be incremented

		"""
		self.start_stats(dsrp)
		incpref = self.inc_rpath.new_index(index)
		if Globals.quoting_enabled: incpref.quote_path()
		if dsrp.isdir():
			self.init_dir(dsrp, diff_rorp, incpref)
			self.mirror_isdirectory = 1
		else: self.init_non_dir(dsrp, diff_rorp, incpref)
		self.setvals(diff_rorp, dsrp, incpref)
		
	def override_changed(self):
		"""Set changed flag to true

		This is used only at the top level of a backup, to make sure
		that a marker is created recording every backup session.

		"""
		self.changed = 1

	def setvals(self, diff_rorp, dsrp, incpref):
		"""Record given values in state dict since in directory

		We don't do these earlier in case of a problem inside the
		init_* functions.  Index isn't given because it is done by the
		superclass.

		"""
		self.diff_rorp = diff_rorp
		self.dsrp = dsrp
		self.incpref = incpref

	def init_dir(self, dsrp, diff_rorp, incpref):
		"""Process a directory (initial pass)

		If the directory is changing into a normal file, we need to
		save the normal file data in a temp file, and then create the
		real file once we are done with everything inside the
		directory.

		"""
		if not (incpref.lstat() and incpref.isdir()): incpref.mkdir()
		if diff_rorp and diff_rorp.isreg() and diff_rorp.file:
			tf = TempFileManager.new(dsrp)
			def init():
				RPathStatic.copy_with_attribs(diff_rorp, tf)
				tf.set_attached_filetype(diff_rorp.get_attached_filetype())
			def error(exc, ran_init, init_val): tf.delete()
			RobustAction(init, None, error).execute()
			self.directory_replacement = tf

	def init_non_dir(self, dsrp, diff_rorp, incpref):
		"""Process a non directory file (initial pass)"""
		if not diff_rorp: return # no diff, so no change necessary
		if diff_rorp.isreg() and (dsrp.isreg() or diff_rorp.isflaglinked()):
			# Write updated mirror to temp file so we can compute
			# reverse diff locally
			mirror_tf = TempFileManager.new(dsrp)
			old_dsrp_tf = TempFileManager.new(dsrp)
			def init_thunk():
				if diff_rorp.isflaglinked():
					Hardlink.link_rp(diff_rorp, mirror_tf, dsrp)
				else: Rdiff.patch_with_attribs_action(dsrp, diff_rorp,
													  mirror_tf).execute()
				self.incrp = Inc.Increment_action(mirror_tf, dsrp,
												  incpref).execute()
				if dsrp.lstat(): RPathStatic.rename(dsrp, old_dsrp_tf)
				mirror_tf.rename(dsrp)

			def final(init_val): old_dsrp_tf.delete()
			def error(exc, ran_init, init_val):
				if ran_init: old_dsrp_tf.delete() # everything is fine
				else: # restore to previous state
					if old_dsrp_tf.lstat(): old_dsrp_tf.rename(dsrp)
					if self.incrp: self.incrp.delete()
					mirror_tf.delete()

			RobustAction(init_thunk, final, error).execute()
		else: self.incrp = Robust.chain(
			Inc.Increment_action(diff_rorp, dsrp, incpref),
			RORPIter.patchonce_action(None, dsrp, diff_rorp)).execute()[0]

		self.changed = 1

	def end_process(self):
		"""Do final work when leaving a tree (directory)"""
		diff_rorp, dsrp, incpref = self.diff_rorp, self.dsrp, self.incpref
		if (self.mirror_isdirectory and (diff_rorp or self.changed)
			or self.directory_replacement):
			if self.directory_replacement:
				tf = self.directory_replacement
				self.incrp = Robust.chain(
					Inc.Increment_action(tf, dsrp, incpref),
					RORPIter.patchonce_action(None, dsrp, tf)).execute()[0]
				tf.delete()
			else:
				self.incrp = Inc.Increment(diff_rorp, dsrp, incpref)
				if diff_rorp:
					RORPIter.patchonce_action(None, dsrp, diff_rorp).execute()

		self.end_stats(diff_rorp, dsrp, self.incrp)
		if self.mirror_isdirectory or dsrp.isdir():
			Stats.write_dir_stats_line(self, dsrp.index)

	def branch_process(self, subinstance):
		"""Update statistics, and the has_changed flag if change in branch"""
		if subinstance.changed: self.changed = 1	
		self.add_file_stats(subinstance)


class MirrorITR(ErrorITR, StatsITR):
	"""Like IncrementITR, but only patch mirror directory, don't increment"""
	# This is always None since no increments will be created
	incrp = None
	def __init__(self, inc_rpath):
		"""Set inc_rpath, an rpath of the base of the inc tree"""
		self.inc_rpath = inc_rpath
		StatsITR.__init__(self, inc_rpath)

	def start_process(self, index, diff_rorp, mirror_dsrp):
		"""Initialize statistics, do actual writing to mirror"""
		self.start_stats(mirror_dsrp)
		if diff_rorp and not diff_rorp.isplaceholder():
			RORPIter.patchonce_action(None, mirror_dsrp, diff_rorp).execute()

		self.incpref = self.inc_rpath.new_index(index)
		self.diff_rorp, self.mirror_dsrp = diff_rorp, mirror_dsrp

	def end_process(self):
		"""Update statistics when leaving"""
		self.end_stats(self.diff_rorp, self.mirror_dsrp)
		if self.mirror_dsrp.isdir():
			Stats.write_dir_stats_line(self, self.mirror_dsrp.index)

	def branch_process(self, subinstance):
		"""Update statistics with subdirectory results"""
		self.add_file_stats(subinstance)


#######################################################################
#
# restore - Read increment files and restore to original
#

class RestoreError(Exception): pass

class Restore:
	def Restore(inc_rpath, mirror, target, rest_time):
		"""Recursively restore inc_rpath and mirror to target at rest_time

		Like restore_recusive below, but with a more friendly
		interface (it converts to DSRPaths if necessary, finds the inc
		files with the appropriate base, and makes rid).

		rest_time is the time in seconds to restore to;

		inc_rpath should not be the name of an increment file, but the
		increment file shorn of its suffixes and thus should have the
		same index as mirror.

		"""
		if not isinstance(mirror, DSRPath): mirror = DSRPath(1, mirror)
		if not isinstance(target, DSRPath): target = DSRPath(None, target)

		mirror_time = Restore.get_mirror_time()
		rest_time = Restore.get_rest_time(rest_time, mirror_time)
		inc_list = Restore.get_inclist(inc_rpath)
		rid = RestoreIncrementData(inc_rpath.index, inc_rpath, inc_list)
		rid.sortincseq(rest_time, mirror_time)
		Restore.check_hardlinks(rest_time)
		Restore.restore_recursive(inc_rpath.index, mirror, rid, target,
								  rest_time, mirror_time)

	def get_mirror_time():
		"""Return the time (in seconds) of latest mirror"""
		current_mirror_incs = \
			Restore.get_inclist(Globals.rbdir.append("current_mirror"))
		if not current_mirror_incs:
			Log.FatalError("Could not get time of current mirror")
		elif len(current_mirror_incs) > 1:
			Log("Warning, two different dates for current mirror found", 2)
		return Time.stringtotime(current_mirror_incs[0].getinctime())

	def get_rest_time(old_rest_time, mirror_time):
		"""If old_rest_time is between two increments, return older time

		There is a slightly tricky reason for doing this:  The rest of
		the code just ignores increments that are older than
		rest_time.  But sometimes we want to consider the very next
		increment older than rest time, because rest_time will be
		between two increments, and what was actually on the mirror
		side will correspond to the older one.

		So here we assume all rdiff-backup events were recorded in
		"increments" increments, and if its in-between we pick the
		older one here.

		"""
		base_incs = Restore.get_inclist(Globals.rbdir.append("increments"))
		if not base_incs: return old_rest_time
		inctimes = [Time.stringtotime(inc.getinctime()) for inc in base_incs]
		return max(filter(lambda time: time <= old_rest_time,
						  inctimes + [mirror_time]))

	def get_inclist(inc_rpath):
		"""Returns increments with given base"""
		dirname, basename = inc_rpath.dirsplit()
		parent_dir = RPath(inc_rpath.conn, dirname, ())
		index = inc_rpath.index

		if index:
			get_inc_ext = lambda filename: \
						  RPath(inc_rpath.conn, inc_rpath.base,
								inc_rpath.index[:-1] + (filename,))
		else: get_inc_ext = lambda filename: \
					 RPath(inc_rpath.conn, os.path.join(dirname, filename))

		inc_list = []
		for filename in parent_dir.listdir():
			inc = get_inc_ext(filename)
			if inc.isincfile() and inc.getincbase_str() == basename:
				inc_list.append(inc)
		return inc_list

	def check_hardlinks(rest_time):
		"""Check for hard links and enable hard link support if found"""
		if (Globals.preserve_hardlinks != 0 and
			Hardlink.retrieve_final(rest_time)):
			Log("Hard link information found, attempting to preserve "
				"hard links.", 5)
			SetConnections.UpdateGlobal('preserve_hardlinks', 1)
		else: SetConnections.UpdateGlobal('preserve_hardlinks', None)

	def restore_recursive(index, mirror, rid, target, time, mirror_time):
		"""Recursive restore function.

		rid is a RestoreIncrementData object whose inclist is already
		sortedincseq'd, and target is the dsrp to restore to.

		Note that target may have a different index than mirror and
		rid, because we may be restoring a file whose index is, say
		('foo','bar') to a target whose path does not contain
		"foo/bar".

		"""
		assert isinstance(mirror, DSRPath) and isinstance(target, DSRPath)
		assert mirror.index == rid.index

		mirror_finalizer = DestructiveSteppingFinalizer()
		target_finalizer = DestructiveSteppingFinalizer()
		for rcd in Restore.yield_rcds(rid.index, mirror, rid,
									  target, time, mirror_time):
			rcd.RestoreFile()
			if rcd.mirror: mirror_finalizer(rcd.index, rcd.mirror)
			target_finalizer(rcd.target.index, rcd.target)
		target_finalizer.Finish()
		mirror_finalizer.Finish()

	def yield_rcds(index, mirrorrp, rid, target, rest_time, mirror_time):
		"""Iterate RestoreCombinedData objects starting with given args

		rid is a RestoreCombinedData object.  target is an rpath where
		the created file should go.

		In this case the "mirror" directory is treated as the source,
		and we are actually copying stuff onto what Select considers
		the source directory.

		"""
		select_result = Globals.select_mirror.Select(target)
		if select_result == 0: return

		if mirrorrp and not Globals.select_source.Select(mirrorrp):
			mirrorrp = None
		rcd = RestoreCombinedData(rid, mirrorrp, target)

		if mirrorrp and mirrorrp.isdir() or \
		   rid and rid.inc_rpath and rid.inc_rpath.isdir():
			sub_rcds = Restore.yield_sub_rcds(index, mirrorrp, rid,
											  target, rest_time, mirror_time)
		else: sub_rcds = None

		if select_result == 1:
			yield rcd
			if sub_rcds:
				for sub_rcd in sub_rcds: yield sub_rcd
		elif select_result == 2:
			if sub_rcds:
				try: first = sub_rcds.next()
				except StopIteration: return # no tuples found inside, skip
				yield rcd
				yield first
				for sub_rcd in sub_rcds: yield sub_rcd

	def yield_sub_rcds(index, mirrorrp, rid, target, rest_time, mirror_time):
		"""Yield collated tuples from inside given args"""
		if not Restore.check_dir_exists(mirrorrp, rid): return
		mirror_iter = Restore.yield_mirrorrps(mirrorrp)
		rid_iter = Restore.yield_rids(rid, rest_time, mirror_time)

		for indexed_tup in RORPIter.CollateIterators(mirror_iter, rid_iter):
			index = indexed_tup.index
			new_mirrorrp, new_rid = indexed_tup
			for rcd in Restore.yield_rcds(index, new_mirrorrp,
				  new_rid, target.append(index[-1]), rest_time, mirror_time):
				yield rcd

	def check_dir_exists(mirrorrp, rid):
		"""Return true if target should be a directory"""
		if rid and rid.inc_list:
			# Incs say dir if last (earliest) one is a dir increment
			return rid.inc_list[-1].getinctype() == "dir"
		elif mirrorrp: return mirrorrp.isdir() # if no incs, copy mirror
		else: return None

	def yield_mirrorrps(mirrorrp):
		"""Yield mirrorrps underneath given mirrorrp"""
		if mirrorrp and mirrorrp.isdir():
			if Globals.quoting_enabled:
				for rp in FilenameMapping.get_quoted_dir_children(mirrorrp):
					yield rp
			else:
				dirlist = mirrorrp.listdir()
				dirlist.sort()
				for filename in dirlist: yield mirrorrp.append(filename)

	def yield_rids(rid, rest_time, mirror_time):
		"""Yield RestoreIncrementData objects within given rid dir

		If the rid doesn't correspond to a directory, don't yield any
		elements.  If there are increments whose corresponding base
		doesn't exist, the first element will be None.  All the rpaths
		involved correspond to files in the increment directory.

		"""
		if not rid or not rid.inc_rpath or not rid.inc_rpath.isdir(): return
		rid_dict = {} # dictionary of basenames:rids
		dirlist = rid.inc_rpath.listdir()		
		if Globals.quoting_enabled:
			dirlist = [FilenameMapping.unquote(fn) for fn in dirlist]

		def affirm_dict_indexed(basename):
			"""Make sure the rid dictionary has given basename as key"""
			if not rid_dict.has_key(basename):
				rid_dict[basename] = RestoreIncrementData(
					rid.index + (basename,), None, []) # init with empty rid

		def add_to_dict(filename):
			"""Add filename to the inc tuple dictionary"""
			rp = rid.inc_rpath.append(filename)
			if Globals.quoting_enabled: rp.quote_path()
			if rp.isincfile() and rp.getinctype() != 'data':
				basename = rp.getincbase_str()
				affirm_dict_indexed(basename)
				rid_dict[basename].inc_list.append(rp)
			elif rp.isdir():
				affirm_dict_indexed(filename)
				rid_dict[filename].inc_rpath = rp

		for filename in dirlist: add_to_dict(filename)
		keys = rid_dict.keys()
		keys.sort()

		# sortincseq now to avoid descending .missing directories later
		for key in keys:
			rid = rid_dict[key]
			if rid.inc_rpath or rid.inc_list:
				rid.sortincseq(rest_time, mirror_time)
				yield rid

MakeStatic(Restore)


class RestoreIncrementData:
	"""Contains information about a specific index from the increments dir

	This is just a container class, used because it would be easier to
	work with than an IndexedTuple.

	"""
	def __init__(self, index, inc_rpath, inc_list):
		self.index = index
		self.inc_rpath = inc_rpath
		self.inc_list = inc_list

	def sortincseq(self, rest_time, mirror_time):
		"""Sort self.inc_list sequence, throwing away irrelevant increments"""
		if not self.inc_list or rest_time >= mirror_time:
			self.inc_list = []
			return

		newer_incs = self.get_newer_incs(rest_time, mirror_time)
		i = 0
		while(i < len(newer_incs)):
			# Only diff type increments require later versions
			if newer_incs[i].getinctype() != "diff": break
			i = i+1
		self.inc_list = newer_incs[:i+1]
		self.inc_list.reverse() # return in reversed order (latest first)
		
	def get_newer_incs(self, rest_time, mirror_time):
		"""Return list of newer incs sorted by time (increasing)

		Also discard increments older than rest_time (rest_time we are
		assuming is the exact time rdiff-backup was run, so no need to
		consider the next oldest increment or any of that)

		"""
		incpairs = []
		for inc in self.inc_list:
			time = Time.stringtotime(inc.getinctime())
			if time >= rest_time: incpairs.append((time, inc))
		incpairs.sort()
		return [pair[1] for pair in incpairs]
				

class RestoreCombinedData:
	"""Combine index information from increment and mirror directories

	This is similar to RestoreIncrementData but has mirror information
	also.

	"""
	def __init__(self, rid, mirror, target):
		"""Init - set values from one or both if they exist

		mirror and target are DSRPaths of the corresponding files in
		the mirror and target directory respectively.  rid is a
		RestoreIncrementData as defined above

		"""
		if rid:
			self.index = rid.index
			self.inc_rpath = rid.inc_rpath
			self.inc_list = rid.inc_list
			if mirror:
				self.mirror = mirror
				assert mirror.index == self.index
			else: self.mirror = None
		elif mirror:
			self.index = mirror.index
			self.mirror = mirror
			self.inc_list = []
			self.inc_rpath = None
		else: assert None, "neither rid nor mirror given"
		self.target = target

	def RestoreFile(self):
		"""Non-recursive restore function """
		if not self.inc_list and not (self.mirror and self.mirror.lstat()):
			return # no increments were applicable
		self.log()

		if self.restore_hardlink(): return

		if not self.inc_list or self.inc_list[0].getinctype() == "diff":
			assert self.mirror and self.mirror.lstat(), \
				   "No base to go with incs for %s" % self.target.path
			RPath.copy_with_attribs(self.mirror, self.target)
		for inc in self.inc_list: self.applyinc(inc, self.target)

	def log(self):
		"""Log current restore action"""
		inc_string = ','.join([inc.path for inc in self.inc_list])
		Log("Restoring %s with increments %s to %s" %
			(self.mirror and self.mirror.path,
			 inc_string, self.target.path), 5)

	def restore_hardlink(self):
		"""Hard link target and return true if hard linking appropriate"""
		if (Globals.preserve_hardlinks and
			Hardlink.restore_link(self.index, self.target)):
			RPath.copy_attribs(self.inc_list and self.inc_list[-1] or
							   self.mirror, self.target)
			return 1
		return None

	def applyinc(self, inc, target):
		"""Apply increment rp inc to targetrp target"""
		Log("Applying increment %s to %s" % (inc.path, target.path), 6)
		inctype = inc.getinctype()
		if inctype == "diff":
			if not target.lstat():
				raise RestoreError("Bad increment sequence at " + inc.path)
			Rdiff.patch_action(target, inc,
							   delta_compressed = inc.isinccompressed()
							   ).execute()
		elif inctype == "dir":
			if not target.isdir():
				if target.lstat():
					raise RestoreError("File %s already exists" % target.path)
				target.mkdir()
		elif inctype == "missing": return
		elif inctype == "snapshot":
			if inc.isinccompressed():
				target.write_from_fileobj(inc.open("rb", compress = 1))
			else: RPath.copy(inc, target)
		else: raise RestoreError("Unknown inctype %s" % inctype)
		RPath.copy_attribs(inc, target)


#######################################################################
#
# manage - list, delete, and otherwise manage increments
#

class ManageException(Exception): pass

class Manage:
	def get_incobjs(datadir):
		"""Return Increments objects given the rdiff-backup data directory"""
		return map(IncObj, Manage.find_incrps_with_base(datadir, "increments"))

	def get_file_type(rp):
		"""Returns one of "regular", "directory", "missing", or "special"."""
		if not rp.lstat(): return "missing"
		elif rp.isdir(): return "directory"
		elif rp.isreg(): return "regular"
		else: return "special"
	
	def get_inc_type(inc):
		"""Return file type increment represents"""
		assert inc.isincfile()
		type = inc.getinctype()
		if type == "dir": return "directory"
		elif type == "diff": return "regular"
		elif type == "missing": return "missing"
		elif type == "snapshot": return Manage.get_file_type(inc)
		else: assert None, "Unknown type %s" % (type,)

	def describe_incs_parsable(incs, mirror_time, mirrorrp):
		"""Return a string parsable by computer describing the increments

		Each line is a time in seconds of the increment, and then the
		type of the file.  It will be sorted oldest to newest.  For example:

		10000 regular
		20000 directory
		30000 special
		40000 missing
		50000 regular    <- last will be the current mirror

		"""
		incpairs = [(Time.stringtotime(inc.getinctime()), inc) for inc in incs]
		incpairs.sort()
		result = ["%s %s" % (time, Manage.get_inc_type(inc))
				  for time, inc in incpairs]
		result.append("%s %s" % (mirror_time, Manage.get_file_type(mirrorrp)))
		return "\n".join(result)

	def describe_incs_human(incs, mirror_time, mirrorrp):
		"""Return a string describing all the the root increments"""
		incpairs = [(Time.stringtotime(inc.getinctime()), inc) for inc in incs]
		incpairs.sort()

		result = ["Found %d increments:" % len(incpairs)]
		for time, inc in incpairs:
			result.append("    %s   %s" %
						  (inc.dirsplit()[1], Time.timetopretty(time)))
		result.append("Current mirror: %s" % Time.timetopretty(mirror_time))
		return "\n".join(result)

	def delete_earlier_than(baserp, time):
		"""Deleting increments older than time in directory baserp

		time is in seconds.  It will then delete any empty directories
		in the tree.  To process the entire backup area, the
		rdiff-backup-data directory should be the root of the tree.

		"""
		baserp.conn.Manage.delete_earlier_than_local(baserp, time)

	def delete_earlier_than_local(baserp, time):
		"""Like delete_earlier_than, but run on local connection for speed"""
		assert baserp.conn is Globals.local_connection
		def yield_files(rp):
			yield rp
			if rp.isdir():
				for filename in rp.listdir():
					for sub_rp in yield_files(rp.append(filename)):
						yield sub_rp

		for rp in yield_files(baserp):
			if ((rp.isincfile() and
				 Time.stringtotime(rp.getinctime()) < time) or
				(rp.isdir() and not rp.listdir())):
				Log("Deleting increment file %s" % rp.path, 5)
				rp.delete()

MakeStatic(Manage)


class IncObj:
	"""Increment object - represent a completed increment"""
	def __init__(self, incrp):
		"""IncObj initializer

		incrp is an RPath of a path like increments.TIMESTR.dir
		standing for the root of the increment.

		"""
		if not incrp.isincfile():
			raise ManageException("%s is not an inc file" % incrp.path)
		self.incrp = incrp
		self.time = Time.stringtotime(incrp.getinctime())

	def getbaserp(self):
		"""Return rp of the incrp without extensions"""
		return self.incrp.getincbase()

	def pretty_time(self):
		"""Return a formatted version of inc's time"""
		return Time.timetopretty(self.time)

	def full_description(self):
		"""Return string describing increment"""
		s = ["Increment file %s" % self.incrp.path,
			 "Date: %s" % self.pretty_time()]
		return "\n".join(s)


#######################################################################
#
# highlevel - High level functions for mirroring, mirror & inc, etc.
#

class SkipFileException(Exception):
	"""Signal that the current file should be skipped but then continue

	This exception will often be raised when there is problem reading
	an individual file, but it makes sense for the rest of the backup
	to keep going.

	"""
	pass


class HighLevel:
	"""High level static functions

	The design of some of these functions is represented on the
	accompanying diagram.

	"""
	def Mirror(src_rpath, dest_rpath, inc_rpath = None, session_info = None):
		"""Turn dest_rpath into a copy of src_rpath

		If inc_rpath is true, then this is the initial mirroring of an
		incremental backup, so checkpoint and write to data_dir.
		Otherwise only mirror and don't create any extra files.

		"""
		SourceS = src_rpath.conn.HLSourceStruct
		DestS = dest_rpath.conn.HLDestinationStruct

		SourceS.set_session_info(session_info)
		DestS.set_session_info(session_info)
		src_init_dsiter = SourceS.split_initial_dsiter()
		dest_sigiter = DestS.get_sigs(dest_rpath, src_init_dsiter)
		diffiter = SourceS.get_diffs_and_finalize(dest_sigiter)
		if inc_rpath:
			DestS.patch_w_datadir_writes(dest_rpath, diffiter, inc_rpath)
		else: DestS.patch_and_finalize(dest_rpath, diffiter)

		dest_rpath.setdata()

	def Mirror_and_increment(src_rpath, dest_rpath, inc_rpath,
							 session_info = None):
		"""Mirror + put increments in tree based at inc_rpath"""
		SourceS = src_rpath.conn.HLSourceStruct
		DestS = dest_rpath.conn.HLDestinationStruct

		SourceS.set_session_info(session_info)
		DestS.set_session_info(session_info)
		if not session_info: dest_rpath.conn.SaveState.touch_last_file()
		src_init_dsiter = SourceS.split_initial_dsiter()
		dest_sigiter = DestS.get_sigs(dest_rpath, src_init_dsiter)
		diffiter = SourceS.get_diffs_and_finalize(dest_sigiter)
		DestS.patch_increment_and_finalize(dest_rpath, diffiter, inc_rpath)

		dest_rpath.setdata()
		inc_rpath.setdata()

MakeStatic(HighLevel)


class HLSourceStruct:
	"""Hold info used by HL on the source side"""
	_session_info = None # set to si if resuming
	def set_session_info(cls, session_info):
		cls._session_info = session_info

	def iterate_from(cls):
		"""Supply more aruments to DestructiveStepping.Iterate_from"""
		if cls._session_info is None: Globals.select_source.set_iter()
		else: Globals.select_source.set_iter(cls._session_info.last_index, 1)
		return Globals.select_source

	def split_initial_dsiter(cls):
		"""Set iterators of all dsrps from rpath, returning one"""
		dsiter = cls.iterate_from()
		initial_dsiter1, cls.initial_dsiter2 = Iter.multiplex(dsiter, 2)
		return initial_dsiter1

	def get_diffs_and_finalize(cls, sigiter):
		"""Return diffs and finalize any dsrp changes remaining

		Return a rorpiterator with files included of signatures of
		dissimilar files.  This is the last operation run on the local
		filestream, so finalize dsrp writes.

		"""
		collated = RORPIter.CollateIterators(cls.initial_dsiter2, sigiter)
		finalizer = DestructiveSteppingFinalizer()
		def error_handler(exc, dest_sig, dsrp):
			Log("Error %s producing a diff of %s" %
				(exc, dsrp and dsrp.path), 2)
			return None
			
		def diffs():
			for dsrp, dest_sig in collated:
				if dest_sig:
					if dest_sig.isplaceholder(): yield dest_sig
					else:
						diff = Robust.check_common_error(
							error_handler, RORPIter.diffonce, dest_sig, dsrp)
						if diff: yield diff
				if dsrp: finalizer(dsrp.index, dsrp)
			finalizer.Finish()
		return diffs()

MakeClass(HLSourceStruct)


class HLDestinationStruct:
	"""Hold info used by HL on the destination side"""
	_session_info = None # set to si if resuming
	def set_session_info(cls, session_info):
		cls._session_info = session_info

	def iterate_from(cls):
		"""Return selection iterator to iterate all the mirror files"""
		if cls._session_info is None: Globals.select_mirror.set_iter()
		else: Globals.select_mirror.set_iter(cls._session_info.last_index)
		return Globals.select_mirror

	def split_initial_dsiter(cls):
		"""Set initial_dsiters (iteration of all dsrps from rpath)"""
		result, cls.initial_dsiter2 = Iter.multiplex(cls.iterate_from(), 2)
		return result

	def get_dissimilar(cls, baserp, src_init_iter, dest_init_iter):
		"""Get dissimilars

		Returns an iterator which enumerates the dsrps which are
		different on the source and destination ends.  The dsrps do
		not necessarily exist on the destination end.

		Also, to prevent the system from getting backed up on the
		remote end, if we don't get enough dissimilars, stick in a
		placeholder every so often, like fiber.  The more
		placeholders, the more bandwidth used, but if there aren't
		enough, lots of memory will be used because files will be
		accumulating on the source side.  How much will accumulate
		will depend on the Globals.conn_bufsize value.

		"""
		collated = RORPIter.CollateIterators(src_init_iter, dest_init_iter)
		def compare(src_rorp, dest_dsrp):
			"""Return dest_dsrp if they are different, None if the same"""
			if not dest_dsrp:
				dest_dsrp = cls.get_dsrp(baserp, src_rorp.index)
				if dest_dsrp.lstat():
					Log("Warning: Found unexpected destination file %s, "
						"not processing it." % dest_dsrp.path, 2)
					return None
			elif (src_rorp and src_rorp == dest_dsrp and
				  (not Globals.preserve_hardlinks or
				   Hardlink.rorp_eq(src_rorp, dest_dsrp))):
				return None
			if src_rorp and src_rorp.isreg() and Hardlink.islinked(src_rorp):
				dest_dsrp.flaglinked()
			return dest_dsrp

		def generate_dissimilar():
			counter = 0
			for src_rorp, dest_dsrp in collated:
				if Globals.preserve_hardlinks:
					if src_rorp: Hardlink.add_rorp(src_rorp, 1)
					if dest_dsrp: Hardlink.add_rorp(dest_dsrp, None)
				dsrp = compare(src_rorp, dest_dsrp)
				if dsrp:
					counter = 0
					yield dsrp
				elif counter == 20:
					placeholder = RORPath(src_rorp.index)
					placeholder.make_placeholder()
					counter = 0
					yield placeholder
				else: counter += 1
		return generate_dissimilar()

	def get_sigs(cls, baserp, src_init_iter):
		"""Return signatures of all dissimilar files"""
		dest_iters1 = cls.split_initial_dsiter()
		dissimilars = cls.get_dissimilar(baserp, src_init_iter, dest_iters1)
		return RORPIter.Signatures(dissimilars)

	def get_dsrp(cls, dest_rpath, index):
		"""Return initialized dsrp based on dest_rpath with given index"""
		dsrp = DSRPath(None, dest_rpath.conn, dest_rpath.base, index)
		if Globals.quoting_enabled: dsrp.quote_path()
		return dsrp

	def get_finalizer(cls):
		"""Return finalizer, starting from session info if necessary"""
		old_finalizer = cls._session_info and cls._session_info.finalizer
		if old_finalizer: return old_finalizer
		else: return DestructiveSteppingFinalizer()

	def get_ITR(cls, inc_rpath):
		"""Return ITR, starting from state if necessary"""
		if cls._session_info and cls._session_info.ITR:
			return cls._session_info.ITR
		else:
			iitr = IncrementITR(inc_rpath)
			iitr.override_changed()
			Globals.ITR = iitr
			iitr.Errors = 0
			return iitr

	def get_MirrorITR(cls, inc_rpath):
		"""Return MirrorITR, starting from state if available"""
		if cls._session_info and cls._session_info.ITR:
			return cls._session_info.ITR
		ITR = MirrorITR(inc_rpath)
		Globals.ITR = ITR
		ITR.Errors = 0
		return ITR

	def patch_and_finalize(cls, dest_rpath, diffs):
		"""Apply diffs and finalize"""
		collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
		finalizer = cls.get_finalizer()
		diff_rorp, dsrp = None, None

		def patch(diff_rorp, dsrp):
			if not dsrp: dsrp = cls.get_dsrp(dest_rpath, diff_rorp.index)
			if diff_rorp and not diff_rorp.isplaceholder():
				RORPIter.patchonce_action(None, dsrp, diff_rorp).execute()
			return dsrp

		def error_handler(exc, diff_rorp, dsrp):
			filename = dsrp and dsrp.path or os.path.join(*diff_rorp.index)
			Log("Error: %s processing file %s" % (exc, filename), 2)
		
		for indexed_tuple in collated:
			Log("Processing %s" % str(indexed_tuple), 7)
			diff_rorp, dsrp = indexed_tuple
			dsrp = Robust.check_common_error(error_handler, patch,
											 diff_rorp, dsrp)
			finalizer(dsrp.index, dsrp)
		finalizer.Finish()

	def patch_w_datadir_writes(cls, dest_rpath, diffs, inc_rpath):
		"""Apply diffs and finalize, with checkpointing and statistics"""
		collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
		finalizer, ITR = cls.get_finalizer(), cls.get_MirrorITR(inc_rpath)
		Stats.open_dir_stats_file()
		dsrp, finished_dsrp = None, None

		try:
			for indexed_tuple in collated:
				Log("Processing %s" % str(indexed_tuple), 7)
				diff_rorp, dsrp = indexed_tuple
				if not dsrp: dsrp = cls.get_dsrp(dest_rpath, diff_rorp.index)
				if diff_rorp and diff_rorp.isplaceholder(): diff_rorp = None
				ITR(dsrp.index, diff_rorp, dsrp)
				finalizer(dsrp.index, dsrp)
				SaveState.checkpoint(ITR, finalizer, dsrp)
				finished_dsrp = dsrp
			ITR.Finish()
			finalizer.Finish()
		except: cls.handle_last_error(finished_dsrp, finalizer, ITR)

		if Globals.preserve_hardlinks: Hardlink.final_writedata()
		Stats.close_dir_stats_file()
		Stats.write_session_statistics(ITR)
		SaveState.checkpoint_remove()

	def patch_increment_and_finalize(cls, dest_rpath, diffs, inc_rpath):
		"""Apply diffs, write increment if necessary, and finalize"""
		collated = RORPIter.CollateIterators(diffs, cls.initial_dsiter2)
		finalizer, ITR = cls.get_finalizer(), cls.get_ITR(inc_rpath)
		Stats.open_dir_stats_file()
		dsrp, finished_dsrp = None, None

		try:
			for indexed_tuple in collated:
				Log("Processing %s" % str(indexed_tuple), 7)
				diff_rorp, dsrp = indexed_tuple
				index = indexed_tuple.index
				if not dsrp: dsrp = cls.get_dsrp(dest_rpath, index)
				if diff_rorp and diff_rorp.isplaceholder(): diff_rorp = None
				ITR(index, diff_rorp, dsrp)
				finalizer(index, dsrp)
				SaveState.checkpoint(ITR, finalizer, dsrp)
				finished_dsrp = dsrp
			ITR.Finish()
			finalizer.Finish()
		except: cls.handle_last_error(finished_dsrp, finalizer, ITR)

		if Globals.preserve_hardlinks: Hardlink.final_writedata()
		Stats.close_dir_stats_file()
		Stats.write_session_statistics(ITR)
		SaveState.checkpoint_remove()

	def handle_last_error(cls, dsrp, finalizer, ITR):
		"""If catch fatal error, try to checkpoint before exiting"""
		Log.exception(1, 2)
		TracebackArchive.log()
		SaveState.checkpoint(ITR, finalizer, dsrp, 1)
		if Globals.preserve_hardlinks: Hardlink.final_checkpoint(Globals.rbdir)
		SaveState.touch_last_file_definitive()
		raise

MakeClass(HLDestinationStruct)


#######################################################################
#
# setconnections - Parse initial arguments and establish connections
#

class SetConnectionsException(Exception): pass

class SetConnections:
	"""Parse args and setup connections

	The methods in this class are used once by Main to parse file
	descriptions like bescoto@folly.stanford.edu:/usr/bin/ls and to
	set up the related connections.

	"""
	# This is the schema that determines how rdiff-backup will open a
	# pipe to the remote system.  If the file is given as A::B, %s will
	# be substituted with A in the schema.
	__cmd_schema = 'ssh -C %s rdiff-backup --server'
	__cmd_schema_no_compress = 'ssh %s rdiff-backup --server'

	# This is a list of remote commands used to start the connections.
	# The first is None because it is the local connection.
	__conn_remote_cmds = [None]

	def InitRPs(cls, arglist, remote_schema = None, remote_cmd = None):
		"""Map the given file descriptions into rpaths and return list"""
		if remote_schema: cls.__cmd_schema = remote_schema
		elif not Globals.ssh_compression:
			cls.__cmd_schema = cls.__cmd_schema_no_compress

		if not arglist: return []
		desc_pairs = map(cls.parse_file_desc, arglist)

		if filter(lambda x: x[0], desc_pairs): # True if any host_info found
			if remote_cmd:
				Log.FatalError("The --remote-cmd flag is not compatible "
							   "with remote file descriptions.")
		elif remote_schema:
			Log("Remote schema option ignored - no remote file "
				"descriptions.", 2)

		cmd_pairs = map(cls.desc2cmd_pairs, desc_pairs)
		if remote_cmd: # last file description gets remote_cmd
			cmd_pairs[-1] = (remote_cmd, cmd_pairs[-1][1])
		return map(cls.cmdpair2rp, cmd_pairs)

	def cmdpair2rp(cls, cmd_pair):
		"""Return RPath from cmd_pair (remote_cmd, filename)"""
		cmd, filename = cmd_pair
		if cmd: conn = cls.init_connection(cmd)
		else: conn = Globals.local_connection
		return RPath(conn, filename)

	def desc2cmd_pairs(cls, desc_pair):
		"""Return pair (remote_cmd, filename) from desc_pair"""
		host_info, filename = desc_pair
		if not host_info: return (None, filename)
		else: return (cls.fill_schema(host_info), filename)

	def parse_file_desc(cls, file_desc):
		"""Parse file description returning pair (host_info, filename)

		In other words, bescoto@folly.stanford.edu::/usr/bin/ls =>
		("bescoto@folly.stanford.edu", "/usr/bin/ls").  The
		complication is to allow for quoting of : by a \.  If the
		string is not separated by :, then the host_info is None.

		"""
		def check_len(i):
			if i >= len(file_desc):
				raise SetConnectionsException(
					"Unexpected end to file description %s" % file_desc)
				
		host_info_list, i, last_was_quoted = [], 0, None
		while 1:
			if i == len(file_desc):
				return (None, file_desc)

			if file_desc[i] == '\\':
				i = i+1
				check_len(i)
				last_was_quoted = 1
			elif (file_desc[i] == ":" and i > 0 and file_desc[i-1] == ":"
				  and not last_was_quoted):
				host_info_list.pop() # Remove last colon from name
				break
			else: last_was_quoted = None
			host_info_list.append(file_desc[i])
			i = i+1
				
		check_len(i+1)
		return ("".join(host_info_list), file_desc[i+1:])

	def fill_schema(cls, host_info):
		"""Fills host_info into the schema and returns remote command"""
		return cls.__cmd_schema % host_info

	def init_connection(cls, remote_cmd):
		"""Run remote_cmd, register connection, and then return it

		If remote_cmd is None, then the local connection will be
		returned.  This also updates some settings on the remote side,
		like global settings, its connection number, and verbosity.

		"""
		if not remote_cmd: return Globals.local_connection

		Log("Executing " + remote_cmd, 4)
		stdin, stdout = os.popen2(remote_cmd)
		conn_number = len(Globals.connections)
		conn = PipeConnection(stdout, stdin, conn_number)

		cls.check_connection_version(conn, remote_cmd)
		Log("Registering connection %d" % conn_number, 7)
		cls.init_connection_routing(conn, conn_number, remote_cmd)
		cls.init_connection_settings(conn)
		return conn

	def check_connection_version(cls, conn, remote_cmd):
		"""Log warning if connection has different version"""
		try: remote_version = conn.Globals.get('version')
		except ConnectionReadError, exception:
			Log.FatalError("""%s

Couldn't start up the remote connection by executing

    %s

Remember that, under the default settings, rdiff-backup must be
installed in the PATH on the remote system.  See the man page for more
information.""" % (exception, remote_cmd))
		
		if remote_version != Globals.version:
			Log("Warning: Local version %s does not match remote version %s."
				% (Globals.version, remote_version), 2)

	def init_connection_routing(cls, conn, conn_number, remote_cmd):
		"""Called by init_connection, establish routing, conn dict"""
		Globals.connection_dict[conn_number] = conn

		conn.SetConnections.init_connection_remote(conn_number)
		for other_remote_conn in Globals.connections[1:]:
			conn.SetConnections.add_redirected_conn(
				other_remote_conn.conn_number)
			other_remote_conn.SetConnections.add_redirected_conn(conn_number)

		Globals.connections.append(conn)
		cls.__conn_remote_cmds.append(remote_cmd)

	def init_connection_settings(cls, conn):
		"""Tell new conn about log settings and updated globals"""
		conn.Log.setverbosity(Log.verbosity)
		conn.Log.setterm_verbosity(Log.term_verbosity)
		for setting_name in Globals.changed_settings:
			conn.Globals.set(setting_name, Globals.get(setting_name))

	def init_connection_remote(cls, conn_number):
		"""Run on server side to tell self that have given conn_number"""
		Globals.connection_number = conn_number
		Globals.local_connection.conn_number = conn_number
		Globals.connection_dict[0] = Globals.connections[1]
		Globals.connection_dict[conn_number] = Globals.local_connection

	def add_redirected_conn(cls, conn_number):
		"""Run on server side - tell about redirected connection"""
		Globals.connection_dict[conn_number] = \
			   RedirectedConnection(conn_number)

	def UpdateGlobal(cls, setting_name, val):
		"""Update value of global variable across all connections"""
		for conn in Globals.connections:
			conn.Globals.set(setting_name, val)

	def BackupInitConnections(cls, reading_conn, writing_conn):
		"""Backup specific connection initialization"""
		reading_conn.Globals.set("isbackup_reader", 1)
		writing_conn.Globals.set("isbackup_writer", 1)
		cls.UpdateGlobal("backup_reader", reading_conn)
		cls.UpdateGlobal("backup_writer", writing_conn)

	def CloseConnections(cls):
		"""Close all connections.  Run by client"""
		assert not Globals.server
		for conn in Globals.connections: conn.quit()
		del Globals.connections[1:] # Only leave local connection
		Globals.connection_dict = {0: Globals.local_connection}
		Globals.backup_reader = Globals.isbackup_reader = \
			  Globals.backup_writer = Globals.isbackup_writer = None

	def TestConnections(cls):
		"""Test connections, printing results"""
		if len(Globals.connections) == 1:
			print "No remote connections specified"
		else:
			for i in range(1, len(Globals.connections)):
				cls.test_connection(i)

	def test_connection(cls, conn_number):
		"""Test connection.  conn_number 0 is the local connection"""
		print "Testing server started by: ", \
			  cls.__conn_remote_cmds[conn_number]
		conn = Globals.connections[conn_number]
		try:
			assert conn.pow(2,3) == 8
			assert conn.os.path.join("a", "b") == "a/b"
			version = conn.reval("lambda: Globals.version")
		except:
			sys.stderr.write("Server tests failed\n")
			raise
		if not version == Globals.version:
			print """Server may work, but there is a version mismatch:
Local version: %s
Remote version: %s""" % (Globals.version, version)
		else: print "Server OK"

MakeClass(SetConnections)


#######################################################################
#
# main - Start here: Read arguments, set global settings, etc.
#

class Main:
	def __init__(self):
		self.action = None
		self.remote_cmd, self.remote_schema = None, None
		self.force = None
		self.select_opts, self.select_mirror_opts = [], []
		self.select_files = []

	def parse_cmdlineoptions(self, arglist):
		"""Parse argument list and set global preferences"""
		def sel_fl(filename):
			"""Helper function for including/excluding filelists below"""
			try: return open(filename, "r")
			except IOError: Log.FatalError("Error opening file %s" % filename)

		try: optlist, self.args = getopt.getopt(arglist, "blmr:sv:V",
			 ["backup-mode", "calculate-average",
			  "change-source-perms", "chars-to-quote=",
			  "checkpoint-interval=", "current-time=", "exclude=",
			  "exclude-device-files", "exclude-filelist=",
			  "exclude-filelist-stdin", "exclude-mirror=",
			  "exclude-regexp=", "force", "include=",
			  "include-filelist=", "include-filelist-stdin",
			  "include-regexp=", "list-increments", "mirror-only",
			  "no-compression", "no-compression-regexp=",
			  "no-hard-links", "no-resume", "null-separator",
			  "parsable-output", "print-statistics", "quoting-char=",
			  "remote-cmd=", "remote-schema=", "remove-older-than=",
			  "restore-as-of=", "resume", "resume-window=", "server",
			  "ssh-no-compression", "terminal-verbosity=",
			  "test-server", "verbosity", "version", "windows-mode",
			  "windows-time-format"])
		except getopt.error, e:
			self.commandline_error("Bad commandline options: %s" % str(e))

		for opt, arg in optlist:
			if opt == "-b" or opt == "--backup-mode": self.action = "backup"
			elif opt == "--calculate-average":
				self.action = "calculate-average"
			elif opt == "--change-source-perms":
				Globals.set('change_source_perms', 1)
			elif opt == "--chars-to-quote":
				Globals.set('chars_to_quote', arg)
				Globals.set('quoting_enabled', 1)
			elif opt == "--checkpoint-interval":
				Globals.set_integer('checkpoint_interval', arg)
			elif opt == "--current-time":
				Globals.set_integer('current_time', arg)
			elif opt == "--exclude": self.select_opts.append((opt, arg))
			elif opt == "--exclude-device-files":
				self.select_opts.append((opt, arg))
			elif opt == "--exclude-filelist":
				self.select_opts.append((opt, arg))
				self.select_files.append(sel_fl(arg))
			elif opt == "--exclude-filelist-stdin":
				self.select_opts.append(("--exclude-filelist",
										 "standard input"))
				self.select_files.append(sys.stdin)
			elif opt == "--exclude-mirror":
				self.select_mirror_opts.append(("--exclude", arg))
			elif opt == "--exclude-regexp": self.select_opts.append((opt, arg))
			elif opt == "--force": self.force = 1
			elif opt == "--include": self.select_opts.append((opt, arg))
			elif opt == "--include-filelist":
				self.select_opts.append((opt, arg))
				self.select_files.append(sel_fl(arg))
			elif opt == "--include-filelist-stdin":
				self.select_opts.append(("--include-filelist",
										 "standard input"))
				self.select_files.append(sys.stdin)
			elif opt == "--include-regexp":
				self.select_opts.append((opt, arg))
			elif opt == "-l" or opt == "--list-increments":
				self.action = "list-increments"
			elif opt == "-m" or opt == "--mirror-only": self.action = "mirror"
			elif opt == "--no-compression": Globals.set("compression", None)
			elif opt == "--no-compression-regexp":
				Globals.set("no_compression_regexp_string", arg)
			elif opt == "--no-hard-links": Globals.set('preserve_hardlinks', 0)
			elif opt == '--no-resume': Globals.resume = 0
			elif opt == "--null-separator": Globals.set("null_separator", 1)
			elif opt == "-r" or opt == "--restore-as-of":
				self.restore_timestr = arg
				self.action = "restore-as-of"
			elif opt == "--parsable-output": Globals.set('parsable_output', 1)
			elif opt == "--print-statistics":
				Globals.set('print_statistics', 1)
			elif opt == "--quoting-char":
				Globals.set('quoting_char', arg)
				Globals.set('quoting_enabled', 1)
			elif opt == "--remote-cmd": self.remote_cmd = arg
			elif opt == "--remote-schema": self.remote_schema = arg
			elif opt == "--remove-older-than":
				self.remove_older_than_string = arg
				self.action = "remove-older-than"
			elif opt == '--resume': Globals.resume = 1
			elif opt == '--resume-window':
				Globals.set_integer('resume_window', arg)
			elif opt == "-s" or opt == "--server": self.action = "server"
			elif opt == "--ssh-no-compression":
				Globals.set('ssh_compression', None)
			elif opt == "--terminal-verbosity": Log.setterm_verbosity(arg)
			elif opt == "--test-server": self.action = "test-server"
			elif opt == "-V" or opt == "--version":
				print "rdiff-backup " + Globals.version
				sys.exit(0)
			elif opt == "-v" or opt == "--verbosity": Log.setverbosity(arg)
			elif opt == "--windows-mode":
				Globals.set('time_separator', "_")
				Globals.set('chars_to_quote', ":")
				Globals.set('quoting_enabled', 1)
			elif opt == '--windows-time-format':
				Globals.set('time_separator', "_")
			else: Log.FatalError("Unknown option %s" % opt)

	def set_action(self):
		"""Check arguments and try to set self.action"""
		l = len(self.args)
		if not self.action:
			if l == 0: self.commandline_error("No arguments given")
			elif l == 1: self.action = "restore"
			elif l == 2:
				if RPath(Globals.local_connection, self.args[0]).isincfile():
					self.action = "restore"
				else: self.action = "backup"
			else: self.commandline_error("Too many arguments given")

		if l == 0 and self.action != "server" and self.action != "test-server":
			self.commandline_error("No arguments given")
		if l > 0 and self.action == "server":
			self.commandline_error("Too many arguments given")
		if l < 2 and (self.action == "backup" or self.action == "mirror" or
					  self.action == "restore-as-of"):
			self.commandline_error("Two arguments are required "
								   "(source, destination).")
		if l == 2 and (self.action == "list-increments" or
					   self.action == "remove-older-than"):
			self.commandline_error("Only use one argument, "
								   "the root of the backup directory")
		if l > 2 and self.action != "calculate-average":
			self.commandline_error("Too many arguments given")

	def commandline_error(self, message):
		sys.stderr.write("Error: %s\n" % message)
		sys.stderr.write("See the rdiff-backup manual page for instructions\n")
		sys.exit(1)

	def misc_setup(self, rps):
		"""Set default change ownership flag, umask, relay regexps"""
		if ((len(rps) == 2 and rps[1].conn.os.getuid() == 0) or
			(len(rps) < 2 and os.getuid() == 0)):
			# Allow change_ownership if destination connection is root
			for conn in Globals.connections:
				conn.Globals.set('change_ownership', 1)
			for rp in rps: rp.setdata() # Update with userinfo

		os.umask(077)
		Time.setcurtime(Globals.current_time)
		FilenameMapping.set_init_quote_vals()
		Globals.set("isclient", 1)
		SetConnections.UpdateGlobal("client_conn", Globals.local_connection)

		# This is because I originally didn't think compiled regexps
		# could be pickled, and so must be compiled on remote side.
		Globals.postset_regexp('no_compression_regexp',
							   Globals.no_compression_regexp_string)

		for conn in Globals.connections: Robust.install_signal_handlers()

	def take_action(self, rps):
		"""Do whatever self.action says"""
		if self.action == "server":
			PipeConnection(sys.stdin, sys.stdout).Server()
		elif self.action == "backup": self.Backup(rps[0], rps[1])
		elif self.action == "restore": self.Restore(*rps)
		elif self.action == "restore-as-of": self.RestoreAsOf(rps[0], rps[1])
		elif self.action == "mirror": self.Mirror(rps[0], rps[1])
		elif self.action == "test-server": SetConnections.TestConnections()
		elif self.action == "list-increments": self.ListIncrements(rps[0])
		elif self.action == "remove-older-than": self.RemoveOlderThan(rps[0])
		elif self.action == "calculate-average": self.CalculateAverage(rps)
		else: raise AssertionError("Unknown action " + self.action)

	def cleanup(self):
		"""Do any last minute cleaning before exiting"""
		Log("Cleaning up", 6)
		Log.close_logfile()
		if not Globals.server: SetConnections.CloseConnections()

	def Main(self, arglist):
		"""Start everything up!"""
		self.parse_cmdlineoptions(arglist)
		self.set_action()
		rps = SetConnections.InitRPs(self.args,
									 self.remote_schema, self.remote_cmd)
		self.misc_setup(rps)
		self.take_action(rps)
		self.cleanup()


	def Mirror(self, src_rp, dest_rp):
		"""Turn dest_path into a copy of src_path"""
		Log("Mirroring %s to %s" % (src_rp.path, dest_rp.path), 5)
		self.mirror_check_paths(src_rp, dest_rp)
		# Since no "rdiff-backup-data" dir, use root of destination.
		SetConnections.UpdateGlobal('rbdir', dest_rp)
		SetConnections.BackupInitConnections(src_rp.conn, dest_rp.conn)
		HighLevel.Mirror(src_rp, dest_rp)

	def mirror_check_paths(self, rpin, rpout):
		"""Check paths and return rpin, rpout"""
		if not rpin.lstat():
			Log.FatalError("Source directory %s does not exist" % rpin.path)
		if rpout.lstat() and not self.force:
			Log.FatalError(
"""Destination %s exists so continuing could mess it up.  Run
rdiff-backup with the --force option if you want to mirror anyway.""" %
			rpout.path)


	def Backup(self, rpin, rpout):
		"""Backup, possibly incrementally, src_path to dest_path."""
		SetConnections.BackupInitConnections(rpin.conn, rpout.conn)
		self.backup_init_select(rpin, rpout)
		self.backup_init_dirs(rpin, rpout)
		RSI = Globals.backup_writer.Resume.ResumeCheck()
		SaveState.init_filenames()
		if self.prevtime:
			Time.setprevtime(self.prevtime)
			HighLevel.Mirror_and_increment(rpin, rpout, self.incdir, RSI)
		else: HighLevel.Mirror(rpin, rpout, self.incdir, RSI)
		self.backup_touch_curmirror(rpin, rpout)

	def backup_init_select(self, rpin, rpout):
		"""Create Select objects on source and dest connections"""
		rpin.conn.Globals.set_select(DSRPath(1, rpin), self.select_opts,
									 None, *self.select_files)
		rpout.conn.Globals.set_select(DSRPath(None, rpout),
									  self.select_mirror_opts, 1)

	def backup_init_dirs(self, rpin, rpout):
		"""Make sure rpin and rpout are valid, init data dir and logging"""
		if rpout.lstat() and not rpout.isdir():
			if not self.force:
				Log.FatalError("Destination %s exists and is not a "
							   "directory" % rpout.path)
			else:
				Log("Deleting %s" % rpout.path, 3)
				rpout.delete()
			
		if not rpin.lstat():
			Log.FatalError("Source directory %s does not exist" % rpin.path)
		elif not rpin.isdir():
			Log.FatalError("Source %s is not a directory" % rpin.path)

		self.datadir = rpout.append("rdiff-backup-data")
		SetConnections.UpdateGlobal('rbdir', self.datadir)
		self.incdir = RPath(rpout.conn, os.path.join(self.datadir.path,
													 "increments"))
		self.prevtime = self.backup_get_mirrortime()

		if rpout.lstat():
			if rpout.isdir() and not rpout.listdir(): # rpout is empty dir
				rpout.chmod(0700) # just make sure permissions aren't too lax
			elif not self.datadir.lstat() and not self.force:
				Log.FatalError(
"""Destination directory %s exists, but does not look like a
rdiff-backup directory.  Running rdiff-backup like this could mess up
what is currently in it.  If you want to overwrite it, run
rdiff-backup with the --force option.""" % rpout.path)

		if not rpout.lstat():
			try: rpout.mkdir()
			except os.error:
				Log.FatalError("Unable to create directory %s" % rpout.path)
		if not self.datadir.lstat(): self.datadir.mkdir()
		if Log.verbosity > 0:
			Log.open_logfile(self.datadir.append("backup.log"))
		self.backup_warn_if_infinite_regress(rpin, rpout)

	def backup_warn_if_infinite_regress(self, rpin, rpout):
		"""Warn user if destination area contained in source area"""
		if rpout.conn is rpin.conn: # it's meaningful to compare paths
			if ((len(rpout.path) > len(rpin.path)+1 and
				 rpout.path[:len(rpin.path)] == rpin.path and
				 rpout.path[len(rpin.path)] == '/') or
				(rpin.path == "." and rpout.path[0] != '/' and
				 rpout.path[:2] != '..')):
				# Just a few heuristics, we don't have to get every case
				if Globals.backup_reader.Globals.select_source \
				   .Select(rpout): Log(
"""Warning: The destination directory '%s' may be contained in the
source directory '%s'.  This could cause an infinite regress.  You
may need to use the --exclude option.""" % (rpout.path, rpin.path), 2)

	def backup_get_mirrorrps(self):
		"""Return list of current_mirror rps"""
		if not self.datadir.isdir(): return []
		mirrorrps = [self.datadir.append(fn) for fn in self.datadir.listdir()
					 if fn.startswith("current_mirror.")]
		return filter(lambda rp: rp.isincfile(), mirrorrps)

	def backup_get_mirrortime(self):
		"""Return time in seconds of previous mirror, or None if cannot"""
		mirrorrps = self.backup_get_mirrorrps()
		if not mirrorrps: return None
		if len(mirrorrps) > 1:
			Log(
"""Warning: duplicate current_mirror files found.  Perhaps something
went wrong during your last backup?  Using """ + mirrorrps[-1].path, 2)

		timestr = mirrorrps[-1].getinctime()
		return Time.stringtotime(timestr)
	
	def backup_touch_curmirror(self, rpin, rpout):
		"""Make a file like current_mirror.time.data to record time

		Also updates rpout so mod times don't get messed up.

		"""
		map(RPath.delete, self.backup_get_mirrorrps())
		mirrorrp = self.datadir.append("current_mirror.%s.%s" %
										  (Time.curtimestr, "data"))
		Log("Touching mirror marker %s" % mirrorrp.path, 6)
		mirrorrp.touch()
		RPath.copy_attribs(rpin, rpout)


	def Restore(self, src_rp, dest_rp = None):
		"""Main restoring function

		Here src_rp should be an increment file, and if dest_rp is
		missing it defaults to the base of the increment.

		"""
		rpin, rpout = self.restore_check_paths(src_rp, dest_rp)
		time = Time.stringtotime(rpin.getinctime())
		self.restore_common(rpin, rpout, time)

	def RestoreAsOf(self, rpin, target):
		"""Secondary syntax for restore operation

		rpin - RPath of mirror file to restore (not nec. with correct index)
		target - RPath of place to put restored file

		"""
		self.restore_check_paths(rpin, target, 1)
		try: time = Time.genstrtotime(self.restore_timestr)
		except TimeException, exc: Log.FatalError(str(exc))
		self.restore_common(rpin, target, time)

	def restore_common(self, rpin, target, time):
		"""Restore operation common to Restore and RestoreAsOf"""
		Log("Starting Restore", 5)
		mirror_root, index = self.restore_get_root(rpin)
		mirror = mirror_root.new_index(index)
		inc_rpath = self.datadir.append_path('increments', index)
		self.restore_init_select(mirror_root, target)
		Log.open_logfile(self.datadir.append("restore.log"))
		Restore.Restore(inc_rpath, mirror, target, time)

	def restore_check_paths(self, rpin, rpout, restoreasof = None):
		"""Check paths and return pair of corresponding rps"""
		if not restoreasof:
			if not rpin.lstat():
				Log.FatalError("Source file %s does not exist" % rpin.path)
			elif not rpin.isincfile():
				Log.FatalError("""File %s does not look like an increment file.

Try restoring from an increment file (the filenames look like
"foobar.2001-09-01T04:49:04-07:00.diff").""" % rpin.path)

		if not rpout: rpout = RPath(Globals.local_connection,
									rpin.getincbase_str())
		if rpout.lstat():
			Log.FatalError("Restore target %s already exists, "
						   "and will not be overwritten." % rpout.path)
		return rpin, rpout

	def restore_init_select(self, rpin, rpout):
		"""Initialize Select

		Unlike the backup selections, here they are on the local
		connection, because the backup operation is pipelined in a way
		the restore operation isn't.

		"""
		Globals.set_select(DSRPath(1, rpin), self.select_mirror_opts, None)
		Globals.set_select(DSRPath(None, rpout), self.select_opts, None,
						   *self.select_files)

	def restore_get_root(self, rpin):
		"""Return (mirror root, index) and set the data dir

		The idea here is to keep backing up on the path until we find
		a directory that contains "rdiff-backup-data".  That is the
		mirror root.  If the path from there starts
		"rdiff-backup-data/increments*", then the index is the
		remainder minus that.  Otherwise the index is just the path
		minus the root.

		All this could fail if the increment file is pointed to in a
		funny way, using symlinks or somesuch.

		"""
		if rpin.isincfile(): relpath = rpin.getincbase().path
		else: relpath = rpin.path
		pathcomps = os.path.join(rpin.conn.os.getcwd(), relpath).split("/")
		assert len(pathcomps) >= 2 # path should be relative to /

		i = len(pathcomps)
		while i >= 2:
			parent_dir = RPath(rpin.conn, "/".join(pathcomps[:i]))
			if (parent_dir.isdir() and
				"rdiff-backup-data" in parent_dir.listdir()): break
			i = i-1
		else: Log.FatalError("Unable to find rdiff-backup-data directory")

		self.rootrp = rootrp = parent_dir
		Log("Using mirror root directory %s" % rootrp.path, 6)

		self.datadir = rootrp.append_path("rdiff-backup-data")
		SetConnections.UpdateGlobal('rbdir', self.datadir)
		if not self.datadir.isdir():
			Log.FatalError("Unable to read rdiff-backup-data directory %s" %
						   self.datadir.path)

		from_datadir = tuple(pathcomps[i:])
		if not from_datadir or from_datadir[0] != "rdiff-backup-data":
			return (rootrp, from_datadir) # in mirror, not increments
		assert from_datadir[1] == "increments"
		return (rootrp, from_datadir[2:])


	def ListIncrements(self, rp):
		"""Print out a summary of the increments and their times"""
		mirror_root, index = self.restore_get_root(rp)
		Globals.rbdir = datadir = \
						mirror_root.append_path("rdiff-backup-data")
		mirrorrp = mirror_root.new_index(index)
		inc_rpath = datadir.append_path('increments', index)
		incs = Restore.get_inclist(inc_rpath)
		mirror_time = Restore.get_mirror_time()
		if Globals.parsable_output:
			print Manage.describe_incs_parsable(incs, mirror_time, mirrorrp)
		else: print Manage.describe_incs_human(incs, mirror_time, mirrorrp)


	def CalculateAverage(self, rps):
		"""Print out the average of the given statistics files"""
		statobjs = map(lambda rp: StatsObj().read_stats_from_rp(rp), rps)
		average_stats = StatsObj().set_to_average(statobjs)
		print average_stats.get_stats_logstring(
			"Average of %d stat files" % len(rps))


	def RemoveOlderThan(self, rootrp):
		"""Remove all increment files older than a certain time"""
		datadir = rootrp.append("rdiff-backup-data")
		if not datadir.lstat() or not datadir.isdir():
			Log.FatalError("Unable to open rdiff-backup-data dir %s" %
						   (datadir.path,))

		try: time = Time.genstrtotime(self.remove_older_than_string)
		except TimeError, exc: Log.FatalError(str(exc))
		timep = Time.timetopretty(time)
		Log("Deleting increment(s) before %s" % timep, 4)

		itimes = [Time.stringtopretty(inc.getinctime())
				  for inc in Restore.get_inclist(datadir.append("increments"))
				  if Time.stringtotime(inc.getinctime()) < time]
		
		if not itimes:
			Log.FatalError("No increments older than %s found" % timep)
		inc_pretty_time = "\n".join(itimes)
		if len(itimes) > 1 and not self.force:
			Log.FatalError("Found %d relevant increments, dated:\n%s"
				"\nIf you want to delete multiple increments in this way, "
				"use the --force." % (len(itimes), inc_pretty_time))
			
		Log("Deleting increment%sat times:\n%s" %
			(len(itimes) == 1 and " " or "s ", inc_pretty_time), 3)
		Manage.delete_earlier_than(datadir, time)
		

Globals.Main = Main()
if __name__ == "__main__" and not globals().has_key('__no_execute__'):
	Globals.Main.Main(sys.argv[1:])


