X-Git-Url: https://mattmccutchen.net/rsync/rsync.git/blobdiff_plain/3b3a2fbcf058b90d9116f48ba26ad9528d450134..740819ef7b3b96451e16b2fa3891d46cfc73ec64:/flist.c

diff --git a/flist.c b/flist.c
index 40158fae..d9b0da7a 100644
--- a/flist.c
+++ b/flist.c
@@ -1,6 +1,7 @@
 /* 
    Copyright (C) Andrew Tridgell 1996
    Copyright (C) Paul Mackerras 1996
+   Copyright (C) 2001 by Martin Pool <mbp@samba.org>
    
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -21,12 +22,11 @@
 
 #include "rsync.h"
 
-extern int csum_length;
+extern struct stats stats;
 
 extern int verbose;
 extern int am_server;
 extern int always_checksum;
-extern off_t total_size;
 
 extern int cvs_exclude;
 
@@ -43,22 +43,132 @@ extern int preserve_gid;
 extern int preserve_times;
 extern int relative_paths;
 extern int copy_links;
+extern int copy_unsafe_links;
+extern int remote_version;
+extern int io_error;
+extern int sanitize_paths;
 
-static char **local_exclude_list = NULL;
+extern int read_batch;
+extern int write_batch;
 
-static void clean_fname(char *name);
+static char topsrcname[MAXPATHLEN];
 
+static struct exclude_struct **local_exclude_list;
 
-int link_stat(const char *Path, struct stat *Buffer) 
+static struct file_struct null_file;
+
+static void clean_flist(struct file_list *flist, int strip_root);
+
+static struct string_area *string_area_new(int size)
+{
+	struct string_area *a;
+
+	if (size <= 0) size = ARENA_SIZE;
+	a = malloc(sizeof(*a));
+	if (!a) out_of_memory("string_area_new");
+	a->current = a->base = malloc(size);
+	if (!a->current) out_of_memory("string_area_new buffer");
+	a->end = a->base + size;
+	a->next = NULL;
+
+	return a;
+}
+
+static void string_area_free(struct string_area *a)
+{
+	struct string_area *next;
+
+	for ( ; a ; a = next) {
+		next = a->next;
+		free(a->base);
+	}
+}
+
+static char *string_area_malloc(struct string_area **ap, int size)
+{
+	char *p;
+	struct string_area *a;
+
+	/* does the request fit into the current space? */
+	a = *ap;
+	if (a->current + size >= a->end) {
+		/* no; get space, move new string_area to front of the list */
+		a = string_area_new(size > ARENA_SIZE ? size : ARENA_SIZE);
+		a->next = *ap;
+		*ap = a;
+	}
+
+	/* have space; do the "allocation." */
+	p = a->current;
+	a->current += size;
+	return p;
+}
+
+static char *string_area_strdup(struct string_area **ap, const char *src)
+{
+	char* dest = string_area_malloc(ap, strlen(src) + 1);
+	return strcpy(dest, src);
+}
+
+static void list_file_entry(struct file_struct *f)
+{
+	char perms[11];
+
+	if (!f->basename)
+		/* this can happen if duplicate names were removed */
+		return;
+
+	permstring(perms, f->mode);
+
+	if (preserve_links && S_ISLNK(f->mode)) {
+		rprintf(FINFO,"%s %11.0f %s %s -> %s\n", 
+			perms, 
+			(double)f->length, timestring(f->modtime), 
+			f_name(f), f->link);
+	} else {
+		rprintf(FINFO,"%s %11.0f %s %s\n", 
+			perms, 
+			(double)f->length, timestring(f->modtime), f_name(f));
+	}
+}
+
+
+int readlink_stat(const char *Path, STRUCT_STAT *Buffer, char *Linkbuf) 
+{
+#if SUPPORT_LINKS
+	if (copy_links) {
+		return do_stat(Path, Buffer);
+	}
+	if (do_lstat(Path, Buffer) == -1) {
+		return -1;
+	}
+	if (S_ISLNK(Buffer->st_mode)) {
+		int l;
+		if ((l = readlink((char *) Path, Linkbuf, MAXPATHLEN-1))== -1) {
+			return -1;
+		}
+		Linkbuf[l] = 0;
+		if (copy_unsafe_links && (topsrcname[0] != '\0') &&
+				    unsafe_symlink(Linkbuf, topsrcname)) {
+			return do_stat(Path, Buffer);
+		}
+	}
+	return 0;
+#else
+	return do_stat(Path, Buffer);
+#endif
+}
+
+int link_stat(const char *Path, STRUCT_STAT *Buffer) 
 {
 #if SUPPORT_LINKS
     if (copy_links) {
-	return stat(Path, Buffer);
+	return do_stat(Path, Buffer);
     } else {
-	return lstat(Path, Buffer);
+	return do_lstat(Path, Buffer);
     }
 #else
-    return stat(Path, Buffer);
+    return do_stat(Path, Buffer);
 #endif
 }
 
@@ -66,14 +176,18 @@ int link_stat(const char *Path, struct stat *Buffer)
   This function is used to check if a file should be included/excluded
   from the list of files based on its name and type etc
  */
-static int match_file_name(char *fname,struct stat *st)
+static int check_exclude_file(int f,char *fname,STRUCT_STAT *st)
 {
-  if (check_exclude(fname,local_exclude_list)) {
-    if (verbose > 2)
-      fprintf(FERROR,"excluding file %s\n",fname);
-    return 0;
-  }
-  return 1;
+	extern int delete_excluded;
+
+	/* f is set to -1 when calculating deletion file list */
+	if ((f == -1) && delete_excluded) {
+		return 0;
+	}
+	if (check_exclude(fname,local_exclude_list,st)) {
+		return 1;
+	}
+	return 0;
 }
 
 /* used by the one_file_system code */
@@ -81,274 +195,444 @@ static dev_t filesystem_dev;
 
 static void set_filesystem(char *fname)
 {
-  struct stat st;
+  STRUCT_STAT st;
   if (link_stat(fname,&st) != 0) return;
   filesystem_dev = st.st_dev;
 }
 
 
-static void send_directory(int f,struct file_list *flist,char *dir);
+static int to_wire_mode(mode_t mode)
+{
+	if (S_ISLNK(mode) && (_S_IFLNK != 0120000)) {
+		return (mode & ~(_S_IFMT)) | 0120000;
+	}
+	return (int)mode;
+}
+
+static mode_t from_wire_mode(int mode)
+{
+	if ((mode & (_S_IFMT)) == 0120000 && (_S_IFLNK != 0120000)) {
+		return (mode & ~(_S_IFMT)) | _S_IFLNK;
+	}
+	return (mode_t)mode;
+}
 
-static char *flist_dir = NULL;
 
-extern void (*send_file_entry)(struct file_struct *file,int f);
-extern void (*receive_file_entry)(struct file_struct *file,
-				  unsigned char flags,int f);
+static void send_directory(int f,struct file_list *flist,char *dir);
+
+static char *flist_dir;
 
 
-void send_file_entry_v11(struct file_struct *file,int f)
+static void send_file_entry(struct file_struct *file,int f,unsigned base_flags)
 {
-  unsigned char flags;
-  static time_t last_time=0;
-  static mode_t last_mode=0;
-  static dev_t last_rdev=0;
-  static uid_t last_uid=0;
-  static gid_t last_gid=0;
-  static char lastname[MAXPATHLEN]="";
-  int l1,l2;
-
-  if (f == -1) return;
-
-  if (!file) {
-    write_byte(f,0);
-    return;
-  }
+	unsigned char flags;
+	static time_t last_time;
+	static mode_t last_mode;
+	static dev_t last_rdev;
+	static uid_t last_uid;
+	static gid_t last_gid;
+	static char lastname[MAXPATHLEN];
+	char *fname;
+	int l1,l2;
+
+	if (f == -1) return;
+
+	if (!file) {
+		write_byte(f,0);
+		return;
+	}
 
-  flags = FILE_VALID;
-
-  if (file->mode == last_mode) flags |= SAME_MODE;
-  if (file->rdev == last_rdev) flags |= SAME_RDEV;
-  if (file->uid == last_uid) flags |= SAME_UID;
-  if (file->gid == last_gid) flags |= SAME_GID;
-  if (file->modtime == last_time) flags |= SAME_TIME;
-
-  for (l1=0;lastname[l1] && file->name[l1] == lastname[l1];l1++) ;
-  l2 = strlen(file->name) - l1;
-
-  if (l1 > 0) flags |= SAME_NAME;
-  if (l2 > 255) flags |= LONG_NAME;
-
-  write_byte(f,flags);  
-  if (flags & SAME_NAME)
-    write_byte(f,l1);
-  if (flags & LONG_NAME)
-    write_int(f,l2);
-  else
-    write_byte(f,l2);
-  write_buf(f,file->name+l1,l2);
-
-  write_int(f,(int)file->length);
-  if (!(flags & SAME_TIME))
-    write_int(f,(int)file->modtime);
-  if (!(flags & SAME_MODE))
-    write_int(f,(int)file->mode);
-  if (preserve_uid && !(flags & SAME_UID))
-    write_int(f,(int)file->uid);
-  if (preserve_gid && !(flags & SAME_GID))
-    write_int(f,(int)file->gid);
-  if (preserve_devices && IS_DEVICE(file->mode) && !(flags & SAME_RDEV))
-    write_int(f,(int)file->rdev);
+	fname = f_name(file);
+
+	flags = base_flags;
+
+	if (file->mode == last_mode) flags |= SAME_MODE;
+	if (file->rdev == last_rdev) flags |= SAME_RDEV;
+	if (file->uid == last_uid) flags |= SAME_UID;
+	if (file->gid == last_gid) flags |= SAME_GID;
+	if (file->modtime == last_time) flags |= SAME_TIME;
+
+	for (l1=0;lastname[l1] && (fname[l1] == lastname[l1]) && (l1 < 255);l1++) ;  
+	l2 = strlen(fname) - l1;
+
+	if (l1 > 0) flags |= SAME_NAME;
+	if (l2 > 255) flags |= LONG_NAME;
+
+	/* we must make sure we don't send a zero flags byte or the other
+	   end will terminate the flist transfer */
+	if (flags == 0 && !S_ISDIR(file->mode)) flags |= FLAG_DELETE;
+	if (flags == 0) flags |= LONG_NAME;
+
+	write_byte(f,flags);  
+	if (flags & SAME_NAME)
+		write_byte(f,l1);
+	if (flags & LONG_NAME)
+		write_int(f,l2);
+	else
+		write_byte(f,l2);
+	write_buf(f,fname+l1,l2);
+
+	write_longint(f,file->length);
+	if (!(flags & SAME_TIME))
+		write_int(f,(int)file->modtime);
+	if (!(flags & SAME_MODE))
+		write_int(f,to_wire_mode(file->mode));
+	if (preserve_uid && !(flags & SAME_UID)) {
+		add_uid(file->uid);
+		write_int(f,(int)file->uid);
+	}
+	if (preserve_gid && !(flags & SAME_GID)) {
+		add_gid(file->gid);
+		write_int(f,(int)file->gid);
+	}
+	if (preserve_devices && IS_DEVICE(file->mode) && !(flags & SAME_RDEV))
+		write_int(f,(int)file->rdev);
 
 #if SUPPORT_LINKS
-  if (preserve_links && S_ISLNK(file->mode)) {
-    write_int(f,strlen(file->link));
-    write_buf(f,file->link,strlen(file->link));
-  }
+	if (preserve_links && S_ISLNK(file->mode)) {
+		write_int(f,strlen(file->link));
+		write_buf(f,file->link,strlen(file->link));
+	}
 #endif
 
 #if SUPPORT_HARD_LINKS
-  if (preserve_hard_links && S_ISREG(file->mode)) {
-    write_int(f,file->dev);
-    write_int(f,file->inode);
-  }
+	if (preserve_hard_links && S_ISREG(file->mode)) {
+		write_int(f,(int)file->dev);
+		write_int(f,(int)file->inode);
+	}
 #endif
 
-  if (always_checksum) {
-    write_buf(f,file->sum,csum_length);
-  }       
+	if (always_checksum) {
+		if (remote_version < 21) {
+			write_buf(f,file->sum,2);
+		} else {
+			write_buf(f,file->sum,MD4_SUM_LENGTH);
+		}
+	}       
 
-  last_mode = file->mode;
-  last_rdev = file->rdev;
-  last_uid = file->uid;
-  last_gid = file->gid;
-  last_time = file->modtime;
+	last_mode = file->mode;
+	last_rdev = file->rdev;
+	last_uid = file->uid;
+	last_gid = file->gid;
+	last_time = file->modtime;
 
-  strncpy(lastname,file->name,MAXPATHLEN-1);
-  lastname[MAXPATHLEN-1] = 0;
+	strlcpy(lastname,fname,MAXPATHLEN);
+	lastname[MAXPATHLEN-1] = 0;
 }
 
 
 
-void receive_file_entry_v11(struct file_struct *file,
-			    unsigned char flags,int f)
+static void receive_file_entry(struct file_struct **fptr,
+			       unsigned flags,int f)
 {
-  static time_t last_time=0;
-  static mode_t last_mode=0;
-  static dev_t last_rdev=0;
-  static uid_t last_uid=0;
-  static gid_t last_gid=0;
-  static char lastname[MAXPATHLEN]="";
-  int l1=0,l2=0;
-
-  if (flags & SAME_NAME)
-    l1 = read_byte(f);
+	static time_t last_time;
+	static mode_t last_mode;
+	static dev_t last_rdev;
+	static uid_t last_uid;
+	static gid_t last_gid;
+	static char lastname[MAXPATHLEN];
+	char thisname[MAXPATHLEN];
+	int l1=0,l2=0;
+	char *p;
+	struct file_struct *file;
+
+	if (flags & SAME_NAME)
+		l1 = read_byte(f);
   
-  if (flags & LONG_NAME)
-    l2 = read_int(f);
-  else
-    l2 = read_byte(f);
-
-  bzero((char *)file,sizeof(*file));
-
-  file->name = (char *)malloc(l1+l2+1);
-  if (!file->name) out_of_memory("receive_file_entry 1");
-
-  strncpy(file->name,lastname,l1);
-  read_buf(f,file->name+l1,l2);
-  file->name[l1+l2] = 0;
-
-  file->length = (off_t)read_int(f);
-  file->modtime = (flags & SAME_TIME) ? last_time : (time_t)read_int(f);
-  file->mode = (flags & SAME_MODE) ? last_mode : (mode_t)read_int(f);
-  if (preserve_uid)
-    file->uid = (flags & SAME_UID) ? last_uid : (uid_t)read_int(f);
-  if (preserve_gid)
-    file->gid = (flags & SAME_GID) ? last_gid : (gid_t)read_int(f);
-  if (preserve_devices && IS_DEVICE(file->mode))
-    file->rdev = (flags & SAME_RDEV) ? last_rdev : (dev_t)read_int(f);
-
-  if (preserve_links && S_ISLNK(file->mode)) {
-    int l = read_int(f);
-    file->link = (char *)malloc(l+1);
-    if (!file->link) out_of_memory("receive_file_entry 2");
-    read_buf(f,file->link,l);
-    file->link[l] = 0;
-  }
+	if (flags & LONG_NAME)
+		l2 = read_int(f);
+	else
+		l2 = read_byte(f);
+
+	file = (struct file_struct *)malloc(sizeof(*file));
+	if (!file) out_of_memory("receive_file_entry");
+	memset((char *)file, 0, sizeof(*file));
+	(*fptr) = file;
+
+	if (l2 >= MAXPATHLEN-l1) {
+		rprintf(FERROR,"overflow: flags=0x%x l1=%d l2=%d lastname=%s\n",
+			flags, l1, l2, lastname);
+		overflow("receive_file_entry");
+	}
+
+	strlcpy(thisname,lastname,l1+1);
+	read_sbuf(f,&thisname[l1],l2);
+	thisname[l1+l2] = 0;
+
+	strlcpy(lastname,thisname,MAXPATHLEN);
+	lastname[MAXPATHLEN-1] = 0;
+
+	clean_fname(thisname);
+
+	if (sanitize_paths) {
+		sanitize_path(thisname, NULL);
+	}
+
+	if ((p = strrchr(thisname,'/'))) {
+		static char *lastdir;
+		*p = 0;
+		if (lastdir && strcmp(thisname, lastdir)==0) {
+			file->dirname = lastdir;
+		} else {
+			file->dirname = strdup(thisname);
+			lastdir = file->dirname;
+		}
+		file->basename = strdup(p+1);
+	} else {
+		file->dirname = NULL;
+		file->basename = strdup(thisname);
+	}
+
+	if (!file->basename) out_of_memory("receive_file_entry 1");
+
+
+	file->flags = flags;
+	file->length = read_longint(f);
+	file->modtime = (flags & SAME_TIME) ? last_time : (time_t)read_int(f);
+	file->mode = (flags & SAME_MODE) ? last_mode : from_wire_mode(read_int(f));
+	if (preserve_uid)
+		file->uid = (flags & SAME_UID) ? last_uid : (uid_t)read_int(f);
+	if (preserve_gid)
+		file->gid = (flags & SAME_GID) ? last_gid : (gid_t)read_int(f);
+	if (preserve_devices && IS_DEVICE(file->mode))
+		file->rdev = (flags & SAME_RDEV) ? last_rdev : (dev_t)read_int(f);
+
+	if (preserve_links && S_ISLNK(file->mode)) {
+		int l = read_int(f);
+		file->link = (char *)malloc(l+1);
+		if (!file->link) out_of_memory("receive_file_entry 2");
+		read_sbuf(f,file->link,l);
+		if (sanitize_paths) {
+			sanitize_path(file->link, file->dirname);
+		}
+	}
 
 #if SUPPORT_HARD_LINKS
-  if (preserve_hard_links && S_ISREG(file->mode)) {
-    file->dev = read_int(f);
-    file->inode = read_int(f);
-  }
+	if (preserve_hard_links && S_ISREG(file->mode)) {
+		file->dev = read_int(f);
+		file->inode = read_int(f);
+	}
 #endif
   
-  if (always_checksum)
-    read_buf(f,file->sum,csum_length);
+	if (always_checksum) {
+		file->sum = (char *)malloc(MD4_SUM_LENGTH);
+		if (!file->sum) out_of_memory("md4 sum");
+		if (remote_version < 21) {
+			read_buf(f,file->sum,2);
+		} else {
+			read_buf(f,file->sum,MD4_SUM_LENGTH);
+		}
+	}
   
-  last_mode = file->mode;
-  last_rdev = file->rdev;
-  last_uid = file->uid;
-  last_gid = file->gid;
-  last_time = file->modtime;
-
-  strncpy(lastname,file->name,MAXPATHLEN-1);
-  lastname[MAXPATHLEN-1] = 0;
+	last_mode = file->mode;
+	last_rdev = file->rdev;
+	last_uid = file->uid;
+	last_gid = file->gid;
+	last_time = file->modtime;
+
+	if (!preserve_perms) {
+		extern int orig_umask;
+		/* set an appropriate set of permissions based on original
+		   permissions and umask. This emulates what GNU cp does */
+		file->mode &= ~orig_umask;
+	}
 }
 
 
+/* determine if a file in a different filesstem should be skipped
+   when one_file_system is set. We bascally only want to include
+   the mount points - but they can be hard to find! */
+static int skip_filesystem(char *fname, STRUCT_STAT *st)
+{
+	STRUCT_STAT st2;
+	char *p = strrchr(fname, '/');
+
+	/* skip all but directories */
+	if (!S_ISDIR(st->st_mode)) return 1;
+
+	/* if its not a subdirectory then allow */
+	if (!p) return 0;
 
-static struct file_struct *make_file(char *fname)
+	*p = 0;
+	if (link_stat(fname, &st2)) {
+		*p = '/';
+		return 0;
+	}
+	*p = '/';
+	
+	return (st2.st_dev != filesystem_dev);
+}
+
+#define STRDUP(ap, p)	(ap ? string_area_strdup(ap, p) : strdup(p))
+#define MALLOC(ap, i)	(ap ? string_area_malloc(ap, i) : malloc(i))
+
+/* create a file_struct for a named file */
+struct file_struct *make_file(int f, char *fname, struct string_area **ap,
+			      int noexcludes)
 {
-  static struct file_struct file;
-  struct stat st;
-  char sum[SUM_LENGTH];
+	struct file_struct *file;
+	STRUCT_STAT st;
+	char sum[SUM_LENGTH];
+	char *p;
+	char cleaned_name[MAXPATHLEN];
+	char linkbuf[MAXPATHLEN];
+	extern int module_id;
+
+	strlcpy(cleaned_name, fname, MAXPATHLEN);
+	cleaned_name[MAXPATHLEN-1] = 0;
+	clean_fname(cleaned_name);
+	if (sanitize_paths) {
+		sanitize_path(cleaned_name, NULL);
+	}
+	fname = cleaned_name;
+
+	memset(sum,0,SUM_LENGTH);
+
+	if (readlink_stat(fname,&st,linkbuf) != 0) {
+		int save_errno = errno;
+		if ((errno == ENOENT) && copy_links && !noexcludes) {
+			/* symlink pointing nowhere, see if excluded */
+			memset((char *)&st, 0, sizeof(st));
+			if (check_exclude_file(f,fname,&st)) {
+				/* file is excluded anyway, ignore silently */
+				return NULL;
+			}
+		}
+		io_error = 1;
+		rprintf(FERROR,"readlink %s: %s\n",
+			fname,strerror(save_errno));
+		return NULL;
+	}
 
-  bzero(sum,SUM_LENGTH);
+	/* we use noexcludes from backup.c */
+	if (noexcludes) goto skip_excludes;
 
-  if (link_stat(fname,&st) != 0) {
-    fprintf(FERROR,"%s: %s\n",
-	    fname,strerror(errno));
-    return NULL;
-  }
+	if (S_ISDIR(st.st_mode) && !recurse) {
+		rprintf(FINFO,"skipping directory %s\n",fname);
+		return NULL;
+	}
+	
+	if (one_file_system && st.st_dev != filesystem_dev) {
+		if (skip_filesystem(fname, &st))
+			return NULL;
+	}
+	
+	if (check_exclude_file(f,fname,&st))
+		return NULL;
 
-  if (S_ISDIR(st.st_mode) && !recurse) {
-    fprintf(FERROR,"skipping directory %s\n",fname);
-    return NULL;
-  }
 
-  if (one_file_system && st.st_dev != filesystem_dev)
-    return NULL;
+	if (lp_ignore_nonreadable(module_id) && access(fname, R_OK) != 0) 
+		return NULL;
 
-  if (!match_file_name(fname,&st))
-    return NULL;
+ skip_excludes:
 
-  if (verbose > 2)
-    fprintf(FERROR,"make_file(%s)\n",fname);
-
-  bzero((char *)&file,sizeof(file));
-
-  file.name = strdup(fname);
-  file.modtime = st.st_mtime;
-  file.length = st.st_size;
-  file.mode = st.st_mode;
-  file.uid = st.st_uid;
-  file.gid = st.st_gid;
-  file.dev = st.st_dev;
-  file.inode = st.st_ino;
+	if (verbose > 2)
+		rprintf(FINFO,"make_file(%d,%s)\n",f,fname);
+	
+	file = (struct file_struct *)malloc(sizeof(*file));
+	if (!file) out_of_memory("make_file");
+	memset((char *)file,0,sizeof(*file));
+
+	if ((p = strrchr(fname,'/'))) {
+		static char *lastdir;
+		*p = 0;
+		if (lastdir && strcmp(fname, lastdir)==0) {
+			file->dirname = lastdir;
+		} else {
+			file->dirname = strdup(fname);
+			lastdir = file->dirname;
+		}
+		file->basename = STRDUP(ap, p+1);
+		*p = '/';
+	} else {
+		file->dirname = NULL;
+		file->basename = STRDUP(ap, fname);
+	}
+
+	file->modtime = st.st_mtime;
+	file->length = st.st_size;
+	file->mode = st.st_mode;
+	file->uid = st.st_uid;
+	file->gid = st.st_gid;
+	file->dev = st.st_dev;
+	file->inode = st.st_ino;
 #ifdef HAVE_ST_RDEV
-  file.rdev = st.st_rdev;
+	file->rdev = st.st_rdev;
 #endif
 
 #if SUPPORT_LINKS
-  if (S_ISLNK(st.st_mode)) {
-    int l;
-    char lnk[MAXPATHLEN];
-    if ((l=readlink(fname,lnk,MAXPATHLEN-1)) == -1) {
-      fprintf(FERROR,"readlink %s : %s\n",fname,strerror(errno));
-      return NULL;
-    }
-    lnk[l] = 0;
-    file.link = strdup(lnk);
-  }
+	if (S_ISLNK(st.st_mode)) {
+		file->link = STRDUP(ap, linkbuf);
+	}
 #endif
 
-  if (always_checksum && S_ISREG(st.st_mode)) {
-    file_checksum(fname,file.sum,st.st_size);
-  }       
+	if (always_checksum) {
+		file->sum = (char *)MALLOC(ap, MD4_SUM_LENGTH);
+		if (!file->sum) out_of_memory("md4 sum");
+		/* drat. we have to provide a null checksum for non-regular
+		   files in order to be compatible with earlier versions
+		   of rsync */
+		if (S_ISREG(st.st_mode)) {
+			file_checksum(fname,file->sum,st.st_size);
+		} else {
+			memset(file->sum, 0, MD4_SUM_LENGTH);
+		}
+	}       
 
-  if (flist_dir)
-    file.dir = strdup(flist_dir);
-  else
-    file.dir = NULL;
+	if (flist_dir) {
+		static char *lastdir;
+		if (lastdir && strcmp(lastdir, flist_dir)==0) {
+			file->basedir = lastdir;
+		} else {
+			file->basedir = strdup(flist_dir);
+			lastdir = file->basedir;
+		}
+	} else {
+		file->basedir = NULL;
+	}
 
-  if (!S_ISDIR(st.st_mode))
-    total_size += st.st_size;
+	if (!S_ISDIR(st.st_mode))
+		stats.total_size += st.st_size;
 
-  return &file;
+	return file;
 }
 
 
 
-static void send_file_name(int f,struct file_list *flist,char *fname)
+void send_file_name(int f,struct file_list *flist,char *fname,
+			   int recursive, unsigned base_flags)
 {
   struct file_struct *file;
 
-  file = make_file(fname);
+  file = make_file(f,fname, &flist->string_area, 0);
 
   if (!file) return;  
   
   if (flist->count >= flist->malloced) {
-	  if (flist->malloced < 100)
-		  flist->malloced += 100;
+	  if (flist->malloced < 1000)
+		  flist->malloced += 1000;
 	  else
-		  flist->malloced *= 1.8;
-	  flist->files = (struct file_struct *)realloc(flist->files,
-						       sizeof(flist->files[0])*
-						       flist->malloced);
+		  flist->malloced *= 2;
+	  flist->files = (struct file_struct **)realloc(flist->files,
+							sizeof(flist->files[0])*
+							flist->malloced);
 	  if (!flist->files)
 		  out_of_memory("send_file_name");
   }
 
-  if (strcmp(file->name,"/")) {
-    flist->files[flist->count++] = *file;    
-    send_file_entry(file,f);
+  if (write_batch) /*  dw  */
+    file->flags = FLAG_DELETE;
+
+  if (strcmp(file->basename,"")) {
+    flist->files[flist->count++] = file;
+    send_file_entry(file,f,base_flags);
   }
 
-  if (S_ISDIR(file->mode) && recurse) {
-    char **last_exclude_list = local_exclude_list;
-    send_directory(f,flist,file->name);
-    local_exclude_list = last_exclude_list;
-    return;
+  if (S_ISDIR(file->mode) && recursive) {
+	  struct exclude_struct **last_exclude_list = local_exclude_list;
+	  send_directory(f,flist,f_name(file));
+	  local_exclude_list = last_exclude_list;
+	  return;
   }
 }
 
@@ -356,153 +640,225 @@ static void send_file_name(int f,struct file_list *flist,char *fname)
 
 static void send_directory(int f,struct file_list *flist,char *dir)
 {
-  DIR *d;
-  struct dirent *di;
-  char fname[MAXPATHLEN];
-  int l;
-  char *p;
-
-  d = opendir(dir);
-  if (!d) {
-    fprintf(FERROR,"%s: %s\n",
-	    dir,strerror(errno));
-    return;
-  }
+	DIR *d;
+	struct dirent *di;
+	char fname[MAXPATHLEN];
+	int l;
+	char *p;
+
+	d = opendir(dir);
+	if (!d) {
+		io_error = 1;
+		rprintf(FERROR,"opendir(%s): %s\n",
+			dir,strerror(errno));
+		return;
+	}
 
-  strncpy(fname,dir,MAXPATHLEN-1);
-  fname[MAXPATHLEN-1]=0;
-  l = strlen(fname);
-  if (fname[l-1] != '/') {
-        if (l == MAXPATHLEN-1) {
-              fprintf(FERROR,"skipping long-named directory %s\n",fname);
-              closedir(d);
-              return;
-        }
-	  strcat(fname,"/");
-	  l++;
-  }
-  p = fname + strlen(fname);
+	strlcpy(fname,dir,MAXPATHLEN);
+	l = strlen(fname);
+	if (fname[l-1] != '/') {
+		if (l == MAXPATHLEN-1) {
+			io_error = 1;
+			rprintf(FERROR,"skipping long-named directory %s\n",fname);
+			closedir(d);
+			return;
+		}
+		strlcat(fname,"/", MAXPATHLEN);
+		l++;
+	}
+	p = fname + strlen(fname);
 
-  if (cvs_exclude) {
-    if (strlen(fname) + strlen(".cvsignore") <= MAXPATHLEN-1) {
-      strcpy(p,".cvsignore");
-      local_exclude_list = make_exclude_list(fname,NULL,0);
-    } else {
-      fprintf(FERROR,"cannot cvs-exclude in long-named directory %s\n",fname);
-    }
-  }  
-
-  for (di=readdir(d); di; di=readdir(d)) {
-    if (strcmp(di->d_name,".")==0 ||
-	strcmp(di->d_name,"..")==0)
-      continue;
-    strncpy(p,di->d_name,MAXPATHLEN-(l+1));
-    send_file_name(f,flist,fname);
-  }
+	local_exclude_list = NULL;
 
-  closedir(d);
-}
+	if (cvs_exclude) {
+		if (strlen(fname) + strlen(".cvsignore") <= MAXPATHLEN-1) {
+			strcpy(p,".cvsignore");
+			local_exclude_list = make_exclude_list(fname,NULL,0,0);
+		} else {
+			io_error = 1;
+			rprintf(FINFO,"cannot cvs-exclude in long-named directory %s\n",fname);
+		}
+	}  
+	
+	for (di=readdir(d); di; di=readdir(d)) {
+		char *dname = d_name(di);
+		if (strcmp(dname,".")==0 ||
+		    strcmp(dname,"..")==0)
+			continue;
+		strlcpy(p,dname,MAXPATHLEN-l);
+		send_file_name(f,flist,fname,recurse,0);
+	}
+
+	if (local_exclude_list) {
+		add_exclude_list("!", &local_exclude_list, 0);
+	}
 
+	closedir(d);
+}
 
 
 struct file_list *send_file_list(int f,int argc,char *argv[])
 {
-  int i,l;
-  struct stat st;
-  char *p,*dir;
-  char dbuf[MAXPATHLEN];
-  struct file_list *flist;
-
-  if (verbose && recurse && !am_server && f != -1) {
-    fprintf(FINFO,"building file list ... ");
-    fflush(FINFO);
-  }
+	int i,l;
+	STRUCT_STAT st;
+	char *p,*dir,*olddir;
+	char lastpath[MAXPATHLEN]="";
+	struct file_list *flist;
+	int64 start_write;
+
+	if (verbose && recurse && !am_server && f != -1) {
+		rprintf(FINFO,"building file list ... ");
+                if (verbose > 1)
+                        rprintf(FINFO, "\n");
+		rflush(FINFO);
+	}
 
-  flist = (struct file_list *)malloc(sizeof(flist[0]));
-  if (!flist) out_of_memory("send_file_list");
+	start_write = stats.total_written;
 
-  flist->count=0;
-  flist->malloced = 100;
-  flist->files = (struct file_struct *)malloc(sizeof(flist->files[0])*
-					      flist->malloced);
-  if (!flist->files) out_of_memory("send_file_list");
+	flist = flist_new();
 
-  for (i=0;i<argc;i++) {
-    char fname2[MAXPATHLEN];
-    char *fname = fname2;
+	if (f != -1) {
+		io_start_buffering(f);
+	}
 
-    strncpy(fname,argv[i],MAXPATHLEN-1);
-    fname[MAXPATHLEN-1] = 0;
+	for (i=0;i<argc;i++) {
+		char *fname = topsrcname;
+
+		strlcpy(fname,argv[i],MAXPATHLEN);
+
+		l = strlen(fname);
+		if (l != 1 && fname[l-1] == '/') {
+			if ((l == 2) && (fname[0] == '.')) {
+				/*  Turn ./ into just . rather than ./.
+				    This was put in to avoid a problem with
+				      rsync -aR --delete from ./
+				    The send_file_name() below of ./ was
+				    mysteriously preventing deletes */
+				fname[1] = 0;
+			} else {
+				strlcat(fname,".",MAXPATHLEN);
+			}
+		}
 
-    l = strlen(fname);
-    if (l != 1 && fname[l-1] == '/') {
-      strcat(fname,".");
-    }
+		if (link_stat(fname,&st) != 0) {
+			if (f != -1) {
+				io_error=1;
+				rprintf(FERROR,"link_stat %s : %s\n",fname,strerror(errno));
+			}
+			continue;
+		}
 
-    if (link_stat(fname,&st) != 0) {
-      fprintf(FERROR,"%s : %s\n",fname,strerror(errno));
-      continue;
-    }
+		if (S_ISDIR(st.st_mode) && !recurse) {
+			rprintf(FINFO,"skipping directory %s\n",fname);
+			continue;
+		}
 
-    if (S_ISDIR(st.st_mode) && !recurse) {
-      fprintf(FERROR,"skipping directory %s\n",fname);
-      continue;
-    }
+		dir = NULL;
+		olddir = NULL;
+
+		if (!relative_paths) {
+			p = strrchr(fname,'/');
+			if (p) {
+				*p = 0;
+				if (p == fname) 
+					dir = "/";
+				else
+					dir = fname;      
+				fname = p+1;      
+			}
+		} else if (f != -1 && (p=strrchr(fname,'/'))) {
+			/* this ensures we send the intermediate directories,
+			   thus getting their permissions right */
+			*p = 0;
+			if (strcmp(lastpath,fname)) {
+				strlcpy(lastpath, fname, sizeof(lastpath));
+				*p = '/';
+				for (p=fname+1; (p=strchr(p,'/')); p++) {
+					int copy_links_saved = copy_links;
+					int recurse_saved = recurse;
+					*p = 0;
+					copy_links = copy_unsafe_links;
+					/* set recurse to 1 to prevent make_file
+					   from ignoring directory, but still
+					   turn off the recursive parameter to
+					   send_file_name */
+					recurse = 1;
+					send_file_name(f, flist, fname, 0, 0);
+					copy_links = copy_links_saved;
+					recurse = recurse_saved;
+					*p = '/';
+				}
+			} else {
+				*p = '/';
+			}
+		}
+		
+		if (!*fname)
+			fname = ".";
+		
+		if (dir && *dir) {
+			olddir = push_dir(dir, 1);
+
+			if (!olddir) {
+				io_error=1;
+				rprintf(FERROR,"push_dir %s : %s\n",
+					dir,strerror(errno));
+				continue;
+			}
+
+			flist_dir = dir;
+		}
+		
+		if (one_file_system)
+			set_filesystem(fname);
+
+		send_file_name(f,flist,fname,recurse,FLAG_DELETE);
+
+		if (olddir != NULL) {
+			flist_dir = NULL;
+			if (pop_dir(olddir) != 0) {
+				rprintf(FERROR,"pop_dir %s : %s\n",
+					dir,strerror(errno));
+				exit_cleanup(RERR_FILESELECT);
+			}
+		}
+	}
 
-    dir = NULL;
-
-    if (!relative_paths) {
-	    p = strrchr(fname,'/');
-	    if (p) {
-		    *p = 0;
-		    if (p == fname) 
-			    dir = "/";
-		    else
-			    dir = fname;      
-		    fname = p+1;      
-	    }
-    }
+	topsrcname[0] = '\0';
 
-    if (!*fname)
-      fname = ".";
-
-    if (dir && *dir) {
-      if (getcwd(dbuf,MAXPATHLEN-1) == NULL) {
-	fprintf(FERROR,"getwd : %s\n",strerror(errno));
-	exit_cleanup(1);
-      }
-      if (chdir(dir) != 0) {
-	fprintf(FERROR,"chdir %s : %s\n",dir,strerror(errno));
-	continue;
-      }
-      flist_dir = dir;
-      if (one_file_system)
-	set_filesystem(fname);
-      send_file_name(f,flist,fname);
-      flist_dir = NULL;
-      if (chdir(dbuf) != 0) {
-	fprintf(FERROR,"chdir %s : %s\n",dbuf,strerror(errno));
-	exit_cleanup(1);
-      }
-      continue;
-    }
+	if (f != -1) {
+		send_file_entry(NULL,f,0);
+	}
 
-    if (one_file_system)
-      set_filesystem(fname);
-    send_file_name(f,flist,fname);
-  }
+	if (verbose && recurse && !am_server && f != -1)
+		rprintf(FINFO,"done\n");
+	
+	clean_flist(flist, 0);
+	
+	/* now send the uid/gid list. This was introduced in protocol
+           version 15 */
+	if (f != -1 && remote_version >= 15) {
+		send_uid_list(f);
+	}
 
-  if (f != -1) {
-    send_file_entry(NULL,f);
-    write_flush(f);
-  }
+	/* if protocol version is >= 17 then send the io_error flag */
+	if (f != -1 && remote_version >= 17) {
+		extern int module_id;
+		write_int(f, lp_ignore_errors(module_id)? 0 : io_error);
+	}
 
-  if (verbose && recurse && !am_server && f != -1)
-    fprintf(FINFO,"done\n");
+	if (f != -1) {
+		io_end_buffering(f);
+		stats.flist_size = stats.total_written - start_write;
+		stats.num_files = flist->count;
+		if (write_batch) /*  dw  */
+		    write_batch_flist_info(flist->count, flist->files);
+	}
 
-  clean_flist(flist);
+	if (verbose > 2)
+		rprintf(FINFO,"send_file_list done\n");
 
-  return flist;
+	return flist;
 }
 
 
@@ -510,20 +866,24 @@ struct file_list *recv_file_list(int f)
 {
   struct file_list *flist;
   unsigned char flags;
+  int64 start_read;
+  extern int list_only;
 
   if (verbose && recurse && !am_server) {
-    fprintf(FINFO,"receiving file list ... ");
-    fflush(FINFO);
+    rprintf(FINFO,"receiving file list ... ");
+    rflush(FINFO);
   }
 
+  start_read = stats.total_read;
+
   flist = (struct file_list *)malloc(sizeof(flist[0]));
   if (!flist)
     goto oom;
 
   flist->count=0;
-  flist->malloced=100;
-  flist->files = (struct file_struct *)malloc(sizeof(flist->files[0])*
-					      flist->malloced);
+  flist->malloced=1000;
+  flist->files = (struct file_struct **)malloc(sizeof(flist->files[0])*
+					       flist->malloced);
   if (!flist->files)
     goto oom;
 
@@ -532,38 +892,68 @@ struct file_list *recv_file_list(int f)
     int i = flist->count;
 
     if (i >= flist->malloced) {
-	  if (flist->malloced < 100)
-		  flist->malloced += 100;
+	  if (flist->malloced < 1000)
+		  flist->malloced += 1000;
 	  else
-		  flist->malloced *= 1.8;
-	  flist->files =(struct file_struct *)realloc(flist->files,
-						      sizeof(flist->files[0])*
-						      flist->malloced);
+		  flist->malloced *= 2;
+	  flist->files =(struct file_struct **)realloc(flist->files,
+						       sizeof(flist->files[0])*
+						       flist->malloced);
 	  if (!flist->files)
 		  goto oom;
     }
 
     receive_file_entry(&flist->files[i],flags,f);
 
-    if (S_ISREG(flist->files[i].mode))
-      total_size += flist->files[i].length;
+    if (S_ISREG(flist->files[i]->mode))
+	    stats.total_size += flist->files[i]->length;
 
     flist->count++;
 
     if (verbose > 2)
-      fprintf(FERROR,"recv_file_name(%s)\n",flist->files[i].name);
+      rprintf(FINFO,"recv_file_name(%s)\n",f_name(flist->files[i]));
   }
 
 
   if (verbose > 2)
-    fprintf(FERROR,"received %d names\n",flist->count);
+    rprintf(FINFO,"received %d names\n",flist->count);
 
-  clean_flist(flist);
+  clean_flist(flist, relative_paths);
 
   if (verbose && recurse && !am_server) {
-    fprintf(FINFO,"done\n");
+    rprintf(FINFO,"done\n");
+  }
+
+  /* now recv the uid/gid list. This was introduced in protocol version 15 */
+  if (f != -1 && remote_version >= 15) {
+	  recv_uid_list(f, flist);
+  }
+
+  /* if protocol version is >= 17 then recv the io_error flag */
+  if (f != -1 && remote_version >= 17  && !read_batch) {  /* dw-added readbatch */
+	  extern int module_id;
+	  extern int ignore_errors;
+	  if (lp_ignore_errors(module_id) || ignore_errors) {
+		  read_int(f);
+	  } else {
+		  io_error |= read_int(f);
+	  }
+  }
+
+  if (list_only) {
+	  int i;
+	  for (i=0;i<flist->count;i++) {
+		  list_file_entry(flist->files[i]);
+	  }
   }
 
+
+  if (verbose > 2)
+    rprintf(FINFO,"recv_file_list done\n");
+
+  stats.flist_size = stats.total_read - start_read;
+  stats.num_files = flist->count;
+
   return flist;
 
 oom:
@@ -572,12 +962,14 @@ oom:
 }
 
 
-int file_compare(struct file_struct *f1,struct file_struct *f2)
+int file_compare(struct file_struct **f1,struct file_struct **f2)
 {
-  if (!f1->name && !f2->name) return 0;
-  if (!f1->name) return -1;
-  if (!f2->name) return 1;
-  return strcmp(f1->name,f2->name);
+	if (!(*f1)->basename && !(*f2)->basename) return 0;
+	if (!(*f1)->basename) return -1;
+	if (!(*f2)->basename) return 1;
+	if ((*f1)->dirname == (*f2)->dirname)
+		return u_strcmp((*f1)->basename, (*f2)->basename);
+	return u_strcmp(f_name(*f1),f_name(*f2));
 }
 
 
@@ -589,7 +981,7 @@ int flist_find(struct file_list *flist,struct file_struct *f)
 
 	while (low != high) {
 		int mid = (low+high)/2;
-		int ret = file_compare(&flist->files[flist_up(flist, mid)],f);
+		int ret = file_compare(&flist->files[flist_up(flist, mid)],&f);
 		if (ret == 0) return flist_up(flist, mid);
 		if (ret > 0) {
 			high=mid;
@@ -598,83 +990,154 @@ int flist_find(struct file_list *flist,struct file_struct *f)
 		}
 	}
 
-	if (file_compare(&flist->files[flist_up(flist,low)],f) == 0)
+	if (file_compare(&flist->files[flist_up(flist,low)],&f) == 0)
 		return flist_up(flist,low);
 	return -1;
 }
 
 
-static void clean_fname(char *name)
+/*
+ * free up one file
+ */
+void free_file(struct file_struct *file)
 {
-  char *p;
-  int l;
-  int modified = 1;
+	if (!file) return;
+	if (file->basename) free(file->basename);
+	if (file->link) free(file->link);
+	if (file->sum) free(file->sum);
+	*file = null_file;
+}
 
-  if (!name) return;
 
-  while (modified) {
-    modified = 0;
+/*
+ * allocate a new file list
+ */
+struct file_list *flist_new()
+{
+	struct file_list *flist;
+
+	flist = (struct file_list *)malloc(sizeof(flist[0]));
+	if (!flist) out_of_memory("send_file_list");
+
+	flist->count=0;
+	flist->malloced = 1000;
+	flist->files = (struct file_struct **)malloc(sizeof(flist->files[0])*
+						     flist->malloced);
+	if (!flist->files) out_of_memory("send_file_list");
+#if ARENA_SIZE > 0
+	flist->string_area = string_area_new(0);
+#else
+	flist->string_area = NULL;
+#endif
+	return flist;
+}
+/*
+ * free up all elements in a flist
+ */
+void flist_free(struct file_list *flist)
+{
+	int i;
+	for (i=1;i<flist->count;i++) {
+		if (!flist->string_area)
+			free_file(flist->files[i]);
+		free(flist->files[i]);
+	}	
+	memset((char *)flist->files, 0, sizeof(flist->files[0])*flist->count);
+	free(flist->files);
+	if (flist->string_area)
+		string_area_free(flist->string_area);
+	memset((char *)flist, 0, sizeof(*flist));
+	free(flist);
+}
 
-    if ((p=strstr(name,"/./"))) {
-      modified = 1;
-      while (*p) {
-	p[0] = p[2];
-	p++;
-      }
-    }
 
-    if ((p=strstr(name,"//"))) {
-      modified = 1;
-      while (*p) {
-	p[0] = p[1];
-	p++;
-      }
-    }
+/*
+ * This routine ensures we don't have any duplicate names in our file list.
+ * duplicate names can cause corruption because of the pipelining 
+ */
+static void clean_flist(struct file_list *flist, int strip_root)
+{
+	int i;
 
-    if (strncmp(p=name,"./",2) == 0) {      
-      modified = 1;
-      do {
-	p[0] = p[2];
-      } while (*p++);
-    }
+	if (!flist || flist->count == 0) 
+		return;
+  
+	qsort(flist->files,flist->count,
+	      sizeof(flist->files[0]),
+	      (int (*)())file_compare);
+
+	for (i=1;i<flist->count;i++) {
+		if (flist->files[i]->basename &&
+		    flist->files[i-1]->basename &&
+		    strcmp(f_name(flist->files[i]),
+			   f_name(flist->files[i-1])) == 0) {
+			if (verbose > 1 && !am_server)
+				rprintf(FINFO,"removing duplicate name %s from file list %d\n",
+					f_name(flist->files[i-1]),i-1);
+			/* it's not great that the flist knows the semantics of the
+			 * file memory usage, but i'd rather not add a flag byte
+			 * to that struct. XXX can i use a bit in the flags field? */
+			if (flist->string_area)
+				flist->files[i][0] = null_file;
+			else
+				free_file(flist->files[i]);
+		} 
+	}
 
-    l = strlen(p=name);
-    if (l > 1 && p[l-1] == '/') {
-      modified = 1;
-      p[l-1] = 0;
-    }
-  }
+	if (strip_root) {
+		/* we need to strip off the root directory in the case
+		   of relative paths, but this must be done _after_
+		   the sorting phase */
+		for (i=0;i<flist->count;i++) {
+			if (flist->files[i]->dirname &&
+			    flist->files[i]->dirname[0] == '/') {
+				memmove(&flist->files[i]->dirname[0],
+					&flist->files[i]->dirname[1],
+					strlen(flist->files[i]->dirname));
+			}
+			
+			if (flist->files[i]->dirname && 
+			    !flist->files[i]->dirname[0]) {
+				flist->files[i]->dirname = NULL;
+			}
+		}
+	}
+
+
+	if (verbose <= 3) return;
+
+	for (i=0;i<flist->count;i++) {
+		rprintf(FINFO,"[%d] i=%d %s %s mode=0%o len=%.0f\n",
+			(int) getpid(), i, 
+			NS(flist->files[i]->dirname),
+			NS(flist->files[i]->basename),
+			(int) flist->files[i]->mode,
+			(double)flist->files[i]->length);
+	}
 }
 
 
 /*
- * This routine ensures we don't have any duplicate names in our file list.
- * duplicate names can cause corruption because of the pipelining 
+ * return the full filename of a flist entry
  */
-void clean_flist(struct file_list *flist)
+char *f_name(struct file_struct *f)
 {
-  int i;
+	static char names[10][MAXPATHLEN];
+	static int n;
+	char *p = names[n];
 
-  if (!flist || flist->count == 0) 
-    return;
-  
-  for (i=0;i<flist->count;i++) {
-    clean_fname(flist->files[i].name);
-  }
-      
-  qsort(flist->files,flist->count,
-	sizeof(flist->files[0]),
-	(int (*)())file_compare);
-
-  for (i=1;i<flist->count;i++) {
-    if (flist->files[i].name &&
-	strcmp(flist->files[i].name,flist->files[i-1].name) == 0) {
-      if (verbose > 1 && !am_server)
-	fprintf(FERROR,"removing duplicate name %s from file list %d\n",
-		flist->files[i-1].name,i-1);
-      free(flist->files[i-1].name);
-      bzero((char *)&flist->files[i-1],sizeof(flist->files[i-1]));
-    } 
-  }
+	if (!f || !f->basename) return NULL;
+
+	n = (n+1)%10;
+
+	if (f->dirname) {
+		strlcpy(p, f->dirname, MAXPATHLEN);
+		strlcat(p, "/", MAXPATHLEN);
+		strlcat(p, f->basename, MAXPATHLEN);
+	} else {
+		strlcpy(p, f->basename, MAXPATHLEN);
+	}
+
+	return p;
 }