From f9df736a3e22f15307f9e304eb9069b06e94208f Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Sat, 1 Mar 2008 20:16:09 +0000 Subject: [PATCH] The patches for 3.0.0. --- acls.diff | 12 +- adaptec_acl_mods.diff | 8 +- atimes.diff | 43 +- backup-dir-dels.diff | 4 +- checksum-reading.diff | 688 ++++++++++++++++++++---- checksum-updating.diff | 884 +++++++++++++++++-------------- checksum-xattr.diff | 4 +- copy-devices.diff | 2 +- crtimes.diff | 159 +++--- date-only.diff | 2 +- detect-renamed-lax.diff | 2 +- detect-renamed.diff | 74 ++- downdate.diff | 6 +- drop-cache.diff | 2 +- fileflags.diff | 934 +++++++++++++++++++++++++-------- ignore-case.diff | 4 +- link-by-hash.diff | 4 +- log-checksum.diff | 2 +- nameconverter.diff | 6 +- omit-dir-changes.diff | 6 +- openssl-support.diff | 14 +- osx-create-time.diff | 2 +- preallocate.diff | 163 ++++-- slow-down.diff | 4 +- slp.diff | 2 +- source-filter_dest-filter.diff | 4 +- time-limit.diff | 2 +- transliterate.diff | 8 +- usermap.diff | 8 +- 29 files changed, 2113 insertions(+), 940 deletions(-) diff --git a/acls.diff b/acls.diff index a7a7aad..81d45fa 100644 --- a/acls.diff +++ b/acls.diff @@ -20,8 +20,8 @@ diff --git a/acls.c b/acls.c /* Flags used to indicate what items are being transmitted for an entry. */ #define XMIT_USER_OBJ (1<<0) -@@ -97,6 +98,18 @@ static const char *str_acl_type(SMB_ACL_TYPE_T type) - : "unknown SMB_ACL_TYPE_T"; +@@ -107,6 +108,18 @@ static const char *str_acl_type(SMB_ACL_TYPE_T type) + return "unknown ACL type!"; } +#define OTHER_TYPE(t) (SMB_ACL_TYPE_ACCESS+SMB_ACL_TYPE_DEFAULT-(t)) @@ -39,7 +39,7 @@ diff --git a/acls.c b/acls.c static int calc_sacl_entries(const rsync_acl *racl) { /* A System ACL always gets user/group/other permission entries. */ -@@ -544,6 +557,96 @@ int get_acl(const char *fname, stat_x *sxp) +@@ -554,6 +567,96 @@ int get_acl(const char *fname, stat_x *sxp) return 0; } @@ -136,7 +136,7 @@ diff --git a/acls.c b/acls.c /* === Send functions === */ /* Send the ida list over the file descriptor. */ -@@ -619,6 +722,11 @@ static void send_rsync_acl(rsync_acl *racl, SMB_ACL_TYPE_T type, +@@ -629,6 +732,11 @@ static void send_rsync_acl(rsync_acl *racl, SMB_ACL_TYPE_T type, * This also frees the ACL data. */ void send_acl(stat_x *sxp, int f) { @@ -148,7 +148,7 @@ diff --git a/acls.c b/acls.c if (!sxp->acc_acl) { sxp->acc_acl = create_racl(); rsync_acl_fake_perms(sxp->acc_acl, sxp->st.st_mode); -@@ -636,6 +744,160 @@ void send_acl(stat_x *sxp, int f) +@@ -646,6 +754,160 @@ void send_acl(stat_x *sxp, int f) } } @@ -309,7 +309,7 @@ diff --git a/acls.c b/acls.c /* === Receive functions === */ static uint32 recv_acl_access(uchar *name_follows_ptr, int f) -@@ -758,6 +1020,11 @@ static int recv_rsync_acl(item_list *racl_list, SMB_ACL_TYPE_T type, int f) +@@ -768,6 +1030,11 @@ static int recv_rsync_acl(item_list *racl_list, SMB_ACL_TYPE_T type, int f) /* Receive the ACL info the sender has included for this file-list entry. */ void receive_acl(struct file_struct *file, int f) { diff --git a/adaptec_acl_mods.diff b/adaptec_acl_mods.diff index c79da0a..d668246 100644 --- a/adaptec_acl_mods.diff +++ b/adaptec_acl_mods.diff @@ -27,7 +27,7 @@ superfluous ACL info. diff --git a/lib/sysacls.c b/lib/sysacls.c --- a/lib/sysacls.c +++ b/lib/sysacls.c -@@ -30,6 +30,18 @@ +@@ -31,6 +31,18 @@ #endif #define DEBUG(x,y) @@ -46,7 +46,7 @@ diff --git a/lib/sysacls.c b/lib/sysacls.c void SAFE_FREE(void *mem) { if (mem) -@@ -99,6 +111,9 @@ int sys_acl_get_info(SMB_ACL_ENTRY_T entry, SMB_ACL_TAG_T *tag_type_p, uint32 *b +@@ -100,6 +112,9 @@ int sys_acl_get_info(SMB_ACL_ENTRY_T entry, SMB_ACL_TAG_T *tag_type_p, uint32 *b return -1; *bits_p = (acl_get_perm(permset, ACL_READ) ? 4 : 0) @@ -56,7 +56,7 @@ diff --git a/lib/sysacls.c b/lib/sysacls.c | (acl_get_perm(permset, ACL_WRITE) ? 2 : 0) | (acl_get_perm(permset, ACL_EXECUTE) ? 1 : 0); -@@ -143,6 +158,12 @@ int sys_acl_set_access_bits(SMB_ACL_ENTRY_T entry, uint32 bits) +@@ -144,6 +159,12 @@ int sys_acl_set_access_bits(SMB_ACL_ENTRY_T entry, uint32 bits) if ((rc = acl_get_permset(entry, &permset)) != 0) return rc; acl_clear_perms(permset); @@ -72,7 +72,7 @@ diff --git a/lib/sysacls.c b/lib/sysacls.c diff --git a/lib/sysacls.h b/lib/sysacls.h --- a/lib/sysacls.h +++ b/lib/sysacls.h -@@ -58,8 +58,8 @@ +@@ -59,8 +59,8 @@ #define SMB_ACL_TYPE_ACCESS ACL_TYPE_ACCESS #define SMB_ACL_TYPE_DEFAULT ACL_TYPE_DEFAULT diff --git a/atimes.diff b/atimes.diff index 66b8404..9b60c91 100644 --- a/atimes.diff +++ b/atimes.diff @@ -36,7 +36,7 @@ diff --git a/compat.c b/compat.c diff --git a/flist.c b/flist.c --- a/flist.c +++ b/flist.c -@@ -53,6 +53,7 @@ extern int preserve_specials; +@@ -55,6 +55,7 @@ extern int preserve_specials; extern int uid_ndx; extern int gid_ndx; extern int eol_nulls; @@ -44,7 +44,7 @@ diff --git a/flist.c b/flist.c extern int relative_paths; extern int implied_dirs; extern int file_extra_cnt; -@@ -342,7 +343,7 @@ int push_pathname(const char *dir, int len) +@@ -344,7 +345,7 @@ int push_pathname(const char *dir, int len) static void send_file_entry(int f, struct file_struct *file, int ndx, int first_ndx) { @@ -53,7 +53,7 @@ diff --git a/flist.c b/flist.c static mode_t mode; #ifdef SUPPORT_HARD_LINKS static int64 dev; -@@ -450,6 +451,13 @@ static void send_file_entry(int f, struct file_struct *file, int ndx, int first_ +@@ -452,6 +453,13 @@ static void send_file_entry(int f, struct file_struct *file, int ndx, int first_ xflags |= XMIT_SAME_TIME; else modtime = file->modtime; @@ -67,16 +67,16 @@ diff --git a/flist.c b/flist.c #ifdef SUPPORT_HARD_LINKS if (tmp_dev != 0) { -@@ -522,6 +530,8 @@ static void send_file_entry(int f, struct file_struct *file, int ndx, int first_ +@@ -524,6 +532,8 @@ static void send_file_entry(int f, struct file_struct *file, int ndx, int first_ } if (!(xflags & XMIT_SAME_MODE)) write_int(f, to_wire_mode(mode)); + if (atimes_ndx && !S_ISDIR(mode) && !(xflags & XMIT_SAME_ATIME)) + write_varlong(f, atime, 4); - if (uid_ndx && !(xflags & XMIT_SAME_UID)) { + if (preserve_uid && !(xflags & XMIT_SAME_UID)) { if (protocol_version < 30) write_int(f, uid); -@@ -608,7 +618,7 @@ static void send_file_entry(int f, struct file_struct *file, int ndx, int first_ +@@ -610,7 +620,7 @@ static void send_file_entry(int f, struct file_struct *file, int ndx, int first_ static struct file_struct *recv_file_entry(struct file_list *flist, int xflags, int f) { @@ -85,7 +85,7 @@ diff --git a/flist.c b/flist.c static mode_t mode; #ifdef SUPPORT_HARD_LINKS static int64 dev; -@@ -741,6 +751,16 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -743,6 +753,16 @@ static struct file_struct *recv_file_entry(struct file_list *flist, } if (!(xflags & XMIT_SAME_MODE)) mode = from_wire_mode(read_int(f)); @@ -102,7 +102,7 @@ diff --git a/flist.c b/flist.c if (chmod_modes && !S_ISLNK(mode)) mode = tweak_mode(mode, chmod_modes); -@@ -871,6 +891,8 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -873,6 +893,8 @@ static struct file_struct *recv_file_entry(struct file_list *flist, F_GROUP(file) = gid; file->flags |= gid_flags; } @@ -111,9 +111,9 @@ diff --git a/flist.c b/flist.c if (unsort_ndx) F_NDX(file) = flist->used + flist->ndx_start; -@@ -1203,6 +1225,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, +@@ -1205,6 +1227,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, F_OWNER(file) = st.st_uid; - if (gid_ndx) + if (gid_ndx) /* Check gid_ndx instead of preserve_gid for del support */ F_GROUP(file) = st.st_gid; + if (atimes_ndx) + f_atime_set(file, st.st_atime); @@ -131,7 +131,7 @@ diff --git a/generator.c b/generator.c extern int verbose; extern int dry_run; -@@ -613,6 +614,9 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre +@@ -633,6 +634,9 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre && (!(iflags & ITEM_XNAME_FOLLOWS) || *xname)) || (keep_time && cmp_time(file->modtime, sxp->st.st_mtime) != 0)) iflags |= ITEM_REPORT_TIME; @@ -141,7 +141,7 @@ diff --git a/generator.c b/generator.c #if !defined HAVE_LCHMOD && !defined HAVE_SETATTRLIST if (S_ISLNK(file->mode)) { ; -@@ -967,6 +971,8 @@ static int try_dests_reg(struct file_struct *file, char *fname, int ndx, +@@ -987,6 +991,8 @@ static int try_dests_reg(struct file_struct *file, char *fname, int ndx, if (link_dest) { if (!hard_link_one(file, fname, cmpbuf, 1)) goto try_a_copy; @@ -150,7 +150,7 @@ diff --git a/generator.c b/generator.c if (preserve_hard_links && F_IS_HLINKED(file)) finish_hard_link(file, fname, ndx, &sxp->st, itemizing, code, j); if (!maybe_ATTRS_REPORT && (verbose > 1 || stdout_format_has_i > 1)) { -@@ -1153,6 +1159,7 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx, +@@ -1173,6 +1179,7 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx, static void list_file_entry(struct file_struct *f) { char permbuf[PERMSTRING_SIZE]; @@ -158,7 +158,7 @@ diff --git a/generator.c b/generator.c double len; if (!F_IS_ACTIVE(f)) { -@@ -1167,14 +1174,16 @@ static void list_file_entry(struct file_struct *f) +@@ -1187,14 +1194,16 @@ static void list_file_entry(struct file_struct *f) #ifdef SUPPORT_LINKS if (preserve_links && S_ISLNK(f->mode)) { @@ -177,7 +177,7 @@ diff --git a/generator.c b/generator.c f_name(f, NULL)); } } -@@ -1929,7 +1938,7 @@ static void touch_up_dirs(struct file_list *flist, int ndx) +@@ -1950,7 +1959,7 @@ static void touch_up_dirs(struct file_list *flist, int ndx) if (!(file->mode & S_IWUSR)) do_chmod(fname, file->mode); if (need_retouch_dir_times) @@ -401,19 +401,18 @@ diff --git a/rsync.yo b/rsync.yo dit(bf(--super)) This tells the receiving side to attempt super-user activities even if the receiving rsync wasn't run by the super-user. These activities include: preserving users via the bf(--owner) option, preserving -@@ -1689,8 +1696,10 @@ quote(itemization( +@@ -1689,7 +1696,10 @@ quote(itemization( sender's value (requires bf(--owner) and super-user privileges). it() A bf(g) means the group is different and is being updated to the sender's value (requires bf(--group) and the authority to set the group). -- it() The bf(u) slot is reserved for reporting update (access) time changes -- (a feature that is not yet released). +- it() The bf(u) slot is reserved for future use. + it() A bf(u) means the access (use) time is different and is being updated to + the sender's value (requires bf(--atimes)). An alternate value of bf(U) + means that the access time will be set to the transfer time, which happens + when a symlink or directory is updated. it() The bf(a) means that the ACL information changed. - it() The bf(x) slot is reserved for reporting extended attribute changes - (a feature that is not yet released). + it() The bf(x) means that the extended attribute information changed. + )) diff --git a/testsuite/atimes.test b/testsuite/atimes.test new file mode 100644 --- /dev/null @@ -439,7 +438,7 @@ new file mode 100644 diff --git a/testsuite/rsync.fns b/testsuite/rsync.fns --- a/testsuite/rsync.fns +++ b/testsuite/rsync.fns -@@ -187,6 +187,10 @@ checkit() { +@@ -192,6 +192,10 @@ checkit() { # We can just write everything to stdout/stderr, because the # wrapper hides it unless there is a problem. @@ -450,7 +449,7 @@ diff --git a/testsuite/rsync.fns b/testsuite/rsync.fns echo "Running: \"$1\"" eval "$1" status=$? -@@ -194,10 +198,13 @@ checkit() { +@@ -199,10 +203,13 @@ checkit() { failed="YES"; fi diff --git a/backup-dir-dels.diff b/backup-dir-dels.diff index 679c7cc..ee24be9 100644 --- a/backup-dir-dels.diff +++ b/backup-dir-dels.diff @@ -118,7 +118,7 @@ diff --git a/generator.c b/generator.c @@ -138,10 +141,15 @@ enum delret { /* Forward declaration for delete_item(). */ - static enum delret delete_dir_contents(char *fname, int flags); + static enum delret delete_dir_contents(char *fname, uint16 flags); + +/* Function now compares both backup_suffix and backup_suffix_dels. */ @@ -133,7 +133,7 @@ diff --git a/generator.c b/generator.c } /* Delete a file or directory. If DEL_RECURSE is set in the flags, this will -@@ -180,9 +188,9 @@ static enum delret delete_item(char *fbuf, int mode, int flags) +@@ -189,9 +197,9 @@ static enum delret delete_item(char *fbuf, uint16 mode, uint16 flags) if (S_ISDIR(mode)) { what = "rmdir"; ok = do_rmdir(fbuf) == 0; diff --git a/checksum-reading.diff b/checksum-reading.diff index 2dd412b..ff96510 100644 --- a/checksum-reading.diff +++ b/checksum-reading.diff @@ -1,14 +1,14 @@ -Optimize the ability of a mirror to send checksums. +Optimize the --checksum option using externally created .rsyncsums files. -This adds a sender optimization feature that allows a cache of checksums -to be used when the client specifies the --checksum option. The checksum -files (.rsyncsums) must be created by some other process (see the perl -script in the support dir for one way). +This adds a new option, --sumfiles=MODE, that allows you to use a cache of +checksums when performing a --checksum transfer. These checksum files +(.rsyncsums) must be created by some other process -- see the perl script, +rsyncsums, in the support dir for one way. -This option should be used by mirrors that contain files that get created and -not changed. There is a minimal amount of sanity-check information in the -.rsyncsums file (size and mtime) so that the sum files can be shared with your -mirror network. +This option can be particularly helpful to a public mirror that wants to +pre-compute their .rsyncsums files, set the "checksum files = strict" option +in their daemon config file, and thus make it quite efficient for a client +rsync to make use of the --checksum option on their server. To use this patch, run these commands for a successful build: @@ -16,49 +16,153 @@ To use this patch, run these commands for a successful build: ./configure (optional if already run) make +diff --git a/checksum.c b/checksum.c +--- a/checksum.c ++++ b/checksum.c +@@ -100,7 +100,7 @@ void get_checksum2(char *buf, int32 len, char *sum) + } + } + +-void file_checksum(char *fname, char *sum, OFF_T size) ++void file_checksum(const char *fname, OFF_T size, char *sum) + { + struct map_struct *buf; + OFF_T i, len = size; +diff --git a/clientserver.c b/clientserver.c +--- a/clientserver.c ++++ b/clientserver.c +@@ -39,6 +39,8 @@ extern int numeric_ids; + extern int filesfrom_fd; + extern int remote_protocol; + extern int protocol_version; ++extern int always_checksum; ++extern int checksum_files; + extern int io_timeout; + extern int no_detach; + extern int write_batch; +@@ -750,6 +752,9 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host) + else if (am_root < 0) /* Treat --fake-super from client as --super. */ + am_root = 2; + ++ checksum_files = always_checksum ? lp_checksum_files(i) ++ : CSF_IGNORE_FILES; ++ + if (filesfrom_fd == 0) + filesfrom_fd = f_in; + diff --git a/flist.c b/flist.c --- a/flist.c +++ b/flist.c -@@ -121,6 +121,7 @@ static char tmp_sum[MAX_DIGEST_LEN]; - static char empty_sum[MAX_DIGEST_LEN]; +@@ -34,6 +34,7 @@ extern int am_generator; + extern int inc_recurse; + extern int do_progress; + extern int always_checksum; ++extern int basis_dir_cnt; + extern int module_id; + extern int ignore_errors; + extern int numeric_ids; +@@ -61,6 +62,7 @@ extern int file_extra_cnt; + extern int ignore_perishable; + extern int non_perishable_cnt; + extern int prune_empty_dirs; ++extern int checksum_files; + extern int copy_links; + extern int copy_unsafe_links; + extern int protocol_version; +@@ -68,6 +70,7 @@ extern int sanitize_paths; + extern int munge_symlinks; + extern int need_unsorted_flist; + extern int unsort_ndx; ++extern char *basis_dir[]; + extern struct stats stats; + extern char *filesfrom_host; + +@@ -83,6 +86,11 @@ extern int filesfrom_convert; + extern iconv_t ic_send, ic_recv; + #endif + ++#define RSYNCSUMS_FILE ".rsyncsums" ++ ++#define CLEAN_STRIP_ROOT (1<<0) ++#define CLEAN_KEEP_LAST (1<<1) ++ + #define PTR_SIZE (sizeof (struct file_struct *)) + + int io_error; +@@ -124,7 +132,11 @@ static char empty_sum[MAX_DIGEST_LEN]; static int flist_count_offset; /* for --delete --progress */ static int dir_count = 0; -+static struct file_list *checksum_flist = NULL; - static void flist_sort_and_clean(struct file_list *flist, int strip_root); +-static void flist_sort_and_clean(struct file_list *flist, int strip_root); ++static struct csum_cache { ++ struct file_list *flist; ++} *csum_cache = NULL; ++ ++static void flist_sort_and_clean(struct file_list *flist, int flags); static void output_flist(struct file_list *flist); -@@ -313,6 +314,186 @@ static void flist_done_allocating(struct file_list *flist) + + void init_flist(void) +@@ -315,6 +327,238 @@ static void flist_done_allocating(struct file_list *flist) flist->pool_boundary = ptr; } -+/* The len count is the length of the basename + 1 for the null. */ -+static int add_checksum(const char *dirname, const char *basename, int len, -+ OFF_T file_length, time_t mtime, const char *sum) ++void reset_checksum_cache() ++{ ++ int slot, slots = am_sender ? 1 : basis_dir_cnt + 1; ++ ++ if (!csum_cache) { ++ csum_cache = new_array0(struct csum_cache, slots); ++ if (!csum_cache) ++ out_of_memory("reset_checksum_cache"); ++ } ++ ++ for (slot = 0; slot < slots; slot++) { ++ struct file_list *flist = csum_cache[slot].flist; ++ ++ if (flist) { ++ /* Reset the pool memory and empty the file-list array. */ ++ pool_free_old(flist->file_pool, ++ pool_boundary(flist->file_pool, 0)); ++ flist->used = 0; ++ } else ++ flist = csum_cache[slot].flist = flist_new(FLIST_TEMP, "reset_checksum_cache"); ++ ++ flist->low = 0; ++ flist->high = -1; ++ flist->next = NULL; ++ } ++} ++ ++ ++/* The basename_len count is the length of the basename + 1 for the null. */ ++static int add_checksum(struct file_list *flist, const char *dirname, ++ const char *basename, int basename_len, OFF_T file_length, ++ time_t mtime, uint32 ctime, uint32 inode, ++ const char *sum) +{ + struct file_struct *file; + int alloc_len, extra_len; + char *bp; + -+ if (len == 10+1 && *basename == '.' && strcmp(basename, ".rsyncsums") == 0) -+ return 0; -+ if (file_length == 0) ++ if (basename_len == 10+1 && *basename == '.' && strcmp(basename, RSYNCSUMS_FILE) == 0) + return 0; + -+ extra_len = (file_extra_cnt + (file_length > 0xFFFFFFFFu) + SUM_EXTRA_CNT) ++ /* "2" is for a 32-bit ctime num and an 32-bit inode num. */ ++ extra_len = (file_extra_cnt + (file_length > 0xFFFFFFFFu) + SUM_EXTRA_CNT + 2) + * EXTRA_LEN; +#if EXTRA_ROUNDING > 0 + if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN)) + extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN; +#endif -+ alloc_len = FILE_STRUCT_LEN + extra_len + len; -+ bp = pool_alloc(checksum_flist->file_pool, alloc_len, "add_checksum"); ++ alloc_len = FILE_STRUCT_LEN + extra_len + basename_len; ++ bp = pool_alloc(flist->file_pool, alloc_len, "add_checksum"); + + memset(bp, 0, extra_len + FILE_STRUCT_LEN); + bp += extra_len; + file = (struct file_struct *)bp; + bp += FILE_STRUCT_LEN; + -+ memcpy(bp, basename, len); ++ memcpy(bp, basename, basename_len); + + file->mode = S_IFREG; + file->modtime = mtime; @@ -68,51 +172,45 @@ diff --git a/flist.c b/flist.c + OPT_EXTRA(file, 0)->unum = (uint32)(file_length >> 32); + } + file->dirname = dirname; ++ F_CTIME(file) = ctime; ++ F_INODE(file) = inode; + bp = F_SUM(file); + memcpy(bp, sum, checksum_len); + -+ flist_expand(checksum_flist, 1); -+ checksum_flist->files[checksum_flist->used++] = file; ++ flist_expand(flist, 1); ++ flist->files[flist->used++] = file; + -+ checksum_flist->sorted = checksum_flist->files; ++ flist->sorted = flist->files; + + return 1; +} + -+/* The direname value must remain unchanged during the lifespan of the -+ * created checksum_flist object because we use it directly. */ -+static void read_checksums(const char *dirname) ++/* The "dirname" arg's data must remain unchanged during the lifespan of ++ * the created csum_cache[].flist object because we use it directly. */ ++static void read_checksums(int slot, struct file_list *flist, const char *dirname) +{ + char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN]; -+ OFF_T file_length; -+ time_t mtime; -+ int len, dlen, i; -+ char *cp; + FILE *fp; ++ char *cp; ++ int len, i; ++ time_t mtime; ++ OFF_T file_length; ++ uint32 ctime, inode; ++ int dlen = dirname ? strlcpy(fbuf, dirname, sizeof fbuf) : 0; + -+ if (checksum_flist) { -+ /* Reset the pool memory and empty the file-list array. */ -+ pool_free_old(checksum_flist->file_pool, -+ pool_boundary(checksum_flist->file_pool, 0)); -+ checksum_flist->used = 0; -+ } else -+ checksum_flist = flist_new(FLIST_TEMP, "read_checksums"); -+ -+ checksum_flist->low = 0; -+ checksum_flist->high = -1; -+ -+ if (!dirname) -+ return; -+ -+ dlen = strlcpy(fbuf, dirname, sizeof fbuf); -+ if (dlen >= (int)sizeof fbuf) ++ if (dlen >= (int)(sizeof fbuf - sizeof RSYNCSUMS_FILE)) + return; + if (dlen) + fbuf[dlen++] = '/'; + else + dirname = NULL; -+ strlcpy(fbuf+dlen, ".rsyncsums", sizeof fbuf - dlen); -+ if (!(fp = fopen(fbuf, "r"))) ++ strlcpy(fbuf+dlen, RSYNCSUMS_FILE, sizeof fbuf - dlen); ++ if (slot) { ++ pathjoin(line, sizeof line, basis_dir[slot-1], fbuf); ++ cp = line; ++ } else ++ cp = fbuf; ++ if (!(fp = fopen(cp, "r"))) + return; + + while (fgets(line, sizeof line, fp)) { @@ -177,16 +275,16 @@ diff --git a/flist.c b/flist.c + break; + while (*++cp == ' ') {} + -+ /* Ignore ctime. */ ++ ctime = 0; + while (isDigit(cp)) -+ cp++; ++ ctime = ctime * 10 + *cp++ - '0'; + if (*cp != ' ') + break; + while (*++cp == ' ') {} + -+ /* Ignore inode. */ ++ inode = 0; + while (isDigit(cp)) -+ cp++; ++ inode = inode * 10 + *cp++ - '0'; + if (*cp != ' ') + break; + while (*++cp == ' ') {} @@ -204,17 +302,46 @@ diff --git a/flist.c b/flist.c + + strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen); + -+ add_checksum(dirname, cp, len, file_length, mtime, sum); ++ add_checksum(flist, dirname, cp, len, file_length, ++ mtime, ctime, inode, ++ sum); + } + fclose(fp); + -+ flist_sort_and_clean(checksum_flist, 0); ++ flist_sort_and_clean(flist, CLEAN_KEEP_LAST); ++} ++ ++void get_cached_checksum(int slot, const char *fname, struct file_struct *file, ++ STRUCT_STAT *stp, char *sum_buf) ++{ ++ struct file_list *flist = csum_cache[slot].flist; ++ int j; ++ ++ if (!flist->next) { ++ flist->next = cur_flist; /* next points from checksum flist to file flist */ ++ read_checksums(slot, flist, file->dirname); ++ } ++ ++ if ((j = flist_find(flist, file)) >= 0) { ++ struct file_struct *fp = flist->sorted[j]; ++ ++ if (F_LENGTH(fp) == stp->st_size ++ && fp->modtime == stp->st_mtime ++ && (checksum_files & CSF_LAX ++ || (F_CTIME(fp) == (uint32)stp->st_ctime ++ && F_INODE(fp) == (uint32)stp->st_ino))) { ++ memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN); ++ return; ++ } ++ } ++ ++ file_checksum(fname, stp->st_size, sum_buf); +} + int push_pathname(const char *dir, int len) { if (dir == pathname) -@@ -1003,7 +1184,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, +@@ -1005,7 +1249,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, STRUCT_STAT *stp, int flags, int filter_level) { static char *lastdir; @@ -223,25 +350,25 @@ diff --git a/flist.c b/flist.c struct file_struct *file; char thisname[MAXPATHLEN]; char linkname[MAXPATHLEN]; -@@ -1136,9 +1317,16 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, +@@ -1138,9 +1382,16 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, memcpy(lastdir, thisname, len); lastdir[len] = '\0'; lastdir_len = len; -+ if (always_checksum && am_sender && flist) -+ read_checksums(lastdir); ++ if (checksum_files && am_sender && flist) ++ reset_checksum_cache(); } - } else + } else { basename = thisname; -+ if (always_checksum && am_sender && flist && lastdir_len == -2) { ++ if (checksum_files && am_sender && flist && lastdir_len == -2) { + lastdir_len = -1; -+ read_checksums(""); ++ reset_checksum_cache(); + } + } basename_len = strlen(basename) + 1; /* count the '\0' */ #ifdef SUPPORT_LINKS -@@ -1214,11 +1402,21 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, +@@ -1216,11 +1467,15 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, } #endif @@ -251,22 +378,16 @@ diff --git a/flist.c b/flist.c F_PATHNAME(file) = pathname; + if (always_checksum && am_sender && S_ISREG(st.st_mode)) { -+ int j; -+ if (flist && (j = flist_find(checksum_flist, file)) >= 0) { -+ struct file_struct *fp = checksum_flist->sorted[j]; -+ if (F_LENGTH(fp) == st.st_size -+ && fp->modtime == st.st_mtime) -+ memcpy(tmp_sum, F_SUM(fp), MAX_DIGEST_LEN); -+ else -+ file_checksum(thisname, tmp_sum, st.st_size); -+ } else -+ file_checksum(thisname, tmp_sum, st.st_size); ++ if (flist && checksum_files) ++ get_cached_checksum(0, thisname, file, &st, tmp_sum); ++ else ++ file_checksum(thisname, st.st_size, tmp_sum); + } + /* This code is only used by the receiver when it is building * a list of files for a delete pass. */ if (keep_dirlinks && linkname_len && flist) { -@@ -2074,7 +2272,8 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) +@@ -2076,7 +2331,8 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) * file-list to check if this is a 1-file xfer. */ send_extra_file_list(f, 1); } @@ -276,6 +397,155 @@ diff --git a/flist.c b/flist.c return flist; } +@@ -2178,7 +2434,7 @@ struct file_list *recv_file_list(int f) + else if (f >= 0) + recv_id_list(f, flist); + +- flist_sort_and_clean(flist, relative_paths); ++ flist_sort_and_clean(flist, relative_paths ? CLEAN_STRIP_ROOT : 0); + + if (protocol_version < 30) { + /* Recv the io_error flag */ +@@ -2376,7 +2632,7 @@ void flist_free(struct file_list *flist) + + /* This routine ensures we don't have any duplicate names in our file list. + * duplicate names can cause corruption because of the pipelining. */ +-static void flist_sort_and_clean(struct file_list *flist, int strip_root) ++static void flist_sort_and_clean(struct file_list *flist, int flags) + { + char fbuf[MAXPATHLEN]; + int i, prev_i; +@@ -2427,7 +2683,7 @@ static void flist_sort_and_clean(struct file_list *flist, int strip_root) + /* If one is a dir and the other is not, we want to + * keep the dir because it might have contents in the + * list. Otherwise keep the first one. */ +- if (S_ISDIR(file->mode)) { ++ if (S_ISDIR(file->mode) || flags & CLEAN_KEEP_LAST) { + struct file_struct *fp = flist->sorted[j]; + if (!S_ISDIR(fp->mode)) + keep = i, drop = j; +@@ -2443,8 +2699,8 @@ static void flist_sort_and_clean(struct file_list *flist, int strip_root) + } else + keep = j, drop = i; + +- if (!am_sender) { +- if (verbose > 1) { ++ if (!am_sender || flags & CLEAN_KEEP_LAST) { ++ if (verbose > 1 && !(flags & CLEAN_KEEP_LAST)) { + rprintf(FINFO, + "removing duplicate name %s from file list (%d)\n", + f_name(file, fbuf), drop + flist->ndx_start); +@@ -2466,7 +2722,7 @@ static void flist_sort_and_clean(struct file_list *flist, int strip_root) + } + flist->high = prev_i; + +- if (strip_root) { ++ if (flags & CLEAN_STRIP_ROOT) { + /* We need to strip off the leading slashes for relative + * paths, but this must be done _after_ the sorting phase. */ + for (i = flist->low; i <= flist->high; i++) { +diff --git a/generator.c b/generator.c +--- a/generator.c ++++ b/generator.c +@@ -53,6 +53,7 @@ extern int delete_during; + extern int delete_after; + extern int msgdone_cnt; + extern int ignore_errors; ++extern int checksum_files; + extern int remove_source_files; + extern int delay_updates; + extern int update_only; +@@ -694,7 +695,7 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre + + + /* Perform our quick-check heuristic for determining if a file is unchanged. */ +-int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st) ++int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st, int slot) + { + if (st->st_size != F_LENGTH(file)) + return 0; +@@ -703,7 +704,10 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st) + of the file time to determine whether to sync */ + if (always_checksum > 0 && S_ISREG(st->st_mode)) { + char sum[MAX_DIGEST_LEN]; +- file_checksum(fn, sum, st->st_size); ++ if (checksum_files && slot >= 0) ++ get_cached_checksum(slot, fn, file, st, sum); ++ else ++ file_checksum(fn, st->st_size, sum); + return memcmp(sum, F_SUM(file), checksum_len) == 0; + } + +@@ -957,7 +961,7 @@ static int try_dests_reg(struct file_struct *file, char *fname, int ndx, + match_level = 1; + /* FALL THROUGH */ + case 1: +- if (!unchanged_file(cmpbuf, file, &sxp->st)) ++ if (!unchanged_file(cmpbuf, file, &sxp->st, j+1)) + continue; + best_match = j; + match_level = 2; +@@ -1219,7 +1223,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, + static const char *parent_dirname = ""; + static struct file_struct *missing_dir = NULL, *excluded_dir = NULL; + static struct file_list *fuzzy_dirlist = NULL; +- static int need_fuzzy_dirlist = 0; ++ static int need_new_dirscan = 0; + struct file_struct *fuzzy_file = NULL; + int fd = -1, f_copy = -1; + stat_x sx, real_sx; +@@ -1309,8 +1313,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, + flist_free(fuzzy_dirlist); + fuzzy_dirlist = NULL; + } +- if (fuzzy_basis) +- need_fuzzy_dirlist = 1; ++ if (fuzzy_basis || checksum_files) ++ need_new_dirscan = 1; + #ifdef SUPPORT_ACLS + if (!preserve_perms) + dflt_perms = default_perms_for_dir(dn); +@@ -1318,10 +1322,15 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, + } + parent_dirname = dn; + +- if (need_fuzzy_dirlist && S_ISREG(file->mode)) { +- strlcpy(fnamecmpbuf, dn, sizeof fnamecmpbuf); +- fuzzy_dirlist = get_dirlist(fnamecmpbuf, -1, 1); +- need_fuzzy_dirlist = 0; ++ if (need_new_dirscan && S_ISREG(file->mode)) { ++ if (fuzzy_basis) { ++ strlcpy(fnamecmpbuf, dn, sizeof fnamecmpbuf); ++ fuzzy_dirlist = get_dirlist(fnamecmpbuf, -1, 1); ++ } ++ if (checksum_files) { ++ reset_checksum_cache(); ++ } ++ need_new_dirscan = 0; + } + + statret = link_stat(fname, &sx.st, keep_dirlinks && is_dir); +@@ -1742,7 +1751,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, + ; + else if (fnamecmp_type == FNAMECMP_FUZZY) + ; +- else if (unchanged_file(fnamecmp, file, &sx.st)) { ++ else if (unchanged_file(fnamecmp, file, &sx.st, fnamecmp_type == FNAMECMP_FNAME ? 0 : -1)) { + if (partialptr) { + do_unlink(partialptr); + handle_partial_dir(partialptr, PDIR_DELETE); +diff --git a/hlink.c b/hlink.c +--- a/hlink.c ++++ b/hlink.c +@@ -351,7 +351,7 @@ int hard_link_check(struct file_struct *file, int ndx, const char *fname, + } + break; + } +- if (!unchanged_file(cmpbuf, file, &alt_sx.st)) ++ if (!unchanged_file(cmpbuf, file, &alt_sx.st, j+1)) + continue; + statret = 1; + if (unchanged_attrs(cmpbuf, file, &alt_sx)) diff --git a/ifuncs.h b/ifuncs.h --- a/ifuncs.h +++ b/ifuncs.h @@ -292,11 +562,247 @@ diff --git a/ifuncs.h b/ifuncs.h isPrint(const char *ptr) { return isprint(*(unsigned char *)ptr); +diff --git a/loadparm.c b/loadparm.c +--- a/loadparm.c ++++ b/loadparm.c +@@ -149,6 +149,7 @@ typedef struct + char *temp_dir; + char *uid; + ++ int checksum_files; + int max_connections; + int max_verbosity; + int syslog_facility; +@@ -200,6 +201,7 @@ static service sDefault = + /* temp_dir; */ NULL, + /* uid; */ NOBODY_USER, + ++ /* checksum_files; */ CSF_IGNORE_FILES, + /* max_connections; */ 0, + /* max_verbosity; */ 1, + /* syslog_facility; */ LOG_DAEMON, +@@ -294,6 +296,12 @@ static struct enum_list enum_facilities[] = { + #endif + { -1, NULL }}; + ++static struct enum_list enum_csum_modes[] = { ++ { CSF_IGNORE_FILES, "none" }, ++ { CSF_LAX_MODE, "lax" }, ++ { CSF_STRICT_MODE, "strict" }, ++ { -1, NULL } ++}; + + /* note that we do not initialise the defaults union - it is not allowed in ANSI C */ + static struct parm_struct parm_table[] = +@@ -306,6 +314,7 @@ static struct parm_struct parm_table[] = + + {"auth users", P_STRING, P_LOCAL, &sDefault.auth_users, NULL,0}, + {"charset", P_STRING, P_LOCAL, &sDefault.charset, NULL,0}, ++ {"checksum files", P_ENUM, P_LOCAL, &sDefault.checksum_files, enum_csum_modes,0}, + {"comment", P_STRING, P_LOCAL, &sDefault.comment, NULL,0}, + {"dont compress", P_STRING, P_LOCAL, &sDefault.dont_compress, NULL,0}, + {"exclude from", P_STRING, P_LOCAL, &sDefault.exclude_from, NULL,0}, +@@ -423,6 +432,7 @@ FN_LOCAL_STRING(lp_secrets_file, secrets_file) + FN_LOCAL_STRING(lp_temp_dir, temp_dir) + FN_LOCAL_STRING(lp_uid, uid) + ++FN_LOCAL_INTEGER(lp_checksum_files, checksum_files) + FN_LOCAL_INTEGER(lp_max_connections, max_connections) + FN_LOCAL_INTEGER(lp_max_verbosity, max_verbosity) + FN_LOCAL_INTEGER(lp_syslog_facility, syslog_facility) +diff --git a/options.c b/options.c +--- a/options.c ++++ b/options.c +@@ -112,6 +112,7 @@ size_t bwlimit_writemax = 0; + int ignore_existing = 0; + int ignore_non_existing = 0; + int need_messages_from_generator = 0; ++int checksum_files = CSF_IGNORE_FILES; + int max_delete = INT_MIN; + OFF_T max_size = 0; + OFF_T min_size = 0; +@@ -316,6 +317,7 @@ void usage(enum logcode F) + rprintf(F," -q, --quiet suppress non-error messages\n"); + rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n"); + rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n"); ++ rprintf(F," --sumfiles=MODE use .rsyncsums to speedup --checksum mode\n"); + rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n"); + rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n"); + rprintf(F," -r, --recursive recurse into directories\n"); +@@ -445,7 +447,7 @@ enum {OPT_VERSION = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM, + OPT_FILTER, OPT_COMPARE_DEST, OPT_COPY_DEST, OPT_LINK_DEST, OPT_HELP, + OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW, OPT_MIN_SIZE, OPT_CHMOD, + OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE, +- OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, ++ OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_SUMFILES, + OPT_SERVER, OPT_REFUSED_BASE = 9000}; + + static struct poptOption long_options[] = { +@@ -566,6 +568,7 @@ static struct poptOption long_options[] = { + {"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 }, + {"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, + {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, ++ {"sumfiles", 0, POPT_ARG_STRING, 0, OPT_SUMFILES, 0, 0 }, + {"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 }, + {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 }, + {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 }, +@@ -1212,6 +1215,23 @@ int parse_arguments(int *argc_p, const char ***argv_p, int frommain) + } + break; + ++ case OPT_SUMFILES: ++ arg = poptGetOptArg(pc); ++ checksum_files = 0; ++ if (strcmp(arg, "lax") == 0) ++ checksum_files |= CSF_LAX_MODE; ++ else if (strcmp(arg, "strict") == 0) ++ checksum_files |= CSF_STRICT_MODE; ++ else if (strcmp(arg, "none") == 0) ++ checksum_files = CSF_IGNORE_FILES; ++ else { ++ snprintf(err_buf, sizeof err_buf, ++ "Invalid argument passed to --sumfiles (%s)\n", ++ arg); ++ return 0; ++ } ++ break; ++ + case OPT_HELP: + usage(FINFO); + exit_cleanup(0); +@@ -1311,6 +1331,9 @@ int parse_arguments(int *argc_p, const char ***argv_p, int frommain) + } + #endif + ++ if (checksum_files && !always_checksum) ++ checksum_files = CSF_IGNORE_FILES; ++ + if (write_batch && read_batch) { + snprintf(err_buf, sizeof err_buf, + "--write-batch and --read-batch can not be used together\n"); +diff --git a/rsync.h b/rsync.h +--- a/rsync.h ++++ b/rsync.h +@@ -680,6 +680,10 @@ extern int xattrs_ndx; + #define F_SUM(f) ((char*)OPT_EXTRA(f, LEN64_BUMP(f) + HLINK_BUMP(f) \ + + SUM_EXTRA_CNT - 1)) + ++/* These are only valid on an entry read from a checksum file. */ ++#define F_CTIME(f) OPT_EXTRA(f, LEN64_BUMP(f) + SUM_EXTRA_CNT)->unum ++#define F_INODE(f) OPT_EXTRA(f, LEN64_BUMP(f) + SUM_EXTRA_CNT + 1)->unum ++ + /* Some utility defines: */ + #define F_IS_ACTIVE(f) (f)->basename[0] + #define F_IS_HLINKED(f) ((f)->flags & FLAG_HLINKED) +@@ -858,6 +862,13 @@ typedef struct { + char fname[1]; /* has variable size */ + } relnamecache; + ++#define CSF_ENABLE (1<<1) ++#define CSF_LAX (1<<2) ++ ++#define CSF_IGNORE_FILES 0 ++#define CSF_LAX_MODE (CSF_ENABLE|CSF_LAX) ++#define CSF_STRICT_MODE (CSF_ENABLE) ++ + #include "byteorder.h" + #include "lib/mdigest.h" + #include "lib/wildmatch.h" +diff --git a/rsync.yo b/rsync.yo +--- a/rsync.yo ++++ b/rsync.yo +@@ -317,6 +317,7 @@ to the detailed description below for a complete description. verb( + -q, --quiet suppress non-error messages + --no-motd suppress daemon-mode MOTD (see caveat) + -c, --checksum skip based on checksum, not mod-time & size ++ --sumfiles=MODE use .rsyncsums to speedup --checksum mode + -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X) + --no-OPTION turn off an implied OPTION (e.g. --no-D) + -r, --recursive recurse into directories +@@ -516,9 +517,9 @@ uses a "quick check" that (by default) checks if each file's size and time + of last modification match between the sender and receiver. This option + changes this to compare a 128-bit MD4 checksum for each file that has a + matching size. Generating the checksums means that both sides will expend +-a lot of disk I/O reading all the data in the files in the transfer (and +-this is prior to any reading that will be done to transfer changed files), +-so this can slow things down significantly. ++a lot of disk I/O reading the data in all the files in the transfer, so ++this can slow things down significantly (and this is prior to any reading ++that will be done to transfer the files that have changed). + + The sending side generates its checksums while it is doing the file-system + scan that builds the list of the available files. The receiver generates +@@ -526,12 +527,44 @@ its checksums when it is scanning for changed files, and will checksum any + file that has the same size as the corresponding sender's file: files with + either a changed size or a changed checksum are selected for transfer. + ++See also the bf(--sumfiles) option for a way to use cached checksum data. ++ + Note that rsync always verifies that each em(transferred) file was + correctly reconstructed on the receiving side by checking a whole-file + checksum that is generated as the file is transferred, but that + automatic after-the-transfer verification has nothing to do with this + option's before-the-transfer "Does this file need to be updated?" check. + ++dit(bf(--sumfiles=MODE)) This option tells rsync to make use of any cached ++checksum information it finds in per-directory .rsyncsums files when the ++current transfer is using the bf(--checksum) option. If the checksum data ++is up-to-date, it is used instead of recomputing it, saving both disk I/O ++and CPU time. If the checksum data is missing or outdated, the checksum is ++computed just as it would be if bf(--sumfiles) was not specified. ++ ++The MODE value is either "lax", for relaxed checking (which compares size ++and mtime), "strict" (which also compares ctime and inode), or "none" to ++ignore any .rsyncsums files ("none" is the default). Rsync does not create ++or update these files, but there is a perl script in the support directory ++named "rsyncsums" that can be used for that. ++ ++This option has no effect unless bf(--checksum, -c) was also specified. It ++also only affects the current side of the transfer, so if you want the ++remote side to parse its own .rsyncsums files, specify the option via the ++bf(--rsync-path) option (e.g. "--rsync-path="rsync --sumfiles=lax"). ++ ++To avoid transferring the system's checksum files, you can use an exclude ++(e.g. bf(--exclude=.rsyncsums)). To make this easier to type, you can use ++a popt alias. For instance, adding the following line in your ~/.popt file ++defines a bf(--cc) option that enables lax checksum files and excludes the ++checksum files: ++ ++verb( rsync alias --cc -c --sumfiles=lax --exclude=.rsyncsums) ++ ++An rsync daemon does not allow the client to control this setting, so see ++the "checksum files" daemon parameter for information on how to make a ++daemon use cached checksum data. ++ + dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick + way of saying you want recursion and want to preserve almost + everything (with -H being a notable omission). +diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo +--- a/rsyncd.conf.yo ++++ b/rsyncd.conf.yo +@@ -281,6 +281,17 @@ locking on this file to ensure that the max connections limit is not + exceeded for the modules sharing the lock file. + The default is tt(/var/run/rsyncd.lock). + ++dit(bf(checksum files)) This option tells rsync to make use of any cached ++checksum information it finds in per-directory .rsyncsums files when the ++current transfer is using the bf(--checksum) option. The value can be set ++to either "lax", "strict", or "none" -- see the client's bf(--sumfiles) ++option for what these choices do. ++ ++Note also that the client's command-line option, bf(--sumfiles), has no ++effect on a daemon. A daemon will only access checksum files if this ++config option tells it to. See also the bf(exclude) directive for a way ++to hide the .rsyncsums files from the user. ++ + dit(bf(read only)) The "read only" option determines whether clients + will be able to upload files or not. If "read only" is true then any + attempted uploads will fail. If "read only" is false then uploads will diff --git a/support/rsyncsums b/support/rsyncsums -new file mode 100644 +new file mode 100755 --- /dev/null +++ b/support/rsyncsums -@@ -0,0 +1,203 @@ +@@ -0,0 +1,201 @@ +#!/usr/bin/perl -w +use strict; + @@ -310,12 +816,14 @@ new file mode 100644 +&Getopt::Long::Configure('bundling'); +&usage if !&GetOptions( + 'recurse|r' => \( my $recurse_opt ), -+ 'simple-cmp|s' => \( my $ignore_ctime_and_inode ), ++ 'mode|m=s' => \( my $cmp_mode = 'strict' ), + 'check|c' => \( my $check_opt ), + 'verbose|v+' => \( my $verbosity = 0 ), + 'help|h' => \( my $help_opt ), +); -+&usage if $help_opt; ++&usage if $help_opt || $cmp_mode !~ /^(lax|strict)$/; ++ ++my $ignore_ctime_and_inode = $cmp_mode eq 'lax' ? 0 : 1; + +my $start_dir = cwd(); + @@ -375,17 +883,9 @@ new file mode 100644 + next unless -f _; + + my($size,$mtime,$ctime,$inode) = (stat(_))[7,9,10,1]; ++ $ctime &= 0xFFFFFFFF; ++ $inode &= 0xFFFFFFFF; + my $ref = $cache{$fn}; -+ if ($size == 0) { -+ if (defined $ref) { -+ delete $cache{$fn}; -+ $f_cnt--; -+ if (!$check_opt && !$update_cnt++) { -+ print "UPDATING\n" if $verbosity; -+ } -+ } -+ next; -+ } + $d_cnt++; + + if (!$check_opt) { @@ -423,6 +923,8 @@ new file mode 100644 + print " $sum4 $sum5" if $verbosity > 2; + print " $fn" if $verbosity > 1; + my($size2,$mtime2,$ctime2,$inode2) = (stat(IN))[7,9,10,1]; ++ $ctime2 &= 0xFFFFFFFF; ++ $inode2 &= 0xFFFFFFFF; + last if $size == $size2 && $mtime == $mtime2 + && ($ignore_ctime_and_inode || ($ctime == $ctime2 && $inode == $inode2)); + $size = $size2; @@ -453,7 +955,7 @@ new file mode 100644 + $exit_code = 1; + } else { + print "\n" if $verbosity > 1; -+ $cache{$fn} = [ 1, $sum4, $sum5, $size, $mtime, $ctime & 0xFFFFFFFF, $inode & 0xFFFFFFFF ]; ++ $cache{$fn} = [ 1, $sum4, $sum5, $size, $mtime, $ctime, $inode ]; + } + } + @@ -494,7 +996,9 @@ new file mode 100644 + +Options: + -r, --recurse Update $SUMS_FILE files in subdirectories too. -+ -s, --simple-cmp Ignore ctime and inode values when comparing identicality. ++ -m, --mode=MODE Compare entries in either "lax" or "strict" mode. Using ++ "lax" compares size and mtime, while "strict" additionally ++ compares ctime and inode. Default: strict. + -c, --check Check if the checksums are right (doesn't update). + -v, --verbose Mention what we're doing. Repeat for more info. + -h, --help Display this help message. diff --git a/checksum-updating.diff b/checksum-updating.diff index 9bd02f0..8a7a5ca 100644 --- a/checksum-updating.diff +++ b/checksum-updating.diff @@ -1,6 +1,10 @@ -This builds on the sender optimization feature of the checksum-reading -patch and adds the ability to create and/or updates the .rsyncsums files -when --checksum-updating (or "checksum updating = true") is specified. +This builds on the checksum-reading patch and adds the ability to +create and/or update the .rsyncsums files using extended mode args to +the --sumfiles=MODE option and the "checksum files = MODE" daemon +parameter. + +CAUTION: This patch is only lightly tested. If you're interested +in using it, please help out. To use this patch, run these commands for a successful build: @@ -9,26 +13,10 @@ To use this patch, run these commands for a successful build: ./configure (optional if already run) make -diff --git a/clientserver.c b/clientserver.c ---- a/clientserver.c -+++ b/clientserver.c -@@ -39,6 +39,7 @@ extern int numeric_ids; - extern int filesfrom_fd; - extern int remote_protocol; - extern int protocol_version; -+extern int checksum_updating; - extern int io_timeout; - extern int no_detach; - extern int write_batch; -@@ -746,6 +747,8 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host) - else if (am_root < 0) /* Treat --fake-super from client as --super. */ - am_root = 2; - -+ checksum_updating = lp_checksum_updating(i); -+ - if (filesfrom_fd == 0) - filesfrom_fd = f_in; - +TODO: + + - Fix the code that removes .rsyncsums files when a dir becomes empty. + diff --git a/flist.c b/flist.c --- a/flist.c +++ b/flist.c @@ -40,15 +28,7 @@ diff --git a/flist.c b/flist.c extern int am_root; extern int am_server; extern int am_daemon; -@@ -58,6 +59,7 @@ extern int implied_dirs; - extern int file_extra_cnt; - extern int ignore_perishable; - extern int non_perishable_cnt; -+extern int checksum_updating; - extern int prune_empty_dirs; - extern int copy_links; - extern int copy_unsafe_links; -@@ -83,6 +85,9 @@ extern iconv_t ic_send, ic_recv; +@@ -93,6 +94,9 @@ extern iconv_t ic_send, ic_recv; #define PTR_SIZE (sizeof (struct file_struct *)) @@ -58,108 +38,59 @@ diff --git a/flist.c b/flist.c int io_error; int checksum_len; dev_t filesystem_dev; /* used to implement -x */ -@@ -121,6 +126,9 @@ static char tmp_sum[MAX_DIGEST_LEN]; - static char empty_sum[MAX_DIGEST_LEN]; +@@ -132,8 +136,13 @@ static char empty_sum[MAX_DIGEST_LEN]; static int flist_count_offset; /* for --delete --progress */ static int dir_count = 0; -+static int checksum_matches = 0; -+static int checksum_updates = 0; -+static int regular_skipped = 0; - static struct file_list *checksum_flist = NULL; - - static void flist_sort_and_clean(struct file_list *flist, int strip_root); -@@ -316,7 +324,8 @@ static void flist_done_allocating(struct file_list *flist) - - /* The len count is the length of the basename + 1 for the null. */ - static int add_checksum(const char *dirname, const char *basename, int len, -- OFF_T file_length, time_t mtime, const char *sum) -+ OFF_T file_length, time_t mtime, int32 ctime, int32 inode, -+ const char *sum, const char *alt_sum, int flags) - { - struct file_struct *file; - int alloc_len, extra_len; -@@ -327,13 +336,14 @@ static int add_checksum(const char *dirname, const char *basename, int len, - if (file_length == 0) - return 0; - -- extra_len = (file_extra_cnt + (file_length > 0xFFFFFFFFu) + SUM_EXTRA_CNT) -+ /* "2" is for a 32-bit ctime num and an 32-bit inode num. */ -+ extra_len = (file_extra_cnt + (file_length > 0xFFFFFFFFu) + SUM_EXTRA_CNT + 2) - * EXTRA_LEN; - #if EXTRA_ROUNDING > 0 - if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN)) - extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN; - #endif -- alloc_len = FILE_STRUCT_LEN + extra_len + len; -+ alloc_len = FILE_STRUCT_LEN + extra_len + len + checksum_len*2 + 1; - bp = pool_alloc(checksum_flist->file_pool, alloc_len, "add_checksum"); - - memset(bp, 0, extra_len + FILE_STRUCT_LEN); -@@ -342,7 +352,14 @@ static int add_checksum(const char *dirname, const char *basename, int len, - bp += FILE_STRUCT_LEN; - memcpy(bp, basename, len); -+ if (alt_sum) -+ strlcpy(bp+len, alt_sum, checksum_len*2 + 1); -+ else { -+ memset(bp+len, '=', checksum_len*2); -+ bp[len+checksum_len*2] = '\0'; -+ } ++#define REGULAR_SKIPPED(flist) ((flist)->to_redo) ++ + static struct csum_cache { + struct file_list *flist; ++ const char *dirname; ++ int checksum_matches; ++ int checksum_updates; + } *csum_cache = NULL; -+ file->flags = flags; - file->mode = S_IFREG; - file->modtime = mtime; - file->len32 = (uint32)file_length; -@@ -353,6 +370,8 @@ static int add_checksum(const char *dirname, const char *basename, int len, - file->dirname = dirname; - bp = F_SUM(file); - memcpy(bp, sum, checksum_len); -+ F_CTIME(file) = ctime; -+ F_INODE(file) = inode; - - flist_expand(checksum_flist, 1); - checksum_flist->files[checksum_flist->used++] = file; -@@ -362,17 +381,104 @@ static int add_checksum(const char *dirname, const char *basename, int len, - return 1; + static void flist_sort_and_clean(struct file_list *flist, int flags); +@@ -327,7 +336,79 @@ static void flist_done_allocating(struct file_list *flist) + flist->pool_boundary = ptr; } -+static void write_checksums(const char *next_dirname, int whole_dir) +-void reset_checksum_cache() ++static void checksum_filename(int slot, const char *dirname, char *fbuf) +{ -+ static const char *dirname_save; -+ char fbuf[MAXPATHLEN]; -+ const char *dirname; -+ int used, new_entries, counts_match, no_skipped; -+ FILE *out_fp; -+ int i; -+ -+ dirname = dirname_save; -+ dirname_save = next_dirname; -+ -+ if (!dirname) -+ return; -+ -+ used = checksum_flist->used; -+ new_entries = checksum_updates != 0; -+ counts_match = used == checksum_matches; -+ no_skipped = whole_dir && regular_skipped == 0; ++ if (dirname && *dirname) { ++ unsigned int len; ++ if (slot) { ++ len = strlcpy(fbuf, basis_dir[slot-1], MAXPATHLEN); ++ if (len >= MAXPATHLEN) ++ return; ++ } else ++ len = 0; ++ if (pathjoin(fbuf+len, MAXPATHLEN-len, dirname, RSYNCSUMS_FILE) >= MAXPATHLEN-len) ++ return; ++ } else ++ strlcpy(fbuf, RSYNCSUMS_FILE, MAXPATHLEN); ++} + -+ flist_sort_and_clean(checksum_flist, 0); ++static void write_checksums(int slot, struct file_list *flist, int whole_dir) ++{ ++ int i; ++ FILE *out_fp; ++ char fbuf[MAXPATHLEN]; ++ int new_entries = csum_cache[slot].checksum_updates != 0; ++ int counts_match = flist->used == csum_cache[slot].checksum_matches; ++ int no_skipped = whole_dir && REGULAR_SKIPPED(flist) == 0; ++ const char *dirname = csum_cache[slot].dirname; + -+ checksum_flist->used = 0; -+ checksum_matches = 0; -+ checksum_updates = 0; -+ regular_skipped = 0; ++ flist_sort_and_clean(flist, 0); + -+ if (dry_run) ++ if (dry_run && !(checksum_files & CSF_AFFECT_DRYRUN)) + return; + -+ if (*dirname) { -+ if (pathjoin(fbuf, sizeof fbuf, dirname, ".rsyncsums") >= sizeof fbuf) -+ return; -+ } else -+ strlcpy(fbuf, ".rsyncsums", sizeof fbuf); ++ checksum_filename(slot, dirname, fbuf); + -+ if (checksum_flist->high - checksum_flist->low < 0 && no_skipped) { ++ if (flist->high - flist->low < 0 && no_skipped) { + unlink(fbuf); + return; + } @@ -170,23 +101,18 @@ diff --git a/flist.c b/flist.c + if (!(out_fp = fopen(fbuf, "w"))) + return; + -+ new_entries = 0; -+ for (i = checksum_flist->low; i <= checksum_flist->high; i++) { -+ struct file_struct *file = checksum_flist->sorted[i]; ++ for (i = flist->low; i <= flist->high; i++) { ++ struct file_struct *file = flist->sorted[i]; + const char *cp = F_SUM(file); + const char *end = cp + checksum_len; + const char *alt_sum = file->basename + strlen(file->basename) + 1; -+ int32 ctime, inode; + if (whole_dir && !(file->flags & FLAG_SUM_KEEP)) + continue; -+ ctime = F_CTIME(file); -+ inode = F_INODE(file); + if (protocol_version >= 30) + fprintf(out_fp, "%s ", alt_sum); + if (file->flags & FLAG_SUM_MISSING) { -+ new_entries++; + do { -+ fprintf(out_fp, "=="); ++ fputs("==", out_fp); + } while (++cp != end); + } else { + do { @@ -195,47 +121,86 @@ diff --git a/flist.c b/flist.c + } + if (protocol_version < 30) + fprintf(out_fp, " %s", alt_sum); -+ if (*alt_sum == '=') -+ new_entries++; + fprintf(out_fp, " %10.0f %10.0f %10lu %10lu %s\n", + (double)F_LENGTH(file), (double)file->modtime, -+ (long)ctime, (long)inode, file->basename); ++ (long)F_CTIME(file), (long)F_INODE(file), file->basename); + } + + fclose(out_fp); +} + - /* The direname value must remain unchanged during the lifespan of the - * created checksum_flist object because we use it directly. */ - static void read_checksums(const char *dirname) ++void reset_checksum_cache(int whole_dir) { + int slot, slots = am_sender ? 1 : basis_dir_cnt + 1; + +@@ -341,6 +422,9 @@ void reset_checksum_cache() + struct file_list *flist = csum_cache[slot].flist; + + if (flist) { ++ if (checksum_files & CSF_UPDATE && flist->next) ++ write_checksums(slot, flist, whole_dir); ++ + /* Reset the pool memory and empty the file-list array. */ + pool_free_old(flist->file_pool, + pool_boundary(flist->file_pool, 0)); +@@ -351,6 +435,10 @@ void reset_checksum_cache() + flist->low = 0; + flist->high = -1; + flist->next = NULL; ++ ++ csum_cache[slot].checksum_matches = 0; ++ csum_cache[slot].checksum_updates = 0; ++ REGULAR_SKIPPED(flist) = 0; + } + } + +@@ -359,7 +447,7 @@ void reset_checksum_cache() + static int add_checksum(struct file_list *flist, const char *dirname, + const char *basename, int basename_len, OFF_T file_length, + time_t mtime, uint32 ctime, uint32 inode, +- const char *sum) ++ const char *sum, const char *alt_sum, int flags) + { + struct file_struct *file; + int alloc_len, extra_len; +@@ -375,7 +463,7 @@ static int add_checksum(struct file_list *flist, const char *dirname, + if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN)) + extra_len = (extra_len | (EXTRA_ROUNDING * EXTRA_LEN)) + EXTRA_LEN; + #endif +- alloc_len = FILE_STRUCT_LEN + extra_len + basename_len; ++ alloc_len = FILE_STRUCT_LEN + extra_len + basename_len + checksum_len*2 + 1; + bp = pool_alloc(flist->file_pool, alloc_len, "add_checksum"); + + memset(bp, 0, extra_len + FILE_STRUCT_LEN); +@@ -384,7 +472,14 @@ static int add_checksum(struct file_list *flist, const char *dirname, + bp += FILE_STRUCT_LEN; + + memcpy(bp, basename, basename_len); ++ if (alt_sum) ++ strlcpy(bp+basename_len, alt_sum, checksum_len*2 + 1); ++ else { ++ memset(bp+basename_len, '=', checksum_len*2); ++ bp[basename_len+checksum_len*2] = '\0'; ++ } + ++ file->flags = flags; + file->mode = S_IFREG; + file->modtime = mtime; + file->len32 = (uint32)file_length; +@@ -413,10 +508,11 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam char line[MAXPATHLEN+1024], fbuf[MAXPATHLEN], sum[MAX_DIGEST_LEN]; -+ const char *alt_sum = NULL; - OFF_T file_length; - time_t mtime; -- int len, dlen, i; -+ int32 ctime, inode; -+ int len, dlen, i, flags; - char *cp; FILE *fp; + char *cp; +- int len, i; + time_t mtime; ++ int len, i, flags; + OFF_T file_length; + uint32 ctime, inode; ++ const char *alt_sum = NULL; + int dlen = dirname ? strlcpy(fbuf, dirname, sizeof fbuf) : 0; -+ if (checksum_updating) -+ write_checksums(dirname, 0); -+ - if (checksum_flist) { - /* Reset the pool memory and empty the file-list array. */ - pool_free_old(checksum_flist->file_pool, -@@ -383,6 +489,9 @@ static void read_checksums(const char *dirname) - - checksum_flist->low = 0; - checksum_flist->high = -1; -+ checksum_matches = 0; -+ checksum_updates = 0; -+ regular_skipped = 0; - - if (!dirname) - return; -@@ -401,7 +510,7 @@ static void read_checksums(const char *dirname) + if (dlen >= (int)(sizeof fbuf - sizeof RSYNCSUMS_FILE)) +@@ -437,7 +533,7 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam while (fgets(line, sizeof line, fp)) { cp = line; if (protocol_version >= 30) { @@ -244,7 +209,7 @@ diff --git a/flist.c b/flist.c if (*cp == '=') while (*++cp == '=') {} else -@@ -412,7 +521,14 @@ static void read_checksums(const char *dirname) +@@ -448,7 +544,14 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam } if (*cp == '=') { @@ -260,7 +225,7 @@ diff --git a/flist.c b/flist.c } else { for (i = 0; i < checksum_len*2; i++, cp++) { int x; -@@ -430,13 +546,14 @@ static void read_checksums(const char *dirname) +@@ -466,13 +569,14 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam else sum[i/2] = x << 4; } @@ -276,288 +241,417 @@ diff --git a/flist.c b/flist.c if (*cp == '=') while (*++cp == '=') {} else -@@ -460,16 +577,16 @@ static void read_checksums(const char *dirname) - break; - while (*++cp == ' ') {} - -- /* Ignore ctime. */ -+ ctime = 0; - while (isDigit(cp)) -- cp++; -+ ctime = ctime * 10 + *cp++ - '0'; - if (*cp != ' ') - break; - while (*++cp == ' ') {} - -- /* Ignore inode. */ -+ inode = 0; - while (isDigit(cp)) -- cp++; -+ inode = inode * 10 + *cp++ - '0'; - if (*cp != ' ') - break; - while (*++cp == ' ') {} -@@ -486,8 +603,13 @@ static void read_checksums(const char *dirname) +@@ -522,24 +626,112 @@ static void read_checksums(int slot, struct file_list *flist, const char *dirnam continue; strlcpy(fbuf+dlen, cp, sizeof fbuf - dlen); + if (is_excluded(fbuf, 0, ALL_FILTERS)) { + flags |= FLAG_SUM_KEEP; -+ checksum_matches++; ++ csum_cache[slot].checksum_matches++; + } -- add_checksum(dirname, cp, len, file_length, mtime, sum); -+ add_checksum(dirname, cp, len, file_length, mtime, ctime, inode, + add_checksum(flist, dirname, cp, len, file_length, + mtime, ctime, inode, +- sum); + sum, alt_sum, flags); } fclose(fp); -@@ -1278,6 +1400,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, + flist_sort_and_clean(flist, CLEAN_KEEP_LAST); + } + ++void set_cached_checksum(struct file_list *file_flist, struct file_struct *file) ++{ ++ int j; ++ FILE *out_fp; ++ STRUCT_STAT st; ++ char fbuf[MAXPATHLEN]; ++ const char *fn = f_name(file, NULL); ++ struct file_list *flist = csum_cache[0].flist; ++ ++ if (dry_run && !(checksum_files & CSF_AFFECT_DRYRUN)) ++ return; ++ ++ if (stat(fn, &st) < 0) ++ return; ++ ++ checksum_filename(0, file->dirname, fbuf); ++ ++ if (file_flist != flist->next) { ++ const char *cp = F_SUM(file); ++ const char *end = cp + checksum_len; ++ ++ if (!(out_fp = fopen(fbuf, "a"))) ++ return; ++ ++ if (protocol_version >= 30) { ++ for (j = 0; j < checksum_len; j++) ++ fputs("==", out_fp); ++ fputc(' ', out_fp); ++ } ++ do { ++ fprintf(out_fp, "%02x", (int)CVAL(cp, 0)); ++ } while (++cp != end); ++ if (protocol_version < 30) { ++ fputc(' ', out_fp); ++ for (j = 0; j < checksum_len; j++) ++ fputs("==", out_fp); ++ } ++ fprintf(out_fp, " %10.0f %10.0f %10lu %10lu %s\n", ++ (double)st.st_size, (double)st.st_mtime, ++ (long)(uint32)st.st_ctime, (long)(uint32)st.st_ino, ++ file->basename); ++ ++ fclose(out_fp); ++ return; ++ } ++ ++ if ((j = flist_find(flist, file)) >= 0) { ++ struct file_struct *fp = flist->sorted[j]; ++ int inc = 0; ++ if (F_LENGTH(fp) != st.st_size) { ++ fp->len32 = (uint32)st.st_size; ++ if (st.st_size > 0xFFFFFFFFu) { ++ OPT_EXTRA(fp, 0)->unum = (uint32)(st.st_size >> 32); ++ fp->flags |= FLAG_LENGTH64; ++ } else ++ fp->flags &= FLAG_LENGTH64; ++ inc = 1; ++ } ++ if (fp->modtime != st.st_mtime) { ++ fp->modtime = st.st_mtime; ++ inc = 1; ++ } ++ if (F_CTIME(fp) != (uint32)st.st_ctime) { ++ F_CTIME(fp) = (uint32)st.st_ctime; ++ inc = 1; ++ } ++ if (F_INODE(fp) != (uint32)st.st_ino) { ++ F_INODE(fp) = (uint32)st.st_ino; ++ inc = 1; ++ } ++ memcpy(F_SUM(fp), F_SUM(file), MAX_DIGEST_LEN); ++ csum_cache[0].checksum_updates += inc; ++ fp->flags &= ~FLAG_SUM_MISSING; ++ fp->flags |= FLAG_SUM_KEEP; ++ return; ++ } ++ ++ csum_cache[0].checksum_updates += ++ add_checksum(flist, file->dirname, file->basename, strlen(file->basename) + 1, ++ st.st_size, (uint32)st.st_mtime, (uint32)st.st_ctime, ++ st.st_ino, F_SUM(file), NULL, FLAG_SUM_KEEP); ++} ++ + void get_cached_checksum(int slot, const char *fname, struct file_struct *file, +- STRUCT_STAT *stp, char *sum_buf) ++ int basename_len, STRUCT_STAT *stp, char *sum_buf) + { + struct file_list *flist = csum_cache[slot].flist; + int j; + + if (!flist->next) { + flist->next = cur_flist; /* next points from checksum flist to file flist */ ++ csum_cache[slot].dirname = file->dirname; + read_checksums(slot, flist, file->dirname); + } + +@@ -551,12 +743,31 @@ void get_cached_checksum(int slot, const char *fname, struct file_struct *file, + && (checksum_files & CSF_LAX + || (F_CTIME(fp) == (uint32)stp->st_ctime + && F_INODE(fp) == (uint32)stp->st_ino))) { +- memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN); ++ if (fp->flags & FLAG_SUM_MISSING) { ++ fp->flags &= ~FLAG_SUM_MISSING; ++ csum_cache[slot].checksum_updates++; ++ file_checksum(fname, stp->st_size, sum_buf); ++ memcpy(F_SUM(fp), sum_buf, MAX_DIGEST_LEN); ++ } else { ++ csum_cache[slot].checksum_matches++; ++ memcpy(sum_buf, F_SUM(fp), MAX_DIGEST_LEN); ++ } ++ fp->flags |= FLAG_SUM_KEEP; + return; + } ++ clear_file(fp); + } + + file_checksum(fname, stp->st_size, sum_buf); ++ ++ if (checksum_files & CSF_UPDATE) { ++ if (basename_len < 0) ++ basename_len = strlen(file->basename) + 1; ++ csum_cache[slot].checksum_updates += ++ add_checksum(flist, file->dirname, file->basename, basename_len, ++ stp->st_size, stp->st_mtime, (uint32)stp->st_ctime, ++ (uint32)stp->st_ino, sum_buf, NULL, FLAG_SUM_KEEP); ++ } + } + + int push_pathname(const char *dir, int len) +@@ -1343,6 +1554,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, if (is_excluded(thisname, S_ISDIR(st.st_mode) != 0, filter_level)) { if (ignore_perishable) non_perishable_cnt++; + if (S_ISREG(st.st_mode)) -+ regular_skipped++; ++ REGULAR_SKIPPED(flist)++; return NULL; } -@@ -1408,13 +1532,36 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, - int j; - if (flist && (j = flist_find(checksum_flist, file)) >= 0) { - struct file_struct *fp = checksum_flist->sorted[j]; -+ int32 ctime = F_CTIME(fp); -+ int32 inode = F_INODE(fp); - if (F_LENGTH(fp) == st.st_size -- && fp->modtime == st.st_mtime) -- memcpy(tmp_sum, F_SUM(fp), MAX_DIGEST_LEN); -- else -- file_checksum(thisname, tmp_sum, st.st_size); -- } else -+ && fp->modtime == st.st_mtime -+ && ctime == (int32)st.st_ctime -+ && inode == (int32)st.st_ino) { -+ if (fp->flags & FLAG_SUM_MISSING) { -+ fp->flags &= ~FLAG_SUM_MISSING; -+ checksum_updates++; -+ file_checksum(thisname, tmp_sum, st.st_size); -+ memcpy(F_SUM(fp), tmp_sum, MAX_DIGEST_LEN); -+ } else { -+ checksum_matches++; -+ memcpy(tmp_sum, F_SUM(fp), MAX_DIGEST_LEN); -+ } -+ fp->flags |= FLAG_SUM_KEEP; -+ } else { -+ clear_file(fp); -+ goto compute_new_checksum; -+ } -+ } else { -+ compute_new_checksum: - file_checksum(thisname, tmp_sum, st.st_size); -+ if (checksum_updating && flist) { -+ checksum_updates += -+ add_checksum(file->dirname, basename, basename_len, -+ st.st_size, st.st_mtime, st.st_ctime, -+ st.st_ino, tmp_sum, NULL, FLAG_SUM_KEEP); -+ } -+ } +@@ -1383,13 +1596,13 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, + lastdir[len] = '\0'; + lastdir_len = len; + if (checksum_files && am_sender && flist) +- reset_checksum_cache(); ++ reset_checksum_cache(0); + } + } else { + basename = thisname; + if (checksum_files && am_sender && flist && lastdir_len == -2) { + lastdir_len = -1; +- reset_checksum_cache(); ++ reset_checksum_cache(0); + } } - - /* This code is only used by the receiver when it is building -@@ -1709,6 +1856,9 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len, + basename_len = strlen(basename) + 1; /* count the '\0' */ +@@ -1471,7 +1684,7 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, + + if (always_checksum && am_sender && S_ISREG(st.st_mode)) { + if (flist && checksum_files) +- get_cached_checksum(0, thisname, file, &st, tmp_sum); ++ get_cached_checksum(0, thisname, file, basename_len, &st, tmp_sum); + else + file_checksum(thisname, st.st_size, tmp_sum); + } +@@ -1768,6 +1981,9 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len, closedir(d); -+ if (checksum_updating && always_checksum && am_sender && f >= 0) -+ write_checksums(NULL, 1); ++ if (checksum_files & CSF_UPDATE && am_sender && f >= 0) ++ reset_checksum_cache(1); + if (f >= 0 && recurse && !divert_dirs) { int i, end = flist->used - 1; /* send_if_directory() bumps flist->used, so use "end". */ -@@ -2274,6 +2424,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) +@@ -2333,6 +2549,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) } } else flist_eof = 1; + -+ if (checksum_updating && always_checksum && flist_eof) -+ read_checksums(NULL); /* writes any last updates */ ++ if (checksum_files & CSF_UPDATE && flist_eof) ++ reset_checksum_cache(0); /* writes any last updates */ return flist; } +diff --git a/generator.c b/generator.c +--- a/generator.c ++++ b/generator.c +@@ -114,6 +114,7 @@ static int lull_mod; + static int dir_tweaking; + static int need_retouch_dir_times; + static int need_retouch_dir_perms; ++static int started_whole_dir, upcoming_whole_dir; + static const char *solo_file = NULL; + + /* For calling delete_item() and delete_dir_contents(). */ +@@ -705,7 +706,7 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st, int slot + if (always_checksum > 0 && S_ISREG(st->st_mode)) { + char sum[MAX_DIGEST_LEN]; + if (checksum_files && slot >= 0) +- get_cached_checksum(slot, fn, file, st, sum); ++ get_cached_checksum(slot, fn, file, -1, st, sum); + else + file_checksum(fn, st->st_size, sum); + return memcmp(sum, F_SUM(file), checksum_len) == 0; +@@ -1328,7 +1329,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, + fuzzy_dirlist = get_dirlist(fnamecmpbuf, -1, 1); + } + if (checksum_files) { +- reset_checksum_cache(); ++ reset_checksum_cache(started_whole_dir); ++ started_whole_dir = upcoming_whole_dir; + } + need_new_dirscan = 0; + } +@@ -1457,9 +1459,12 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, + DEV_MINOR(devp) = minor(real_sx.st.st_dev); + } + } +- else if (delete_during && f_out != -1 && !phase && dry_run < 2 +- && (file->flags & FLAG_CONTENT_DIR)) +- delete_in_dir(fname, file, &real_sx.st.st_dev); ++ else if (file->flags & FLAG_CONTENT_DIR && f_out != -1) { ++ if (delete_during && !phase && dry_run < 2) ++ delete_in_dir(fname, file, &real_sx.st.st_dev); ++ upcoming_whole_dir = 1; ++ } else ++ upcoming_whole_dir = 0; + goto cleanup; + } + +@@ -1757,6 +1762,8 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, + handle_partial_dir(partialptr, PDIR_DELETE); + } + set_file_attrs(fname, file, &sx, NULL, maybe_ATTRS_REPORT); ++ if (checksum_files & CSF_UPDATE) ++ set_cached_checksum(cur_flist, file); + if (itemizing) + itemize(fnamecmp, file, ndx, statret, &sx, 0, 0, NULL); + #ifdef SUPPORT_HARD_LINKS +@@ -2132,6 +2139,7 @@ void generate_files(int f_out, const char *local_name) + delete_in_dir(f_name(fp, fbuf), fp, &dirdev); + } + } ++ upcoming_whole_dir = fp->flags & FLAG_CONTENT_DIR ? 1 : 0; + } + for (i = cur_flist->low; i <= cur_flist->high; i++) { + struct file_struct *file = cur_flist->sorted[i]; +@@ -2212,6 +2220,9 @@ void generate_files(int f_out, const char *local_name) + wait_for_receiver(); + } + ++ if (checksum_files) ++ reset_checksum_cache(started_whole_dir); ++ + do_progress = save_do_progress; + if (delete_during == 2) + do_delayed_deletions(fbuf); +diff --git a/io.c b/io.c +--- a/io.c ++++ b/io.c +@@ -49,6 +49,7 @@ extern int read_batch; + extern int csum_length; + extern int protect_args; + extern int checksum_seed; ++extern int checksum_files; + extern int protocol_version; + extern int remove_source_files; + extern int preserve_hard_links; +@@ -200,6 +201,9 @@ static void got_flist_entry_status(enum festatus status, const char *buf) + flist_ndx_push(&hlink_list, ndx); + flist->in_progress++; + } ++ } else if (checksum_files & CSF_UPDATE) { ++ struct file_struct *file = flist->files[ndx - flist->ndx_start]; ++ set_cached_checksum(flist, file); + } + break; + case FES_REDO: diff --git a/loadparm.c b/loadparm.c --- a/loadparm.c +++ b/loadparm.c -@@ -154,6 +154,7 @@ typedef struct - int syslog_facility; - int timeout; - -+ BOOL checksum_updating; - BOOL fake_super; - BOOL ignore_errors; - BOOL ignore_nonreadable; -@@ -205,6 +206,7 @@ static service sDefault = - /* syslog_facility; */ LOG_DAEMON, - /* timeout; */ 0, - -+ /* checksum_updating; */ False, - /* fake_super; */ False, - /* ignore_errors; */ False, - /* ignore_nonreadable; */ False, -@@ -306,6 +308,7 @@ static struct parm_struct parm_table[] = - - {"auth users", P_STRING, P_LOCAL, &sDefault.auth_users, NULL,0}, - {"charset", P_STRING, P_LOCAL, &sDefault.charset, NULL,0}, -+ {"checksum updating", P_BOOL, P_LOCAL, &sDefault.checksum_updating, NULL,0}, - {"comment", P_STRING, P_LOCAL, &sDefault.comment, NULL,0}, - {"dont compress", P_STRING, P_LOCAL, &sDefault.dont_compress, NULL,0}, - {"exclude from", P_STRING, P_LOCAL, &sDefault.exclude_from, NULL,0}, -@@ -428,6 +431,7 @@ FN_LOCAL_INTEGER(lp_max_verbosity, max_verbosity) - FN_LOCAL_INTEGER(lp_syslog_facility, syslog_facility) - FN_LOCAL_INTEGER(lp_timeout, timeout) - -+FN_LOCAL_BOOL(lp_checksum_updating, checksum_updating) - FN_LOCAL_BOOL(lp_fake_super, fake_super) - FN_LOCAL_BOOL(lp_ignore_errors, ignore_errors) - FN_LOCAL_BOOL(lp_ignore_nonreadable, ignore_nonreadable) +@@ -300,6 +300,10 @@ static struct enum_list enum_csum_modes[] = { + { CSF_IGNORE_FILES, "none" }, + { CSF_LAX_MODE, "lax" }, + { CSF_STRICT_MODE, "strict" }, ++ { CSF_LAX_MODE|CSF_UPDATE, "+lax" }, ++ { CSF_STRICT_MODE|CSF_UPDATE, "+strict" }, ++ { CSF_LAX_MODE|CSF_UPDATE|CSF_AFFECT_DRYRUN, "++lax" }, ++ { CSF_STRICT_MODE|CSF_UPDATE|CSF_AFFECT_DRYRUN, "++strict" }, + { -1, NULL } + }; + diff --git a/options.c b/options.c --- a/options.c +++ b/options.c -@@ -112,6 +112,7 @@ size_t bwlimit_writemax = 0; - int ignore_existing = 0; - int ignore_non_existing = 0; - int need_messages_from_generator = 0; -+int checksum_updating = 0; - int max_delete = INT_MIN; - OFF_T max_size = 0; - OFF_T min_size = 0; -@@ -316,6 +317,7 @@ void usage(enum logcode F) - rprintf(F," -q, --quiet suppress non-error messages\n"); - rprintf(F," --no-motd suppress daemon-mode MOTD (see manpage caveat)\n"); - rprintf(F," -c, --checksum skip based on checksum, not mod-time & size\n"); -+ rprintf(F," --checksum-updating sender updates .rsyncsums files\n"); - rprintf(F," -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X)\n"); - rprintf(F," --no-OPTION turn off an implied OPTION (e.g. --no-D)\n"); - rprintf(F," -r, --recursive recurse into directories\n"); -@@ -566,6 +568,7 @@ static struct poptOption long_options[] = { - {"checksum", 'c', POPT_ARG_VAL, &always_checksum, 1, 0, 0 }, - {"no-checksum", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, - {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 }, -+ {"checksum-updating",0, POPT_ARG_NONE, &checksum_updating, 0, 0, 0 }, - {"block-size", 'B', POPT_ARG_LONG, &block_size, 0, 0, 0 }, - {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 }, - {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 }, -@@ -1988,7 +1991,9 @@ void server_options(char **args, int *argc_p) - args[ac++] = basis_dir[i]; - } - } -- } -+ } else if (checksum_updating) -+ args[ac++] = "--checksum-updating"; -+ - - if (append_mode) { - if (append_mode > 1) +@@ -1217,7 +1217,15 @@ int parse_arguments(int *argc_p, const char ***argv_p, int frommain) + + case OPT_SUMFILES: + arg = poptGetOptArg(pc); +- checksum_files = 0; ++ if (*arg == '+') { ++ arg++; ++ checksum_files = CSF_UPDATE; ++ if (*arg == '+') { ++ arg++; ++ checksum_files |= CSF_AFFECT_DRYRUN; ++ } ++ } else ++ checksum_files = 0; + if (strcmp(arg, "lax") == 0) + checksum_files |= CSF_LAX_MODE; + else if (strcmp(arg, "strict") == 0) +diff --git a/receiver.c b/receiver.c +--- a/receiver.c ++++ b/receiver.c +@@ -47,6 +47,7 @@ extern int append_mode; + extern int sparse_files; + extern int keep_partial; + extern int checksum_seed; ++extern int checksum_files; + extern int inplace; + extern int delay_updates; + extern mode_t orig_umask; +@@ -339,7 +340,7 @@ static void handle_delayed_updates(char *local_name) + "rename failed for %s (from %s)", + full_fname(fname), partialptr); + } else { +- if (remove_source_files ++ if (remove_source_files || checksum_files & CSF_UPDATE + || (preserve_hard_links && F_IS_HLINKED(file))) + send_msg_int(MSG_SUCCESS, ndx); + handle_partial_dir(partialptr, PDIR_DELETE); +@@ -716,7 +717,7 @@ int recv_files(int f_in, char *local_name) + + switch (recv_ok) { + case 1: +- if (remove_source_files || inc_recurse ++ if (remove_source_files || inc_recurse || checksum_files & CSF_UPDATE + || (preserve_hard_links && F_IS_HLINKED(file))) + send_msg_int(MSG_SUCCESS, ndx); + break; diff --git a/rsync.h b/rsync.h --- a/rsync.h +++ b/rsync.h -@@ -680,6 +680,10 @@ extern int xattrs_ndx; - #define F_SUM(f) ((char*)OPT_EXTRA(f, LEN64_BUMP(f) + HLINK_BUMP(f) \ - + SUM_EXTRA_CNT - 1)) +@@ -864,6 +864,8 @@ typedef struct { -+/* These are only valid on an entry read from a checksum file. */ -+#define F_CTIME(f) OPT_EXTRA(f, LEN64_BUMP(f) + SUM_EXTRA_CNT)->num -+#define F_INODE(f) OPT_EXTRA(f, LEN64_BUMP(f) + SUM_EXTRA_CNT + 1)->num -+ - /* Some utility defines: */ - #define F_IS_ACTIVE(f) (f)->basename[0] - #define F_IS_HLINKED(f) ((f)->flags & FLAG_HLINKED) + #define CSF_ENABLE (1<<1) + #define CSF_LAX (1<<2) ++#define CSF_UPDATE (1<<3) ++#define CSF_AFFECT_DRYRUN (1<<4) + + #define CSF_IGNORE_FILES 0 + #define CSF_LAX_MODE (CSF_ENABLE|CSF_LAX) diff --git a/rsync.yo b/rsync.yo --- a/rsync.yo +++ b/rsync.yo -@@ -317,6 +317,7 @@ to the detailed description below for a complete description. verb( - -q, --quiet suppress non-error messages - --no-motd suppress daemon-mode MOTD (see caveat) - -c, --checksum skip based on checksum, not mod-time & size -+ --checksum-updating sender updates .rsyncsums files - -a, --archive archive mode; equals -rlptgoD (no -H,-A,-X) - --no-OPTION turn off an implied OPTION (e.g. --no-D) - -r, --recursive recurse into directories -@@ -516,9 +517,9 @@ uses a "quick check" that (by default) checks if each file's size and time - of last modification match between the sender and receiver. This option - changes this to compare a 128-bit MD4 checksum for each file that has a - matching size. Generating the checksums means that both sides will expend --a lot of disk I/O reading all the data in the files in the transfer (and --this is prior to any reading that will be done to transfer changed files), --so this can slow things down significantly. -+a lot of disk I/O reading the data in all the files in the transfer, so -+this can slow things down significantly (and this is prior to any reading -+that will be done to transfer the files that have changed). - - The sending side generates its checksums while it is doing the file-system - scan that builds the list of the available files. The receiver generates -@@ -526,12 +527,42 @@ its checksums when it is scanning for changed files, and will checksum any - file that has the same size as the corresponding sender's file: files with - either a changed size or a changed checksum are selected for transfer. - -+Starting with version 3.0.0, the sending side will look for a checksum -+summary file and use a pre-generated checksum that it reads out of the file -+(as long as it matches the file's size and modified time). This allows a -+server to support the --checksum option to clients without having to -+recompute the checksums for each client. See the bf(--checksum-updating) -+option for a way to have rsync create/update these checksum files. -+ - Note that rsync always verifies that each em(transferred) file was - correctly reconstructed on the receiving side by checking a whole-file - checksum that is generated as the file is transferred, but that - automatic after-the-transfer verification has nothing to do with this - option's before-the-transfer "Does this file need to be updated?" check. - -+dit(bf(--checksum-updating)) This option tells the sending side to create -+and/or update per-directory checksum files that are used by the -+bf(--checksum) option. The file that is updated is named .rsyncsums. If -+pre-transfer checksums are not being computed, this option has no effect. -+ -+The checksum files stores the computed checksum, last-known size, -+modification time, and name for each file in the current directory. If a -+later transfer finds that a file matches its prior size and modification -+time, the checksum is assumed to still be correct. Otherwise it is -+recomputed and udpated in the file. -+ -+To avoid transferring the system's checksum files, you can use an exclude -+(e.g. bf(--exclude=.rsyncsums)). To make this easier to type, you can use -+a popt alias. For instance, adding the following line in your ~/.popt file -+defines a bf(-cc) option that enables checksum updating and excludes the -+checksum files: -+ -+verb( rsync alias --cc --checksum-updating --exclude=.rsyncsums) -+ -+An rsync daemon does not allow the client to control this setting, so see -+the "checksum updating" daemon config option for information on how to make -+a daemon maintain these checksum files. -+ - dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick - way of saying you want recursion and want to preserve almost - everything (with -H being a notable omission). +@@ -544,9 +544,13 @@ computed just as it would be if bf(--sumfiles) was not specified. + + The MODE value is either "lax", for relaxed checking (which compares size + and mtime), "strict" (which also compares ctime and inode), or "none" to +-ignore any .rsyncsums files ("none" is the default). Rsync does not create +-or update these files, but there is a perl script in the support directory +-named "rsyncsums" that can be used for that. ++ignore any .rsyncsums files ("none" is the default). ++If you want rsync to create and/or update these files, specify a prefixed ++plus ("+lax" or "+strict"). ++Adding a second prefixed '+' causes the checksum-file updates to happen ++even when the transfer is in bf(--dry-run) mode ("++lax" or "++strict"). ++There is also a perl script in the support directory named "rsyncsums" ++that can be used to update the .rsyncsums files. + + This option has no effect unless bf(--checksum, -c) was also specified. It + also only affects the current side of the transfer, so if you want the diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo --- a/rsyncd.conf.yo +++ b/rsyncd.conf.yo -@@ -281,6 +281,20 @@ locking on this file to ensure that the max connections limit is not - exceeded for the modules sharing the lock file. - The default is tt(/var/run/rsyncd.lock). - -+dit(bf(checksum updating)) This option tells rsync to update/create the -+checksum information in the per-directory checksum files when users copy -+files using the bf(--checksum) option. Any file that has changed since it -+was last checksummed (or is not mentioned) has its data updated in the -+.rsyncsums file. -+ -+Note that this updating will occur even if the module is listed as being -+read-only. If you want to hide these files (and you will almost always -+want to do), add ".rsyncsums" to the module's exclude setting. -+ -+Note also that the client's command-line option, bf(--checksum-updating), -+has no effect on a daemon. A daemon will only update/create checksum files -+if this config option is true. -+ +@@ -284,13 +284,15 @@ The default is tt(/var/run/rsyncd.lock). + dit(bf(checksum files)) This option tells rsync to make use of any cached + checksum information it finds in per-directory .rsyncsums files when the + current transfer is using the bf(--checksum) option. The value can be set +-to either "lax", "strict", or "none" -- see the client's bf(--sumfiles) +-option for what these choices do. ++to either "lax", "strict", "+lax", "+strict", "++lax", "++strict", or ++"none". See the client's bf(--sumfiles) option for what these choices do. + + Note also that the client's command-line option, bf(--sumfiles), has no + effect on a daemon. A daemon will only access checksum files if this +-config option tells it to. See also the bf(exclude) directive for a way +-to hide the .rsyncsums files from the user. ++config option tells it to. You can configure updating of the .rsyncsums ++files even if the module itself is configured to be read-only. See also ++the bf(exclude) directive for a way to hide the .rsyncsums files from the ++user. + dit(bf(read only)) The "read only" option determines whether clients will be able to upload files or not. If "read only" is true then any - attempted uploads will fail. If "read only" is false then uploads will diff --git a/checksum-xattr.diff b/checksum-xattr.diff index c707287..b5020e6 100644 --- a/checksum-xattr.diff +++ b/checksum-xattr.diff @@ -11,7 +11,7 @@ To use this patch, run these commands for a successful build: diff --git a/flist.c b/flist.c --- a/flist.c +++ b/flist.c -@@ -1214,7 +1214,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, +@@ -1216,7 +1216,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, } #endif @@ -24,7 +24,7 @@ diff --git a/flist.c b/flist.c diff --git a/generator.c b/generator.c --- a/generator.c +++ b/generator.c -@@ -683,7 +683,8 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st) +@@ -703,7 +703,8 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st) of the file time to determine whether to sync */ if (always_checksum > 0 && S_ISREG(st->st_mode)) { char sum[MAX_DIGEST_LEN]; diff --git a/copy-devices.diff b/copy-devices.diff index a278af4..cd87197 100644 --- a/copy-devices.diff +++ b/copy-devices.diff @@ -19,7 +19,7 @@ diff --git a/generator.c b/generator.c extern int preserve_specials; extern int preserve_hard_links; extern int preserve_executability; -@@ -1611,7 +1612,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, +@@ -1632,7 +1633,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, goto cleanup; } diff --git a/crtimes.diff b/crtimes.diff index 09491f0..5a989c8 100644 --- a/crtimes.diff +++ b/crtimes.diff @@ -3,7 +3,7 @@ create times on OS X. To use this patch, run these commands for a successful build: - patch -p1 modtime; @@ -71,7 +71,7 @@ diff --git a/flist.c b/flist.c #ifdef SUPPORT_HARD_LINKS if (tmp_dev != 0) { -@@ -532,6 +540,8 @@ static void send_file_entry(int f, struct file_struct *file, int ndx, int first_ +@@ -534,6 +542,8 @@ static void send_file_entry(int f, struct file_struct *file, int ndx, int first_ else write_int(f, modtime); } @@ -79,17 +79,17 @@ diff --git a/flist.c b/flist.c + write_varlong(f, crtime, 4); if (!(xflags & XMIT_SAME_MODE)) write_int(f, to_wire_mode(mode)); - #ifdef SUPPORT_FLAGS -@@ -624,7 +634,7 @@ static void send_file_entry(int f, struct file_struct *file, int ndx, int first_ + #ifdef SUPPORT_FILEFLAGS +@@ -626,7 +636,7 @@ static void send_file_entry(int f, struct file_struct *file, int ndx, int first_ static struct file_struct *recv_file_entry(struct file_list *flist, int xflags, int f) { - static int64 modtime; + static int64 modtime, crtime; static mode_t mode; - #ifdef SUPPORT_FLAGS + #ifdef SUPPORT_FILEFLAGS static uint32 fileflags; -@@ -758,6 +768,19 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -760,6 +770,19 @@ static struct file_struct *recv_file_entry(struct file_list *flist, } else modtime = read_int(f); } @@ -109,7 +109,7 @@ diff --git a/flist.c b/flist.c if (!(xflags & XMIT_SAME_MODE)) mode = from_wire_mode(read_int(f)); -@@ -898,6 +921,8 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -900,6 +923,8 @@ static struct file_struct *recv_file_entry(struct file_list *flist, F_GROUP(file) = gid; file->flags |= gid_flags; } @@ -118,9 +118,9 @@ diff --git a/flist.c b/flist.c if (unsort_ndx) F_NDX(file) = flist->used + flist->ndx_start; -@@ -1234,6 +1259,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, +@@ -1236,6 +1261,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, F_OWNER(file) = st.st_uid; - if (gid_ndx) + if (gid_ndx) /* Check gid_ndx instead of preserve_gid for del support */ F_GROUP(file) = st.st_gid; + if (crtimes_ndx) + f_crtime_set(file, get_create_time(fname)); @@ -145,16 +145,8 @@ diff --git a/generator.c b/generator.c +extern int preserve_fileflags; extern int preserve_hard_links; extern int preserve_executability; - extern int preserve_perms; -@@ -139,6 +141,7 @@ enum delret { - /* Forward declaration for delete_item(). */ - static enum delret delete_dir_contents(char *fname, int flags); - -+ - static int is_backup_file(char *fn) - { - int k = strlen(fn) - backup_suffix_len; -@@ -603,6 +606,13 @@ int unchanged_attrs(const char *fname, struct file_struct *file, stat_x *sxp) + extern int preserve_fileflags; +@@ -612,6 +614,13 @@ int unchanged_attrs(const char *fname, struct file_struct *file, stat_x *sxp) if (gid_ndx && !(file->flags & FLAG_SKIP_GROUP) && sxp->st.st_gid != (gid_t)F_GROUP(file)) return 0; @@ -168,7 +160,7 @@ diff --git a/generator.c b/generator.c #ifdef SUPPORT_ACLS if (preserve_acls && !S_ISLNK(file->mode)) { if (!ACL_READY(*sxp)) -@@ -642,6 +652,12 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre +@@ -651,6 +660,12 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre && (!(iflags & ITEM_XNAME_FOLLOWS) || *xname)) || (keep_time && cmp_time(file->modtime, sxp->st.st_mtime) != 0)) iflags |= ITEM_REPORT_TIME; @@ -181,7 +173,7 @@ diff --git a/generator.c b/generator.c #if !defined HAVE_LCHMOD && !defined HAVE_SETATTRLIST if (S_ISLNK(file->mode)) { ; -@@ -1182,6 +1198,7 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx, +@@ -1196,6 +1211,7 @@ static int try_dests_non(struct file_struct *file, char *fname, int ndx, static void list_file_entry(struct file_struct *f) { char permbuf[PERMSTRING_SIZE]; @@ -189,7 +181,7 @@ diff --git a/generator.c b/generator.c double len; if (!F_IS_ACTIVE(f)) { -@@ -1196,14 +1213,16 @@ static void list_file_entry(struct file_struct *f) +@@ -1210,14 +1226,16 @@ static void list_file_entry(struct file_struct *f) #ifdef SUPPORT_LINKS if (preserve_links && S_ISLNK(f->mode)) { @@ -208,7 +200,7 @@ diff --git a/generator.c b/generator.c f_name(f, NULL)); } } -@@ -1290,6 +1309,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, +@@ -1304,6 +1322,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, return; } } @@ -251,15 +243,16 @@ diff --git a/ifuncs.h b/ifuncs.h diff --git a/log.c b/log.c --- a/log.c +++ b/log.c -@@ -644,7 +644,7 @@ static void log_formatted(enum logcode code, const char *format, const char *op, - c[5] = !(iflags & ITEM_REPORT_PERMS) ? '.' : 'p'; - c[6] = !(iflags & ITEM_REPORT_OWNER) ? '.' : 'o'; - c[7] = !(iflags & ITEM_REPORT_GROUP) ? '.' : 'g'; -- c[8] = !(iflags & ITEM_REPORT_ATIME) ? '.' : 'u'; -+ c[8] = !(iflags & ITEM_REPORT_CRTIME) ? '.' : 'n'; +@@ -647,7 +647,8 @@ static void log_formatted(enum logcode code, const char *format, const char *op, + c[8] = !(iflags & ITEM_REPORT_FFLAGS) ? '.' : 'f'; c[9] = !(iflags & ITEM_REPORT_ACL) ? '.' : 'a'; c[10] = !(iflags & ITEM_REPORT_XATTR) ? '.' : 'x'; - c[11] = '\0'; +- c[11] = '\0'; ++ c[11] = !(iflags & ITEM_REPORT_CRTIME) ? '.' : 'n'; ++ c[12] = '\0'; + + if (iflags & (ITEM_IS_NEW|ITEM_MISSING_DATA)) { + char ch = iflags & ITEM_IS_NEW ? '+' : '?'; diff --git a/options.c b/options.c --- a/options.c +++ b/options.c @@ -271,15 +264,15 @@ diff --git a/options.c b/options.c int update_only = 0; int cvs_exclude = 0; int dry_run = 0; -@@ -358,6 +359,7 @@ void usage(enum logcode F) +@@ -361,6 +362,7 @@ void usage(enum logcode F) rprintf(F," -D same as --devices --specials\n"); rprintf(F," -t, --times preserve modification times\n"); rprintf(F," -O, --omit-dir-times omit directories from --times\n"); -+ rprintf(F," -N, --crtimes preserve create (newness) times\n"); ++ rprintf(F," -N, --crtimes preserve create times (newness)\n"); rprintf(F," --super receiver attempts super-user activities\n"); #ifdef SUPPORT_XATTRS rprintf(F," --fake-super store/recover privileged attrs using xattrs\n"); -@@ -495,6 +497,9 @@ static struct poptOption long_options[] = { +@@ -505,6 +507,9 @@ static struct poptOption long_options[] = { {"times", 't', POPT_ARG_VAL, &preserve_times, 2, 0, 0 }, {"no-times", 0, POPT_ARG_VAL, &preserve_times, 0, 0, 0 }, {"no-t", 0, POPT_ARG_VAL, &preserve_times, 0, 0, 0 }, @@ -289,7 +282,7 @@ diff --git a/options.c b/options.c {"omit-dir-times", 'O', POPT_ARG_VAL, &omit_dir_times, 1, 0, 0 }, {"no-omit-dir-times",0, POPT_ARG_VAL, &omit_dir_times, 0, 0, 0 }, {"no-O", 0, POPT_ARG_VAL, &omit_dir_times, 0, 0, 0 }, -@@ -1764,6 +1769,8 @@ void server_options(char **args, int *argc_p) +@@ -1772,6 +1777,8 @@ void server_options(char **args, int *argc_p) argstr[x++] = 'D'; if (preserve_times) argstr[x++] = 't'; @@ -301,7 +294,7 @@ diff --git a/options.c b/options.c diff --git a/rsync.c b/rsync.c --- a/rsync.c +++ b/rsync.c -@@ -439,6 +439,14 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp, +@@ -437,6 +437,14 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp, else if (receiver_symlink_times) file->flags |= FLAG_TIME_FAILED; } @@ -316,15 +309,15 @@ diff --git a/rsync.c b/rsync.c change_uid = am_root && uid_ndx && sxp->st.st_uid != (uid_t)F_OWNER(file); change_gid = gid_ndx && !(file->flags & FLAG_SKIP_GROUP) -@@ -576,7 +584,7 @@ int finish_transfer(const char *fname, const char *fnametmp, - +@@ -580,7 +588,7 @@ int finish_transfer(const char *fname, const char *fnametmp, /* Change permissions before putting the file into place. */ set_file_attrs(fnametmp, file, NULL, fnamecmp, -- ok_to_set_time ? 0 : ATTRS_SKIP_MTIME); -+ ok_to_set_time ? 0 : ATTRS_SKIP_MTIME | ATTRS_SKIP_CRTIME); + ATTRS_DELAY_IMMUTABLE +- | (ok_to_set_time ? 0 : ATTRS_SKIP_MTIME)); ++ | (ok_to_set_time ? 0 : ATTRS_SKIP_MTIME | ATTRS_SKIP_CRTIME)); - #ifdef SUPPORT_FLAGS - if (preserve_fileflags) + /* move tmp file over real file */ + if (verbose > 2) @@ -611,7 +619,7 @@ int finish_transfer(const char *fname, const char *fnametmp, do_set_file_attrs: @@ -345,15 +338,15 @@ diff --git a/rsync.h b/rsync.h #define XMIT_SAME_FLAGS (1<<14) /* protocols ?? - now */ /* These flags are used in the live flist data. */ -@@ -150,6 +151,7 @@ - +@@ -151,6 +152,7 @@ #define ATTRS_REPORT (1<<0) #define ATTRS_SKIP_MTIME (1<<1) -+#define ATTRS_SKIP_CRTIME (1<<2) + #define ATTRS_DELAY_IMMUTABLE (1<<2) ++#define ATTRS_SKIP_CRTIME (1<<3) #define FULL_FLUSH 1 #define NORMAL_FLUSH 0 -@@ -166,7 +168,7 @@ +@@ -167,7 +169,7 @@ #define FNAMECMP_FUZZY 0x83 /* For use by the itemize_changes code */ @@ -362,7 +355,7 @@ diff --git a/rsync.h b/rsync.h #define ITEM_REPORT_CHECKSUM (1<<1) #define ITEM_REPORT_SIZE (1<<2) #define ITEM_REPORT_TIME (1<<3) -@@ -636,6 +638,7 @@ extern int file_extra_cnt; +@@ -645,6 +647,7 @@ extern int file_extra_cnt; extern int inc_recurse; extern int uid_ndx; extern int gid_ndx; @@ -370,7 +363,7 @@ diff --git a/rsync.h b/rsync.h extern int fileflags_ndx; extern int acls_ndx; extern int xattrs_ndx; -@@ -643,6 +646,7 @@ extern int xattrs_ndx; +@@ -652,6 +655,7 @@ extern int xattrs_ndx; #define FILE_STRUCT_LEN (offsetof(struct file_struct, basename)) #define EXTRA_LEN (sizeof (union file_extras)) #define PTR_EXTRA_CNT ((sizeof (char *) + EXTRA_LEN - 1) / EXTRA_LEN) @@ -378,7 +371,7 @@ diff --git a/rsync.h b/rsync.h #define DEV_EXTRA_CNT 2 #define DIRNODE_EXTRA_CNT 3 #define SUM_EXTRA_CNT ((MAX_DIGEST_LEN + EXTRA_LEN - 1) / EXTRA_LEN) -@@ -897,6 +901,7 @@ typedef struct { +@@ -910,6 +914,7 @@ typedef struct { typedef struct { STRUCT_STAT st; @@ -393,44 +386,42 @@ diff --git a/rsync.yo b/rsync.yo -D same as --devices --specials -t, --times preserve modification times -O, --omit-dir-times omit directories from --times -+ -N, --crtimes preserve create (newness) times ++ -N, --crtimes preserve create times (newness) --super receiver attempts super-user activities --fake-super store/recover privileged attrs using xattrs -S, --sparse handle sparse files efficiently -@@ -996,6 +997,9 @@ it is preserving modification times (see bf(--times)). If NFS is sharing +@@ -1015,6 +1016,9 @@ it is preserving modification times (see bf(--times)). If NFS is sharing the directories on the receiving side, it is a good idea to use bf(-O). This option is inferred if you use bf(--backup) without bf(--backup-dir). -+dit(bf(-N, --crtimes)) This tells rsync to set the create (newness) times of ++dit(bf(-N, --crtimes)) This tells rsync to set the create times (newness) of +the destination files to the same value as the source files. + dit(bf(--super)) This tells the receiving side to attempt super-user activities even if the receiving rsync wasn't run by the super-user. These activities include: preserving users via the bf(--owner) option, preserving -@@ -1647,7 +1651,7 @@ with older versions of rsync, but that also turns on the output of other +@@ -1667,7 +1671,7 @@ with older versions of rsync, but that also turns on the output of other verbose messages). The "%i" escape has a cryptic output that is 11 letters long. The general --format is like the string bf(YXcstpoguax), where bf(Y) is replaced by the -+format is like the string bf(YXcstpognax), where bf(Y) is replaced by the +-format is like the string bf(YXcstpogfax), where bf(Y) is replaced by the ++format is like the string bf(YXcstpogfaxn), where bf(Y) is replaced by the type of update being done, bf(X) is replaced by the file-type, and the other letters represent attributes that may be output if they are being modified. -@@ -1698,8 +1702,8 @@ quote(itemization( - sender's value (requires bf(--owner) and super-user privileges). - it() A bf(g) means the group is different and is being updated to the - sender's value (requires bf(--group) and the authority to set the group). -- it() The bf(u) slot is reserved for reporting update (access) time changes -- (a feature that is not yet released). -+ it() A bf(n) means the create (newness) time is different and is being -+ updated to the sender's value (requires bf(--crtimes)). +@@ -1721,6 +1725,8 @@ quote(itemization( + it() The bf(f) means that the fileflags information changed. it() The bf(a) means that the ACL information changed. - it() The bf(x) slot is reserved for reporting extended attribute changes - (a feature that is not yet released). + it() The bf(x) means that the extended attribute information changed. ++ it() A bf(n) means the create time (newness) is different and is being ++ updated to the sender's value (requires bf(--crtimes)). + )) + + One other output is possible: when deleting files, the "%i" will output diff --git a/syscall.c b/syscall.c --- a/syscall.c +++ b/syscall.c -@@ -36,6 +36,11 @@ extern int list_only; +@@ -37,6 +37,11 @@ extern int force_change; extern int preserve_perms; extern int preserve_executability; @@ -442,7 +433,7 @@ diff --git a/syscall.c b/syscall.c #define RETURN_ERROR_IF(x,e) \ do { \ if (x) { \ -@@ -300,3 +305,33 @@ char *d_name(struct dirent *di) +@@ -403,3 +408,33 @@ char *d_name(struct dirent *di) return di->d_name; #endif } @@ -505,6 +496,22 @@ new file mode 100644 + +# The script would have aborted on error, so getting here means we've won. +exit 0 +diff --git a/testsuite/rsync.fns b/testsuite/rsync.fns +--- a/testsuite/rsync.fns ++++ b/testsuite/rsync.fns +@@ -24,9 +24,9 @@ todir="$tmpdir/to" + chkdir="$tmpdir/chk" + + # For itemized output: +-all_plus='+++++++++' +-allspace=' ' +-dots='.....' # trailing dots after changes ++all_plus='++++++++++' ++allspace=' ' ++dots='......' # trailing dots after changes + + # Berkley's nice. + PATH="$PATH:/usr/ucb" diff --git a/tls.c b/tls.c --- a/tls.c +++ b/tls.c @@ -513,7 +520,7 @@ diff --git a/tls.c b/tls.c #endif +static int display_crtimes = 0; -+ ++ static void failed(char const *what, char const *where) { fprintf(stderr, PROGRAM ": %s %s: %s\n", @@ -602,7 +609,7 @@ diff --git a/tls.c b/tls.c fprintf(F,"usage: " PROGRAM " [OPTIONS] FILE ...\n"); fprintf(F,"Trivial file listing program for portably checking rsync\n"); fprintf(F,"\nOptions:\n"); -+ fprintf(F," -N, --crtimes display create (newness) times\n"); ++ fprintf(F," -N, --crtimes display create times (newness)\n"); #ifdef SUPPORT_XATTRS fprintf(F," -f, --fake-super display attributes including fake-super xattrs\n"); #endif diff --git a/date-only.diff b/date-only.diff index e5c2819..f9dcb98 100644 --- a/date-only.diff +++ b/date-only.diff @@ -25,7 +25,7 @@ diff --git a/generator.c b/generator.c extern int size_only; extern OFF_T max_size; extern OFF_T min_size; -@@ -676,6 +677,9 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre +@@ -696,6 +697,9 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre /* Perform our quick-check heuristic for determining if a file is unchanged. */ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st) { diff --git a/detect-renamed-lax.diff b/detect-renamed-lax.diff index 20fac0f..7be49e2 100644 --- a/detect-renamed-lax.diff +++ b/detect-renamed-lax.diff @@ -34,7 +34,7 @@ diff --git a/generator.c b/generator.c diff = u_strcmp(fmid->basename, f->basename); if (diff == 0) { good_match = mid; -@@ -1907,6 +1909,21 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, +@@ -1927,6 +1929,21 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, fnamecmp = partialptr; fnamecmp_type = FNAMECMP_PARTIAL_DIR; statret = 0; diff --git a/detect-renamed.diff b/detect-renamed.diff index d4103e0..1243447 100644 --- a/detect-renamed.diff +++ b/detect-renamed.diff @@ -54,7 +54,7 @@ diff --git a/compat.c b/compat.c diff --git a/flist.c b/flist.c --- a/flist.c +++ b/flist.c -@@ -61,6 +61,7 @@ extern int non_perishable_cnt; +@@ -63,6 +63,7 @@ extern int non_perishable_cnt; extern int prune_empty_dirs; extern int copy_links; extern int copy_unsafe_links; @@ -62,7 +62,7 @@ diff --git a/flist.c b/flist.c extern int protocol_version; extern int sanitize_paths; extern int munge_symlinks; -@@ -118,6 +119,8 @@ static int64 tmp_dev, tmp_ino; +@@ -120,6 +121,8 @@ static int64 tmp_dev, tmp_ino; #endif static char tmp_sum[MAX_DIGEST_LEN]; @@ -71,7 +71,7 @@ diff --git a/flist.c b/flist.c static char empty_sum[MAX_DIGEST_LEN]; static int flist_count_offset; /* for --delete --progress */ static int dir_count = 0; -@@ -261,6 +264,45 @@ static int is_excluded(char *fname, int is_dir, int filter_level) +@@ -263,6 +266,45 @@ static int is_excluded(char *fname, int is_dir, int filter_level) return 0; } @@ -117,7 +117,7 @@ diff --git a/flist.c b/flist.c static void send_directory(int f, struct file_list *flist, char *fbuf, int len, int flags); -@@ -2178,6 +2220,25 @@ struct file_list *recv_file_list(int f) +@@ -2180,6 +2222,25 @@ struct file_list *recv_file_list(int f) flist_sort_and_clean(flist, relative_paths); @@ -176,7 +176,7 @@ diff --git a/generator.c b/generator.c -/* For calling delete_item() and delete_dir_contents(). */ +/* For calling delete_item(), delete_dir_contents(), and delete_in_dir(). */ - #define DEL_OWNED_BY_US (1<<0) /* file/dir has our uid */ + #define DEL_NO_UID_WRITE (1<<0) /* file/dir has our uid w/o write perm */ #define DEL_RECURSE (1<<1) /* if dir, delete all contents */ #define DEL_DIR_IS_EMPTY (1<<2) /* internal delete_FUNCTIONS use only */ @@ -124,6 +127,7 @@ static const char *solo_file = NULL; @@ -307,9 +307,9 @@ diff --git a/generator.c b/generator.c + * + * Also note: --detect-rename may use this routine with DEL_NO_DELETIONS set! */ - static enum delret delete_item(char *fbuf, int mode, int flags) + static enum delret delete_item(char *fbuf, uint16 mode, uint16 flags) { -@@ -173,6 +287,8 @@ static enum delret delete_item(char *fbuf, int mode, int flags) +@@ -182,6 +296,8 @@ static enum delret delete_item(char *fbuf, uint16 mode, uint16 flags) goto check_ret; /* OK: try to delete the directory. */ } @@ -318,16 +318,16 @@ diff --git a/generator.c b/generator.c if (!(flags & DEL_MAKE_ROOM) && max_delete >= 0 && ++deletion_count > max_delete) return DR_AT_LIMIT; -@@ -228,6 +344,8 @@ static enum delret delete_item(char *fbuf, int mode, int flags) +@@ -237,6 +353,8 @@ static enum delret delete_item(char *fbuf, uint16 mode, uint16 flags) * its contents, otherwise just checks for content. Returns DR_SUCCESS or * DR_NOT_EMPTY. Note that fname must point to a MAXPATHLEN buffer! (The * buffer is used for recursion, but returned unchanged.) + * + * Note: --detect-rename may use this routine with DEL_NO_DELETIONS set! */ - static enum delret delete_dir_contents(char *fname, int flags) + static enum delret delete_dir_contents(char *fname, uint16 flags) { -@@ -247,7 +365,9 @@ static enum delret delete_dir_contents(char *fname, int flags) +@@ -256,7 +374,9 @@ static enum delret delete_dir_contents(char *fname, uint16 flags) save_filters = push_local_filters(fname, dlen); non_perishable_cnt = 0; @@ -337,8 +337,8 @@ diff --git a/generator.c b/generator.c ret = non_perishable_cnt ? DR_NOT_EMPTY : DR_SUCCESS; if (!dirlist->used) -@@ -290,7 +410,8 @@ static enum delret delete_dir_contents(char *fname, int flags) - do_chmod(fname, fp->mode |= S_IWUSR); +@@ -296,7 +416,8 @@ static enum delret delete_dir_contents(char *fname, uint16 flags) + if (S_ISDIR(fp->mode)) { if (delete_dir_contents(fname, flags | DEL_RECURSE) != DR_SUCCESS) ret = DR_NOT_EMPTY; - } @@ -347,7 +347,7 @@ diff --git a/generator.c b/generator.c if (delete_item(fname, fp->mode, flags) != DR_SUCCESS) ret = DR_NOT_EMPTY; } -@@ -451,13 +572,18 @@ static void do_delayed_deletions(char *delbuf) +@@ -461,13 +582,18 @@ static void do_delayed_deletions(char *delbuf) * all the --delete-WHEN options. Note that the fbuf pointer must point to a * MAXPATHLEN buffer with the name of the directory in it (the functions we * call will append names onto the end, but the old dir value will be restored @@ -358,7 +358,7 @@ diff --git a/generator.c b/generator.c + * Note: --detect-rename may use this routine with DEL_NO_DELETIONS set! + */ +static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t *fs_dev, -+ int flags) ++ int del_flags) { static int already_warned = 0; struct file_list *dirlist; @@ -367,16 +367,10 @@ diff --git a/generator.c b/generator.c + char *p, delbuf[MAXPATHLEN]; + unsigned remainder; + int dlen, i, restore_dot = 0; + int save_uid_ndx = uid_ndx; if (!fbuf) { - change_local_filter_dir(NULL, 0, 0); -@@ -467,21 +593,28 @@ static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t *fs_dev) - if (verbose > 2) - rprintf(FINFO, "delete_in_dir(%s)\n", fbuf); - -+ flags |= DEL_RECURSE; -+ - if (allowed_lull) +@@ -482,17 +608,22 @@ static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t *fs_dev) maybe_send_keepalive(); if (io_error && !ignore_errors) { @@ -392,7 +386,7 @@ diff --git a/generator.c b/generator.c - "IO error encountered -- skipping file deletion\n"); - already_warned = 1; - return; -+ flags |= DEL_NO_DELETIONS; ++ del_flags |= DEL_NO_DELETIONS; } dlen = strlen(fbuf); @@ -404,7 +398,7 @@ diff --git a/generator.c b/generator.c if (one_file_system) { if (file->flags & FLAG_TOP_DIR) filesystem_dev = *fs_dev; -@@ -491,6 +624,14 @@ static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t *fs_dev) +@@ -505,6 +636,14 @@ static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t *fs_dev) dirlist = get_dirlist(fbuf, dlen, 0); @@ -419,7 +413,7 @@ diff --git a/generator.c b/generator.c /* If an item in dirlist is not found in flist, delete it * from the filesystem. */ for (i = dirlist->used; i--; ) { -@@ -503,18 +644,26 @@ static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t *fs_dev) +@@ -517,19 +656,28 @@ static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t *fs_dev) f_name(fp, NULL)); continue; } @@ -428,19 +422,19 @@ diff --git a/generator.c b/generator.c + look_for_rename(fp, fbuf); + } if (flist_find(cur_flist, fp) < 0) { -- int flags = DEL_RECURSE -- | (!uid_ndx || (uid_t)F_OWNER(fp) == our_uid ? DEL_OWNED_BY_US : 0); -+ int own_flag = (!uid_ndx || (uid_t)F_OWNER(fp) == our_uid ? DEL_OWNED_BY_US : 0); + int flags = DEL_RECURSE; + if (!(fp->mode & S_IWUSR) && !am_root && (uid_t)F_OWNER(fp) == our_uid) + flags |= DEL_NO_UID_WRITE; f_name(fp, delbuf); - if (delete_during == 2) { - if (!remember_delete(fp, delbuf, flags)) -+ if (delete_during == 2 && !(flags & DEL_NO_DELETIONS)) { -+ if (!remember_delete(fp, delbuf, own_flag | flags)) ++ if (delete_during == 2 && !(del_flags & DEL_NO_DELETIONS)) { ++ if (!remember_delete(fp, delbuf, del_flags | flags)) break; } else - delete_item(delbuf, fp->mode, flags); - } -+ delete_item(delbuf, fp->mode, own_flag | flags); ++ delete_item(delbuf, fp->mode, del_flags | flags); + } else if (detect_renamed && S_ISDIR(fp->mode)) + unexplored_dirs++; } @@ -450,9 +444,9 @@ diff --git a/generator.c b/generator.c + fbuf[dlen] = '\0'; + flist_free(dirlist); - } -@@ -544,9 +693,9 @@ static void do_delete_pass(void) + if (!save_uid_ndx) { +@@ -564,9 +712,9 @@ static void do_delete_pass(void) || !S_ISDIR(st.st_mode)) continue; @@ -464,7 +458,7 @@ diff --git a/generator.c b/generator.c if (do_progress && !am_server) rprintf(FINFO, " \r"); -@@ -1179,6 +1328,7 @@ static void list_file_entry(struct file_struct *f) +@@ -1199,6 +1347,7 @@ static void list_file_entry(struct file_struct *f) } } @@ -472,7 +466,7 @@ diff --git a/generator.c b/generator.c static int phase = 0; static int dflt_perms; -@@ -1428,8 +1578,12 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, +@@ -1449,8 +1598,12 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, } } else if (delete_during && f_out != -1 && !phase && dry_run < 2 @@ -487,7 +481,7 @@ diff --git a/generator.c b/generator.c goto cleanup; } -@@ -1707,8 +1861,14 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, +@@ -1728,8 +1881,14 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, goto cleanup; } #endif @@ -503,7 +497,7 @@ diff --git a/generator.c b/generator.c rsyserr(FERROR_XFER, stat_errno, "recv_generator: failed to stat %s", full_fname(fname)); goto cleanup; -@@ -2051,6 +2211,12 @@ void generate_files(int f_out, const char *local_name) +@@ -2072,6 +2231,12 @@ void generate_files(int f_out, const char *local_name) if (verbose > 2) rprintf(FINFO, "generator starting pid=%ld\n", (long)getpid()); @@ -516,7 +510,7 @@ diff --git a/generator.c b/generator.c if (delete_before && !solo_file && cur_flist->used > 0) do_delete_pass(); if (delete_during == 2) { -@@ -2061,7 +2227,7 @@ void generate_files(int f_out, const char *local_name) +@@ -2082,7 +2247,7 @@ void generate_files(int f_out, const char *local_name) } do_progress = 0; @@ -525,7 +519,7 @@ diff --git a/generator.c b/generator.c whole_file = 0; if (verbose >= 2) { rprintf(FINFO, "delta-transmission %s\n", -@@ -2099,7 +2265,7 @@ void generate_files(int f_out, const char *local_name) +@@ -2120,7 +2285,7 @@ void generate_files(int f_out, const char *local_name) dirdev = MAKEDEV(DEV_MAJOR(devp), DEV_MINOR(devp)); } else dirdev = MAKEDEV(0, 0); @@ -534,7 +528,7 @@ diff --git a/generator.c b/generator.c } } } -@@ -2142,7 +2308,21 @@ void generate_files(int f_out, const char *local_name) +@@ -2163,7 +2328,21 @@ void generate_files(int f_out, const char *local_name) } while ((cur_flist = cur_flist->next) != NULL); if (delete_during) diff --git a/downdate.diff b/downdate.diff index fb8094d..36683c0 100644 --- a/downdate.diff +++ b/downdate.diff @@ -18,7 +18,7 @@ diff --git a/generator.c b/generator.c extern int ignore_existing; extern int ignore_non_existing; extern int inplace; -@@ -1648,6 +1649,13 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, +@@ -1669,6 +1670,13 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, goto cleanup; } @@ -32,7 +32,7 @@ diff --git a/generator.c b/generator.c fnamecmp = fname; fnamecmp_type = FNAMECMP_FNAME; -@@ -1964,6 +1972,7 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo) +@@ -1985,6 +1993,7 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo) ignore_existing = -ignore_existing; ignore_non_existing = -ignore_non_existing; update_only = -update_only; @@ -40,7 +40,7 @@ diff --git a/generator.c b/generator.c always_checksum = -always_checksum; size_only = -size_only; append_mode = -append_mode; -@@ -1989,6 +1998,7 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo) +@@ -2010,6 +2019,7 @@ void check_for_finished_files(int itemizing, enum logcode code, int check_redo) ignore_existing = -ignore_existing; ignore_non_existing = -ignore_non_existing; update_only = -update_only; diff --git a/drop-cache.diff b/drop-cache.diff index 6c7f277..aca5d63 100644 --- a/drop-cache.diff +++ b/drop-cache.diff @@ -134,7 +134,7 @@ diff --git a/generator.c b/generator.c +#endif + /* For calling delete_item() and delete_dir_contents(). */ - #define DEL_OWNED_BY_US (1<<0) /* file/dir has our uid */ + #define DEL_NO_UID_WRITE (1<<0) /* file/dir has our uid w/o write perm */ #define DEL_RECURSE (1<<1) /* if dir, delete all contents */ diff --git a/options.c b/options.c --- a/options.c diff --git a/fileflags.diff b/fileflags.diff index cbe6989..eead498 100644 --- a/fileflags.diff +++ b/fileflags.diff @@ -3,15 +3,40 @@ Modified from a patch that was written by Rolf Grossmann. To use this patch, run these commands for a successful build: - patch -p1 mode; -+#ifdef SUPPORT_FLAGS -+ if (fileflags_ndx) { ++#ifdef SUPPORT_FILEFLAGS ++ if (preserve_fileflags) { + if (F_FFLAGS(file) == fileflags) + xflags |= XMIT_SAME_FLAGS; + else @@ -85,195 +110,202 @@ diff --git a/flist.c b/flist.c if ((preserve_devices && IS_DEVICE(mode)) || (preserve_specials && IS_SPECIAL(mode))) { -@@ -522,6 +534,10 @@ static void send_file_entry(int f, struct file_struct *file, int ndx, int first_ +@@ -524,6 +536,10 @@ static void send_file_entry(int f, struct file_struct *file, int ndx, int first_ } if (!(xflags & XMIT_SAME_MODE)) write_int(f, to_wire_mode(mode)); -+#ifdef SUPPORT_FLAGS -+ if (fileflags_ndx && !(xflags & XMIT_SAME_FLAGS)) ++#ifdef SUPPORT_FILEFLAGS ++ if (preserve_fileflags && !(xflags & XMIT_SAME_FLAGS)) + write_int(f, (int)fileflags); +#endif - if (uid_ndx && !(xflags & XMIT_SAME_UID)) { + if (preserve_uid && !(xflags & XMIT_SAME_UID)) { if (protocol_version < 30) write_int(f, uid); -@@ -610,6 +626,9 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -612,6 +628,9 @@ static struct file_struct *recv_file_entry(struct file_list *flist, { static int64 modtime; static mode_t mode; -+#ifdef SUPPORT_FLAGS ++#ifdef SUPPORT_FILEFLAGS + static uint32 fileflags; +#endif #ifdef SUPPORT_HARD_LINKS static int64 dev; #endif -@@ -744,6 +763,10 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -746,6 +765,10 @@ static struct file_struct *recv_file_entry(struct file_list *flist, if (chmod_modes && !S_ISLNK(mode)) mode = tweak_mode(mode, chmod_modes); -+#ifdef SUPPORT_FLAGS -+ if (fileflags_ndx && !(xflags & XMIT_SAME_FLAGS)) ++#ifdef SUPPORT_FILEFLAGS ++ if (preserve_fileflags && !(xflags & XMIT_SAME_FLAGS)) + fileflags = (uint32)read_int(f); +#endif - if (uid_ndx && !(xflags & XMIT_SAME_UID)) { + if (preserve_uid && !(xflags & XMIT_SAME_UID)) { if (protocol_version < 30) -@@ -865,6 +888,10 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -867,6 +890,10 @@ static struct file_struct *recv_file_entry(struct file_list *flist, OPT_EXTRA(file, 0)->unum = (uint32)(file_length >> 32); } file->mode = mode; -+#ifdef SUPPORT_FLAGS -+ if (fileflags_ndx) ++#ifdef SUPPORT_FILEFLAGS ++ if (preserve_fileflags) + F_FFLAGS(file) = fileflags; +#endif - if (uid_ndx) + if (preserve_uid) F_OWNER(file) = uid; - if (gid_ndx) { -@@ -1199,6 +1226,10 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, + if (preserve_gid) { +@@ -1201,6 +1228,10 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, OPT_EXTRA(file, 0)->unum = (uint32)(st.st_size >> 32); } file->mode = st.st_mode; -+#ifdef SUPPORT_FLAGS ++#if defined SUPPORT_FILEFLAGS || defined SUPPORT_FORCE_CHANGE + if (fileflags_ndx) + F_FFLAGS(file) = st.st_flags; +#endif - if (uid_ndx) + if (uid_ndx) /* Check uid_ndx instead of preserve_uid for del support */ F_OWNER(file) = st.st_uid; - if (gid_ndx) + if (gid_ndx) /* Check gid_ndx instead of preserve_gid for del support */ diff --git a/generator.c b/generator.c --- a/generator.c +++ b/generator.c -@@ -124,6 +124,7 @@ static const char *solo_file = NULL; - #define DEL_FOR_SYMLINK (1<<5) /* making room for a replacement symlink */ - #define DEL_FOR_DEVICE (1<<6) /* making room for a replacement device */ - #define DEL_FOR_SPECIAL (1<<7) /* making room for a replacement special */ -+#define DEL_AN_IMMUTABLE (1<<8) /* item has an immutable flag set */ - - #define DEL_MAKE_ROOM (DEL_FOR_FILE|DEL_FOR_DIR|DEL_FOR_SYMLINK|DEL_FOR_DEVICE|DEL_FOR_SPECIAL) - -@@ -163,6 +164,10 @@ static enum delret delete_item(char *fbuf, int mode, int flags) +@@ -43,8 +43,10 @@ extern int preserve_devices; + extern int preserve_specials; + extern int preserve_hard_links; + extern int preserve_executability; ++extern int preserve_fileflags; + extern int preserve_perms; + extern int preserve_times; ++extern int force_change; + extern int uid_ndx; + extern int gid_ndx; + extern int delete_mode; +@@ -162,7 +164,7 @@ static enum delret delete_item(char *fbuf, uint16 mode, uint16 flags) + } - if (!am_root && !(mode & S_IWUSR) && flags & DEL_OWNED_BY_US) - do_chmod(fbuf, mode |= S_IWUSR); -+#ifdef SUPPORT_FLAGS -+ if (fileflags_ndx && flags & DEL_AN_IMMUTABLE) -+ make_mutable(fbuf, mode, NODELETE_FLAGS); -+#endif + if (flags & DEL_NO_UID_WRITE) +- do_chmod(fbuf, mode | S_IWUSR); ++ do_chmod(fbuf, mode | S_IWUSR, NO_FFLAGS); if (S_ISDIR(mode) && !(flags & DEL_DIR_IS_EMPTY)) { - ignore_perishable = 1; -@@ -284,6 +289,14 @@ static enum delret delete_dir_contents(char *fname, int flags) - flags |= DEL_OWNED_BY_US; - else - flags &= ~DEL_OWNED_BY_US; -+#ifdef SUPPORT_FLAGS -+ if (fileflags_ndx) { -+ if (F_FFLAGS(fp) & NODELETE_FLAGS) -+ flags |= DEL_AN_IMMUTABLE; -+ else -+ flags &= ~DEL_AN_IMMUTABLE; + int save_uid_ndx = uid_ndx; +@@ -170,6 +172,13 @@ static enum delret delete_item(char *fbuf, uint16 mode, uint16 flags) + * delete_dir_contents() always calls us w/DEL_DIR_IS_EMPTY. */ + if (!uid_ndx) + uid_ndx = ++file_extra_cnt; ++#ifdef SUPPORT_FORCE_CHANGE ++ if (force_change) { ++ STRUCT_STAT st; ++ if (x_lstat(fbuf, &st, NULL) == 0) ++ make_mutable(fbuf, st.st_mode, st.st_flags, force_change); + } -+#endif ++#endif + ignore_perishable = 1; + /* If DEL_RECURSE is not set, this just reports emptiness. */ + ret = delete_dir_contents(fbuf, flags); +@@ -290,8 +299,12 @@ static enum delret delete_dir_contents(char *fname, uint16 flags) + } + + strlcpy(p, fp->basename, remainder); ++#ifdef SUPPORT_FORCE_CHANGE ++ if (force_change) ++ make_mutable(fname, fp->mode, F_FFLAGS(fp), force_change); ++#endif + if (!(fp->mode & S_IWUSR) && !am_root && (uid_t)F_OWNER(fp) == our_uid) +- do_chmod(fname, fp->mode | S_IWUSR); ++ do_chmod(fname, fp->mode | S_IWUSR, NO_FFLAGS); /* Save stack by recursing to ourself directly. */ if (S_ISDIR(fp->mode)) { - if (!am_root && !(fp->mode & S_IWUSR) && flags & DEL_OWNED_BY_US) -@@ -343,15 +356,22 @@ static int flush_delete_delay(void) + if (delete_dir_contents(fname, flags | DEL_RECURSE) != DR_SUCCESS) +@@ -585,6 +598,11 @@ int unchanged_attrs(const char *fname, struct file_struct *file, stat_x *sxp) + if (preserve_perms && !BITS_EQUAL(sxp->st.st_mode, file->mode, CHMOD_BITS)) + return 0; - static int remember_delete(struct file_struct *file, const char *fname, int flags) - { -- const char *plus = (!am_root && !(file->mode & S_IWUSR) && flags & DEL_OWNED_BY_US) -- ? "+" : ""; -+ char buf[16], *bp = buf; - int len; - -+ if (!am_root && !(file->mode & S_IWUSR) && flags & DEL_OWNED_BY_US) -+ *bp++ = '+'; -+#ifdef SUPPORT_FLAGS -+ if (flags & DEL_AN_IMMUTABLE) -+ *bp++ = '-'; -+#endif -+ *bp = '\0'; ++#ifdef SUPPORT_FILEFLAGS ++ if (preserve_fileflags && !S_ISLNK(file->mode) && sxp->st.st_flags != F_FFLAGS(file)) ++ return 0; ++#endif + - while (1) { - len = snprintf(deldelay_buf + deldelay_cnt, - deldelay_size - deldelay_cnt, - "%s%x %s%c", -- plus, (int)file->mode, fname, '\0'); -+ buf, (int)file->mode, fname, '\0'); - if ((deldelay_cnt += len) <= deldelay_size) - break; - if (deldelay_fd < 0 && !start_delete_delay_temp()) -@@ -364,10 +384,10 @@ static int remember_delete(struct file_struct *file, const char *fname, int flag - return 1; - } + if (preserve_executability && ((sxp->st.st_mode & 0111 ? 1 : 0) ^ (file->mode & 0111 ? 1 : 0))) + return 0; --static int read_delay_line(char *buf, int *own_flag_p) -+static int read_delay_line(char *buf, int *flags_p) - { - static int read_pos = 0; -- int j, len, mode; -+ int j, len, mode, flags = 0; - char *bp, *past_space; - - while (1) { -@@ -407,9 +427,15 @@ static int read_delay_line(char *buf, int *own_flag_p) - bp = deldelay_buf + read_pos; - if (*bp == '+') { - bp++; -- *own_flag_p = DEL_OWNED_BY_US; -- } else -- *own_flag_p = 0; -+ flags |= DEL_OWNED_BY_US; -+ } -+#ifdef SUPPORT_FLAGS -+ if (*bp == '-') { -+ bp++; -+ flags |= DEL_AN_IMMUTABLE; -+ } +@@ -646,6 +664,11 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre + if (gid_ndx && !(file->flags & FLAG_SKIP_GROUP) + && sxp->st.st_gid != (gid_t)F_GROUP(file)) + iflags |= ITEM_REPORT_GROUP; ++#ifdef SUPPORT_FILEFLAGS ++ if (preserve_fileflags && !S_ISLNK(file->mode) ++ && sxp->st.st_flags != F_FFLAGS(file)) ++ iflags |= ITEM_REPORT_FFLAGS; +#endif -+ *flags_p = flags; - - if (sscanf(bp, "%x ", &mode) != 1) { - invalid_data: -@@ -434,15 +460,15 @@ static int read_delay_line(char *buf, int *own_flag_p) - - static void do_delayed_deletions(char *delbuf) - { -- int mode, own_flag; -+ int mode, flags; - - if (deldelay_fd >= 0) { - if (deldelay_cnt && !flush_delete_delay()) - return; - lseek(deldelay_fd, 0, 0); - } -- while ((mode = read_delay_line(delbuf, &own_flag)) >= 0) -- delete_item(delbuf, mode, own_flag | DEL_RECURSE); -+ while ((mode = read_delay_line(delbuf, &flags)) >= 0) -+ delete_item(delbuf, mode, DEL_RECURSE | flags); - if (deldelay_fd >= 0) - close(deldelay_fd); - } -@@ -505,6 +531,9 @@ static void delete_in_dir(char *fbuf, struct file_struct *file, dev_t *fs_dev) + #ifdef SUPPORT_ACLS + if (preserve_acls && !S_ISLNK(file->mode)) { + if (!ACL_READY(*sxp)) +@@ -1391,6 +1414,10 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, + file->mode = dest_mode(file->mode, sx.st.st_mode, + dflt_perms, statret == 0); } - if (flist_find(cur_flist, fp) < 0) { - int flags = DEL_RECURSE -+#ifdef SUPPORT_FLAGS -+ | (fileflags_ndx && F_FFLAGS(fp) & NODELETE_FLAGS ? DEL_AN_IMMUTABLE : 0) ++#ifdef SUPPORT_FORCE_CHANGE ++ if (force_change && !preserve_fileflags) ++ F_FFLAGS(file) = sx.st.st_flags; +#endif - | (!uid_ndx || (uid_t)F_OWNER(fp) == our_uid ? DEL_OWNED_BY_US : 0); - f_name(fp, delbuf); - if (delete_during == 2) { -@@ -1329,6 +1358,10 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, - - if (statret == 0 && sx.st.st_uid == our_uid) - del_opts |= DEL_OWNED_BY_US; -+#ifdef SUPPORT_FLAGS -+ if (statret == 0 && fileflags_ndx && sx.st.st_flags & NODELETE_FLAGS) -+ del_opts |= DEL_AN_IMMUTABLE; + if (statret != 0 && basis_dir[0] != NULL) { + int j = try_dests_non(file, fname, ndx, fnamecmpbuf, &sx, + itemizing, code); +@@ -1427,10 +1454,15 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, + /* We need to ensure that the dirs in the transfer have writable + * permissions during the time we are putting files within them. + * This is then fixed after the transfer is done. */ ++#ifdef SUPPORT_FORCE_CHANGE ++ if (force_change && F_FFLAGS(file) & force_change ++ && make_mutable(fname, file->mode, F_FFLAGS(file), force_change)) ++ need_retouch_dir_perms = 1; ++#endif + #ifdef HAVE_CHMOD + if (!am_root && !(file->mode & S_IWUSR) && dir_tweaking) { + mode_t mode = file->mode | S_IWUSR; +- if (do_chmod(fname, mode) < 0) { ++ if (do_chmod(fname, mode, 0) < 0) { + rsyserr(FERROR_XFER, errno, + "failed to modify permissions on %s", + full_fname(fname)); +@@ -1461,6 +1493,10 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, + file->mode = dest_mode(file->mode, sx.st.st_mode, dflt_perms, + exists); + } ++#ifdef SUPPORT_FORCE_CHANGE ++ if (force_change && !preserve_fileflags) ++ F_FFLAGS(file) = sx.st.st_flags; +#endif - if (is_dir) { - if (!implied_dirs && file->flags & FLAG_IMPLIED_DIR) + #ifdef SUPPORT_HARD_LINKS + if (preserve_hard_links && F_HLINK_NOT_FIRST(file) +@@ -1948,9 +1984,13 @@ static void touch_up_dirs(struct file_list *flist, int ndx) + continue; + fname = f_name(file, NULL); + if (!(file->mode & S_IWUSR)) +- do_chmod(fname, file->mode); ++ do_chmod(fname, file->mode, 0); + if (need_retouch_dir_times) +- set_modtime(fname, file->modtime, file->mode); ++ set_modtime(fname, file->modtime, file->mode, 0); ++#ifdef SUPPORT_FORCE_CHANGE ++ if (force_change && F_FFLAGS(file) & force_change) ++ undo_make_mutable(fname, F_FFLAGS(file)); ++#endif + if (allowed_lull && !(counter % lull_mod)) + maybe_send_keepalive(); + else if (!(counter & 0xFF)) +diff --git a/log.c b/log.c +--- a/log.c ++++ b/log.c +@@ -644,7 +644,7 @@ static void log_formatted(enum logcode code, const char *format, const char *op, + c[5] = !(iflags & ITEM_REPORT_PERMS) ? '.' : 'p'; + c[6] = !(iflags & ITEM_REPORT_OWNER) ? '.' : 'o'; + c[7] = !(iflags & ITEM_REPORT_GROUP) ? '.' : 'g'; +- c[8] = !(iflags & ITEM_REPORT_ATIME) ? '.' : 'u'; ++ c[8] = !(iflags & ITEM_REPORT_FFLAGS) ? '.' : 'f'; + c[9] = !(iflags & ITEM_REPORT_ACL) ? '.' : 'a'; + c[10] = !(iflags & ITEM_REPORT_XATTR) ? '.' : 'x'; + c[11] = '\0'; diff --git a/options.c b/options.c --- a/options.c +++ b/options.c @@ -285,7 +317,15 @@ diff --git a/options.c b/options.c int preserve_executability = 0; int preserve_devices = 0; int preserve_specials = 0; -@@ -224,6 +225,7 @@ static void print_rsync_version(enum logcode f) +@@ -84,6 +85,7 @@ int implied_dirs = 1; + int numeric_ids = 0; + int allow_8bit_chars = 0; + int force_delete = 0; ++int force_change = 0; + int io_timeout = 0; + int allowed_lull = 0; + int prune_empty_dirs = 0; +@@ -224,6 +226,7 @@ static void print_rsync_version(enum logcode f) char const *links = "no "; char const *iconv = "no "; char const *ipv6 = "no "; @@ -293,17 +333,17 @@ diff --git a/options.c b/options.c STRUCT_STAT *dumstat; #if SUBPROTOCOL_VERSION != 0 -@@ -256,6 +258,9 @@ static void print_rsync_version(enum logcode f) +@@ -256,6 +259,9 @@ static void print_rsync_version(enum logcode f) #if defined HAVE_LUTIMES && defined HAVE_UTIMES symtimes = ""; #endif -+#ifdef SUPPORT_FLAGS ++#ifdef SUPPORT_FILEFLAGS + fileflags = ""; +#endif rprintf(f, "%s version %s protocol version %d%s\n", RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol); -@@ -269,8 +274,8 @@ static void print_rsync_version(enum logcode f) +@@ -269,8 +275,8 @@ static void print_rsync_version(enum logcode f) (int)(sizeof (int64) * 8)); rprintf(f, " %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n", got_socketpair, hardlinks, links, ipv6, have_inplace); @@ -314,40 +354,56 @@ diff --git a/options.c b/options.c #ifdef MAINTAINER_MODE rprintf(f, "Panic Action: \"%s\"\n", get_panic_action()); -@@ -337,6 +342,7 @@ void usage(enum logcode F) +@@ -337,6 +343,9 @@ void usage(enum logcode F) rprintf(F," -K, --keep-dirlinks treat symlinked dir on receiver as dir\n"); rprintf(F," -H, --hard-links preserve hard links\n"); rprintf(F," -p, --perms preserve permissions\n"); -+ rprintf(F," --fileflags preserve file-flags\n"); ++#ifdef SUPPORT_FILEFLAGS ++ rprintf(F," --fileflags preserve file-flags (aka chflags)\n"); ++#endif rprintf(F," -E, --executability preserve the file's executability\n"); rprintf(F," --chmod=CHMOD affect file and/or directory permissions\n"); #ifdef SUPPORT_ACLS -@@ -477,6 +483,8 @@ static struct poptOption long_options[] = { +@@ -374,7 +383,12 @@ void usage(enum logcode F) + rprintf(F," --delete-after receiver deletes after transfer, not during\n"); + rprintf(F," --delete-excluded also delete excluded files from destination dirs\n"); + rprintf(F," --ignore-errors delete even if there are I/O errors\n"); +- rprintf(F," --force force deletion of directories even if not empty\n"); ++ rprintf(F," --force-delete force deletion of directories even if not empty\n"); ++#ifdef SUPPORT_FORCE_CHANGE ++ rprintf(F," --force-change affect user-/system-immutable files/dirs\n"); ++ rprintf(F," --force-uchange affect user-immutable files/dirs\n"); ++ rprintf(F," --force-schange affect system-immutable files/dirs\n"); ++#endif + rprintf(F," --max-delete=NUM don't delete more than NUM files\n"); + rprintf(F," --max-size=SIZE don't transfer any file larger than SIZE\n"); + rprintf(F," --min-size=SIZE don't transfer any file smaller than SIZE\n"); +@@ -477,6 +491,10 @@ static struct poptOption long_options[] = { {"perms", 'p', POPT_ARG_VAL, &preserve_perms, 1, 0, 0 }, {"no-perms", 0, POPT_ARG_VAL, &preserve_perms, 0, 0, 0 }, {"no-p", 0, POPT_ARG_VAL, &preserve_perms, 0, 0, 0 }, ++#ifdef SUPPORT_FILEFLAGS + {"fileflags", 0, POPT_ARG_VAL, &preserve_fileflags, 1, 0, 0 }, + {"no-fileflags", 0, POPT_ARG_VAL, &preserve_fileflags, 0, 0, 0 }, ++#endif {"executability", 'E', POPT_ARG_NONE, &preserve_executability, 0, 0, 0 }, {"acls", 'A', POPT_ARG_NONE, 0, 'A', 0, 0 }, {"no-acls", 0, POPT_ARG_VAL, &preserve_acls, 0, 0, 0 }, -@@ -1311,6 +1319,15 @@ int parse_arguments(int *argc_p, const char ***argv_p, int frommain) - } - #endif - -+#ifndef SUPPORT_FLAGS -+ if (preserve_fileflags) { -+ snprintf(err_buf, sizeof err_buf, -+ "the --fileflags option is not supported on this %s\n", -+ am_server ? "server" : "client"); -+ return 0; -+ } +@@ -551,6 +569,13 @@ static struct poptOption long_options[] = { + {"remove-sent-files",0, POPT_ARG_VAL, &remove_source_files, 2, 0, 0 }, /* deprecated */ + {"remove-source-files",0,POPT_ARG_VAL, &remove_source_files, 1, 0, 0 }, + {"force", 0, POPT_ARG_NONE, &force_delete, 0, 0, 0 }, ++ {"force-delete", 0, POPT_ARG_NONE, &force_delete, 0, 0, 0 }, ++#ifdef SUPPORT_FORCE_CHANGE ++ {"force-change", 0, POPT_ARG_VAL, &force_change, ALL_IMMUTABLE, 0, 0 }, ++ {"no-force-change", 0, POPT_ARG_VAL, &force_change, 0, 0, 0 }, ++ {"force-uchange", 0, POPT_ARG_VAL, &force_change, USR_IMMUTABLE, 0, 0 }, ++ {"force-schange", 0, POPT_ARG_VAL, &force_change, SYS_IMMUTABLE, 0, 0 }, +#endif -+ - if (write_batch && read_batch) { - snprintf(err_buf, sizeof err_buf, - "--write-batch and --read-batch can not be used together\n"); -@@ -1826,6 +1843,9 @@ void server_options(char **args, int *argc_p) + {"ignore-errors", 0, POPT_ARG_NONE, &ignore_errors, 0, 0, 0 }, + {"max-delete", 0, POPT_ARG_INT, &max_delete, 0, 0, 0 }, + {0, 'F', POPT_ARG_NONE, 0, 'F', 0, 0 }, +@@ -1826,6 +1851,9 @@ void server_options(char **args, int *argc_p) if (xfer_dirs && !recurse && delete_mode && am_sender) args[ac++] = "--no-r"; @@ -357,6 +413,23 @@ diff --git a/options.c b/options.c if (do_compression && def_compress_level != Z_DEFAULT_COMPRESSION) { if (asprintf(&arg, "--compress-level=%d", def_compress_level) < 0) goto oom; +@@ -1913,6 +1941,16 @@ void server_options(char **args, int *argc_p) + args[ac++] = "--delete-excluded"; + if (force_delete) + args[ac++] = "--force"; ++#ifdef SUPPORT_FORCE_CHANGE ++ if (force_change) { ++ if (force_change == ALL_IMMUTABLE) ++ args[ac++] = "--force-change"; ++ else if (force_change == USR_IMMUTABLE) ++ args[ac++] = "--force-uchange"; ++ else if (force_change == SYS_IMMUTABLE) ++ args[ac++] = "--force-schange"; ++ } ++#endif + if (write_batch < 0) + args[ac++] = "--only-write-batch=X"; + if (am_root > 1) diff --git a/rsync.c b/rsync.c --- a/rsync.c +++ b/rsync.c @@ -368,11 +441,11 @@ diff --git a/rsync.c b/rsync.c extern int preserve_executability; extern int preserve_times; extern int am_root; -@@ -339,6 +340,41 @@ mode_t dest_mode(mode_t flist_mode, mode_t stat_mode, int dflt_perms, +@@ -339,6 +340,39 @@ mode_t dest_mode(mode_t flist_mode, mode_t stat_mode, int dflt_perms, return new_mode; } -+#ifdef SUPPORT_FLAGS ++#if defined SUPPORT_FILEFLAGS || defined SUPPORT_FORCE_CHANGE +/* Set a file's st_flags. */ +static int set_fileflags(const char *fname, uint32 fileflags) +{ @@ -387,37 +460,67 @@ diff --git a/rsync.c b/rsync.c +} + +/* Remove immutable flags from an object, so it can be altered/removed. */ -+void make_mutable(const char *fname, mode_t mode, uint32 fileflags) ++int make_mutable(const char *fname, mode_t mode, uint32 fileflags, uint32 iflags) +{ -+ if (S_ISLNK(mode)) -+ return; -+ -+ if (fileflags & NOCHANGE_FLAGS) -+ set_fileflags(fname, fileflags & ~NOCHANGE_FLAGS); ++ if (S_ISLNK(mode) || !(fileflags & iflags)) ++ return 0; ++ if (!set_fileflags(fname, fileflags & ~iflags)) ++ return -1; ++ return 1; +} + -+/* Undo a prior make_mutable() call. */ -+void undo_make_mutable(const char *fname, mode_t mode, uint32 fileflags) ++/* Undo a prior make_mutable() call that returned a 1. */ ++int undo_make_mutable(const char *fname, uint32 fileflags) +{ -+ if (S_ISLNK(mode)) -+ return; -+ -+ if (fileflags & NOCHANGE_FLAGS) -+ set_fileflags(fname, fileflags); ++ if (!set_fileflags(fname, fileflags)) ++ return -1; ++ return 1; +} +#endif + int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp, const char *fnamecmp, int flags) { -@@ -472,6 +508,15 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp, +@@ -392,7 +426,7 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp, + flags |= ATTRS_SKIP_MTIME; + if (!(flags & ATTRS_SKIP_MTIME) + && cmp_time(sxp->st.st_mtime, file->modtime) != 0) { +- int ret = set_modtime(fname, file->modtime, sxp->st.st_mode); ++ int ret = set_modtime(fname, file->modtime, sxp->st.st_mode, ST_FLAGS(sxp->st)); + if (ret < 0) { + rsyserr(FERROR_XFER, errno, "failed to set times on %s", + full_fname(fname)); +@@ -428,7 +462,8 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp, + if (am_root >= 0) { + if (do_lchown(fname, + change_uid ? (uid_t)F_OWNER(file) : sxp->st.st_uid, +- change_gid ? (gid_t)F_GROUP(file) : sxp->st.st_gid) != 0) { ++ change_gid ? (gid_t)F_GROUP(file) : sxp->st.st_gid, ++ sxp->st.st_mode, ST_FLAGS(sxp->st)) != 0) { + /* We shouldn't have attempted to change uid + * or gid unless have the privilege. */ + rsyserr(FERROR_XFER, errno, "%s %s failed", +@@ -460,7 +495,7 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp, + + #ifdef HAVE_CHMOD + if (!BITS_EQUAL(sxp->st.st_mode, new_mode, CHMOD_BITS)) { +- int ret = am_root < 0 ? 0 : do_chmod(fname, new_mode); ++ int ret = am_root < 0 ? 0 : do_chmod(fname, new_mode, ST_FLAGS(sxp->st)); + if (ret < 0) { + rsyserr(FERROR_XFER, errno, + "failed to set permissions on %s", +@@ -472,6 +507,19 @@ int set_file_attrs(const char *fname, struct file_struct *file, stat_x *sxp, } #endif -+#ifdef SUPPORT_FLAGS ++#ifdef SUPPORT_FILEFLAGS + if (preserve_fileflags && !S_ISLNK(sxp->st.st_mode) + && sxp->st.st_flags != F_FFLAGS(file)) { -+ if (!set_fileflags(fname, F_FFLAGS(file))) ++ uint32 fileflags = F_FFLAGS(file); ++ if (flags & ATTRS_DELAY_IMMUTABLE) ++ fileflags &= ~ALL_IMMUTABLE; ++ if (sxp->st.st_flags != fileflags ++ && !set_fileflags(fname, fileflags)) + goto cleanup; + updated = 1; + } @@ -426,24 +529,23 @@ diff --git a/rsync.c b/rsync.c if (verbose > 1 && flags & ATTRS_REPORT) { if (updated) rprintf(FCLIENT, "%s\n", fname); -@@ -533,6 +578,10 @@ int finish_transfer(const char *fname, const char *fnametmp, +@@ -531,7 +579,8 @@ int finish_transfer(const char *fname, const char *fnametmp, + + /* Change permissions before putting the file into place. */ set_file_attrs(fnametmp, file, NULL, fnamecmp, - ok_to_set_time ? 0 : ATTRS_SKIP_MTIME); +- ok_to_set_time ? 0 : ATTRS_SKIP_MTIME); ++ ATTRS_DELAY_IMMUTABLE ++ | (ok_to_set_time ? 0 : ATTRS_SKIP_MTIME)); -+#ifdef SUPPORT_FLAGS -+ if (preserve_fileflags) -+ make_mutable(fnametmp, file->mode, F_FFLAGS(file)); -+#endif /* move tmp file over real file */ if (verbose > 2) - rprintf(FINFO, "renaming %s to %s\n", fnametmp, fname); @@ -550,6 +599,10 @@ int finish_transfer(const char *fname, const char *fnametmp, } if (ret == 0) { /* The file was moved into place (not copied), so it's done. */ -+#ifdef SUPPORT_FLAGS -+ if (preserve_fileflags) -+ undo_make_mutable(fname, file->mode, F_FFLAGS(file)); ++#ifdef SUPPORT_FILEFLAGS ++ if (preserve_fileflags && F_FFLAGS(file) & ALL_IMMUTABLE) ++ set_fileflags(fname, F_FFLAGS(file)); +#endif return 1; } @@ -459,29 +561,52 @@ diff --git a/rsync.h b/rsync.h /* These flags are used in the live flist data. */ -@@ -452,6 +453,21 @@ typedef unsigned int size_t; +@@ -149,6 +150,7 @@ + + #define ATTRS_REPORT (1<<0) + #define ATTRS_SKIP_MTIME (1<<1) ++#define ATTRS_DELAY_IMMUTABLE (1<<2) + + #define FULL_FLUSH 1 + #define NORMAL_FLUSH 0 +@@ -174,6 +176,7 @@ + #define ITEM_REPORT_GROUP (1<<6) + #define ITEM_REPORT_ACL (1<<7) + #define ITEM_REPORT_XATTR (1<<8) ++#define ITEM_REPORT_FFLAGS (1<<9) + #define ITEM_BASIS_TYPE_FOLLOWS (1<<11) + #define ITEM_XNAME_FOLLOWS (1<<12) + #define ITEM_IS_NEW (1<<13) +@@ -452,6 +455,28 @@ typedef unsigned int size_t; #endif #endif ++#define NO_FFLAGS ((uint32)-1) ++ +#ifdef HAVE_CHFLAGS -+#define SUPPORT_FLAGS 1 ++#define SUPPORT_FILEFLAGS 1 ++#define SUPPORT_FORCE_CHANGE 1 +#endif + -+#ifdef SUPPORT_FLAGS ++#if defined SUPPORT_FILEFLAGS || defined SUPPORT_FORCE_CHANGE +#ifndef UF_NOUNLINK +#define UF_NOUNLINK 0 +#endif +#ifndef SF_NOUNLINK +#define SF_NOUNLINK 0 +#endif -+#define NODELETE_FLAGS (UF_IMMUTABLE|UF_NOUNLINK|SF_IMMUTABLE|SF_NOUNLINK) -+#define NOCHANGE_FLAGS (NODELETE_FLAGS|UF_APPEND|SF_APPEND) ++#define USR_IMMUTABLE (UF_IMMUTABLE|UF_NOUNLINK) ++#define SYS_IMMUTABLE (SF_IMMUTABLE|SF_NOUNLINK) ++#define ALL_IMMUTABLE (USR_IMMUTABLE|SYS_IMMUTABLE) ++#define ST_FLAGS(st) (st.st_flags) ++#else ++#define ST_FLAGS(st) NO_FFLAGS +#endif + /* Find a variable that is either exactly 32-bits or longer. * If some code depends on 32-bit truncation, it will need to * take special action in a "#if SIZEOF_INT32 > 4" section. */ -@@ -620,6 +636,7 @@ extern int file_extra_cnt; +@@ -620,6 +645,7 @@ extern int file_extra_cnt; extern int inc_recurse; extern int uid_ndx; extern int gid_ndx; @@ -489,11 +614,15 @@ diff --git a/rsync.h b/rsync.h extern int acls_ndx; extern int xattrs_ndx; -@@ -657,6 +674,7 @@ extern int xattrs_ndx; +@@ -657,6 +683,11 @@ extern int xattrs_ndx; /* When the associated option is on, all entries will have these present: */ #define F_OWNER(f) REQ_EXTRA(f, uid_ndx)->unum #define F_GROUP(f) REQ_EXTRA(f, gid_ndx)->unum ++#if defined SUPPORT_FILEFLAGS || defined SUPPORT_FORCE_CHANGE +#define F_FFLAGS(f) REQ_EXTRA(f, fileflags_ndx)->unum ++#else ++#define F_FFLAGS(f) NO_FFLAGS ++#endif #define F_ACL(f) REQ_EXTRA(f, acls_ndx)->num #define F_XATTR(f) REQ_EXTRA(f, xattrs_ndx)->num #define F_NDX(f) REQ_EXTRA(f, unsort_ndx)->num @@ -504,11 +633,23 @@ diff --git a/rsync.yo b/rsync.yo -K, --keep-dirlinks treat symlinked dir on receiver as dir -H, --hard-links preserve hard links -p, --perms preserve permissions -+ --fileflags preserve file-flags ++ --fileflags preserve file-flags (aka chflags) -E, --executability preserve executability --chmod=CHMOD affect file and/or directory permissions -A, --acls preserve ACLs (implies -p) -@@ -540,7 +541,8 @@ specified, in which case bf(-r) is not implied. +@@ -369,7 +370,10 @@ to the detailed description below for a complete description. verb( + --delete-after receiver deletes after transfer, not before + --delete-excluded also delete excluded files from dest dirs + --ignore-errors delete even if there are I/O errors +- --force force deletion of dirs even if not empty ++ --force-delete force deletion of dirs even if not empty ++ --force-change affect user/system immutable files/dirs ++ --force-uchange affect user-immutable files/dirs ++ --force-schange affect system-immutable files/dirs + --max-delete=NUM don't delete more than NUM files + --max-size=SIZE don't transfer any file larger than SIZE + --min-size=SIZE don't transfer any file smaller than SIZE +@@ -540,7 +544,8 @@ specified, in which case bf(-r) is not implied. Note that bf(-a) bf(does not preserve hardlinks), because finding multiply-linked files is expensive. You must separately @@ -518,36 +659,389 @@ diff --git a/rsync.yo b/rsync.yo dit(--no-OPTION) You may turn off one or more implied options by prefixing the option name with "no-". Not all options may be prefixed with a "no-": -@@ -922,6 +924,13 @@ super-user copies all namespaces except system.*. A normal user only copies +@@ -785,7 +790,7 @@ they would be using bf(--copy-links). + Without this option, if the sending side has replaced a directory with a + symlink to a directory, the receiving side will delete anything that is in + the way of the new symlink, including a directory hierarchy (as long as +-bf(--force) or bf(--delete) is in effect). ++bf(--force-delete) or bf(--delete) is in effect). + + See also bf(--keep-dirlinks) for an analogous option for the receiving + side. +@@ -922,6 +927,29 @@ super-user copies all namespaces except system.*. A normal user only copies the user.* namespace. To be able to backup and restore non-user namespaces as a normal user, see the bf(--fake-super) option. -+dit(bf(--fileflags)) This option causes rsync to update the file-flags -+to be the same as the source file, if your OS supports the bf(chflags)(2) -+system call. In any case, an attempt is made to remove flags that would -+prevent a file to be altered. Some flags can only be altered by the -+super-user and can only be unset below a certain secure-level (usually -+single-user mode). ++dit(bf(--fileflags)) This option causes rsync to update the file-flags to be ++the same as the source files and directories (if your OS supports the ++bf(chflags)(2) system call). Some flags can only be altered by the super-user ++and some might only be unset below a certain secure-level (usually single-user ++mode). It will not make files alterable that are set to immutable on the ++receiver. To do that, see bf(--force-change), bf(--force-uchange), and ++bf(--force-schange). ++ ++dit(bf(--force-change)) This option causes rsync to disable both user-immutable ++and system-immutable flags on files and directories that are being updated or ++deleted on the receiving side. This option overrides bf(--force-uchange) and ++bf(--force-schange). ++ ++dit(bf(--force-uchange)) This option causes rsync to disable user-immutable ++flags on files and directories that are being updated or deleted on the ++receiving side. It does not try to affect system flags. This option overrides ++bf(--force-change) and bf(--force-schange). ++ ++dit(bf(--force-schange)) This option causes rsync to disable system-immutable ++flags on files and directories that are being updated or deleted on the ++receiving side. It does not try to affect user flags. This option overrides ++bf(--force-change) and bf(--force-schange). + dit(bf(--chmod)) This option tells rsync to apply one or more comma-separated "chmod" strings to the permission of the files in the transfer. The resulting value is treated as though it was the permissions +@@ -1177,12 +1205,13 @@ See bf(--delete) (which is implied) for more details on file-deletion. + dit(bf(--ignore-errors)) Tells bf(--delete) to go ahead and delete files + even when there are I/O errors. + +-dit(bf(--force)) This option tells rsync to delete a non-empty directory ++dit(bf(--force-delete)) This option tells rsync to delete a non-empty directory + when it is to be replaced by a non-directory. This is only relevant if + deletions are not active (see bf(--delete) for details). + +-Note for older rsync versions: bf(--force) used to still be required when +-using bf(--delete-after), and it used to be non-functional unless the ++This option can be abbreviated bf(--force) for backward compatibility. ++Note that some older rsync versions used to still require bf(--force) ++when using bf(--delete-after), and it used to be non-functional unless the + bf(--recursive) option was also enabled. + + dit(bf(--max-delete=NUM)) This tells rsync not to delete more than NUM +@@ -1638,7 +1667,7 @@ with older versions of rsync, but that also turns on the output of other + verbose messages). + + The "%i" escape has a cryptic output that is 11 letters long. The general +-format is like the string bf(YXcstpoguax), where bf(Y) is replaced by the ++format is like the string bf(YXcstpogfax), where bf(Y) is replaced by the + type of update being done, bf(X) is replaced by the file-type, and the + other letters represent attributes that may be output if they are being + modified. +@@ -1689,7 +1718,7 @@ quote(itemization( + sender's value (requires bf(--owner) and super-user privileges). + it() A bf(g) means the group is different and is being updated to the + sender's value (requires bf(--group) and the authority to set the group). +- it() The bf(u) slot is reserved for future use. ++ it() The bf(f) means that the fileflags information changed. + it() The bf(a) means that the ACL information changed. + it() The bf(x) means that the extended attribute information changed. + )) diff --git a/syscall.c b/syscall.c --- a/syscall.c +++ b/syscall.c -@@ -174,6 +174,15 @@ int do_chmod(const char *path, mode_t mode) +@@ -33,6 +33,7 @@ extern int dry_run; + extern int am_root; + extern int read_only; + extern int list_only; ++extern int force_change; + extern int preserve_perms; + extern int preserve_executability; + +@@ -50,7 +51,23 @@ int do_unlink(const char *fname) + { + if (dry_run) return 0; + RETURN_ERROR_IF_RO_OR_LO; +- return unlink(fname); ++ if (unlink(fname) == 0) ++ return 0; ++#ifdef SUPPORT_FORCE_CHANGE ++ if (force_change && errno == EPERM) { ++ STRUCT_STAT st; ++ ++ if (x_lstat(fname, &st, NULL) == 0 ++ && make_mutable(fname, st.st_mode, st.st_flags, force_change) > 0) { ++ if (unlink(fname) == 0) ++ return 0; ++ undo_make_mutable(fname, st.st_flags); ++ } ++ /* TODO: handle immutable directories */ ++ errno = EPERM; ++ } ++#endif ++ return -1; + } + + int do_symlink(const char *fname1, const char *fname2) +@@ -69,14 +86,37 @@ int do_link(const char *fname1, const char *fname2) + } + #endif + +-int do_lchown(const char *path, uid_t owner, gid_t group) ++int do_lchown(const char *path, uid_t owner, gid_t group, mode_t mode, uint32 fileflags) + { + if (dry_run) return 0; + RETURN_ERROR_IF_RO_OR_LO; + #ifndef HAVE_LCHOWN + #define lchown chown + #endif +- return lchown(path, owner, group); ++ if (lchown(path, owner, group) == 0) ++ return 0; ++#ifdef SUPPORT_FORCE_CHANGE ++ if (force_change && errno == EPERM) { ++ if (fileflags == NO_FFLAGS) { ++ STRUCT_STAT st; ++ if (x_lstat(path, &st, NULL) == 0) { ++ mode = st.st_mode; ++ fileflags = st.st_flags; ++ } ++ } ++ if (fileflags != NO_FFLAGS ++ && make_mutable(path, mode, fileflags, force_change) > 0) { ++ int ret = lchown(path, owner, group); ++ undo_make_mutable(path, fileflags); ++ if (ret == 0) ++ return 0; ++ } ++ errno = EPERM; ++ } ++#else ++ mode = fileflags = 0; /* avoid compiler warning */ ++#endif ++ return -1; } + + int do_mknod(const char *pathname, mode_t mode, dev_t dev) +@@ -116,7 +156,7 @@ int do_mknod(const char *pathname, mode_t mode, dev_t dev) + return -1; + close(sock); + #ifdef HAVE_CHMOD +- return do_chmod(pathname, mode); ++ return do_chmod(pathname, mode, 0); + #else + return 0; #endif +@@ -133,7 +173,22 @@ int do_rmdir(const char *pathname) + { + if (dry_run) return 0; + RETURN_ERROR_IF_RO_OR_LO; +- return rmdir(pathname); ++ if (rmdir(pathname) == 0) ++ return 0; ++#ifdef SUPPORT_FORCE_CHANGE ++ if (force_change && errno == EPERM) { ++ STRUCT_STAT st; ++ ++ if (x_lstat(pathname, &st, NULL) == 0 ++ && make_mutable(pathname, st.st_mode, st.st_flags, force_change) > 0) { ++ if (rmdir(pathname) == 0) ++ return 0; ++ undo_make_mutable(pathname, st.st_flags); ++ } ++ errno = EPERM; ++ } ++#endif ++ return -1; + } + + int do_open(const char *pathname, int flags, mode_t mode) +@@ -147,7 +202,7 @@ int do_open(const char *pathname, int flags, mode_t mode) + } -+#ifdef SUPPORT_FLAGS -+int do_chflags(const char *path, u_long flags) + #ifdef HAVE_CHMOD +-int do_chmod(const char *path, mode_t mode) ++int do_chmod(const char *path, mode_t mode, uint32 fileflags) + { + int code; + if (dry_run) return 0; +@@ -168,17 +223,74 @@ int do_chmod(const char *path, mode_t mode) + #endif + } else + code = chmod(path, mode & CHMOD_BITS); ++#ifdef SUPPORT_FORCE_CHANGE ++ if (code < 0 && force_change && errno == EPERM && !S_ISLNK(mode)) { ++ if (fileflags == NO_FFLAGS) { ++ STRUCT_STAT st; ++ if (x_lstat(path, &st, NULL) == 0) ++ fileflags = st.st_flags; ++ } ++ if (fileflags != NO_FFLAGS ++ && make_mutable(path, mode, fileflags, force_change) > 0) { ++ code = chmod(path, mode & CHMOD_BITS); ++ undo_make_mutable(path, fileflags); ++ if (code == 0) ++ return 0; ++ } ++ errno = EPERM; ++ } ++#else ++ fileflags = 0; /* avoid compiler warning */ ++#endif + if (code != 0 && (preserve_perms || preserve_executability)) + return code; + return 0; + } + #endif + ++#ifdef HAVE_CHFLAGS ++int do_chflags(const char *path, uint32 fileflags) +{ + if (dry_run) return 0; + RETURN_ERROR_IF_RO_OR_LO; -+ return chflags(path, flags); ++ return chflags(path, fileflags); +} +#endif + int do_rename(const char *fname1, const char *fname2) { if (dry_run) return 0; + RETURN_ERROR_IF_RO_OR_LO; +- return rename(fname1, fname2); ++ if (rename(fname1, fname2) == 0) ++ return 0; ++#ifdef SUPPORT_FORCE_CHANGE ++ if (force_change && errno == EPERM) { ++ STRUCT_STAT st1, st2; ++ int became_mutable; ++ ++ if (x_lstat(fname1, &st1, NULL) != 0) ++ goto failed; ++ became_mutable = make_mutable(fname1, st1.st_mode, st1.st_flags, force_change) > 0; ++ if (became_mutable && rename(fname1, fname2) == 0) ++ goto success; ++ if (x_lstat(fname2, &st2, NULL) == 0 ++ && make_mutable(fname2, st2.st_mode, st2.st_flags, force_change) > 0) { ++ if (rename(fname1, fname2) == 0) { ++ success: ++ if (became_mutable) /* Yes, use fname2 and st1! */ ++ undo_make_mutable(fname2, st1.st_flags); ++ return 0; ++ } ++ undo_make_mutable(fname2, st2.st_flags); ++ } ++ /* TODO: handle immutable directories */ ++ if (became_mutable) ++ undo_make_mutable(fname1, st1.st_flags); ++ failed: ++ errno = EPERM; ++ } ++#endif ++ return -1; + } + + void trim_trailing_slashes(char *name) +diff --git a/t_stub.c b/t_stub.c +--- a/t_stub.c ++++ b/t_stub.c +@@ -26,6 +26,7 @@ int module_id = -1; + int relative_paths = 0; + int human_readable = 0; + int module_dirlen = 0; ++int force_change = 0; + mode_t orig_umask = 002; + char *partial_dir; + char *module_dir; +@@ -83,3 +84,23 @@ struct filter_list_struct server_filter_list; + { + return "tester"; + } ++ ++#if defined SUPPORT_FILEFLAGS || defined SUPPORT_FORCE_CHANGE ++ int make_mutable(UNUSED(const char *fname), UNUSED(mode_t mode), UNUSED(uint32 fileflags), UNUSED(uint32 iflags)) ++{ ++ return 0; ++} ++ ++/* Undo a prior make_mutable() call that returned a 1. */ ++ int undo_make_mutable(UNUSED(const char *fname), UNUSED(uint32 fileflags)) ++{ ++ return 0; ++} ++#endif ++ ++#ifdef SUPPORT_XATTRS ++ int x_lstat(UNUSED(const char *fname), UNUSED(STRUCT_STAT *fst), UNUSED(STRUCT_STAT *xst)) ++{ ++ return -1; ++} ++#endif +diff --git a/util.c b/util.c +--- a/util.c ++++ b/util.c +@@ -29,6 +29,7 @@ extern int module_id; + extern int modify_window; + extern int relative_paths; + extern int human_readable; ++extern int force_change; + extern char *module_dir; + extern unsigned int module_dirlen; + extern mode_t orig_umask; +@@ -122,7 +123,7 @@ NORETURN void overflow_exit(const char *str) + exit_cleanup(RERR_MALLOC); + } + +-int set_modtime(const char *fname, time_t modtime, mode_t mode) ++int set_modtime(const char *fname, time_t modtime, mode_t mode, uint32 fileflags) + { + #if !defined HAVE_LUTIMES || !defined HAVE_UTIMES + if (S_ISLNK(mode)) +@@ -139,6 +140,7 @@ int set_modtime(const char *fname, time_t modtime, mode_t mode) + return 0; + + { ++ int ret; + #ifdef HAVE_UTIMES + struct timeval t[2]; + t[0].tv_sec = time(NULL); +@@ -152,20 +154,39 @@ int set_modtime(const char *fname, time_t modtime, mode_t mode) + return 0; + } + # endif +- return utimes(fname, t); ++#define SET_THE_TIME(fn) utimes(fn, t) + #elif defined HAVE_STRUCT_UTIMBUF + struct utimbuf tbuf; + tbuf.actime = time(NULL); + tbuf.modtime = modtime; +- return utime(fname,&tbuf); ++#define SET_THE_TIME(fn) utime(fn, &tbuf) + #elif defined HAVE_UTIME + time_t t[2]; + t[0] = time(NULL); + t[1] = modtime; +- return utime(fname,t); ++#define SET_THE_TIME(fn) utime(fn, t) + #else + #error No file-time-modification routine found! + #endif ++ ret = SET_THE_TIME(fname); ++#ifdef SUPPORT_FORCE_CHANGE ++ if (ret != 0 && force_change && errno == EPERM) { ++ if (fileflags == NO_FFLAGS) { ++ STRUCT_STAT st; ++ if (x_lstat(fname, &st, NULL) == 0) ++ fileflags = st.st_flags; ++ } ++ if (fileflags != NO_FFLAGS ++ && make_mutable(fname, mode, fileflags, force_change) > 0) { ++ ret = SET_THE_TIME(fname); ++ undo_make_mutable(fname, fileflags); ++ } ++ errno = EPERM; ++ } ++#else ++ fileflags = 0; /* avoid compiler warning */ ++#endif ++ return ret; + } + } + +diff --git a/xattrs.c b/xattrs.c +--- a/xattrs.c ++++ b/xattrs.c +@@ -915,7 +915,7 @@ int set_stat_xattr(const char *fname, struct file_struct *file, mode_t new_mode) + mode = (fst.st_mode & _S_IFMT) | (fmode & ACCESSPERMS) + | (S_ISDIR(fst.st_mode) ? 0700 : 0600); + if (fst.st_mode != mode) +- do_chmod(fname, mode); ++ do_chmod(fname, mode, ST_FLAGS(fst)); + if (!IS_DEVICE(fst.st_mode) && !IS_SPECIAL(fst.st_mode)) + fst.st_rdev = 0; /* just in case */ + diff --git a/ignore-case.diff b/ignore-case.diff index 0b22d21..7fbcbc6 100644 --- a/ignore-case.diff +++ b/ignore-case.diff @@ -55,7 +55,7 @@ diff --git a/flist.c b/flist.c extern int ignore_errors; extern int numeric_ids; extern int recurse; -@@ -2615,6 +2616,7 @@ int f_name_cmp(const struct file_struct *f1, const struct file_struct *f2) +@@ -2617,6 +2618,7 @@ int f_name_cmp(const struct file_struct *f1, const struct file_struct *f2) { int dif; const uchar *c1, *c2; @@ -63,7 +63,7 @@ diff --git a/flist.c b/flist.c enum fnc_state state1, state2; enum fnc_type type1, type2; enum fnc_type t_path = protocol_version >= 29 ? t_PATH : t_ITEM; -@@ -2725,7 +2727,15 @@ int f_name_cmp(const struct file_struct *f1, const struct file_struct *f2) +@@ -2727,7 +2729,15 @@ int f_name_cmp(const struct file_struct *f1, const struct file_struct *f2) if (type1 != type2) return type1 == t_PATH ? 1 : -1; } diff --git a/link-by-hash.diff b/link-by-hash.diff index 7470051..8a934ba 100644 --- a/link-by-hash.diff +++ b/link-by-hash.diff @@ -27,7 +27,7 @@ diff --git a/Makefile.in b/Makefile.in diff --git a/flist.c b/flist.c --- a/flist.c +++ b/flist.c -@@ -68,6 +68,7 @@ extern int need_unsorted_flist; +@@ -70,6 +70,7 @@ extern int need_unsorted_flist; extern int unsort_ndx; extern struct stats stats; extern char *filesfrom_host; @@ -35,7 +35,7 @@ diff --git a/flist.c b/flist.c extern char curr_dir[MAXPATHLEN]; -@@ -822,7 +823,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -824,7 +825,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist, extra_len += (S_ISDIR(mode) ? 2 : 1) * EXTRA_LEN; #endif diff --git a/log-checksum.diff b/log-checksum.diff index 2f15ed6..f06824b 100644 --- a/log-checksum.diff +++ b/log-checksum.diff @@ -155,7 +155,7 @@ diff --git a/receiver.c b/receiver.c diff --git a/rsync.yo b/rsync.yo --- a/rsync.yo +++ b/rsync.yo -@@ -2077,7 +2077,7 @@ by the server and defaults to the current code(time()). This option +@@ -2075,7 +2075,7 @@ by the server and defaults to the current code(time()). This option is used to set a specific checksum seed, which is useful for applications that want repeatable block and file checksums, or in the case where the user wants a more random checksum seed. diff --git a/nameconverter.diff b/nameconverter.diff index 8ff8fd3..33a990c 100644 --- a/nameconverter.diff +++ b/nameconverter.diff @@ -39,7 +39,7 @@ diff --git a/clientserver.c b/clientserver.c #ifdef HAVE_SIGACTION static struct sigaction sigact; -@@ -535,7 +537,7 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host) +@@ -539,7 +541,7 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host) log_init(1); #ifdef HAVE_PUTENV @@ -48,7 +48,7 @@ diff --git a/clientserver.c b/clientserver.c char *modname, *modpath, *hostaddr, *hostname, *username; int status; -@@ -631,6 +633,44 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host) +@@ -635,6 +637,44 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host) set_blocking(fds[1]); pre_exec_fd = fds[1]; } @@ -93,7 +93,7 @@ diff --git a/clientserver.c b/clientserver.c umask(0); } #endif -@@ -844,6 +884,44 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host) +@@ -848,6 +888,44 @@ static int rsync_module(int f_in, int f_out, int i, char *addr, char *host) return 0; } diff --git a/omit-dir-changes.diff b/omit-dir-changes.diff index a7220f9..b750253 100644 --- a/omit-dir-changes.diff +++ b/omit-dir-changes.diff @@ -18,7 +18,7 @@ diff --git a/generator.c b/generator.c extern int uid_ndx; extern int gid_ndx; extern int delete_mode; -@@ -605,6 +606,7 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre +@@ -625,6 +626,7 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre (receiver_symlink_times && !(file->flags & FLAG_TIME_FAILED)) || #endif !S_ISLNK(file->mode); @@ -26,7 +26,7 @@ diff --git a/generator.c b/generator.c if (S_ISREG(file->mode) && F_LENGTH(file) != sxp->st.st_size) iflags |= ITEM_REPORT_SIZE; -@@ -621,10 +623,11 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre +@@ -641,10 +643,11 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre if ((preserve_perms || preserve_executability) && !BITS_EQUAL(sxp->st.st_mode, file->mode, CHMOD_BITS)) iflags |= ITEM_REPORT_PERMS; @@ -41,7 +41,7 @@ diff --git a/generator.c b/generator.c iflags |= ITEM_REPORT_GROUP; #ifdef SUPPORT_ACLS if (preserve_acls && !S_ISLNK(file->mode)) { -@@ -1366,7 +1369,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, +@@ -1387,7 +1390,7 @@ static void recv_generator(char *fname, struct file_struct *file, int ndx, real_sx = sx; if (file->flags & FLAG_DIR_CREATED) statret = -1; diff --git a/openssl-support.diff b/openssl-support.diff index a61902e..2974811 100644 --- a/openssl-support.diff +++ b/openssl-support.diff @@ -88,8 +88,8 @@ diff --git a/clientserver.c b/clientserver.c extern int rsync_port; extern int ignore_errors; extern int kluge_around_eof; -@@ -121,8 +124,18 @@ int start_socket_client(char *host, int remote_argc, char *remote_argv[], - set_socket_options(fd, sockopts); +@@ -125,8 +128,18 @@ int start_socket_client(char *host, int remote_argc, char *remote_argv[], + #endif ret = start_inband_exchange(fd, fd, user, remote_argc, remote_argv); + if (ret) @@ -108,7 +108,7 @@ diff --git a/clientserver.c b/clientserver.c } static int exchange_protocols(int f_in, int f_out, char *buf, size_t bufsiz, int am_client) -@@ -265,6 +278,32 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char +@@ -269,6 +282,32 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char if (verbose > 1) print_child_argv("sending daemon args:", sargs); @@ -141,7 +141,7 @@ diff --git a/clientserver.c b/clientserver.c io_printf(f_out, "%.*s\n", modlen, modname); /* Old servers may just drop the connection here, -@@ -290,6 +329,10 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char +@@ -294,6 +333,10 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char * server to terminate the listing of modules. * We don't want to go on and transfer * anything; just exit. */ @@ -152,7 +152,7 @@ diff --git a/clientserver.c b/clientserver.c exit(0); } -@@ -297,6 +340,10 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char +@@ -301,6 +344,10 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char rprintf(FERROR, "%s\n", line); /* This is always fatal; the server will now * close the socket. */ @@ -163,7 +163,7 @@ diff --git a/clientserver.c b/clientserver.c return -1; } -@@ -894,6 +941,9 @@ int start_daemon(int f_in, int f_out) +@@ -898,6 +945,9 @@ int start_daemon(int f_in, int f_out) if (exchange_protocols(f_in, f_out, line, sizeof line, 0) < 0) return -1; @@ -173,7 +173,7 @@ diff --git a/clientserver.c b/clientserver.c line[0] = 0; if (!read_line_old(f_in, line, sizeof line)) return -1; -@@ -905,6 +955,20 @@ int start_daemon(int f_in, int f_out) +@@ -909,6 +959,20 @@ int start_daemon(int f_in, int f_out) return -1; } diff --git a/osx-create-time.diff b/osx-create-time.diff index 862f427..d868a7c 100644 --- a/osx-create-time.diff +++ b/osx-create-time.diff @@ -11,7 +11,7 @@ To use this patch, run these commands for a successful build: diff --git a/lib/sysxattrs.c b/lib/sysxattrs.c --- a/lib/sysxattrs.c +++ b/lib/sysxattrs.c -@@ -52,29 +52,118 @@ ssize_t sys_llistxattr(const char *path, char *list, size_t size) +@@ -53,29 +53,118 @@ ssize_t sys_llistxattr(const char *path, char *list, size_t size) #elif HAVE_OSX_XATTRS diff --git a/preallocate.diff b/preallocate.diff index 74f3238..97c0b96 100644 --- a/preallocate.diff +++ b/preallocate.diff @@ -9,18 +9,78 @@ To use this patch, run these commands for a successful build: ./configure make +diff --git a/compat.c b/compat.c +--- a/compat.c ++++ b/compat.c +@@ -33,6 +33,7 @@ extern int inplace; + extern int recurse; + extern int use_qsort; + extern int allow_inc_recurse; ++extern int preallocate_files; + extern int append_mode; + extern int fuzzy_basis; + extern int read_batch; +@@ -184,6 +185,15 @@ void setup_protocol(int f_out,int f_in) + if (read_batch) + check_batch_flags(); + ++#ifndef SUPPORT_PREALLOCATION ++ if (preallocate_files && !am_sender) { ++ rprintf(FERROR, ++ "preallocation is not supported on this %s\n", ++ am_server ? "server" : "client"); ++ exit_cleanup(RERR_SYNTAX); ++ } ++#endif ++ + if (protocol_version < 30) { + if (append_mode == 1) + append_mode = 2; diff --git a/configure.in b/configure.in --- a/configure.in +++ b/configure.in -@@ -552,7 +552,7 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strdup chown chmod lchmod mknod mkfifo \ +@@ -552,13 +552,40 @@ AC_CHECK_FUNCS(waitpid wait4 getcwd strdup chown chmod lchmod mknod mkfifo \ strlcat strlcpy strtol mallinfo getgroups setgroups geteuid getegid \ setlocale setmode open64 lseek64 mkstemp64 mtrace va_copy __va_copy \ strerror putenv iconv_open locale_charset nl_langinfo getxattr \ - extattr_get_link sigaction sigprocmask setattrlist) -+ extattr_get_link sigaction sigprocmask setattrlist posix_fallocate) ++ extattr_get_link sigaction sigprocmask setattrlist fallocate posix_fallocate) dnl cygwin iconv.h defines iconv_open as libiconv_open if test x"$ac_cv_func_iconv_open" != x"yes"; then + AC_CHECK_FUNC(libiconv_open, [ac_cv_func_iconv_open=yes; AC_DEFINE(HAVE_ICONV_OPEN, 1)]) + fi + ++dnl Preallocation stuff (also fallocate, posix_fallocate function tests above): ++ ++AC_CACHE_CHECK([for SYS_fallocate],rsync_cv_have_sys_fallocate,[ ++AC_TRY_COMPILE([#include ++#include ], ++[syscall(SYS_fallocate, 0, 0, (loff_t) 0, (loff_t) 0);], ++rsync_cv_have_sys_fallocate=yes,rsync_cv_have_sys_fallocate=no)]) ++if test x"$rsync_cv_have_sys_fallocate" = x"yes"; then ++ AC_DEFINE(HAVE_SYS_FALLOCATE, 1, [Define to 1 if you have the SYS_fallocate syscall number]) ++fi ++ ++if test x"$ac_cv_func_posix_fallocate" = x"yes"; then ++ AC_MSG_CHECKING([whether posix_fallocate is efficient]) ++ case $host_os in ++ *cygwin*) ++ AC_MSG_RESULT(yes) ++ AC_DEFINE(HAVE_EFFICIENT_POSIX_FALLOCATE, 1, ++ [Define if posix_fallocate is efficient (Cygwin)]) ++ ;; ++ *) ++ AC_MSG_RESULT(no) ++ ;; ++ esac ++fi ++ ++dnl End of preallocation stuff ++ + AC_CHECK_FUNCS(getpgrp tcgetpgrp) + if test $ac_cv_func_getpgrp = yes; then + AC_FUNC_GETPGRP diff --git a/options.c b/options.c --- a/options.c +++ b/options.c @@ -66,7 +126,7 @@ diff --git a/options.c b/options.c #endif rprintf(F," -S, --sparse handle sparse files efficiently\n"); +#ifdef SUPPORT_PREALLOCATION -+ rprintf(F," --preallocate posix_fallocate dest files before writing them\n"); ++ rprintf(F," --preallocate allocate dest files before writing them\n"); +#endif rprintf(F," -n, --dry-run perform a trial run with no changes made\n"); rprintf(F," -W, --whole-file copy files whole (without delta-xfer algorithm)\n"); @@ -79,23 +139,7 @@ diff --git a/options.c b/options.c {"inplace", 0, POPT_ARG_NONE, &inplace, 0, 0, 0 }, {"append", 0, POPT_ARG_NONE, 0, OPT_APPEND, 0, 0 }, {"append-verify", 0, POPT_ARG_VAL, &append_mode, 2, 0, 0 }, -@@ -1311,6 +1320,15 @@ int parse_arguments(int *argc_p, const char ***argv_p, int frommain) - } - #endif - -+#ifndef SUPPORT_PREALLOCATION -+ if (preallocate_files && !am_sender) { -+ snprintf(err_buf, sizeof err_buf, -+ "preallocation is not supported on this %s\n", -+ am_server ? "server" : "client"); -+ return 0; -+ } -+#endif -+ - if (write_batch && read_batch) { - snprintf(err_buf, sizeof err_buf, - "--write-batch and --read-batch can not be used together\n"); -@@ -2022,6 +2040,9 @@ void server_options(char **args, int *argc_p) +@@ -2022,6 +2031,9 @@ void server_options(char **args, int *argc_p) else if (remove_source_files) args[ac++] = "--remove-sent-files"; @@ -116,27 +160,26 @@ diff --git a/receiver.c b/receiver.c extern int keep_partial; extern int checksum_seed; extern int inplace; -@@ -175,6 +176,19 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, +@@ -174,6 +175,18 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, + char *data; int32 i; char *map = NULL; - +#ifdef SUPPORT_PREALLOCATION + OFF_T preallocated_len = 0; + + if (preallocate_files && fd != -1 && total_size > 0) { + /* Preallocate enough space for file's eventual length if + * possible; seems to reduce fragmentation on Windows. */ -+ if (posix_fallocate(fd, 0, total_size) == 0) ++ if (do_fallocate(fd, 0, total_size) == 0) + preallocated_len = total_size; + else -+ rsyserr(FINFO, errno, "preallocate %s", full_fname(fname)); ++ rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(fname)); + } +#endif -+ + read_sum_head(f_in, &sum); - if (fd_r >= 0 && size_r > 0) { -@@ -284,8 +298,18 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, +@@ -284,8 +297,18 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, goto report_write_error; #ifdef HAVE_FTRUNCATE @@ -160,11 +203,14 @@ diff --git a/receiver.c b/receiver.c diff --git a/rsync.h b/rsync.h --- a/rsync.h +++ b/rsync.h -@@ -602,6 +602,10 @@ struct ht_int64_node { +@@ -602,6 +602,13 @@ struct ht_int64_node { #define ACLS_NEED_MASK 1 #endif -+#if defined HAVE_FTRUNCATE && defined HAVE_POSIX_FALLOCATE ++#if defined HAVE_FTRUNCATE \ ++ && (defined HAVE_FALLOCATE \ ++ || defined HAVE_SYS_FALLOCATE \ ++ || defined HAVE_EFFICIENT_POSIX_FALLOCATE) +#define SUPPORT_PREALLOCATION 1 +#endif + @@ -178,30 +224,65 @@ diff --git a/rsync.yo b/rsync.yo --super receiver attempts super-user activities --fake-super store/recover privileged attrs using xattrs -S, --sparse handle sparse files efficiently -+ --preallocate posix_fallocate dest files before writing ++ --preallocate allocate dest files before writing -n, --dry-run perform a trial run with no changes made -W, --whole-file copy files whole (w/o delta-xfer algorithm) -x, --one-file-system don't cross filesystem boundaries -@@ -1036,6 +1037,19 @@ NOTE: Don't use this option when the destination is a Solaris "tmpfs" +@@ -1036,6 +1037,18 @@ NOTE: Don't use this option when the destination is a Solaris "tmpfs" filesystem. It doesn't seem to handle seeks over null regions correctly and ends up corrupting the files. +dit(bf(--preallocate)) This tells the receiver to allocate each destination -+file to its eventual size using bf(posix_fallocate)(3) before writing data -+to the file. If the receiver is remote, this nonstandard option only works -+if the receiver also has the preallocation patch. Furthermore, this option -+only works if the receiver found the bf(posix_fallocate)(3) call at -+configure time. ++file to its eventual size before writing data to the file. Rsync will only use ++the real filesystem-level preallocation support provided by bf(fallocate)(2) or ++Cygwin's bf(posix_fallocate)(3), not the slow glibc implementation that writes ++a zero byte into each block. If the receiver is remote, this nonstandard ++option only works if the receiver also has the preallocation patch. + +Without this option on MS Windows, very large destination files tend to be +broken into thousands of fragments; advising Windows ahead of time of the +eventual file size using this option usually reduces the number of -+fragments to one. However, on Linux, this option appears to just waste -+disk I/O. ++fragments to one. The usefulness of this option on Linux is yet to be tested. + dit(bf(-n, --dry-run)) This makes rsync perform a trial run that doesn't make any changes (and produces mostly the same output as a real run). It is most commonly used in combination with the bf(-v, --verbose) and/or +diff --git a/syscall.c b/syscall.c +--- a/syscall.c ++++ b/syscall.c +@@ -29,6 +29,10 @@ + #include + #endif + ++#if defined HAVE_SYS_FALLOCATE && !defined HAVE_FALLOCATE ++#include ++#endif ++ + extern int dry_run; + extern int am_root; + extern int read_only; +@@ -291,3 +295,21 @@ char *d_name(struct dirent *di) + return di->d_name; + #endif + } ++ ++#ifdef SUPPORT_PREALLOCATION ++int do_fallocate(int fd, OFF_T offset, OFF_T length) ++{ ++ RETURN_ERROR_IF(dry_run, 0); ++ RETURN_ERROR_IF_RO_OR_LO; ++ /* TODO: Use FALLOC_FL_KEEP_SIZE to avoid the need to truncate. */ ++#if defined HAVE_FALLOCATE ++ return fallocate(fd, 0, offset, length); ++#elif defined HAVE_SYS_FALLOCATE ++ return syscall(SYS_fallocate, fd, 0, (loff_t) offset, (loff_t) length); ++#elif defined HAVE_EFFICIENT_POSIX_FALLOCATE ++ return posix_fallocate(fd, offset, length); ++#else ++#error coding error in SUPPORT_PREALLOCATION ++#endif ++} ++#endif diff --git a/t_stub.c b/t_stub.c --- a/t_stub.c +++ b/t_stub.c @@ -246,13 +327,13 @@ diff --git a/util.c b/util.c + STRUCT_STAT srcst; + if (do_fstat(ifd, &srcst) == 0) { + if (srcst.st_size > 0) { -+ if (posix_fallocate(ofd, 0, srcst.st_size) == 0) ++ if (do_fallocate(ofd, 0, srcst.st_size) == 0) + preallocated_len = srcst.st_size; + else -+ rsyserr(FINFO, errno, "posix_fallocate %s", full_fname(dest)); ++ rsyserr(FWARNING, errno, "do_fallocate %s", full_fname(dest)); + } + } else -+ rsyserr(FINFO, errno, "fstat %s", full_fname(source)); ++ rsyserr(FWARNING, errno, "fstat %s", full_fname(source)); + } +#endif + diff --git a/slow-down.diff b/slow-down.diff index 95d91e2..a889f7d 100644 --- a/slow-down.diff +++ b/slow-down.diff @@ -17,7 +17,7 @@ To use this patch, run these commands for a successful build: diff --git a/flist.c b/flist.c --- a/flist.c +++ b/flist.c -@@ -66,6 +66,7 @@ extern int sanitize_paths; +@@ -68,6 +68,7 @@ extern int sanitize_paths; extern int munge_symlinks; extern int need_unsorted_flist; extern int unsort_ndx; @@ -25,7 +25,7 @@ diff --git a/flist.c b/flist.c extern struct stats stats; extern char *filesfrom_host; -@@ -1500,6 +1501,9 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len, +@@ -1502,6 +1503,9 @@ static void send_directory(int f, struct file_list *flist, char *fbuf, int len, } send_file_name(f, flist, fbuf, NULL, flags, filter_level); diff --git a/slp.diff b/slp.diff index bdb5e23..f509693 100644 --- a/slp.diff +++ b/slp.diff @@ -43,7 +43,7 @@ diff --git a/Makefile.in b/Makefile.in diff --git a/clientserver.c b/clientserver.c --- a/clientserver.c +++ b/clientserver.c -@@ -1028,6 +1028,13 @@ int daemon_main(void) +@@ -1032,6 +1032,13 @@ int daemon_main(void) * address too. In fact, why not just do inet_ntop on the * local address??? */ diff --git a/source-filter_dest-filter.diff b/source-filter_dest-filter.diff index f6035bb..cd6d0c7 100644 --- a/source-filter_dest-filter.diff +++ b/source-filter_dest-filter.diff @@ -41,7 +41,7 @@ diff --git a/generator.c b/generator.c extern int size_only; extern OFF_T max_size; extern OFF_T min_size; -@@ -676,7 +677,7 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre +@@ -696,7 +697,7 @@ void itemize(const char *fnamecmp, struct file_struct *file, int ndx, int statre /* Perform our quick-check heuristic for determining if a file is unchanged. */ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st) { @@ -342,7 +342,7 @@ diff --git a/rsync.yo b/rsync.yo --protocol=NUM force an older protocol version to be used --iconv=CONVERT_SPEC request charset conversion of filenames --checksum-seed=NUM set block/file checksum seed (advanced) -@@ -2023,6 +2026,33 @@ file previously generated by bf(--write-batch). +@@ -2021,6 +2024,33 @@ file previously generated by bf(--write-batch). If em(FILE) is bf(-), the batch data will be read from standard input. See the "BATCH MODE" section for details. diff --git a/time-limit.diff b/time-limit.diff index 326c804..fb6c36c 100644 --- a/time-limit.diff +++ b/time-limit.diff @@ -151,7 +151,7 @@ diff --git a/rsync.yo b/rsync.yo --write-batch=FILE write a batched update to FILE --only-write-batch=FILE like --write-batch but w/o updating dest --read-batch=FILE read a batched update from FILE -@@ -1997,6 +1999,19 @@ transfer was too fast, it will wait before sending the next data block. The +@@ -1995,6 +1997,19 @@ transfer was too fast, it will wait before sending the next data block. The result is an average transfer rate equaling the specified limit. A value of zero specifies no limit. diff --git a/transliterate.diff b/transliterate.diff index f3ea0a3..47f5f00 100644 --- a/transliterate.diff +++ b/transliterate.diff @@ -16,7 +16,7 @@ To use this patch, run these commands for a successful build: diff --git a/flist.c b/flist.c --- a/flist.c +++ b/flist.c -@@ -81,6 +81,9 @@ extern int filesfrom_convert; +@@ -83,6 +83,9 @@ extern int filesfrom_convert; extern iconv_t ic_send, ic_recv; #endif @@ -26,7 +26,7 @@ diff --git a/flist.c b/flist.c #define PTR_SIZE (sizeof (struct file_struct *)) int io_error; -@@ -605,6 +608,24 @@ static void send_file_entry(int f, struct file_struct *file, int ndx, int first_ +@@ -607,6 +610,24 @@ static void send_file_entry(int f, struct file_struct *file, int ndx, int first_ stats.total_size += F_LENGTH(file); } @@ -51,7 +51,7 @@ diff --git a/flist.c b/flist.c static struct file_struct *recv_file_entry(struct file_list *flist, int xflags, int f) { -@@ -673,6 +694,9 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -675,6 +696,9 @@ static struct file_struct *recv_file_entry(struct file_list *flist, } #endif @@ -145,7 +145,7 @@ diff --git a/rsync.yo b/rsync.yo --checksum-seed=NUM set block/file checksum seed (advanced) -4, --ipv4 prefer IPv4 -6, --ipv6 prefer IPv6 -@@ -2061,6 +2062,22 @@ daemon uses the charset specified in its "charset" configuration parameter +@@ -2059,6 +2060,22 @@ daemon uses the charset specified in its "charset" configuration parameter regardless of the remote charset you actually pass. Thus, you may feel free to specify just the local charset for a daemon transfer (e.g. bf(--iconv=utf8)). diff --git a/usermap.diff b/usermap.diff index 05548a6..a6128a0 100644 --- a/usermap.diff +++ b/usermap.diff @@ -10,7 +10,7 @@ To use this patch, run these commands for a successful build: diff --git a/flist.c b/flist.c --- a/flist.c +++ b/flist.c -@@ -68,6 +68,7 @@ extern int need_unsorted_flist; +@@ -70,6 +70,7 @@ extern int need_unsorted_flist; extern int unsort_ndx; extern struct stats stats; extern char *filesfrom_host; @@ -18,7 +18,7 @@ diff --git a/flist.c b/flist.c extern char curr_dir[MAXPATHLEN]; -@@ -752,7 +753,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -754,7 +755,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist, uid = (uid_t)read_varint(f); if (xflags & XMIT_USER_NAME_FOLLOWS) uid = recv_user_name(f, uid); @@ -27,7 +27,7 @@ diff --git a/flist.c b/flist.c uid = match_uid(uid); } } -@@ -764,7 +765,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist, +@@ -766,7 +767,7 @@ static struct file_struct *recv_file_entry(struct file_list *flist, gid_flags = 0; if (xflags & XMIT_GROUP_NAME_FOLLOWS) gid = recv_group_name(f, gid, &gid_flags); @@ -36,7 +36,7 @@ diff --git a/flist.c b/flist.c gid = match_gid(gid, &gid_flags); } } -@@ -2085,8 +2086,13 @@ struct file_list *recv_file_list(int f) +@@ -2087,8 +2088,13 @@ struct file_list *recv_file_list(int f) int dstart, flags; int64 start_read; -- 2.34.1