Make sure that any weird mode bits (if they exist up beyond the

[rsync/rsync-patches.git] / link-by-hash.diff

diff --git a/link-by-hash.diff b/link-by-hash.diff
index 5589bd5..ed8bf5b 100644 (file)
--- a/link-by-hash.diff
+++ b/link-by-hash.diff
@@ -1,30 +1,31 @@
-After applying this patch and running configure, you MUST run this
-command before "make":
-
-    make proto
-
-Jason M. Felice writes:
+Jason M. Felice wrote:

 
 This patch adds the --link-by-hash=DIR option, which hard links received
 files in a link farm arranged by MD4 file hash.  The result is that the system
 will only store one copy of the unique contents of each file, regardless of
 the file's name.
 
 
 This patch adds the --link-by-hash=DIR option, which hard links received
 files in a link farm arranged by MD4 file hash.  The result is that the system
 will only store one copy of the unique contents of each file, regardless of
 the file's name.
 

+To use this patch, run these commands for a successful build:
+
+    patch -p1 <patches/link-by-hash.diff
+    ./prepare-source
+    ./configure
+    make

 
 

---- orig/Makefile.in   2004-11-03 11:56:03
-+++ Makefile.in        2004-07-03 20:20:15
+--- old/Makefile.in
++++ new/Makefile.in

 @@ -35,7 +35,7 @@ OBJS1=rsync.o generator.o receiver.o cle
        main.o checksum.o match.o syscall.o log.o backup.o
  OBJS2=options.o flist.o io.o compat.o hlink.o token.o uidlist.o socket.o \
 @@ -35,7 +35,7 @@ OBJS1=rsync.o generator.o receiver.o cle
        main.o checksum.o match.o syscall.o log.o backup.o
  OBJS2=options.o flist.o io.o compat.o hlink.o token.o uidlist.o socket.o \

-       fileio.o batch.o clientname.o
+       fileio.o batch.o clientname.o chmod.o

 -OBJS3=progress.o pipe.o
 +OBJS3=progress.o pipe.o hashlink.o
  DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
  popt_OBJS=popt/findme.o  popt/popt.o  popt/poptconfig.o \
        popt/popthelp.o popt/poptparse.o
 -OBJS3=progress.o pipe.o
 +OBJS3=progress.o pipe.o hashlink.o
  DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
  popt_OBJS=popt/findme.o  popt/popt.o  popt/poptconfig.o \
        popt/popthelp.o popt/poptparse.o

---- orig/hashlink.c    2004-09-24 16:44:25
-+++ hashlink.c 2004-09-24 16:44:25
-@@ -0,0 +1,340 @@
+--- old/hashlink.c
++++ new/hashlink.c
+@@ -0,0 +1,339 @@

 +/*
 +   Copyright (C) Cronosys, LLC 2004
 +
 +/*
 +   Copyright (C) Cronosys, LLC 2004
 +


@@ -130,7 +131,7 @@ the file's name.
 +              if (this_fnbr > *fnbr)
 +                      *fnbr = this_fnbr;
 +
 +              if (this_fnbr > *fnbr)
 +                      *fnbr = this_fnbr;
 +

-+              hashfile = (struct hashfile_struct*)malloc(sizeof(struct hashfile_struct));
++              hashfile = new_array(struct hashfile_struct, 1);

 +              asprintf(&hashfile->name,"%s/%s",hashname,
 +                       di->d_name);
 +              if (do_stat(hashfile->name,&st) == -1) {
 +              asprintf(&hashfile->name,"%s/%s",hashname,
 +                       di->d_name);
 +              if (do_stat(hashfile->name,&st) == -1) {


@@ -254,9 +255,8 @@ the file's name.
 +      char *linkname;
 +      long last_fnbr;
 +
 +      char *linkname;
 +      long last_fnbr;
 +

-+      if (file->length == 0) {
-+              return robust_rename(fnametmp,fname,0644);
-+      }
++      if (file->length == 0)
++              return robust_rename(fnametmp, fname, NULL, 0644);

 +
 +      if (do_stat(hashname, &st) == -1) {
 +              char *dirname;
 +
 +      if (do_stat(hashname, &st) == -1) {
 +              char *dirname;


@@ -268,14 +268,14 @@ the file's name.
 +                      rsyserr(FERROR, errno, "mkdir failed: %s", dirname);
 +                      free(hashname);
 +                      free(dirname);
 +                      rsyserr(FERROR, errno, "mkdir failed: %s", dirname);
 +                      free(hashname);
 +                      free(dirname);

-+                      return robust_rename(fnametmp,fname,0644);
++                      return robust_rename(fnametmp, fname, NULL, 0644);

 +              }
 +              free(dirname);
 +
 +              if (do_mkdir(hashname, 0755) == -1 && errno != EEXIST) {
 +                      rsyserr(FERROR, errno, "mkdir failed: %s", hashname);
 +                      free(hashname);
 +              }
 +              free(dirname);
 +
 +              if (do_mkdir(hashname, 0755) == -1 && errno != EEXIST) {
 +                      rsyserr(FERROR, errno, "mkdir failed: %s", hashname);
 +                      free(hashname);

-+                      return robust_rename(fnametmp,fname,0644);
++                      return robust_rename(fnametmp, fname, NULL, 0644);

 +              }
 +
 +              first = 1;
 +              }
 +
 +              first = 1;


@@ -336,7 +336,7 @@ the file's name.
 +                      } else {
 +                              rsyserr(FERROR, errno, "link \"%s\" -> \"%s\"",
 +                                      linkname, full_fname(fname));
 +                      } else {
 +                              rsyserr(FERROR, errno, "link \"%s\" -> \"%s\"",
 +                                      linkname, full_fname(fname));

-+                              rc = robust_rename(fnametmp,fname,0644);
++                              rc = robust_rename(fnametmp, fname, NULL, 0644);

 +                      }
 +              } else {
 +                      do_unlink(fnametmp);
 +                      }
 +              } else {
 +                      do_unlink(fnametmp);


@@ -347,7 +347,7 @@ the file's name.
 +              rprintf(FINFO, "link-by-hash (new): %s -> \"%s\"\n",
 +                              full_fname(fname),linkname);
 +
 +              rprintf(FINFO, "link-by-hash (new): %s -> \"%s\"\n",
 +                              full_fname(fname),linkname);
 +

-+              rc = robust_rename(fnametmp,fname,0644);
++              rc = robust_rename(fnametmp, fname, NULL, 0644);

 +              if (rc != 0) {
 +                      rsyserr(FERROR, errno, "rename \"%s\" -> \"%s\"",
 +                              full_fname(fnametmp), full_fname(fname));
 +              if (rc != 0) {
 +                      rsyserr(FERROR, errno, "rename \"%s\" -> \"%s\"",
 +                              full_fname(fnametmp), full_fname(fname));


@@ -365,50 +365,50 @@ the file's name.
 +}
 +
 +#endif
 +}
 +
 +#endif

---- orig/options.c     2005-02-21 10:51:52
-+++ options.c  2005-02-14 02:52:09
-@@ -139,6 +139,7 @@ char *log_format = NULL;
- char *password_file = NULL;
- char *rsync_path = RSYNC_PATH;
- char *backup_dir = NULL;
+--- old/options.c
++++ new/options.c
+@@ -145,6 +145,7 @@ char *backup_suffix = NULL;
+ char *tmpdir = NULL;
+ char *partial_dir = NULL;
+ char *basis_dir[MAX_BASIS_DIRS+1];

 +char *link_by_hash_dir = NULL;
 +char *link_by_hash_dir = NULL;

- char backup_dir_buf[MAXPATHLEN];
- int rsync_port = 0;
- int compare_dest = 0;
-@@ -319,6 +320,7 @@ void usage(enum logcode F)
+ char *config_file = NULL;
+ char *shell_cmd = NULL;
+ char *logfile_name = NULL;
+@@ -349,6 +350,7 @@ void usage(enum logcode F)

    rprintf(F,"     --compare-dest=DIR      also compare destination files relative to DIR\n");
    rprintf(F,"     --copy-dest=DIR         ... and include copies of unchanged files\n");
    rprintf(F,"     --link-dest=DIR         hardlink to files in DIR when unchanged\n");
 +  rprintf(F,"     --link-by-hash=DIR      create hardlinks by hash into DIR\n");
    rprintf(F," -z, --compress              compress file data during the transfer\n");
    rprintf(F,"     --compare-dest=DIR      also compare destination files relative to DIR\n");
    rprintf(F,"     --copy-dest=DIR         ... and include copies of unchanged files\n");
    rprintf(F,"     --link-dest=DIR         hardlink to files in DIR when unchanged\n");
 +  rprintf(F,"     --link-by-hash=DIR      create hardlinks by hash into DIR\n");
    rprintf(F," -z, --compress              compress file data during the transfer\n");

+   rprintf(F,"     --compress-level=NUM    explicitly set compression level\n");

    rprintf(F," -C, --cvs-exclude           auto-ignore files the same way CVS does\n");
    rprintf(F," -C, --cvs-exclude           auto-ignore files the same way CVS does\n");

-   rprintf(F," -f, --filter=RULE           add a file-filtering RULE\n");
-@@ -358,7 +360,7 @@ void usage(enum logcode F)
- enum {OPT_VERSION = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
-       OPT_FILTER, OPT_COMPARE_DEST, OPT_COPY_DEST, OPT_LINK_DEST,
-       OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW,
--      OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_TIMEOUT, OPT_MAX_SIZE,
-+      OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_TIMEOUT, OPT_MAX_SIZE, OPT_LINK_BY_HASH,
-       OPT_REFUSED_BASE = 9000};
+@@ -398,7 +400,7 @@ enum {OPT_VERSION = 1000, OPT_DAEMON, OP
+       OPT_FILTER, OPT_COMPARE_DEST, OPT_COPY_DEST, OPT_LINK_DEST, OPT_HELP,
+       OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW, OPT_MIN_SIZE, OPT_CHMOD,
+       OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE,
+-      OPT_NO_D,
++      OPT_NO_D, OPT_LINK_BY_HASH,
+       OPT_SERVER, OPT_REFUSED_BASE = 9000};

  
  static struct poptOption long_options[] = {
  
  static struct poptOption long_options[] = {

-@@ -427,6 +429,7 @@ static struct poptOption long_options[] 
+@@ -499,6 +501,7 @@ static struct poptOption long_options[] 

    {"compare-dest",     0,  POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
    {"copy-dest",        0,  POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
    {"link-dest",        0,  POPT_ARG_STRING, 0, OPT_LINK_DEST, 0, 0 },
 +  {"link-by-hash",     0,  POPT_ARG_STRING, 0, OPT_LINK_BY_HASH, 0, 0},
    {"fuzzy",           'y', POPT_ARG_NONE,   &fuzzy_basis, 0, 0, 0 },
    {"compare-dest",     0,  POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
    {"copy-dest",        0,  POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
    {"link-dest",        0,  POPT_ARG_STRING, 0, OPT_LINK_DEST, 0, 0 },
 +  {"link-by-hash",     0,  POPT_ARG_STRING, 0, OPT_LINK_BY_HASH, 0, 0},
    {"fuzzy",           'y', POPT_ARG_NONE,   &fuzzy_basis, 0, 0, 0 },

-   /* TODO: Should this take an optional int giving the compression level? */
-   {"compress",        'z', POPT_ARG_NONE,   &do_compression, 0, 0, 0 },
-@@ -860,6 +863,21 @@ int parse_arguments(int *argc, const cha
-                       basis_dir[basis_dir_cnt++] = (char *)arg;
-                       break;
+   {"compress",        'z', POPT_ARG_NONE,   0, 'z', 0, 0 },
+   {"compress-level",   0,  POPT_ARG_INT,    &def_compress_level, 'z', 0, 0 },
+@@ -1089,6 +1092,21 @@ int parse_arguments(int *argc, const cha
+                       usage(FINFO);
+                       exit_cleanup(0);

  
 +                case OPT_LINK_BY_HASH:
 +#if HAVE_LINK
 +                      arg = poptGetOptArg(pc);
 +                      if (sanitize_paths)
  
 +                case OPT_LINK_BY_HASH:
 +#if HAVE_LINK
 +                      arg = poptGetOptArg(pc);
 +                      if (sanitize_paths)

-+                              arg = sanitize_path(NULL, arg, NULL, 0);
++                              arg = sanitize_path(NULL, arg, NULL, 0, NULL);

 +                      link_by_hash_dir = (char *)arg;
 +                      break;
 +#else
 +                      link_by_hash_dir = (char *)arg;
 +                      break;
 +#else


@@ -422,7 +422,7 @@ the file's name.
                default:
                        /* A large opt value means that set_refuse_options()
                         * turned this option off. */
                default:
                        /* A large opt value means that set_refuse_options()
                         * turned this option off. */

-@@ -1440,6 +1458,11 @@ void server_options(char **args,int *arg
+@@ -1739,6 +1757,11 @@ void server_options(char **args,int *arg

                }
        }
  
                }
        }
  


@@ -431,20 +431,20 @@ the file's name.
 +              args[ac++] = link_by_hash_dir;
 +      }
 +
 +              args[ac++] = link_by_hash_dir;
 +      }
 +

-       if (files_from && (!am_sender || remote_filesfrom_file)) {
-               if (remote_filesfrom_file) {
+       if (files_from && (!am_sender || filesfrom_host)) {
+               if (filesfrom_host) {

                        args[ac++] = "--files-from";
                        args[ac++] = "--files-from";

---- orig/receiver.c    2005-02-20 20:55:51
-+++ receiver.c 2005-01-15 21:29:13
-@@ -52,6 +52,7 @@ extern int delay_updates;
+--- old/receiver.c
++++ new/receiver.c
+@@ -50,6 +50,7 @@ extern int delay_updates;

  extern struct stats stats;
  extern struct stats stats;

- extern char *log_format;
+ extern char *stdout_format;

  extern char *tmpdir;
 +extern char *link_by_hash_dir;
  extern char *partial_dir;
  extern char *basis_dir[];
  extern char *tmpdir;
 +extern char *link_by_hash_dir;
  extern char *partial_dir;
  extern char *basis_dir[];

- 
-@@ -206,12 +207,13 @@ static int get_tmpname(char *fnametmp, c
+ extern struct file_list *the_file_list;
+@@ -124,12 +125,13 @@ static int get_tmpname(char *fnametmp, c

  
  
  static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,
  
  
  static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r,


@@ -459,7 +459,7 @@ the file's name.
        int32 len;
        OFF_T offset = 0;
        OFF_T offset2;
        int32 len;
        OFF_T offset = 0;
        OFF_T offset2;

-@@ -231,6 +233,9 @@ static int receive_data(int f_in, char *
+@@ -149,6 +151,9 @@ static int receive_data(int f_in, char *

        } else
                mapbuf = NULL;
  
        } else
                mapbuf = NULL;
  


@@ -468,26 +468,26 @@ the file's name.
 +
        sum_init(checksum_seed);
  
 +
        sum_init(checksum_seed);
  

-       while ((i = recv_token(f_in, &data)) != 0) {
-@@ -247,6 +252,8 @@ static int receive_data(int f_in, char *
+       if (append_mode) {
+@@ -191,6 +196,8 @@ static int receive_data(int f_in, char *

                        cleanup_got_literal = 1;
  
                        sum_update(data, i);
 +                      if (md4)
                        cleanup_got_literal = 1;
  
                        sum_update(data, i);
 +                      if (md4)

-+                              mdfour_update(&mdfour_data,data,i);
++                              mdfour_update(&mdfour_data, (uchar*)data, i);

  
                        if (fd != -1 && write_file(fd,data,i) != i)
                                goto report_write_error;
  
                        if (fd != -1 && write_file(fd,data,i) != i)
                                goto report_write_error;

-@@ -273,6 +280,8 @@ static int receive_data(int f_in, char *
+@@ -217,6 +224,8 @@ static int receive_data(int f_in, char *

  
                        see_token(map, len);
                        sum_update(map, len);
 +                      if (md4)
  
                        see_token(map, len);
                        sum_update(map, len);
 +                      if (md4)

-+                              mdfour_update(&mdfour_data,map,len);
++                              mdfour_update(&mdfour_data, (uchar*)map, len);

                }
  
                }
  

-               if (inplace) {
-@@ -313,6 +322,8 @@ static int receive_data(int f_in, char *
+               if (updating_basis) {
+@@ -259,6 +268,8 @@ static int receive_data(int f_in, char *

        }
  
        sum_end(file_sum1);
        }
  
        sum_end(file_sum1);


@@ -496,7 +496,7 @@ the file's name.
  
        if (mapbuf)
                unmap_file(mapbuf);
  
        if (mapbuf)
                unmap_file(mapbuf);

-@@ -358,7 +369,7 @@ static void read_gen_name(int fd, char *
+@@ -274,7 +285,7 @@ static int receive_data(int f_in, char *

  
  static void discard_receive_data(int f_in, OFF_T length)
  {
  
  static void discard_receive_data(int f_in, OFF_T length)
  {


@@ -504,48 +504,52 @@ the file's name.
 +      receive_data(f_in, NULL, -1, 0, NULL, -1, length, NULL);
  }
  
 +      receive_data(f_in, NULL, -1, 0, NULL, -1, length, NULL);
  }
  

- 
-@@ -621,8 +632,12 @@ int recv_files(int f_in, struct file_lis
-                       rprintf(FINFO, "%s\n", safe_fname(fname));
+ static void handle_delayed_updates(struct file_list *flist, char *local_name)
+@@ -611,8 +622,12 @@ int recv_files(int f_in, struct file_lis
+                       rprintf(FINFO, "%s\n", fname);

  
                /* recv file data */
 +#if HAVE_LINK
 +              if (link_by_hash_dir)
  
                /* recv file data */
 +#if HAVE_LINK
 +              if (link_by_hash_dir)

-+                      file->u.sum = (char*)malloc(MD4_SUM_LENGTH);
++                      file->u.sum = new_array(char, MD4_SUM_LENGTH);

 +#endif
                recv_ok = receive_data(f_in, fnamecmp, fd1, st.st_size,
 -                                     fname, fd2, file->length);
 +                                     fname, fd2, file->length, file->u.sum);
  
 +#endif
                recv_ok = receive_data(f_in, fnamecmp, fd1, st.st_size,
 -                                     fname, fd2, file->length);
 +                                     fname, fd2, file->length, file->u.sum);
  

-               if (!log_before_transfer)
-                       log_recv(file, &initial_stats, iflags);
---- orig/rsync.c       2005-02-21 10:51:52
-+++ rsync.c    2005-02-21 11:04:36
-@@ -38,6 +38,7 @@ extern int recurse;
+               log_item(log_code, file, &initial_stats, iflags, NULL);
+ 
+--- old/rsync.c
++++ new/rsync.c
+@@ -48,6 +48,7 @@ extern int inplace;

  extern int keep_dirlinks;
  extern int make_backups;
  extern int keep_dirlinks;
  extern int make_backups;

- extern struct stats stats;
+ extern mode_t orig_umask;

 +extern char *link_by_hash_dir;
 +extern char *link_by_hash_dir;

+ extern struct stats stats;
+ extern struct chmod_mode_struct *daemon_chmod_modes;

  
  

- 
- /*
-@@ -188,7 +189,12 @@ void finish_transfer(char *fname, char *
-               rprintf(FINFO, "renaming %s to %s\n",
-                       safe_fname(fnametmp), safe_fname(fname));
-       }
--      ret = robust_rename(fnametmp, fname, file->mode & INITACCESSPERMS);
+@@ -271,8 +272,15 @@ void finish_transfer(char *fname, char *
+       /* move tmp file over real file */
+       if (verbose > 2)
+               rprintf(FINFO, "renaming %s to %s\n", fnametmp, fname);
+-      ret = robust_rename(fnametmp, fname, partialptr,
+-                          file->mode & INITACCESSPERMS);

 +#if HAVE_LINK
 +      if (link_by_hash_dir)
 +              ret = link_by_hash(fnametmp, fname, file);
 +      else
 +#endif
 +#if HAVE_LINK
 +      if (link_by_hash_dir)
 +              ret = link_by_hash(fnametmp, fname, file);
 +      else
 +#endif

-+              ret = robust_rename(fnametmp, fname, file->mode & INITACCESSPERMS);
++      {
++              ret = robust_rename(fnametmp, fname, partialptr,
++                                  file->mode & INITACCESSPERMS);
++      }

        if (ret < 0) {
                rsyserr(FERROR, errno, "%s %s -> \"%s\"",
        if (ret < 0) {
                rsyserr(FERROR, errno, "%s %s -> \"%s\"",

-                   ret == -2 ? "copy" : "rename",
---- orig/rsync.h       2005-02-20 00:02:23
-+++ rsync.h    2004-07-03 20:20:15
-@@ -620,6 +620,14 @@ struct stats {
+                       ret == -2 ? "copy" : "rename",
+--- old/rsync.h
++++ new/rsync.h
+@@ -651,6 +651,14 @@ struct stats {

        int current_file_index;
  };
  
        int current_file_index;
  };
  


@@ -557,16 +561,16 @@ the file's name.
 +      uint32 nlink;
 +};
 +
 +      uint32 nlink;
 +};
 +

+ struct chmod_mode_struct;

  
  #include "byteorder.h"
  
  #include "byteorder.h"

- #include "lib/mdfour.h"
---- orig/rsync.yo      2005-02-20 01:12:43
-+++ rsync.yo   2005-02-13 06:58:47
-@@ -356,6 +356,7 @@ to the detailed description below for a 
+--- old/rsync.yo
++++ new/rsync.yo
+@@ -366,6 +366,7 @@ to the detailed description below for a 

       --compare-dest=DIR      also compare received files relative to DIR
       --copy-dest=DIR         ... and include copies of unchanged files
       --link-dest=DIR         hardlink to files in DIR when unchanged
 +     --link-by-hash=DIR      create hardlinks by hash into DIR
   -z, --compress              compress file data during the transfer
       --compare-dest=DIR      also compare received files relative to DIR
       --copy-dest=DIR         ... and include copies of unchanged files
       --link-dest=DIR         hardlink to files in DIR when unchanged
 +     --link-by-hash=DIR      create hardlinks by hash into DIR
   -z, --compress              compress file data during the transfer

+      --compress-level=NUM    explicitly set compression level

   -C, --cvs-exclude           auto-ignore files in the same way CVS does
   -C, --cvs-exclude           auto-ignore files in the same way CVS does

-  -f, --filter=RULE           add a file-filtering RULE