[rsync/rsync.git] / generator.c

/* 
   Copyright (C) Andrew Tridgell 1996
   Copyright (C) Paul Mackerras 1996
   
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.
   
   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.
   
   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/

#include "rsync.h"

extern int verbose;
extern int dry_run;
extern int relative_paths;
extern int preserve_links;
extern int am_root;
extern int preserve_devices;
extern int preserve_hard_links;
extern int update_only;
extern int whole_file;
extern int block_size;
extern int csum_length;
extern int ignore_times;
extern int io_timeout;
extern int remote_version;
extern int always_checksum;


/* choose whether to skip a particular file */
static int skip_file(char *fname,
		     struct file_struct *file, STRUCT_STAT *st)
{
	if (st->st_size != file->length) {
		return 0;
	}
	
	/* if always checksum is set then we use the checksum instead 
	   of the file time to determine whether to sync */
	if (always_checksum && S_ISREG(st->st_mode)) {
		char sum[MD4_SUM_LENGTH];
		file_checksum(fname,sum,st->st_size);
		return (memcmp(sum,file->sum,csum_length) == 0);
	}

	if (ignore_times) {
		return 0;
	}

	return (st->st_mtime == file->modtime);
}


/* use a larger block size for really big files */
static int adapt_block_size(struct file_struct *file, int bsize)
{
	int ret;

	if (bsize != BLOCK_SIZE) return bsize;

	ret = file->length / (10000); /* rough heuristic */
	ret = ret & ~15; /* multiple of 16 */
	if (ret < bsize) ret = bsize;
	if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
	return ret;
}


/*
  send a sums struct down a fd
  */
static void send_sums(struct sum_struct *s,int f_out)
{
	int i;

  /* tell the other guy how many we are going to be doing and how many
     bytes there are in the last chunk */
	write_int(f_out,s?s->count:0);
	write_int(f_out,s?s->n:block_size);
	write_int(f_out,s?s->remainder:0);
	if (s)
		for (i=0;i<s->count;i++) {
			write_int(f_out,s->sums[i].sum1);
			write_buf(f_out,s->sums[i].sum2,csum_length);
		}
}


/*
  generate a stream of signatures/checksums that describe a buffer

  generate approximately one checksum every n bytes
  */
static struct sum_struct *generate_sums(struct map_struct *buf,OFF_T len,int n)
{
	int i;
	struct sum_struct *s;
	int count;
	int block_len = n;
	int remainder = (len%block_len);
	OFF_T offset = 0;

	count = (len+(block_len-1))/block_len;

	s = (struct sum_struct *)malloc(sizeof(*s));
	if (!s) out_of_memory("generate_sums");

	s->count = count;
	s->remainder = remainder;
	s->n = n;
	s->flength = len;

	if (count==0) {
		s->sums = NULL;
		return s;
	}

	if (verbose > 3)
		rprintf(FINFO,"count=%d rem=%d n=%d flength=%d\n",
			s->count,s->remainder,s->n,(int)s->flength);

	s->sums = (struct sum_buf *)malloc(sizeof(s->sums[0])*s->count);
	if (!s->sums) out_of_memory("generate_sums");
  
	for (i=0;i<count;i++) {
		int n1 = MIN(len,n);
		char *map = map_ptr(buf,offset,n1);

		s->sums[i].sum1 = get_checksum1(map,n1);
		get_checksum2(map,n1,s->sums[i].sum2);

		s->sums[i].offset = offset;
		s->sums[i].len = n1;
		s->sums[i].i = i;

		if (verbose > 3)
			rprintf(FINFO,"chunk[%d] offset=%d len=%d sum1=%08x\n",
				i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);

		len -= n1;
		offset += n1;
	}

	return s;
}


void recv_generator(char *fname,struct file_list *flist,int i,int f_out)
{  
	int fd;
	STRUCT_STAT st;
	struct map_struct *buf;
	struct sum_struct *s;
	int statret;
	struct file_struct *file = flist->files[i];

	if (verbose > 2)
		rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);

	statret = link_stat(fname,&st);

	if (S_ISDIR(file->mode)) {
		if (dry_run) return;
		if (statret == 0 && !S_ISDIR(st.st_mode)) {
			if (do_unlink(fname) != 0) {
				rprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
				return;
			}
			statret = -1;
		}
		if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
			if (!(relative_paths && errno==ENOENT && 
			      create_directory_path(fname)==0 && 
			      do_mkdir(fname,file->mode)==0)) {
				rprintf(FERROR,"mkdir %s : %s (2)\n",
					fname,strerror(errno));
			}
		}
		if (set_perms(fname,file,NULL,0) && verbose) 
			rprintf(FINFO,"%s/\n",fname);
		return;
	}

	if (preserve_links && S_ISLNK(file->mode)) {
#if SUPPORT_LINKS
		char lnk[MAXPATHLEN];
		int l;
		extern int safe_symlinks;

		if (safe_symlinks && unsafe_symlink(file->link, fname)) {
			if (verbose) {
				rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
					fname,file->link);
			}
			return;
		}
		if (statret == 0) {
			l = readlink(fname,lnk,MAXPATHLEN-1);
			if (l > 0) {
				lnk[l] = 0;
				if (strcmp(lnk,file->link) == 0) {
					set_perms(fname,file,&st,1);
					return;
				}
			}
		}
		delete_file(fname);
		if (do_symlink(file->link,fname) != 0) {
			rprintf(FERROR,"link %s -> %s : %s\n",
				fname,file->link,strerror(errno));
		} else {
			set_perms(fname,file,NULL,0);
			if (verbose) {
				rprintf(FINFO,"%s -> %s\n",
					fname,file->link);
			}
		}
#endif
		return;
	}

#ifdef HAVE_MKNOD
	if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
		if (statret != 0 || 
		    st.st_mode != file->mode ||
		    st.st_rdev != file->rdev) {	
			delete_file(fname);
			if (verbose > 2)
				rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
					fname,(int)file->mode,(int)file->rdev);
			if (do_mknod(fname,file->mode,file->rdev) != 0) {
				rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
			} else {
				set_perms(fname,file,NULL,0);
				if (verbose)
					rprintf(FINFO,"%s\n",fname);
			}
		} else {
			set_perms(fname,file,&st,1);
		}
		return;
	}
#endif

	if (preserve_hard_links && check_hard_link(file)) {
		if (verbose > 1)
			rprintf(FINFO,"%s is a hard link\n",f_name(file));
		return;
	}

	if (!S_ISREG(file->mode)) {
		rprintf(FINFO,"skipping non-regular file %s\n",fname);
		return;
	}

	if (statret == -1) {
		if (errno == ENOENT) {
			write_int(f_out,i);
			if (!dry_run) send_sums(NULL,f_out);
		} else {
			if (verbose > 1)
				rprintf(FERROR,"recv_generator failed to open %s\n",fname);
		}
		return;
	}

	if (!S_ISREG(st.st_mode)) {
		if (delete_file(fname) != 0) {
			return;
		}

		/* now pretend the file didn't exist */
		write_int(f_out,i);
		if (!dry_run) send_sums(NULL,f_out);    
		return;
	}

	if (update_only && st.st_mtime > file->modtime) {
		if (verbose > 1)
			rprintf(FINFO,"%s is newer\n",fname);
		return;
	}

	if (skip_file(fname, file, &st)) {
		set_perms(fname,file,&st,1);
		return;
	}

	if (dry_run) {
		write_int(f_out,i);
		return;
	}

	if (whole_file) {
		write_int(f_out,i);
		send_sums(NULL,f_out);    
		return;
	}

	/* open the file */  
	fd = open(fname,O_RDONLY);

	if (fd == -1) {
		rprintf(FERROR,"failed to open %s : %s\n",fname,strerror(errno));
		rprintf(FERROR,"skipping %s\n",fname);
		return;
	}

	if (st.st_size > 0) {
		buf = map_file(fd,st.st_size);
	} else {
		buf = NULL;
	}

	if (verbose > 3)
		rprintf(FINFO,"gen mapped %s of size %d\n",fname,(int)st.st_size);

	s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));

	if (verbose > 2)
		rprintf(FINFO,"sending sums for %d\n",i);

	write_int(f_out,i);
	send_sums(s,f_out);

	close(fd);
	if (buf) unmap_file(buf);

	free_sums(s);
}


void generate_files(int f,struct file_list *flist,char *local_name,int f_recv)
{
	int i;
	int phase=0;

	if (verbose > 2)
		rprintf(FINFO,"generator starting pid=%d count=%d\n",
			(int)getpid(),flist->count);

	for (i = 0; i < flist->count; i++) {
		struct file_struct *file = flist->files[i];
		mode_t saved_mode = file->mode;
		if (!file->basename) continue;

		/* we need to ensure that any directories we create have writeable
		   permissions initially so that we can create the files within
		   them. This is then fixed after the files are transferred */
		if (!am_root && S_ISDIR(file->mode)) {
			file->mode |= S_IWUSR; /* user write */
		}

		recv_generator(local_name?local_name:f_name(file),
			       flist,i,f);

		file->mode = saved_mode;
	}

	phase++;
	csum_length = SUM_LENGTH;
	ignore_times=1;

	if (verbose > 2)
		rprintf(FINFO,"generate_files phase=%d\n",phase);

	write_int(f,-1);

	/* we expect to just sit around now, so don't exit on a
	   timeout. If we really get a timeout then the other process should
	   exit */
	io_timeout = 0;

	if (remote_version >= 13) {
		/* in newer versions of the protocol the files can cycle through
		   the system more than once to catch initial checksum errors */
		for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
			struct file_struct *file = flist->files[i];
			recv_generator(local_name?local_name:f_name(file),
				       flist,i,f);    
		}

		phase++;
		if (verbose > 2)
			rprintf(FINFO,"generate_files phase=%d\n",phase);

		write_int(f,-1);
	}
}
Commit	Line	Data
2f03f956 AT	1	/*
	2	Copyright (C) Andrew Tridgell 1996
	3	Copyright (C) Paul Mackerras 1996
	4
	5	This program is free software; you can redistribute it and/or modify
	6	it under the terms of the GNU General Public License as published by
	7	the Free Software Foundation; either version 2 of the License, or
	8	(at your option) any later version.
	9
	10	This program is distributed in the hope that it will be useful,
	11	but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	GNU General Public License for more details.
	14
	15	You should have received a copy of the GNU General Public License
	16	along with this program; if not, write to the Free Software
	17	Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
	18	*/
	19
	20	#include "rsync.h"
	21
	22	extern int verbose;
	23	extern int dry_run;
	24	extern int relative_paths;
	25	extern int preserve_links;
	26	extern int am_root;
	27	extern int preserve_devices;
	28	extern int preserve_hard_links;
	29	extern int update_only;
	30	extern int whole_file;
	31	extern int block_size;
	32	extern int csum_length;
	33	extern int ignore_times;
	34	extern int io_timeout;
	35	extern int remote_version;
	36	extern int always_checksum;
	37
	38
	39	/* choose whether to skip a particular file */
	40	static int skip_file(char *fname,
	41	struct file_struct file, STRUCT_STAT st)
	42	{
	43	if (st->st_size != file->length) {
	44	return 0;
	45	}
	46
	47	/* if always checksum is set then we use the checksum instead
	48	of the file time to determine whether to sync */
	49	if (always_checksum && S_ISREG(st->st_mode)) {
	50	char sum[MD4_SUM_LENGTH];
	51	file_checksum(fname,sum,st->st_size);
	52	return (memcmp(sum,file->sum,csum_length) == 0);
	53	}
	54
	55	if (ignore_times) {
	56	return 0;
	57	}
	58
	59	return (st->st_mtime == file->modtime);
	60	}
	61
	62
	63	/* use a larger block size for really big files */
	64	static int adapt_block_size(struct file_struct *file, int bsize)
65	{
66	int ret;
67
68	if (bsize != BLOCK_SIZE) return bsize;
69
70	ret = file->length / (10000); /* rough heuristic */
71	ret = ret & ~15; /* multiple of 16 */
72	if (ret < bsize) ret = bsize;
73	if (ret > CHUNK_SIZE/2) ret = CHUNK_SIZE/2;
74	return ret;
75	}
76
77
78	/*
79	send a sums struct down a fd
80	*/
81	static void send_sums(struct sum_struct *s,int f_out)
82	{
83	int i;
84
85	/* tell the other guy how many we are going to be doing and how many
86	bytes there are in the last chunk */
87	write_int(f_out,s?s->count:0);
88	write_int(f_out,s?s->n:block_size);
89	write_int(f_out,s?s->remainder:0);
90	if (s)
91	for (i=0;i<s->count;i++) {
92	write_int(f_out,s->sums[i].sum1);
93	write_buf(f_out,s->sums[i].sum2,csum_length);
94	}
95	}
96
97
98	/*
99	generate a stream of signatures/checksums that describe a buffer
100
101	generate approximately one checksum every n bytes
102	*/
103	static struct sum_struct generate_sums(struct map_struct buf,OFF_T len,int n)
104	{
105	int i;
106	struct sum_struct *s;
107	int count;
108	int block_len = n;
109	int remainder = (len%block_len);
110	OFF_T offset = 0;
111
112	count = (len+(block_len-1))/block_len;
113
114	s = (struct sum_struct )malloc(sizeof(s));
115	if (!s) out_of_memory("generate_sums");
116
117	s->count = count;
118	s->remainder = remainder;
119	s->n = n;
120	s->flength = len;
121
122	if (count==0) {
123	s->sums = NULL;
124	return s;
125	}
126
127	if (verbose > 3)
128	rprintf(FINFO,"count=%d rem=%d n=%d flength=%d\n",
129	s->count,s->remainder,s->n,(int)s->flength);
130
131	s->sums = (struct sum_buf )malloc(sizeof(s->sums[0])s->count);
132	if (!s->sums) out_of_memory("generate_sums");
133
134	for (i=0;i<count;i++) {
135	int n1 = MIN(len,n);
136	char *map = map_ptr(buf,offset,n1);
137
138	s->sums[i].sum1 = get_checksum1(map,n1);
139	get_checksum2(map,n1,s->sums[i].sum2);
140
141	s->sums[i].offset = offset;
142	s->sums[i].len = n1;
143	s->sums[i].i = i;
144
145	if (verbose > 3)
146	rprintf(FINFO,"chunk[%d] offset=%d len=%d sum1=%08x\n",
147	i,(int)s->sums[i].offset,s->sums[i].len,s->sums[i].sum1);
148
149	len -= n1;
150	offset += n1;
151	}
152
153	return s;
154	}
155
156
157	void recv_generator(char fname,struct file_list flist,int i,int f_out)
158	{
159	int fd;
160	STRUCT_STAT st;
161	struct map_struct *buf;
162	struct sum_struct *s;
163	int statret;
164	struct file_struct *file = flist->files[i];
165
166	if (verbose > 2)
167	rprintf(FINFO,"recv_generator(%s,%d)\n",fname,i);
168
169	statret = link_stat(fname,&st);
170
171	if (S_ISDIR(file->mode)) {
172	if (dry_run) return;
173	if (statret == 0 && !S_ISDIR(st.st_mode)) {
174	if (do_unlink(fname) != 0) {
175	rprintf(FERROR,"unlink %s : %s\n",fname,strerror(errno));
176	return;
177	}
178	statret = -1;
179	}
180	if (statret != 0 && do_mkdir(fname,file->mode) != 0 && errno != EEXIST) {
181	if (!(relative_paths && errno==ENOENT &&
182	create_directory_path(fname)==0 &&
183	do_mkdir(fname,file->mode)==0)) {
184	rprintf(FERROR,"mkdir %s : %s (2)\n",
185	fname,strerror(errno));
186	}
187	}
188	if (set_perms(fname,file,NULL,0) && verbose)
189	rprintf(FINFO,"%s/\n",fname);
190	return;
191	}
192
193	if (preserve_links && S_ISLNK(file->mode)) {
194	#if SUPPORT_LINKS
195	char lnk[MAXPATHLEN];
196	int l;
197	extern int safe_symlinks;
198
199	if (safe_symlinks && unsafe_symlink(file->link, fname)) {
200	if (verbose) {
201	rprintf(FINFO,"ignoring unsafe symlink %s -> %s\n",
202	fname,file->link);
203	}
204	return;
205	}
206	if (statret == 0) {
207	l = readlink(fname,lnk,MAXPATHLEN-1);
208	if (l > 0) {
209	lnk[l] = 0;
210	if (strcmp(lnk,file->link) == 0) {
211	set_perms(fname,file,&st,1);
212	return;
213	}
214	}
215	}
216	delete_file(fname);
217	if (do_symlink(file->link,fname) != 0) {
218	rprintf(FERROR,"link %s -> %s : %s\n",
219	fname,file->link,strerror(errno));
220	} else {
221	set_perms(fname,file,NULL,0);
222	if (verbose) {
223	rprintf(FINFO,"%s -> %s\n",
224	fname,file->link);
225	}
226	}
227	#endif
228	return;
229	}
230
231	#ifdef HAVE_MKNOD
232	if (am_root && preserve_devices && IS_DEVICE(file->mode)) {
233	if (statret != 0 \|\|
234	st.st_mode != file->mode \|\|
235	st.st_rdev != file->rdev) {
236	delete_file(fname);
237	if (verbose > 2)
238	rprintf(FINFO,"mknod(%s,0%o,0x%x)\n",
239	fname,(int)file->mode,(int)file->rdev);
240	if (do_mknod(fname,file->mode,file->rdev) != 0) {
241	rprintf(FERROR,"mknod %s : %s\n",fname,strerror(errno));
242	} else {
243	set_perms(fname,file,NULL,0);
244	if (verbose)
245	rprintf(FINFO,"%s\n",fname);
246	}
247	} else {
248	set_perms(fname,file,&st,1);
249	}
250	return;
251	}
252	#endif
253
254	if (preserve_hard_links && check_hard_link(file)) {
255	if (verbose > 1)
256	rprintf(FINFO,"%s is a hard link\n",f_name(file));
257	return;
258	}
259
260	if (!S_ISREG(file->mode)) {
261	rprintf(FINFO,"skipping non-regular file %s\n",fname);
262	return;
263	}
264
265	if (statret == -1) {
266	if (errno == ENOENT) {
267	write_int(f_out,i);
268	if (!dry_run) send_sums(NULL,f_out);
269	} else {
270	if (verbose > 1)
271	rprintf(FERROR,"recv_generator failed to open %s\n",fname);
272	}
273	return;
274	}
275
276	if (!S_ISREG(st.st_mode)) {
277	if (delete_file(fname) != 0) {
278	return;
279	}
280
281	/* now pretend the file didn't exist */
282	write_int(f_out,i);
283	if (!dry_run) send_sums(NULL,f_out);
284	return;
285	}
286
287	if (update_only && st.st_mtime > file->modtime) {
288	if (verbose > 1)
289	rprintf(FINFO,"%s is newer\n",fname);
290	return;
291	}
292
293	if (skip_file(fname, file, &st)) {
294	set_perms(fname,file,&st,1);
295	return;
296	}
297
298	if (dry_run) {
299	write_int(f_out,i);
300	return;
301	}
302
303	if (whole_file) {
304	write_int(f_out,i);
305	send_sums(NULL,f_out);
306	return;
307	}
308
309	/* open the file */
310	fd = open(fname,O_RDONLY);
311
312	if (fd == -1) {
313	rprintf(FERROR,"failed to open %s : %s\n",fname,strerror(errno));
314	rprintf(FERROR,"skipping %s\n",fname);
315	return;
316	}
317
318	if (st.st_size > 0) {
319	buf = map_file(fd,st.st_size);
320	} else {
321	buf = NULL;
322	}
323
324	if (verbose > 3)
325	rprintf(FINFO,"gen mapped %s of size %d\n",fname,(int)st.st_size);
326
327	s = generate_sums(buf,st.st_size,adapt_block_size(file, block_size));
328
329	if (verbose > 2)
330	rprintf(FINFO,"sending sums for %d\n",i);
331
332	write_int(f_out,i);
333	send_sums(s,f_out);
334
335	close(fd);
336	if (buf) unmap_file(buf);
337
338	free_sums(s);
339	}
340
341
342
343	void generate_files(int f,struct file_list flist,char local_name,int f_recv)
344	{
345	int i;
346	int phase=0;
347
348	if (verbose > 2)
349	rprintf(FINFO,"generator starting pid=%d count=%d\n",
350	(int)getpid(),flist->count);
351
352	for (i = 0; i < flist->count; i++) {
353	struct file_struct *file = flist->files[i];
354	mode_t saved_mode = file->mode;
355	if (!file->basename) continue;
356
357	/* we need to ensure that any directories we create have writeable
358	permissions initially so that we can create the files within
359	them. This is then fixed after the files are transferred */
360	if (!am_root && S_ISDIR(file->mode)) {
361	file->mode \|= S_IWUSR; /* user write */
362	}
363
364	recv_generator(local_name?local_name:f_name(file),
365	flist,i,f);
366
367	file->mode = saved_mode;
368	}
369
370	phase++;
371	csum_length = SUM_LENGTH;
372	ignore_times=1;
373
374	if (verbose > 2)
375	rprintf(FINFO,"generate_files phase=%d\n",phase);
376
377	write_int(f,-1);
378
379	/* we expect to just sit around now, so don't exit on a
380	timeout. If we really get a timeout then the other process should
381	exit */
382	io_timeout = 0;
383
384	if (remote_version >= 13) {
385	/* in newer versions of the protocol the files can cycle through
386	the system more than once to catch initial checksum errors */
387	for (i=read_int(f_recv); i != -1; i=read_int(f_recv)) {
388	struct file_struct *file = flist->files[i];
389	recv_generator(local_name?local_name:f_name(file),
390	flist,i,f);
391	}
392
393	phase++;
394	if (verbose > 2)
395	rprintf(FINFO,"generate_files phase=%d\n",phase);
396
397	write_int(f,-1);
398	}
399	}