using mmap for copying large files

rohanak · 05-12-2008, 11:27 AM

Hello everyone,
I tried using mmap to copy very large files in the following way :

Code:

if((fps = open(source, O_RDONLY)) == -1)
			printf("error : can't open source file for reading\n");
		if((fpd = open(dest, O_RDWR | O_CREAT | O_TRUNC, 0777)) == -1)
			printf("error : cant open destination file for writing\n");
		printf("\nsource = %s\n",source);
		//while((bytes = read(fps, buffer, BSIZE)) > 0)
		//	write(fpd, buffer, bytes);
		fstat(fps,&statbuf);
		filesize=statbuf.st_size;
		lseek(fpd,filesize-1,SEEK_SET);
		write(fpd,"",1);
		lseek(fpd,0,SEEK_SET);
		bytes=2097152;
		while(filesize > 0)
		{
			if(filesize < bytes)
			{
				bytes=filesize;
				filesize=0;
			}
			else
				filesize-=bytes;
			if((src=mmap((caddr_t)0,bytes,PROT_READ,MAP_SHARED,fps,0))==MAP_FAILED)
			{	
				printf("mmap error : fps\n");
				exit(1);
			}
			if((dst=mmap((caddr_t)0,bytes,PROT_READ | PROT_WRITE,MAP_SHARED,fpd,0))==MAP_FAILED)
			{
				printf("mmap error : fpd\n");
				exit(1);
			}
			memcpy(dst,src,bytes);
			if((munmap(src,bytes))==-1)
			{
				printf("munmap error : src\n");
				exit(1);
			}
			if((munmap(dst,bytes)==-1))
			{
				printf("munmap error : dst\n");
				exit(1);
			}
			lseek(fps,bytes,SEEK_SET);
			lseek(fpd,bytes,SEEK_SET);
		}

I used the variables bytes instead of the filesize for mmap as I guess mapping wont work for very large files.This is working for text files but not for binary files.Can anyone suggest changes?

rohanak · 05-12-2008, 11:42 AM

Hello again,

The problem with copying binary files especially video files using mmap is that the destination file is broken (as reported by VLC media player).How can I solve this??

rohanak · 05-13-2008, 08:55 AM

I changed the code to this :

Code:

#define BSIZE 1048576
.......................

if((fps = open(source, O_RDONLY)) == -1)
			printf("error : can't open source file for reading\n");
		if((fpd = open(dest, O_RDWR | O_CREAT | O_TRUNC, 0777)) == -1)
			printf("error : cant open destination file for writing\n");
		
		fstat(fps,&statbuf);
		filesize=statbuf.st_size;
		lseek(fpd,filesize-1,SEEK_SET);
		
		write(fpd,"",1);
		bytes=BSIZE;
		offset=0;
		while(filesize > 0)
		{
			if(filesize < bytes)
			{
				bytes=filesize;
				filesize=0;
				
			}
			else
			{
				filesize-=bytes;
				
			}
			if((src=mmap((caddr_t)0,bytes,PROT_READ,MAP_SHARED,fps,offset))==MAP_FAILED)
			{	
				printf("mmap error : fps\n");
				exit(1);
			}
			if((dst=mmap((caddr_t)0,bytes,PROT_READ | PROT_WRITE,MAP_SHARED,fpd,offset))==MAP_FAILED)
			{
				printf("mmap error : fpd\n");
				exit(1);
			}
			memcpy(dst,src,bytes);
			if((munmap(src,bytes))==-1)
			{
				printf("munmap error : src\n");
				exit(1);
			}
			if((munmap(dst,bytes)==-1))
			{
				printf("munmap error : dst\n");
				exit(1);
			}
			offset=offset+bytes;
		}

This works fine for all kinds of files...However it is much slower than read/write.
First I used mmap to copy all files in a directory totalling to 3GB.It took 5m48s while a read/write took only 4m48s.

bigearsbilly · 05-15-2008, 07:15 PM

what's the loop for?

you sure it's not looping over and over?
the mmap doesn't need a loop.

jim mcnamara · 05-18-2008, 06:28 AM

mmap does not save you anything; it costs more memory. If you are copying one file to one destination.
If you are copying 1 file to multiple destinations then it makes sense. Otherwise just fopen(), loop{ fread() fwrite() }, then fclose().