Syncr.java - syncr - Simple program to synchronise directories fast

Source path: svn/ trunk/ de.xam.syncr/ src/ main/ java/ de/ xam/ syncr/ Syncr.java

‹r4

package de.xam.syncr;

import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.WritableByteChannel;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
import java.util.zip.ZipOutputStream;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


/**
 * @author voelkel
 * 
 *         TODO preserve flags
 * 
 *         FIXME preserve timestamps
 * 
 *         TODO propagate deletion of files, too
 * 
 *         TODO write info file to targetRoot about sync date + job
 * 
 *         TODO add timeouts when reading a file takes too long
 * 
 *         FIXME uses too much memory sometimes - probably when copying really
 *         large files
 * 
 */
public class Syncr {
	
	private static final long KB = 1024;
	private static final long MB = 1024 * KB;
	@SuppressWarnings("unused")
	private static final long GB = 1024 * MB;
	
	private static Logger log = LoggerFactory.getLogger(Syncr.class);
	
	public static Set<String> skipFilenames = new HashSet<String>();
	
	// FIXME
	public static Set<String> skipDirnames = new HashSet<String>();
	
	public static void main(String[] args) {
		// example
		// sync("C:/", "D:/_syncr");
	}
	
	/** FIXME 5 x 18 hours */
	// TODO find out how change dates are stored in zip files - maybe a metadata
	// standard
	public static long skipAge = 5 * 18 * 60 * 60 * 1000;
	
	public static long sleepMsAfterFileCopy = 0;// 100;
	
	public static long sleepMsAfterFileRead = 0;// 10;
	
	private static long dirProcessed = 0;
	
	private static long kb_written = 0;
	private static long kb_read = 0;
	
	private static long start = 0;
	
	private static long lastStats = 0;
	
	/**
	 * Makes sure that all files residing at sourceDir are also present at
	 * targetDir. If files in sourceDir have a more recent changeDate than the
	 * one in targetDir, the file is copied again.
	 * 
	 * @param sourceDirName
	 * @param targetDirName
	 * @param archiveDeleted if true, files deleted on 'source' are moved to a
	 *            special folder __deleted. So in effect, they are not really
	 *            deleted.
	 * @param compressSomeFolders if true, eclipse projects end up in a zip
	 *            file. This results in much faster writes, since writing a
	 *            single zip file is faster than writing 1000 of small files.
	 * @throws IOException
	 */
	public static void sync(String sourceDirName, String targetDirName, boolean archiveDeleted,
	        boolean compressSomeFolders) throws IOException {
		skipFilenames.addAll(Arrays.asList(new String[] { "hiberfil.sys", "pagefile.sys",
		        "Thumbs.db", "autorun.inf", "UsrClass.dat", "UsrClass.dat.LOG", "ntuser.dat.LOG",
		        "ntuser.dat", "parent.lock" /* firefox */}));
		
		skipDirnames.addAll(Arrays.asList(new String[] { "Temp", "RECYCLER",
		        "System Volume Information", "Google Desktop" }));
		
		start = System.currentTimeMillis();
		System.out.println("Start " + new Date());
		
		System.out.println("Config: sleepMsAfterFileCopy=" + sleepMsAfterFileCopy
		        + " sleepMsAfterFileRead=" + sleepMsAfterFileRead);
		
		Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
		File source = new File(sourceDirName);
		if(!source.exists()) {
			log.error("Source dir " + source.getAbsolutePath() + " does not exist");
		}
		File target = new File(targetDirName);
		if(!target.exists()) {
			target.mkdirs();
			log.info("Target dir " + target.getAbsolutePath() + " has been created");
		}
		syncDirectory(source, source, target, archiveDeleted, compressSomeFolders);
		
		System.out.println("Done at " + new Date());
		printStats();
	}
	
	private static long runningSeconds() {
		long stop = System.currentTimeMillis();
		long seconds = ((stop - start) / 1000);
		return seconds;
	}
	
	private static void printStats() {
		long seconds = runningSeconds() + 1;
		log.info("Speed: " + (kb_read / seconds) + " KB/s, " + (dirProcessed / seconds)
		        + " directory/s. " + seconds + "s. " + filesCopied + " files. In "
		        + (kb_read / 1024) + " MB. Out " + (kb_written / 1024) + " MB.");
	}
	
	private static void printStats(File dir) {
		printStats();
		log.info("Processed " + dirProcessed + " directories. Syncing " + dir.getAbsolutePath());
	}
	
	/**
	 * @param sourceRoot
	 * @param sourceDir
	 * @param targetRoot
	 * @param archiveDeleted TODO
	 * @param compressSomeFolders TODO
	 * @return true if any write operations happened in this directory or
	 *         subdirectories thereof
	 * @throws IOException
	 */
	public static boolean syncDirectory(File sourceRoot, File sourceDir, File targetRoot,
	        boolean archiveDeleted, boolean compressSomeFolders) throws IOException {
		assert sourceDir != null;
		assert sourceDir.isDirectory() : "sourcedir " + sourceDir.getAbsolutePath()
		        + " is not a directoy";
		
		dirProcessed++;
		
		if(isExclude(sourceDir)) {
			log.debug("Excluded: " + sourceDir.getAbsolutePath());
			return false;
		}
		
		if(compressSomeFolders && shouldZipDirOnSyncDrive(sourceDir)) {
			// create target zip file
			File targetZipFile = new File(targetRoot, getLocalName(sourceRoot, sourceDir) + ".zip");
			log.debug("Syncing to zip file " + targetZipFile.getAbsolutePath());
			syncDirectoryToZipFile(sourceRoot, sourceDir, targetRoot, sourceDir, targetZipFile);
			return true;
		}
		
		/** normal copy */
		log.trace("Syncing " + sourceDir.getAbsolutePath());
		
		// print stats every 30 seconds
		long runtime = runningSeconds();
		if(runtime - lastStats >= 30) {
			lastStats = runtime;
			printStats(sourceDir);
		}
		
		// if(dirProcessed % 100 == 0) {
		// printStats(sourceDir);
		// }
		
		// TODO speed up with a local database of synced files
		
		File targetDir = targetDir(sourceRoot, sourceDir, targetRoot);
		
		// backup empty directories
		targetDir.mkdirs();
		
		File[] sourceFiles = sourceDir.listFiles();
		boolean targetDirChanged = false;
		if(sourceFiles == null) {
			log.warn("Could not process (or even list) files in " + sourceDir.getAbsolutePath());
		} else {
			Set<String> syncedSourceFileNames = new HashSet<String>();
			for(File sourceFile : sourceFiles) {
				syncedSourceFileNames.add(sourceFile.getName());
				
				if(!sourceFile.exists()) {
					log.warn("While Syncr is running, this file has been removed: "
					        + sourceFile.getAbsolutePath());
				} else if(sourceFile.isFile()) {
					File targetFile = targetFile(sourceRoot, sourceFile, targetRoot);
					if(shouldCopy(sourceFile, targetFile)) {
						copy(sourceFile, targetFile);
						targetDirChanged = true;
						try {
							if(sleepMsAfterFileCopy > 0)
								Thread.sleep(sleepMsAfterFileCopy);
						} catch(InterruptedException e) {
						}
					}
					try {
						if(sleepMsAfterFileRead > 0)
							Thread.sleep(sleepMsAfterFileRead);
					} catch(InterruptedException e) {
					}
				} else if(sourceFile.isDirectory()) {
					targetDirChanged = syncDirectory(sourceRoot, sourceFile, targetRoot,
					        archiveDeleted, compressSomeFolders);
				} else {
					log.warn("Cannot copy " + sourceFile.getAbsolutePath()
					        + " - it is neither a file, nor a directory");
				}
			}
			if(archiveDeleted) {
				// move files in target that are no longer in sourceDir
				File targetFiles[] = targetDir.listFiles();
				if(targetFiles != null) {
					File backupRoot = new File(targetRoot, "__deleted");
					boolean localBackupRootCreated = false;
					for(File targetFile : targetFiles) {
						if(!localBackupRootCreated) {
							backupRoot.mkdirs();
							localBackupRootCreated = true;
						}
						if(!syncedSourceFileNames.contains(targetFile.getName())) {
							File backupFile = targetFile(targetRoot, targetFile, backupRoot);
							backupFile.getParentFile().mkdirs();
							targetFile.renameTo(backupFile);
						}
					}
				}
			}
			
			if(targetDirChanged) {
				targetDir.setLastModified(
				/**
				 * fix for: 'java.lang.IllegalArgumentException: Negative time
				 * at java.io.File.setLastModified(File.java:1222)'
				 */
				Math.max(sourceDir.lastModified(), 0));
			}
		}
		return targetDirChanged;
	}
	
	/**
	 * Archive all files in sourceDir to the zipArchive targetZipFile
	 * 
	 * @param sourceRoot
	 * @param sourceDir
	 * @param targetRoot
	 * @param sourceZippedDirectory TODO
	 * @param targetZipFile
	 * @throws IOException
	 * @throws IOException
	 * @throws ZipException
	 */
	private static void syncDirectoryToZipFile(File sourceRoot, File sourceDir, File targetRoot,
	        File sourceZippedDirectory, File targetZipFile) throws IOException {
		targetZipFile.getParentFile().mkdirs();
		
		// avoid creating the same zip file if we interrupted a backup session
		if(targetZipFile.exists()) {
			if(targetZipFile.lastModified() + skipAge >= System.currentTimeMillis()) {
				log.debug("Skipping zip file creation for source dir "
				        + sourceDir.getAbsolutePath());
				return;
			}
		}
		
		log.info("Zipping " + sourceDir.getAbsolutePath());
		File targetZipFileTemp = new File(targetZipFile.getParentFile(), targetZipFile.getName()
		        + ".temp");
		
		try {
			targetZipFileTemp.createNewFile();
			FileOutputStream fos;
			try {
				fos = new FileOutputStream(targetZipFileTemp);
				ZipOutputStream zos = new ZipOutputStream(fos);
				zos.setLevel(ZipOutputStream.DEFLATED);
				syncDirectoryToZipOutputStream(sourceRoot, sourceZippedDirectory, sourceDir, zos);
				try {
					zos.close();
				} catch(IOException e) {
					log.error("", e);
				}
				try {
					fos.close();
				} catch(IOException e) {
					log.error("", e);
				}
				// rename
				targetZipFileTemp.renameTo(targetZipFile);
			} catch(FileNotFoundException e) {
				log.error("", e);
				assert false;
			}
		} catch(IOException e) {
			throw new RuntimeException("Could not create " + targetZipFileTemp.getAbsolutePath(), e);
		} finally {
			// delete temp file
			targetZipFileTemp.deleteOnExit();
		}
		
	}
	
	private static void syncDirectoryToZipOutputStream(File sourceRoot, File sourceZippedDirectory,
	        File sourceDir, ZipOutputStream zos) throws IOException {
		if(isExclude(sourceDir)) {
			log.debug("Excluded: " + sourceDir.getAbsolutePath());
			return;
		}
		
		for(File sourceFile : sourceDir.listFiles()) {
			if(sourceFile.isFile()) {
				if(shouldArchive(sourceFile)) {
					String zipEntryName = targetNameInZipFile(
					        sourceZippedDirectory.getParentFile(), sourceFile);
					ZipEntry zipEntry = new ZipEntry(zipEntryName);
					/* modification time */
					zipEntry.setTime(sourceFile.lastModified());
					zipEntry.setSize(sourceFile.length());
					zipEntry.setComment("");
					zos.putNextEntry(zipEntry);
					copyFileToStream(sourceFile, zos);
					zos.closeEntry();
					// maybe sleep here
					try {
						Thread.sleep(sleepMsAfterFileRead);
					} catch(InterruptedException e) {
					}
				}
				
			} else {
				// recurse
				syncDirectoryToZipOutputStream(sourceRoot, sourceZippedDirectory, sourceFile, zos);
			}
		}
	}
	
	/** for Zip streams */
	private static void copyFileToStream(File file, OutputStream os) throws IOException {
		FileChannel inChannel = new FileInputStream(file).getChannel();
		WritableByteChannel outChannel = Channels.newChannel(os);
		try {
			inChannel.transferTo(0, inChannel.size(), outChannel);
		} catch(IOException e) {
			throw e;
		} finally {
			if(inChannel != null)
				inChannel.close();
			// do not close out channel
		}
		kb_read += file.length() / 1024;
		filesCopied++;
	}
	
	private static boolean shouldArchive(File sourceFile) {
		assert sourceFile.isFile();
		return !isSkipFileName(sourceFile.getName());
	}
	
	private static boolean isSkipFileName(String name) {
		return skipFilenames.contains(name);
	}
	
	private static boolean isSkipDirName(String name) {
		return skipDirnames.contains(name);
	}
	
	// FIXME kill
	static long LARGE = 1 * 1024 * 1024 * 1024;
	
	private static boolean shouldCopy(File sourceFile, File targetFile) {
		return shouldArchive(sourceFile)
		        && (!targetFile.exists() || targetFile.length() != sourceFile.length() || targetFile
		                .lastModified() < sourceFile.lastModified())
		        && (sourceFile.length() <= LARGE);
	}
	
	private static long filesCopied = 0;
	
	/** reuse buffer to allocate that memory only once */
	private static byte[] buffer;
	
	/**
	 * Create targetFile parent directories and copy sourceFile into targetFile
	 * 
	 * @param sourceFile
	 * @param targetFile
	 * @throws IOException
	 */
	public static void copy(File sourceFile, File targetFile) {
		// handle copying of very large files
		long length = sourceFile.length();
		if(length > 128 * MB) {
			Progress p = new Progress();
			p.start();
			log.info("Copy large file with " + (length / MB) + " MB: "
			        + sourceFile.getAbsolutePath());
			FileInputStream fin;
			try {
				fin = new FileInputStream(sourceFile);
				FileOutputStream fout = new FileOutputStream(targetFile);
				if(buffer == null) {
					buffer = new byte[(int)(8 * MB)];
				}
				long count = 0;
				int n = 0;
				try {
					while(-1 != (n = fin.read(buffer))) {
						p.tick();
						if(p.intervalElapsed(20)) {
							log.info("Writing block of " + n / MB + " MB, "
							        + ((100 * count) / length) + "% done. Speed: "
							        + ((count / MB) / (p.tick() / 1000d)) + " MB/s");
						}
						fout.write(buffer, 0, n);
						count += n;
					}
				} catch(IOException e) {
					log.warn("Could not copy large file " + sourceFile.getAbsolutePath()
					        + " due to ", e);
					targetFile.deleteOnExit();
				}
				fout.flush();
				fout.close();
				return;
			} catch(FileNotFoundException e) {
				log.warn("Cannot copy large file " + sourceFile.getAbsolutePath() + " due to ", e);
			} catch(IOException e) {
				log.warn("Cannot copy large file " + sourceFile.getAbsolutePath() + " due to ", e);
			}
		}
		
		log.trace("Copy small file with just " + sourceFile.length() + " bytes");
		try {
			targetFile.getParentFile().mkdirs();
			FileChannel inChannel = new FileInputStream(sourceFile).getChannel();
			FileChannel outChannel = new FileOutputStream(targetFile).getChannel();
			try {
				inChannel.transferTo(0, inChannel.size(), outChannel);
			} catch(IOException e) {
				log.warn(e.getMessage(), e);
				throw e;
			} finally {
				if(inChannel != null)
					inChannel.close();
				if(outChannel != null)
					outChannel.close();
			}
			// preserve file date
			targetFile.setLastModified(sourceFile.lastModified());
			filesCopied++;
			kb_read += sourceFile.length() / 1024;
			kb_written += sourceFile.length() / 1024;
		} catch(FileNotFoundException e) {
			log.warn("Cannot copy " + sourceFile.getAbsolutePath() + " due to ", e);
		} catch(IOException e) {
			log.warn("Cannot copy " + sourceFile.getAbsolutePath() + " due to ", e);
		}
	}
	
	/**
	 * @param sourceRoot
	 * @param sourceFile can also be a directory
	 * @param targetRoot
	 * @return a File representing the source File but with a modified path so
	 *         that it is in the targetRoot
	 */
	private static File targetFile(File sourceRoot, File sourceFile, File targetRoot) {
		return new File(targetRoot, getLocalName(sourceRoot, sourceFile));
	}
	
	private static File targetDir(File sourceRoot, File sourceDir, File targetRoot) {
		return new File(targetRoot, getLocalName(sourceRoot, sourceDir) + "/");
	}
	
	/**
	 * @param rootDir
	 * @param file
	 * @return the local name of 'file' in 'rootDir', with a leading '/' or '\'
	 */
	public static String getLocalName(File rootDir, File file) {
		return file.getAbsolutePath().substring(rootDir.getAbsolutePath().length());
	}
	
	/**
	 * @param zipFileRoot
	 * @param sourceFile
	 * @param targetRoot
	 * @return a File representing the source File but with a modified path so
	 *         that it is in the targetRoot
	 */
	private static String targetNameInZipFile(File zipFileRoot, File sourceFile) {
		return getLocalName(zipFileRoot, sourceFile).substring(1);
	}
	
	/**
	 * @param dir
	 * @return true if this directory should result in a zip archive on the sync
	 *         drive
	 */
	public static boolean shouldZipDirOnSyncDrive(File dir) {
		assert dir.isDirectory();
		File[] fileList = dir.listFiles();
		return isSoftwareProjectRootDir(fileList) | isEclipseWorkspaceMetadata(dir)
		        | containsZipMaxSyncFile(fileList);
	}
	
	private static boolean containsZipMaxSyncFile(File[] fileList) {
		if(fileList == null) {
			return false;
		}
		
		for(File f : fileList) {
			String name = f.getName();
			if(name.equals("zip.maxsync")) {
				return true;
			}
		}
		return false;
	}
	
	private static boolean isEclipseWorkspaceMetadata(File dir) {
		return dir.getName().equals(".metadata");
	}
	
	/**
	 * @param dir
	 * @return true if directory should not be synced
	 */
	public static boolean isExclude(File dir) {
		return isMavenTargetDir(dir) | isSkipDirName(dir.getName());
	}
	
	private static boolean isMavenTargetDir(File dir) {
		return dir.getParentFile() != null
		        && isSoftwareProjectRootDir(dir.getParentFile().listFiles())
		        && dir.getName().equals("target");
	}
	
	/**
	 * Detect Eclipse projects
	 * 
	 * @param dir
	 * @return
	 */
	public static boolean isSoftwareProjectRootDir(File[] fileList) {
		if(fileList == null) {
			return false;
		}
		
		for(File f : fileList) {
			String name = f.getName();
			if(name.equals(".classpath") || name.equals(".project")) {
				return true;
			}
		}
		return false;
	}
	
	public static void copyFilesLargerThan(File sourceRoot, File targetRoot, final long minSize) {
		for(File f : sourceRoot.listFiles(new FileFilter() {
			@Override
			public boolean accept(File f) {
				return f.length() >= minSize;
			}
		})) {
			File targetFile = targetFile(sourceRoot, f, targetRoot);
			if(shouldCopy(f, targetFile)) {
				copy(f, targetFile);
			}
		}
		// recurse
		for(File dir : sourceRoot.listFiles(new FileFilter() {
			
			@Override
			public boolean accept(File dir) {
				return dir.isDirectory();
			}
		})) {
			copyFilesLargerThan(dir, targetDir(sourceRoot, dir, targetRoot), minSize);
		}
	}
	
}

Show details Hide details

Change log

r5 by max.at.xam.de on Feb 11, 2012 Diff

Improve copying large files

Go to:

Project members, sign in to write a code review

Older revisions

r4 by max.at.xam.de on Oct 13, 2010 Diff

[No log message]

r3 by max.at.xam.de on Jun 28, 2010 Diff

Public release

All revisions of this file

File info

Size: 18980 bytes, 605 lines

View raw file