diff --git a/jsync-engine/src/main/java/com/fizzed/jsync/engine/DefaultJsyncEventHandler.java b/jsync-engine/src/main/java/com/fizzed/jsync/engine/DefaultJsyncEventHandler.java index 902e17d..2e9af15 100644 --- a/jsync-engine/src/main/java/com/fizzed/jsync/engine/DefaultJsyncEventHandler.java +++ b/jsync-engine/src/main/java/com/fizzed/jsync/engine/DefaultJsyncEventHandler.java @@ -25,8 +25,13 @@ public void willEnd(VirtualFileSystem sourceVfs, VirtualPath sourcePath, Virtual } @Override - public void willExcludePath(VirtualPath targetPath) { - log.debug("Excluding path {}", targetPath); + public void willExcludePath(VirtualPath sourcePath) { + log.debug("Excluding path {}", sourcePath); + } + + @Override + public void willIgnorePath(VirtualPath sourcePath) { + log.debug("Ignoring path {}", sourcePath); } @Override diff --git a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java index 029c9a1..b0a060d 100644 --- a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java +++ b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java @@ -1,6 +1,7 @@ package com.fizzed.jsync.engine; import com.fizzed.jsync.vfs.*; +import com.fizzed.jsync.vfs.util.Permissions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -8,11 +9,9 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; +import java.util.*; +import static com.fizzed.jsync.vfs.util.Permissions.isOwnerPermissionEqual; import static java.util.Arrays.asList; import static java.util.stream.Collectors.toList; @@ -26,8 +25,15 @@ public class JsyncEngine { private boolean force; private boolean parents; private boolean ignoreTimes; + private boolean skipPermissions; private int maxFilesMaybeModifiedLimit; private List excludes; + private List ignores; + // when running a sync + private Checksum negotiatedChecksum; + private List excludePaths; + private List ignoreSourcePaths; + private List ignoreTargetPaths; public JsyncEngine() { this.eventHandler = new DefaultJsyncEventHandler(); @@ -35,9 +41,9 @@ public JsyncEngine() { this.force = false; this.parents = false; this.ignoreTimes = false; + this.skipPermissions = false; this.preferredChecksums = new ArrayList<>(asList(Checksum.CK, Checksum.MD5)); this.maxFilesMaybeModifiedLimit = 256; - this.excludes = null; } public JsyncEventHandler getEventHandler() { @@ -86,6 +92,15 @@ public JsyncEngine setIgnoreTimes(boolean ignoreTimes) { return this; } + public boolean isSkipPermissions() { + return skipPermissions; + } + + public JsyncEngine setSkipPermissions(boolean skipPermissions) { + this.skipPermissions = skipPermissions; + return this; + } + public List getPreferredChecksums() { return this.preferredChecksums; } @@ -122,6 +137,23 @@ public JsyncEngine addExclude(String exclude) { return this; } + public List getIgnores() { + return ignores; + } + + public JsyncEngine setIgnores(List ignores) { + this.ignores = ignores; + return this; + } + + public JsyncEngine addIgnore(String ignore) { + if (this.ignores == null) { + this.ignores = new ArrayList<>(); + } + this.ignores.add(ignore); + return this; + } + public JsyncResult sync(Path sourcePath, Path targetPath, JsyncMode mode) throws IOException { // local -> local final LocalVirtualFileSystem localVfs = LocalVirtualFileSystem.open(); @@ -198,8 +230,32 @@ public JsyncResult sync(VirtualFileSystem sourceVfs, String sourcePath, VirtualF // Negotiate checksum methods between source and target filesystems if necessary // - // find the best common checksum to use - final Checksum checksum = this.negotiateChecksum(sourceVfs, targetVfs); + // find the best common checksum + this.negotiatedChecksum = this.negotiateChecksum(sourceVfs, targetVfs); + + // build exclude and ignore paths + if (this.excludes != null) { + this.excludePaths = this.excludes.stream() + .map(VirtualPath::parse) + .map(sourcePathAbsFinal::resolve) + .collect(toList()); + } else { + this.excludePaths = Collections.emptyList(); + } + + if (this.ignores != null) { + this.ignoreSourcePaths = this.ignores.stream() + .map(VirtualPath::parse) + .map(sourcePathAbsFinal::resolve) + .collect(toList()); + this.ignoreTargetPaths = this.ignores.stream() + .map(VirtualPath::parse) + .map(targetPathAbsFinal::resolve) + .collect(toList()); + } else { + this.ignoreSourcePaths = Collections.emptyList(); + this.ignoreTargetPaths = Collections.emptyList(); + } final long now = System.currentTimeMillis(); @@ -227,11 +283,11 @@ public JsyncResult sync(VirtualFileSystem sourceVfs, String sourcePath, VirtualF // as we process files, only a subset may require more advanced methods of detecting whether they were modified // since that process could be "expensive", we keep a list of files on source/target that we will defer processing // until we have a chance to do some bulk processing of checksums, etc. - this.syncDirectory(0, result, excludePaths, deferredFiles, sourceVfs, sourcePathAbsFinal, targetVfs, targetPathAbsFinal, checksum); + this.syncDirectory(0, result, deferredFiles, sourceVfs, sourcePathAbsFinal, targetVfs, targetPathAbsFinal); } else { // we are only syncing a file, we may need to do some more expensive checks to determine if it needs to be updated this.syncFile(result, deferredFiles, sourceVfs, sourcePathAbsFinal, targetVfs, targetPathAbsFinal); - this.syncDeferredFiles(result, deferredFiles, sourceVfs, targetVfs, checksum); + this.syncDeferredFiles(result, deferredFiles, sourceVfs, targetVfs); } final long timeMillis = System.currentTimeMillis() - now; @@ -263,7 +319,7 @@ protected void syncFile(JsyncResult result, List deferredFiles, } // detect what changes exists between source & target paths - final JsyncPathChanges changes = this.detectChanges(sourcePath, targetPath); + final JsyncPathChanges changes = this.detectChanges(sourceVfs, sourcePath, targetVfs, targetPath); // first, check if we should defer syncing the file till later on if (deferredFiles != null && changes.isDeferredProcessing(this.ignoreTimes)) { @@ -282,23 +338,23 @@ protected void syncFile(JsyncResult result, List deferredFiles, if (changes.isStatModified()) { // stat will need updated if the file is either new, updated, or if only the perms/times need updating - this.updateStat(result, sourcePath, targetVfs, targetPath, changes, fileWasTransferred); + this.updateStat(result, sourceVfs, sourcePath, targetVfs, targetPath, changes, fileWasTransferred); } } - protected void syncDeferredFiles(JsyncResult result, List deferredFiles, VirtualFileSystem sourceVfs, VirtualFileSystem targetVfs, Checksum checksum) throws IOException { + protected void syncDeferredFiles(JsyncResult result, List deferredFiles, VirtualFileSystem sourceVfs, VirtualFileSystem targetVfs) throws IOException { // we need to calculate checksums for source and target files final List sourceFiles = deferredFiles.stream() .map(VirtualPathPair::getSource) .collect(toList()); - sourceVfs.checksums(checksum, sourceFiles); + sourceVfs.checksums(this.negotiatedChecksum, sourceFiles); final List targetFiles = deferredFiles.stream() .map(VirtualPathPair::getTarget) .collect(toList()); - targetVfs.checksums(checksum, targetFiles); + targetVfs.checksums(this.negotiatedChecksum, targetFiles); result.incrementChecksums(targetFiles.size()); @@ -310,7 +366,7 @@ protected void syncDeferredFiles(JsyncResult result, List defer deferredFiles.clear(); } - protected void syncDirectory(int level, JsyncResult result, List excludePaths, List deferredFiles, VirtualFileSystem sourceVfs, VirtualPath sourcePath, VirtualFileSystem targetVfs, VirtualPath targetPath, Checksum checksum) throws IOException { + protected void syncDirectory(int level, JsyncResult result, List deferredFiles, VirtualFileSystem sourceVfs, VirtualPath sourcePath, VirtualFileSystem targetVfs, VirtualPath targetPath) throws IOException { // source needs to be a directory if (!sourcePath.isDirectory()) { @@ -322,7 +378,8 @@ protected void syncDirectory(int level, JsyncResult result, List ex log.warn("Type mismatch: source {} is a directory but target '{}' is a file!", sourcePath, targetPath); if (!this.force) { - throw new PathOverwriteException("Type mismatch: source " + sourcePath + " is a directory but target " + targetPath + " is a file. Either delete the target file manually or use the 'force' option to have jsync do it for you."); + throw new PathOverwriteException("Type mismatch: source " + sourcePath + " is a directory but target " + targetPath + " is a file. " + + "Either delete the target file manually or use the 'force' option to have jsync do it for you."); } // delete the target file @@ -335,7 +392,7 @@ protected void syncDirectory(int level, JsyncResult result, List ex } // detect what changes exists between source & target paths - final JsyncPathChanges changes = this.detectChanges(sourcePath, targetPath); + final JsyncPathChanges changes = this.detectChanges(sourceVfs, sourcePath, targetVfs, targetPath); if (changes.isMissing()) { this.createDirectory(result, targetVfs, targetPath, false, false); @@ -352,15 +409,23 @@ protected void syncDirectory(int level, JsyncResult result, List ex List sourceChildPaths = sourceVfs.ls(sourcePath).stream() // apply filter to source files if they are on the exclude list .filter(v -> { - //log.info("Checking for exlcude of path {} with excludes {}", v, excludePaths); - for (VirtualPath excludePath : excludePaths) { - if (v.startsWith(excludePath)) { + for (VirtualPath p : this.excludePaths) { + if (v.startsWith(p)) { this.eventHandler.willExcludePath(v); return false; } } return true; }) + .filter(v -> { + for (VirtualPath p : this.ignoreSourcePaths) { + if (v.startsWith(p)) { + this.eventHandler.willIgnorePath(v); + return false; + } + } + return true; + }) // apply filter to excluding non-regular files (such as symlinks) .filter(v -> { switch (v.getStat().getType()) { @@ -376,7 +441,16 @@ protected void syncDirectory(int level, JsyncResult result, List ex }) .collect(toList()); - final List targetChildPaths = targetVfs.ls(targetPath); + final List targetChildPaths = targetVfs.ls(targetPath).stream() + .filter(v -> { + for (VirtualPath p : this.ignoreTargetPaths) { + if (v.startsWith(p)) { + return false; + } + } + return true; + }) + .collect(toList()); // its better to work with all dirs first, then files, so we sort the files before we process them this.sortPaths(sourceChildPaths); @@ -388,7 +462,7 @@ protected void syncDirectory(int level, JsyncResult result, List ex // find a matching target path entirely by name VirtualPath targetChildPath = targetChildPaths.stream() - .filter(p -> targetVfs.areFileNamesEqual(p.getName(), sourceChildPath.getName())) + .filter(p -> targetVfs.isFileNameEqual(p.getName(), sourceChildPath.getName())) .findFirst() .orElse(null); @@ -398,7 +472,7 @@ protected void syncDirectory(int level, JsyncResult result, List ex } if (sourceChildPath.isDirectory()) { - this.syncDirectory(level+1, result, excludePaths, deferredFiles, sourceVfs, sourceChildPath, targetVfs, targetChildPath, checksum); + this.syncDirectory(level+1, result, deferredFiles, sourceVfs, sourceChildPath, targetVfs, targetChildPath); } else { // NOTE: it's possible syncFile will "defer" processing if a checksum is required this.syncFile(result, deferredFiles, sourceVfs, sourceChildPath, targetVfs, targetChildPath); @@ -407,7 +481,7 @@ protected void syncDirectory(int level, JsyncResult result, List ex // handle any deferred files that need to be processed if (level == 0 || deferredFiles.size() >= this.maxFilesMaybeModifiedLimit) { - this.syncDeferredFiles(result, deferredFiles, sourceVfs, targetVfs, checksum); + this.syncDeferredFiles(result, deferredFiles, sourceVfs, targetVfs); } // handle any paths that need to be deleted @@ -415,7 +489,7 @@ protected void syncDirectory(int level, JsyncResult result, List ex for (VirtualPath targetChildPath : targetChildPaths) { // find a matching source path entirely by name final VirtualPath sourceChildPath = sourceChildPaths.stream() - .filter(p -> sourceVfs.areFileNamesEqual(p.getName(), targetChildPath.getName())) + .filter(p -> sourceVfs.isFileNameEqual(p.getName(), targetChildPath.getName())) .findFirst() .orElse(null); @@ -436,11 +510,11 @@ protected void syncDirectory(int level, JsyncResult result, List ex // To successfully preserve directory timestamps, you must set the directory attributes after you have finished touching every single file inside that directory. if (changes.isStatModified()) { // stat will need updated if the dir is new OR if the dir stats have changed - this.updateStat(result, sourcePath, targetVfs, targetPath, changes, changes.isMissing()); + this.updateStat(result, sourceVfs, sourcePath, targetVfs, targetPath, changes, changes.isMissing()); } } - protected JsyncPathChanges detectChanges(VirtualPath sourcePath, VirtualPath targetPath) throws IOException { + protected JsyncPathChanges detectChanges(VirtualFileSystem sourceVfs, VirtualPath sourcePath, VirtualFileSystem targetVfs, VirtualPath targetPath) throws IOException { // source "stats" MUST exist Objects.requireNonNull(sourcePath, "sourceFile cannot be null"); @@ -479,6 +553,19 @@ protected JsyncPathChanges detectChanges(VirtualPath sourcePath, VirtualPath tar timestamps = true; } + if (!this.skipPermissions) { + // we only support syncing changes to POSIX targets, based on testing BASIC targets can break easily + // if posix -> posix we will compare entire permission value + // if basic -> posix we will only compare the owner permission bits + if (targetVfs.getStatModel() == StatModel.POSIX) { + if ((sourceVfs.getStatModel() == StatModel.POSIX && sourcePath.getStat().getPermissions() != targetPath.getStat().getPermissions()) + || (sourceVfs.getStatModel() == StatModel.BASIC && !isOwnerPermissionEqual(sourcePath.getStat().getPermissions(), targetPath.getStat().getPermissions()))) { + log.trace("Source path {} perms {} != target perms {}", sourcePath, sourcePath.getStat().getPermissions(), targetPath.getStat().getPermissions()); + permissions = true; + } + } + } + // if we have "cksum" values on both sides, we can compare those if (sourcePath.getStat().getCksum() != null && targetPath.getStat().getCksum() != null) { if (!sourcePath.getStat().getCksum().equals(targetPath.getStat().getCksum())) { @@ -509,7 +596,7 @@ protected JsyncPathChanges detectChanges(VirtualPath sourcePath, VirtualPath tar } } - return new JsyncPathChanges(sourcePath.isDirectory(), false, size, timestamps, ownership, permissions, checksums); + return new JsyncPathChanges(sourcePath.isDirectory(), false, size, timestamps, permissions, ownership, checksums); } protected void transferFile(JsyncResult result, VirtualFileSystem sourceVfs, VirtualPath sourceFile, VirtualFileSystem targetVfs, VirtualPath targetFile, JsyncPathChanges changes) throws IOException { @@ -532,12 +619,42 @@ protected void transferFile(JsyncResult result, VirtualFileSystem sourceVfs, Vir } } - protected void updateStat(JsyncResult result, VirtualPath sourcePath, VirtualFileSystem targetVfs, VirtualPath targetPath, JsyncPathChanges changes, boolean associatedWithFileModifiedOrDirCreated) throws IOException { + protected void updateStat(JsyncResult result, VirtualFileSystem sourceVfs, VirtualPath sourcePath, + VirtualFileSystem targetVfs, VirtualPath targetPath, JsyncPathChanges changes, boolean associatedWithFileModifiedOrDirCreated) throws IOException { + this.eventHandler.willUpdateStat(sourcePath, targetPath, changes, associatedWithFileModifiedOrDirCreated); - targetVfs.updateStat(targetPath, sourcePath.getStat()); + final Set options = EnumSet.noneOf(StatUpdateOption.class); + // in posix -> posix, we can use the stat of the source, but if we're changing permissions and a BASIC vfs + // is involved, we only want to try and change the "owner" permission, and leave everything else as-is + VirtualFileStat updateStat = sourcePath.getStat(); + + if (changes.isPermissionModified()) { + options.add(StatUpdateOption.PERMISSIONS); + + // if the source of perms is BASIC and the target currently has perms, we will only want to change the + // owner bits and retain the targets group & world bits + if (sourceVfs.getStatModel() == StatModel.BASIC && targetPath.getStat() != null) { + // TODO: simplify this + log.info("current target perms: {}", targetPath.getStat().getPermissionsOctal()); + int targetPerms = targetPath.getStat().getPermissions(); + int newTargetPerms = Permissions.mergeOwnerPermissions(sourcePath.getStat().getPermissions(), targetPerms); + updateStat = updateStat.withPermissions(newTargetPerms); + } + } - result.incrementStatsUpdated(); + if (changes.isTimestampsModified()) { + options.add(StatUpdateOption.TIMESTAMPS); + } + +// log.debug("Updating stats with options {} (perms {})", options, updateStat.getPermissionsOctal()); + + if (!options.isEmpty()) { + targetVfs.updateStat(targetPath, updateStat, options); + result.incrementStatsUpdated(); + } else { + log.warn("updateStat was called, but nothing to update (options empty)"); + } } protected void createDirectory(JsyncResult result, VirtualFileSystem vfs, VirtualPath path, boolean verifyParentExists, boolean parents) throws IOException { @@ -610,41 +727,35 @@ protected Checksum negotiateChecksum(VirtualFileSystem sourceVfs, VirtualFileSys log.debug("Negotiating checksums supported on both source and target filesystems..."); // negotiate the checksums to use - Checksum checksum = null; - List sourceChecksumsSupported = new ArrayList<>(); - List targetChecksumsSupported = new ArrayList<>(); + final Set sourceChecksums = sourceVfs.getChecksumsSupported(); + final Set targetChecksums = targetVfs.getChecksumsSupported(); - for (Checksum preferredChecksum : this.preferredChecksums) { - log.debug("Detecting if {} checksum is supported on source/target", preferredChecksum); - - // check supported checksums, keep a tally of which are supported by both sides, so we can log them out - boolean sourceSupported = sourceVfs.isSupported(preferredChecksum); + log.debug("Source filesystem supports checksums {}", sourceChecksums); + log.debug("Target filesystem supports checksums {}", targetChecksums); - if (sourceSupported) { - sourceChecksumsSupported.add(preferredChecksum); - } - - boolean targetSupported = targetVfs.isSupported(preferredChecksum); - - if (targetSupported) { - targetChecksumsSupported.add(preferredChecksum); + for (Checksum preferredChecksum : this.preferredChecksums) { + if (sourceChecksums.contains(preferredChecksum) && targetChecksums.contains(preferredChecksum)) { + return preferredChecksum; } + } - log.debug("Detected {} checksum supported on source={}, target={}", preferredChecksum, sourceSupported, targetSupported); + throw new IOException("Unable to find a checksum that is supported by both source and target filesystems. " + + "Source filesystem " + sourceVfs.getName() + " supports checksums " + sourceChecksums + + " and target filesystem " + targetVfs.getName() + " supports checksums " + targetChecksums); + } - if (sourceSupported && targetSupported) { - checksum = preferredChecksum; - break; - } + protected StatModel negotiateStatModel(VirtualFileSystem sourceVfs, VirtualFileSystem targetVfs) throws IOException { + // do both support posix? + if (sourceVfs.getStatModel() == StatModel.POSIX && targetVfs.getStatModel() == StatModel.POSIX) { + return StatModel.POSIX; } - if (checksum == null) { - throw new IOException("Unable to find a checksum that is supported by both source and target filesystems. " + - "Source filesystem " + sourceVfs.getName() + " supports checksums " + sourceChecksumsSupported - + " and target filesystem " + targetVfs.getName() + " supports checksums " + targetChecksumsSupported); + // do either support basic? + if (sourceVfs.getStatModel() == StatModel.BASIC || targetVfs.getStatModel() == StatModel.BASIC) { + return StatModel.BASIC; } - return checksum; + throw new IOException("Unable to find a stat model that is supported by source and target filesystems"); } protected void sortPaths(List paths) { diff --git a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEventHandler.java b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEventHandler.java index 30b7da2..d9abbe5 100644 --- a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEventHandler.java +++ b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEventHandler.java @@ -15,6 +15,8 @@ public interface JsyncEventHandler { void willExcludePath(VirtualPath targetPath); + void willIgnorePath(VirtualPath targetPath); + void willCreateDirectory(VirtualPath targetPath, boolean recursively); void willDeleteDirectory(VirtualPath targetPath, boolean recursively); diff --git a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncPathChanges.java b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncPathChanges.java index 2361905..96e77c5 100644 --- a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncPathChanges.java +++ b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncPathChanges.java @@ -79,6 +79,16 @@ public boolean isDeferredProcessing(boolean ignoreTimes) { && checksum == null; } + public boolean isPermissionModified() { + return this.missing + || this.permissions; + } + + public boolean isTimestampsModified() { + return this.missing + || this.timestamps; + } + public boolean isStatModified() { return this.missing || this.timestamps diff --git a/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncDemo.java b/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncDemo.java index 597c184..27c7582 100644 --- a/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncDemo.java +++ b/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncDemo.java @@ -17,7 +17,7 @@ static public void main(String[] args) throws Exception { // final String sourceDir = Paths.get("/home/jjlauer/test-sync").toString(); // final VirtualVolume source = localVolume(Paths.get("/home/jjlauer/workspace/third-party/jsch")); // final String sourceDir = Paths.get("/home/jjlauer/workspace/third-party/coredns").toString(); - final VirtualVolume source = localVolume(Paths.get("/home/jjlauer/workspace/third-party/nats.java")); + final VirtualVolume source = localVolume(Paths.get( System.getProperty("user.home") + "/workspace/third-party/nats.java")); // final VirtualVolume source = localVolume(Paths.get("/home/jjlauer/workspace/third-party/tokyocabinet-1.4.48")); // final String sourceDir = Paths.get("C:\\Users\\jjlauer\\test-sync").toString(); // final String sourceDir = Paths.get("C:\\Users\\jjlauer\\workspace\\third-party\\tokyocabinet-1.4.48").toString(); @@ -28,8 +28,8 @@ static public void main(String[] args) throws Exception { final String targetDir = "test-sync"; // final VirtualVolume target = sftpVolume("bmh-build-x64-win11-1", targetDir); - final VirtualVolume target = sftpVolume("bmh-dev-x64-indy25-1", targetDir); -// final VirtualVolume target = sftpVolume("bmh-dev-x64-fedora43-1", targetDir); +// final VirtualVolume target = sftpVolume("bmh-dev-x64-indy25-1", targetDir); + final VirtualVolume target = sftpVolume("bmh-dev-x64-fedora43-1", targetDir); // final VirtualVolume target = sftpVolume("bmh-build-x64-freebsd15-1", targetDir); final JsyncResult result = new JsyncEngine() @@ -39,6 +39,7 @@ static public void main(String[] args) throws Exception { .setDelete(true) .setParents(true) .setForce(true) + //.setSkipPermissions(true) //.setIgnoreTimes(true) //.setMaxFilesMaybeModifiedLimit(256) // .setProgress(true) diff --git a/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncEngineTest.java b/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncEngineTest.java index 6a837ea..d96e559 100644 --- a/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncEngineTest.java +++ b/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncEngineTest.java @@ -4,6 +4,7 @@ import com.fizzed.crux.util.Resources; import com.fizzed.jsync.vfs.ParentDirectoryMissingException; import com.fizzed.jsync.vfs.PathOverwriteException; +import com.fizzed.jsync.vfs.util.Permissions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.Logger; @@ -19,6 +20,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.within; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assumptions.assumeTrue; class JsyncEngineTest { static private final Logger log = LoggerFactory.getLogger(JsyncEngineTest.class); @@ -502,6 +504,53 @@ public void syncDirTimestamp() throws Exception { assertThat(modifiedTime(targetADir)).isCloseTo(ts, within(2, ChronoUnit.SECONDS)); } + @Test + public void syncFilePermission() throws Exception { + assumeTrue(Permissions.isPosix()); + + Path sourceAFile = this.syncSourceDir.resolve("a.txt"); + Files.write(sourceAFile, "hello".getBytes()); + + Path targetAFile = this.syncTargetDir.resolve("a.txt"); + Files.write(targetAFile, "hello".getBytes()); + + Permissions.setPosixInt(sourceAFile, 0755); + Permissions.setPosixInt(targetAFile, 0644); + + JsyncResult result = new JsyncEngine() + .sync(sourceAFile, this.syncTargetDir, JsyncMode.NEST); + + assertThat(Permissions.getPosixInt(targetAFile)).isEqualTo(0755); + assertThat(result.getStatsUpdated()).isEqualTo(1); + assertThat(result.getFilesCreated()).isEqualTo(0); + assertThat(result.getFilesDeleted()).isEqualTo(0); + assertThat(result.getFilesUpdated()).isEqualTo(0); + } + + @Test + public void syncFileSkipPermissions() throws Exception { + assumeTrue(Permissions.isPosix()); + + Path sourceAFile = this.syncSourceDir.resolve("a.txt"); + Files.write(sourceAFile, "hello".getBytes()); + + Path targetAFile = this.syncTargetDir.resolve("a.txt"); + Files.write(targetAFile, "hello".getBytes()); + + Permissions.setPosixInt(sourceAFile, 0755); + Permissions.setPosixInt(targetAFile, 0644); + + JsyncResult result = new JsyncEngine() + .setSkipPermissions(true) + .sync(sourceAFile, this.syncTargetDir, JsyncMode.NEST); + + assertThat(Permissions.getPosixInt(targetAFile)).isEqualTo(0644); + assertThat(result.getStatsUpdated()).isEqualTo(0); + assertThat(result.getFilesCreated()).isEqualTo(0); + assertThat(result.getFilesDeleted()).isEqualTo(0); + assertThat(result.getFilesUpdated()).isEqualTo(0); + } + @Test public void syncExcludeDir() throws Exception { Path sourceADir = this.syncSourceDir.resolve("a"); @@ -547,4 +596,37 @@ public void syncExcludeNonRegularFiles() throws Exception { assertThat(this.syncTargetDir.resolve("a/b.txt")).hasContent("hello"); } + @Test + public void syncIgnoreDir() throws Exception { + Path sourceADir = this.syncSourceDir.resolve("a"); + Path sourceBFile = this.syncSourceDir.resolve("a/b.txt"); + this.writeFile(sourceBFile, "hello"); + + Path sourceBDir = this.syncSourceDir.resolve("b"); + Path sourceCFile = this.syncSourceDir.resolve("b/c.txt"); + Path sourceDFile = this.syncSourceDir.resolve("b/sub/sub2/d.txt"); + this.writeFile(sourceCFile, "hello"); + this.writeFile(sourceDFile, "hello"); + + // with directory "a" fully excluded, it actually should be deleted with --exclude + Path targetADir = this.syncTargetDir.resolve("a"); + Path targetBFile = this.syncTargetDir.resolve("a/b.txt"); + this.writeFile(targetBFile, "hello"); + Path targetBDir = this.syncTargetDir.resolve("b"); + Path targetCDir = this.syncTargetDir.resolve("c"); + Path targetEFile = this.syncTargetDir.resolve("c/sub/sub2/e.txt"); + this.writeFile(targetEFile, "hello"); + + final JsyncResult result = new JsyncEngine() + .addIgnore("c") // target dir should not be deleted + .addIgnore("b") // b should not be synced to target + .setDelete(true) + .sync(this.syncSourceDir, this.syncTargetDir, JsyncMode.MERGE); + + assertThat(targetADir).isDirectory(); + assertThat(targetBDir).doesNotExist(); + assertThat(targetCDir).exists(); + assertThat(targetEFile).isRegularFile(); + } + } \ No newline at end of file diff --git a/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java b/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java index 6576657..b7a5335 100644 --- a/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java +++ b/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java @@ -17,6 +17,8 @@ import java.util.*; import java.util.concurrent.CountDownLatch; +import static java.util.Arrays.asList; + public class SftpVirtualFileSystem extends AbstractVirtualFileSystem { static private final Logger log = LoggerFactory.getLogger(SftpVirtualFileSystem.class); @@ -109,27 +111,27 @@ static public SftpVirtualFileSystem open(Session ssh, boolean closeSsh) throws I static public SftpVirtualFileSystem open(Session ssh, boolean closeSsh, ChannelSftp sftp, boolean closeSftp) throws IOException { final String name = ssh.getHost(); - log.info("Opening filesystem {}...", name); + log.debug("Opening filesystem {}...", name); - final String pwd2; + final String pwdRaw; try { - pwd2 = sftp.pwd(); + pwdRaw = sftp.pwd(); } catch (SftpException e) { throw toIOException(e); } - final VirtualPath pwd = VirtualPath.parse(pwd2, true); + final VirtualPath pwd = VirtualPath.parse(pwdRaw, true); - log.debug("Detected filesystem {} has pwd {}", name, pwd); + log.debug("Detected pwd {}", pwd); boolean windows = false; // this is likely a "windows" system if the 2nd char is : - if (pwd2.length() > 2 && pwd2.charAt(2) == ':') { + if (pwdRaw.length() > 2 && pwdRaw.charAt(2) == ':') { // TODO: should we confirm by running a command that exists only windows to confirm? // for now we'll just assume it is windows = true; - log.debug("Detected filesystem {} is running on windows (changes standard checksums, native filepaths, case sensitivity, etc.)", name); + log.debug("Detected windows-based sftp server"); } return new SftpVirtualFileSystem(name, pwd, ssh, closeSsh, sftp, closeSftp, windows); @@ -154,12 +156,22 @@ public void close() throws Exception { } @Override - public String toString() { - return this.getName(); + public boolean isRemote() { + return true; + } + + @Override + public StatModel getStatModel() { + // for now, we'll claim full POSIX as the sftp server itself does the POSIX translation + if (this.windows) { + return StatModel.BASIC; + } else { + return StatModel.POSIX; + } } @Override - public boolean isSupported(Checksum checksum) throws IOException { + public boolean isChecksumSupported(Checksum checksum) throws IOException { if (this.windows) { switch (checksum) { case MD5: @@ -189,6 +201,37 @@ public boolean isSupported(Checksum checksum) throws IOException { } } + @Override + protected List doDetectChecksums() throws IOException { + // windows is easy, return what powershell supports + if (this.windows) { + return asList(Checksum.MD5, Checksum.SHA1); + } + + // otherwise, we are on posix and we can actually check whether these would work or not + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + // check if anything is supported + int exitValue = this.exec(this.ssh, "which cksum md5sum sha1sum", null, baos, null); + if (exitValue != 0) { + return Collections.emptyList(); + } + + final String output = baos.toString(StandardCharsets.UTF_8.name()); + final List checksums = new ArrayList<>(); + if (output.contains("cksum")) { + checksums.add(Checksum.CK); + } + if (output.contains("md5sum")) { + checksums.add(Checksum.MD5); + } + if (output.contains("sha1sum")) { + checksums.add(Checksum.SHA1); + } + + return checksums; + } + public int getMaxCommandLength() { return maxCommandLength; } @@ -198,10 +241,12 @@ public SftpVirtualFileSystem setMaxCommandLength(int maxCommandLength) { return this; } - private VirtualPath toVirtualPathWithStats(VirtualPath path, SftpATTRS attrs) throws IOException { + protected VirtualPath withStats(VirtualPath path, SftpATTRS attrs) throws IOException { final long size = attrs.getSize(); final long modifiedTime = attrs.getMTime() * 1000L; final long accessedTime = attrs.getATime() * 1000L; + // sftp stuffs extra stuff like the file type in the permissions value, we don't care about it + final int perms = attrs.getPermissions() & 07777; final VirtualFileType type; if (attrs.isDir()) { @@ -214,44 +259,40 @@ private VirtualPath toVirtualPathWithStats(VirtualPath path, SftpATTRS attrs) th type = VirtualFileType.OTHER; } - final VirtualFileStat stat = new VirtualFileStat(type, size, modifiedTime, accessedTime); + final VirtualFileStat stat = new VirtualFileStat(type, size, modifiedTime, accessedTime, perms); return new VirtualPath(path.getParentPath(), path.getName(), type == VirtualFileType.DIR, stat); } - @Override - public boolean isRemote() { - return true; - } - @Override public VirtualPath stat(VirtualPath path) throws IOException { try { final SftpATTRS attrs = this.sftp.lstat(path.toString()); - return this.toVirtualPathWithStats(path, attrs); + return this.withStats(path, attrs); } catch (SftpException e) { throw toIOException(e); } } @Override - public void updateStat(VirtualPath path, VirtualFileStat stat) throws IOException { + public void updateStat(VirtualPath path, VirtualFileStat stat, Collection options) throws IOException { try { + final SftpATTRS attrs = SftpATTRSAccessor.createSftpATTRS(); + // TODO: are we updating uid/gid? Integer uid = null; Integer gid = null; - // TODO: are we updating permissions? - Integer perms = null; - - // are we updating mtime/atime?d - int mtime = (int)(stat.getModifiedTime()/1000); - int atime = (int)(stat.getAccessedTime()/1000); - - final SftpATTRS attrs = SftpATTRSAccessor.createSftpATTRS(); + if (options.contains(StatUpdateOption.PERMISSIONS)) { + attrs.setPERMISSIONS(stat.getPermissions()); + } - attrs.setACMODTIME(atime, mtime); + if (options.contains(StatUpdateOption.TIMESTAMPS)) { + int mtime = (int) (stat.getModifiedTime() / 1000); + int atime = (int) (stat.getAccessedTime() / 1000); + attrs.setACMODTIME(atime, mtime); + } this.sftp.setStat(path.toString(), attrs); } catch (SftpException e) { @@ -277,7 +318,7 @@ public List ls(VirtualPath path) throws IOException { // dir true/false doesn't matter, stats call next will correct it VirtualPath childPathWithoutStats = path.resolve(entry.getFilename(), false); - VirtualPath childPath = this.toVirtualPathWithStats(childPathWithoutStats, entry.getAttrs()); + VirtualPath childPath = this.withStats(childPathWithoutStats, entry.getAttrs()); childPaths.add(childPath); } diff --git a/jsync-sftp/src/test/java/com/fizzed/jsync/sftp/SftpVirtualFileSystemDemo.java b/jsync-sftp/src/test/java/com/fizzed/jsync/sftp/SftpVirtualFileSystemDemo.java new file mode 100644 index 0000000..e80e445 --- /dev/null +++ b/jsync-sftp/src/test/java/com/fizzed/jsync/sftp/SftpVirtualFileSystemDemo.java @@ -0,0 +1,27 @@ +package com.fizzed.jsync.sftp; + +import com.fizzed.jsync.vfs.VirtualPath; +import com.fizzed.jsync.vfs.VirtualVolume; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Paths; + +import static com.fizzed.jsync.sftp.SftpVirtualVolume.sftpVolume; +import static com.fizzed.jsync.vfs.LocalVirtualVolume.localVolume; + +public class SftpVirtualFileSystemDemo { + static private final Logger log = LoggerFactory.getLogger(SftpVirtualFileSystemDemo.class); + + static public void main(String[] args) throws Exception { + +// final SftpVirtualFileSystem vfs = SftpVirtualFileSystem.open("bmh-dev-x64-indy25-1"); + final SftpVirtualFileSystem vfs = SftpVirtualFileSystem.open("bmh-build-x64-win11-1"); + +// final VirtualPath stat = vfs.stat(VirtualPath.parse(".ssh/authorized_keys")); + final VirtualPath stat = vfs.stat(VirtualPath.parse("remote-build")); + + log.info("file: {}, perms={}", stat, stat.getStat().getPermissionsOctal()); + } + +} \ No newline at end of file diff --git a/jsync-sftp/src/test/java/com/fizzed/jsync/sftp/SftpVirtualFileSystemTest.java b/jsync-sftp/src/test/java/com/fizzed/jsync/sftp/SftpVirtualFileSystemTest.java index d9be42b..65d8803 100644 --- a/jsync-sftp/src/test/java/com/fizzed/jsync/sftp/SftpVirtualFileSystemTest.java +++ b/jsync-sftp/src/test/java/com/fizzed/jsync/sftp/SftpVirtualFileSystemTest.java @@ -135,6 +135,11 @@ public void stat() throws Exception { assertThat(vpWithStat.getStat().getType()).isEqualTo(VirtualFileType.FILE); assertThat(vpWithStat.getStat().getSize()).isEqualTo(Files.size(sftpRootDir.resolve("root.txt"))); assertThat(vpWithStat.getStat().getModifiedTime()).isCloseTo(Files.getLastModifiedTime(sftpRootDir.resolve("root.txt")).toMillis(), within(1500L)); + + // permissions + assertThat(vpWithStat.getStat().getPermissions()).isGreaterThan(0); + // if we got rid of the extra stuff stacked by sftp, this value should be less than this + assertThat(vpWithStat.getStat().getPermissions()).isLessThan(4096); } @Test diff --git a/jsync-vfs/pom.xml b/jsync-vfs/pom.xml index adeeff4..afbbec0 100644 --- a/jsync-vfs/pom.xml +++ b/jsync-vfs/pom.xml @@ -19,7 +19,13 @@ - + + + com.fizzed + crux-util + test + + org.junit.jupiter junit-jupiter diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/AbstractVirtualFileSystem.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/AbstractVirtualFileSystem.java index 8a9ebf8..7d2e37f 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/AbstractVirtualFileSystem.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/AbstractVirtualFileSystem.java @@ -1,15 +1,22 @@ package com.fizzed.jsync.vfs; +import java.io.IOException; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + abstract public class AbstractVirtualFileSystem implements VirtualFileSystem { protected final String name; protected final VirtualPath pwd; protected final boolean caseSensitive; + protected volatile Set checksums; public AbstractVirtualFileSystem(String name, VirtualPath pwd, boolean caseSensitive) { this.name = name; this.pwd = pwd; this.caseSensitive = caseSensitive; + this.checksums = null; } @Override @@ -27,6 +34,27 @@ public boolean isCaseSensitive() { return this.caseSensitive; } + @Override + public boolean isChecksumSupported(Checksum checksum) throws IOException { + return this.getChecksumsSupported().contains(checksum); + } + + @Override + public Set getChecksumsSupported() throws IOException { + if (this.checksums == null) { + // we need to calculate them + synchronized (this) { + if (this.checksums == null) { + final List values = this.doDetectChecksums(); + this.checksums = new LinkedHashSet<>(values); + } + } + } + return this.checksums; + } + + abstract protected List doDetectChecksums() throws IOException; + @Override public String toString() { return this.name; diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java index 4044ece..cad5320 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java @@ -1,6 +1,7 @@ package com.fizzed.jsync.vfs; import com.fizzed.jsync.vfs.util.Checksums; +import com.fizzed.jsync.vfs.util.Permissions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -8,19 +9,20 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.*; -import java.nio.file.attribute.BasicFileAttributeView; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.FileTime; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; +import java.nio.file.attribute.*; +import java.util.*; import java.util.stream.Stream; +import static java.util.Arrays.asList; + public class LocalVirtualFileSystem extends AbstractVirtualFileSystem { static private final Logger log = LoggerFactory.getLogger(LocalVirtualFileSystem.class); - public LocalVirtualFileSystem(String name, VirtualPath pwd, boolean caseSensitive) { + private final boolean posix; + + public LocalVirtualFileSystem(String name, VirtualPath pwd, boolean caseSensitive, boolean posix) { super(name, pwd, caseSensitive); + this.posix = posix; } static public LocalVirtualFileSystem open() { @@ -37,14 +39,11 @@ static public LocalVirtualFileSystem open(Path workingDir) { final VirtualPath pwd = VirtualPath.parse(currentWorkingDir.toString(), true); - log.debug("Detected filesystem {} has pwd {}", name, pwd); - - // everything is case-sensitive except windows - final boolean caseSensitive = !System.getProperty("os.name").toLowerCase().contains("windows"); + final boolean posix = Permissions.isPosix(); - log.debug("Detected filesystem {} is case-sensitive={}", name, caseSensitive); + log.debug("Detected filesystem {} with pwd={}, posix={}, caseSensitive={}", name, pwd, posix, posix); - return new LocalVirtualFileSystem(name, pwd, caseSensitive); + return new LocalVirtualFileSystem(name, pwd, posix, posix); } @Override @@ -52,18 +51,49 @@ public void close() throws Exception { // nothing to do } + public boolean isPosix() { + return this.posix; + } + + @Override + public boolean isRemote() { + return false; + } + + @Override + public StatModel getStatModel() { + if (this.posix) { + return StatModel.POSIX; + } else { + return StatModel.BASIC; + } + } + + @Override + protected List doDetectChecksums() throws IOException { + // everything is supported + return asList(Checksum.CK, Checksum.MD5, Checksum.SHA1); + } + protected Path toNativePath(VirtualPath path) { return Paths.get(path.toString()); } - protected VirtualPath toVirtualPathWithStat(VirtualPath path) throws IOException { + protected VirtualPath withStat(VirtualPath path) throws IOException { final Path nativePath = this.toNativePath(path); // Fetch all attributes in ONE operation (and don't follow symlinks, we need to know the type) - final BasicFileAttributes attrs = Files.readAttributes(nativePath, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS); - // TODO: if we're on posix, we can also do this - // fetches size, times, PLUS owner, group, and permissions - // PosixFileAttributes attrs = Files.readAttributes(path, PosixFileAttributes.class); + final PosixFileAttributes posixAttrs; + final BasicFileAttributes attrs; + if (this.posix) { + posixAttrs = Files.readAttributes(nativePath, PosixFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + attrs = posixAttrs; + } else { + posixAttrs = null; + attrs = Files.readAttributes(nativePath, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + } + + // basic attributes get us much of what we need final long size = attrs.size(); final long modifiedTime = attrs.lastModifiedTime().toMillis(); final long accessedTime = attrs.lastAccessTime().toMillis(); @@ -84,36 +114,59 @@ protected VirtualPath toVirtualPathWithStat(VirtualPath path) throws IOException type = VirtualFileType.OTHER; } - final VirtualFileStat stat = new VirtualFileStat(type, size, modifiedTime, accessedTime); + // permissions are a tad trickier if they aren't really supported + final int perms; + if (posixAttrs != null) { + perms = Permissions.toPosixInt(posixAttrs.permissions()); + } else { + // use basic permissions, usually ends up being 700 from what I can gather + final Set basicPermissions = Permissions.toBasicPermissions(nativePath); + perms = Permissions.toPosixInt(basicPermissions); + } - return new VirtualPath(path.getParentPath(), path.getName(), type == VirtualFileType.DIR, stat); - } + final VirtualFileStat stat = new VirtualFileStat(type, size, modifiedTime, accessedTime, perms); - @Override - public boolean isRemote() { - return false; + return new VirtualPath(path.getParentPath(), path.getName(), type == VirtualFileType.DIR, stat); } @Override public VirtualPath stat(VirtualPath path) throws IOException { - return this.toVirtualPathWithStat(path); + return this.withStat(path); } @Override - public void updateStat(VirtualPath path, VirtualFileStat stat) throws IOException { + public void updateStat(VirtualPath path, VirtualFileStat stat, Collection options) throws IOException { final Path nativePath = this.toNativePath(path); - // Get the "View" (This is a lightweight handle to the attributes) - BasicFileAttributeView view = Files.getFileAttributeView(nativePath, BasicFileAttributeView.class); + final PosixFileAttributeView posixView; + final BasicFileAttributeView view; + if (this.posix) { + posixView = Files.getFileAttributeView(nativePath, PosixFileAttributeView.class); + view = posixView; + } else { + posixView = null; + view = Files.getFileAttributeView(nativePath, BasicFileAttributeView.class); + } + + if (options.contains(StatUpdateOption.PERMISSIONS)) { + final Set posixFilePermissions = Permissions.toPosixFilePermissions(stat.getPermissions()); + if (posixView != null) { + posixView.setPermissions(posixFilePermissions); + } else { + Permissions.setBasicPermissions(nativePath, posixFilePermissions); + } + } - // 2. Prepare the times - FileTime newModifiedTime = FileTime.fromMillis(stat.getModifiedTime()); - FileTime newAccessedTime = FileTime.fromMillis(stat.getAccessedTime()); + if (options.contains(StatUpdateOption.TIMESTAMPS)) { + // 2. Prepare the times + FileTime newModifiedTime = FileTime.fromMillis(stat.getModifiedTime()); + FileTime newAccessedTime = FileTime.fromMillis(stat.getAccessedTime()); - // 3. Update all three in ONE operation - // Signature: setTimes(lastModified, lastAccess, createTime) - // Pass 'null' if you want to leave a specific timestamp unchanged. - view.setTimes(newModifiedTime, newAccessedTime, null); + // 3. Update all three in ONE operation + // Signature: setTimes(lastModified, lastAccess, createTime) + // Pass 'null' if you want to leave a specific timestamp unchanged. + view.setTimes(newModifiedTime, newAccessedTime, null); + } } @Override @@ -129,7 +182,7 @@ public List ls(VirtualPath path) throws IOException { // dir true/false doesn't matter, stats call next will correct it VirtualPath childPathWithoutStats = path.resolve(nativeChildPath.getFileName().toString(), false); - VirtualPath childPath = this.toVirtualPathWithStat(childPathWithoutStats); + VirtualPath childPath = this.withStat(childPathWithoutStats); childPaths.add(childPath); } } @@ -179,12 +232,6 @@ public OutputStream writeStream(VirtualPath path) throws IOException { return Files.newOutputStream(nativePath, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); } - @Override - public boolean isSupported(Checksum checksum) throws IOException { - // all are supported! - return true; - } - @Override public void cksums(List paths) throws IOException { for (VirtualPath path : paths) { diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatModel.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatModel.java new file mode 100644 index 0000000..9d68e6d --- /dev/null +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatModel.java @@ -0,0 +1,8 @@ +package com.fizzed.jsync.vfs; + +public enum StatModel { + + POSIX, + BASIC + +} \ No newline at end of file diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatUpdateOption.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatUpdateOption.java new file mode 100644 index 0000000..b34d986 --- /dev/null +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatUpdateOption.java @@ -0,0 +1,9 @@ +package com.fizzed.jsync.vfs; + +public enum StatUpdateOption { + + OWNERSHIP, + PERMISSIONS, + TIMESTAMPS; + +} \ No newline at end of file diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileStat.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileStat.java index 98d612b..70a75c8 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileStat.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileStat.java @@ -6,16 +6,22 @@ public class VirtualFileStat { final private long size; final private long modifiedTime; final private long accessedTime; + final private int permissions; // there are values that can be populated later as they are expensive operations private Long cksum; private String md5; private String sha1; - public VirtualFileStat(VirtualFileType type, long size, long modifiedTime, long accessedTime) { + public VirtualFileStat(VirtualFileType type, long size, long modifiedTime, long accessedTime, int permissions) { this.size = size; this.type = type; this.modifiedTime = modifiedTime; this.accessedTime = accessedTime; + this.permissions = permissions; + } + + public VirtualFileStat withPermissions(int permissions) { + return new VirtualFileStat(type, size, modifiedTime, accessedTime, permissions); } public VirtualFileType getType() { @@ -34,6 +40,14 @@ public long getAccessedTime() { return accessedTime; } + public int getPermissions() { + return this.permissions; + } + + public String getPermissionsOctal() { + return Integer.toOctalString(this.permissions); + } + public Long getCksum() { return cksum; } diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileSystem.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileSystem.java index ee4ab8f..1c0f583 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileSystem.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileSystem.java @@ -4,7 +4,9 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.NoSuchFileException; +import java.util.Collection; import java.util.List; +import java.util.Set; public interface VirtualFileSystem extends AutoCloseable { @@ -12,9 +14,17 @@ public interface VirtualFileSystem extends AutoCloseable { boolean isRemote(); + // features of this filesystem, which changes the behavior of the engine + boolean isCaseSensitive(); - default boolean areFileNamesEqual(String name1, String name2) { + StatModel getStatModel(); + + boolean isChecksumSupported(Checksum checksum) throws IOException; + + Set getChecksumsSupported() throws IOException; + + default boolean isFileNameEqual(String name1, String name2) { if (this.isCaseSensitive()) { return name1.equals(name2); } else { @@ -52,7 +62,7 @@ default VirtualPath exists(VirtualPath path) throws IOException { */ VirtualPath stat(VirtualPath path) throws IOException; - void updateStat(VirtualPath path, VirtualFileStat stats) throws IOException; + void updateStat(VirtualPath path, VirtualFileStat stats, Collection options) throws IOException; List ls(VirtualPath path) throws IOException; @@ -68,8 +78,6 @@ default VirtualPath exists(VirtualPath path) throws IOException { OutputStream writeStream(VirtualPath path) throws IOException; - boolean isSupported(Checksum checksum) throws IOException; - default void checksums(Checksum checksum, List paths) throws IOException { switch (checksum) { case CK: diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/IoHelper.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/IoHelper.java deleted file mode 100644 index f55a387..0000000 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/IoHelper.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.fizzed.jsync.vfs.util; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -public class IoHelper { - - static private final int BUFFER_SIZE = 8192; // 8KB - - static public long copy(InputStream input, OutputStream output) throws IOException { - long nread = 0L; - byte[] buf = new byte[BUFFER_SIZE]; - int n; - while ((n = input.read(buf)) > 0) { - output.write(buf, 0, n); - nread += n; - } - return nread; - } - -} \ No newline at end of file diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java new file mode 100644 index 0000000..a872e3a --- /dev/null +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java @@ -0,0 +1,168 @@ +package com.fizzed.jsync.vfs.util; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Set; + +public class Permissions { + + static public boolean isPosix() { + return FileSystems.getDefault() + .supportedFileAttributeViews() + .contains("posix"); + } + + static public int getPosixInt(Path path) throws IOException { + final Set permissions = Files.getPosixFilePermissions(path); + return toPosixInt(permissions); + } + + static public void setPosixInt(Path path, int perms) throws IOException { + final Set permissions = toPosixFilePermissions(perms); + Files.setPosixFilePermissions(path, permissions); + } + + /** + * Converts a set of {@link PosixFilePermission} to its corresponding POSIX integer representation. + * The resulting integer is a bitmask representing the file permission mode. + * + * @param permissions the set of {@link PosixFilePermission} to convert; if null, the result will be 0 + * @return an integer representing the POSIX file permission mode derived from the given set of permissions + */ + static public int toPosixInt(Set permissions) { + int mode = 0; + + // Null check safety + if (permissions == null) return mode; + + for (PosixFilePermission perm : permissions) { + switch (perm) { + case OWNER_READ: mode |= 0400; break; // 256 in decimal + case OWNER_WRITE: mode |= 0200; break; // 128 + case OWNER_EXECUTE: mode |= 0100; break; // 64 + + case GROUP_READ: mode |= 0040; break; // 32 + case GROUP_WRITE: mode |= 0020; break; // 16 + case GROUP_EXECUTE: mode |= 0010; break; // 8 + + case OTHERS_READ: mode |= 0004; break; // 4 + case OTHERS_WRITE: mode |= 0002; break; // 2 + case OTHERS_EXECUTE: mode |= 0001; break; // 1 + } + } + + return mode; + } + + /** + * Converts a POSIX file permission integer into a set of {@link PosixFilePermission}. + * The input integer is interpreted as the POSIX file permission bitmask, where each bit + * corresponds to a specific owner, group, or others permission (read, write, execute). + * + * @param permissions the integer representing the POSIX file permission bitmask + * @return a set of {@link PosixFilePermission} that represents the provided integer bitmask + */ + public static Set toPosixFilePermissions(int permissions) { + // Create an empty set specifically for this Enum type + Set perms = EnumSet.noneOf(PosixFilePermission.class); + + // Owner (User) + if ((permissions & 0400) != 0) perms.add(PosixFilePermission.OWNER_READ); + if ((permissions & 0200) != 0) perms.add(PosixFilePermission.OWNER_WRITE); + if ((permissions & 0100) != 0) perms.add(PosixFilePermission.OWNER_EXECUTE); + + // Group + if ((permissions & 0040) != 0) perms.add(PosixFilePermission.GROUP_READ); + if ((permissions & 0020) != 0) perms.add(PosixFilePermission.GROUP_WRITE); + if ((permissions & 0010) != 0) perms.add(PosixFilePermission.GROUP_EXECUTE); + + // Others (World) + if ((permissions & 0004) != 0) perms.add(PosixFilePermission.OTHERS_READ); + if ((permissions & 0002) != 0) perms.add(PosixFilePermission.OTHERS_WRITE); + if ((permissions & 0001) != 0) perms.add(PosixFilePermission.OTHERS_EXECUTE); + + return perms; + } + + static public Set toBasicPermissions(Path path) throws IOException { + // 2. On Windows: Synthesize permissions based on "DOS" attributes + Set perms = new HashSet<>(); + + // We use the basic Files.check methods which abstract the OS details + boolean isReadable = Files.isReadable(path); + boolean isWritable = Files.isWritable(path); + boolean isExecutable = Files.isExecutable(path); + + // it seems like on windows, its sftp server only really sets the "owner" permissions and uses zeroes for the rest + if (isReadable) perms.add(PosixFilePermission.OWNER_READ); + if (isWritable) perms.add(PosixFilePermission.OWNER_WRITE); + if (isExecutable) perms.add(PosixFilePermission.OWNER_EXECUTE); + + // --- GROUP (Mirror Owner or Default to Read) --- + //if (isReadable) perms.add(PosixFilePermission.GROUP_READ); + //if (isWritable) perms.add(PosixFilePermission.GROUP_WRITE); // Optional: usually safer to omit on Windows + //if (isExecutable) perms.add(PosixFilePermission.GROUP_EXECUTE); + + // --- OTHERS (Mirror Owner or Default to Read) --- + //if (isReadable) perms.add(PosixFilePermission.OTHERS_READ); + //if (isExecutable) perms.add(PosixFilePermission.OTHERS_EXECUTE); + // Usually we don't give OTHERS_WRITE on a best-effort basis + + return perms; + } + + static public void setBasicPermissions(Path path, Set perms) { + File file = path.toFile(); + + // --- READ --- + if (perms.contains(PosixFilePermission.OTHERS_READ)) { + // If World needs read, open to Everyone (false = not owner only) + file.setReadable(true, false); + } else { + // Otherwise, set specifically for Owner (true = owner only) + // If OWNER_READ is missing, this sets readable to FALSE for owner + file.setReadable(perms.contains(PosixFilePermission.OWNER_READ), true); + } + + // --- WRITE --- + if (perms.contains(PosixFilePermission.OTHERS_WRITE)) { + file.setWritable(true, false); + } else { + file.setWritable(perms.contains(PosixFilePermission.OWNER_WRITE), true); + } + + // --- EXECUTE --- + if (perms.contains(PosixFilePermission.OTHERS_EXECUTE)) { + file.setExecutable(true, false); + } else { + file.setExecutable(perms.contains(PosixFilePermission.OWNER_EXECUTE), true); + } + } + + static public int mergeOwnerPermissions(int sourcePerms, int targetPerms) { + // 0700 is the octal mask for Owner Read/Write/Execute + final int OWNER_MASK = 0700; + + // 1. Get only Owner bits from source + int ownerBits = sourcePerms & OWNER_MASK; + + // 2. Clear Owner bits from target, keeping Group, World, and special bits (SUID/SGID) + int targetWithoutOwner = targetPerms & ~OWNER_MASK; + + // 3. Combine them + return targetWithoutOwner | ownerBits; + } + + static public boolean isOwnerPermissionEqual(int perm1, int perm2) { + final int OWNER_MASK = 0700; + // Isolate owner bits for both and check equality + return (perm1 & OWNER_MASK) == (perm2 & OWNER_MASK); + } + +} \ No newline at end of file diff --git a/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java b/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java new file mode 100644 index 0000000..5827ca3 --- /dev/null +++ b/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java @@ -0,0 +1,57 @@ +package com.fizzed.jsync.vfs; + +import com.fizzed.crux.util.MoreFiles; +import com.fizzed.crux.util.Resources; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +class LocalVirtualFileSystemTest { + + static Path projectDir; + private Path sourceDir; + private LocalVirtualFileSystem defaultVfs; + + @BeforeAll + static public void setup() throws Exception { + projectDir = Resources.file("/locator.txt").resolve("../..").toAbsolutePath().normalize(); + } + + @BeforeEach + public void before() throws IOException { + this.sourceDir = projectDir.resolve("local-vfs-source"); + MoreFiles.deleteDirectoryIfExists(this.sourceDir); + Files.createDirectories(this.sourceDir); + this.defaultVfs = LocalVirtualFileSystem.open(this.sourceDir); + } + + @Test + public void permissions() throws Exception { + // we want to use a .txt file, which isn't naturally executable on windows or linux/mac + Path file = this.sourceDir.resolve("test.txt"); + Files.write(file, "#!/bin/sh\necho hello".getBytes()); + + if (this.defaultVfs.getStatModel() == StatModel.POSIX) { + // set permissions to 755 (rwxr-xr-x) + Files.setPosixFilePermissions(file, PosixFilePermissions.fromString("rwxr-xr-x")); + + final VirtualPath fileWithStat = this.defaultVfs.stat(VirtualPath.parse(file.toString())); + + assertThat(fileWithStat.getStat().getPermissionsOctal()).isEqualTo("755"); + } else { + // on windows this will always end up returning 700 + final VirtualPath fileWithStat = this.defaultVfs.stat(VirtualPath.parse(file.toString())); + + assertThat(fileWithStat.getStat().getPermissionsOctal()).isEqualTo("700"); + } + } + +} \ No newline at end of file diff --git a/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/util/PermissionsTest.java b/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/util/PermissionsTest.java new file mode 100644 index 0000000..ac8fa2a --- /dev/null +++ b/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/util/PermissionsTest.java @@ -0,0 +1,26 @@ +package com.fizzed.jsync.vfs.util; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; + +class PermissionsTest { + + @Test + public void mergeOwnerPermissions() { + int source = 0755; + int target = 0640; + int result = Permissions.mergeOwnerPermissions(source, target); + + assertThat(result).isEqualTo(0740); + } + + @Test + public void isOwnerPermissionEqual() { + assertThat(Permissions.isOwnerPermissionEqual(0755, 0655)).isFalse(); + assertThat(Permissions.isOwnerPermissionEqual(0755, 0744)).isTrue(); + assertThat(Permissions.isOwnerPermissionEqual(01755, 0744)).isTrue(); + } + +} \ No newline at end of file