From 0c757e4db2b3f9ea35a02d66fd716f536d21d554 Mon Sep 17 00:00:00 2001 From: Joe Lauer Date: Wed, 26 Nov 2025 13:56:49 -0500 Subject: [PATCH 1/9] Initial work supporting permissions sync --- .../jsync/sftp/SftpVirtualFileSystem.java | 3 +- .../jsync/sftp/SftpVirtualFileSystemDemo.java | 27 +++++++ jsync-vfs/pom.xml | 8 ++- .../jsync/vfs/LocalVirtualFileSystem.java | 42 ++++++++--- .../com/fizzed/jsync/vfs/VirtualFileStat.java | 12 +++- .../fizzed/jsync/vfs/util/Permissions.java | 71 +++++++++++++++++++ .../jsync/vfs/LocalVirtualFileSystemTest.java | 50 +++++++++++++ 7 files changed, 202 insertions(+), 11 deletions(-) create mode 100644 jsync-sftp/src/test/java/com/fizzed/jsync/sftp/SftpVirtualFileSystemDemo.java create mode 100644 jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java create mode 100644 jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java diff --git a/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java b/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java index 6576657..91e3788 100644 --- a/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java +++ b/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java @@ -202,6 +202,7 @@ private VirtualPath toVirtualPathWithStats(VirtualPath path, SftpATTRS attrs) th final long size = attrs.getSize(); final long modifiedTime = attrs.getMTime() * 1000L; final long accessedTime = attrs.getATime() * 1000L; + final int perms = attrs.getPermissions(); final VirtualFileType type; if (attrs.isDir()) { @@ -214,7 +215,7 @@ private VirtualPath toVirtualPathWithStats(VirtualPath path, SftpATTRS attrs) th type = VirtualFileType.OTHER; } - final VirtualFileStat stat = new VirtualFileStat(type, size, modifiedTime, accessedTime); + final VirtualFileStat stat = new VirtualFileStat(type, size, modifiedTime, accessedTime, perms); return new VirtualPath(path.getParentPath(), path.getName(), type == VirtualFileType.DIR, stat); } diff --git a/jsync-sftp/src/test/java/com/fizzed/jsync/sftp/SftpVirtualFileSystemDemo.java b/jsync-sftp/src/test/java/com/fizzed/jsync/sftp/SftpVirtualFileSystemDemo.java new file mode 100644 index 0000000..e80e445 --- /dev/null +++ b/jsync-sftp/src/test/java/com/fizzed/jsync/sftp/SftpVirtualFileSystemDemo.java @@ -0,0 +1,27 @@ +package com.fizzed.jsync.sftp; + +import com.fizzed.jsync.vfs.VirtualPath; +import com.fizzed.jsync.vfs.VirtualVolume; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Paths; + +import static com.fizzed.jsync.sftp.SftpVirtualVolume.sftpVolume; +import static com.fizzed.jsync.vfs.LocalVirtualVolume.localVolume; + +public class SftpVirtualFileSystemDemo { + static private final Logger log = LoggerFactory.getLogger(SftpVirtualFileSystemDemo.class); + + static public void main(String[] args) throws Exception { + +// final SftpVirtualFileSystem vfs = SftpVirtualFileSystem.open("bmh-dev-x64-indy25-1"); + final SftpVirtualFileSystem vfs = SftpVirtualFileSystem.open("bmh-build-x64-win11-1"); + +// final VirtualPath stat = vfs.stat(VirtualPath.parse(".ssh/authorized_keys")); + final VirtualPath stat = vfs.stat(VirtualPath.parse("remote-build")); + + log.info("file: {}, perms={}", stat, stat.getStat().getPermissionsOctal()); + } + +} \ No newline at end of file diff --git a/jsync-vfs/pom.xml b/jsync-vfs/pom.xml index adeeff4..afbbec0 100644 --- a/jsync-vfs/pom.xml +++ b/jsync-vfs/pom.xml @@ -19,7 +19,13 @@ - + + + com.fizzed + crux-util + test + + org.junit.jupiter junit-jupiter diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java index 4044ece..6e3bf0e 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java @@ -1,6 +1,7 @@ package com.fizzed.jsync.vfs; import com.fizzed.jsync.vfs.util.Checksums; +import com.fizzed.jsync.vfs.util.Permissions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,6 +12,7 @@ import java.nio.file.attribute.BasicFileAttributeView; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; +import java.nio.file.attribute.PosixFileAttributes; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -19,8 +21,11 @@ public class LocalVirtualFileSystem extends AbstractVirtualFileSystem { static private final Logger log = LoggerFactory.getLogger(LocalVirtualFileSystem.class); - public LocalVirtualFileSystem(String name, VirtualPath pwd, boolean caseSensitive) { + private final boolean posix; + + public LocalVirtualFileSystem(String name, VirtualPath pwd, boolean caseSensitive, boolean posix) { super(name, pwd, caseSensitive); + this.posix = posix; } static public LocalVirtualFileSystem open() { @@ -37,14 +42,18 @@ static public LocalVirtualFileSystem open(Path workingDir) { final VirtualPath pwd = VirtualPath.parse(currentWorkingDir.toString(), true); - log.debug("Detected filesystem {} has pwd {}", name, pwd); + final boolean isPosixAttributes = FileSystems.getDefault() + .supportedFileAttributeViews() + .contains("posix"); + + log.debug("Detected filesystem {} has pwd {}, posixAttrs {}", name, pwd, isPosixAttributes); // everything is case-sensitive except windows final boolean caseSensitive = !System.getProperty("os.name").toLowerCase().contains("windows"); log.debug("Detected filesystem {} is case-sensitive={}", name, caseSensitive); - return new LocalVirtualFileSystem(name, pwd, caseSensitive); + return new LocalVirtualFileSystem(name, pwd, caseSensitive, isPosixAttributes); } @Override @@ -52,6 +61,10 @@ public void close() throws Exception { // nothing to do } + public boolean isPosix() { + return this.posix; + } + protected Path toNativePath(VirtualPath path) { return Paths.get(path.toString()); } @@ -60,10 +73,17 @@ protected VirtualPath toVirtualPathWithStat(VirtualPath path) throws IOException final Path nativePath = this.toNativePath(path); // Fetch all attributes in ONE operation (and don't follow symlinks, we need to know the type) - final BasicFileAttributes attrs = Files.readAttributes(nativePath, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS); - // TODO: if we're on posix, we can also do this - // fetches size, times, PLUS owner, group, and permissions - // PosixFileAttributes attrs = Files.readAttributes(path, PosixFileAttributes.class); + final PosixFileAttributes posixAttrs; + final BasicFileAttributes attrs; + if (this.posix) { + posixAttrs = Files.readAttributes(nativePath, PosixFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + attrs = posixAttrs; + } else { + posixAttrs = null; + attrs = Files.readAttributes(nativePath, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS); + } + + // basic attributes get us much of what we need final long size = attrs.size(); final long modifiedTime = attrs.lastModifiedTime().toMillis(); final long accessedTime = attrs.lastAccessTime().toMillis(); @@ -84,7 +104,13 @@ protected VirtualPath toVirtualPathWithStat(VirtualPath path) throws IOException type = VirtualFileType.OTHER; } - final VirtualFileStat stat = new VirtualFileStat(type, size, modifiedTime, accessedTime); + // permissions are a tad trickier if they aren't really supported + int perms = -1; + if (posixAttrs != null) { + perms = Permissions.toPosixInt(posixAttrs.permissions()); + } + + final VirtualFileStat stat = new VirtualFileStat(type, size, modifiedTime, accessedTime, perms); return new VirtualPath(path.getParentPath(), path.getName(), type == VirtualFileType.DIR, stat); } diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileStat.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileStat.java index 98d612b..0365d56 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileStat.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileStat.java @@ -6,16 +6,18 @@ public class VirtualFileStat { final private long size; final private long modifiedTime; final private long accessedTime; + final private int permissions; // there are values that can be populated later as they are expensive operations private Long cksum; private String md5; private String sha1; - public VirtualFileStat(VirtualFileType type, long size, long modifiedTime, long accessedTime) { + public VirtualFileStat(VirtualFileType type, long size, long modifiedTime, long accessedTime, int permissions) { this.size = size; this.type = type; this.modifiedTime = modifiedTime; this.accessedTime = accessedTime; + this.permissions = permissions; } public VirtualFileType getType() { @@ -34,6 +36,14 @@ public long getAccessedTime() { return accessedTime; } + public int getPermissions() { + return this.permissions; + } + + public String getPermissionsOctal() { + return Integer.toOctalString(this.permissions); + } + public Long getCksum() { return cksum; } diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java new file mode 100644 index 0000000..ec30cf3 --- /dev/null +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java @@ -0,0 +1,71 @@ +package com.fizzed.jsync.vfs.util; + +import java.nio.file.attribute.PosixFilePermission; +import java.util.EnumSet; +import java.util.Set; + +public class Permissions { + + /** + * Converts a set of {@link PosixFilePermission} to its corresponding POSIX integer representation. + * The resulting integer is a bitmask representing the file permission mode. + * + * @param permissions the set of {@link PosixFilePermission} to convert; if null, the result will be 0 + * @return an integer representing the POSIX file permission mode derived from the given set of permissions + */ + static public int toPosixInt(Set permissions) { + int mode = 0; + + // Null check safety + if (permissions == null) return mode; + + for (PosixFilePermission perm : permissions) { + switch (perm) { + case OWNER_READ: mode |= 0400; break; // 256 in decimal + case OWNER_WRITE: mode |= 0200; break; // 128 + case OWNER_EXECUTE: mode |= 0100; break; // 64 + + case GROUP_READ: mode |= 0040; break; // 32 + case GROUP_WRITE: mode |= 0020; break; // 16 + case GROUP_EXECUTE: mode |= 0010; break; // 8 + + case OTHERS_READ: mode |= 0004; break; // 4 + case OTHERS_WRITE: mode |= 0002; break; // 2 + case OTHERS_EXECUTE: mode |= 0001; break; // 1 + } + } + + return mode; + } + + /** + * Converts a POSIX file permission integer into a set of {@link PosixFilePermission}. + * The input integer is interpreted as the POSIX file permission bitmask, where each bit + * corresponds to a specific owner, group, or others permission (read, write, execute). + * + * @param permissions the integer representing the POSIX file permission bitmask + * @return a set of {@link PosixFilePermission} that represents the provided integer bitmask + */ + public static Set toPosixFilePermissions(int permissions) { + // Create an empty set specifically for this Enum type + Set perms = EnumSet.noneOf(PosixFilePermission.class); + + // Owner (User) + if ((permissions & 0400) != 0) perms.add(PosixFilePermission.OWNER_READ); + if ((permissions & 0200) != 0) perms.add(PosixFilePermission.OWNER_WRITE); + if ((permissions & 0100) != 0) perms.add(PosixFilePermission.OWNER_EXECUTE); + + // Group + if ((permissions & 0040) != 0) perms.add(PosixFilePermission.GROUP_READ); + if ((permissions & 0020) != 0) perms.add(PosixFilePermission.GROUP_WRITE); + if ((permissions & 0010) != 0) perms.add(PosixFilePermission.GROUP_EXECUTE); + + // Others (World) + if ((permissions & 0004) != 0) perms.add(PosixFilePermission.OTHERS_READ); + if ((permissions & 0002) != 0) perms.add(PosixFilePermission.OTHERS_WRITE); + if ((permissions & 0001) != 0) perms.add(PosixFilePermission.OTHERS_EXECUTE); + + return perms; + } + +} \ No newline at end of file diff --git a/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java b/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java new file mode 100644 index 0000000..1807188 --- /dev/null +++ b/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java @@ -0,0 +1,50 @@ +package com.fizzed.jsync.vfs; + +import com.fizzed.crux.util.MoreFiles; +import com.fizzed.crux.util.Resources; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +class LocalVirtualFileSystemTest { + + static Path projectDir; + private Path sourceDir; + private LocalVirtualFileSystem defaultVfs; + + @BeforeAll + static public void setup() throws Exception { + projectDir = Resources.file("/locator.txt").resolve("../..").toAbsolutePath().normalize(); + } + + @BeforeEach + public void before() throws IOException { + this.sourceDir = projectDir.resolve("local-vfs-source"); + MoreFiles.deleteDirectoryIfExists(this.sourceDir); + Files.createDirectories(this.sourceDir); + this.defaultVfs = LocalVirtualFileSystem.open(this.sourceDir); + } + + @Test + public void readPermissions() throws Exception { + if (this.defaultVfs.isPosix()) { + Path file = this.sourceDir.resolve("test.sh"); + Files.write(file, "#!/bin/sh\necho hello".getBytes()); + // 1. Set permissions to 755 (rwxr-xr-x) + Files.setPosixFilePermissions(file, PosixFilePermissions.fromString("rwxr-xr-x")); + + final VirtualPath fileWithStat = this.defaultVfs.stat(VirtualPath.parse(file.toString())); + + assertThat(fileWithStat.getStat().getPermissionsOctal()).isEqualTo("755"); + } + } + +} \ No newline at end of file From 09e4d1e4de62cf42030796f8dc5afbbdcb2e3053 Mon Sep 17 00:00:00 2001 From: Joe Lauer Date: Wed, 26 Nov 2025 15:04:27 -0500 Subject: [PATCH 2/9] Support for non-posix perms --- .../com/fizzed/jsync/engine/JsyncEngine.java | 8 +-- .../jsync/sftp/SftpVirtualFileSystem.java | 70 ++++++++++++++----- .../jsync/vfs/AbstractVirtualFileSystem.java | 28 ++++++++ .../jsync/vfs/LocalVirtualFileSystem.java | 51 +++++++++----- .../java/com/fizzed/jsync/vfs/StatKind.java | 8 +++ .../fizzed/jsync/vfs/VirtualFileSystem.java | 13 +++- .../fizzed/jsync/vfs/util/Permissions.java | 31 ++++++++ .../jsync/vfs/LocalVirtualFileSystemTest.java | 19 +++-- 8 files changed, 180 insertions(+), 48 deletions(-) create mode 100644 jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatKind.java diff --git a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java index 029c9a1..357b0ee 100644 --- a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java +++ b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java @@ -388,7 +388,7 @@ protected void syncDirectory(int level, JsyncResult result, List ex // find a matching target path entirely by name VirtualPath targetChildPath = targetChildPaths.stream() - .filter(p -> targetVfs.areFileNamesEqual(p.getName(), sourceChildPath.getName())) + .filter(p -> targetVfs.isFileNameEqual(p.getName(), sourceChildPath.getName())) .findFirst() .orElse(null); @@ -415,7 +415,7 @@ protected void syncDirectory(int level, JsyncResult result, List ex for (VirtualPath targetChildPath : targetChildPaths) { // find a matching source path entirely by name final VirtualPath sourceChildPath = sourceChildPaths.stream() - .filter(p -> sourceVfs.areFileNamesEqual(p.getName(), targetChildPath.getName())) + .filter(p -> sourceVfs.isFileNameEqual(p.getName(), targetChildPath.getName())) .findFirst() .orElse(null); @@ -618,13 +618,13 @@ protected Checksum negotiateChecksum(VirtualFileSystem sourceVfs, VirtualFileSys log.debug("Detecting if {} checksum is supported on source/target", preferredChecksum); // check supported checksums, keep a tally of which are supported by both sides, so we can log them out - boolean sourceSupported = sourceVfs.isSupported(preferredChecksum); + boolean sourceSupported = sourceVfs.isChecksumSupported(preferredChecksum); if (sourceSupported) { sourceChecksumsSupported.add(preferredChecksum); } - boolean targetSupported = targetVfs.isSupported(preferredChecksum); + boolean targetSupported = targetVfs.isChecksumSupported(preferredChecksum); if (targetSupported) { targetChecksumsSupported.add(preferredChecksum); diff --git a/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java b/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java index 91e3788..940718f 100644 --- a/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java +++ b/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java @@ -17,6 +17,8 @@ import java.util.*; import java.util.concurrent.CountDownLatch; +import static java.util.Arrays.asList; + public class SftpVirtualFileSystem extends AbstractVirtualFileSystem { static private final Logger log = LoggerFactory.getLogger(SftpVirtualFileSystem.class); @@ -109,27 +111,27 @@ static public SftpVirtualFileSystem open(Session ssh, boolean closeSsh) throws I static public SftpVirtualFileSystem open(Session ssh, boolean closeSsh, ChannelSftp sftp, boolean closeSftp) throws IOException { final String name = ssh.getHost(); - log.info("Opening filesystem {}...", name); + log.debug("Opening filesystem {}...", name); - final String pwd2; + final String pwdRaw; try { - pwd2 = sftp.pwd(); + pwdRaw = sftp.pwd(); } catch (SftpException e) { throw toIOException(e); } - final VirtualPath pwd = VirtualPath.parse(pwd2, true); + final VirtualPath pwd = VirtualPath.parse(pwdRaw, true); - log.debug("Detected filesystem {} has pwd {}", name, pwd); + log.debug("Detected pwd {}", pwd); boolean windows = false; // this is likely a "windows" system if the 2nd char is : - if (pwd2.length() > 2 && pwd2.charAt(2) == ':') { + if (pwdRaw.length() > 2 && pwdRaw.charAt(2) == ':') { // TODO: should we confirm by running a command that exists only windows to confirm? // for now we'll just assume it is windows = true; - log.debug("Detected filesystem {} is running on windows (changes standard checksums, native filepaths, case sensitivity, etc.)", name); + log.debug("Detected windows-based sftp server"); } return new SftpVirtualFileSystem(name, pwd, ssh, closeSsh, sftp, closeSftp, windows); @@ -154,12 +156,18 @@ public void close() throws Exception { } @Override - public String toString() { - return this.getName(); + public boolean isRemote() { + return true; } @Override - public boolean isSupported(Checksum checksum) throws IOException { + public StatKind getStatKind() { + // for now, we'll claim full POSIX as the sftp server itself does the POSIX translation + return StatKind.POSIX; + } + + @Override + public boolean isChecksumSupported(Checksum checksum) throws IOException { if (this.windows) { switch (checksum) { case MD5: @@ -189,6 +197,37 @@ public boolean isSupported(Checksum checksum) throws IOException { } } + @Override + protected List doDetectChecksums() throws IOException { + // windows is easy, return what powershell supports + if (this.windows) { + return asList(Checksum.MD5, Checksum.SHA1); + } + + // otherwise, we are on posix and we can actually check whether these would work or not + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + // check if anything is supported + int exitValue = this.exec(this.ssh, "which cksum md5sum sha1sum", null, baos, null); + if (exitValue != 0) { + return Collections.emptyList(); + } + + final String output = baos.toString(StandardCharsets.UTF_8.name()); + final List checksums = new ArrayList<>(); + if (output.contains("cksum")) { + checksums.add(Checksum.CK); + } + if (output.contains("md5sum")) { + checksums.add(Checksum.MD5); + } + if (output.contains("sha1sum")) { + checksums.add(Checksum.SHA1); + } + + return checksums; + } + public int getMaxCommandLength() { return maxCommandLength; } @@ -198,7 +237,7 @@ public SftpVirtualFileSystem setMaxCommandLength(int maxCommandLength) { return this; } - private VirtualPath toVirtualPathWithStats(VirtualPath path, SftpATTRS attrs) throws IOException { + protected VirtualPath withStats(VirtualPath path, SftpATTRS attrs) throws IOException { final long size = attrs.getSize(); final long modifiedTime = attrs.getMTime() * 1000L; final long accessedTime = attrs.getATime() * 1000L; @@ -220,17 +259,12 @@ private VirtualPath toVirtualPathWithStats(VirtualPath path, SftpATTRS attrs) th return new VirtualPath(path.getParentPath(), path.getName(), type == VirtualFileType.DIR, stat); } - @Override - public boolean isRemote() { - return true; - } - @Override public VirtualPath stat(VirtualPath path) throws IOException { try { final SftpATTRS attrs = this.sftp.lstat(path.toString()); - return this.toVirtualPathWithStats(path, attrs); + return this.withStats(path, attrs); } catch (SftpException e) { throw toIOException(e); } @@ -278,7 +312,7 @@ public List ls(VirtualPath path) throws IOException { // dir true/false doesn't matter, stats call next will correct it VirtualPath childPathWithoutStats = path.resolve(entry.getFilename(), false); - VirtualPath childPath = this.toVirtualPathWithStats(childPathWithoutStats, entry.getAttrs()); + VirtualPath childPath = this.withStats(childPathWithoutStats, entry.getAttrs()); childPaths.add(childPath); } diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/AbstractVirtualFileSystem.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/AbstractVirtualFileSystem.java index 8a9ebf8..7d2e37f 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/AbstractVirtualFileSystem.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/AbstractVirtualFileSystem.java @@ -1,15 +1,22 @@ package com.fizzed.jsync.vfs; +import java.io.IOException; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + abstract public class AbstractVirtualFileSystem implements VirtualFileSystem { protected final String name; protected final VirtualPath pwd; protected final boolean caseSensitive; + protected volatile Set checksums; public AbstractVirtualFileSystem(String name, VirtualPath pwd, boolean caseSensitive) { this.name = name; this.pwd = pwd; this.caseSensitive = caseSensitive; + this.checksums = null; } @Override @@ -27,6 +34,27 @@ public boolean isCaseSensitive() { return this.caseSensitive; } + @Override + public boolean isChecksumSupported(Checksum checksum) throws IOException { + return this.getChecksumsSupported().contains(checksum); + } + + @Override + public Set getChecksumsSupported() throws IOException { + if (this.checksums == null) { + // we need to calculate them + synchronized (this) { + if (this.checksums == null) { + final List values = this.doDetectChecksums(); + this.checksums = new LinkedHashSet<>(values); + } + } + } + return this.checksums; + } + + abstract protected List doDetectChecksums() throws IOException; + @Override public String toString() { return this.name; diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java index 6e3bf0e..49a67c2 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java @@ -9,15 +9,15 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.*; -import java.nio.file.attribute.BasicFileAttributeView; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.FileTime; -import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.attribute.*; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.Set; import java.util.stream.Stream; +import static java.util.Arrays.asList; + public class LocalVirtualFileSystem extends AbstractVirtualFileSystem { static private final Logger log = LoggerFactory.getLogger(LocalVirtualFileSystem.class); @@ -65,11 +65,31 @@ public boolean isPosix() { return this.posix; } + @Override + public boolean isRemote() { + return false; + } + + @Override + public StatKind getStatKind() { + if (this.posix) { + return StatKind.POSIX; + } else { + return StatKind.BASIC; + } + } + + @Override + protected List doDetectChecksums() throws IOException { + // everything is supported + return asList(Checksum.CK, Checksum.MD5, Checksum.SHA1); + } + protected Path toNativePath(VirtualPath path) { return Paths.get(path.toString()); } - protected VirtualPath toVirtualPathWithStat(VirtualPath path) throws IOException { + protected VirtualPath withStat(VirtualPath path) throws IOException { final Path nativePath = this.toNativePath(path); // Fetch all attributes in ONE operation (and don't follow symlinks, we need to know the type) @@ -105,9 +125,13 @@ protected VirtualPath toVirtualPathWithStat(VirtualPath path) throws IOException } // permissions are a tad trickier if they aren't really supported - int perms = -1; + final int perms; if (posixAttrs != null) { perms = Permissions.toPosixInt(posixAttrs.permissions()); + } else { + // use basic permissions + final Set simulatedPosixPermissions = Permissions.getPosixPermissions(nativePath); + perms = Permissions.toPosixInt(simulatedPosixPermissions); } final VirtualFileStat stat = new VirtualFileStat(type, size, modifiedTime, accessedTime, perms); @@ -115,14 +139,11 @@ protected VirtualPath toVirtualPathWithStat(VirtualPath path) throws IOException return new VirtualPath(path.getParentPath(), path.getName(), type == VirtualFileType.DIR, stat); } - @Override - public boolean isRemote() { - return false; - } + @Override public VirtualPath stat(VirtualPath path) throws IOException { - return this.toVirtualPathWithStat(path); + return this.withStat(path); } @Override @@ -155,7 +176,7 @@ public List ls(VirtualPath path) throws IOException { // dir true/false doesn't matter, stats call next will correct it VirtualPath childPathWithoutStats = path.resolve(nativeChildPath.getFileName().toString(), false); - VirtualPath childPath = this.toVirtualPathWithStat(childPathWithoutStats); + VirtualPath childPath = this.withStat(childPathWithoutStats); childPaths.add(childPath); } } @@ -205,12 +226,6 @@ public OutputStream writeStream(VirtualPath path) throws IOException { return Files.newOutputStream(nativePath, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); } - @Override - public boolean isSupported(Checksum checksum) throws IOException { - // all are supported! - return true; - } - @Override public void cksums(List paths) throws IOException { for (VirtualPath path : paths) { diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatKind.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatKind.java new file mode 100644 index 0000000..6001e40 --- /dev/null +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatKind.java @@ -0,0 +1,8 @@ +package com.fizzed.jsync.vfs; + +public enum StatKind { + + POSIX, + BASIC + +} \ No newline at end of file diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileSystem.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileSystem.java index ee4ab8f..7c521a9 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileSystem.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileSystem.java @@ -5,6 +5,7 @@ import java.io.OutputStream; import java.nio.file.NoSuchFileException; import java.util.List; +import java.util.Set; public interface VirtualFileSystem extends AutoCloseable { @@ -12,9 +13,17 @@ public interface VirtualFileSystem extends AutoCloseable { boolean isRemote(); + // features of this filesystem, which changes the behavior of the engine + boolean isCaseSensitive(); - default boolean areFileNamesEqual(String name1, String name2) { + StatKind getStatKind(); + + boolean isChecksumSupported(Checksum checksum) throws IOException; + + Set getChecksumsSupported() throws IOException; + + default boolean isFileNameEqual(String name1, String name2) { if (this.isCaseSensitive()) { return name1.equals(name2); } else { @@ -68,8 +77,6 @@ default VirtualPath exists(VirtualPath path) throws IOException { OutputStream writeStream(VirtualPath path) throws IOException; - boolean isSupported(Checksum checksum) throws IOException; - default void checksums(Checksum checksum, List paths) throws IOException { switch (checksum) { case CK: diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java index ec30cf3..0797c93 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java @@ -1,7 +1,11 @@ package com.fizzed.jsync.vfs.util; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.attribute.PosixFilePermission; import java.util.EnumSet; +import java.util.HashSet; import java.util.Set; public class Permissions { @@ -68,4 +72,31 @@ public static Set toPosixFilePermissions(int permissions) { return perms; } + static public Set getPosixPermissions(Path path) throws IOException { + // 2. On Windows: Synthesize permissions based on "DOS" attributes + Set perms = new HashSet<>(); + + // We use the basic Files.check methods which abstract the OS details + boolean isReadable = Files.isReadable(path); + boolean isWritable = Files.isWritable(path); + boolean isExecutable = Files.isExecutable(path); + + // it seems like on windows, its sftp server only really sets the "owner" permissions and uses zeroes for the rest + if (isReadable) perms.add(PosixFilePermission.OWNER_READ); + if (isWritable) perms.add(PosixFilePermission.OWNER_WRITE); + if (isExecutable) perms.add(PosixFilePermission.OWNER_EXECUTE); + + // --- GROUP (Mirror Owner or Default to Read) --- + //if (isReadable) perms.add(PosixFilePermission.GROUP_READ); + //if (isWritable) perms.add(PosixFilePermission.GROUP_WRITE); // Optional: usually safer to omit on Windows + //if (isExecutable) perms.add(PosixFilePermission.GROUP_EXECUTE); + + // --- OTHERS (Mirror Owner or Default to Read) --- + //if (isReadable) perms.add(PosixFilePermission.OTHERS_READ); + //if (isExecutable) perms.add(PosixFilePermission.OTHERS_EXECUTE); + // Usually we don't give OTHERS_WRITE on a best-effort basis + + return perms; + } + } \ No newline at end of file diff --git a/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java b/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java index 1807188..d08de7b 100644 --- a/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java +++ b/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java @@ -34,15 +34,24 @@ public void before() throws IOException { } @Test - public void readPermissions() throws Exception { - if (this.defaultVfs.isPosix()) { - Path file = this.sourceDir.resolve("test.sh"); - Files.write(file, "#!/bin/sh\necho hello".getBytes()); - // 1. Set permissions to 755 (rwxr-xr-x) + public void permissions() throws Exception { + + Path file = this.sourceDir.resolve("test.sh"); + Files.write(file, "#!/bin/sh\necho hello".getBytes()); + + if (this.defaultVfs.getStatKind() == StatKind.POSIX) { + // set permissions to 755 (rwxr-xr-x) Files.setPosixFilePermissions(file, PosixFilePermissions.fromString("rwxr-xr-x")); final VirtualPath fileWithStat = this.defaultVfs.stat(VirtualPath.parse(file.toString())); + assertThat(fileWithStat.getStat().getPermissionsOctal()).isEqualTo("755"); + } else { + // set permissions to execute + file.toFile().setExecutable(true); + + final VirtualPath fileWithStat = this.defaultVfs.stat(VirtualPath.parse(file.toString())); + assertThat(fileWithStat.getStat().getPermissionsOctal()).isEqualTo("755"); } } From 759cf723b19c9d29874cd1cb7301ff3625d4cd11 Mon Sep 17 00:00:00 2001 From: Joe Lauer Date: Wed, 26 Nov 2025 15:15:12 -0500 Subject: [PATCH 3/9] Windows basic permissions calculated --- .../com/fizzed/jsync/vfs/LocalVirtualFileSystem.java | 4 ++-- .../fizzed/jsync/vfs/LocalVirtualFileSystemTest.java | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java index 49a67c2..ecad653 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java @@ -46,7 +46,7 @@ static public LocalVirtualFileSystem open(Path workingDir) { .supportedFileAttributeViews() .contains("posix"); - log.debug("Detected filesystem {} has pwd {}, posixAttrs {}", name, pwd, isPosixAttributes); + log.debug("Detected filesystem {} has pwd={}, posix={}", name, pwd, isPosixAttributes); // everything is case-sensitive except windows final boolean caseSensitive = !System.getProperty("os.name").toLowerCase().contains("windows"); @@ -129,7 +129,7 @@ protected VirtualPath withStat(VirtualPath path) throws IOException { if (posixAttrs != null) { perms = Permissions.toPosixInt(posixAttrs.permissions()); } else { - // use basic permissions + // use basic permissions, usually ends up being 700 from what I can gather final Set simulatedPosixPermissions = Permissions.getPosixPermissions(nativePath); perms = Permissions.toPosixInt(simulatedPosixPermissions); } diff --git a/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java b/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java index d08de7b..179e066 100644 --- a/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java +++ b/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java @@ -35,8 +35,8 @@ public void before() throws IOException { @Test public void permissions() throws Exception { - - Path file = this.sourceDir.resolve("test.sh"); + // we want to use a .txt file, which isn't naturally executable on windows or linux/mac + Path file = this.sourceDir.resolve("test.txt"); Files.write(file, "#!/bin/sh\necho hello".getBytes()); if (this.defaultVfs.getStatKind() == StatKind.POSIX) { @@ -47,12 +47,10 @@ public void permissions() throws Exception { assertThat(fileWithStat.getStat().getPermissionsOctal()).isEqualTo("755"); } else { - // set permissions to execute - file.toFile().setExecutable(true); - + // on windows this will always end up returning 700 final VirtualPath fileWithStat = this.defaultVfs.stat(VirtualPath.parse(file.toString())); - assertThat(fileWithStat.getStat().getPermissionsOctal()).isEqualTo("755"); + assertThat(fileWithStat.getStat().getPermissionsOctal()).isEqualTo("700"); } } From 050f73f4d802119f0956fbcb86e6f370fbb8a01b Mon Sep 17 00:00:00 2001 From: Joe Lauer Date: Wed, 26 Nov 2025 15:42:09 -0500 Subject: [PATCH 4/9] Permissions sync is sorta working --- .../java/com/fizzed/jsync/engine/JsyncEngine.java | 7 ++++++- .../test/java/com/fizzed/jsync/engine/JsyncDemo.java | 2 +- .../com/fizzed/jsync/sftp/SftpVirtualFileSystem.java | 12 +++++++----- .../fizzed/jsync/sftp/SftpVirtualFileSystemTest.java | 5 +++++ 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java index 357b0ee..8ce895d 100644 --- a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java +++ b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java @@ -479,6 +479,11 @@ protected JsyncPathChanges detectChanges(VirtualPath sourcePath, VirtualPath tar timestamps = true; } + if (sourcePath.getStat().getPermissions() != targetPath.getStat().getPermissions()) { + log.trace("Source path {} perms {} != target perms {}", sourcePath, sourcePath.getStat().getPermissions(), targetPath.getStat().getPermissions()); + permissions = true; + } + // if we have "cksum" values on both sides, we can compare those if (sourcePath.getStat().getCksum() != null && targetPath.getStat().getCksum() != null) { if (!sourcePath.getStat().getCksum().equals(targetPath.getStat().getCksum())) { @@ -509,7 +514,7 @@ protected JsyncPathChanges detectChanges(VirtualPath sourcePath, VirtualPath tar } } - return new JsyncPathChanges(sourcePath.isDirectory(), false, size, timestamps, ownership, permissions, checksums); + return new JsyncPathChanges(sourcePath.isDirectory(), false, size, timestamps, permissions, ownership, checksums); } protected void transferFile(JsyncResult result, VirtualFileSystem sourceVfs, VirtualPath sourceFile, VirtualFileSystem targetVfs, VirtualPath targetFile, JsyncPathChanges changes) throws IOException { diff --git a/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncDemo.java b/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncDemo.java index 597c184..317c192 100644 --- a/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncDemo.java +++ b/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncDemo.java @@ -17,7 +17,7 @@ static public void main(String[] args) throws Exception { // final String sourceDir = Paths.get("/home/jjlauer/test-sync").toString(); // final VirtualVolume source = localVolume(Paths.get("/home/jjlauer/workspace/third-party/jsch")); // final String sourceDir = Paths.get("/home/jjlauer/workspace/third-party/coredns").toString(); - final VirtualVolume source = localVolume(Paths.get("/home/jjlauer/workspace/third-party/nats.java")); + final VirtualVolume source = localVolume(Paths.get( System.getProperty("user.home") + "/workspace/third-party/nats.java")); // final VirtualVolume source = localVolume(Paths.get("/home/jjlauer/workspace/third-party/tokyocabinet-1.4.48")); // final String sourceDir = Paths.get("C:\\Users\\jjlauer\\test-sync").toString(); // final String sourceDir = Paths.get("C:\\Users\\jjlauer\\workspace\\third-party\\tokyocabinet-1.4.48").toString(); diff --git a/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java b/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java index 940718f..187f3a2 100644 --- a/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java +++ b/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java @@ -241,7 +241,8 @@ protected VirtualPath withStats(VirtualPath path, SftpATTRS attrs) throws IOExce final long size = attrs.getSize(); final long modifiedTime = attrs.getMTime() * 1000L; final long accessedTime = attrs.getATime() * 1000L; - final int perms = attrs.getPermissions(); + // sftp stuffs extra stuff like the file type in the permissions value, we don't care about it + final int perms = attrs.getPermissions() & 07777; final VirtualFileType type; if (attrs.isDir()) { @@ -273,19 +274,20 @@ public VirtualPath stat(VirtualPath path) throws IOException { @Override public void updateStat(VirtualPath path, VirtualFileStat stat) throws IOException { try { + final SftpATTRS attrs = SftpATTRSAccessor.createSftpATTRS(); + // TODO: are we updating uid/gid? Integer uid = null; Integer gid = null; // TODO: are we updating permissions? - Integer perms = null; +// Integer perms = null; + int perms = stat.getPermissions(); + attrs.setPERMISSIONS(perms); // are we updating mtime/atime?d int mtime = (int)(stat.getModifiedTime()/1000); int atime = (int)(stat.getAccessedTime()/1000); - - final SftpATTRS attrs = SftpATTRSAccessor.createSftpATTRS(); - attrs.setACMODTIME(atime, mtime); this.sftp.setStat(path.toString(), attrs); diff --git a/jsync-sftp/src/test/java/com/fizzed/jsync/sftp/SftpVirtualFileSystemTest.java b/jsync-sftp/src/test/java/com/fizzed/jsync/sftp/SftpVirtualFileSystemTest.java index d9be42b..65d8803 100644 --- a/jsync-sftp/src/test/java/com/fizzed/jsync/sftp/SftpVirtualFileSystemTest.java +++ b/jsync-sftp/src/test/java/com/fizzed/jsync/sftp/SftpVirtualFileSystemTest.java @@ -135,6 +135,11 @@ public void stat() throws Exception { assertThat(vpWithStat.getStat().getType()).isEqualTo(VirtualFileType.FILE); assertThat(vpWithStat.getStat().getSize()).isEqualTo(Files.size(sftpRootDir.resolve("root.txt"))); assertThat(vpWithStat.getStat().getModifiedTime()).isCloseTo(Files.getLastModifiedTime(sftpRootDir.resolve("root.txt")).toMillis(), within(1500L)); + + // permissions + assertThat(vpWithStat.getStat().getPermissions()).isGreaterThan(0); + // if we got rid of the extra stuff stacked by sftp, this value should be less than this + assertThat(vpWithStat.getStat().getPermissions()).isLessThan(4096); } @Test From 01cd40b2360df8da142d304f483f0af5a14af8dc Mon Sep 17 00:00:00 2001 From: Joe Lauer Date: Wed, 26 Nov 2025 19:21:20 -0500 Subject: [PATCH 5/9] Perms kinda working --- .../com/fizzed/jsync/engine/JsyncEngine.java | 37 +++++++++++---- .../fizzed/jsync/engine/JsyncPathChanges.java | 14 +++++- .../com/fizzed/jsync/engine/JsyncDemo.java | 1 + .../jsync/sftp/SftpVirtualFileSystem.java | 18 ++++---- .../jsync/vfs/LocalVirtualFileSystem.java | 45 ++++++++++++------- .../fizzed/jsync/vfs/StatUpdateOption.java | 9 ++++ .../fizzed/jsync/vfs/VirtualFileSystem.java | 3 +- .../fizzed/jsync/vfs/util/Permissions.java | 31 ++++++++++++- .../jsync/vfs/util/PermissionsTest.java | 11 +++++ 9 files changed, 133 insertions(+), 36 deletions(-) create mode 100644 jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatUpdateOption.java create mode 100644 jsync-vfs/src/test/java/com/fizzed/jsync/vfs/util/PermissionsTest.java diff --git a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java index 8ce895d..bac1997 100644 --- a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java +++ b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java @@ -8,10 +8,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; +import java.util.*; import static java.util.Arrays.asList; import static java.util.stream.Collectors.toList; @@ -26,6 +23,7 @@ public class JsyncEngine { private boolean force; private boolean parents; private boolean ignoreTimes; + private boolean skipPermissions; private int maxFilesMaybeModifiedLimit; private List excludes; @@ -35,6 +33,7 @@ public JsyncEngine() { this.force = false; this.parents = false; this.ignoreTimes = false; + this.skipPermissions = false; this.preferredChecksums = new ArrayList<>(asList(Checksum.CK, Checksum.MD5)); this.maxFilesMaybeModifiedLimit = 256; this.excludes = null; @@ -86,6 +85,15 @@ public JsyncEngine setIgnoreTimes(boolean ignoreTimes) { return this; } + public boolean isSkipPermissions() { + return skipPermissions; + } + + public JsyncEngine setSkipPermissions(boolean skipPermissions) { + this.skipPermissions = skipPermissions; + return this; + } + public List getPreferredChecksums() { return this.preferredChecksums; } @@ -280,7 +288,7 @@ protected void syncFile(JsyncResult result, List deferredFiles, if (log.isDebugEnabled()) log.debug("Verified file {} ({})", targetPath, changes); } - if (changes.isStatModified()) { + if (changes.isStatModified(this.skipPermissions)) { // stat will need updated if the file is either new, updated, or if only the perms/times need updating this.updateStat(result, sourcePath, targetVfs, targetPath, changes, fileWasTransferred); } @@ -434,7 +442,7 @@ protected void syncDirectory(int level, JsyncResult result, List ex // last step is to update the stat of the target dir // To successfully preserve directory timestamps, you must set the directory attributes after you have finished touching every single file inside that directory. - if (changes.isStatModified()) { + if (changes.isStatModified(this.skipPermissions)) { // stat will need updated if the dir is new OR if the dir stats have changed this.updateStat(result, sourcePath, targetVfs, targetPath, changes, changes.isMissing()); } @@ -479,7 +487,7 @@ protected JsyncPathChanges detectChanges(VirtualPath sourcePath, VirtualPath tar timestamps = true; } - if (sourcePath.getStat().getPermissions() != targetPath.getStat().getPermissions()) { + if (!this.skipPermissions && sourcePath.getStat().getPermissions() != targetPath.getStat().getPermissions()) { log.trace("Source path {} perms {} != target perms {}", sourcePath, sourcePath.getStat().getPermissions(), targetPath.getStat().getPermissions()); permissions = true; } @@ -540,7 +548,20 @@ protected void transferFile(JsyncResult result, VirtualFileSystem sourceVfs, Vir protected void updateStat(JsyncResult result, VirtualPath sourcePath, VirtualFileSystem targetVfs, VirtualPath targetPath, JsyncPathChanges changes, boolean associatedWithFileModifiedOrDirCreated) throws IOException { this.eventHandler.willUpdateStat(sourcePath, targetPath, changes, associatedWithFileModifiedOrDirCreated); - targetVfs.updateStat(targetPath, sourcePath.getStat()); + final Set options = EnumSet.noneOf(StatUpdateOption.class); + // in posix -> posix, we can use the stat of the source, but if we're changing permissions and a BASIC vfs + // is involved, we only want to try and change the "owner" permission, and leave everything else as-is + VirtualFileStat updateStat = sourcePath.getStat(); + + if (changes.isPermissionModified(this.skipPermissions)) { + options.add(StatUpdateOption.PERMISSIONS); + } + + if (changes.isTimestampsModified()) { + options.add(StatUpdateOption.TIMESTAMPS); + } + + targetVfs.updateStat(targetPath, updateStat, options); result.incrementStatsUpdated(); } diff --git a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncPathChanges.java b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncPathChanges.java index 2361905..4fc15b1 100644 --- a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncPathChanges.java +++ b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncPathChanges.java @@ -79,10 +79,20 @@ public boolean isDeferredProcessing(boolean ignoreTimes) { && checksum == null; } - public boolean isStatModified() { + public boolean isPermissionModified(boolean skipPermissions) { + return this.missing + || (!skipPermissions && this.permissions); + } + + public boolean isTimestampsModified() { + return this.missing + || this.timestamps; + } + + public boolean isStatModified(boolean skipPermissions) { return this.missing || this.timestamps - || this.permissions + || (!skipPermissions && this.permissions) || this.ownership; } diff --git a/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncDemo.java b/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncDemo.java index 317c192..4492a4a 100644 --- a/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncDemo.java +++ b/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncDemo.java @@ -39,6 +39,7 @@ static public void main(String[] args) throws Exception { .setDelete(true) .setParents(true) .setForce(true) + //.setSkipPermissions(true) //.setIgnoreTimes(true) //.setMaxFilesMaybeModifiedLimit(256) // .setProgress(true) diff --git a/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java b/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java index 187f3a2..99c671c 100644 --- a/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java +++ b/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java @@ -272,7 +272,7 @@ public VirtualPath stat(VirtualPath path) throws IOException { } @Override - public void updateStat(VirtualPath path, VirtualFileStat stat) throws IOException { + public void updateStat(VirtualPath path, VirtualFileStat stat, Collection options) throws IOException { try { final SftpATTRS attrs = SftpATTRSAccessor.createSftpATTRS(); @@ -280,15 +280,15 @@ public void updateStat(VirtualPath path, VirtualFileStat stat) throws IOExceptio Integer uid = null; Integer gid = null; - // TODO: are we updating permissions? -// Integer perms = null; - int perms = stat.getPermissions(); - attrs.setPERMISSIONS(perms); + if (options.contains(StatUpdateOption.PERMISSIONS)) { + attrs.setPERMISSIONS(stat.getPermissions()); + } - // are we updating mtime/atime?d - int mtime = (int)(stat.getModifiedTime()/1000); - int atime = (int)(stat.getAccessedTime()/1000); - attrs.setACMODTIME(atime, mtime); + if (options.contains(StatUpdateOption.TIMESTAMPS)) { + int mtime = (int) (stat.getModifiedTime() / 1000); + int atime = (int) (stat.getAccessedTime() / 1000); + attrs.setACMODTIME(atime, mtime); + } this.sftp.setStat(path.toString(), attrs); } catch (SftpException e) { diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java index ecad653..4504042 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java @@ -10,10 +10,7 @@ import java.io.OutputStream; import java.nio.file.*; import java.nio.file.attribute.*; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.stream.Stream; import static java.util.Arrays.asList; @@ -130,7 +127,7 @@ protected VirtualPath withStat(VirtualPath path) throws IOException { perms = Permissions.toPosixInt(posixAttrs.permissions()); } else { // use basic permissions, usually ends up being 700 from what I can gather - final Set simulatedPosixPermissions = Permissions.getPosixPermissions(nativePath); + final Set simulatedPosixPermissions = Permissions.toBasicPermissions(nativePath); perms = Permissions.toPosixInt(simulatedPosixPermissions); } @@ -147,20 +144,38 @@ public VirtualPath stat(VirtualPath path) throws IOException { } @Override - public void updateStat(VirtualPath path, VirtualFileStat stat) throws IOException { + public void updateStat(VirtualPath path, VirtualFileStat stat, Collection options) throws IOException { final Path nativePath = this.toNativePath(path); - // Get the "View" (This is a lightweight handle to the attributes) - BasicFileAttributeView view = Files.getFileAttributeView(nativePath, BasicFileAttributeView.class); + final PosixFileAttributeView posixView; + final BasicFileAttributeView view; + if (this.posix) { + posixView = Files.getFileAttributeView(nativePath, PosixFileAttributeView.class); + view = posixView; + } else { + posixView = null; + view = Files.getFileAttributeView(nativePath, BasicFileAttributeView.class); + } - // 2. Prepare the times - FileTime newModifiedTime = FileTime.fromMillis(stat.getModifiedTime()); - FileTime newAccessedTime = FileTime.fromMillis(stat.getAccessedTime()); + if (options.contains(StatUpdateOption.PERMISSIONS)) { + final Set posixFilePermissions = Permissions.toPosixFilePermissions(stat.getPermissions()); + if (posixView != null) { + posixView.setPermissions(posixFilePermissions); + } else { + Permissions.setBasicPermissions(nativePath, posixFilePermissions); + } + } + + if (options.contains(StatUpdateOption.TIMESTAMPS)) { + // 2. Prepare the times + FileTime newModifiedTime = FileTime.fromMillis(stat.getModifiedTime()); + FileTime newAccessedTime = FileTime.fromMillis(stat.getAccessedTime()); - // 3. Update all three in ONE operation - // Signature: setTimes(lastModified, lastAccess, createTime) - // Pass 'null' if you want to leave a specific timestamp unchanged. - view.setTimes(newModifiedTime, newAccessedTime, null); + // 3. Update all three in ONE operation + // Signature: setTimes(lastModified, lastAccess, createTime) + // Pass 'null' if you want to leave a specific timestamp unchanged. + view.setTimes(newModifiedTime, newAccessedTime, null); + } } @Override diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatUpdateOption.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatUpdateOption.java new file mode 100644 index 0000000..b34d986 --- /dev/null +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatUpdateOption.java @@ -0,0 +1,9 @@ +package com.fizzed.jsync.vfs; + +public enum StatUpdateOption { + + OWNERSHIP, + PERMISSIONS, + TIMESTAMPS; + +} \ No newline at end of file diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileSystem.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileSystem.java index 7c521a9..37211f6 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileSystem.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileSystem.java @@ -4,6 +4,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.NoSuchFileException; +import java.util.Collection; import java.util.List; import java.util.Set; @@ -61,7 +62,7 @@ default VirtualPath exists(VirtualPath path) throws IOException { */ VirtualPath stat(VirtualPath path) throws IOException; - void updateStat(VirtualPath path, VirtualFileStat stats) throws IOException; + void updateStat(VirtualPath path, VirtualFileStat stats, Collection options) throws IOException; List ls(VirtualPath path) throws IOException; diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java index 0797c93..175ac15 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java @@ -1,5 +1,6 @@ package com.fizzed.jsync.vfs.util; +import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -72,7 +73,7 @@ public static Set toPosixFilePermissions(int permissions) { return perms; } - static public Set getPosixPermissions(Path path) throws IOException { + static public Set toBasicPermissions(Path path) throws IOException { // 2. On Windows: Synthesize permissions based on "DOS" attributes Set perms = new HashSet<>(); @@ -99,4 +100,32 @@ static public Set getPosixPermissions(Path path) throws IOE return perms; } + static public void setBasicPermissions(Path path, Set perms) { + File file = path.toFile(); + + // --- READ --- + if (perms.contains(PosixFilePermission.OTHERS_READ)) { + // If World needs read, open to Everyone (false = not owner only) + file.setReadable(true, false); + } else { + // Otherwise, set specifically for Owner (true = owner only) + // If OWNER_READ is missing, this sets readable to FALSE for owner + file.setReadable(perms.contains(PosixFilePermission.OWNER_READ), true); + } + + // --- WRITE --- + if (perms.contains(PosixFilePermission.OTHERS_WRITE)) { + file.setWritable(true, false); + } else { + file.setWritable(perms.contains(PosixFilePermission.OWNER_WRITE), true); + } + + // --- EXECUTE --- + if (perms.contains(PosixFilePermission.OTHERS_EXECUTE)) { + file.setExecutable(true, false); + } else { + file.setExecutable(perms.contains(PosixFilePermission.OWNER_EXECUTE), true); + } + } + } \ No newline at end of file diff --git a/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/util/PermissionsTest.java b/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/util/PermissionsTest.java new file mode 100644 index 0000000..cf39a03 --- /dev/null +++ b/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/util/PermissionsTest.java @@ -0,0 +1,11 @@ +package com.fizzed.jsync.vfs.util; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class PermissionsTest { + + + +} \ No newline at end of file From 7fb9420bb7641bd4b7b785a41cc86310d9adba67 Mon Sep 17 00:00:00 2001 From: Joe Lauer Date: Wed, 26 Nov 2025 21:42:32 -0500 Subject: [PATCH 6/9] Reworked permissions to ignore sending them to BASIC filesystems --- .../com/fizzed/jsync/engine/JsyncEngine.java | 138 ++++++++++++------ .../com/fizzed/jsync/engine/JsyncDemo.java | 4 +- .../jsync/sftp/SftpVirtualFileSystem.java | 8 +- .../jsync/vfs/LocalVirtualFileSystem.java | 6 +- .../vfs/{StatKind.java => StatModel.java} | 2 +- .../com/fizzed/jsync/vfs/VirtualFileStat.java | 4 + .../fizzed/jsync/vfs/VirtualFileSystem.java | 2 +- .../com/fizzed/jsync/vfs/util/IoHelper.java | 22 --- .../fizzed/jsync/vfs/util/Permissions.java | 28 ++++ .../jsync/vfs/LocalVirtualFileSystemTest.java | 2 +- .../jsync/vfs/util/PermissionsTest.java | 15 ++ 11 files changed, 151 insertions(+), 80 deletions(-) rename jsync-vfs/src/main/java/com/fizzed/jsync/vfs/{StatKind.java => StatModel.java} (71%) delete mode 100644 jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/IoHelper.java diff --git a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java index bac1997..655a4bc 100644 --- a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java +++ b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java @@ -1,6 +1,7 @@ package com.fizzed.jsync.engine; import com.fizzed.jsync.vfs.*; +import com.fizzed.jsync.vfs.util.Permissions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -10,6 +11,7 @@ import java.nio.file.Path; import java.util.*; +import static com.fizzed.jsync.vfs.util.Permissions.isOwnerPermissionEqual; import static java.util.Arrays.asList; import static java.util.stream.Collectors.toList; @@ -26,6 +28,8 @@ public class JsyncEngine { private boolean skipPermissions; private int maxFilesMaybeModifiedLimit; private List excludes; + // when running a sync + private Checksum negotiatedChecksum; public JsyncEngine() { this.eventHandler = new DefaultJsyncEventHandler(); @@ -206,8 +210,8 @@ public JsyncResult sync(VirtualFileSystem sourceVfs, String sourcePath, VirtualF // Negotiate checksum methods between source and target filesystems if necessary // - // find the best common checksum to use - final Checksum checksum = this.negotiateChecksum(sourceVfs, targetVfs); + // find the best common checksum + this.negotiatedChecksum = this.negotiateChecksum(sourceVfs, targetVfs); final long now = System.currentTimeMillis(); @@ -235,11 +239,11 @@ public JsyncResult sync(VirtualFileSystem sourceVfs, String sourcePath, VirtualF // as we process files, only a subset may require more advanced methods of detecting whether they were modified // since that process could be "expensive", we keep a list of files on source/target that we will defer processing // until we have a chance to do some bulk processing of checksums, etc. - this.syncDirectory(0, result, excludePaths, deferredFiles, sourceVfs, sourcePathAbsFinal, targetVfs, targetPathAbsFinal, checksum); + this.syncDirectory(0, result, excludePaths, deferredFiles, sourceVfs, sourcePathAbsFinal, targetVfs, targetPathAbsFinal); } else { // we are only syncing a file, we may need to do some more expensive checks to determine if it needs to be updated this.syncFile(result, deferredFiles, sourceVfs, sourcePathAbsFinal, targetVfs, targetPathAbsFinal); - this.syncDeferredFiles(result, deferredFiles, sourceVfs, targetVfs, checksum); + this.syncDeferredFiles(result, deferredFiles, sourceVfs, targetVfs); } final long timeMillis = System.currentTimeMillis() - now; @@ -271,7 +275,7 @@ protected void syncFile(JsyncResult result, List deferredFiles, } // detect what changes exists between source & target paths - final JsyncPathChanges changes = this.detectChanges(sourcePath, targetPath); + final JsyncPathChanges changes = this.detectChanges(sourceVfs, sourcePath, targetVfs, targetPath); // first, check if we should defer syncing the file till later on if (deferredFiles != null && changes.isDeferredProcessing(this.ignoreTimes)) { @@ -290,23 +294,23 @@ protected void syncFile(JsyncResult result, List deferredFiles, if (changes.isStatModified(this.skipPermissions)) { // stat will need updated if the file is either new, updated, or if only the perms/times need updating - this.updateStat(result, sourcePath, targetVfs, targetPath, changes, fileWasTransferred); + this.updateStat(result, sourceVfs, sourcePath, targetVfs, targetPath, changes, fileWasTransferred); } } - protected void syncDeferredFiles(JsyncResult result, List deferredFiles, VirtualFileSystem sourceVfs, VirtualFileSystem targetVfs, Checksum checksum) throws IOException { + protected void syncDeferredFiles(JsyncResult result, List deferredFiles, VirtualFileSystem sourceVfs, VirtualFileSystem targetVfs) throws IOException { // we need to calculate checksums for source and target files final List sourceFiles = deferredFiles.stream() .map(VirtualPathPair::getSource) .collect(toList()); - sourceVfs.checksums(checksum, sourceFiles); + sourceVfs.checksums(this.negotiatedChecksum, sourceFiles); final List targetFiles = deferredFiles.stream() .map(VirtualPathPair::getTarget) .collect(toList()); - targetVfs.checksums(checksum, targetFiles); + targetVfs.checksums(this.negotiatedChecksum, targetFiles); result.incrementChecksums(targetFiles.size()); @@ -318,7 +322,7 @@ protected void syncDeferredFiles(JsyncResult result, List defer deferredFiles.clear(); } - protected void syncDirectory(int level, JsyncResult result, List excludePaths, List deferredFiles, VirtualFileSystem sourceVfs, VirtualPath sourcePath, VirtualFileSystem targetVfs, VirtualPath targetPath, Checksum checksum) throws IOException { + protected void syncDirectory(int level, JsyncResult result, List excludePaths, List deferredFiles, VirtualFileSystem sourceVfs, VirtualPath sourcePath, VirtualFileSystem targetVfs, VirtualPath targetPath) throws IOException { // source needs to be a directory if (!sourcePath.isDirectory()) { @@ -330,7 +334,8 @@ protected void syncDirectory(int level, JsyncResult result, List ex log.warn("Type mismatch: source {} is a directory but target '{}' is a file!", sourcePath, targetPath); if (!this.force) { - throw new PathOverwriteException("Type mismatch: source " + sourcePath + " is a directory but target " + targetPath + " is a file. Either delete the target file manually or use the 'force' option to have jsync do it for you."); + throw new PathOverwriteException("Type mismatch: source " + sourcePath + " is a directory but target " + targetPath + " is a file. " + + "Either delete the target file manually or use the 'force' option to have jsync do it for you."); } // delete the target file @@ -343,7 +348,7 @@ protected void syncDirectory(int level, JsyncResult result, List ex } // detect what changes exists between source & target paths - final JsyncPathChanges changes = this.detectChanges(sourcePath, targetPath); + final JsyncPathChanges changes = this.detectChanges(sourceVfs, sourcePath, targetVfs, targetPath); if (changes.isMissing()) { this.createDirectory(result, targetVfs, targetPath, false, false); @@ -406,7 +411,7 @@ protected void syncDirectory(int level, JsyncResult result, List ex } if (sourceChildPath.isDirectory()) { - this.syncDirectory(level+1, result, excludePaths, deferredFiles, sourceVfs, sourceChildPath, targetVfs, targetChildPath, checksum); + this.syncDirectory(level+1, result, excludePaths, deferredFiles, sourceVfs, sourceChildPath, targetVfs, targetChildPath); } else { // NOTE: it's possible syncFile will "defer" processing if a checksum is required this.syncFile(result, deferredFiles, sourceVfs, sourceChildPath, targetVfs, targetChildPath); @@ -415,7 +420,7 @@ protected void syncDirectory(int level, JsyncResult result, List ex // handle any deferred files that need to be processed if (level == 0 || deferredFiles.size() >= this.maxFilesMaybeModifiedLimit) { - this.syncDeferredFiles(result, deferredFiles, sourceVfs, targetVfs, checksum); + this.syncDeferredFiles(result, deferredFiles, sourceVfs, targetVfs); } // handle any paths that need to be deleted @@ -444,11 +449,11 @@ protected void syncDirectory(int level, JsyncResult result, List ex // To successfully preserve directory timestamps, you must set the directory attributes after you have finished touching every single file inside that directory. if (changes.isStatModified(this.skipPermissions)) { // stat will need updated if the dir is new OR if the dir stats have changed - this.updateStat(result, sourcePath, targetVfs, targetPath, changes, changes.isMissing()); + this.updateStat(result, sourceVfs, sourcePath, targetVfs, targetPath, changes, changes.isMissing()); } } - protected JsyncPathChanges detectChanges(VirtualPath sourcePath, VirtualPath targetPath) throws IOException { + protected JsyncPathChanges detectChanges(VirtualFileSystem sourceVfs, VirtualPath sourcePath, VirtualFileSystem targetVfs, VirtualPath targetPath) throws IOException { // source "stats" MUST exist Objects.requireNonNull(sourcePath, "sourceFile cannot be null"); @@ -487,9 +492,23 @@ protected JsyncPathChanges detectChanges(VirtualPath sourcePath, VirtualPath tar timestamps = true; } - if (!this.skipPermissions && sourcePath.getStat().getPermissions() != targetPath.getStat().getPermissions()) { - log.trace("Source path {} perms {} != target perms {}", sourcePath, sourcePath.getStat().getPermissions(), targetPath.getStat().getPermissions()); - permissions = true; + if (!this.skipPermissions) { + // we only support syncing changes to POSIX targets, based on testing BASIC targets can break easily + // if posix -> posix we will compare entire permission value + // if basic -> posix we will only compare the owner permission bits + if (targetVfs.getStatModel() == StatModel.POSIX) { + if ((sourceVfs.getStatModel() == StatModel.POSIX && sourcePath.getStat().getPermissions() != targetPath.getStat().getPermissions()) + || (sourceVfs.getStatModel() == StatModel.BASIC && !isOwnerPermissionEqual(sourcePath.getStat().getPermissions(), targetPath.getStat().getPermissions()))) { + log.trace("Source path {} perms {} != target perms {}", sourcePath, sourcePath.getStat().getPermissions(), targetPath.getStat().getPermissions()); + permissions = true; + } + } + + /*if ((this.negotiatedStatModel == StatModel.POSIX && sourcePath.getStat().getPermissions() != targetPath.getStat().getPermissions()) + || (targetVfs == StatModel.BASIC && !isOwnerPermissionEqual(sourcePath.getStat().getPermissions(), targetPath.getStat().getPermissions()))) { + log.trace("Source path {} perms {} != target perms {}", sourcePath, sourcePath.getStat().getPermissions(), targetPath.getStat().getPermissions()); + permissions = true; + }*/ } // if we have "cksum" values on both sides, we can compare those @@ -545,7 +564,9 @@ protected void transferFile(JsyncResult result, VirtualFileSystem sourceVfs, Vir } } - protected void updateStat(JsyncResult result, VirtualPath sourcePath, VirtualFileSystem targetVfs, VirtualPath targetPath, JsyncPathChanges changes, boolean associatedWithFileModifiedOrDirCreated) throws IOException { + protected void updateStat(JsyncResult result, VirtualFileSystem sourceVfs, VirtualPath sourcePath, + VirtualFileSystem targetVfs, VirtualPath targetPath, JsyncPathChanges changes, boolean associatedWithFileModifiedOrDirCreated) throws IOException { + this.eventHandler.willUpdateStat(sourcePath, targetPath, changes, associatedWithFileModifiedOrDirCreated); final Set options = EnumSet.noneOf(StatUpdateOption.class); @@ -555,15 +576,42 @@ protected void updateStat(JsyncResult result, VirtualPath sourcePath, VirtualFil if (changes.isPermissionModified(this.skipPermissions)) { options.add(StatUpdateOption.PERMISSIONS); + + // if the source of perms is BASIC and the target currently has perms, we will only want to change the + // owner bits and retain the targets group & world bits + if (sourceVfs.getStatModel() == StatModel.BASIC && targetPath.getStat() != null) { + log.info("current target perms: {}", targetPath.getStat().getPermissionsOctal()); + int targetPerms = targetPath.getStat().getPermissions(); + int newTargetPerms = Permissions.mergeOwnerPermissions(sourcePath.getStat().getPermissions(), targetPerms); +// int newTargetPerms = Permissions.onlyOwnerPermissions(sourcePath.getStat().getPermissions()); + updateStat = updateStat.withPermissions(newTargetPerms); + } + + // posix permissions we don't need to change, but if basic we only want to send over the "owner" perm + // and merge it with the existing source + /*log.info("targetVfs stat model: {}", targetVfs.getStatModel()); + if (targetVfs.getStatModel() == StatModel.BASIC && targetPath.getStat() != null) { + log.info("current target perms: {}", targetPath.getStat().getPermissionsOctal()); + int targetPerms = targetPath.getStat().getPermissions(); + int newTargetPerms = Permissions.mergeOwnerPermissions(sourcePath.getStat().getPermissions(), targetPerms); +// int newTargetPerms = Permissions.onlyOwnerPermissions(sourcePath.getStat().getPermissions()); + updateStat = updateStat.withPermissions(newTargetPerms); + }*/ } if (changes.isTimestampsModified()) { options.add(StatUpdateOption.TIMESTAMPS); } - targetVfs.updateStat(targetPath, updateStat, options); + log.debug("Updating stats with options {} (perms {})", options, updateStat.getPermissionsOctal()); + + if (!options.isEmpty()) { + targetVfs.updateStat(targetPath, updateStat, options); - result.incrementStatsUpdated(); + result.incrementStatsUpdated(); + } else { + log.warn("updateStat was called, but nothing to update (options empty)"); + } } protected void createDirectory(JsyncResult result, VirtualFileSystem vfs, VirtualPath path, boolean verifyParentExists, boolean parents) throws IOException { @@ -636,41 +684,35 @@ protected Checksum negotiateChecksum(VirtualFileSystem sourceVfs, VirtualFileSys log.debug("Negotiating checksums supported on both source and target filesystems..."); // negotiate the checksums to use - Checksum checksum = null; - List sourceChecksumsSupported = new ArrayList<>(); - List targetChecksumsSupported = new ArrayList<>(); + final Set sourceChecksums = sourceVfs.getChecksumsSupported(); + final Set targetChecksums = targetVfs.getChecksumsSupported(); - for (Checksum preferredChecksum : this.preferredChecksums) { - log.debug("Detecting if {} checksum is supported on source/target", preferredChecksum); - - // check supported checksums, keep a tally of which are supported by both sides, so we can log them out - boolean sourceSupported = sourceVfs.isChecksumSupported(preferredChecksum); - - if (sourceSupported) { - sourceChecksumsSupported.add(preferredChecksum); - } - - boolean targetSupported = targetVfs.isChecksumSupported(preferredChecksum); + log.debug("Source filesystem supports checksums {}", sourceChecksums); + log.debug("Target filesystem supports checksums {}", targetChecksums); - if (targetSupported) { - targetChecksumsSupported.add(preferredChecksum); + for (Checksum preferredChecksum : this.preferredChecksums) { + if (sourceChecksums.contains(preferredChecksum) && targetChecksums.contains(preferredChecksum)) { + return preferredChecksum; } + } - log.debug("Detected {} checksum supported on source={}, target={}", preferredChecksum, sourceSupported, targetSupported); + throw new IOException("Unable to find a checksum that is supported by both source and target filesystems. " + + "Source filesystem " + sourceVfs.getName() + " supports checksums " + sourceChecksums + + " and target filesystem " + targetVfs.getName() + " supports checksums " + targetChecksums); + } - if (sourceSupported && targetSupported) { - checksum = preferredChecksum; - break; - } + protected StatModel negotiateStatModel(VirtualFileSystem sourceVfs, VirtualFileSystem targetVfs) throws IOException { + // do both support posix? + if (sourceVfs.getStatModel() == StatModel.POSIX && targetVfs.getStatModel() == StatModel.POSIX) { + return StatModel.POSIX; } - if (checksum == null) { - throw new IOException("Unable to find a checksum that is supported by both source and target filesystems. " + - "Source filesystem " + sourceVfs.getName() + " supports checksums " + sourceChecksumsSupported - + " and target filesystem " + targetVfs.getName() + " supports checksums " + targetChecksumsSupported); + // do either support basic? + if (sourceVfs.getStatModel() == StatModel.BASIC || targetVfs.getStatModel() == StatModel.BASIC) { + return StatModel.BASIC; } - return checksum; + throw new IOException("Unable to find a stat model that is supported by source and target filesystems"); } protected void sortPaths(List paths) { diff --git a/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncDemo.java b/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncDemo.java index 4492a4a..27c7582 100644 --- a/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncDemo.java +++ b/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncDemo.java @@ -28,8 +28,8 @@ static public void main(String[] args) throws Exception { final String targetDir = "test-sync"; // final VirtualVolume target = sftpVolume("bmh-build-x64-win11-1", targetDir); - final VirtualVolume target = sftpVolume("bmh-dev-x64-indy25-1", targetDir); -// final VirtualVolume target = sftpVolume("bmh-dev-x64-fedora43-1", targetDir); +// final VirtualVolume target = sftpVolume("bmh-dev-x64-indy25-1", targetDir); + final VirtualVolume target = sftpVolume("bmh-dev-x64-fedora43-1", targetDir); // final VirtualVolume target = sftpVolume("bmh-build-x64-freebsd15-1", targetDir); final JsyncResult result = new JsyncEngine() diff --git a/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java b/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java index 99c671c..b7a5335 100644 --- a/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java +++ b/jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java @@ -161,9 +161,13 @@ public boolean isRemote() { } @Override - public StatKind getStatKind() { + public StatModel getStatModel() { // for now, we'll claim full POSIX as the sftp server itself does the POSIX translation - return StatKind.POSIX; + if (this.windows) { + return StatModel.BASIC; + } else { + return StatModel.POSIX; + } } @Override diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java index 4504042..9321e48 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java @@ -68,11 +68,11 @@ public boolean isRemote() { } @Override - public StatKind getStatKind() { + public StatModel getStatModel() { if (this.posix) { - return StatKind.POSIX; + return StatModel.POSIX; } else { - return StatKind.BASIC; + return StatModel.BASIC; } } diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatKind.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatModel.java similarity index 71% rename from jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatKind.java rename to jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatModel.java index 6001e40..9d68e6d 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatKind.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/StatModel.java @@ -1,6 +1,6 @@ package com.fizzed.jsync.vfs; -public enum StatKind { +public enum StatModel { POSIX, BASIC diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileStat.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileStat.java index 0365d56..70a75c8 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileStat.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileStat.java @@ -20,6 +20,10 @@ public VirtualFileStat(VirtualFileType type, long size, long modifiedTime, long this.permissions = permissions; } + public VirtualFileStat withPermissions(int permissions) { + return new VirtualFileStat(type, size, modifiedTime, accessedTime, permissions); + } + public VirtualFileType getType() { return type; } diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileSystem.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileSystem.java index 37211f6..1c0f583 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileSystem.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/VirtualFileSystem.java @@ -18,7 +18,7 @@ public interface VirtualFileSystem extends AutoCloseable { boolean isCaseSensitive(); - StatKind getStatKind(); + StatModel getStatModel(); boolean isChecksumSupported(Checksum checksum) throws IOException; diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/IoHelper.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/IoHelper.java deleted file mode 100644 index f55a387..0000000 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/IoHelper.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.fizzed.jsync.vfs.util; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -public class IoHelper { - - static private final int BUFFER_SIZE = 8192; // 8KB - - static public long copy(InputStream input, OutputStream output) throws IOException { - long nread = 0L; - byte[] buf = new byte[BUFFER_SIZE]; - int n; - while ((n = input.read(buf)) > 0) { - output.write(buf, 0, n); - nread += n; - } - return nread; - } - -} \ No newline at end of file diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java index 175ac15..596f324 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java @@ -128,4 +128,32 @@ static public void setBasicPermissions(Path path, Set perms } } + static public int mergeOwnerPermissions(int sourcePerms, int targetPerms) { + // 0700 is the octal mask for Owner Read/Write/Execute + final int OWNER_MASK = 0700; + + // 1. Get only Owner bits from source + int ownerBits = sourcePerms & OWNER_MASK; + + // 2. Clear Owner bits from target, keeping Group, World, and special bits (SUID/SGID) + int targetWithoutOwner = targetPerms & ~OWNER_MASK; + + // 3. Combine them + return targetWithoutOwner | ownerBits; + } + + static public int onlyOwnerPermissions(int sourcePerms) { + // 0700 is the octal mask for Owner Read/Write/Execute + final int OWNER_MASK = 0700; + + // 1. Get only Owner bits from source + return sourcePerms & OWNER_MASK; + } + + static public boolean isOwnerPermissionEqual(int perm1, int perm2) { + final int OWNER_MASK = 0700; + // Isolate owner bits for both and check equality + return (perm1 & OWNER_MASK) == (perm2 & OWNER_MASK); + } + } \ No newline at end of file diff --git a/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java b/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java index 179e066..5827ca3 100644 --- a/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java +++ b/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/LocalVirtualFileSystemTest.java @@ -39,7 +39,7 @@ public void permissions() throws Exception { Path file = this.sourceDir.resolve("test.txt"); Files.write(file, "#!/bin/sh\necho hello".getBytes()); - if (this.defaultVfs.getStatKind() == StatKind.POSIX) { + if (this.defaultVfs.getStatModel() == StatModel.POSIX) { // set permissions to 755 (rwxr-xr-x) Files.setPosixFilePermissions(file, PosixFilePermissions.fromString("rwxr-xr-x")); diff --git a/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/util/PermissionsTest.java b/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/util/PermissionsTest.java index cf39a03..ac8fa2a 100644 --- a/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/util/PermissionsTest.java +++ b/jsync-vfs/src/test/java/com/fizzed/jsync/vfs/util/PermissionsTest.java @@ -2,10 +2,25 @@ import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.*; class PermissionsTest { + @Test + public void mergeOwnerPermissions() { + int source = 0755; + int target = 0640; + int result = Permissions.mergeOwnerPermissions(source, target); + assertThat(result).isEqualTo(0740); + } + + @Test + public void isOwnerPermissionEqual() { + assertThat(Permissions.isOwnerPermissionEqual(0755, 0655)).isFalse(); + assertThat(Permissions.isOwnerPermissionEqual(0755, 0744)).isTrue(); + assertThat(Permissions.isOwnerPermissionEqual(01755, 0744)).isTrue(); + } } \ No newline at end of file From 63177653748a288faaf1d034f87612441da1698a Mon Sep 17 00:00:00 2001 From: Joe Lauer Date: Wed, 26 Nov 2025 22:20:17 -0500 Subject: [PATCH 7/9] Working across windows and posix --- .../com/fizzed/jsync/engine/JsyncEngine.java | 28 ++++--------------- .../fizzed/jsync/engine/JsyncPathChanges.java | 8 +++--- .../fizzed/jsync/vfs/util/Permissions.java | 8 ------ 3 files changed, 9 insertions(+), 35 deletions(-) diff --git a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java index 655a4bc..7031f25 100644 --- a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java +++ b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java @@ -292,7 +292,7 @@ protected void syncFile(JsyncResult result, List deferredFiles, if (log.isDebugEnabled()) log.debug("Verified file {} ({})", targetPath, changes); } - if (changes.isStatModified(this.skipPermissions)) { + if (changes.isStatModified()) { // stat will need updated if the file is either new, updated, or if only the perms/times need updating this.updateStat(result, sourceVfs, sourcePath, targetVfs, targetPath, changes, fileWasTransferred); } @@ -447,7 +447,7 @@ protected void syncDirectory(int level, JsyncResult result, List ex // last step is to update the stat of the target dir // To successfully preserve directory timestamps, you must set the directory attributes after you have finished touching every single file inside that directory. - if (changes.isStatModified(this.skipPermissions)) { + if (changes.isStatModified()) { // stat will need updated if the dir is new OR if the dir stats have changed this.updateStat(result, sourceVfs, sourcePath, targetVfs, targetPath, changes, changes.isMissing()); } @@ -503,12 +503,6 @@ protected JsyncPathChanges detectChanges(VirtualFileSystem sourceVfs, VirtualPat permissions = true; } } - - /*if ((this.negotiatedStatModel == StatModel.POSIX && sourcePath.getStat().getPermissions() != targetPath.getStat().getPermissions()) - || (targetVfs == StatModel.BASIC && !isOwnerPermissionEqual(sourcePath.getStat().getPermissions(), targetPath.getStat().getPermissions()))) { - log.trace("Source path {} perms {} != target perms {}", sourcePath, sourcePath.getStat().getPermissions(), targetPath.getStat().getPermissions()); - permissions = true; - }*/ } // if we have "cksum" values on both sides, we can compare those @@ -574,40 +568,28 @@ protected void updateStat(JsyncResult result, VirtualFileSystem sourceVfs, Virtu // is involved, we only want to try and change the "owner" permission, and leave everything else as-is VirtualFileStat updateStat = sourcePath.getStat(); - if (changes.isPermissionModified(this.skipPermissions)) { + if (changes.isPermissionModified()) { options.add(StatUpdateOption.PERMISSIONS); // if the source of perms is BASIC and the target currently has perms, we will only want to change the // owner bits and retain the targets group & world bits if (sourceVfs.getStatModel() == StatModel.BASIC && targetPath.getStat() != null) { + // TODO: simplify this log.info("current target perms: {}", targetPath.getStat().getPermissionsOctal()); int targetPerms = targetPath.getStat().getPermissions(); int newTargetPerms = Permissions.mergeOwnerPermissions(sourcePath.getStat().getPermissions(), targetPerms); -// int newTargetPerms = Permissions.onlyOwnerPermissions(sourcePath.getStat().getPermissions()); updateStat = updateStat.withPermissions(newTargetPerms); } - - // posix permissions we don't need to change, but if basic we only want to send over the "owner" perm - // and merge it with the existing source - /*log.info("targetVfs stat model: {}", targetVfs.getStatModel()); - if (targetVfs.getStatModel() == StatModel.BASIC && targetPath.getStat() != null) { - log.info("current target perms: {}", targetPath.getStat().getPermissionsOctal()); - int targetPerms = targetPath.getStat().getPermissions(); - int newTargetPerms = Permissions.mergeOwnerPermissions(sourcePath.getStat().getPermissions(), targetPerms); -// int newTargetPerms = Permissions.onlyOwnerPermissions(sourcePath.getStat().getPermissions()); - updateStat = updateStat.withPermissions(newTargetPerms); - }*/ } if (changes.isTimestampsModified()) { options.add(StatUpdateOption.TIMESTAMPS); } - log.debug("Updating stats with options {} (perms {})", options, updateStat.getPermissionsOctal()); +// log.debug("Updating stats with options {} (perms {})", options, updateStat.getPermissionsOctal()); if (!options.isEmpty()) { targetVfs.updateStat(targetPath, updateStat, options); - result.incrementStatsUpdated(); } else { log.warn("updateStat was called, but nothing to update (options empty)"); diff --git a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncPathChanges.java b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncPathChanges.java index 4fc15b1..96e77c5 100644 --- a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncPathChanges.java +++ b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncPathChanges.java @@ -79,9 +79,9 @@ public boolean isDeferredProcessing(boolean ignoreTimes) { && checksum == null; } - public boolean isPermissionModified(boolean skipPermissions) { + public boolean isPermissionModified() { return this.missing - || (!skipPermissions && this.permissions); + || this.permissions; } public boolean isTimestampsModified() { @@ -89,10 +89,10 @@ public boolean isTimestampsModified() { || this.timestamps; } - public boolean isStatModified(boolean skipPermissions) { + public boolean isStatModified() { return this.missing || this.timestamps - || (!skipPermissions && this.permissions) + || this.permissions || this.ownership; } diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java index 596f324..555d2f1 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java @@ -142,14 +142,6 @@ static public int mergeOwnerPermissions(int sourcePerms, int targetPerms) { return targetWithoutOwner | ownerBits; } - static public int onlyOwnerPermissions(int sourcePerms) { - // 0700 is the octal mask for Owner Read/Write/Execute - final int OWNER_MASK = 0700; - - // 1. Get only Owner bits from source - return sourcePerms & OWNER_MASK; - } - static public boolean isOwnerPermissionEqual(int perm1, int perm2) { final int OWNER_MASK = 0700; // Isolate owner bits for both and check equality From aaedc8823512c68b2bd6dccc843869ad3e508e07 Mon Sep 17 00:00:00 2001 From: Joe Lauer Date: Wed, 26 Nov 2025 23:34:41 -0500 Subject: [PATCH 8/9] Support for ignoring paths --- .../engine/DefaultJsyncEventHandler.java | 9 ++- .../com/fizzed/jsync/engine/JsyncEngine.java | 77 ++++++++++++++++-- .../jsync/engine/JsyncEventHandler.java | 2 + .../fizzed/jsync/engine/JsyncEngineTest.java | 81 +++++++++++++++++++ .../jsync/vfs/LocalVirtualFileSystem.java | 19 ++--- .../fizzed/jsync/vfs/util/Permissions.java | 17 ++++ 6 files changed, 181 insertions(+), 24 deletions(-) diff --git a/jsync-engine/src/main/java/com/fizzed/jsync/engine/DefaultJsyncEventHandler.java b/jsync-engine/src/main/java/com/fizzed/jsync/engine/DefaultJsyncEventHandler.java index 902e17d..2e9af15 100644 --- a/jsync-engine/src/main/java/com/fizzed/jsync/engine/DefaultJsyncEventHandler.java +++ b/jsync-engine/src/main/java/com/fizzed/jsync/engine/DefaultJsyncEventHandler.java @@ -25,8 +25,13 @@ public void willEnd(VirtualFileSystem sourceVfs, VirtualPath sourcePath, Virtual } @Override - public void willExcludePath(VirtualPath targetPath) { - log.debug("Excluding path {}", targetPath); + public void willExcludePath(VirtualPath sourcePath) { + log.debug("Excluding path {}", sourcePath); + } + + @Override + public void willIgnorePath(VirtualPath sourcePath) { + log.debug("Ignoring path {}", sourcePath); } @Override diff --git a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java index 7031f25..b0a060d 100644 --- a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java +++ b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEngine.java @@ -28,8 +28,12 @@ public class JsyncEngine { private boolean skipPermissions; private int maxFilesMaybeModifiedLimit; private List excludes; + private List ignores; // when running a sync private Checksum negotiatedChecksum; + private List excludePaths; + private List ignoreSourcePaths; + private List ignoreTargetPaths; public JsyncEngine() { this.eventHandler = new DefaultJsyncEventHandler(); @@ -40,7 +44,6 @@ public JsyncEngine() { this.skipPermissions = false; this.preferredChecksums = new ArrayList<>(asList(Checksum.CK, Checksum.MD5)); this.maxFilesMaybeModifiedLimit = 256; - this.excludes = null; } public JsyncEventHandler getEventHandler() { @@ -134,6 +137,23 @@ public JsyncEngine addExclude(String exclude) { return this; } + public List getIgnores() { + return ignores; + } + + public JsyncEngine setIgnores(List ignores) { + this.ignores = ignores; + return this; + } + + public JsyncEngine addIgnore(String ignore) { + if (this.ignores == null) { + this.ignores = new ArrayList<>(); + } + this.ignores.add(ignore); + return this; + } + public JsyncResult sync(Path sourcePath, Path targetPath, JsyncMode mode) throws IOException { // local -> local final LocalVirtualFileSystem localVfs = LocalVirtualFileSystem.open(); @@ -213,6 +233,30 @@ public JsyncResult sync(VirtualFileSystem sourceVfs, String sourcePath, VirtualF // find the best common checksum this.negotiatedChecksum = this.negotiateChecksum(sourceVfs, targetVfs); + // build exclude and ignore paths + if (this.excludes != null) { + this.excludePaths = this.excludes.stream() + .map(VirtualPath::parse) + .map(sourcePathAbsFinal::resolve) + .collect(toList()); + } else { + this.excludePaths = Collections.emptyList(); + } + + if (this.ignores != null) { + this.ignoreSourcePaths = this.ignores.stream() + .map(VirtualPath::parse) + .map(sourcePathAbsFinal::resolve) + .collect(toList()); + this.ignoreTargetPaths = this.ignores.stream() + .map(VirtualPath::parse) + .map(targetPathAbsFinal::resolve) + .collect(toList()); + } else { + this.ignoreSourcePaths = Collections.emptyList(); + this.ignoreTargetPaths = Collections.emptyList(); + } + final long now = System.currentTimeMillis(); @@ -239,7 +283,7 @@ public JsyncResult sync(VirtualFileSystem sourceVfs, String sourcePath, VirtualF // as we process files, only a subset may require more advanced methods of detecting whether they were modified // since that process could be "expensive", we keep a list of files on source/target that we will defer processing // until we have a chance to do some bulk processing of checksums, etc. - this.syncDirectory(0, result, excludePaths, deferredFiles, sourceVfs, sourcePathAbsFinal, targetVfs, targetPathAbsFinal); + this.syncDirectory(0, result, deferredFiles, sourceVfs, sourcePathAbsFinal, targetVfs, targetPathAbsFinal); } else { // we are only syncing a file, we may need to do some more expensive checks to determine if it needs to be updated this.syncFile(result, deferredFiles, sourceVfs, sourcePathAbsFinal, targetVfs, targetPathAbsFinal); @@ -322,7 +366,7 @@ protected void syncDeferredFiles(JsyncResult result, List defer deferredFiles.clear(); } - protected void syncDirectory(int level, JsyncResult result, List excludePaths, List deferredFiles, VirtualFileSystem sourceVfs, VirtualPath sourcePath, VirtualFileSystem targetVfs, VirtualPath targetPath) throws IOException { + protected void syncDirectory(int level, JsyncResult result, List deferredFiles, VirtualFileSystem sourceVfs, VirtualPath sourcePath, VirtualFileSystem targetVfs, VirtualPath targetPath) throws IOException { // source needs to be a directory if (!sourcePath.isDirectory()) { @@ -365,15 +409,23 @@ protected void syncDirectory(int level, JsyncResult result, List ex List sourceChildPaths = sourceVfs.ls(sourcePath).stream() // apply filter to source files if they are on the exclude list .filter(v -> { - //log.info("Checking for exlcude of path {} with excludes {}", v, excludePaths); - for (VirtualPath excludePath : excludePaths) { - if (v.startsWith(excludePath)) { + for (VirtualPath p : this.excludePaths) { + if (v.startsWith(p)) { this.eventHandler.willExcludePath(v); return false; } } return true; }) + .filter(v -> { + for (VirtualPath p : this.ignoreSourcePaths) { + if (v.startsWith(p)) { + this.eventHandler.willIgnorePath(v); + return false; + } + } + return true; + }) // apply filter to excluding non-regular files (such as symlinks) .filter(v -> { switch (v.getStat().getType()) { @@ -389,7 +441,16 @@ protected void syncDirectory(int level, JsyncResult result, List ex }) .collect(toList()); - final List targetChildPaths = targetVfs.ls(targetPath); + final List targetChildPaths = targetVfs.ls(targetPath).stream() + .filter(v -> { + for (VirtualPath p : this.ignoreTargetPaths) { + if (v.startsWith(p)) { + return false; + } + } + return true; + }) + .collect(toList()); // its better to work with all dirs first, then files, so we sort the files before we process them this.sortPaths(sourceChildPaths); @@ -411,7 +472,7 @@ protected void syncDirectory(int level, JsyncResult result, List ex } if (sourceChildPath.isDirectory()) { - this.syncDirectory(level+1, result, excludePaths, deferredFiles, sourceVfs, sourceChildPath, targetVfs, targetChildPath); + this.syncDirectory(level+1, result, deferredFiles, sourceVfs, sourceChildPath, targetVfs, targetChildPath); } else { // NOTE: it's possible syncFile will "defer" processing if a checksum is required this.syncFile(result, deferredFiles, sourceVfs, sourceChildPath, targetVfs, targetChildPath); diff --git a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEventHandler.java b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEventHandler.java index 30b7da2..d9abbe5 100644 --- a/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEventHandler.java +++ b/jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEventHandler.java @@ -15,6 +15,8 @@ public interface JsyncEventHandler { void willExcludePath(VirtualPath targetPath); + void willIgnorePath(VirtualPath targetPath); + void willCreateDirectory(VirtualPath targetPath, boolean recursively); void willDeleteDirectory(VirtualPath targetPath, boolean recursively); diff --git a/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncEngineTest.java b/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncEngineTest.java index 6a837ea..27552b3 100644 --- a/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncEngineTest.java +++ b/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncEngineTest.java @@ -4,6 +4,7 @@ import com.fizzed.crux.util.Resources; import com.fizzed.jsync.vfs.ParentDirectoryMissingException; import com.fizzed.jsync.vfs.PathOverwriteException; +import com.fizzed.jsync.vfs.util.Permissions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.Logger; @@ -19,6 +20,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.within; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assumptions.assumeTrue; class JsyncEngineTest { static private final Logger log = LoggerFactory.getLogger(JsyncEngineTest.class); @@ -502,6 +504,53 @@ public void syncDirTimestamp() throws Exception { assertThat(modifiedTime(targetADir)).isCloseTo(ts, within(2, ChronoUnit.SECONDS)); } + @Test + public void syncFilePermission() throws Exception { + assumeTrue(Permissions.isPosix()); + + Path sourceAFile = this.syncSourceDir.resolve("a.txt"); + Files.write(sourceAFile, "hello".getBytes()); + + Path targetAFile = this.syncTargetDir.resolve("a.txt"); + Files.write(targetAFile, "hello".getBytes()); + + Permissions.setPosixInt(sourceAFile, 0755); + Permissions.setPosixInt(targetAFile, 0644); + + JsyncResult result = new JsyncEngine() + .sync(sourceAFile, this.syncTargetDir, JsyncMode.NEST); + + assertThat(Permissions.getPosixInt(targetAFile)).isEqualTo(0755); + assertThat(result.getStatsUpdated()).isEqualTo(1); + assertThat(result.getFilesCreated()).isEqualTo(0); + assertThat(result.getFilesDeleted()).isEqualTo(0); + assertThat(result.getFilesUpdated()).isEqualTo(0); + } + + @Test + public void syncFileSkipPermissions() throws Exception { + assumeTrue(Permissions.isPosix()); + + Path sourceAFile = this.syncSourceDir.resolve("a.txt"); + Files.write(sourceAFile, "hello".getBytes()); + + Path targetAFile = this.syncTargetDir.resolve("a.txt"); + Files.write(targetAFile, "hello".getBytes()); + + Permissions.setPosixInt(sourceAFile, 0755); + Permissions.setPosixInt(targetAFile, 0644); + + JsyncResult result = new JsyncEngine() + .setSkipPermissions(true) + .sync(sourceAFile, this.syncTargetDir, JsyncMode.NEST); + + assertThat(Permissions.getPosixInt(targetAFile)).isEqualTo(0644); + assertThat(result.getStatsUpdated()).isEqualTo(0); + assertThat(result.getFilesCreated()).isEqualTo(0); + assertThat(result.getFilesDeleted()).isEqualTo(0); + assertThat(result.getFilesUpdated()).isEqualTo(0); + } + @Test public void syncExcludeDir() throws Exception { Path sourceADir = this.syncSourceDir.resolve("a"); @@ -547,4 +596,36 @@ public void syncExcludeNonRegularFiles() throws Exception { assertThat(this.syncTargetDir.resolve("a/b.txt")).hasContent("hello"); } + @Test + public void syncIgnoreDir() throws Exception { + Path sourceADir = this.syncSourceDir.resolve("a"); + Path sourceBFile = this.syncSourceDir.resolve("a/b.txt"); + this.writeFile(sourceBFile, "hello"); + + Path sourceBDir = this.syncSourceDir.resolve("b"); + Path sourceCFile = this.syncSourceDir.resolve("b/c.txt"); + Path sourceDFile = this.syncSourceDir.resolve("b/d.txt"); + this.writeFile(sourceCFile, "hello"); + this.writeFile(sourceDFile, "hello"); + + // with directory "a" fully excluded, it actually should be deleted with --exclude + Path targetADir = this.syncTargetDir.resolve("a"); + Path targetBFile = this.syncTargetDir.resolve("a/b.txt"); + this.writeFile(targetBFile, "hello"); + Path targetBDir = this.syncTargetDir.resolve("b"); + Path targetCDir = this.syncTargetDir.resolve("c"); + Path targetEFile = this.syncTargetDir.resolve("c/e.txt"); + this.writeFile(targetEFile, "hello"); + + final JsyncResult result = new JsyncEngine() + .addIgnore("c") + .addIgnore("b") + .setDelete(true) + .sync(this.syncSourceDir, this.syncTargetDir, JsyncMode.MERGE); + + assertThat(targetADir).isDirectory(); + assertThat(targetBDir).doesNotExist(); + assertThat(targetEFile).isRegularFile(); + } + } \ No newline at end of file diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java index 9321e48..cad5320 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/LocalVirtualFileSystem.java @@ -39,18 +39,11 @@ static public LocalVirtualFileSystem open(Path workingDir) { final VirtualPath pwd = VirtualPath.parse(currentWorkingDir.toString(), true); - final boolean isPosixAttributes = FileSystems.getDefault() - .supportedFileAttributeViews() - .contains("posix"); + final boolean posix = Permissions.isPosix(); - log.debug("Detected filesystem {} has pwd={}, posix={}", name, pwd, isPosixAttributes); + log.debug("Detected filesystem {} with pwd={}, posix={}, caseSensitive={}", name, pwd, posix, posix); - // everything is case-sensitive except windows - final boolean caseSensitive = !System.getProperty("os.name").toLowerCase().contains("windows"); - - log.debug("Detected filesystem {} is case-sensitive={}", name, caseSensitive); - - return new LocalVirtualFileSystem(name, pwd, caseSensitive, isPosixAttributes); + return new LocalVirtualFileSystem(name, pwd, posix, posix); } @Override @@ -127,8 +120,8 @@ protected VirtualPath withStat(VirtualPath path) throws IOException { perms = Permissions.toPosixInt(posixAttrs.permissions()); } else { // use basic permissions, usually ends up being 700 from what I can gather - final Set simulatedPosixPermissions = Permissions.toBasicPermissions(nativePath); - perms = Permissions.toPosixInt(simulatedPosixPermissions); + final Set basicPermissions = Permissions.toBasicPermissions(nativePath); + perms = Permissions.toPosixInt(basicPermissions); } final VirtualFileStat stat = new VirtualFileStat(type, size, modifiedTime, accessedTime, perms); @@ -136,8 +129,6 @@ protected VirtualPath withStat(VirtualPath path) throws IOException { return new VirtualPath(path.getParentPath(), path.getName(), type == VirtualFileType.DIR, stat); } - - @Override public VirtualPath stat(VirtualPath path) throws IOException { return this.withStat(path); diff --git a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java index 555d2f1..a872e3a 100644 --- a/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java +++ b/jsync-vfs/src/main/java/com/fizzed/jsync/vfs/util/Permissions.java @@ -2,6 +2,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.PosixFilePermission; @@ -11,6 +12,22 @@ public class Permissions { + static public boolean isPosix() { + return FileSystems.getDefault() + .supportedFileAttributeViews() + .contains("posix"); + } + + static public int getPosixInt(Path path) throws IOException { + final Set permissions = Files.getPosixFilePermissions(path); + return toPosixInt(permissions); + } + + static public void setPosixInt(Path path, int perms) throws IOException { + final Set permissions = toPosixFilePermissions(perms); + Files.setPosixFilePermissions(path, permissions); + } + /** * Converts a set of {@link PosixFilePermission} to its corresponding POSIX integer representation. * The resulting integer is a bitmask representing the file permission mode. From e04248cb34403e413f336e5e2ba1bd973eaf2707 Mon Sep 17 00:00:00 2001 From: Joe Lauer Date: Thu, 27 Nov 2025 08:03:00 -0500 Subject: [PATCH 9/9] Unit test for ignoring paths --- .../java/com/fizzed/jsync/engine/JsyncEngineTest.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncEngineTest.java b/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncEngineTest.java index 27552b3..d96e559 100644 --- a/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncEngineTest.java +++ b/jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncEngineTest.java @@ -604,7 +604,7 @@ public void syncIgnoreDir() throws Exception { Path sourceBDir = this.syncSourceDir.resolve("b"); Path sourceCFile = this.syncSourceDir.resolve("b/c.txt"); - Path sourceDFile = this.syncSourceDir.resolve("b/d.txt"); + Path sourceDFile = this.syncSourceDir.resolve("b/sub/sub2/d.txt"); this.writeFile(sourceCFile, "hello"); this.writeFile(sourceDFile, "hello"); @@ -614,17 +614,18 @@ public void syncIgnoreDir() throws Exception { this.writeFile(targetBFile, "hello"); Path targetBDir = this.syncTargetDir.resolve("b"); Path targetCDir = this.syncTargetDir.resolve("c"); - Path targetEFile = this.syncTargetDir.resolve("c/e.txt"); + Path targetEFile = this.syncTargetDir.resolve("c/sub/sub2/e.txt"); this.writeFile(targetEFile, "hello"); final JsyncResult result = new JsyncEngine() - .addIgnore("c") - .addIgnore("b") + .addIgnore("c") // target dir should not be deleted + .addIgnore("b") // b should not be synced to target .setDelete(true) .sync(this.syncSourceDir, this.syncTargetDir, JsyncMode.MERGE); assertThat(targetADir).isDirectory(); assertThat(targetBDir).doesNotExist(); + assertThat(targetCDir).exists(); assertThat(targetEFile).isRegularFile(); }