Feature/perms

jsync-engine/src/main/java/com/fizzed/jsync/engine/DefaultJsyncEventHandler.java

@@ -25,8 +25,13 @@ public void willEnd(VirtualFileSystem sourceVfs, VirtualPath sourcePath, Virtual
     }
 
     @Override
-    public void willExcludePath(VirtualPath targetPath) {
-        log.debug("Excluding path {}", targetPath);
+    public void willExcludePath(VirtualPath sourcePath) {
+        log.debug("Excluding path {}", sourcePath);
+    }
+
+    @Override
+    public void willIgnorePath(VirtualPath sourcePath) {
+        log.debug("Ignoring path {}", sourcePath);
     }
 
     @Override

jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncEventHandler.java

@@ -15,6 +15,8 @@ public interface JsyncEventHandler {
 
     void willExcludePath(VirtualPath targetPath);
 
+    void willIgnorePath(VirtualPath targetPath);
+
     void willCreateDirectory(VirtualPath targetPath, boolean recursively);
 
     void willDeleteDirectory(VirtualPath targetPath, boolean recursively);

jsync-engine/src/main/java/com/fizzed/jsync/engine/JsyncPathChanges.java

@@ -79,6 +79,16 @@ public boolean isDeferredProcessing(boolean ignoreTimes) {
             && checksum == null;
     }
 
+    public boolean isPermissionModified() {
+        return this.missing
+            || this.permissions;
+    }
+
+    public boolean isTimestampsModified() {
+        return this.missing
+            || this.timestamps;
+    }
+
     public boolean isStatModified() {
         return this.missing
             || this.timestamps

jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncDemo.java

@@ -17,7 +17,7 @@ static public void main(String[] args) throws Exception {
 //        final String sourceDir = Paths.get("/home/jjlauer/test-sync").toString();
 //        final VirtualVolume source = localVolume(Paths.get("/home/jjlauer/workspace/third-party/jsch"));
 //        final String sourceDir = Paths.get("/home/jjlauer/workspace/third-party/coredns").toString();
-        final VirtualVolume source = localVolume(Paths.get("/home/jjlauer/workspace/third-party/nats.java"));
+        final VirtualVolume source = localVolume(Paths.get( System.getProperty("user.home") + "/workspace/third-party/nats.java"));
 //        final VirtualVolume source = localVolume(Paths.get("/home/jjlauer/workspace/third-party/tokyocabinet-1.4.48"));
 //        final String sourceDir = Paths.get("C:\\Users\\jjlauer\\test-sync").toString();
 //        final String sourceDir = Paths.get("C:\\Users\\jjlauer\\workspace\\third-party\\tokyocabinet-1.4.48").toString();
@@ -28,8 +28,8 @@ static public void main(String[] args) throws Exception {
         final String targetDir = "test-sync";
 
 //        final VirtualVolume target = sftpVolume("bmh-build-x64-win11-1", targetDir);
-        final VirtualVolume target = sftpVolume("bmh-dev-x64-indy25-1", targetDir);
-//        final VirtualVolume target = sftpVolume("bmh-dev-x64-fedora43-1", targetDir);
+//        final VirtualVolume target = sftpVolume("bmh-dev-x64-indy25-1", targetDir);
+        final VirtualVolume target = sftpVolume("bmh-dev-x64-fedora43-1", targetDir);
 //        final VirtualVolume target = sftpVolume("bmh-build-x64-freebsd15-1", targetDir);
 
         final JsyncResult result = new JsyncEngine()
@@ -39,6 +39,7 @@ static public void main(String[] args) throws Exception {
             .setDelete(true)
             .setParents(true)
             .setForce(true)
+            //.setSkipPermissions(true)
             //.setIgnoreTimes(true)
             //.setMaxFilesMaybeModifiedLimit(256)
 //            .setProgress(true)

jsync-engine/src/test/java/com/fizzed/jsync/engine/JsyncEngineTest.java

@@ -4,6 +4,7 @@
 import com.fizzed.crux.util.Resources;
 import com.fizzed.jsync.vfs.ParentDirectoryMissingException;
 import com.fizzed.jsync.vfs.PathOverwriteException;
+import com.fizzed.jsync.vfs.util.Permissions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.slf4j.Logger;
@@ -19,6 +20,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.within;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 class JsyncEngineTest {
     static private final Logger log = LoggerFactory.getLogger(JsyncEngineTest.class);
@@ -502,6 +504,53 @@ public void syncDirTimestamp() throws Exception {
         assertThat(modifiedTime(targetADir)).isCloseTo(ts, within(2, ChronoUnit.SECONDS));
     }
 
+    @Test
+    public void syncFilePermission() throws Exception {
+        assumeTrue(Permissions.isPosix());
+
+        Path sourceAFile = this.syncSourceDir.resolve("a.txt");
+        Files.write(sourceAFile, "hello".getBytes());
+
+        Path targetAFile = this.syncTargetDir.resolve("a.txt");
+        Files.write(targetAFile, "hello".getBytes());
+
+        Permissions.setPosixInt(sourceAFile, 0755);
+        Permissions.setPosixInt(targetAFile, 0644);
+
+        JsyncResult result = new JsyncEngine()
+            .sync(sourceAFile, this.syncTargetDir, JsyncMode.NEST);
+
+        assertThat(Permissions.getPosixInt(targetAFile)).isEqualTo(0755);
+        assertThat(result.getStatsUpdated()).isEqualTo(1);
+        assertThat(result.getFilesCreated()).isEqualTo(0);
+        assertThat(result.getFilesDeleted()).isEqualTo(0);
+        assertThat(result.getFilesUpdated()).isEqualTo(0);
+    }
+
+    @Test
+    public void syncFileSkipPermissions() throws Exception {
+        assumeTrue(Permissions.isPosix());
+
+        Path sourceAFile = this.syncSourceDir.resolve("a.txt");
+        Files.write(sourceAFile, "hello".getBytes());
+
+        Path targetAFile = this.syncTargetDir.resolve("a.txt");
+        Files.write(targetAFile, "hello".getBytes());
+
+        Permissions.setPosixInt(sourceAFile, 0755);
+        Permissions.setPosixInt(targetAFile, 0644);
+
+        JsyncResult result = new JsyncEngine()
+            .setSkipPermissions(true)
+            .sync(sourceAFile, this.syncTargetDir, JsyncMode.NEST);
+
+        assertThat(Permissions.getPosixInt(targetAFile)).isEqualTo(0644);
+        assertThat(result.getStatsUpdated()).isEqualTo(0);
+        assertThat(result.getFilesCreated()).isEqualTo(0);
+        assertThat(result.getFilesDeleted()).isEqualTo(0);
+        assertThat(result.getFilesUpdated()).isEqualTo(0);
+    }
+
     @Test
     public void syncExcludeDir() throws Exception {
         Path sourceADir = this.syncSourceDir.resolve("a");
@@ -547,4 +596,37 @@ public void syncExcludeNonRegularFiles() throws Exception {
         assertThat(this.syncTargetDir.resolve("a/b.txt")).hasContent("hello");
     }
 
+    @Test
+    public void syncIgnoreDir() throws Exception {
+        Path sourceADir = this.syncSourceDir.resolve("a");
+        Path sourceBFile = this.syncSourceDir.resolve("a/b.txt");
+        this.writeFile(sourceBFile, "hello");
+
+        Path sourceBDir = this.syncSourceDir.resolve("b");
+        Path sourceCFile = this.syncSourceDir.resolve("b/c.txt");
+        Path sourceDFile = this.syncSourceDir.resolve("b/sub/sub2/d.txt");
+        this.writeFile(sourceCFile, "hello");
+        this.writeFile(sourceDFile, "hello");
+
+        // with directory "a" fully excluded, it actually should be deleted with --exclude
+        Path targetADir = this.syncTargetDir.resolve("a");
+        Path targetBFile = this.syncTargetDir.resolve("a/b.txt");
+        this.writeFile(targetBFile, "hello");
+        Path targetBDir = this.syncTargetDir.resolve("b");
+        Path targetCDir = this.syncTargetDir.resolve("c");
+        Path targetEFile = this.syncTargetDir.resolve("c/sub/sub2/e.txt");
+        this.writeFile(targetEFile, "hello");
+
+        final JsyncResult result = new JsyncEngine()
+            .addIgnore("c")     // target dir should not be deleted
+            .addIgnore("b")     // b should not be synced to target
+            .setDelete(true)
+            .sync(this.syncSourceDir, this.syncTargetDir, JsyncMode.MERGE);
+
+        assertThat(targetADir).isDirectory();
+        assertThat(targetBDir).doesNotExist();
+        assertThat(targetCDir).exists();
+        assertThat(targetEFile).isRegularFile();
+    }
+
 }

jsync-sftp/src/main/java/com/fizzed/jsync/sftp/SftpVirtualFileSystem.java

@@ -17,6 +17,8 @@
 import java.util.*;
 import java.util.concurrent.CountDownLatch;
 
+import static java.util.Arrays.asList;
+
 public class SftpVirtualFileSystem extends AbstractVirtualFileSystem {
     static private final Logger log = LoggerFactory.getLogger(SftpVirtualFileSystem.class);
 
@@ -109,27 +111,27 @@ static public SftpVirtualFileSystem open(Session ssh, boolean closeSsh) throws I
     static public SftpVirtualFileSystem open(Session ssh, boolean closeSsh, ChannelSftp sftp, boolean closeSftp) throws IOException {
         final String name = ssh.getHost();
 
-        log.info("Opening filesystem {}...", name);
+        log.debug("Opening filesystem {}...", name);
 
-        final String pwd2;
+        final String pwdRaw;
         try {
-            pwd2 = sftp.pwd();
+            pwdRaw = sftp.pwd();
         } catch (SftpException e) {
             throw toIOException(e);
         }
 
-        final VirtualPath pwd = VirtualPath.parse(pwd2, true);
+        final VirtualPath pwd = VirtualPath.parse(pwdRaw, true);
 
-        log.debug("Detected filesystem {} has pwd {}", name, pwd);
+        log.debug("Detected pwd {}", pwd);
 
         boolean windows = false;
 
         // this is likely a "windows" system if the 2nd char is :
-        if (pwd2.length() > 2 && pwd2.charAt(2) == ':') {
+        if (pwdRaw.length() > 2 && pwdRaw.charAt(2) == ':') {
             // TODO: should we confirm by running a command that exists only windows to confirm?
             // for now we'll just assume it is
             windows = true;
-            log.debug("Detected filesystem {} is running on windows (changes standard checksums, native filepaths, case sensitivity, etc.)", name);
+            log.debug("Detected windows-based sftp server");
         }
 
         return new SftpVirtualFileSystem(name, pwd, ssh, closeSsh, sftp, closeSftp, windows);
@@ -154,12 +156,22 @@ public void close() throws Exception {
     }
 
     @Override
-    public String toString() {
-        return this.getName();
+    public boolean isRemote() {
+        return true;
+    }
+
+    @Override
+    public StatModel getStatModel() {
+        // for now, we'll claim full POSIX as the sftp server itself does the POSIX translation
+        if (this.windows) {
+            return StatModel.BASIC;
+        } else {
+            return StatModel.POSIX;
+        }
     }
 
     @Override
-    public boolean isSupported(Checksum checksum) throws IOException {
+    public boolean isChecksumSupported(Checksum checksum) throws IOException {
         if (this.windows) {
             switch (checksum) {
                 case MD5:
@@ -189,6 +201,37 @@ public boolean isSupported(Checksum checksum) throws IOException {
         }
     }
 
+    @Override
+    protected List<Checksum> doDetectChecksums() throws IOException {
+        // windows is easy, return what powershell supports
+        if (this.windows) {
+            return asList(Checksum.MD5, Checksum.SHA1);
+        }
+
+        // otherwise, we are on posix and we can actually check whether these would work or not
+        final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+        // check if anything is supported
+        int exitValue = this.exec(this.ssh, "which cksum md5sum sha1sum", null, baos, null);
+        if (exitValue != 0) {
+            return Collections.emptyList();
+        }
+
+        final String output = baos.toString(StandardCharsets.UTF_8.name());
+        final List<Checksum> checksums = new ArrayList<>();
+        if (output.contains("cksum")) {
+            checksums.add(Checksum.CK);
+        }
+        if (output.contains("md5sum")) {
+            checksums.add(Checksum.MD5);
+        }
+        if (output.contains("sha1sum")) {
+            checksums.add(Checksum.SHA1);
+        }
+
+        return checksums;
+    }
+
     public int getMaxCommandLength() {
         return maxCommandLength;
     }
@@ -198,10 +241,12 @@ public SftpVirtualFileSystem setMaxCommandLength(int maxCommandLength) {
         return this;
     }
 
-    private VirtualPath toVirtualPathWithStats(VirtualPath path, SftpATTRS attrs) throws IOException {
+    protected VirtualPath withStats(VirtualPath path, SftpATTRS attrs) throws IOException {
         final long size = attrs.getSize();
         final long modifiedTime = attrs.getMTime() * 1000L;
         final long accessedTime = attrs.getATime() * 1000L;
+        // sftp stuffs extra stuff like the file type in the permissions value, we don't care about it
+        final int perms = attrs.getPermissions() & 07777;
 
         final VirtualFileType type;
         if (attrs.isDir()) {
@@ -214,44 +259,40 @@ private VirtualPath toVirtualPathWithStats(VirtualPath path, SftpATTRS attrs) th
             type = VirtualFileType.OTHER;
         }
 
-        final VirtualFileStat stat = new VirtualFileStat(type, size, modifiedTime, accessedTime);
+        final VirtualFileStat stat = new VirtualFileStat(type, size, modifiedTime, accessedTime, perms);
 
         return new VirtualPath(path.getParentPath(), path.getName(), type == VirtualFileType.DIR, stat);
     }
 
-    @Override
-    public boolean isRemote() {
-        return true;
-    }
-
     @Override
     public VirtualPath stat(VirtualPath path) throws IOException {
         try {
             final SftpATTRS attrs = this.sftp.lstat(path.toString());
 
-            return this.toVirtualPathWithStats(path, attrs);
+            return this.withStats(path, attrs);
         } catch (SftpException e) {
             throw toIOException(e);
         }
     }
 
     @Override
-    public void updateStat(VirtualPath path, VirtualFileStat stat) throws IOException {
+    public void updateStat(VirtualPath path, VirtualFileStat stat, Collection<StatUpdateOption> options) throws IOException {
         try {
+            final SftpATTRS attrs = SftpATTRSAccessor.createSftpATTRS();
+
             // TODO: are we updating uid/gid?
             Integer uid = null;
             Integer gid = null;
 
-            // TODO: are we updating permissions?
-            Integer perms = null;
-
-            // are we updating mtime/atime?d
-            int mtime = (int)(stat.getModifiedTime()/1000);
-            int atime = (int)(stat.getAccessedTime()/1000);
-
-            final SftpATTRS attrs = SftpATTRSAccessor.createSftpATTRS();
+            if (options.contains(StatUpdateOption.PERMISSIONS)) {
+                attrs.setPERMISSIONS(stat.getPermissions());
+            }
 
-            attrs.setACMODTIME(atime, mtime);
+            if (options.contains(StatUpdateOption.TIMESTAMPS)) {
+                int mtime = (int) (stat.getModifiedTime() / 1000);
+                int atime = (int) (stat.getAccessedTime() / 1000);
+                attrs.setACMODTIME(atime, mtime);
+            }
 
             this.sftp.setStat(path.toString(), attrs);
         } catch (SftpException e) {
@@ -277,7 +318,7 @@ public List<VirtualPath> ls(VirtualPath path) throws IOException {
 
             // dir true/false doesn't matter, stats call next will correct it
             VirtualPath childPathWithoutStats = path.resolve(entry.getFilename(), false);
-            VirtualPath childPath = this.toVirtualPathWithStats(childPathWithoutStats, entry.getAttrs());
+            VirtualPath childPath = this.withStats(childPathWithoutStats, entry.getAttrs());
             childPaths.add(childPath);
         }
 

jsync-sftp/src/test/java/com/fizzed/jsync/sftp/SftpVirtualFileSystemDemo.java

@@ -0,0 +1,27 @@
+package com.fizzed.jsync.sftp;
+
+import com.fizzed.jsync.vfs.VirtualPath;
+import com.fizzed.jsync.vfs.VirtualVolume;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.file.Paths;
+
+import static com.fizzed.jsync.sftp.SftpVirtualVolume.sftpVolume;
+import static com.fizzed.jsync.vfs.LocalVirtualVolume.localVolume;
+
+public class SftpVirtualFileSystemDemo {
+    static private final Logger log = LoggerFactory.getLogger(SftpVirtualFileSystemDemo.class);
+
+    static public void main(String[] args) throws Exception {
+
+//        final SftpVirtualFileSystem vfs = SftpVirtualFileSystem.open("bmh-dev-x64-indy25-1");
+        final SftpVirtualFileSystem vfs = SftpVirtualFileSystem.open("bmh-build-x64-win11-1");
+
+//        final VirtualPath stat = vfs.stat(VirtualPath.parse(".ssh/authorized_keys"));
+        final VirtualPath stat = vfs.stat(VirtualPath.parse("remote-build"));
+
+        log.info("file: {}, perms={}", stat, stat.getStat().getPermissionsOctal());
+    }
+
+}