Implemented WebDAV reader and writer

pom.xml

@@ -178,6 +178,11 @@
             <artifactId>json</artifactId>
             <version>20211205</version>
         </dependency>
+        <dependency>
+            <groupId>com.github.lookfirst</groupId>
+            <artifactId>sardine</artifactId>
+            <version>5.10</version>
+        </dependency>
     </dependencies>
     <dependencyManagement>
         <dependencies>

src/main/java/org/onedatashare/transferservice/odstransferservice/Enum/EndpointType.java

@@ -1,5 +1,5 @@
 package org.onedatashare.transferservice.odstransferservice.Enum;
 
 public enum EndpointType {
-    dropbox, gdrive, sftp, ftp, box, s3, gftp, http, vfs, scp
+    dropbox, gdrive, sftp, ftp, box, s3, gftp, http, vfs, scp, webdav
 }

src/main/java/org/onedatashare/transferservice/odstransferservice/service/JobControl.java

@@ -30,6 +30,8 @@
 import org.onedatashare.transferservice.odstransferservice.service.step.sftp.SFTPWriter;
 import org.onedatashare.transferservice.odstransferservice.service.step.vfs.VfsReader;
 import org.onedatashare.transferservice.odstransferservice.service.step.vfs.VfsWriter;
+import org.onedatashare.transferservice.odstransferservice.service.step.webdav.WebDAVReader;
+import org.onedatashare.transferservice.odstransferservice.service.step.webdav.WebDAVWriter;
 import org.onedatashare.transferservice.odstransferservice.utility.ODSUtility;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -174,6 +176,9 @@ protected AbstractItemCountingItemStreamItemReader<DataChunk> getRightReader(End
             case gdrive:
                 GDriveReader dDriveReader = new GDriveReader(request.getSource().getOauthSourceCredential(), fileInfo);
                 return dDriveReader;
+            case webdav:
+                WebDAVReader webDAVReader = new WebDAVReader(request.getSource().getVfsSourceCredential(),fileInfo);
+                return webDAVReader;
         }
         return null;
     }
@@ -227,6 +232,9 @@ protected ItemWriter<DataChunk> getRightWriter(EndpointType type, EntityInfo fil
                     writer.setPool(connectionBag.getGoogleDriveWriterPool());
                     return writer;
                 }
+            case webdav:
+                WebDAVWriter writer = new WebDAVWriter(request.getDestination().getVfsDestCredential(),fileInfo,this.metricsCollector,this.influxCache);
+                return writer;
         }
         return null;
     }

src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/webdav/WebDAVReader.java

@@ -0,0 +1,75 @@
+package org.onedatashare.transferservice.odstransferservice.service.step.webdav;
+
+import com.github.sardine.Sardine;
+import com.github.sardine.SardineFactory;
+import org.onedatashare.transferservice.odstransferservice.constant.ODSConstants;
+import org.onedatashare.transferservice.odstransferservice.model.DataChunk;
+import org.onedatashare.transferservice.odstransferservice.model.EntityInfo;
+import org.onedatashare.transferservice.odstransferservice.model.FilePart;
+import org.onedatashare.transferservice.odstransferservice.model.credential.AccountEndpointCredential;
+import org.onedatashare.transferservice.odstransferservice.service.FilePartitioner;
+import org.onedatashare.transferservice.odstransferservice.utility.ODSUtility;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.batch.core.StepExecution;
+import org.springframework.batch.core.annotation.BeforeStep;
+import org.springframework.batch.item.support.AbstractItemCountingItemStreamItemReader;
+import org.springframework.util.ClassUtils;
+
+import java.io.InputStream;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+public class WebDAVReader extends AbstractItemCountingItemStreamItemReader<DataChunk> {
+
+    Logger logger = LoggerFactory.getLogger(WebDAVReader.class);
+    private final EntityInfo fileInfo;
+    private final AccountEndpointCredential credential;
+    private final FilePartitioner filePartitioner;
+    private Sardine client;
+
+    private String fileName;
+    private String uri;
+
+    public WebDAVReader(AccountEndpointCredential credential, EntityInfo fileInfo){
+        this.fileInfo = fileInfo;
+        this.filePartitioner = new FilePartitioner(fileInfo.getChunkSize());
+        this.setName(ClassUtils.getShortName(WebDAVReader.class));
+        this.credential = credential;
+    }
+
+    @BeforeStep
+    public void beforeStep(StepExecution stepExecution){
+
+        this.filePartitioner.createParts(this.fileInfo.getSize(), this.fileInfo.getId());
+        this.fileName = fileInfo.getId();
+        this.uri = credential.getUri() + Paths.get(fileInfo.getPath()).toString();
+    }
+    @Override
+    protected DataChunk doRead() throws Exception {
+        FilePart filePart = this.filePartitioner.nextPart();
+        if (filePart == null) return null;
+        Map<String,String> headers = new HashMap<>();
+        headers.put(ODSConstants.RANGE,String.format(ODSConstants.byteRange,filePart.getStart(), filePart.getEnd()));
+        InputStream response = client.get(this.uri,headers);
+        byte[] body = response.readAllBytes();
+        DataChunk chunk = ODSUtility.makeChunk(body.length, body, filePart.getStart(), Long.valueOf(filePart.getPartIdx()).intValue(), this.fileName);
+        logger.info(chunk.toString());
+        return chunk;
+    }
+
+    @Override
+    protected void doOpen() throws Exception {
+        if(this.credential!=null && this.credential.getUsername()!=null){
+            client = SardineFactory.begin(this.credential.getUsername(),this.credential.getSecret());
+        }else{
+            client = SardineFactory.begin();
+        }
+
+    }
+
+    @Override
+    protected void doClose() throws Exception {
+        client.shutdown();
+    }
+}

src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/webdav/WebDAVWriter.java

@@ -0,0 +1,68 @@
+package org.onedatashare.transferservice.odstransferservice.service.step.webdav;
+
+import com.github.sardine.Sardine;
+import com.github.sardine.SardineFactory;
+import org.apache.http.HttpHeaders;
+import org.onedatashare.transferservice.odstransferservice.model.DataChunk;
+import org.onedatashare.transferservice.odstransferservice.model.EntityInfo;
+import org.onedatashare.transferservice.odstransferservice.model.credential.AccountEndpointCredential;
+import org.onedatashare.transferservice.odstransferservice.service.InfluxCache;
+import org.onedatashare.transferservice.odstransferservice.service.cron.MetricsCollector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.batch.core.annotation.AfterStep;
+import org.springframework.batch.core.annotation.BeforeWrite;
+import org.springframework.batch.item.ItemWriter;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.net.URLConnection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+public class WebDAVWriter implements ItemWriter<DataChunk> {
+    Logger logger = LoggerFactory.getLogger(WebDAVWriter.class);
+    private EntityInfo fileInfo;
+    private Sardine client;
+    private AccountEndpointCredential credential;
+    private String uri;
+    private String fileName;
+
+    public WebDAVWriter(AccountEndpointCredential credential, EntityInfo fileInfo, MetricsCollector metricsCollector, InfluxCache influxCache) {
+        this.credential = credential;
+        this.fileInfo = fileInfo;
+        this.uri = credential.getUri();
+    }
+
+    @BeforeWrite
+    public void beforeWrite(List<DataChunk> items){
+        this.fileName = items.get(0).getFileName();
+        if(client==null){
+            if(this.credential.getUsername()!=null){
+                logger.debug("Setting authentication credentials for WebDAV client");
+                client= SardineFactory.begin(this.credential.getUsername(),this.credential.getSecret());
+            }else{
+                client= SardineFactory.begin();
+            }
+        }
+    }
+
+
+    @Override
+    public void write(List<? extends DataChunk> items) throws Exception {
+        for(DataChunk item : items){
+            boolean fileExist = client.exists(this.uri+item.getFileName());
+            if(fileExist==true){
+                logger.debug("File already exists. Overriding the file");
+            }
+            client.put(this.uri+item.getFileName(),new ByteArrayInputStream(item.getData()), URLConnection.guessContentTypeFromName(item.getFileName()), true);
+        }
+    }
+
+    @AfterStep
+    public void afterStep() throws IOException {
+        client.shutdown();
+    }
+}