diff --git a/pom.xml b/pom.xml
index ab72a72f..c36238ac 100644
--- a/pom.xml
+++ b/pom.xml
@@ -178,6 +178,11 @@
json
20211205
+
+ com.github.lookfirst
+ sardine
+ 5.10
+
diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/Enum/EndpointType.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/Enum/EndpointType.java
index 5dd3ca7f..72ba49aa 100644
--- a/src/main/java/org/onedatashare/transferservice/odstransferservice/Enum/EndpointType.java
+++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/Enum/EndpointType.java
@@ -1,5 +1,5 @@
package org.onedatashare.transferservice.odstransferservice.Enum;
public enum EndpointType {
- dropbox, gdrive, sftp, ftp, box, s3, gftp, http, vfs, scp
+ dropbox, gdrive, sftp, ftp, box, s3, gftp, http, vfs, scp, webdav
}
\ No newline at end of file
diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/JobControl.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/JobControl.java
index 13d6092c..52fa437f 100644
--- a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/JobControl.java
+++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/JobControl.java
@@ -30,6 +30,8 @@
import org.onedatashare.transferservice.odstransferservice.service.step.sftp.SFTPWriter;
import org.onedatashare.transferservice.odstransferservice.service.step.vfs.VfsReader;
import org.onedatashare.transferservice.odstransferservice.service.step.vfs.VfsWriter;
+import org.onedatashare.transferservice.odstransferservice.service.step.webdav.WebDAVReader;
+import org.onedatashare.transferservice.odstransferservice.service.step.webdav.WebDAVWriter;
import org.onedatashare.transferservice.odstransferservice.utility.ODSUtility;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -174,6 +176,9 @@ protected AbstractItemCountingItemStreamItemReader getRightReader(End
case gdrive:
GDriveReader dDriveReader = new GDriveReader(request.getSource().getOauthSourceCredential(), fileInfo);
return dDriveReader;
+ case webdav:
+ WebDAVReader webDAVReader = new WebDAVReader(request.getSource().getVfsSourceCredential(),fileInfo);
+ return webDAVReader;
}
return null;
}
@@ -227,6 +232,9 @@ protected ItemWriter getRightWriter(EndpointType type, EntityInfo fil
writer.setPool(connectionBag.getGoogleDriveWriterPool());
return writer;
}
+ case webdav:
+ WebDAVWriter writer = new WebDAVWriter(request.getDestination().getVfsDestCredential(),fileInfo,this.metricsCollector,this.influxCache);
+ return writer;
}
return null;
}
diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/webdav/WebDAVReader.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/webdav/WebDAVReader.java
new file mode 100644
index 00000000..8d973def
--- /dev/null
+++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/webdav/WebDAVReader.java
@@ -0,0 +1,75 @@
+package org.onedatashare.transferservice.odstransferservice.service.step.webdav;
+
+import com.github.sardine.Sardine;
+import com.github.sardine.SardineFactory;
+import org.onedatashare.transferservice.odstransferservice.constant.ODSConstants;
+import org.onedatashare.transferservice.odstransferservice.model.DataChunk;
+import org.onedatashare.transferservice.odstransferservice.model.EntityInfo;
+import org.onedatashare.transferservice.odstransferservice.model.FilePart;
+import org.onedatashare.transferservice.odstransferservice.model.credential.AccountEndpointCredential;
+import org.onedatashare.transferservice.odstransferservice.service.FilePartitioner;
+import org.onedatashare.transferservice.odstransferservice.utility.ODSUtility;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.batch.core.StepExecution;
+import org.springframework.batch.core.annotation.BeforeStep;
+import org.springframework.batch.item.support.AbstractItemCountingItemStreamItemReader;
+import org.springframework.util.ClassUtils;
+
+import java.io.InputStream;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+public class WebDAVReader extends AbstractItemCountingItemStreamItemReader {
+
+ Logger logger = LoggerFactory.getLogger(WebDAVReader.class);
+ private final EntityInfo fileInfo;
+ private final AccountEndpointCredential credential;
+ private final FilePartitioner filePartitioner;
+ private Sardine client;
+
+ private String fileName;
+ private String uri;
+
+ public WebDAVReader(AccountEndpointCredential credential, EntityInfo fileInfo){
+ this.fileInfo = fileInfo;
+ this.filePartitioner = new FilePartitioner(fileInfo.getChunkSize());
+ this.setName(ClassUtils.getShortName(WebDAVReader.class));
+ this.credential = credential;
+ }
+
+ @BeforeStep
+ public void beforeStep(StepExecution stepExecution){
+
+ this.filePartitioner.createParts(this.fileInfo.getSize(), this.fileInfo.getId());
+ this.fileName = fileInfo.getId();
+ this.uri = credential.getUri() + Paths.get(fileInfo.getPath()).toString();
+ }
+ @Override
+ protected DataChunk doRead() throws Exception {
+ FilePart filePart = this.filePartitioner.nextPart();
+ if (filePart == null) return null;
+ Map headers = new HashMap<>();
+ headers.put(ODSConstants.RANGE,String.format(ODSConstants.byteRange,filePart.getStart(), filePart.getEnd()));
+ InputStream response = client.get(this.uri,headers);
+ byte[] body = response.readAllBytes();
+ DataChunk chunk = ODSUtility.makeChunk(body.length, body, filePart.getStart(), Long.valueOf(filePart.getPartIdx()).intValue(), this.fileName);
+ logger.info(chunk.toString());
+ return chunk;
+ }
+
+ @Override
+ protected void doOpen() throws Exception {
+ if(this.credential!=null && this.credential.getUsername()!=null){
+ client = SardineFactory.begin(this.credential.getUsername(),this.credential.getSecret());
+ }else{
+ client = SardineFactory.begin();
+ }
+
+ }
+
+ @Override
+ protected void doClose() throws Exception {
+ client.shutdown();
+ }
+}
diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/webdav/WebDAVWriter.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/webdav/WebDAVWriter.java
new file mode 100644
index 00000000..3f8d9750
--- /dev/null
+++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/webdav/WebDAVWriter.java
@@ -0,0 +1,68 @@
+package org.onedatashare.transferservice.odstransferservice.service.step.webdav;
+
+import com.github.sardine.Sardine;
+import com.github.sardine.SardineFactory;
+import org.apache.http.HttpHeaders;
+import org.onedatashare.transferservice.odstransferservice.model.DataChunk;
+import org.onedatashare.transferservice.odstransferservice.model.EntityInfo;
+import org.onedatashare.transferservice.odstransferservice.model.credential.AccountEndpointCredential;
+import org.onedatashare.transferservice.odstransferservice.service.InfluxCache;
+import org.onedatashare.transferservice.odstransferservice.service.cron.MetricsCollector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.batch.core.annotation.AfterStep;
+import org.springframework.batch.core.annotation.BeforeWrite;
+import org.springframework.batch.item.ItemWriter;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.net.URLConnection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+public class WebDAVWriter implements ItemWriter {
+ Logger logger = LoggerFactory.getLogger(WebDAVWriter.class);
+ private EntityInfo fileInfo;
+ private Sardine client;
+ private AccountEndpointCredential credential;
+ private String uri;
+ private String fileName;
+
+ public WebDAVWriter(AccountEndpointCredential credential, EntityInfo fileInfo, MetricsCollector metricsCollector, InfluxCache influxCache) {
+ this.credential = credential;
+ this.fileInfo = fileInfo;
+ this.uri = credential.getUri();
+ }
+
+ @BeforeWrite
+ public void beforeWrite(List items){
+ this.fileName = items.get(0).getFileName();
+ if(client==null){
+ if(this.credential.getUsername()!=null){
+ logger.debug("Setting authentication credentials for WebDAV client");
+ client= SardineFactory.begin(this.credential.getUsername(),this.credential.getSecret());
+ }else{
+ client= SardineFactory.begin();
+ }
+ }
+ }
+
+
+ @Override
+ public void write(List extends DataChunk> items) throws Exception {
+ for(DataChunk item : items){
+ boolean fileExist = client.exists(this.uri+item.getFileName());
+ if(fileExist==true){
+ logger.debug("File already exists. Overriding the file");
+ }
+ client.put(this.uri+item.getFileName(),new ByteArrayInputStream(item.getData()), URLConnection.guessContentTypeFromName(item.getFileName()), true);
+ }
+ }
+
+ @AfterStep
+ public void afterStep() throws IOException {
+ client.shutdown();
+ }
+}