From a7073efeafdeef2323c8d3868154e201d323de61 Mon Sep 17 00:00:00 2001 From: steveglowplunk <28670916+steveglowplunk@users.noreply.github.com> Date: Fri, 11 Aug 2023 17:27:37 +0800 Subject: [PATCH 1/6] Add extra features --- .../CombinedSpeedProgressDownloadHandler.java | 2 +- src/me/marnic/jdl/Downloader.java | 197 ++++++++++++++++-- 2 files changed, 181 insertions(+), 18 deletions(-) diff --git a/src/me/marnic/jdl/CombinedSpeedProgressDownloadHandler.java b/src/me/marnic/jdl/CombinedSpeedProgressDownloadHandler.java index cd93bd8..3148544 100644 --- a/src/me/marnic/jdl/CombinedSpeedProgressDownloadHandler.java +++ b/src/me/marnic/jdl/CombinedSpeedProgressDownloadHandler.java @@ -33,7 +33,7 @@ public void run() { onDownloadProgress(downloader.downloadedBytes,downloader.downloadLength,percent); onDownloadSpeedProgress(downloader.downloadedBytes,downloader.downloadLength,percent,deltaDownload); } - },0,1000); + },0,250); } public void onDownloadTickPerSec(int bytesPerSec) {} diff --git a/src/me/marnic/jdl/Downloader.java b/src/me/marnic/jdl/Downloader.java index 9961ca4..37fdf7c 100644 --- a/src/me/marnic/jdl/Downloader.java +++ b/src/me/marnic/jdl/Downloader.java @@ -1,10 +1,11 @@ package me.marnic.jdl; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.io.ObjectInputStream; -import java.net.URL; -import java.net.URLConnection; +import java.io.*; +import java.net.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.*; /** * Copyright (c) 12.01.2019 @@ -12,14 +13,38 @@ * GitHub: https://github.com/MrMarnic */ public class Downloader { - private DownloadHandler downloadHandler; - int downloadedBytes; int downloadLength; + private HashMap cookiesMap; + private Boolean bUseCookies = false; + private String cookiesLocation; + private String domainFilter; + private boolean bIsCancelled = false; public Downloader(boolean createDefaultHandler) { - if(createDefaultHandler) { + if (createDefaultHandler) { + downloadHandler = new DownloadHandler(this) { + @Override + public void onDownloadStart() { + + } + + @Override + public void onDownloadFinish() { + + } + + @Override + public void onDownloadError() { + + } + }; + } + } + + public Downloader(boolean createDefaultHandler, String cookiesLocation, String domainFilter) { + if (createDefaultHandler) { downloadHandler = new DownloadHandler(this) { @Override public void onDownloadStart() { @@ -37,38 +62,176 @@ public void onDownloadError() { } }; } + this.bUseCookies = true; + this.cookiesLocation = cookiesLocation; + this.domainFilter = domainFilter; + this.cookiesMap = readCookies(); + } + + public HashMap getCookiesMap() { + return cookiesMap; + } + + public HashMap readCookies() { + try { + File myObj = new File(this.cookiesLocation); + Scanner myReader = new Scanner(myObj); + String data; + List partList; + HashMap result = new HashMap<>(); + while (myReader.hasNextLine()) { + data = myReader.nextLine(); + // skip comment and empty lines + if (!data.contains("#") && !data.trim().isEmpty() && data.contains(this.domainFilter)) { + partList = Arrays.asList(data.split("\\s+")); // split line as array by space + result.put(partList.get(5), partList.get(6)); // name and key columns in netscape format cookies + } + } + myReader.close(); + return result; + } catch (FileNotFoundException e) { + System.out.println("Cookies not found"); + e.printStackTrace(); + } + return null; + } + + private URLConnection setupConnectionWithCookies(HashMap cookiesMap, String urlStr) { + URLConnection connection = null; + URL url = null; + try { + url = new URL(urlStr); + } catch (MalformedURLException e) { + System.out.println("An error occurred."); + e.printStackTrace(); + } + try { + connection = url.openConnection(); + } catch (IOException e) { + System.out.println("An error occurred."); + e.printStackTrace(); + } + String cookies = ""; + + for (Map.Entry entry : cookiesMap.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + cookies += key + "=" + value + "; "; + connection.addRequestProperty("Cookie", cookies); + } + return connection; + } + + public String getServerFileName(String urlStr) { + HttpURLConnection con = null; + try { + URL url = new URL(urlStr); + if (bUseCookies) { + con = (HttpURLConnection) setupConnectionWithCookies(cookiesMap, urlStr); + } else { + con = (HttpURLConnection) url.openConnection(); + } + con.connect(); + + if (con.getResponseCode() != HttpURLConnection.HTTP_OK) { + return "Server returned HTTP " + con.getResponseCode() + " " + con.getResponseMessage(); + } + } catch (Exception e) { + System.out.println("An error occurred."); + return e.toString(); + } finally { + if (con != null) con.disconnect(); + } + + String decodedURL = URLDecoder.decode(con.getURL().toString(), StandardCharsets.UTF_8); // for Chinese + // characters + return new File(decodedURL).getName(); + } + + public void setbIsCancelled(boolean bIsCancelled) { + this.bIsCancelled = bIsCancelled; + } + + private void resetValues() { + downloadedBytes = 0; + downloadLength = 0; } public void downloadFileToLocation(String urlStr, String pathToDownload) { try { URL url = new URL(urlStr); - URLConnection con = url.openConnection(); + HttpURLConnection con; + if (bUseCookies) { + con = (HttpURLConnection) setupConnectionWithCookies(cookiesMap, urlStr); + } else { + con = (HttpURLConnection) url.openConnection(); + } + + con.connect(); downloadLength = con.getContentLength(); InputStream in = con.getInputStream(); - FileOutputStream out = new FileOutputStream(pathToDownload); + String fileName = getServerFileName(urlStr); + Files.createDirectories(Paths.get(pathToDownload)); + FileOutputStream out = new FileOutputStream(pathToDownload + fileName); byte[] data = new byte[1024]; int length; downloadHandler.onDownloadStart(); - while ((length = in.read(data,0,1024))!=-1) { - out.write(data,0,length); - downloadedBytes+=length; + while (!bIsCancelled && (length = in.read(data, 0, 1024)) != -1) { + out.write(data, 0, length); + downloadedBytes += length; } + out.flush(); in.close(); out.close(); + resetValues(); // for repeated use of the same downloader object, in order to get correct download speed + // values downloadHandler.onDownloadFinish(); - }catch (Exception e) { + } catch (Exception e) { + e.printStackTrace(); + downloadHandler.onDownloadError(); + } + } + + public String downloadJSONString(String urlStr) { + try { + URL url = new URL(urlStr); + HttpURLConnection con; + if (bUseCookies) { + con = (HttpURLConnection) setupConnectionWithCookies(cookiesMap, urlStr); + } else { + con = (HttpURLConnection) url.openConnection(); + } + + con.connect(); + downloadLength = con.getContentLength(); + + InputStream in = con.getInputStream(); + BufferedReader br = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(); + String line; + while ((line = br.readLine()) != null) { + sb.append(line).append("\n"); + } + br.close(); + return sb.toString(); + } catch (UnknownHostException e) { + e.printStackTrace(); + System.out.println("No Internet"); + } catch (Exception e) { e.printStackTrace(); downloadHandler.onDownloadError(); } + return null; } + // This method never worked anyway public Object downloadObject(String urlStr) { try { @@ -83,15 +246,15 @@ public Object downloadObject(String urlStr) { downloadHandler.onDownloadStart(); - while ((length = in.read(data,0,1024))!=-1) { - downloadedBytes+=length; + while ((length = in.read(data, 0, 1024)) != -1) { + downloadedBytes += length; } in.close(); downloadHandler.onDownloadFinish(); return in.readObject(); - }catch (Exception e) { + } catch (Exception e) { e.printStackTrace(); downloadHandler.onDownloadError(); } From 04a8036be596efbbd5fd604015f96ef9dee58e1b Mon Sep 17 00:00:00 2001 From: steveglowplunk <28670916+steveglowplunk@users.noreply.github.com> Date: Mon, 14 Aug 2023 15:37:27 +0800 Subject: [PATCH 2/6] Changes to Downloader constructor; Fixes in SizeUtil --- .../jdl/DownloadProgressDownloadHandler.java | 2 +- .../jdl/DownloadSpeedDownloadHandler.java | 2 +- src/me/marnic/jdl/Downloader.java | 58 +++++++++++++++++-- src/me/marnic/jdl/SizeUtil.java | 32 +++++++--- 4 files changed, 79 insertions(+), 15 deletions(-) diff --git a/src/me/marnic/jdl/DownloadProgressDownloadHandler.java b/src/me/marnic/jdl/DownloadProgressDownloadHandler.java index 27907be..eb666c8 100644 --- a/src/me/marnic/jdl/DownloadProgressDownloadHandler.java +++ b/src/me/marnic/jdl/DownloadProgressDownloadHandler.java @@ -25,7 +25,7 @@ public void onDownloadStart() { public void run() { onDownloadProgress(downloader.downloadedBytes,downloader.downloadLength,(int)(((double)downloader.downloadedBytes/downloader.downloadLength)*100)); } - },0,1000); + },0,250); } public abstract void onDownloadProgress(int downloaded,int maxDownload,int percent); diff --git a/src/me/marnic/jdl/DownloadSpeedDownloadHandler.java b/src/me/marnic/jdl/DownloadSpeedDownloadHandler.java index 4a95005..95c184d 100644 --- a/src/me/marnic/jdl/DownloadSpeedDownloadHandler.java +++ b/src/me/marnic/jdl/DownloadSpeedDownloadHandler.java @@ -29,7 +29,7 @@ public void run() { onDownloadTickPerSec(deltaDownload); lastDownloadSize = downloader.downloadedBytes; } - },0,1000); + },0,250); } public abstract void onDownloadTickPerSec(int bytesPerSec); diff --git a/src/me/marnic/jdl/Downloader.java b/src/me/marnic/jdl/Downloader.java index 37fdf7c..5ee0083 100644 --- a/src/me/marnic/jdl/Downloader.java +++ b/src/me/marnic/jdl/Downloader.java @@ -43,6 +43,30 @@ public void onDownloadError() { } } + public Downloader(boolean createDefaultHandler, String cookiesLocation) { + if (createDefaultHandler) { + downloadHandler = new DownloadHandler(this) { + @Override + public void onDownloadStart() { + + } + + @Override + public void onDownloadFinish() { + + } + + @Override + public void onDownloadError() { + + } + }; + } + this.bUseCookies = true; + this.cookiesLocation = cookiesLocation; + this.cookiesMap = readCookies(); + } + public Downloader(boolean createDefaultHandler, String cookiesLocation, String domainFilter) { if (createDefaultHandler) { downloadHandler = new DownloadHandler(this) { @@ -79,10 +103,11 @@ public HashMap readCookies() { String data; List partList; HashMap result = new HashMap<>(); + boolean bHasDomainFilter = (domainFilter != null) && !domainFilter.isEmpty(); while (myReader.hasNextLine()) { data = myReader.nextLine(); // skip comment and empty lines - if (!data.contains("#") && !data.trim().isEmpty() && data.contains(this.domainFilter)) { + if (!data.contains("#") && !data.trim().isEmpty() && (bHasDomainFilter ? data.contains(this.domainFilter) : true)) { partList = Arrays.asList(data.split("\\s+")); // split line as array by space result.put(partList.get(5), partList.get(6)); // name and key columns in netscape format cookies } @@ -122,7 +147,7 @@ private URLConnection setupConnectionWithCookies(HashMap cookies return connection; } - public String getServerFileName(String urlStr) { + public String getServerFileName(String urlStr) throws UnsupportedEncodingException { HttpURLConnection con = null; try { URL url = new URL(urlStr); @@ -131,6 +156,7 @@ public String getServerFileName(String urlStr) { } else { con = (HttpURLConnection) url.openConnection(); } + con.addRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"); con.connect(); if (con.getResponseCode() != HttpURLConnection.HTTP_OK) { @@ -143,13 +169,13 @@ public String getServerFileName(String urlStr) { if (con != null) con.disconnect(); } - String decodedURL = URLDecoder.decode(con.getURL().toString(), StandardCharsets.UTF_8); // for Chinese + String decodedURL = URLDecoder.decode(con.getURL().toString(), String.valueOf(StandardCharsets.UTF_8)); // for Chinese // characters return new File(decodedURL).getName(); } - public void setbIsCancelled(boolean bIsCancelled) { - this.bIsCancelled = bIsCancelled; + public void cancelDownload() { + bIsCancelled = true; } private void resetValues() { @@ -157,6 +183,26 @@ private void resetValues() { downloadLength = 0; } + public Integer getDownloadLength(String urlStr) { + try { + URL url = new URL(urlStr); + HttpURLConnection con; + if (bUseCookies) { + con = (HttpURLConnection) setupConnectionWithCookies(cookiesMap, urlStr); + } else { + con = (HttpURLConnection) url.openConnection(); + } + + con.addRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"); + con.connect(); + downloadLength = con.getContentLength(); + return downloadLength; + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + public void downloadFileToLocation(String urlStr, String pathToDownload) { try { @@ -168,6 +214,7 @@ public void downloadFileToLocation(String urlStr, String pathToDownload) { con = (HttpURLConnection) url.openConnection(); } + con.addRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"); con.connect(); downloadLength = con.getContentLength(); @@ -209,6 +256,7 @@ public String downloadJSONString(String urlStr) { con = (HttpURLConnection) url.openConnection(); } + con.addRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"); con.connect(); downloadLength = con.getContentLength(); diff --git a/src/me/marnic/jdl/SizeUtil.java b/src/me/marnic/jdl/SizeUtil.java index 9ef07b6..8e20d68 100644 --- a/src/me/marnic/jdl/SizeUtil.java +++ b/src/me/marnic/jdl/SizeUtil.java @@ -7,30 +7,46 @@ */ public class SizeUtil { public static double toKBFB(int bytes) { - return ((double)bytes/1000); + return ((double) bytes / 1024); } public static double toMBFB(int bytes) { - return ((double)bytes/1000000); + return ((double) bytes / (1024 * 1024)); } public static double toGBFB(int bytes) { - return ((double)bytes/1000000000); + return ((double) bytes / 1024 * 1024 * 1024); } - public static double toMBFKB(int kb) { - return ((double)kb/1000); + return ((double) kb / 1024); } public static double toGBFKB(int kb) { - return ((double)kb/1000000); + return ((double) kb / (1024 * 1024)); } - public static double toGBFMB(int mb) { - return ((double)mb/1000); + return ((double) mb / 1024); + } + + public static String toHumanReadableFromBytes(int bytes) { + if (bytes < 0) { + return "unknown size"; + } + + String[] units = {"B", "KB", "MB", "GB"}; + int unitIndex = 0; + double value = bytes; + + while (value >= 1024 && unitIndex < units.length - 1) { + value /= 1024.0; + unitIndex++; + } + + String formattedValue = String.format("%.2f", value); + return formattedValue + " " + units[unitIndex]; } } From a3346f20c5a8ea932253195255c479f1a21bb130 Mon Sep 17 00:00:00 2001 From: steveglowplunk <28670916+steveglowplunk@users.noreply.github.com> Date: Mon, 21 Aug 2023 10:05:02 +0800 Subject: [PATCH 3/6] Add custom user agent string feature --- src/me/marnic/jdl/Downloader.java | 129 +++++++++++++++--------------- 1 file changed, 64 insertions(+), 65 deletions(-) diff --git a/src/me/marnic/jdl/Downloader.java b/src/me/marnic/jdl/Downloader.java index 5ee0083..f926d3c 100644 --- a/src/me/marnic/jdl/Downloader.java +++ b/src/me/marnic/jdl/Downloader.java @@ -17,10 +17,12 @@ public class Downloader { int downloadedBytes; int downloadLength; private HashMap cookiesMap; - private Boolean bUseCookies = false; - private String cookiesLocation; + private boolean bUseCookies = false; + private boolean bUseDomainFilter = false; private String domainFilter; private boolean bIsCancelled = false; + private boolean bUseCustomUserAgentString = false; + private String customUserAgentString; public Downloader(boolean createDefaultHandler) { if (createDefaultHandler) { @@ -43,79 +45,51 @@ public void onDownloadError() { } } - public Downloader(boolean createDefaultHandler, String cookiesLocation) { - if (createDefaultHandler) { - downloadHandler = new DownloadHandler(this) { - @Override - public void onDownloadStart() { - - } - - @Override - public void onDownloadFinish() { - - } - - @Override - public void onDownloadError() { - - } - }; - } + public void setCookies(File cookiesFile) { this.bUseCookies = true; - this.cookiesLocation = cookiesLocation; - this.cookiesMap = readCookies(); + this.cookiesMap = readCookies(cookiesFile); } - public Downloader(boolean createDefaultHandler, String cookiesLocation, String domainFilter) { - if (createDefaultHandler) { - downloadHandler = new DownloadHandler(this) { - @Override - public void onDownloadStart() { - - } - - @Override - public void onDownloadFinish() { - - } - - @Override - public void onDownloadError() { - - } - }; - } - this.bUseCookies = true; - this.cookiesLocation = cookiesLocation; + public void setDomainFilter(String domainFilter) { + this.bUseDomainFilter = true; this.domainFilter = domainFilter; - this.cookiesMap = readCookies(); } + public void setCustomUserAgentString(String userAgentString) { + this.bUseCustomUserAgentString = true; + this.customUserAgentString = userAgentString; + } + + // get HashMap formatted cookies that have finished reading from a txt file public HashMap getCookiesMap() { return cookiesMap; } - public HashMap readCookies() { + // read HashMap formatted cookies from txt file for use with libraries like jsoup + public HashMap readCookies(File cookiesFile) { try { - File myObj = new File(this.cookiesLocation); - Scanner myReader = new Scanner(myObj); + Scanner myReader = new Scanner(cookiesFile); String data; List partList; HashMap result = new HashMap<>(); - boolean bHasDomainFilter = (domainFilter != null) && !domainFilter.isEmpty(); while (myReader.hasNextLine()) { data = myReader.nextLine(); // skip comment and empty lines - if (!data.contains("#") && !data.trim().isEmpty() && (bHasDomainFilter ? data.contains(this.domainFilter) : true)) { + if (!data.contains("#") && !data.trim().isEmpty()) { partList = Arrays.asList(data.split("\\s+")); // split line as array by space - result.put(partList.get(5), partList.get(6)); // name and key columns in netscape format cookies + if (bUseDomainFilter) { + if (partList.get(0).contains(this.domainFilter)) { + result.put(partList.get(5), partList.get(6)); // name and value fields in Netscape format cookies + } + } else { + result.put(partList.get(5), partList.get(6)); + } } } myReader.close(); return result; } catch (FileNotFoundException e) { - System.out.println("Cookies not found"); + System.err.println("Cookies not found"); e.printStackTrace(); } return null; @@ -127,13 +101,13 @@ private URLConnection setupConnectionWithCookies(HashMap cookies try { url = new URL(urlStr); } catch (MalformedURLException e) { - System.out.println("An error occurred."); + System.err.println("An error occurred."); e.printStackTrace(); } try { connection = url.openConnection(); } catch (IOException e) { - System.out.println("An error occurred."); + System.err.println("An error occurred."); e.printStackTrace(); } String cookies = ""; @@ -156,24 +130,29 @@ public String getServerFileName(String urlStr) throws UnsupportedEncodingExcepti } else { con = (HttpURLConnection) url.openConnection(); } - con.addRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"); + if (bUseCustomUserAgentString) { + con.addRequestProperty("User-Agent", customUserAgentString); + } con.connect(); if (con.getResponseCode() != HttpURLConnection.HTTP_OK) { return "Server returned HTTP " + con.getResponseCode() + " " + con.getResponseMessage(); } } catch (Exception e) { - System.out.println("An error occurred."); + System.err.println("An error occurred."); return e.toString(); } finally { if (con != null) con.disconnect(); } - String decodedURL = URLDecoder.decode(con.getURL().toString(), String.valueOf(StandardCharsets.UTF_8)); // for Chinese - // characters + String decodedURL = URLDecoder.decode(con.getURL().toString(), String.valueOf(StandardCharsets.UTF_8)); // for Chinese characters return new File(decodedURL).getName(); } + public void setbIsCancelled(boolean bIsCancelled) { + this.bIsCancelled = bIsCancelled; + } + public void cancelDownload() { bIsCancelled = true; } @@ -193,18 +172,28 @@ public Integer getDownloadLength(String urlStr) { con = (HttpURLConnection) url.openConnection(); } - con.addRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"); + if (bUseCustomUserAgentString) { + con.addRequestProperty("User-Agent", customUserAgentString); + } con.connect(); downloadLength = con.getContentLength(); return downloadLength; } catch (Exception e) { + System.err.println("An error occurred."); e.printStackTrace(); } return null; } public void downloadFileToLocation(String urlStr, String pathToDownload) { + handleDownloadFileToLocation(urlStr, pathToDownload, null); + } + public void downloadFileToLocation(String urlStr, String pathToDownload, String customFileName) { + handleDownloadFileToLocation(urlStr, pathToDownload, customFileName); + } + + private void handleDownloadFileToLocation(String urlStr, String pathToDownload, String customFileName) { try { URL url = new URL(urlStr); HttpURLConnection con; @@ -214,12 +203,19 @@ public void downloadFileToLocation(String urlStr, String pathToDownload) { con = (HttpURLConnection) url.openConnection(); } - con.addRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"); + if (bUseCustomUserAgentString) { + con.addRequestProperty("User-Agent", customUserAgentString); + } con.connect(); downloadLength = con.getContentLength(); InputStream in = con.getInputStream(); - String fileName = getServerFileName(urlStr); + String fileName; + if (customFileName != null) { + fileName = customFileName; + } else { + fileName = getServerFileName(urlStr); + } Files.createDirectories(Paths.get(pathToDownload)); FileOutputStream out = new FileOutputStream(pathToDownload + fileName); @@ -237,10 +233,10 @@ public void downloadFileToLocation(String urlStr, String pathToDownload) { in.close(); out.close(); - resetValues(); // for repeated use of the same downloader object, in order to get correct download speed - // values + resetValues(); // for repeated use of the same downloader object, in order to get correct download speed values downloadHandler.onDownloadFinish(); } catch (Exception e) { + System.err.println("An error occurred."); e.printStackTrace(); downloadHandler.onDownloadError(); } @@ -256,7 +252,9 @@ public String downloadJSONString(String urlStr) { con = (HttpURLConnection) url.openConnection(); } - con.addRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"); + if (bUseCustomUserAgentString) { + con.addRequestProperty("User-Agent", customUserAgentString); + } con.connect(); downloadLength = con.getContentLength(); @@ -270,8 +268,8 @@ public String downloadJSONString(String urlStr) { br.close(); return sb.toString(); } catch (UnknownHostException e) { + System.err.println("No Internet available"); e.printStackTrace(); - System.out.println("No Internet"); } catch (Exception e) { e.printStackTrace(); downloadHandler.onDownloadError(); @@ -303,6 +301,7 @@ public Object downloadObject(String urlStr) { downloadHandler.onDownloadFinish(); return in.readObject(); } catch (Exception e) { + System.err.println("An error occurred."); e.printStackTrace(); downloadHandler.onDownloadError(); } From cc69ade403a73eaeb5f8b1f47a3372883b89f095 Mon Sep 17 00:00:00 2001 From: steveglowplunk <28670916+steveglowplunk@users.noreply.github.com> Date: Mon, 21 Aug 2023 12:04:45 +0800 Subject: [PATCH 4/6] Better filename retrieval mechanism --- src/me/marnic/jdl/Downloader.java | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/me/marnic/jdl/Downloader.java b/src/me/marnic/jdl/Downloader.java index f926d3c..e47a517 100644 --- a/src/me/marnic/jdl/Downloader.java +++ b/src/me/marnic/jdl/Downloader.java @@ -121,7 +121,7 @@ private URLConnection setupConnectionWithCookies(HashMap cookies return connection; } - public String getServerFileName(String urlStr) throws UnsupportedEncodingException { + public String getServerFileName(String urlStr) { HttpURLConnection con = null; try { URL url = new URL(urlStr); @@ -136,17 +136,30 @@ public String getServerFileName(String urlStr) throws UnsupportedEncodingExcepti con.connect(); if (con.getResponseCode() != HttpURLConnection.HTTP_OK) { - return "Server returned HTTP " + con.getResponseCode() + " " + con.getResponseMessage(); + System.err.println("Server returned HTTP " + con.getResponseCode() + " " + con.getResponseMessage()); + return "error"; } } catch (Exception e) { System.err.println("An error occurred."); - return e.toString(); + return "error"; } finally { if (con != null) con.disconnect(); } - String decodedURL = URLDecoder.decode(con.getURL().toString(), String.valueOf(StandardCharsets.UTF_8)); // for Chinese characters - return new File(decodedURL).getName(); + String filename; + try { + if (con.getHeaderField("Content-Disposition") != null) { + filename = con.getHeaderField("Content-Disposition").replaceFirst("(?i)^.*filename=\"?([^\"]+)\"?.*$", "$1"); + } else { + filename = Paths.get(new URI(con.getURL().toString()).getPath()).getFileName().toString(); + } + return filename; + } + catch (Exception e) { + System.err.println("An error occurred."); + e.printStackTrace(); + } + return null; } public void setbIsCancelled(boolean bIsCancelled) { From fd4753e07af96def3de0586e2cdae78e630b29a7 Mon Sep 17 00:00:00 2001 From: steveglowplunk <28670916+steveglowplunk@users.noreply.github.com> Date: Mon, 21 Aug 2023 14:59:21 +0800 Subject: [PATCH 5/6] Update README.md --- README.md | 61 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 8feec9f..0517450 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # JavaDownloadLibrary -A java library to download files and process the download speed,progress and other things +A Java library to download files and process the download speed, progress and other things Features: @@ -8,37 +8,60 @@ Features: - Check download progress - Directly cast download objects - Download Text -- Convert file sizes (MB;GB;KB...) +- Convert file sizes (MB, GB, KB...) +- Use (Netscape format) cookies to download files that require login +- Get JSON (e.g. from a website API) as String objects # Download: -https://github.com/MrMarnic/JDL/releases/download/1.0/JDL.jar - - -# Getting Started: - -# Download File: +https://github.com/MrMarnic/JavaDownloadLibrary/releases/ +# Example Usage: +**Simple download, with auto filename retrieval (without cookies/progress/speed/custom filename/custom user agent string)** +Note that a '\\' is expected at the end of the file path ``` -Downloader downloader = new Downloader(false); -downloader.downloadFileToLocation("https://github.com/MrMarnic/JIconExtract/releases/download/1.0/JIconExtract.jar","C:\\Downloads\\download.zip"); +Downloader downloader = new Downloader(true); +downloader.downloadFileToLocation("https://github.com/MrMarnic/JIconExtractReloaded/releases/download/v1.0/JIconExtractReloaded.jar", "C:\\demo\\test downloads\\"); ``` -# Add Handler (Check Speed,progress...): +**Download with file size preview, progress and speed display** ``` Downloader downloader = new Downloader(false); downloader.setDownloadHandler(new CombinedSpeedProgressDownloadHandler(downloader) { + @Override + public void onDownloadStart() { + super.onDownloadStart(); + // define custom actions to do before download starts + System.out.println("Download starting..."); + } @Override public void onDownloadSpeedProgress(int downloaded, int maxDownload, int percent, int bytesPerSec) { - System.out.println(SizeUtil.toMBFB(bytesPerSec)+"/s " + percent + "%"); + // define actions to do on each progress update + // (by default updates once every 250ms as defined in CombinedSpeedProgressDownloadHandler's onDownloadStart()) + System.out.println(SizeUtil.toHumanReadableFromBytes(bytesPerSec) + "/s " + percent + "%"); } - @Override public void onDownloadFinish() { super.onDownloadFinish(); + // define custom actions to do after download finishes System.out.println("Download finished"); } }); -downloader.downloadFileToLocation("https://github.com/MrMarnic/JIconExtract/releases/download/1.0/JIconExtract.jar","C:\\Downloads\\download.zi"); +System.out.println(SizeUtil.toHumanReadableFromBytes(downloader.getDownloadLength("https://github.com/MrMarnic/JIconExtractReloaded/releases/download/v1.0/JIconExtractReloaded.jar"))); +downloader.downloadFileToLocation("https://github.com/MrMarnic/JIconExtractReloaded/releases/download/v1.0/JIconExtractReloaded.jar", "C:\\demo\\test downloads\\"); +``` + +**Download files that require login using cookies, with custom file name** +``` +Downloader downloader = new Downloader(true); +downloader.setCookies(new File("C:\\demo\\cookies-github-com.txt")); +downloader.downloadFileToLocation("https://github.com/settings/profile", "C:\\demo\\test downloads\\", "GitHub Settings page.html"); +``` + +**Use custom agent string to pass through sites that block Java** +``` +Downloader downloader = new Downloader(true); +downloader.setCustomUserAgentString("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"); +downloader.downloadFileToLocation("https://www.whatismybrowser.com/detect/what-is-my-user-agent/", "C:\\demo\\test downloads\\", "UserAgentTest.html"); ``` # Handler @@ -76,12 +99,14 @@ public class ExampleDownloadHandler extends DownloadHandler{ # Convert File sizes -Syntax: SizeUtil.toMBFB() = toMegaBytesFromBytes - SizeUtil.toGBFB() = toGigiaBytesFromBytes - - +**Explaination:** +SizeUtil.toMBFB() = toMegaBytesFromBytes +SizeUtil.toGBFB() = toGigiaBytesFromBytes +SizeUtil.toHumanReadableFromBytes() = convert bytes to KB, MB, GB (e.g. for displaying download size / download speed); +**Usage** ``` double mb = SizeUtil.toMBFB(2000000000); double kb = SizeUtil.toKBFB(1000000); +String size = SizeUtil.toHumanReadableFromBytes(4096); ``` From 64696ca87103e51bcf4263172c8dde9ea4a33d6b Mon Sep 17 00:00:00 2001 From: steveglowplunk <28670916+steveglowplunk@users.noreply.github.com> Date: Mon, 21 Aug 2023 15:27:52 +0800 Subject: [PATCH 6/6] Update README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0517450..8df8b69 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,10 @@ Features: - Check download progress - Directly cast download objects - Download Text -- Convert file sizes (MB, GB, KB...) -- Use (Netscape format) cookies to download files that require login +- Use (Netscape format) cookies to download files that require login, optionally filter which domain name to load (to check if the cookie file contains the domain name that the programme expects) +- Able to cancel during download - Get JSON (e.g. from a website API) as String objects +- Convert file sizes from bytes to KB, MB, GB dynamically # Download: https://github.com/MrMarnic/JavaDownloadLibrary/releases/