mitigation of Screenscraper.fr long running responses on some games

src/netcomm.cpp

@@ -34,7 +34,6 @@ constexpr int DL_MAXSIZE = 100 * 1000 * 1000;
 NetComm::NetComm(QSharedPointer<NetManager> manager, int timeout)
     : manager(manager), timeout(timeout) {
     requestTimer.setSingleShot(true);
-    requestTimer.setInterval(timeout * 1000);
     connect(&requestTimer, &QTimer::timeout, this, &NetComm::requestTimeout);
 }
 
@@ -71,7 +70,7 @@ void NetComm::request(QString query, QString postData,
     connect(reply, &QNetworkReply::finished, this, &NetComm::replyReady);
     connect(reply, &QNetworkReply::downloadProgress, this,
             &NetComm::dataDownloaded);
-    requestTimer.start();
+    requestTimer.start(timeout * 1000);
 }
 
 void NetComm::replyReady() {
@@ -172,3 +171,5 @@ void NetComm::requestTimeout() {
            timeout);
     reply->abort();
 }
+
+void NetComm::setTimeout(int secs) { timeout = secs; }

src/netcomm.h

@@ -47,6 +47,7 @@ class NetComm : public QObject {
     QByteArray getContentType();
     QByteArray getRedirUrl();
     QString getHeaderValue(const QString headerKey);
+    void setTimeout(int secs);
 
 private slots:
     void replyReady();

src/screenscraper.cpp

@@ -36,8 +36,10 @@
 #include <QProcess>
 #include <QRegularExpression>
 
-constexpr int RETRIESMAX = 4;
-constexpr int TIMEOUT_SEC = 60;
+constexpr int RETRIESMAX = 3;
+constexpr int TIMEOUT_SEC = 75;
+constexpr int DELAY_NOTE_AFTER_SEC = 15;
+
 constexpr int MINARTSIZE = 256;
 
 ScreenScraper::ScreenScraper(Settings *config,
@@ -50,6 +52,16 @@ ScreenScraper::ScreenScraper(Settings *config,
     limitTimer.setSingleShot(false);
     limitTimer.start();
 
+    connect(&statusTimer, &QTimer::timeout, this, [=]() {
+        tctr = tctr + 1;
+        if (tctr >= DELAY_NOTE_AFTER_SEC) {
+            printf(" \033[1;32m%2s\033[0mResponse delayed for "
+                   "%ds\r",
+                   tctr % 2 ? " •" : "• ", 5 * ((int)(tctr / 5)));
+            fflush(stdout);
+        }
+    });
+
     baseUrl = "http://www.screenscraper.fr";
 
     fetchOrder.append(GameEntry::Elem::PUBLISHER);
@@ -94,36 +106,48 @@ void ScreenScraper::getSearchResults(QList<GameEntry> &gameEntries,
         (platformId == -1 ? "" : "&systemeid=" + QString::number(platformId)) +
         "&output=json&" + searchName;
 
+    tctr = 0;
+    statusTimer.start(1000);
+
     for (int retries = 0; retries < RETRIESMAX; ++retries) {
         limiter.exec();
         netComm->request(gameUrl);
         q.exec();
         data = netComm->getData();
 
-        QByteArray headerData =
-            data.left(1024); // Minor optimization with minimal more RAM usage
+        // Minor optimization with minimal more RAM usage
+        QByteArray headerData = data.left(1024);
         // Do error checks on headerData. It's more stable than checking the
         // potentially faulty JSON
         if (headerData.isEmpty()) {
-            printf("\033[1;33mRetrying request...\033[0m\n\n");
+            int timeout = TIMEOUT_SEC << (1 + retries);
+            printf(
+                "\033[1;33mRetrying request with timeout of %ds...\033[0m\n\n",
+                timeout);
+            netComm->setTimeout(timeout);
+            tctr = 0;
             continue;
         } else if (headerData.contains("non trouvée")) {
+            statusTimer.stop();
             return;
         } else if (headerData.contains("API totalement fermé")) {
             printf("\033[1;31mThe ScreenScraper API is currently closed, "
                    "exiting nicely...\033[0m\n\n");
+            statusTimer.stop();
             reqRemaining = 0;
             return;
         } else if (headerData.contains(
                        "Le logiciel de scrape utilisé a été blacklisté")) {
             printf("\033[1;31mSkyscraper has apparently been blacklisted at "
                    "ScreenScraper, exiting nicely...\033[0m\n\n");
+            statusTimer.stop();
             reqRemaining = 0;
             return;
         } else if (headerData.contains("Votre quota de scrape est")) {
             printf("\033[1;31mYour daily ScreenScraper request limit has been "
                    "reached, exiting nicely...\033[0m\n\n");
             reqRemaining = 0;
+            statusTimer.stop();
             return;
         } else if (
             headerData.contains("API fermé pour les non membres") ||
@@ -147,12 +171,18 @@ void ScreenScraper::getSearchResults(QList<GameEntry> &gameEntries,
                 Config::getSkyFolder().toStdString().c_str());
             if (retries == RETRIESMAX - 1) {
                 reqRemaining = 0;
+                statusTimer.stop();
                 return;
             } else {
                 continue;
             }
         }
 
+        if (tctr >= DELAY_NOTE_AFTER_SEC) {
+            printf("Response after %ds           \n", tctr);
+        }
+        statusTimer.stop();
+
         // Fix faulty JSON that is sometimes received back from ScreenScraper
         data.replace("],\n\t\t}", "]\n\t\t}");
 

src/screenscraper.h

@@ -47,6 +47,7 @@ class ScreenScraper : public AbstractScraper {
 
 private:
     QTimer limitTimer;
+    QTimer statusTimer;
     QEventLoop limiter;
     QList<QString> getSearchNames(const QFileInfo &info,
                                   QString &debug) override;
@@ -91,6 +92,8 @@ class ScreenScraper : public AbstractScraper {
     QString region;
     QString lang;
     QJsonObject jsonObj;
+    int timeout;
+    int tctr = 0;
 };
 
 #endif // SCREENSCRAPER_H