From 9612dbacb623a232daa6f8bbbb357b5644fcafbd Mon Sep 17 00:00:00 2001 From: Gemba Date: Thu, 5 Feb 2026 18:38:01 +0100 Subject: [PATCH] mitigation of Screenscraper.fr long running responses on some games --- src/netcomm.cpp | 5 +++-- src/netcomm.h | 1 + src/screenscraper.cpp | 40 +++++++++++++++++++++++++++++++++++----- src/screenscraper.h | 3 +++ 4 files changed, 42 insertions(+), 7 deletions(-) diff --git a/src/netcomm.cpp b/src/netcomm.cpp index f93035b6..899bf662 100644 --- a/src/netcomm.cpp +++ b/src/netcomm.cpp @@ -34,7 +34,6 @@ constexpr int DL_MAXSIZE = 100 * 1000 * 1000; NetComm::NetComm(QSharedPointer manager, int timeout) : manager(manager), timeout(timeout) { requestTimer.setSingleShot(true); - requestTimer.setInterval(timeout * 1000); connect(&requestTimer, &QTimer::timeout, this, &NetComm::requestTimeout); } @@ -71,7 +70,7 @@ void NetComm::request(QString query, QString postData, connect(reply, &QNetworkReply::finished, this, &NetComm::replyReady); connect(reply, &QNetworkReply::downloadProgress, this, &NetComm::dataDownloaded); - requestTimer.start(); + requestTimer.start(timeout * 1000); } void NetComm::replyReady() { @@ -172,3 +171,5 @@ void NetComm::requestTimeout() { timeout); reply->abort(); } + +void NetComm::setTimeout(int secs) { timeout = secs; } diff --git a/src/netcomm.h b/src/netcomm.h index 66625db7..73812ca8 100644 --- a/src/netcomm.h +++ b/src/netcomm.h @@ -47,6 +47,7 @@ class NetComm : public QObject { QByteArray getContentType(); QByteArray getRedirUrl(); QString getHeaderValue(const QString headerKey); + void setTimeout(int secs); private slots: void replyReady(); diff --git a/src/screenscraper.cpp b/src/screenscraper.cpp index 3e678a57..31fa3065 100644 --- a/src/screenscraper.cpp +++ b/src/screenscraper.cpp @@ -36,8 +36,10 @@ #include #include -constexpr int RETRIESMAX = 4; -constexpr int TIMEOUT_SEC = 60; +constexpr int RETRIESMAX = 3; +constexpr int TIMEOUT_SEC = 75; +constexpr int DELAY_NOTE_AFTER_SEC = 15; + constexpr int MINARTSIZE = 256; ScreenScraper::ScreenScraper(Settings *config, @@ -50,6 +52,16 @@ ScreenScraper::ScreenScraper(Settings *config, limitTimer.setSingleShot(false); limitTimer.start(); + connect(&statusTimer, &QTimer::timeout, this, [=]() { + tctr = tctr + 1; + if (tctr >= DELAY_NOTE_AFTER_SEC) { + printf(" \033[1;32m%2s\033[0mResponse delayed for " + "%ds\r", + tctr % 2 ? " •" : "• ", 5 * ((int)(tctr / 5))); + fflush(stdout); + } + }); + baseUrl = "http://www.screenscraper.fr"; fetchOrder.append(GameEntry::Elem::PUBLISHER); @@ -94,36 +106,48 @@ void ScreenScraper::getSearchResults(QList &gameEntries, (platformId == -1 ? "" : "&systemeid=" + QString::number(platformId)) + "&output=json&" + searchName; + tctr = 0; + statusTimer.start(1000); + for (int retries = 0; retries < RETRIESMAX; ++retries) { limiter.exec(); netComm->request(gameUrl); q.exec(); data = netComm->getData(); - QByteArray headerData = - data.left(1024); // Minor optimization with minimal more RAM usage + // Minor optimization with minimal more RAM usage + QByteArray headerData = data.left(1024); // Do error checks on headerData. It's more stable than checking the // potentially faulty JSON if (headerData.isEmpty()) { - printf("\033[1;33mRetrying request...\033[0m\n\n"); + int timeout = TIMEOUT_SEC << (1 + retries); + printf( + "\033[1;33mRetrying request with timeout of %ds...\033[0m\n\n", + timeout); + netComm->setTimeout(timeout); + tctr = 0; continue; } else if (headerData.contains("non trouvée")) { + statusTimer.stop(); return; } else if (headerData.contains("API totalement fermé")) { printf("\033[1;31mThe ScreenScraper API is currently closed, " "exiting nicely...\033[0m\n\n"); + statusTimer.stop(); reqRemaining = 0; return; } else if (headerData.contains( "Le logiciel de scrape utilisé a été blacklisté")) { printf("\033[1;31mSkyscraper has apparently been blacklisted at " "ScreenScraper, exiting nicely...\033[0m\n\n"); + statusTimer.stop(); reqRemaining = 0; return; } else if (headerData.contains("Votre quota de scrape est")) { printf("\033[1;31mYour daily ScreenScraper request limit has been " "reached, exiting nicely...\033[0m\n\n"); reqRemaining = 0; + statusTimer.stop(); return; } else if ( headerData.contains("API fermé pour les non membres") || @@ -147,12 +171,18 @@ void ScreenScraper::getSearchResults(QList &gameEntries, Config::getSkyFolder().toStdString().c_str()); if (retries == RETRIESMAX - 1) { reqRemaining = 0; + statusTimer.stop(); return; } else { continue; } } + if (tctr >= DELAY_NOTE_AFTER_SEC) { + printf("Response after %ds \n", tctr); + } + statusTimer.stop(); + // Fix faulty JSON that is sometimes received back from ScreenScraper data.replace("],\n\t\t}", "]\n\t\t}"); diff --git a/src/screenscraper.h b/src/screenscraper.h index 4ef11f5e..0346d960 100644 --- a/src/screenscraper.h +++ b/src/screenscraper.h @@ -47,6 +47,7 @@ class ScreenScraper : public AbstractScraper { private: QTimer limitTimer; + QTimer statusTimer; QEventLoop limiter; QList getSearchNames(const QFileInfo &info, QString &debug) override; @@ -91,6 +92,8 @@ class ScreenScraper : public AbstractScraper { QString region; QString lang; QJsonObject jsonObj; + int timeout; + int tctr = 0; }; #endif // SCREENSCRAPER_H