From db7cec6935a66ed1fda9b18286f7df040ac59cd4 Mon Sep 17 00:00:00 2001 From: Eugene Sandulenko Date: Thu, 18 Nov 2021 21:19:18 +0100 Subject: [PATCH] CLOUD: SessionRequest now can save downloaded files to disk --- backends/networking/curl/session.cpp | 25 +++++--- backends/networking/curl/session.h | 2 +- backends/networking/curl/sessionrequest.cpp | 71 ++++++++++++++++++--- backends/networking/curl/sessionrequest.h | 25 +++++++- gui/downloadiconsdialog.cpp | 64 ++++++++++++------- gui/downloadiconsdialog.h | 1 + 6 files changed, 144 insertions(+), 44 deletions(-) diff --git a/backends/networking/curl/session.cpp b/backends/networking/curl/session.cpp index 958ede8c7b5..2a15d73727f 100644 --- a/backends/networking/curl/session.cpp +++ b/backends/networking/curl/session.cpp @@ -33,17 +33,17 @@ Session::~Session() { close(); } -SessionRequest *Session::get(Common::String url, DataCallback cb, ErrorCallback ecb, bool binary) { +static Common::String constructUrl(Common::String prefix, Common::String url) { // check url prefix - if (!_prefix.empty()) { + if (!prefix.empty()) { if (url.contains("://")) { - if (url.size() < _prefix.size() || url.find(_prefix) != 0) { - warning("Session: given URL does not match the prefix!\n\t%s\n\t%s", url.c_str(), _prefix.c_str()); - return nullptr; + if (url.size() < prefix.size() || url.find(prefix) != 0) { + warning("Session: given URL does not match the prefix!\n\t%s\n\t%s", url.c_str(), prefix.c_str()); + return ""; } } else { // if no schema given, just append to <_prefix> - Common::String newUrl = _prefix; + Common::String newUrl = prefix; if (newUrl.lastChar() != '/' && (url.size() > 0 && url.firstChar() != '/')) newUrl += "/"; newUrl += url; @@ -51,6 +51,15 @@ SessionRequest *Session::get(Common::String url, DataCallback cb, ErrorCallback } } + return url; +} + +SessionRequest *Session::get(Common::String url, Common::String localFile, DataCallback cb, ErrorCallback ecb, bool binary) { + url = constructUrl(_prefix, url); + + if (url.empty()) + return nullptr; + // check if request has finished (ready to be replaced) if (_request) { if (!_request->complete()) { @@ -60,10 +69,10 @@ SessionRequest *Session::get(Common::String url, DataCallback cb, ErrorCallback } if (!_request) { - _request = new Networking::SessionRequest(url, cb, ecb, binary); // automatically added to ConnMan + _request = new SessionRequest(url, localFile, cb, ecb, binary); // automatically added to ConnMan _request->connectionKeepAlive(); } else { - _request->reuse(url, cb, ecb); + _request->reuse(url, localFile, cb, ecb, binary); } return _request; diff --git a/backends/networking/curl/session.h b/backends/networking/curl/session.h index c3e82ff63f1..da8c3ca05c2 100644 --- a/backends/networking/curl/session.h +++ b/backends/networking/curl/session.h @@ -36,7 +36,7 @@ public: Session(Common::String prefix = ""); ~Session(); - SessionRequest *get(Common::String url, DataCallback cb = nullptr, ErrorCallback ecb = nullptr, bool binary = false); + SessionRequest *get(Common::String url, Common::String localFile, DataCallback cb = nullptr, ErrorCallback ecb = nullptr, bool binary = false); void close(); }; diff --git a/backends/networking/curl/sessionrequest.cpp b/backends/networking/curl/sessionrequest.cpp index c3943093862..8ad933ea0e8 100644 --- a/backends/networking/curl/sessionrequest.cpp +++ b/backends/networking/curl/sessionrequest.cpp @@ -27,15 +27,18 @@ #include "backends/networking/curl/networkreadstream.h" #include "backends/networking/curl/sessionrequest.h" #include "common/debug.h" +#include "common/file.h" #include "common/json.h" namespace Networking { -SessionRequest::SessionRequest(Common::String url, DataCallback cb, ErrorCallback ecb, bool binary): +SessionRequest::SessionRequest(Common::String url, Common::String localFile, DataCallback cb, ErrorCallback ecb, bool binary): CurlRequest(cb, ecb, url), _contentsStream(DisposeAfterUse::YES), - _buffer(new byte[CURL_SESSION_REQUEST_BUFFER_SIZE]), _text(nullptr), + _buffer(new byte[CURL_SESSION_REQUEST_BUFFER_SIZE]), _text(nullptr), _localFile(nullptr), _started(false), _complete(false), _success(false), _binary(binary) { + openLocalFile(localFile); + // automatically go under ConnMan control so nobody would be able to leak the memory // but, we don't need it to be working just yet _state = PAUSED; @@ -46,6 +49,22 @@ SessionRequest::~SessionRequest() { delete[] _buffer; } +void SessionRequest::openLocalFile(Common::String localFile) { + if (localFile.empty()) + return; + + _localFile = new Common::DumpFile(); + if (!_localFile->open(localFile, true)) { + warning("SessionRequestFile: unable to open file to download into"); + ErrorResponse error(this, false, true, "SessionRequestFile: unable to open file to download into", -1); + finishError(error); + delete _localFile; + return; + } + + _binary = true; // Enforce binary +} + bool SessionRequest::reuseStream() { if (!_stream) { return false; @@ -86,8 +105,18 @@ void SessionRequest::finishSuccess() { _complete = true; _success = true; - if (_callback) - (*_callback)(DataResponse(this, text())); + if (_callback && !_localFile) { // If localfile is present, we already called the callback + _response.buffer = _contentsStream.getData(); + _response.len = _contentsStream.size(); + _response.eos = true; + + (*_callback)(DataResponse(this, &_response)); + } + + if (_localFile) { + _localFile->close(); + _localFile = nullptr; + } } void SessionRequest::start() { @@ -105,7 +134,7 @@ void SessionRequest::startAndWait() { wait(); } -void SessionRequest::reuse(Common::String url, DataCallback cb, ErrorCallback ecb) { +void SessionRequest::reuse(Common::String url, Common::String localFile, DataCallback cb, ErrorCallback ecb, bool binary) { _url = url; delete _callback; @@ -113,6 +142,10 @@ void SessionRequest::reuse(Common::String url, DataCallback cb, ErrorCallback ec _callback = cb; _errorCallback = ecb; + _binary = binary; + + openLocalFile(localFile); + restart(); } @@ -127,9 +160,25 @@ void SessionRequest::handle() { return; } uint32 readBytes = _stream->read(_buffer, CURL_SESSION_REQUEST_BUFFER_SIZE); - if (readBytes != 0) - if (_contentsStream.write(_buffer, readBytes) != readBytes) - warning("SessionRequest: unable to write all the bytes into MemoryWriteStreamDynamic"); + if (readBytes != 0) { + if (!_localFile) { + if (_contentsStream.write(_buffer, readBytes) != readBytes) + warning("SessionRequest: unable to write all the bytes into MemoryWriteStreamDynamic"); + } else { + _response.buffer = _buffer; + _response.len = readBytes; + _response.eos = _stream->eos(); + + if (_localFile->write(_buffer, readBytes) != readBytes) { + warning("DownloadRequest: unable to write all received bytes into output file"); + finishError(Networking::ErrorResponse(this, false, true, "DownloadRequest::handle: failed to write all bytes into a file", -1)); + return; + } + + if (_callback) + (*_callback)(DataResponse(this, &_response)); + } + } if (_stream->eos()) { finishSuccess(); @@ -171,7 +220,7 @@ bool SessionRequest::success() { } char *SessionRequest::text() { - if (_binary) + if (_binary || _localFile) return nullptr; if (_text == nullptr) @@ -180,6 +229,10 @@ char *SessionRequest::text() { } Common::JSONValue *SessionRequest::json() { + if (_binary) + error("SessionRequest::json() is called for binary stream"); + if (_localFile) + error("SessionRequest::json() is called for localFile stream"); return Common::JSON::parse(text()); } diff --git a/backends/networking/curl/sessionrequest.h b/backends/networking/curl/sessionrequest.h index d3a1eca816a..7de34fafe1b 100644 --- a/backends/networking/curl/sessionrequest.h +++ b/backends/networking/curl/sessionrequest.h @@ -25,12 +25,28 @@ #include "backends/networking/curl/curlrequest.h" #include "common/memstream.h" -#include "common/json.h" + +namespace Common { +class DumpFile; +class JSONValue; +} namespace Networking { #define CURL_SESSION_REQUEST_BUFFER_SIZE 512 * 1024 +struct SessionFileResponse { + byte *buffer; + uint32 len; + bool eos; +}; + +/** + * @brief Class for reading file and storing locally + * + * @return Returns SessionFileResponse in the callback + */ + class SessionRequest: public CurlRequest { protected: Common::MemoryWriteStreamDynamic _contentsStream; @@ -38,6 +54,8 @@ protected: char *_text; bool _started, _complete, _success; bool _binary; + Common::DumpFile *_localFile; + SessionFileResponse _response; bool reuseStream(); @@ -46,15 +64,16 @@ protected: virtual void finishError(ErrorResponse error, RequestState state = PAUSED); virtual void finishSuccess(); + void openLocalFile(Common::String localFile); public: - SessionRequest(Common::String url, DataCallback cb = nullptr, ErrorCallback ecb = nullptr, bool binary = false); + SessionRequest(Common::String url, Common::String localFile, DataCallback cb = nullptr, ErrorCallback ecb = nullptr, bool binary = false); virtual ~SessionRequest(); void start(); void startAndWait(); - void reuse(Common::String url, DataCallback cb = nullptr, ErrorCallback ecb = nullptr); + void reuse(Common::String url, Common::String localFile, DataCallback cb = nullptr, ErrorCallback ecb = nullptr, bool binary = false); virtual void handle(); virtual void restart(); diff --git a/gui/downloadiconsdialog.cpp b/gui/downloadiconsdialog.cpp index 096dcea7dd5..5798ffb1c41 100644 --- a/gui/downloadiconsdialog.cpp +++ b/gui/downloadiconsdialog.cpp @@ -20,7 +20,7 @@ * */ -#include "backends/cloud/cloudmanager.h" +#include "backends/networking/curl/request.h" #include "gui/downloadiconsdialog.h" #include "gui/downloaddialog.h" #include "backends/networking/curl/session.h" @@ -51,9 +51,23 @@ struct DialogState { Networking::Session session; Common::HashMap fileHash; IconProcessState state; + uint32 downloadedsize; uint32 totalsize; + + DialogState() { state = kDownloadStateNone; downloadedsize = totalsize = 0; dialog = nullptr; } } static *g_state; +static uint32 getDownloadingProgress() { + if (!g_state) + return 0; + + return 100 * g_state->downloadedsize / (g_state->totalsize ? g_state->totalsize : 1); +} + +static uint32 getDownloadSpeed() { + return 0; +} + DownloadIconsDialog::DownloadIconsDialog() : Dialog("GlobalOptions_DownloadIconsDialog"), CommandSender(this), _close(false) { @@ -62,7 +76,7 @@ DownloadIconsDialog::DownloadIconsDialog() : _statusText = new StaticTextWidget(this, "GlobalOptions_DownloadIconsDialog.StatusText", _("Downloading icons list...")); _errorText = new StaticTextWidget(this, "GlobalOptions_DownloadIconsDialog.ErrorText", Common::U32String("")); - uint32 progress = (uint32)(100 * CloudMan.getDownloadingProgress()); + uint32 progress = getDownloadingProgress(); _progressBar = new SliderWidget(this, "GlobalOptions_DownloadIconsDialog.ProgressBar"); _progressBar->setMinValue(0); _progressBar->setMaxValue(100); @@ -74,8 +88,6 @@ DownloadIconsDialog::DownloadIconsDialog() : _cancelButton = new ButtonWidget(this, "GlobalOptions_DownloadIconsDialog.MainButton", _("Cancel download"), Common::U32String(), kDownloadCancelCmd); _closeButton = new ButtonWidget(this, "GlobalOptions_DownloadIconsDialog.CloseButton", _("Hide"), Common::U32String(), kCloseCmd); - CloudMan.setDownloadTarget(this); - if (!g_state) { g_state = new DialogState; @@ -94,7 +106,6 @@ DownloadIconsDialog::DownloadIconsDialog() : } DownloadIconsDialog::~DownloadIconsDialog() { - CloudMan.setDownloadTarget(nullptr); } void DownloadIconsDialog::open() { @@ -104,8 +115,6 @@ void DownloadIconsDialog::open() { } void DownloadIconsDialog::close() { - CloudMan.setDownloadTarget(nullptr); - if (g_state) g_state->dialog = nullptr; @@ -116,6 +125,7 @@ void DownloadIconsDialog::setState(IconProcessState state) { g_state->state = state; switch (state) { + case kDownloadStateNone: case kDownloadStateList: _statusText->setLabel(_("Downloading icons list...")); _cancelButton->setLabel(_("Cancel download")); @@ -201,7 +211,7 @@ void DownloadIconsDialog::handleTickle() { return; } - int32 progress = (int32)(100 * CloudMan.getDownloadingProgress()); + int32 progress = getDownloadingProgress(); if (_progressBar->getValue() != progress) { refreshWidgets(); g_gui.scheduleTopDialogRedraw(); @@ -217,30 +227,30 @@ void DownloadIconsDialog::reflowLayout() { Common::U32String DownloadIconsDialog::getSizeLabelText() { Common::String downloaded, downloadedUnits, total, totalUnits; - downloaded = getHumanReadableBytes(CloudMan.getDownloadBytesNumber(), downloadedUnits); - total = getHumanReadableBytes(CloudMan.getDownloadTotalBytesNumber(), totalUnits); + downloaded = getHumanReadableBytes(g_state->downloadedsize, downloadedUnits); + total = getHumanReadableBytes(g_state->totalsize, totalUnits); return Common::U32String::format(_("Downloaded %s %S / %s %S"), downloaded.c_str(), _(downloadedUnits).c_str(), total.c_str(), _(totalUnits).c_str()); } Common::U32String DownloadIconsDialog::getSpeedLabelText() { Common::String speed, speedUnits; - speed = getHumanReadableBytes(CloudMan.getDownloadSpeed(), speedUnits); + speed = getHumanReadableBytes(getDownloadSpeed(), speedUnits); speedUnits += "/s"; return Common::U32String::format(_("Download speed: %s %S"), speed.c_str(), _(speedUnits).c_str()); } void DownloadIconsDialog::refreshWidgets() { - uint32 progress = (uint32)(100 * CloudMan.getDownloadingProgress()); + uint32 progress = getDownloadingProgress(); _percentLabel->setLabel(Common::String::format("%u %%", progress)); _downloadSizeLabel->setLabel(getSizeLabelText()); _downloadSpeedLabel->setLabel(getSpeedLabelText()); _progressBar->setValue(progress); } -void DownloadIconsDialog::downloadListCallback(Networking::DataResponse response) { - Networking::SessionRequest *req = dynamic_cast(response.request); +void DownloadIconsDialog::downloadListCallback(Networking::DataResponse r) { + Networking::SessionFileResponse *response = static_cast(r.value); - Common::MemoryReadStream stream(req->getData(), req->getSize()); + Common::MemoryReadStream stream(response->buffer, response->len); int nline = 0; @@ -280,7 +290,7 @@ void DownloadIconsDialog::errorCallback(Networking::ErrorResponse error) { } void DownloadIconsDialog::downloadList() { - Networking::SessionRequest *rq = g_state->session.get("https://downloads.scummvm.org/frs/icons/LIST", + Networking::SessionRequest *rq = g_state->session.get("https://downloads.scummvm.org/frs/icons/LIST", "", new Common::Callback(this, &DownloadIconsDialog::downloadListCallback), new Common::Callback(this, &DownloadIconsDialog::errorCallback), true); @@ -318,27 +328,35 @@ void DownloadIconsDialog::calculateList() { } if (g_state->totalsize == 0) { - _statusText->setLabel(_("No new icons packs available")); + Common::U32String error(_("No new icons packs available")); + setError(error); return; } setState(kDownloadStateListCalculated); } -void DownloadIconsDialog::downloadFileCallback(Networking::DataResponse response) { - Networking::SessionRequest *req = dynamic_cast(response.request); +void DownloadIconsDialog::downloadFileCallback(Networking::DataResponse r) { + Networking::SessionFileResponse *response = static_cast(r.value); - warning("Got %d bytes", req->getSize()); + warning("Read %u bytes", response->len); + + if (response->eos) { + sendCommand(kDownloadEndedCmd, 0); + return; + } + + sendCommand(kDownloadProgressCmd, 0); } void DownloadIconsDialog::proceedDownload() { for (auto f = g_state->fileHash.begin(); f != g_state->fileHash.end(); ++f) { Common::String url = Common::String::format("https://downloads.scummvm.org/frs/icons/%s", f->_key.c_str()); + Common::String localFile = normalizePath(ConfMan.get("iconspath") + "/" + f->_key, '/'); - Networking::SessionRequest *rq = g_state->session.get(url, + Networking::SessionRequest *rq = g_state->session.get(url, localFile, new Common::Callback(this, &DownloadIconsDialog::downloadFileCallback), - new Common::Callback(this, &DownloadIconsDialog::errorCallback), - true); + new Common::Callback(this, &DownloadIconsDialog::errorCallback)); rq->start(); } diff --git a/gui/downloadiconsdialog.h b/gui/downloadiconsdialog.h index 0acdd41ce39..64e3fd66179 100644 --- a/gui/downloadiconsdialog.h +++ b/gui/downloadiconsdialog.h @@ -38,6 +38,7 @@ class ButtonWidget; class SliderWidget; enum IconProcessState { + kDownloadStateNone, kDownloadStateList, kDownloadStateListDownloaded, kDownloadStateListCalculated,