diff --git a/CMakeLists.txt b/CMakeLists.txt index cb8bb6c..450bb1c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ set(CLIENT_VERSION_REVISION 1) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -find_package(Qt6Core REQUIRED) +find_package(Qt6 COMPONENTS Core Network REQUIRED) if (CMAKE_VERSION VERSION_GREATER "3.29.9") # use the upstream boost info instead of the cmake-shipped one for finding diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9a1a95b..ffe483f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -21,11 +21,13 @@ set(CMAKE_AUTORCC ON) add_executable(processor main.cpp Processor.h Processor.cpp + DownloadJob.h DownloadJob.cpp ) target_compile_definitions(processor PRIVATE LOG_DEFAULT_SECTION=100) target_link_libraries(processor flowee_utils Qt6::Core + Qt6::Network Boost::filesystem ) install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/processor DESTINATION ${CMAKE_BINARY_DIR}/bin) diff --git a/src/DownloadJob.cpp b/src/DownloadJob.cpp new file mode 100644 index 0000000..fa8da29 --- /dev/null +++ b/src/DownloadJob.cpp @@ -0,0 +1,60 @@ +#include "DownloadJob.h" + +#include +#include +#include +#include +#include +#include + +#include + +DownloadJob::DownloadJob(QNetworkAccessManager *parent) + : QObject{parent}, + m_net(parent) +{} + +void DownloadJob::start() +{ + assert(m_sourceUrl.isValid()); + QNetworkRequest request(m_sourceUrl); + m_reply = m_net->get(request); + connect (m_reply, &QNetworkReply::finished, this, [=]() { + if (m_reply->error() != QNetworkReply::NoError) { + // TODO mark file as not available somehow. + logCritical() << "Download failed" << m_sourceUrl.toString(); + } + else { + QFileInfo info(m_targetFilePath); + QDir root("/"); + root.mkpath(info.absolutePath()); + QFile out(m_targetFilePath); + if (!out.open(QIODevice::WriteOnly)) { + logCritical() << "Opening file for writing failed" << m_targetFilePath; + } + out.write(m_reply->readAll()); + out.flush(); + } + emit this->finished(); + }); +} + +QString DownloadJob::targetFilePath() const +{ + return m_targetFilePath; +} + +QUrl DownloadJob::sourceUrl() const +{ + return m_sourceUrl; +} + +void DownloadJob::setSourceUrl(const QUrl &newSourceUrl) +{ + m_sourceUrl = newSourceUrl; +} + +void DownloadJob::setTargetFilePath(const QString &newTargetFilePath) +{ + m_targetFilePath = newTargetFilePath; +} diff --git a/src/DownloadJob.h b/src/DownloadJob.h new file mode 100644 index 0000000..2b6c972 --- /dev/null +++ b/src/DownloadJob.h @@ -0,0 +1,34 @@ +#ifndef DOWNLOADJOB_H +#define DOWNLOADJOB_H + +#include +#include + +class QNetworkAccessManager; +class QNetworkReply; + +class DownloadJob : public QObject +{ + Q_OBJECT +public: + explicit DownloadJob(QNetworkAccessManager *parent = nullptr); + + void start(); + + void setSourceUrl(const QUrl &newSourceUrl); + QUrl sourceUrl() const; + void setTargetFilePath(const QString &newTargetFilePath); + QString targetFilePath() const; + +signals: + void finished(); + +private: + QString m_targetFilePath; + QUrl m_sourceUrl; + + QNetworkReply *m_reply = nullptr; + QNetworkAccessManager *m_net; +}; + +#endif diff --git a/src/Processor.cpp b/src/Processor.cpp index 11f59a3..7eb0b2e 100644 --- a/src/Processor.cpp +++ b/src/Processor.cpp @@ -1,10 +1,14 @@ #include "Processor.h" +#include "DownloadJob.h" + #include +#include #include #include #include #include +#include #include #include @@ -21,11 +25,11 @@ Processor::Processor(const QString &inDir, const QString &outDir) { } -int Processor::run() +bool Processor::run() { if (!QDir::current().mkpath(m_outDir)) { logCritical() << "Failed to create target dir"; - return 1; + return false; } QDirIterator iter(m_inDir, QDir::NoDotAndDotDot | QDir::Files | QDir::Dirs, QDirIterator::Subdirectories); while (iter.hasNext()) { @@ -42,6 +46,27 @@ int Processor::run() logCritical() << "Placing BCMR " << source->name << "as" << source->hash; QFile::copy(source->origFilename, outPath); } + + for (const auto &resource : std::as_const(source->resources)) { + QUrl url(resource); + if (!url.isValid()) + continue; + QString path = m_outDir + source->hash + "/" + url.host() + url.path(); + if (QFile::exists(path)) + continue; + + if (url.scheme() == "http" || url.scheme() == "https") { + // schedule for download. + DownloadJob *job = new DownloadJob(&m_network); + job->setTargetFilePath(path); + job->setSourceUrl(url); + connect (job, &DownloadJob::finished, this, [=]() { + m_downloadJobs.removeAll(job); + QTimer::singleShot(1, this, SLOT(runDownloadQueue())); + }); + m_downloadJobs.append(job); + } + } } logInfo() << "num categories found:" << m_categories.size(); @@ -91,12 +116,25 @@ int Processor::run() if (!out.open(QIODevice::WriteOnly)) { logCritical() << "Failed to open file for writing:" << outPath; logFatal() << "Giving up"; - return 10; + return false; } out.write(memData); } - return 0; + QTimer::singleShot(1, this, SLOT(runDownloadQueue())); + + return true; +} + +void Processor::runDownloadQueue() +{ + if (m_downloadJobs.isEmpty()) { + QCoreApplication::quit(); + return; + } + logInfo() << "DownloadQueue has" << m_downloadJobs.size() << "jobs"; + auto *one = m_downloadJobs.at(0); + one->start(); } void Processor::parseBCMR(const QString &path) @@ -172,6 +210,14 @@ void Processor::parseBCMR(const QString &path) m_sources.push_back(me); } logInfo() << "Found BCMR for" << me->name; + const auto uris_ = revision["uris"]; + if (uris_.isObject()) { + const auto uris = uris_.toObject(); + auto icon = uris["icon"]; + if (icon.isString()) + me->resources.insert(icon.toString()); + } + const auto token_ = revision["token"]; if (token_.isObject()) { const auto token = token_.toObject(); diff --git a/src/Processor.h b/src/Processor.h index 490b9f3..9a5c8a5 100644 --- a/src/Processor.h +++ b/src/Processor.h @@ -2,17 +2,25 @@ #define PROCESSOR_H #include +#include #include #include #include #include +#include -class Processor +class DownloadJob; + +class Processor : QObject { + Q_OBJECT public: Processor(const QString &inDir, const QString &outDir); - int run(); + bool run(); + +private slots: + void runDownloadQueue(); private: void parseBCMR(const QString &path); @@ -27,6 +35,7 @@ private: QString timestampStr; QDateTime timestamp; std::vector tokens; + QSet resources; }; struct MetaCategory { @@ -38,6 +47,9 @@ private: std::vector m_sources; typedef std::unordered_map CatMap; CatMap m_categories; + + QNetworkAccessManager m_network; + QList m_downloadJobs; }; #endif diff --git a/src/main.cpp b/src/main.cpp index 6a50cc3..efae055 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -44,5 +44,8 @@ int main(int x, char **y) { logger->addConsoleChannel(); Processor processor(args.at(0), args.at(1)); - return processor.run(); + if (!processor.run()) + return 1; + + return app.exec(); }