Add downloading of resources (icon for now)

CMakeLists.txt

@@ -27,7 +27,7 @@ set(CLIENT_VERSION_REVISION 1)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-find_package(Qt6Core REQUIRED)
+find_package(Qt6 COMPONENTS Core Network REQUIRED)
 
 if (CMAKE_VERSION VERSION_GREATER "3.29.9")
     # use the upstream boost info instead of the cmake-shipped one for finding

src/CMakeLists.txt

@@ -21,11 +21,13 @@ set(CMAKE_AUTORCC ON)
 add_executable(processor
     main.cpp
     Processor.h Processor.cpp
+    DownloadJob.h DownloadJob.cpp
 )
 target_compile_definitions(processor PRIVATE LOG_DEFAULT_SECTION=100)
 target_link_libraries(processor
     flowee_utils
     Qt6::Core
+    Qt6::Network
     Boost::filesystem
 )
 install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/processor DESTINATION ${CMAKE_BINARY_DIR}/bin)

src/DownloadJob.cpp

@@ -0,0 +1,60 @@
+#include "DownloadJob.h"
+
+#include <QNetworkRequest>
+#include <QNetworkReply>
+#include <QNetworkAccessManager>
+#include <QFile>
+#include <QFileInfo>
+#include <QDir>
+
+#include <utils/Logger.h>
+
+DownloadJob::DownloadJob(QNetworkAccessManager *parent)
+    : QObject{parent},
+    m_net(parent)
+{}
+
+void DownloadJob::start()
+{
+    assert(m_sourceUrl.isValid());
+    QNetworkRequest request(m_sourceUrl);
+    m_reply = m_net->get(request);
+    connect (m_reply, &QNetworkReply::finished, this, [=]() {
+        if (m_reply->error() != QNetworkReply::NoError) {
+            // TODO mark file as not available somehow.
+            logCritical() << "Download failed" << m_sourceUrl.toString();
+        }
+        else {
+            QFileInfo info(m_targetFilePath);
+            QDir root("/");
+            root.mkpath(info.absolutePath());
+            QFile out(m_targetFilePath);
+            if (!out.open(QIODevice::WriteOnly)) {
+                logCritical() << "Opening file for writing failed" << m_targetFilePath;
+            }
+            out.write(m_reply->readAll());
+            out.flush();
+        }
+        emit this->finished();
+    });
+}
+
+QString DownloadJob::targetFilePath() const
+{
+    return m_targetFilePath;
+}
+
+QUrl DownloadJob::sourceUrl() const
+{
+    return m_sourceUrl;
+}
+
+void DownloadJob::setSourceUrl(const QUrl &newSourceUrl)
+{
+    m_sourceUrl = newSourceUrl;
+}
+
+void DownloadJob::setTargetFilePath(const QString &newTargetFilePath)
+{
+    m_targetFilePath = newTargetFilePath;
+}

src/DownloadJob.h

@@ -0,0 +1,34 @@
+#ifndef DOWNLOADJOB_H
+#define DOWNLOADJOB_H
+
+#include <QObject>
+#include <QUrl>
+
+class QNetworkAccessManager;
+class QNetworkReply;
+
+class DownloadJob : public QObject
+{
+    Q_OBJECT
+public:
+    explicit DownloadJob(QNetworkAccessManager *parent = nullptr);
+
+    void start();
+
+    void setSourceUrl(const QUrl &newSourceUrl);
+    QUrl sourceUrl() const;
+    void setTargetFilePath(const QString &newTargetFilePath);
+    QString targetFilePath() const;
+
+signals:
+    void finished();
+
+private:
+    QString m_targetFilePath;
+    QUrl m_sourceUrl;
+
+    QNetworkReply *m_reply = nullptr;
+    QNetworkAccessManager *m_net;
+};
+
+#endif

src/Processor.cpp

@@ -1,10 +1,14 @@
 #include "Processor.h"
 
+#include "DownloadJob.h"
+
 #include <utils/Logger.h>
+#include <QCoreApplication>
 #include <QDirIterator>
 #include <QJsonArray>
 #include <QJsonDocument>
 #include <QJsonObject>
+#include <QTimer>
 #include <sha256.h>
 #include <uint256.h>
 
@@ -21,11 +25,11 @@ Processor::Processor(const QString &inDir, const QString &outDir)
 {
 }
 
-int Processor::run()
+bool Processor::run()
 {
     if (!QDir::current().mkpath(m_outDir)) {
         logCritical() << "Failed to create target dir";
-        return 1;
+        return false;
     }
     QDirIterator iter(m_inDir, QDir::NoDotAndDotDot | QDir::Files | QDir::Dirs, QDirIterator::Subdirectories);
     while (iter.hasNext()) {
@@ -42,6 +46,27 @@ int Processor::run()
             logCritical() << "Placing BCMR " << source->name << "as" << source->hash;
             QFile::copy(source->origFilename, outPath);
         }
+
+        for (const auto &resource : std::as_const(source->resources)) {
+            QUrl url(resource);
+            if (!url.isValid())
+                continue;
+            QString path = m_outDir + source->hash + "/" + url.host() + url.path();
+            if (QFile::exists(path))
+                continue;
+
+            if (url.scheme() == "http" || url.scheme() == "https") {
+                // schedule for download.
+                DownloadJob *job = new DownloadJob(&m_network);
+                job->setTargetFilePath(path);
+                job->setSourceUrl(url);
+                connect (job, &DownloadJob::finished, this, [=]() {
+                    m_downloadJobs.removeAll(job);
+                    QTimer::singleShot(1, this, SLOT(runDownloadQueue()));
+                });
+                m_downloadJobs.append(job);
+            }
+        }
     }
 
     logInfo() << "num categories found:" << m_categories.size();
@@ -91,12 +116,25 @@ int Processor::run()
         if (!out.open(QIODevice::WriteOnly)) {
             logCritical() << "Failed to open file for writing:" << outPath;
             logFatal() << "Giving up";
-            return 10;
+            return false;
         }
         out.write(memData);
     }
 
-    return 0;
+    QTimer::singleShot(1, this, SLOT(runDownloadQueue()));
+
+    return true;
+}
+
+void Processor::runDownloadQueue()
+{
+    if (m_downloadJobs.isEmpty()) {
+        QCoreApplication::quit();
+        return;
+    }
+    logInfo() << "DownloadQueue has" << m_downloadJobs.size() << "jobs";
+    auto *one = m_downloadJobs.at(0);
+    one->start();
 }
 
 void Processor::parseBCMR(const QString &path)
@@ -172,6 +210,14 @@ void Processor::parseBCMR(const QString &path)
             m_sources.push_back(me);
         }
         logInfo() << "Found BCMR for" << me->name;
+        const auto uris_ = revision["uris"];
+        if (uris_.isObject()) {
+            const auto uris = uris_.toObject();
+            auto icon = uris["icon"];
+            if (icon.isString())
+                me->resources.insert(icon.toString());
+        }
+
         const auto token_ = revision["token"];
         if (token_.isObject()) {
             const auto token = token_.toObject();

src/Processor.h

@@ -2,17 +2,25 @@
 #define PROCESSOR_H
 
 #include <QString>
+#include <QObject>
 #include <QFile>
 #include <vector>
 #include <set>
 #include <QDateTime>
+#include <QNetworkAccessManager>
 
-class Processor
+class DownloadJob;
+
+class Processor : QObject
 {
+    Q_OBJECT
 public:
     Processor(const QString &inDir, const QString &outDir);
 
-    int run();
+    bool run();
+
+private slots:
+    void runDownloadQueue();
 
 private:
     void parseBCMR(const QString &path);
@@ -27,6 +35,7 @@ private:
         QString timestampStr;
         QDateTime timestamp;
         std::vector<struct MetaCategory*> tokens;
+        QSet<QString> resources;
     };
 
     struct MetaCategory {
@@ -38,6 +47,9 @@ private:
     std::vector<MetaBCMR*> m_sources;
     typedef std::unordered_map<QString, MetaCategory*> CatMap;
     CatMap m_categories;
+
+    QNetworkAccessManager m_network;
+    QList<DownloadJob*> m_downloadJobs;
 };
 
 #endif

src/main.cpp

@@ -44,5 +44,8 @@ int main(int x, char **y) {
     logger->addConsoleChannel();
 
     Processor processor(args.at(0), args.at(1));
-    return processor.run();
+    if (!processor.run())
+        return 1;
+
+    return app.exec();
 }