registry/src/Processor.cpp

#include "Processor.h"

#include "DownloadJob.h"

#include <utils/Logger.h>
#include <QCoreApplication>
#include <QDirIterator>
#include <QJsonArray>
#include <QJsonDocument>
#include <QJsonObject>
#include <QTimer>
#include <sha256.h>
#include <uint256.h>

// if the passed path doesn't have a trailing slash, add one.
static QString pathWithSlash(const QString &input) {
    if (input.endsWith('/'))
        return input;
    return input + '/';
}

static bool walk(QJsonObject &item, const QStringList &steps) {
    for (const auto &step : steps) {
        auto x = item[step];
        if (!x.isObject())
            return false;
        item = x.toObject();
    }
    return true;
}


Processor::Processor(const QString &inDir, const QString &outDir)
    : m_inDir(inDir),
    m_outDir(pathWithSlash(outDir))
{
}

bool Processor::run()
{
    if (!QDir::current().mkpath(m_outDir)) {
        logCritical() << "Failed to create target dir";
        return false;
    }
    QDirIterator iter(m_inDir, QDir::NoDotAndDotDot | QDir::Files | QDir::Dirs, QDirIterator::Subdirectories);
    while (iter.hasNext()) {
        const QString path = iter.next();
        if (path.indexOf("bcmrs/", m_inDir.size()) == m_inDir.size()) {
            parseBCMR(path);
        }
    }

    for (auto *source : m_sources) {
        assert(source);
        assert(source->identities.size() > 0);
        QString outPath = m_outDir + source->hash + ".json";
        if (!QFile::exists(outPath)) {
            logCritical() << "Placing BCMR " << source->identities.at(0)->name << "as" << source->hash;
            QFile::copy(source->origFilename, outPath);
        }

        for (auto *id : source->identities) {
            for (const auto &resource : std::as_const(id->resources)) {
                QUrl url(resource);
                if (!url.isValid())
                    continue;
                QString path = m_outDir + source->hash + "/" + url.host() + url.path();
                if (QFile::exists(path))
                    continue;
                if (url.scheme() == "ipfs") {
                    // use the main gateway for download
                    url = QUrl(QString("https://ipfs.io/ipfs/%1").arg(resource.mid(7)));
                }
                if (url.scheme() == "http" || url.scheme() == "https") {
                    // schedule for download.
                    DownloadJob *job = new DownloadJob(&m_network);
                    job->setTargetFilePath(path);
                    job->setSourceUrl(url);
                    connect (job, &DownloadJob::finished, this, [=]() {
                        m_downloadJobs.removeAll(job);
                        QTimer::singleShot(1, this, SLOT(runDownloadQueue()));
                    });
                    m_downloadJobs.append(job);
                }
            }
        }
    }

    logInfo() << "num categories found:" << m_categories.size();
    for (auto i = m_categories.begin(); i != m_categories.end(); ++i) {
        MetaCategory *mc = i->second;
        QJsonArray sources;
        auto owners = std::vector<MetaBCMR*>(mc->owners.begin(), mc->owners.end());
        std::sort(owners.begin(), owners.end(), [mc](MetaBCMR *a, MetaBCMR *b) {
            // we're sorting the bcmr itself, but we only really care about a
            // single identity inside each of them. THAT is what we'll compare.
            MetaIdentity *ai = a->idForCategory(mc->category);
            assert(ai);
            MetaIdentity *bi = b->idForCategory(mc->category);
            assert(bi);
            return ai->timestamp > bi->timestamp;
        });

        for (auto owner : owners) {
            QJsonObject o;
            o.insert("bcmr", owner->hash);
            MetaIdentity *id = owner->idForCategory(mc->category);
            o.insert("name", id->name);
            o.insert("timestamp", id->timestampStr);
            sources.append(o);
        }
        QJsonObject root;
        root.insert("sources", sources);
        QJsonDocument doc;
        doc.setObject(root);
        auto memData = doc.toJson(QJsonDocument::Compact);

        QString outPath = m_outDir + mc->category + ".json";
        if (QFile::exists(outPath)) {
            // only overwrite if different.
            QFile disk(outPath);
            if (disk.open(QIODevice::ReadOnly)) {
                CSHA256 hasher;
                auto fileData = disk.readAll();
                disk.close();
                hasher.write(fileData.constData(), fileData.size());
                fileData = QByteArray();
                char fileHash[CSHA256::OUTPUT_SIZE];
                hasher.finalize(fileHash);
                hasher.reset();
                hasher.write(memData.constData(), memData.size());
                char memHash[CSHA256::OUTPUT_SIZE];
                hasher.finalize(memHash);
                if (memcmp(fileHash, memHash, sizeof(memHash)) == 0)
                    continue;
            }
        }
        logCritical() << "Placing or updating category file" << mc->category;
        QFile out(outPath);
        if (!out.open(QIODevice::WriteOnly)) {
            logCritical() << "Failed to open file for writing:" << outPath;
            logFatal() << "Giving up";
            return false;
        }
        out.write(memData);
    }

    QTimer::singleShot(1, this, SLOT(runDownloadQueue()));

    return true;
}

void Processor::runDownloadQueue()
{
    if (m_offline || m_downloadJobs.isEmpty()) {
        QCoreApplication::quit();
        return;
    }
    logInfo() << "DownloadQueue has" << m_downloadJobs.size() << "jobs";
    auto *one = m_downloadJobs.at(0);
    one->start();
}

void Processor::parseBCMR(const QString &path)
{
    QFile in(path);
    if (in.open(QIODevice::ReadOnly)) {
        char signature[4];
        in.read(signature, 4);
        bool isTx = true;
        int isJson = 0;
        for (int i = 0; i < 4; ++i) {
            uint8_t k = static_cast<uint8_t>(signature[i]);
            if (k < 10) {
                isTx = isTx && true;
                isJson = 10;
            }
            if (k == ' ' || k == '\n' || k == '\r' || k == '\t') {
                if (isJson < 1)
                    isJson = 1;
            } else if (isJson < 2 && k == '{') {
                isJson = 2;
            }
            else if (isJson == 2 && (k < 34 || k > 'z')) {
                isJson = 10; // fail
            }
        }
        if (isJson == 0 && isTx) {
            parseBCHTx(in);
            return;
        }
        else if (isJson != 2) {
            logCritical() << "Magic detection of file:" << path << "failed. What is it?";
            return;
        }
    }
    // still here, then it is a json.
    in.seek(0);
    auto data = in.readAll();
    QJsonDocument doc = QJsonDocument::fromJson(data);
    in.close();
    if (!doc.isObject()) {
        logCritical() << "Failed to parse json document:" << path;
        return;
    }
    auto ids = doc.object().value("identities").toObject();
    if (ids.isEmpty()) {
        logCritical() << "No identities in BCMR:" << path;
        return;
    }
    MetaBCMR *me = nullptr;
    for (auto identity = ids.begin(); identity != ids.end(); ++identity) {
        // The 'key' can be a category or an auth-base, relevant unless
        // duplicated in the 'token' section below.
        auto revisions = identity->toObject();
        auto revisionDates = revisions.keys(); // sorted list.
        if (revisionDates.isEmpty())
            continue;
        const auto revision = revisions[revisionDates.back()].toObject();
        if (revision.isEmpty())
            continue;
        if (me == nullptr) {
            me = new MetaBCMR();
            me->origFilename = path;
            // calc hash
            CSHA256 hasher;
            hasher.write(data.constData(), data.size());
            char buf[CSHA256::OUTPUT_SIZE];
            hasher.finalize(buf);
            me->hash = QByteArray(buf, sizeof(buf)).toHex();

            m_sources.push_back(me);
        }
        MetaIdentity *id = new MetaIdentity();
        id->name = revision["name"].toString();
        id->timestampStr = revisionDates.back();
        id->timestamp = QDateTime::fromString(id->timestampStr, Qt::ISODate);
        id->identity = identity.key();
        me->identities.push_back(id);

        logInfo() << "Found BCMR" << id->name << identity.key();
        const auto uris_ = revision["uris"];
        if (uris_.isObject()) {
            const auto uris = uris_.toObject();
            auto icon = uris["icon"];
            if (icon.isString())
                id->resources.insert(icon.toString());
        }

        const auto token_ = revision["token"];
        if (token_.isObject()) {
            const auto token = token_.toObject();
            const auto cat = token["category"].toString();
            if (cat.size() != 64) {
                logCritical() << "found token, but category is invalid length" << cat.size();
            } else {
                id->category = cat;
                MetaCategory *mc = fetchOrCreate(cat);
                id->tokens.push_back(mc);
                mc->owners.insert(me);
                // walk all nfts and extract image and icon resource urls
                auto types = token;
                if (!walk(types, QStringList() << "nfts" << "parse" << "types"))
                    continue;
                for (auto iter = types.begin(); iter != types.end(); ++iter) {
                    if (!iter->isObject())
                        continue;
                    auto uris = iter->toObject();
                    if (!walk(uris, QStringList() << "uris"))
                        continue;
                    auto image = uris["image"];
                    if (image.isString())
                        id->resources.insert(image.toString());
                    auto icon = uris["icon"];
                    if (icon.isString())
                        id->resources.insert(icon.toString());

                }
            }
        }
    }
}

void Processor::parseBCHTx(QFile &file)
{
    // TODO
    logFatal() << "parse tx is a TODO";
}

bool Processor::offline() const
{
    return m_offline;
}

void Processor::setOffline(bool newOffline)
{
    m_offline = newOffline;
}

Processor::MetaCategory *Processor::fetchOrCreate(const QString &catId)
{
    auto iter = m_categories.find(catId);
    if (iter != m_categories.end()) {
        return iter->second;
    }

    auto *mc = new MetaCategory();
    mc->category = catId;
    m_categories.insert({mc->category, mc});

    return mc;
}

Processor::MetaIdentity *Processor::MetaBCMR::idForCategory(const QString &category) const
{
    for (auto *i : identities) {
        if (i->category == category) {
            return i;
        }
    }
    return nullptr;
}