thehub/unspentdb/ExportCommand.cpp

/*
 * This file is part of the Flowee project
 * Copyright (C) 2018-2024 Tom Zander <tom@flowee.org>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "ExportCommand.h"

#include <boost/iostreams/device/mapped_file.hpp>
#include <boost/filesystem.hpp>
#include <qfile.h>

static void nothing(const char *){}

ExportCommand::ExportCommand()
    : m_filename(QStringList() << "o" << "output", "The [FILE] to output to", "FILE"),
    m_forceHeight(QStringList() << "height", "Read DB at the state 'height'"),
    m_noOutput("no-output", "Skip the actual export")
{
}

ExportCommand::~ExportCommand()
{
    delete m_outFileStream;
    delete m_device;
}

QString ExportCommand::commandDescription() const
{
    return "Export\nExports the database to either stdout or to a file.";
}

void ExportCommand::addArguments(QCommandLineParser &commandLineParser)
{
    commandLineParser.addOption(m_filename);
    commandLineParser.addOption(m_forceHeight);
    commandLineParser.addOption(m_noOutput);
}

void ExportCommand::write(const AbstractCommand::Leaf &leaf)
{
    assert(m_outFileStream);
    QTextStream &out = *m_outFileStream;

    out << QString::fromStdString(leaf.txid.GetHex());
    out << "," << leaf.outIndex;
    out << "," << leaf.blockHeight;
    out << "," << leaf.offsetInBlock << Qt::endl;
}

Flowee::ReturnCodes ExportCommand::run()
{
    assert (!dbDataFiles().isEmpty());
    if (dbDataFiles().at(0).filetype() == Datadir) {

    }
    else if (dbDataFiles().length() != 1
            || dbDataFiles().first().databaseFiles().length() != 1) {
        err << "Please select exactly one database file" << Qt::endl;
        return Flowee::InvalidOptions;
    }

    int target = commandLineParser().value(m_forceHeight).toInt();
    const bool targetGiven = target;
    if (!targetGiven) {
        // find the highest
        for (auto const &db : dbDataFiles()) {
            for (auto const &info : db.infoFiles()) {
                const auto checkpoint = readInfoFile(info.filepath());
                if (checkpoint.lastBlockHeight > target)
                    target = checkpoint.lastBlockHeight;
            }
        }
    }

    out << "Starting Export" << Qt::endl;
    out << " height: " << target << Qt::endl;
    QList<DatabaseFile> infoFiles;
    for (auto const &db : dbDataFiles()) {
        bool included = true;
        for (auto const &info : db.infoFiles()) {
            const auto checkpoint = readInfoFile(info.filepath());
            if (checkpoint.lastBlockHeight == target) {
                assert(!included); // can't have two info files that are duplicates
                infoFiles.append(info);
                included = true;
            }
        }
        if (!included)
            out << " WARNING: chosen height means we skip export of" << db.filepath() << Qt::endl;
    }

    const bool noOutput = commandLineParser().isSet(m_noOutput);
    if (!noOutput && commandLineParser().isSet(m_filename)) {
        QString filename = commandLineParser().value(m_filename);
        QFile *file = new QFile(filename);
        if (!file->open(QIODevice::WriteOnly)) {
            err << "Failed to open out file" << Qt::endl;
            delete file;
            return Flowee::CommandFailed;
        } else {
            m_device = file;
            m_outFileStream = new QTextStream(m_device);
        }
    }

    if (!noOutput) {
        if (m_outFileStream == nullptr)
            m_outFileStream = new QTextStream(stdout);
        *m_outFileStream << "# txid,outindex,blockheight,offsetinblock" << Qt::endl;
    }

    qint64 recordCount = 0;
    for (auto const &infoFile : infoFiles) {
        qint64 dbRecordCount = 0;
        const auto dbfiles = infoFile.databaseFiles();
        assert(dbfiles.count() == 1);
        out << "Exporting: " << dbfiles.at(0).filepath() << Qt::endl;
        auto info = infoFile.filepath();
        if (info.indexOf('/') >= 0)
            info = info.mid(info.lastIndexOf('/'));
        out << " + using: " << info << Qt::endl;
        // get data from info file.
        const auto checkpoint = readInfoFile(infoFile.filepath());
        if (checkpoint.jumptableFilepos < 0)
            return Flowee::CommandFailed;
        uint32_t jumptables[0x100000];
        if (!readJumptables(infoFile.filepath(), checkpoint.jumptableFilepos, jumptables))
            return Flowee::CommandFailed;
        if (checkpoint.jumptableHash != calcChecksum(jumptables))
            out << "Checkpoint CHECKSUM Failed" << Qt::endl;

        auto files = infoFile.databaseFiles();
        const DatabaseFile db = files.first();
        boost::iostreams::mapped_file file;
        file.open(db.filepath().toStdString(), std::ios_base::binary | std::ios_base::in);
        if (!file.is_open()) {
            err << "Failed to open db file " << db.filepath() << Qt::endl;
            return Flowee::CommandFailed;
        }
        std::shared_ptr<char> buffer = std::shared_ptr<char>(const_cast<char*>(file.const_data()), nothing);

        // read buckets
        for (int shorthash = 0; shorthash < 0x100000; ++shorthash) {
            if (jumptables[shorthash] == 0)
                continue;
            int32_t bucketOffsetInFile = static_cast<int>(jumptables[shorthash]);
            Streaming::ConstBuffer buf(buffer, buffer.get() + bucketOffsetInFile, buffer.get() + file.size());
            std::vector<LeafRef> leafPositions = readBucket(buf, bucketOffsetInFile);
            for (auto leaf : leafPositions) {
                Streaming::ConstBuffer leafBuf(buffer, buffer.get() + leaf.pos, buffer.get() + file.size());
                AbstractCommand::Leaf l = readLeaf(leafBuf, leaf.cheapHash);
                if (!noOutput)
                    write(l);
                ++recordCount;
                ++dbRecordCount;
            }
        }
        out << " + Contains records: " << dbRecordCount << Qt::endl;
    }
    if (infoFiles.size() > 1)
        out << "Total records: " << recordCount << Qt::endl;
    return Flowee::Ok;
}