pay/modules/blocks/BlockHeadersChecker.cpp

/*
 * This file is part of the Flowee project
 * Copyright (C) 2024-2025 Tom Zander <tom@flowee.org>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "BlockHeadersChecker.h"
#include "FloweePay.h"

#include <utils/Logger.h>
#include <utils/streaming/BufferPools.h>
#include <p2p/Blockchain.h>
#include <QTimer>
#include <QNetworkReply>

static constexpr const char * HEADERFILE = "https://flowee.org/products/pay/blockheaders";

#define TEMP_DOWNLOAD_FILENAME "module_blocks_newheaders"
#define PAY_STATIC_HEADERS "staticHeaders"

BlockHeadersChecker::BlockHeadersChecker(QObject *parent)
    : QObject(parent)
{
    connect (this, &BlockHeadersChecker::finishUp, this, [=](Status status) {
        this->setStatus(status);
    }, Qt::QueuedConnection);
}

int BlockHeadersChecker::wantedHeight() const
{
    return m_wantedHeight;
}

void BlockHeadersChecker::setWantedHeight(int h)
{
    if (m_wantedHeight == h)
        return;
    m_wantedHeight = h;
    emit wantedHeightChanged();

    QTimer::singleShot(0, this, SLOT(startChecking()));
}

void BlockHeadersChecker::startChecking()
{
    logInfo(10054) << "starting" << m_wantedHeight;
    assert(m_checkpoint >= 0);
    if (m_wantedHeight == 0 || FloweePay::instance()->p2pNet()->chain() != P2PNet::MainChain) {
        setStatus(NoDownloadNeeded);
        return;
    }
    setStatus(Checking);
    const auto &blockchain = FloweePay::instance()->p2pNet()->blockchain();
    bool have = false;
    const auto sources = blockchain.dataSources();
    assert(!sources.empty());
    for (const auto &source : sources) {
        if (source.startBlock <= m_wantedHeight)
            have = true;
    }
    if (have) {
        logDebug(10054) << "Nothing to do, the blockchain has the needed headers";
        setStatus(NoDownloadNeeded);
        return;
    }

    int startHeight = 0;
    for (const auto &cp : blockchain.checkpoints()) {
        if (cp.height < m_wantedHeight)
            startHeight = cp.height;
        else
            break; // checkpoints are ordered.
    }

    m_checkpoint = startHeight;
    m_downloadTo = sources.front().startBlock;

    // do a HEAD
    if (m_headerReply == nullptr) {
        QNetworkRequest headRequest((QUrl(HEADERFILE)));
        headRequest.setTransferTimeout(6000);
        m_headerReply = FloweePay::instance()->network()->head(headRequest);
        connect(m_headerReply, SIGNAL(finished()), this, SLOT(headerReturned()));
    }
}

void BlockHeadersChecker::headerReturned()
{
    assert(m_headerReply);
    auto reply = m_headerReply;
    m_headerReply->deleteLater();
    m_headerReply = nullptr;

    if (m_newHeaders) {
        logFatal(10054) << "Already downloading...";
        return;
    }
    if (reply->error() != QNetworkReply::NoError) {
        logDebug(10054) << reply->errorString();
        setStatus(NetworkFailure);
        return;
    }

    uint64_t length = reply->header(QNetworkRequest::ContentLengthHeader).toLongLong();
    auto allowedRange = reply->rawHeader("Accept-Ranges");

    if (allowedRange != "bytes") {
        setStatus(NetworkFailure);
        logFatal(10054) << "Can't download partial files. Failing";
        return;
    }

    // Check if this is a continuation download.
    assert(m_downloadTo > m_checkpoint);
    assert(m_checkpoint > 0);
    m_newHeaders = new QFile(TEMP_DOWNLOAD_FILENAME, this);
    bool isContinuation = false;
    if (m_newHeaders->open(QIODevice::ReadOnly)) {
        char bytes[80];
        auto read = m_newHeaders->read(bytes, 80);
        if (read == 80) {
            BlockHeader *bh = reinterpret_cast<BlockHeader*>(bytes);
            auto hash = bh->createHash();
            auto &blockchain = FloweePay::instance()->p2pNet()->blockchain();
            for (const auto &cp : blockchain.checkpoints()) {
                if (cp.height == m_checkpoint) {
                    isContinuation = cp.hash == hash;
                    logCritical(10054) << "Partial file found of" << m_newHeaders->size()
                            << "bytes. Continuing!";
                    break;
                }
            }
        }
        m_newHeaders->close();
    }

    // We're going to download the headers.
    // On the remote server there is one file that has the genesis
    // all the way up to a recent height=((length / 80) - 1)

    // We use the 'range' feature of the webserver to only download
    // what we need, so lets calculate the offsets here.
    const int from = m_checkpoint * 80 + (isContinuation ? m_newHeaders->size() : 0);
    const int to = m_downloadTo * 80;
    if (to > static_cast<int>(length)) {
        logCritical(10054) << "Remote file does not have the data we want";
        setStatus(NetworkFailure);
        return;
    }

    QNetworkRequest downloadRequest((QUrl(HEADERFILE)));
    downloadRequest.setTransferTimeout(30000);
    QString range("bytes=%1-%2");
    range = range.arg(from);
    range = range.arg(to - 1); // -1 because ranges are inclusive.
    downloadRequest.setRawHeader("Range", range.toLatin1());

    if (!m_newHeaders->open(QIODevice::WriteOnly
                | (isContinuation ? QIODevice::Append : QIODevice::Truncate))) {
        logFatal(10054) << "Can not open file...";
        setStatus(DiskFailure);
        return;
    }
    setTotalDownload(to - from);
    setStatus(Downloading);

    // actually start the download
    m_downloadReply = FloweePay::instance()->network()->get(downloadRequest);
    connect(m_downloadReply, SIGNAL(readyRead()), this, SLOT(onBytesDownloaded()));
    connect(m_downloadReply, SIGNAL(finished()), this, SLOT(downloadFinished()));
}

void BlockHeadersChecker::downloadFinished()
{
    assert(m_downloadReply);
    auto reply = m_downloadReply;
    m_downloadReply->deleteLater();
    m_downloadReply = nullptr;

    if (reply->error() != QNetworkReply::NoError) {
        logCritical(10054) << "headers download gave:" << reply->errorString();
        setStatus(NetworkFailure);
        m_newHeaders->close();
        m_newHeaders->deleteLater();
        m_newHeaders = nullptr;
        return;
    }

    assert(m_newHeaders->isOpen());
    logDebug(10054) << "Wrote out:" << m_newHeaders->size() << "bytes" << (m_newHeaders->size() / 80);
    // --- use the file:
    // the new file is the headers I didn't have yet. We need to
    // create a new file that has both the old static headers as well
    // as the new ones concatenated in one.
    // the file we put this in has to have an application-long lifetime.
    // so lets give ownership to the main app.

    QFile oldStaticHeaders(PAY_STATIC_HEADERS);
    if (!oldStaticHeaders.open(QIODevice::ReadOnly)) {
        logFatal(10054) << "Failed to find old static headers";
        setStatus(DiskFailure);
        return;
    }
    logDebug(10054) << " old ones:" << oldStaticHeaders.size() << "=>"  << oldStaticHeaders.size() / 80;
    setStatus(Verifying);
    char buf[4096];
    while (true) {
        auto count = oldStaticHeaders.read(buf, sizeof(buf));
        if (count <= 0)
            break;
        m_newHeaders->write(buf, count);
    }
    oldStaticHeaders.close();
    m_newHeaders->close();

    // make sure that the UI can update the changes we pushed here
    // before we start the process new headers which is going to take
    // some actual processing time, depending on how big our headers
    // are now.
    // keep the file object from getting garbage collected.
    // the map() requires the QFile to stay alive.
    m_newHeaders->setParent(FloweePay::instance());
    boost::asio::post(FloweePay::instance()->ioContext(), std::bind(&BlockHeadersChecker::processNewHeaders, this));
}

void BlockHeadersChecker::processNewHeaders()
{
    // the currenty in use static headers are in the location I want the new ones to be
    // because when we restart, we open by path.
    //
    // After a successful attempt to replace staticChain we'll thus remove the old
    // staticHeaders file and move the one on top of that.
    // "staticHeaders_new"  -> finished,
    // static headers path...
    // Knowing that this will never run on Windowz, lets just rename the open file and
    // place the newly created file at the known path.

    struct RAII {
        explicit RAII(BlockHeadersChecker *dd) : d(dd) {}
        ~RAII() {
            emit d->finishUp(status);
        }
        BlockHeadersChecker *d;
        Status status = Success;
    };
    RAII raii(this);

    if (!m_newHeaders->open(QIODevice::ReadOnly)) {
        logFatal(10054) << "Failed to re-open replacement static headers";
        raii.status = DiskFailure;
        return;
    }
    try {
        logCritical(10054) << "Prepared a new static headers chain. Size:" << m_newHeaders->size()
                      << "Replacing the current one with this new file";
        auto &blockchain = FloweePay::instance()->p2pNet()->blockchain();
        blockchain.replaceStaticChain(m_newHeaders->map(0, m_newHeaders->size()),
                m_newHeaders->size(), TEMP_DOWNLOAD_FILENAME ".info");
        m_newHeaders->close();

        // make sure we use the new ones after restart.
        QFile::remove(PAY_STATIC_HEADERS);
        QFile::remove(PAY_STATIC_HEADERS ".info");
        QFile::rename(TEMP_DOWNLOAD_FILENAME ".info", PAY_STATIC_HEADERS ".info");
        m_newHeaders->rename(PAY_STATIC_HEADERS);
    } catch (const std::exception &e) {
        logCritical(10054) << "Replace static chain failed. Reason:" << e;
        raii.status = NetworkFailure;
        m_newHeaders->close();
        m_newHeaders->remove();
        m_newHeaders->deleteLater();
        m_newHeaders = nullptr;
    }
}

void BlockHeadersChecker::onBytesDownloaded()
{
    assert(m_downloadReply);
    assert(m_newHeaders);
    assert(m_newHeaders->isOpen());

    int downloaded = 0;
    char buf[4096];
    while (true) {
        auto readCount = m_downloadReply->read(buf, sizeof(buf));
        if (readCount <= 0)
            break;
        m_newHeaders->write(buf, readCount);
        downloaded += readCount;
    }
    setBytesDownloaded(m_bytesDownloaded + downloaded);
}

BlockHeadersChecker::Status BlockHeadersChecker::status() const
{
    return m_status;
}

void BlockHeadersChecker::setStatus(Status newStatus)
{
    if (m_status == newStatus)
        return;
    m_status = newStatus;
    emit statusChanged();
}

void BlockHeadersChecker::restart()
{
    if (m_headerReply == nullptr
            && m_downloadReply == nullptr
            && m_newHeaders == nullptr)
        startChecking();
}

int BlockHeadersChecker::totalDownload() const
{
    return m_totalDownload;
}

void BlockHeadersChecker::setTotalDownload(int newTotalDownload)
{
    if (m_totalDownload == newTotalDownload)
        return;
    m_totalDownload = newTotalDownload;
    emit totalDownloadChanged();
}

void BlockHeadersChecker::setBytesDownloaded(int count)
{
    if (m_bytesDownloaded == count)
        return;
    m_bytesDownloaded = count;
    emit bytesDownloadedChanged();
}

int BlockHeadersChecker::bytesDownloaded()
{
    return m_bytesDownloaded;
}