thehub/testing/p2pnet/test_blockchain.cpp

/*
 * This file is part of the Flowee project
 * Copyright (C) 2021-2024 Tom Zander <tom@flowee.org>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "test_blockchain.h"

#include <p2p/Blockchain.h>
#include <p2p/DownloadManager.h>
#include <streaming/BufferPool.h>
#include <streaming/P2PBuilder.h>
#include <utiltime.h>

constexpr const char *genesisHash = "0x000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f";

void TestP2PBlockchain::init()
{
    m_tmpPath = QDir::tempPath() + QString("/flowee-test-%1").arg(getpid());
}

void TestP2PBlockchain::cleanup()
{
     // delete tmp dir.
    if (!m_tmpPath.isEmpty())
        QDir(m_tmpPath).removeRecursively();
    m_tmpPath.clear();
    Blockchain::setStaticChain(nullptr, 0, "");
    SetMockTime(0);
}

void TestP2PBlockchain::basics()
{
    boost::asio::io_context ioContext;
    boost::filesystem::path basedir(m_tmpPath.toStdString());
    DownloadManager dlm(ioContext, basedir, P2PNet::MainChain);
    Blockchain blockchain(&dlm, basedir, P2PNet::MainChain);

    // empty, no blocks other than genesis
    QCOMPARE(blockchain.height(), 0);
    const auto genesis = uint256S(genesisHash);
    QCOMPARE(blockchain.block(0).createHash(), genesis);
    QVERIFY(blockchain.isKnown(genesis));
    QCOMPARE(blockchain.blockHeightFor(genesis), 0);
}

void TestP2PBlockchain::staticChain()
{
    QByteArray bytes;
    prepareStaticFile(bytes);
    // support
    SetMockTime(1232000000);
    boost::asio::io_context ioContext;
    boost::filesystem::path basedir(m_tmpPath.toStdString());
    DownloadManager dlm(ioContext, basedir, P2PNet::MainChain);

    // First just load only the static stuff.
    {
        Blockchain blockchain(&dlm, basedir, P2PNet::MainChain);
        QCOMPARE(blockchain.height(), 99);
        QCOMPARE(blockchain.block(99).createHash(), uint256S("00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995")); // block at height 99 on mainchain
        QCOMPARE(blockchain.blockHeightFor(uint256S("00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995")), 99);
        QCOMPARE(blockchain.expectedBlockHeight(), 665);

        auto dat = blockchain.dataSources();
        QCOMPARE(dat.size(), 1);
        auto source = dat.at(0);
        QCOMPARE(source.source, Blockchain::StaticHeadersSource);
        QCOMPARE(source.startBlock, 0);
        QCOMPARE(source.endBlock, 99);
    }

    const QString dest(m_tmpPath + "/blockchain");
    QFile::remove(dest);
    // The static chain and the blockchain have the same blocks, make sure
    // we are not seeing any confusion.
    QVERIFY(QFile::copy(QString("%1/headers0-99").arg(SRCDIR), dest));
    {
        Blockchain blockchain(&dlm, basedir, P2PNet::MainChain);
        QCOMPARE(blockchain.height(), 99);
        QCOMPARE(blockchain.block(99).createHash(), uint256S("00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995")); // block at height 99 on mainchain
        QCOMPARE(blockchain.blockHeightFor(uint256S("00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995")), 99);
        QCOMPARE(blockchain.expectedBlockHeight(), 665);

        auto dat = blockchain.dataSources();
        QCOMPARE(dat.size(), 1);
        auto source = dat.at(0);
        QCOMPARE(source.source, Blockchain::StaticHeadersSource);
        QCOMPARE(source.startBlock, 0);
        QCOMPARE(source.endBlock, 99);
    }

    // Now follow the static load with a nicely fitting one
    QFile::remove(dest);
    QFile::remove(dest + ".info");
    QVERIFY(QFile::copy(QString("%1/headers100-111").arg(SRCDIR), dest));
    {
        Blockchain blockchain(&dlm, basedir, P2PNet::MainChain);
        QCOMPARE(blockchain.height(), 111);
        QCOMPARE(blockchain.block(99).createHash(), uint256S("00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995"));
        QCOMPARE(blockchain.blockHeightFor(uint256S("00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995")), 99);
        QCOMPARE(blockchain.block(111).createHash(), uint256S("000000004d6a6dd8b882deec7b54421949dddd2c166bd51ee7f62a52091a6c35"));
        QCOMPARE(blockchain.blockHeightFor(uint256S("000000004d6a6dd8b882deec7b54421949dddd2c166bd51ee7f62a52091a6c35")), 111);
        QCOMPARE(blockchain.expectedBlockHeight(), 662);

        auto dat = blockchain.dataSources();
        QCOMPARE(dat.size(), 2);
        auto source = dat.at(0);
        QCOMPARE(source.source, Blockchain::StaticHeadersSource);
        QCOMPARE(source.startBlock, 0);
        QCOMPARE(source.endBlock, 99);
        source = dat.at(1);
        QCOMPARE(source.source, Blockchain::DataFileSource);
        QCOMPARE(source.startBlock, 100);
        QCOMPARE(source.endBlock, 111);
    }

    // ensure that we didn't save anything.
    QVERIFY(!QFile::exists(dest + "1"));

    // Now follow the static load with one that slightly overlaps
    QFile::remove(dest);
    QFile::remove(dest + ".info");
    QVERIFY(QFile::copy(QString("%1/headers91-104").arg(SRCDIR), dest));
    {
        Blockchain blockchain(&dlm, basedir, P2PNet::MainChain);
        QCOMPARE(blockchain.height(), 104);
        QCOMPARE(blockchain.block(99).createHash(), uint256S("00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995"));
        QCOMPARE(blockchain.blockHeightFor(uint256S("00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995")), 99);
        QCOMPARE(blockchain.block(104).createHash(), uint256S("00000000fb11ef25014e02b315285a22f80c8f97689d7e36d723317defaabe5b"));
        QCOMPARE(blockchain.blockHeightFor(uint256S("00000000fb11ef25014e02b315285a22f80c8f97689d7e36d723317defaabe5b")), 104);
        QCOMPARE(blockchain.expectedBlockHeight(), 664);

        auto dat = blockchain.dataSources();
        QCOMPARE(dat.size(), 2);
        auto source = dat.at(0);
        QCOMPARE(source.source, Blockchain::StaticHeadersSource);
        QCOMPARE(source.startBlock, 0);
        QCOMPARE(source.endBlock, 99);
        source = dat.at(1);
        QCOMPARE(source.source, Blockchain::DataFileSource);
        QCOMPARE(source.startBlock, 100);
        QCOMPARE(source.endBlock, 104);
    }

    // ensure that we didn't save anything.
    QVERIFY(!QFile::exists(dest + "1"));

#if 0
    {
        QFile out("headers0-99");
        if (!out.open(QIODevice::WriteOnly))
            logFatal() << "failed to write file";
        for (int i = 0; i <= 99; ++i) {
            BlockHeader bh = blockchain.block(i);
            out.write(reinterpret_cast<const char*>(&bh), 80);
        }
    }
#endif
}

void TestP2PBlockchain::staticChain2()
{
    QByteArray bytes;
    prepareStaticFile(bytes);
    // support
    SetMockTime(1232000000);
    boost::asio::io_context ioContext;
    boost::filesystem::path basedir(m_tmpPath.toStdString());
    DownloadManager dlm(ioContext, basedir, P2PNet::MainChain);

    auto moreHeaders = QFile(QString("%1/headers100-111").arg(SRCDIR));
    QVERIFY(moreHeaders.open(QIODevice::ReadOnly));
    Streaming::BufferPool pool;
    for (int i = 100; i <= 111; ++i) {
        pool.reserve(82);
        pool.data()[0] = 1;
        pool.markUsed(1);
        auto readSize = moreHeaders.read(pool.data(), 80);
        QVERIFY(readSize == 80);
        pool.markUsed(readSize);
        pool.data()[81] = 0;

        Blockchain blockchain(&dlm, basedir, P2PNet::MainChain);
        QCOMPARE(blockchain.height(), i - 1);
        QCOMPARE(blockchain.block(99).createHash(), uint256S("00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995")); // block at height 99 on mainchain
        QCOMPARE(blockchain.blockHeightFor(uint256S("00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995")), 99);

        // add one header.
        blockchain.processBlockHeaders(Message(pool.commit(1), 1), 1);
        QCOMPARE(blockchain.height(), i);
        if (i >= 104){
            QCOMPARE(blockchain.block(104).createHash(), uint256S("00000000fb11ef25014e02b315285a22f80c8f97689d7e36d723317defaabe5b"));
            QCOMPARE(blockchain.blockHeightFor(uint256S("00000000fb11ef25014e02b315285a22f80c8f97689d7e36d723317defaabe5b")), 104);
        }
        blockchain.save();

        auto dat = blockchain.dataSources();
        QCOMPARE(dat.size(), 2);
        auto source = dat.at(0);
        QCOMPARE(source.source, Blockchain::StaticHeadersSource);
        QCOMPARE(source.startBlock, 0);
        QCOMPARE(source.endBlock, 99);
        source = dat.at(1);
        QCOMPARE(source.source, Blockchain::DataFileSource);
        QCOMPARE(source.startBlock, 100);
        QCOMPARE(source.endBlock, i);
    }
}

void TestP2PBlockchain::blockHeightAtTime()
{
    boost::asio::io_context ioContext;
    boost::filesystem::path basedir(m_tmpPath.toStdString());
    DownloadManager dlm(ioContext, basedir, P2PNet::MainChain);
    QByteArray bytes;
    prepareStaticFile(bytes);

    // block at height 80 is mined at: 1231646077
    // asking for all timestamps until block 81 is mined  (nearly 6 minutes later)
    // should return blockheight 80.
    {
        Blockchain blockchain(&dlm, basedir, P2PNet::MainChain);
        QCOMPARE(blockchain.blockHeightAtTime(1231646077), 80);
        QCOMPARE(blockchain.blockHeightAtTime(1231646078), 80);
        QCOMPARE(blockchain.blockHeightAtTime(1231646400), 80);
    }

    // block 101 is mined at 1231661741
    // block 102 is mined at 1231662670
    QVERIFY(QFile::copy(QString("%1/headers100-111").arg(SRCDIR), m_tmpPath + "/blockchain"));
    {
        Blockchain blockchain(&dlm, basedir, P2PNet::MainChain);
        QCOMPARE(blockchain.height(), 111);
        QCOMPARE(blockchain.blockHeightAtTime(1231646080), 80);
        QCOMPARE(blockchain.blockHeightAtTime(1231662000), 101);
        QCOMPARE(blockchain.blockHeightAtTime(1800000000), 111);
    }
}

void TestP2PBlockchain::startAtCheckpoint()
{
    QByteArray bytes;
    prepareStaticFile(bytes, "headers-295000-295020");
    boost::asio::io_context ioContext;
    boost::filesystem::path basedir(m_tmpPath.toStdString());
    DownloadManager dlm(ioContext, basedir, P2PNet::MainChain);

    {
        Blockchain blockchain(&dlm, basedir, P2PNet::MainChain);
        QCOMPARE(blockchain.height(), 295020);
        logFatal() << blockchain.block(295000).createHash();
        logFatal() << blockchain.block(295010).createHash();
        logFatal() << blockchain.block(295020).createHash();
        QCOMPARE(blockchain.block(295000).createHash(), uint256S("4d9b4ef50f0f9d686fd69db2e03af35a100370c64632a983"));
        QCOMPARE(blockchain.blockHeightFor(uint256S("4d9b4ef50f0f9d686fd69db2e03af35a100370c64632a983")), 295000);
        QCOMPARE(blockchain.block(295010).createHash(), uint256S("a6b67d5c5bd85bf8d3fe11212211e1d8c8b43961ac04c01"));
        QCOMPARE(blockchain.blockHeightFor(uint256S("a6b67d5c5bd85bf8d3fe11212211e1d8c8b43961ac04c01")), 295010);
        QCOMPARE(blockchain.block(295020).createHash(), uint256S("ca07ade2023489b5de067b1262f1d5fd738de28dcda1af0"));
        QCOMPARE(blockchain.blockHeightFor(uint256S("ca07ade2023489b5de067b1262f1d5fd738de28dcda1af0")), 295020);

        auto g = blockchain.block(0);
        Q_UNUSED(g);
        try {
            auto b = blockchain.block(10);
            Q_UNUSED(b);
            QFAIL("Expected exception on fetch non existing block");
        } catch (...) {}

        auto dat = blockchain.dataSources();
        QCOMPARE(dat.size(), 1);
        auto source = dat.at(0);
        QCOMPARE(source.source, Blockchain::StaticHeadersSource);
        QCOMPARE(source.startBlock, 295000);
        QCOMPARE(source.endBlock, 295020);
    }

    // split our static headers. Making sure there is some overlap between them.
    QCOMPARE(bytes.size(), 1680); // 21 blocks.
    QFile blks(m_tmpPath + "/blockchain");
    QVERIFY(blks.open(QIODevice::WriteOnly));
    blks.write(bytes.mid(800)); // last 11 blocks
    blks.close();

    QFile staticHeaders(m_tmpPath + "/staticHeaders");
    QVERIFY(staticHeaders.open(QIODevice::WriteOnly));
    bytes = bytes.mid(0, 1200); // first 15 blocks
    staticHeaders.write(bytes);
    staticHeaders.close();
    // create info file for new staticHeaders
    QFile::remove(m_tmpPath + "/staticHeaders.info");
    std::string info = staticHeaders.fileName().toStdString() + ".info";
    Blockchain::createStaticHeaders(staticHeaders.fileName().toStdString(), info);
    Blockchain::setStaticChain(reinterpret_cast<const uint8_t*>(bytes.constData()), bytes.size(), info);

    {
        // loading now should load the staticHeaders file which extends
        // a checkpoint at 279000. The static file covers 15 blocks (279000-279014 inc).
        // Then it should load the blockchain file which overlaps the
        // static one and has 6 additional blocks (15 - 20 inc).
        Blockchain blockchain(&dlm, basedir, P2PNet::MainChain);
        QCOMPARE(blockchain.height(), 295020);
        QCOMPARE(blockchain.block(295000).createHash(), uint256S("4d9b4ef50f0f9d686fd69db2e03af35a100370c64632a983"));
        QCOMPARE(blockchain.blockHeightFor(uint256S("4d9b4ef50f0f9d686fd69db2e03af35a100370c64632a983")), 295000);
        QCOMPARE(blockchain.block(295010).createHash(), uint256S("a6b67d5c5bd85bf8d3fe11212211e1d8c8b43961ac04c01"));
        QCOMPARE(blockchain.blockHeightFor(uint256S("a6b67d5c5bd85bf8d3fe11212211e1d8c8b43961ac04c01")), 295010);
        QCOMPARE(blockchain.block(295020).createHash(), uint256S("ca07ade2023489b5de067b1262f1d5fd738de28dcda1af0"));
        QCOMPARE(blockchain.blockHeightFor(uint256S("ca07ade2023489b5de067b1262f1d5fd738de28dcda1af0")), 295020);

        auto g = blockchain.block(0);
        Q_UNUSED(g);
        try {
            auto b = blockchain.block(10);
            Q_UNUSED(b);
            QFAIL("Expected exception on fetch non existing block");
        } catch (...) {}

        auto dat = blockchain.dataSources();
        QCOMPARE(dat.size(), 2);
        auto source = dat.at(0);
        QCOMPARE(source.source, Blockchain::StaticHeadersSource);
        QCOMPARE(source.startBlock, 295000);
        QCOMPARE(source.endBlock, 295014);
        source = dat.at(1);
        QCOMPARE(source.source, Blockchain::DataFileSource);
        QCOMPARE(source.startBlock, 295015);
        QCOMPARE(source.endBlock, 295020);
    }

    // and one more time, now after the .info file has been created for the save file.
    {
        Blockchain blockchain(&dlm, basedir, P2PNet::MainChain);
        QCOMPARE(blockchain.height(), 295020);
        QCOMPARE(blockchain.block(295000).createHash(), uint256S("4d9b4ef50f0f9d686fd69db2e03af35a100370c64632a983"));
        QCOMPARE(blockchain.blockHeightFor(uint256S("4d9b4ef50f0f9d686fd69db2e03af35a100370c64632a983")), 295000);
        QCOMPARE(blockchain.block(295010).createHash(), uint256S("a6b67d5c5bd85bf8d3fe11212211e1d8c8b43961ac04c01"));
        QCOMPARE(blockchain.blockHeightFor(uint256S("a6b67d5c5bd85bf8d3fe11212211e1d8c8b43961ac04c01")), 295010);
        QCOMPARE(blockchain.block(295020).createHash(), uint256S("ca07ade2023489b5de067b1262f1d5fd738de28dcda1af0"));
        QCOMPARE(blockchain.blockHeightFor(uint256S("ca07ade2023489b5de067b1262f1d5fd738de28dcda1af0")), 295020);

        auto g = blockchain.block(0);
        Q_UNUSED(g);
        try {
            auto b = blockchain.block(10);
            Q_UNUSED(b);
            QFAIL("Expected exception on fetch non existing block");
        } catch (...) {}
    }

    // avoid loading the file in the constructor.
    QFile::remove(blks.fileName() + ".info");
    blks.rename(m_tmpPath + "/blocks");
    {
        Blockchain blockchain(&dlm, basedir, P2PNet::MainChain);
        QCOMPARE(blockchain.height(), 295014);
    }
    QVERIFY(blks.open(QIODevice::ReadOnly));
    Streaming::BufferPool pool;
    for (int i = 0; i < 11; ++i) {
        pool.reserve(82);
        pool.data()[0] = 1;
        pool.markUsed(1);
        auto readSize = blks.read(pool.data(), 80);
        QVERIFY(readSize == 80);
        pool.markUsed(readSize);
        pool.data()[81] = 0;

        Blockchain blockchain(&dlm, basedir, P2PNet::MainChain);
        QCOMPARE(blockchain.height(), 295014 + (std::max(0, i - 5)));
        // add one header.
        blockchain.processBlockHeaders(Message(pool.commit(1), 1), 1);
        QCOMPARE(blockchain.height(), 295014 + (std::max(0, i - 4)));
        QCOMPARE(blockchain.block(295000).createHash(), uint256S("4d9b4ef50f0f9d686fd69db2e03af35a100370c64632a983"));
        blockchain.save();

        auto dat = blockchain.dataSources();
        QCOMPARE(dat.size(), i <= 4 ? 1 : 2);
        auto source = dat.at(0);
        QCOMPARE(source.source, Blockchain::StaticHeadersSource);
        QCOMPARE(source.startBlock, 295000);
        QCOMPARE(source.endBlock, 295014);
        if (i > 4) {
            source = dat.at(1);
            QCOMPARE(source.source, Blockchain::DataFileSource);
            QCOMPARE(source.startBlock, 295015);
            QCOMPARE(source.endBlock, blockchain.height());
        }
    }
}

void TestP2PBlockchain::prepareStaticFile(QByteArray &bytes, const char *filename)
{
    // we copy our static data from a file, more flexible.
    // Suggestion for real apps is to use something like QResource.
    static const char *first100headers = "headers0-99";
    if (filename == nullptr)
        filename = first100headers;
    QFile staticHeaders(QString("%1/%2").arg(SRCDIR, filename));
    std::string info = (m_tmpPath + "/staticHeaders.info").toStdString();
    boost::system::error_code error;
    boost::filesystem::create_directories(m_tmpPath.toStdString(), error);
    Blockchain::createStaticHeaders(staticHeaders.fileName().toStdString(), info);

    bool readOk = staticHeaders.open(QIODevice::ReadOnly);
    assert(readOk);
    bytes = staticHeaders.readAll();
    Blockchain::setStaticChain(reinterpret_cast<const uint8_t*>(bytes.constData()), bytes.size(), info);
}

QTEST_MAIN(TestP2PBlockchain)