thehub/libs/p2p/Blockchain.cpp

/*
 * This file is part of the Flowee project
 * Copyright (C) 2020-2024 Tom Zander <tom@flowee.org>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#include "Blockchain.h"
#include "DownloadManager.h"
#include "Peer.h"
#include "streaming/BufferPools.h"

#include <filesystem>
#include <streaming/BufferPool.h>
#include <streaming/P2PBuilder.h>
#include <streaming/P2PParser.h>
#include <utils/utiltime.h>
#include <interfaces/BitcoinVersion.h> // for PROTOCOL_VERSION

#include <stdexcept>
#include <fstream>
#include <cstdio> // for rename()

constexpr char Version = 1;
constexpr char MinVersion = 1;

struct StaticChain {
    const unsigned char *data = nullptr;
    std::string infoFile;
    int64_t size = 0;
};
static StaticChain s_staticChain = StaticChain();

/*
 * This class holds as many block-headers as are available.
 * The datasource for these blockheaders ranges from them being downloaded from
 * the p2p network, to having a static file provided with headers.
 *
 * - static-file.
 *   A simple file with headers. Sorted by height. Starting with block 1.
 * - static-info-file.
 *   A list of block-hashes matching the static file.
 * - blockchain files. See load() for more details.
 *
 * The static file loading is practically instant. It reads the actual file once,
 * and on a laptop that takes 22ms (warm). Additionally, it inserts the hashes
 * from the info file into an unsorted map. This adds another 200ms.
 *
 * The _really_ slow part is to process the blockheaders we did not yet create an
 * info file for before. My test of 3.6 MB of headers (44k headers) took 600ms.
 * So 3 times as long for 1/20th the blocks...
 */

Blockchain::Blockchain(DownloadManager *downloadManager, const boost::filesystem::path &basedir, P2PNet::Chain chain)
    : m_basedir(basedir),
      m_blockHeight(chain == P2PNet::MainChain ? 60001 : 1001),
      m_dlmanager(downloadManager),
      m_saveTimer(m_dlmanager->service())
{
    assert(m_dlmanager);

    switch (chain) {
    case P2PNet::MainChain:
        createMainchainGenesis();
        loadMainchainCheckpoints();
        break;
    case P2PNet::Testnet4Chain:
        createTestnet4Genesis();
        loadTestnet4Checkpoints();
        break;
    default:
        assert(false);
    }
    std::ifstream hashes;
    hashes.open(s_staticChain.infoFile);
    if (hashes.is_open()) // if it doesn't exist, silently skip loading
        loadStaticChain(s_staticChain.data, s_staticChain.size, hashes);
    load();
}

Message Blockchain::createGetHeadersRequest(int skipHeaders)
{
    assert(skipHeaders >= 0);
    std::unique_lock<std::recursive_mutex> lock(m_lock);
    Streaming::P2PBuilder builder(Streaming::pool(4 + 1 + 32 * 11));
    builder.writeInt(PROTOCOL_VERSION);
    if (m_tip.height <= 1000 + skipHeaders) {
        builder.writeCompactSize(1);
        builder.writeByteArray(m_tip.tip.begin(), 32, Streaming::RawBytes);
        builder.writeByteArray(m_tip.tip.begin(), 32, Streaming::RawBytes);
    } else {
        builder.writeCompactSize(10);
        std::array<int, 10> offsets = {
            m_tip.height - skipHeaders,
            m_tip.height - 5 - skipHeaders,
            m_tip.height - 20 - skipHeaders,
            m_tip.height - 60 - skipHeaders,
            m_tip.height - 100 - skipHeaders,
            m_tip.height - 200 - skipHeaders,
            m_tip.height - 400 - skipHeaders,
            m_tip.height - 600 - skipHeaders,
            m_tip.height - 800 - skipHeaders,
            m_tip.height - 1000 - skipHeaders
        };
        for (auto i : offsets) {
            auto bh = block(i);
            uint256 hash = bh.createHash();
            builder.writeByteArray(hash.begin(), 32, Streaming::RawBytes);
        }
        // add the 'stop-at-hash', we don't want them to stop, so we set that to zero's
        uint8_t empty[32];
        memset(empty, 0, 32);
        builder.writeByteArray(empty, 32, Streaming::RawBytes);
    }
    return builder.message(Api::P2P::GetHeaders);
}

void Blockchain::processBlockHeaders(Message message, int peerId)
{
    int newTip = 0;
    bool goodHeadersSameTip = false;
    int height = 0;
    bool justDisconnect = false;
    try {
        std::unique_lock<std::recursive_mutex> lock(m_lock);
        Streaming::P2PParser parser(message);
        auto count = parser.readCompactInt();
        if (count > 2000) {
            logInfo() << "Peer:" << peerId << "Sent too many headers" << count << "p2p protocol violation";
            m_dlmanager->reportDataFailure(peerId);
            return;
        }
        const uint32_t maxFuture = time(nullptr) + 7200; // headers can not be more than 2 hours in the future.

        uint256 prevHash;
        int startHeight = -1;
        arith_uint256 chainWork;
        for (size_t i = 0; i < count; ++i) {
            BlockHeader header = BlockHeader::fromMessage(parser);
            /*int txCount =*/ parser.readCompactInt(); // always zero

            // timestamp not more than 2h in the future.
            if (header.nTime > maxFuture) {
                logWarning() << "Peer:" << peerId << "sent bogus headers. Too far in future";
                m_dlmanager->reportDataFailure(peerId);
                return;
            }

            if (i == 0) { // first header in the sequence.
                assert(startHeight == -1);
                auto iter = m_blockHeight.find(header.hashPrevBlock);
                if (iter == m_blockHeight.end()) {
                    if (header.createHash() == block(0).createHash()) // genesis
                        height = 0;
                    else
                        throw std::runtime_error("is on a different chain, headers don't extend ours");
                }
                else
                    height = iter->second; // the height _before_ we add the above header.
                if (m_tip.height == height) {
                    chainWork = m_tip.chainWork;
                }
                else if (m_tip.height > height + static_cast<int>(count)) { // too old
                    assert(!m_checkpoints.empty());
                    auto cp = m_checkpoints.crbegin();
                    if (height > cp->height) { // remote has a valid block after our last checkpoint.
                        // the remote is behind. We won't punish it, but we should disconnect a useless peer
                        logInfo() << "Peer:" << peerId << "is behind, not useful to us.";
                        justDisconnect = true;
                        throw std::runtime_error("Peer behind");
                    }
                    throw std::runtime_error("is on a different chain, headers don't extend ours");
                }
                else {
                    // rollback the chainWork to branch-point
                    const auto curHeight = blockHeight_priv();
                    assert(m_tip.height == curHeight);
                    chainWork = m_tip.chainWork;
                    for (int h = m_tip.height; h > height; --h) {
                        chainWork -= block(h).blockProof();
                    }
                }
            }
            else if (prevHash != header.hashPrevBlock) { // check if we are really a sequence.
                throw std::runtime_error("sent bogus headers. Not in sequence");
            }
            uint256 hash = header.createHash();
            // check POW
            {
                bool fNegative;
                bool fOverflow;
                arith_uint256 bnTarget;
                bnTarget.SetCompact(header.nBits, &fNegative, &fOverflow);
                if (fNegative || bnTarget == 0 || fOverflow || bnTarget > UintToArith256(powLimit)
                        || UintToArith256(hash) > bnTarget) {// Check proof of work matches claimed amount
                    throw std::runtime_error("sent bogus headers. POW failed");
                }
            }
            chainWork += header.blockProof();

            if (startHeight == -1)
                startHeight = height;
            ++height; // Header is known to fit at the new 'height'

            for (auto iter = m_checkpoints.rbegin(); iter != m_checkpoints.rend(); ++iter) {
                // we're going backwards through the list, so if its height is lower,
                // we stop looking.
                if (iter->height < height)
                    break;
                if (iter->height == height && iter->hash != hash)
                    throw std::runtime_error("is on a different chain, checkpoint failure");
            }
            prevHash = std::move(hash);
        }

        if (chainWork > m_tip.chainWork) {
            // The new chain has more PoW, apply it.
            parser = Streaming::P2PParser(message);
            count = parser.readCompactInt();
            int offset = startHeight - m_chainStartPoint + 1;
            if (m_numStaticHeaders)
                offset -= m_numStaticHeaders - 1;
            m_longestChain.resize(count + offset);
            for (size_t i = 0; i < count; ++i) {
                BlockHeader header = BlockHeader::fromMessage(parser);
                /*int txCount =*/ parser.readCompactInt(); // always zero
                m_blockHeight.insert(std::make_pair(header.createHash(), startHeight + i + 1));
                m_longestChain[offset + i] = header;
            }
            m_tip.height = startHeight + count;
            m_tip.tip = prevHash;
            m_tip.chainWork = chainWork;
            newTip = m_tip.height;
            assert(m_tip.height == blockHeight_priv());
            logCritical() << "Headers now at" << newTip << m_tip.tip <<
                         DateTimeStrFormat("%Y-%m-%d %H:%M:%S", m_longestChain.back().nTime).c_str();
        }
        else if (chainWork == m_tip.chainWork) { // Good headers, same tip we already had
            // we go and do nice things with the peer, but we have to do that outside of the
            // mutex to avoid deadlocks. As such we simply set a bool and delay the action.
            goodHeadersSameTip = true;
        }
    } catch (const std::runtime_error &err) {
        logWarning() << "Peer:" << peerId << "is" << err.what();
        if (justDisconnect) {
            m_dlmanager->connectionManager().disconnect(peerId);
        } else {
            m_dlmanager->reportDataFailure(peerId);
        }
        return;
    }

    if (goodHeadersSameTip) {
        // Lets tell the PeerAddressDB that
        // this peer got good headers, since that DB is persisted between
        // restarts, we improve our performance by remembering success.
        auto peer = m_dlmanager->connectionManager().peer(peerId);
        if (peer.get()) {
            peer->peerAddress().gotGoodHeaders();
            peer->updatePeerHeight(height);
        }
    }
    else if (newTip > 0) {
        m_dlmanager->headersDownloadFinished(newTip, peerId);
        if (!m_saveTimerStarted && newTip - m_lastSavedHeader > 300) {
            // schedule task to compress all files
            m_saveTimer.expires_from_now(boost::posix_time::seconds(45));
            m_saveTimer.async_wait(std::bind(&Blockchain::save, this));
            m_saveTimerStarted = true;
        }
    }
}

// static
void Blockchain::setStaticChain(const unsigned char *data, int64_t size, const std::string &infoFile)
{
    assert(size == 0 || data);
    s_staticChain.data = data;
    s_staticChain.size = size;
    s_staticChain.infoFile = infoFile;
}

// static
bool Blockchain::createStaticHeaders(const std::string &blockchain, const std::string &meta)
{
    std::ifstream data;
    data.open(blockchain, std::ios_base::binary);
    if (!data.is_open()) {
        logFatal() << "FAIL Could not read input file:" << blockchain;
        return false;
    }
    std::ofstream out;
    out.open(meta, std::ios_base::trunc | std::ios_base::binary);
    if (!out.is_open()) {
        logFatal() << "FAIL Could not write info file:" << meta;
        return false;
    }

    arith_uint256 chainWork;
    while (true) {
        BlockHeader bh;
        data.read(reinterpret_cast<char*>(&bh), 80);
        if (data.fail())
            break;
        chainWork += bh.blockProof();
        auto blockhash = bh.createHash();
        out.write(reinterpret_cast<const char*>(blockhash.begin()), 32);
        if (out.fail()) {
            logFatal() << "Out of disk space";
            return false;
        }
    }
    data.close();
    out.write(reinterpret_cast<const char*>(&chainWork), 32);
    if (out.fail()) {
        logFatal() << "Out of disk space";
        return false;
    }
    out.close();

    return true;
}

void Blockchain::replaceStaticChain(const unsigned char *data, int64_t size, const std::string &infoFile)
{
    assert(data);
    assert(size > 0);
    assert((size % 80) == 0);
    std::unique_lock<std::recursive_mutex> lock(m_lock);
    if (m_chainStartPoint == 0) // can't replace a static chain if there wasn't one.
        throw std::runtime_error("Missing static chain");

    std::ofstream out;
    out.open(infoFile + ".part", std::ios_base::trunc | std::ios_base::binary);
    if (!out.is_open())
        throw std::runtime_error("Can't replace / create info file");

    BlockHeightMap newblockHeight = m_blockHeight;
    // int headersRead = 0;
    int startingPoint = -1;
    int curHeight = -1;
    auto bytesLeft = size;

    arith_uint256 chainWork;
    const uint8_t *ptr = data;
    uint256 prevHash;
    while (bytesLeft >= 80) {
        const BlockHeader *bh = reinterpret_cast<const BlockHeader*>(ptr);
        const uint256 hash = bh->createHash();
        if (startingPoint == -1) {
            // check if we're actaully extending a checkpoint.
            for (const auto &cp : m_checkpoints) {
                if (cp.hash == hash) {
                    startingPoint = cp.height;
                    curHeight = startingPoint;
                    chainWork = cp.chainWork;
                    break;
                }
            }
            if (startingPoint == -1)
                throw std::runtime_error("Bad static file. Unknown starting point");
            if (startingPoint >= m_chainStartPoint)
                throw std::runtime_error("Replacement not bigger, cowardly refusing");
            if ((size / 80) != m_numStaticHeaders + (m_chainStartPoint - startingPoint))
                throw std::runtime_error("Wrong headers count in new static file");
        }
        else if (bh->hashPrevBlock != prevHash)
            throw std::runtime_error("Broken chain");
        prevHash = hash;
        out.write(reinterpret_cast<const char*>(hash.begin()), 32);
        chainWork += bh->blockProof();
        newblockHeight.insert(std::make_pair(hash, curHeight++));
        ptr += 80;
        bytesLeft -= 80;
    }
    // finish with writing the cumulative work.
    out.write(reinterpret_cast<const char*>(&chainWork), 32);
    out.close();
    std::filesystem::rename(infoFile + ".part", infoFile );

    m_blockHeight = newblockHeight;
    m_chainStartPoint = startingPoint;
    m_numStaticHeaders = size / 80;
    m_staticChain = data;
}

int Blockchain::height() const
{
    return m_tip.height;
}

int Blockchain::expectedBlockHeight() const
{
    std::unique_lock<std::recursive_mutex> lock(m_lock);
    const int ourHeight = blockHeight_priv();
    int secsSinceLastBlock = 0;
    if (m_numStaticHeaders > ourHeight) {
        const BlockHeader *bh = reinterpret_cast<const BlockHeader*>(m_staticChain + 80
                                                 * (ourHeight - m_chainStartPoint));
        secsSinceLastBlock = GetTime() - bh->nTime;
    }
    else {
        secsSinceLastBlock = GetTime() - m_longestChain.back().nTime;
    }
    return m_tip.height + (secsSinceLastBlock + 300) / 600; // add how many 10 minutes chunks fit in the time-span.
}

bool Blockchain::isKnown(const uint256 &blockId) const
{
    std::unique_lock<std::recursive_mutex> lock(m_lock);
    return m_blockHeight.find(blockId) != m_blockHeight.end();
}

int Blockchain::blockHeightFor(const uint256 &blockId) const
{
    std::unique_lock<std::recursive_mutex> lock(m_lock);
    auto iter = m_blockHeight.find(blockId);
    if (iter == m_blockHeight.end())
        return -1;
    if (blockHeight_priv() < iter->second)
        return -1;
    return iter->second;
}

int Blockchain::blockHeightAtTime(uint32_t timestamp) const
{
    std::unique_lock<std::recursive_mutex> lock(m_lock);
    int l = m_chainStartPoint;
    int r = blockHeight_priv();
    while (l <= r) {
        int m = (l + r) / 2;
        uint32_t val = block(m).nTime;
        if (val < timestamp)
            l = m + 1;
        else if (val == timestamp)
            return m;
        else
            r = m - 1;
    }

    return std::max(m_chainStartPoint, l - 1);
}

int Blockchain::oldestKnownBlockHeight() const
{
    return m_chainStartPoint;
}

BlockHeader Blockchain::block(int height) const
{
    assert(height >= 0);
    std::unique_lock<std::recursive_mutex> lock(m_lock);
    if (blockHeight_priv() < height)
        return BlockHeader();
    if (height == 0)
        return m_longestChain.at(0); // coinbase is in this list.
    if (height < m_chainStartPoint)
        throw std::runtime_error("Requested unknown block");

    int offset = m_chainStartPoint;
    if (m_numStaticHeaders) {
        // -1 because we skip the first static header since it duplicates one we already know.
        offset += m_numStaticHeaders - 1;
    }
    if (offset >= height) {
        const BlockHeader *bh = reinterpret_cast<const BlockHeader*>(
            m_staticChain + 80 * (height - m_chainStartPoint));
        return *bh;
    }
    return m_longestChain.at(height - offset);
}

void Blockchain::createMainchainGenesis()
{
    if (!m_longestChain.empty())
        return;
    BlockHeader genesis;
    genesis.nBits = 0x1d00ffff;
    genesis.nTime = 1231006505;
    genesis.nNonce = 2083236893;
    genesis.nVersion = 1;
    createGenericGenesis(genesis);
}

void Blockchain::loadMainchainCheckpoints()
{
    CheckPoint cp;
    cp.hash = uint256S("573993a3c9e41ce34471c079dcf5f52a0e824a81e7f953b8661a20");
    cp.chainWork = 0x1b32578bc31800;
    cp.height = 74000;
    m_checkpoints.push_back(cp);
    cp.hash = uint256S("5b12ffd4cd315cd34ffd4a594f430ac814c91184a0d42d2b0fe");
    cp.chainWork = arith_uint256("1d3d101c6a0418476");
    cp.height = 134444;
    m_checkpoints.push_back(cp);
    cp.hash = uint256S("59f452a5f7340de6682a977387c17010ff6e6c3bd83ca8b1317");
    cp.chainWork = arith_uint256("16c80a189db0468c5e");
    cp.height = 193000;
    m_checkpoints.push_back(cp);
    cp.hash = uint256S("3887df1f29024b06fc2200b55f8af8f35453d7be294df2d214");
    cp.chainWork = arith_uint256("8113ee25e591519297");
    cp.height = 250000;
    m_checkpoints.push_back(cp);
    cp.hash = uint256S("4d9b4ef50f0f9d686fd69db2e03af35a100370c64632a983");
    cp.chainWork = arith_uint256("3a485160e85f201be9fd");
    cp.height = 295000;
    m_checkpoints.push_back(cp);
    cp.height = 350000;
    cp.hash = uint256S("53cf64f0400bb38e0c4b3872c38795ddde27acb40a112bb");
    cp.chainWork = arith_uint256("5c7fd68dd37dbdbc5f63d");
    m_checkpoints.push_back(cp);
    cp.height = 400000;
    cp.hash = uint256S("4ec466ce4732fe6f1ed1cddc2ed4b328fff5224276e3f6f");
    cp.chainWork = arith_uint256("122a24b77c62cc75ff4cdd");
    m_checkpoints.push_back(cp);
    cp.height = 450000;
    cp.hash = uint256S("14083723ed311a461c648068af8cef8a19dcd620c07a20b");
    cp.chainWork = arith_uint256("3ab70ae0412d1cb5e6f455");
    m_checkpoints.push_back(cp);
    cp.height = 500000;
    cp.hash = uint256S("5e14d3f9fdfb70745308706615cfa9edca4f4558332b201");
    cp.chainWork = arith_uint256("7ae48aca46e3b349ac9713");
    m_checkpoints.push_back(cp);
    cp.height = 550000;
    cp.hash = uint256S("7965cc5602c50d0ace7d4ba12b44fb14256bf36baad67d");
    cp.chainWork = arith_uint256("c5d1790f2be43002fc6cb1");
    m_checkpoints.push_back(cp);
    cp.height = 600000;
    cp.hash = uint256S("8e8d83cba6d45a9314bc2ef4538d4e0577c6bed8593536");
    cp.chainWork = arith_uint256("fd72e720ff6acb690235bc");
    m_checkpoints.push_back(cp);
    cp.height = 650000;
    cp.hash = uint256S("1e5a8e11a9a523e15ad985b8123df0f7b364ad8f83d82b0");
    cp.chainWork = arith_uint256("144be7da24c53af81a05c93");
    m_checkpoints.push_back(cp);
    cp.height = 700000;
    cp.hash = uint256S("3d908ba7463fd154902eda89ac015a2e2630f4130674e77");
    cp.chainWork = arith_uint256("176dde1e204b768e9ede4fe");
    m_checkpoints.push_back(cp);
    cp.height = 750000;
    cp.hash = uint256S("621b8f668b02530bcf77446e85e56cb0257a51b4ec980ba");
    cp.chainWork = arith_uint256("19b74375ad1de115c4c20dd");
    m_checkpoints.push_back(cp);
    cp.height = 800000;
    cp.hash = uint256S("1012aaf4b6aa033ddda9d9bbf832b7be597705be5635311");
    cp.chainWork = arith_uint256("1c01c5dbaf91a1b0a125128");
    m_checkpoints.push_back(cp);
    cp.height = 850000;
    cp.hash = uint256S("d3efae5f8652243331d4b72870363bdc659eefa7360974");
    cp.chainWork = arith_uint256("20d787fe53be001b15bb727");
    m_checkpoints.push_back(cp);
}

void Blockchain::createTestnet4Genesis()
{
    if (!m_longestChain.empty())
        return;
    BlockHeader genesis;
    genesis.nBits = 0x1d00ffff;
    genesis.nTime = 1597811185;
    genesis.nNonce = 114152193;
    genesis.nVersion = 1;
    createGenericGenesis(genesis);
}

void Blockchain::loadTestnet4Checkpoints()
{
    CheckPoint cp;
    cp.hash = uint256S("0x0000000019df558b6686b1a1c3e7aee0535c38052651b711f84eebafc0cc4b5e");
    cp.chainWork = 0x32bd583ca267;
    cp.height = 5677;
    m_checkpoints.push_back(cp);
    cp.hash = uint256S("0x00000000016522b7506939b23734bca7681c42a53997f2943ab4c8013936b419");
    cp.chainWork = 0x2bd3345121f1a;
    cp.height = 9999;
    m_checkpoints.push_back(cp);
}

void Blockchain::loadStaticChain(const unsigned char *data, int64_t dataSize, std::ifstream &infoFile)
{
    m_chainStartPoint = -1; // checking for invalid.
    if (dataSize > 80 ) {
        // find starting point.
        assert(m_longestChain.size() >= 1); // always mininum one block, the genesis one.
        const BlockHeader *firstBlock = reinterpret_cast<const BlockHeader*>(data);
        const uint256 firstBlockHash = firstBlock->createHash();
        if (firstBlockHash == m_longestChain.front().createHash()) {
            m_chainStartPoint = 0; // genesis block
        }
        else {
            for (const auto &cp : m_checkpoints) {
                if (cp.hash == firstBlockHash) {
                    m_chainStartPoint = cp.height;
                    m_tip.chainWork = cp.chainWork;
                    m_tip.chainWork -= firstBlock->blockProof(); // don't count the checkpoint block twice
                    break;
                }
            }
        }
    }
    if (m_chainStartPoint == -1) {
        logWarning() << "Ignoring static blockchain, not for this network (genesis does not match)";
        m_chainStartPoint = 0;
        return;
    }

    int numHeadersFound = 0;
    uint256 blockhash;
    for (int64_t pos = 0; pos + 80 <= dataSize; pos += 80) {
        infoFile.read(reinterpret_cast<char*>(blockhash.begin()), 32);
        if (infoFile.fail())
            throw std::runtime_error("loadStaticChain: info file truncated");
        if (m_chainStartPoint != 0 || pos > 0) // genesis was inserted in the createGenesis method
            m_blockHeight.insert(std::make_pair(blockhash, numHeadersFound + m_chainStartPoint));
        ++numHeadersFound;
    }
    if (numHeadersFound > 1) {
        m_staticChain = data;
        m_numStaticHeaders = numHeadersFound;

        m_tip.height = blockHeight_priv();
        m_tip.tip = block(m_tip.height).createHash();

        arith_uint256 staticChainWork;
        infoFile.read(reinterpret_cast<char*>(&staticChainWork), 32);
        if (infoFile.fail())
            throw std::runtime_error("loadStaticChain: info file truncated");
        m_tip.chainWork += staticChainWork;
    }
}

void Blockchain::createGenericGenesis(BlockHeader genesis)
{
    genesis.hashMerkleRoot = uint256S("4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b");
    m_longestChain.push_back(genesis);

    const uint256 genesisHash = genesis.createHash();
    m_blockHeight.insert(std::make_pair(genesisHash, 0));
    m_tip.tip = genesisHash;
    m_tip.height = 0;
    m_tip.chainWork += genesis.blockProof();
}

void Blockchain::load()
{
    /*
     * We check a series of files starting with 'blockchain' and load them in sequence.
     * When there are more than 60 files we schedule a write to replace all of them with
     * a single one. Since headers generate about 4MB per year, this should all be quite
     * small due to the usage of the static file which comes from the install.
     *
     * The idea is that we cut down a lot on dupicate-saving while also minimizing the
     * load of saving at shutdown.
     * So, for saving we save at a timeout after we received a large number of headers,
     * to a new file. At shutdown we only save the stuff we have not saved before, again
     * to a new file.
     */
    std::unique_lock<std::recursive_mutex> lock(m_lock);

    Streaming::BufferPool pool;
    for (m_fileCount = 0; m_fileCount < 100; ++m_fileCount) {
        std::string suffix;
        if (m_fileCount > 0)
            suffix = std::to_string(m_fileCount);
        std::string path((m_basedir / (std::string("blockchain") + suffix)).string());

        std::ifstream in(path);
        if (!in.is_open()) {
            if (m_fileCount > 60) {
                // schedule task to compress all files
                m_saveTimer.expires_from_now(boost::posix_time::seconds(45));
                m_saveTimer.async_wait(std::bind(&Blockchain::compressSaveFiles, this,
                                                       std::placeholders::_1));
                m_saveTimerStarted = true;
            }
            break;
        }
        logInfo() << "Starting to load the blockchain" << (suffix.empty() ? "" : "number:") << suffix;

        const auto infoPath = path + ".info";
        std::ifstream info(infoPath);
        if (info.is_open()) {
            const auto inSize = boost::filesystem::file_size(path);
            const auto infoSize = boost::filesystem::file_size(infoPath);
            const int inHeaderCount = inSize / 80;
            const int infoHeaderCount = infoSize / 32;

            /* two size comparisons.
             *
             *      inHeaderCount == infoHeaderCount - 1
             *    this simply checks that we have the right count of headers vs blockIds.
             *    with the '-1' there to account for the info file having an additional
             *    chain-work 32-bytes row.
             *
             * second comparison or check:
             *
             *      infoHeaderCount * 32 + 1 != infoSize
             *    the infoHeaderCount is the result of an int division, so our accuracy is 32 bytes.
             *    We need exactly one more byte in the file, which is the opening version byte.
             */
            if (inHeaderCount != infoHeaderCount - 1 || infoHeaderCount * 32 + 1 != infoSize) {
                logWarning() << "Skipping usage of info file, it has the wrong size:" << infoPath;
                info.close();
            }
        }
        if (info.is_open()) {
            char version;
            info.read(&version, 1);
            if (version < MinVersion) {
                logWarning() << "Skipping usage of info file, too old version" << version;
                info.close();
            }
        }
        if (info.is_open()) {
            int skipNumber = -1;
            // load the data from the 'in' file and also the 'info' file.
            while (true) {
                char headerData[80];
                in.read(headerData, sizeof(headerData));
                if (in.fail()) // end of file.
                    break;
                const BlockHeader *bh = reinterpret_cast<const BlockHeader*>(headerData);

                if (skipNumber == -1) {
                    skipNumber = skipBlocksOfFile(*bh);
                    if (skipNumber == -1) {
                        logWarning() << "Blockchain WARN: Loaded blocksdata do not match our chain" << infoPath;
                        logWarning() << "  += starts at" << bh->createHash();
                        return;
                    }
                }

                uint256 blockhash;
                info.read(reinterpret_cast<char*>(blockhash.begin()), 32);
                if (info.fail()) {
                    logWarning() << "Failed to read from 'info' file." << infoPath;
                    break;
                }
                if (skipNumber > 0) {
                    --skipNumber;
                    continue;
                }

                m_longestChain.push_back(*bh);
                m_blockHeight.insert(std::make_pair(blockhash, blockHeight_priv()));
            }

            info.read(reinterpret_cast<char*>(&m_tip.chainWork), 32);
            assert(!info.fail());
        }
        else {
            // if there is no 'info' file for this one yet,
            // load the blockchain file the 'slow' way (which includes hashing) and
            // also write the info file afterwards.
            loadSlow(in, path + ".info");
        }
    }

    if (m_tip.height < blockHeight_priv()) {
        m_tip.tip = m_longestChain.back().createHash();
        m_tip.height = blockHeight_priv();
    }
    logCritical() << "Blockchain loading completed. Tip:" << m_tip.height << m_tip.tip;
    m_lastSavedHeader = blockHeight_priv();
}

void Blockchain::save()
{
    boost::system::error_code error;
    boost::filesystem::create_directories(m_basedir, error);
    if (error && !boost::filesystem::exists(m_basedir) && !boost::filesystem::is_directory(m_basedir)) {
        logFatal() << "P2P.Blockchain can't save. Failed creating the dir:" << m_basedir.string();
        return;
    }
    std::unique_lock<std::recursive_mutex> lock(m_lock);
    m_saveTimerStarted = false;
    if (m_lastSavedHeader == blockHeight_priv()) // nothing to do
        return;

    /*
     * This method saves the not yet saved blocks into a new file.
     * We skip creating the info file here because it would be a duplication of
     * code and the actual gain in time is minuscule. The lack of the metadata
     * file simply means that when it will be loaded with the 'loadSlow()' method
     * (only once ever) this creates the metadata.
     */
    std::string suffix;
    if (m_fileCount > 0)
        suffix = std::to_string(m_fileCount);
    ++m_fileCount;
    std::ofstream out((m_basedir / (std::string("blockchain") + suffix)).string());
    std::shared_ptr<Streaming::BufferPool> pool = std::make_shared<Streaming::BufferPool>();
    const size_t end = blockHeight_priv();
    for (size_t i = m_lastSavedHeader + 1; i <= end; ++i) {
        const auto &header = block(i);
        assert(!header.hashMerkleRoot.IsNull());
        auto cd = header.write(pool);
        assert(cd.size() == 80);
        out.write(cd.begin(), cd.size());
    }
    m_lastSavedHeader = blockHeight_priv();
    out.close();
}

std::vector<Blockchain::DataSource> Blockchain::dataSources() const
{
    std::vector<DataSource> answer;
    if (m_numStaticHeaders > 0) {
        DataSource ds;
        ds.source = StaticHeadersSource;
        ds.startBlock = m_chainStartPoint;
        ds.endBlock = ds.startBlock + m_numStaticHeaders - 1;
        answer.push_back(ds);
    }
    DataSource ds;
    ds.source = DataFileSource;
    if (answer.size() == 1)
        ds.startBlock = answer.at(0).endBlock + 1;
    ds.endBlock = height();
    if (ds.endBlock >= ds.startBlock)
        answer.push_back(ds);
    return answer;
}

void Blockchain::compressSaveFiles(const boost::system::error_code &error)
{
    if (error) // operation aborted is the main one we expect here.
        return;
    std::unique_lock<std::recursive_mutex> lock(m_lock);
    auto path = (m_basedir / "blockchain.part").string();
    std::ofstream out(path);
    std::shared_ptr<Streaming::BufferPool> pool = std::make_shared<Streaming::BufferPool>();
    for (size_t i = 1; i < m_longestChain.size(); ++i) {
        const auto &header = m_longestChain.at(i);
        auto cd = header.write(pool);
        assert(cd.size() == 80);
        out.write(cd.begin(), cd.size());
    }

    auto newPath(path);
    newPath.resize(newPath.size() - 5);
    out.flush();
    out.close();
    if (rename(path.c_str(), newPath.c_str()) != 0) {
        logFatal() << "Rename of blockchain part to blockchain failed";
        return;
    }
    for (int i = 1; i < m_fileCount; ++i) {
        path = newPath + std::to_string(i);
        if (remove(path.c_str()) != 0)
            logFatal() << "Failed to remove blockchain file" << path;
        path += ".info";
        if (remove(path.c_str()) != 0)
            logFatal() << "Failed to remove blockchain meta file" << path;
    }

    m_lastSavedHeader = blockHeight_priv();
    m_saveTimerStarted = false;
    m_fileCount = 1;
}

int Blockchain::skipBlocksOfFile(const BlockHeader &bh) const
{
    // On finding the first block in the file, check how it relates to the existing blockheaders already
    // known. Most importantly from the static data.
    const uint256 blockhash = bh.createHash();
    if (blockhash == m_longestChain.at(0).createHash()) {
        // external file starts at genesis.
        return blockHeight_priv() + 1;
    }
    // do we know the parent?
    auto former = m_blockHeight.find(bh.hashPrevBlock);
    if (former == m_blockHeight.end())  {
        // the file doesn't append data. You may used to have a static file and no longer do.
        return -1; // this will cause the caller to skip loading this file.
    }
    return blockHeight_priv() - former->second;
}

std::vector<Blockchain::CheckPoint> Blockchain::checkpoints() const
{
    return m_checkpoints;
}

void Blockchain::loadSlow(std::ifstream &in, const std::string &meta)
{
    std::ofstream out;
    if (!meta.empty()) {
        out.open(meta, std::ios_base::trunc | std::ios_base::binary);
        if (!out.is_open()) {
            logFatal() << "FAIL Could not write info file:" << meta;
        } else {
            char version = Version;
            out.write(&version, 1);
        }
    }

    int skipNumber = -1;
    while (true) {
        char headerData[80];
        in.read(headerData, sizeof(headerData));
        if (in.fail())
            break;

        const BlockHeader *bh = reinterpret_cast<const BlockHeader*>(headerData);
        if (skipNumber == -1) {
            skipNumber = skipBlocksOfFile(*bh);
            if (skipNumber == -1) {
                logWarning() << "Blockchain WARN: Loaded blocksdata do not match our chain" << meta;
                logWarning() << "  += starts at" << bh->createHash();

                // remove not fully created info file.
                std::error_code ec; // we ignore failures.
                std::filesystem::remove(meta, ec);
                return;
            }
        }
        const uint256 blockhash = bh->createHash();
        if (out.is_open())
            out.write(reinterpret_cast<const char*>(blockhash.begin()), 32);

        if (skipNumber > 0) {
            --skipNumber;
            continue;
        }

        m_tip.chainWork += bh->blockProof();
        m_longestChain.push_back(*bh);
        m_blockHeight.insert(std::make_pair(blockhash, blockHeight_priv()));
    }
    // finish with writing the cumulative work.
    if (out.is_open()) {
        out.write(reinterpret_cast<const char*>(&m_tip.chainWork), 32);
        if (out.fail())
            logFatal() << "Out of disk space";
        out.close();
    }
}

BlockHeader Blockchain::blockAt_priv(size_t i) const
{
    if (i == 0)
        return m_longestChain.at(0);
    // this is an accessor for m_longestHeaders, static headers are not in there.
    assert(m_numStaticHeaders >= 0);
    assert(i > static_cast<size_t>(m_numStaticHeaders));
    return m_longestChain.at(i - m_numStaticHeaders);
}

int Blockchain::blockHeight_priv() const
{
    // m_longestChain has the genesis, followed by the non-static headers.
    // the m_chainStartPoint variable holds the checkpoint we started at.

    // the static headers repeat the genesis block, which we need to ignore.
    int start = m_chainStartPoint;
    if (m_numStaticHeaders) // if static blocks present:
        start += m_numStaticHeaders - 1;   // -1 to ignore the duplicate 'startPoint' block

    // and then by tradition, the blockheight is counted without the genesis block,
    // so we do a minus one.
    return static_cast<int>(m_longestChain.size() + start - 1);
}