tomFlowee
8e833c0a63
The system writes a new file every single run (provided new headers were received) and we sometimes compact them into a big file again. The code forgot to remove the newly introduced info files of the old files it compacted. Leaving confusing things happening after. This solves that by making the first run remove all info files and re- build them, adding a version byte to allow us freedom to do that in the future again.
869 lines
33 KiB
C++
869 lines
33 KiB
C++
/*
|
|
* This file is part of the Flowee project
|
|
* Copyright (C) 2020-2023 Tom Zander <tom@flowee.org>
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
#include "Blockchain.h"
|
|
#include "DownloadManager.h"
|
|
#include "Peer.h"
|
|
#include "streaming/BufferPools.h"
|
|
|
|
#include <filesystem>
|
|
#include <streaming/BufferPool.h>
|
|
#include <streaming/P2PBuilder.h>
|
|
#include <streaming/P2PParser.h>
|
|
#include <utils/utiltime.h>
|
|
#include <interfaces/BitcoinVersion.h> // for PROTOCOL_VERSION
|
|
|
|
#include <stdexcept>
|
|
#include <fstream>
|
|
#include <cstdio> // for rename()
|
|
|
|
constexpr char Version = 1;
|
|
constexpr char MinVersion = 1;
|
|
|
|
struct StaticChain {
|
|
const unsigned char *data = nullptr;
|
|
std::string infoFile;
|
|
int64_t size = 0;
|
|
};
|
|
static StaticChain s_staticChain = StaticChain();
|
|
|
|
/*
|
|
* This class holds as many block-headers as are available.
|
|
* The datasource for these blockheaders ranges from them being downloaded from
|
|
* the p2p network, to having a static file provided with headers.
|
|
*
|
|
* - static-file.
|
|
* A simple file with headers. Sorted by height. Starting with block 1.
|
|
* - static-info-file.
|
|
* A list of block-hashes matching the static file.
|
|
* - blockchain files. See load() for more details.
|
|
*
|
|
* The static file loading is practically instant. It reads the actual file once,
|
|
* and on a laptop that takes 22ms (warm). Additionally, it inserts the hashes
|
|
* from the info file into an unsorted map. This adds another 200ms.
|
|
*
|
|
* The _really_ slow part is to process the blockheaders we did not yet create an
|
|
* info file for before. My test of 3.6 MB of headers (44k headers) took 600ms.
|
|
* So 3 times as long for 1/20th the blocks...
|
|
*/
|
|
|
|
Blockchain::Blockchain(DownloadManager *downloadManager, const boost::filesystem::path &basedir, P2PNet::Chain chain)
|
|
: m_basedir(basedir),
|
|
m_blockHeight(chain == P2PNet::MainChain ? 60001 : 1001),
|
|
m_dlmanager(downloadManager),
|
|
m_saveTimer(m_dlmanager->service())
|
|
{
|
|
assert(m_dlmanager);
|
|
|
|
switch (chain) {
|
|
case P2PNet::MainChain:
|
|
createMainchainGenesis();
|
|
loadMainchainCheckpoints();
|
|
break;
|
|
case P2PNet::Testnet4Chain:
|
|
createTestnet4Genesis();
|
|
loadTestnet4Checkpoints();
|
|
break;
|
|
default:
|
|
assert(false);
|
|
}
|
|
std::ifstream hashes;
|
|
hashes.open(s_staticChain.infoFile);
|
|
if (hashes.is_open()) // if it doesn't exist, silently skip loading
|
|
loadStaticChain(s_staticChain.data, s_staticChain.size, hashes);
|
|
load();
|
|
}
|
|
|
|
Message Blockchain::createGetHeadersRequest()
|
|
{
|
|
std::unique_lock<std::recursive_mutex> lock(m_lock);
|
|
Streaming::P2PBuilder builder(Streaming::pool(4 + 1 + 32 * 11));
|
|
builder.writeInt(PROTOCOL_VERSION);
|
|
if (m_tip.height <= 1000) {
|
|
builder.writeCompactSize(1);
|
|
builder.writeByteArray(m_tip.tip.begin(), 32, Streaming::RawBytes);
|
|
builder.writeByteArray(m_tip.tip.begin(), 32, Streaming::RawBytes);
|
|
} else {
|
|
builder.writeCompactSize(10);
|
|
std::array<int, 10> offsets = {
|
|
m_tip.height,
|
|
m_tip.height - 5,
|
|
m_tip.height - 20,
|
|
m_tip.height - 60,
|
|
m_tip.height - 100,
|
|
m_tip.height - 200,
|
|
m_tip.height - 400,
|
|
m_tip.height - 600,
|
|
m_tip.height - 800,
|
|
m_tip.height - 1000
|
|
};
|
|
for (auto i : offsets) {
|
|
auto bh = block(i);
|
|
uint256 hash = bh.createHash();
|
|
builder.writeByteArray(hash.begin(), 32, Streaming::RawBytes);
|
|
}
|
|
// add the 'stop-at-hash', we don't want them to stop, so we set that to zero's
|
|
uint8_t empty[32];
|
|
memset(empty, 0, 32);
|
|
builder.writeByteArray(empty, 32, Streaming::RawBytes);
|
|
}
|
|
return builder.message(Api::P2P::GetHeaders);
|
|
}
|
|
|
|
void Blockchain::processBlockHeaders(Message message, int peerId)
|
|
{
|
|
int newTip = 0;
|
|
bool goodHeadersSameTip = false;
|
|
int height = 0;
|
|
bool justDisconnect = false;
|
|
try {
|
|
std::unique_lock<std::recursive_mutex> lock(m_lock);
|
|
Streaming::P2PParser parser(message);
|
|
auto count = parser.readCompactInt();
|
|
if (count > 2000) {
|
|
logInfo() << "Peer:" << peerId << "Sent too many headers" << count << "p2p protocol violation";
|
|
m_dlmanager->reportDataFailure(peerId);
|
|
return;
|
|
}
|
|
const uint32_t maxFuture = time(nullptr) + 7200; // headers can not be more than 2 hours in the future.
|
|
|
|
uint256 prevHash;
|
|
int startHeight = -1;
|
|
arith_uint256 chainWork;
|
|
for (size_t i = 0; i < count; ++i) {
|
|
BlockHeader header = BlockHeader::fromMessage(parser);
|
|
/*int txCount =*/ parser.readCompactInt(); // always zero
|
|
|
|
// timestamp not more than 2h in the future.
|
|
if (header.nTime > maxFuture) {
|
|
logWarning() << "Peer:" << peerId << "sent bogus headers. Too far in future";
|
|
m_dlmanager->reportDataFailure(peerId);
|
|
return;
|
|
}
|
|
|
|
if (i == 0) { // first header in the sequence.
|
|
assert(startHeight == -1);
|
|
auto iter = m_blockHeight.find(header.hashPrevBlock);
|
|
if (iter == m_blockHeight.end()) {
|
|
if (header.createHash() == block(0).createHash()) // genesis
|
|
height = 0;
|
|
else
|
|
throw std::runtime_error("is on a different chain, headers don't extend ours");
|
|
}
|
|
else
|
|
height = iter->second; // the height _before_ we add the above header.
|
|
if (m_tip.height == height) {
|
|
chainWork = m_tip.chainWork;
|
|
}
|
|
else if (m_tip.height > height + static_cast<int>(count)) { // too old
|
|
assert(!m_checkpoints.empty());
|
|
auto cp = m_checkpoints.crbegin();
|
|
if (height > cp->first) { // remote has a valid block after our last checkpoint.
|
|
// the remote is behind. We won't punish it, but we should disconnect a useless peer
|
|
logInfo() << "Peer:" << peerId << "is behind, not useful to us.";
|
|
justDisconnect = true;
|
|
throw std::runtime_error("Peer behind");
|
|
}
|
|
throw std::runtime_error("is on a different chain, headers don't extend ours");
|
|
}
|
|
else {
|
|
// rollback the chainWork to branch-point
|
|
const auto curHeight = blockHeight_priv();
|
|
assert(m_tip.height == curHeight);
|
|
chainWork = m_tip.chainWork;
|
|
for (int h = m_tip.height; h >= height; --h) {
|
|
chainWork -= block(h).blockProof();
|
|
}
|
|
}
|
|
}
|
|
else if (prevHash != header.hashPrevBlock) { // check if we are really a sequence.
|
|
throw std::runtime_error("sent bogus headers. Not in sequence");
|
|
}
|
|
uint256 hash = header.createHash();
|
|
// check POW
|
|
{
|
|
bool fNegative;
|
|
bool fOverflow;
|
|
arith_uint256 bnTarget;
|
|
bnTarget.SetCompact(header.nBits, &fNegative, &fOverflow);
|
|
if (fNegative || bnTarget == 0 || fOverflow || bnTarget > UintToArith256(powLimit)
|
|
|| UintToArith256(hash) > bnTarget) {// Check proof of work matches claimed amount
|
|
throw std::runtime_error("sent bogus headers. POW failed");
|
|
}
|
|
}
|
|
chainWork += header.blockProof();
|
|
|
|
if (startHeight == -1)
|
|
startHeight = height;
|
|
++height; // Header is known to fit at the new 'height'
|
|
|
|
auto cpIter = m_checkpoints.find(height);
|
|
if (cpIter != m_checkpoints.end()) {
|
|
if (cpIter->second.hash != hash)
|
|
throw std::runtime_error("is on a different chain, checkpoint failure");
|
|
}
|
|
prevHash = std::move(hash);
|
|
}
|
|
|
|
if (chainWork > m_tip.chainWork) {
|
|
// The new chain has more PoW, apply it.
|
|
parser = Streaming::P2PParser(message);
|
|
count = parser.readCompactInt();
|
|
int offset = startHeight - m_chainStartPoint + 1;
|
|
if (m_numStaticHeaders)
|
|
offset -= m_numStaticHeaders - 1;
|
|
m_longestChain.resize(count + offset);
|
|
for (size_t i = 0; i < count; ++i) {
|
|
BlockHeader header = BlockHeader::fromMessage(parser);
|
|
/*int txCount =*/ parser.readCompactInt(); // always zero
|
|
m_blockHeight.insert(std::make_pair(header.createHash(), startHeight + i + 1));
|
|
m_longestChain[offset + i] = header;
|
|
}
|
|
m_tip.height = startHeight + count;
|
|
m_tip.tip = prevHash;
|
|
m_tip.chainWork = chainWork;
|
|
newTip = m_tip.height;
|
|
assert(m_tip.height == blockHeight_priv());
|
|
logCritical() << "Headers now at" << newTip << m_tip.tip <<
|
|
DateTimeStrFormat("%Y-%m-%d %H:%M:%S", m_longestChain.back().nTime).c_str();
|
|
}
|
|
else if (chainWork == m_tip.chainWork) { // Good headers, same tip we already had
|
|
// we go and do nice things with the peer, but we have to do that outside of the
|
|
// mutex to avoid deadlocks. As such we simply set a bool and delay the action.
|
|
goodHeadersSameTip = true;
|
|
}
|
|
} catch (const std::runtime_error &err) {
|
|
logWarning() << "Peer:" << peerId << "is" << err.what();
|
|
if (justDisconnect) {
|
|
m_dlmanager->connectionManager().disconnect(peerId);
|
|
} else {
|
|
m_dlmanager->reportDataFailure(peerId);
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (goodHeadersSameTip) {
|
|
// Lets tell the PeerAddressDB that
|
|
// this peer got good headers, since that DB is persisted between
|
|
// restarts, we improve our performance by remembering success.
|
|
auto peer = m_dlmanager->connectionManager().peer(peerId);
|
|
if (peer.get()) {
|
|
peer->peerAddress().gotGoodHeaders();
|
|
peer->updatePeerHeight(height);
|
|
}
|
|
}
|
|
else if (newTip > 0) {
|
|
m_dlmanager->headersDownloadFinished(newTip, peerId);
|
|
if (!m_saveTimerStarted && newTip - m_lastSavedHeader > 300) {
|
|
// schedule task to compress all files
|
|
m_saveTimer.expires_from_now(boost::posix_time::seconds(45));
|
|
m_saveTimer.async_wait(std::bind(&Blockchain::save, this));
|
|
m_saveTimerStarted = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// static
|
|
void Blockchain::setStaticChain(const unsigned char *data, int64_t size, const std::string &infoFile)
|
|
{
|
|
assert(size == 0 || data);
|
|
s_staticChain.data = data;
|
|
s_staticChain.size = size;
|
|
s_staticChain.infoFile = infoFile;
|
|
}
|
|
|
|
// static
|
|
bool Blockchain::createStaticHeaders(const std::string &blockchain, const std::string &meta)
|
|
{
|
|
std::ifstream data;
|
|
data.open(blockchain, std::ios_base::binary);
|
|
if (!data.is_open()) {
|
|
logFatal() << "FAIL Could not read input file:" << blockchain;
|
|
return false;
|
|
}
|
|
std::ofstream out;
|
|
out.open(meta, std::ios_base::trunc | std::ios_base::binary);
|
|
if (!out.is_open()) {
|
|
logFatal() << "FAIL Could not write info file:" << meta;
|
|
return false;
|
|
}
|
|
|
|
arith_uint256 chainWork;
|
|
while (true) {
|
|
BlockHeader bh;
|
|
data.read(reinterpret_cast<char*>(&bh), 80);
|
|
if (data.fail())
|
|
break;
|
|
chainWork += bh.blockProof();
|
|
auto blockhash = bh.createHash();
|
|
out.write(reinterpret_cast<const char*>(blockhash.begin()), 32);
|
|
if (out.fail()) {
|
|
logFatal() << "Out of disk space";
|
|
return false;
|
|
}
|
|
}
|
|
data.close();
|
|
out.write(reinterpret_cast<const char*>(&chainWork), 32);
|
|
if (out.fail()) {
|
|
logFatal() << "Out of disk space";
|
|
return false;
|
|
}
|
|
out.close();
|
|
|
|
return true;
|
|
}
|
|
|
|
int Blockchain::height() const
|
|
{
|
|
return m_tip.height;
|
|
}
|
|
|
|
int Blockchain::expectedBlockHeight() const
|
|
{
|
|
std::unique_lock<std::recursive_mutex> lock(m_lock);
|
|
const int ourHeight = blockHeight_priv();
|
|
int secsSinceLastBlock = 0;
|
|
if (m_numStaticHeaders > ourHeight) {
|
|
const BlockHeader *bh = reinterpret_cast<const BlockHeader*>(m_staticChain + 80
|
|
* (ourHeight - m_chainStartPoint));
|
|
secsSinceLastBlock = GetTime() - bh->nTime;
|
|
}
|
|
else {
|
|
secsSinceLastBlock = GetTime() - m_longestChain.back().nTime;
|
|
}
|
|
return m_tip.height + (secsSinceLastBlock + 300) / 600; // add how many 10 minutes chunks fit in the time-span.
|
|
}
|
|
|
|
bool Blockchain::isKnown(const uint256 &blockId) const
|
|
{
|
|
std::unique_lock<std::recursive_mutex> lock(m_lock);
|
|
return m_blockHeight.find(blockId) != m_blockHeight.end();
|
|
}
|
|
|
|
int Blockchain::blockHeightFor(const uint256 &blockId) const
|
|
{
|
|
std::unique_lock<std::recursive_mutex> lock(m_lock);
|
|
auto iter = m_blockHeight.find(blockId);
|
|
if (iter == m_blockHeight.end())
|
|
return -1;
|
|
if (blockHeight_priv() < iter->second)
|
|
return -1;
|
|
return iter->second;
|
|
}
|
|
|
|
int Blockchain::blockHeightAtTime(uint32_t timestamp) const
|
|
{
|
|
std::unique_lock<std::recursive_mutex> lock(m_lock);
|
|
int l = 1;
|
|
int r = blockHeight_priv();
|
|
while (l <= r) {
|
|
int m = (l + r) / 2;
|
|
uint32_t val = block(m).nTime;
|
|
if (val < timestamp)
|
|
l = m + 1;
|
|
else if (val == timestamp)
|
|
return m;
|
|
else
|
|
r = m - 1;
|
|
}
|
|
|
|
return l - 1;
|
|
}
|
|
|
|
BlockHeader Blockchain::block(int height) const
|
|
{
|
|
assert(height >= 0);
|
|
std::unique_lock<std::recursive_mutex> lock(m_lock);
|
|
if (blockHeight_priv() < height)
|
|
return BlockHeader();
|
|
if (height == 0)
|
|
return m_longestChain.at(0); // coinbase is in this list.
|
|
if (height < m_chainStartPoint)
|
|
throw std::runtime_error("Requested unknown block"); // TODO better exception?
|
|
|
|
int offset = m_chainStartPoint;
|
|
if (m_numStaticHeaders) {
|
|
// -1 because we skip the first static header since it duplicates one we already know.
|
|
offset += m_numStaticHeaders - 1;
|
|
}
|
|
if (offset >= height) {
|
|
const BlockHeader *bh = reinterpret_cast<const BlockHeader*>(
|
|
m_staticChain + 80 * (height - m_chainStartPoint));
|
|
return *bh;
|
|
}
|
|
return m_longestChain.at(height - offset);
|
|
}
|
|
|
|
void Blockchain::createMainchainGenesis()
|
|
{
|
|
if (!m_longestChain.empty())
|
|
return;
|
|
BlockHeader genesis;
|
|
genesis.nBits = 0x1d00ffff;
|
|
genesis.nTime = 1231006505;
|
|
genesis.nNonce = 2083236893;
|
|
genesis.nVersion = 1;
|
|
createGenericGenesis(genesis);
|
|
}
|
|
|
|
void Blockchain::loadMainchainCheckpoints()
|
|
{
|
|
CheckPoint cp;
|
|
cp.hash = uint256S("69e244f73d78e8fd29ba2fd2ed618bd6fa2ee92559f542fdb26e7c1d");
|
|
cp.chainWork = 47726404840296l;
|
|
m_checkpoints.insert(std::make_pair( 11111, cp));
|
|
cp.hash = uint256S("2dd5588a74784eaa7ab0507a18ad16a236e7b1ce69f00d7ddfb5d0a6");
|
|
cp.chainWork = 144017457601612;
|
|
m_checkpoints.insert(std::make_pair( 33333, cp));
|
|
cp.hash = uint256S("573993a3c9e41ce34471c079dcf5f52a0e824a81e7f953b8661a20");
|
|
cp.chainWork = 0x1b32578bc31800;
|
|
m_checkpoints.insert(std::make_pair( 74000, cp));
|
|
cp.hash = uint256S("291ce28027faea320c8d2b054b2e0fe44a773f3eefb151d6bdc97");
|
|
cp.chainWork = 0xb5afc0363d92ee0;
|
|
m_checkpoints.insert(std::make_pair(105000, cp));
|
|
cp.hash = uint256S("5b12ffd4cd315cd34ffd4a594f430ac814c91184a0d42d2b0fe");
|
|
cp.chainWork = arith_uint256("1d3d101c6a0418476");
|
|
m_checkpoints.insert(std::make_pair(134444, cp));
|
|
cp.hash = uint256S("99e61ea72015e79632f216fe6cb33d7899acb35b75c8303b763");
|
|
cp.chainWork = arith_uint256("d37e4a7ffd6c0f802");
|
|
m_checkpoints.insert(std::make_pair(168000, cp));
|
|
cp.hash = uint256S("59f452a5f7340de6682a977387c17010ff6e6c3bd83ca8b1317");
|
|
cp.chainWork = arith_uint256("16c80a189db0468c5e");
|
|
m_checkpoints.insert(std::make_pair(193000, cp));
|
|
cp.hash = uint256S("48b95347e83192f69cf0366076336c639f9b7228e9ba171342e");
|
|
cp.chainWork = arith_uint256("2218a01c35910b0060");
|
|
m_checkpoints.insert(std::make_pair(210000, cp));
|
|
cp.hash = uint256S("1b4f4b433e81ee46494af945cf96014816a4e2370f11b23df4e");
|
|
cp.chainWork = arith_uint256("26ba40d1ab30f19b8b");
|
|
m_checkpoints.insert(std::make_pair(216116, cp));
|
|
cp.hash = uint256S("1c108384350f74090433e7fcf79a606b8e797f065b130575932");
|
|
cp.chainWork = arith_uint256("2e463ab5728a736a10");
|
|
m_checkpoints.insert(std::make_pair(225430, cp));
|
|
cp.hash = uint256S("3887df1f29024b06fc2200b55f8af8f35453d7be294df2d214");
|
|
cp.chainWork = arith_uint256("8113ee25e591519297");
|
|
m_checkpoints.insert(std::make_pair(250000, cp));
|
|
cp.hash = uint256S("1ae8c72a0b0c301f67e3afca10e819efa9041e458e9bd7e40");
|
|
cp.chainWork = arith_uint256("b2f4e230c399a9618e4");
|
|
m_checkpoints.insert(std::make_pair(279000, cp));
|
|
cp.hash = uint256S("4d9b4ef50f0f9d686fd69db2e03af35a100370c64632a983");
|
|
cp.chainWork = arith_uint256("3a485160e85f201be9fd");
|
|
m_checkpoints.insert(std::make_pair(295000, cp));
|
|
cp.hash = uint256S("651ef99cb9fcbe0dadde1d424bd9f15ff20136191a5eec");
|
|
cp.chainWork = arith_uint256("74489ee978b0489ded1f20");
|
|
m_checkpoints.insert(std::make_pair(478559, cp));
|
|
cp.hash = uint256S("4626ff6e3b936941d341c5932ece4357eeccac44e6d56c");
|
|
cp.chainWork = arith_uint256("d336f3c3e9334d7fa41ee6 ");
|
|
m_checkpoints.insert(std::make_pair(556767, cp));
|
|
cp.hash = uint256S("1b4b8e36aec7d4f9671a47872cb9a74dc16ca398c7dcc18");
|
|
cp.chainWork = arith_uint256("eafdfa9271c11ceae5513a");
|
|
m_checkpoints.insert(std::make_pair(582680, cp));
|
|
cp.hash = uint256S("b48bb207faac5ac655c313e41ac909322eaa694f5bc5b1");
|
|
cp.chainWork = arith_uint256("1090f04074679a2ec4f811d");
|
|
m_checkpoints.insert(std::make_pair(609136, cp));
|
|
cp.hash = uint256S("33dfef1fc2d6a5d5520b078c55193a9bf498c5b27530f7");
|
|
cp.chainWork = arith_uint256("1313793fe89fc0038d83086");
|
|
m_checkpoints.insert(std::make_pair(635259, cp));
|
|
cp.hash = uint256S("29e471c41818d24b8b74c911071c4ef0b4a0509f9b5a8ce");
|
|
cp.chainWork = arith_uint256("15353b8fc95f466c3c16744");
|
|
m_checkpoints.insert(std::make_pair(661648, cp));
|
|
cp.hash = uint256S("480527d21eb07089d8390caef5008ba2971fd554777fae");
|
|
cp.chainWork = arith_uint256("1a19604fff5408f9e752840");
|
|
m_checkpoints.insert(std::make_pair(760000, cp)); // End Sept 2022
|
|
}
|
|
|
|
void Blockchain::createTestnet4Genesis()
|
|
{
|
|
if (!m_longestChain.empty())
|
|
return;
|
|
BlockHeader genesis;
|
|
genesis.nBits = 0x1d00ffff;
|
|
genesis.nTime = 1597811185;
|
|
genesis.nNonce = 114152193;
|
|
genesis.nVersion = 1;
|
|
createGenericGenesis(genesis);
|
|
}
|
|
|
|
void Blockchain::loadTestnet4Checkpoints()
|
|
{
|
|
CheckPoint cp;
|
|
cp.hash = uint256S("0x0000000019df558b6686b1a1c3e7aee0535c38052651b711f84eebafc0cc4b5e");
|
|
cp.chainWork = 0x32bd583ca267;
|
|
m_checkpoints.insert(std::make_pair(5677, cp));
|
|
cp.hash = uint256S("0x00000000016522b7506939b23734bca7681c42a53997f2943ab4c8013936b419");
|
|
cp.chainWork = 0x2bd3345121f1a;
|
|
m_checkpoints.insert(std::make_pair(9999, cp));
|
|
}
|
|
|
|
void Blockchain::loadStaticChain(const unsigned char *data, int64_t dataSize, std::ifstream &infoFile)
|
|
{
|
|
m_chainStartPoint = -1; // checking for invalid.
|
|
if (dataSize > 80 ) {
|
|
// find starting point.
|
|
assert(m_longestChain.size() >= 1); // always mininum one block, the genesis one.
|
|
const BlockHeader *firstBlock = reinterpret_cast<const BlockHeader*>(data);
|
|
const uint256 firstBlockHash = firstBlock->createHash();
|
|
if (firstBlockHash == m_longestChain.front().createHash()) {
|
|
m_chainStartPoint = 0; // genesis block
|
|
}
|
|
else {
|
|
for (const auto &cp : m_checkpoints) {
|
|
if (cp.second.hash == firstBlockHash) {
|
|
m_chainStartPoint = cp.first;
|
|
m_tip.chainWork = cp.second.chainWork;
|
|
m_tip.chainWork -= firstBlock->blockProof(); // don't count the checkpoint block twice
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (m_chainStartPoint == -1) {
|
|
logWarning() << "Ignoring static blockchain, not for this network (genesis does not match)";
|
|
m_chainStartPoint = 0;
|
|
return;
|
|
}
|
|
|
|
int numHeadersFound = 0;
|
|
uint256 blockhash;
|
|
for (int64_t pos = 0; pos + 80 <= dataSize; pos += 80) {
|
|
infoFile.read(reinterpret_cast<char*>(blockhash.begin()), 32);
|
|
if (infoFile.fail())
|
|
throw std::runtime_error("loadStaticChain: info file truncated");
|
|
if (m_chainStartPoint != 0 || pos > 0) // genesis was inserted in the createGenesis method
|
|
m_blockHeight.insert(std::make_pair(blockhash, numHeadersFound + m_chainStartPoint));
|
|
++numHeadersFound;
|
|
}
|
|
if (numHeadersFound > 1) {
|
|
m_staticChain = data;
|
|
m_numStaticHeaders = numHeadersFound;
|
|
|
|
m_tip.height = blockHeight_priv();
|
|
m_tip.tip = block(m_tip.height).createHash();
|
|
|
|
arith_uint256 staticChainWork;
|
|
infoFile.read(reinterpret_cast<char*>(&staticChainWork), 32);
|
|
if (infoFile.fail())
|
|
throw std::runtime_error("loadStaticChain: info file truncated");
|
|
m_tip.chainWork += staticChainWork;
|
|
}
|
|
}
|
|
|
|
void Blockchain::createGenericGenesis(BlockHeader genesis)
|
|
{
|
|
genesis.hashMerkleRoot = uint256S("4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b");
|
|
m_longestChain.push_back(genesis);
|
|
|
|
const uint256 genesisHash = genesis.createHash();
|
|
m_blockHeight.insert(std::make_pair(genesisHash, 0));
|
|
m_tip.tip = genesisHash;
|
|
m_tip.height = 0;
|
|
m_tip.chainWork += genesis.blockProof();
|
|
}
|
|
|
|
void Blockchain::load()
|
|
{
|
|
/*
|
|
* We check a series of files starting with 'blockchain' and load them in sequence.
|
|
* When there are more than 60 files we schedule a write to replace all of them with
|
|
* a single one. Since headers generate about 4MB per year, this should all be quite
|
|
* small due to the usage of the static file which comes from the install.
|
|
*
|
|
* The idea is that we cut down a lot on dupicate-saving while also minimizing the
|
|
* load of saving at shutdown.
|
|
* So, for saving we save at a timeout after we received a large number of headers,
|
|
* to a new file. At shutdown we only save the stuff we have not saved before, again
|
|
* to a new file.
|
|
*/
|
|
std::unique_lock<std::recursive_mutex> lock(m_lock);
|
|
|
|
Streaming::BufferPool pool;
|
|
for (m_fileCount = 0; m_fileCount < 100; ++m_fileCount) {
|
|
std::string suffix;
|
|
if (m_fileCount > 0)
|
|
suffix = std::to_string(m_fileCount);
|
|
std::string path((m_basedir / (std::string("blockchain") + suffix)).string());
|
|
|
|
std::ifstream in(path);
|
|
if (!in.is_open()) {
|
|
if (m_fileCount > 60) {
|
|
// schedule task to compress all files
|
|
m_saveTimer.expires_from_now(boost::posix_time::seconds(45));
|
|
m_saveTimer.async_wait(std::bind(&Blockchain::compressSaveFiles, this,
|
|
std::placeholders::_1));
|
|
m_saveTimerStarted = true;
|
|
}
|
|
break;
|
|
}
|
|
logInfo() << "Starting to load the blockchain" << (suffix.empty() ? "" : "number:") << suffix;
|
|
|
|
const auto infoPath = path + ".info";
|
|
std::ifstream info(infoPath);
|
|
if (info.is_open()) {
|
|
const auto inSize = boost::filesystem::file_size(path);
|
|
const auto infoSize = boost::filesystem::file_size(infoPath);
|
|
const int inHeaderCount = inSize / 80;
|
|
const int infoHeaderCount = infoSize / 32;
|
|
|
|
/* two size comparisons.
|
|
*
|
|
* inHeaderCount == infoHeaderCount - 1
|
|
* this simply checks that we have the right count of headers vs blockIds.
|
|
* with the '-1' there to account for the info file having an additional
|
|
* chain-work 32-bytes row.
|
|
*
|
|
* second comparison or check:
|
|
*
|
|
* infoHeaderCount * 32 + 1 != infoSize
|
|
* the infoHeaderCount is the result of an int division, so our accuracy is 32 bytes.
|
|
* We need exactly one more byte in the file, which is the opening version byte.
|
|
*/
|
|
if (inHeaderCount != infoHeaderCount - 1 || infoHeaderCount * 32 + 1 != infoSize) {
|
|
logWarning() << "Skipping usage of info file, it has the wrong size:" << infoPath;
|
|
info.close();
|
|
}
|
|
}
|
|
if (info.is_open()) {
|
|
char version;
|
|
info.read(&version, 1);
|
|
if (version < MinVersion) {
|
|
logWarning() << "Skipping usage of info file, too old version" << version;
|
|
info.close();
|
|
}
|
|
}
|
|
if (info.is_open()) {
|
|
int skipNumber = -1;
|
|
// load the data from the 'in' file and also the 'info' file.
|
|
while (true) {
|
|
char headerData[80];
|
|
in.read(headerData, sizeof(headerData));
|
|
if (in.fail()) // end of file.
|
|
break;
|
|
const BlockHeader *bh = reinterpret_cast<const BlockHeader*>(headerData);
|
|
|
|
if (skipNumber == -1) {
|
|
skipNumber = skipBlocksOfFile(*bh);
|
|
if (skipNumber == -1) {
|
|
logWarning() << "Blockchain WARN: Loaded blocksdata do not match our chain" << infoPath;
|
|
logWarning() << " += starts at" << bh->createHash();
|
|
return;
|
|
}
|
|
}
|
|
|
|
uint256 blockhash;
|
|
info.read(reinterpret_cast<char*>(blockhash.begin()), 32);
|
|
if (info.fail()) {
|
|
logWarning() << "Failed to read from 'info' file." << infoPath;
|
|
break;
|
|
}
|
|
if (skipNumber > 0) {
|
|
--skipNumber;
|
|
continue;
|
|
}
|
|
|
|
m_longestChain.push_back(*bh);
|
|
m_blockHeight.insert(std::make_pair(blockhash, blockHeight_priv()));
|
|
}
|
|
|
|
info.read(reinterpret_cast<char*>(&m_tip.chainWork), 32);
|
|
assert(!info.fail());
|
|
}
|
|
else {
|
|
// if there is no 'info' file for this one yet,
|
|
// load the blockchain file the 'slow' way (which includes hashing) and
|
|
// also write the info file afterwards.
|
|
loadSlow(in, path + ".info");
|
|
}
|
|
}
|
|
|
|
if (m_tip.height < blockHeight_priv()) {
|
|
m_tip.tip = m_longestChain.back().createHash();
|
|
m_tip.height = blockHeight_priv();
|
|
}
|
|
logCritical() << "Blockchain loading completed. Tip:" << m_tip.height << m_tip.tip;
|
|
m_lastSavedHeader = blockHeight_priv();
|
|
}
|
|
|
|
void Blockchain::save()
|
|
{
|
|
boost::system::error_code error;
|
|
boost::filesystem::create_directories(m_basedir, error);
|
|
if (error && !boost::filesystem::exists(m_basedir) && !boost::filesystem::is_directory(m_basedir)) {
|
|
logFatal() << "P2P.Blockchain can't save. Failed creating the dir:" << m_basedir.string();
|
|
return;
|
|
}
|
|
std::unique_lock<std::recursive_mutex> lock(m_lock);
|
|
m_saveTimerStarted = false;
|
|
if (m_lastSavedHeader == blockHeight_priv()) // nothing to do
|
|
return;
|
|
|
|
/*
|
|
* This method saves the not yet saved blocks into a new file.
|
|
* We skip creating the info file here because it would be a duplication of
|
|
* code and the actual gain in time is minuscule. The lack of the metadata
|
|
* file simply means that exactly once it will be loaded with the 'loadSlow()' method
|
|
* which creates the metadata.
|
|
*/
|
|
std::string suffix;
|
|
if (m_fileCount > 0)
|
|
suffix = std::to_string(m_fileCount);
|
|
++m_fileCount;
|
|
std::ofstream out((m_basedir / (std::string("blockchain") + suffix)).string());
|
|
Streaming::BufferPool pool;
|
|
const size_t end = blockHeight_priv();
|
|
for (size_t i = m_lastSavedHeader + 1; i <= end; ++i) {
|
|
const auto &header = block(i);
|
|
assert(!header.hashMerkleRoot.IsNull());
|
|
auto cd = header.write(pool);
|
|
assert(cd.size() == 80);
|
|
out.write(cd.begin(), cd.size());
|
|
}
|
|
m_lastSavedHeader = blockHeight_priv();
|
|
out.close();
|
|
}
|
|
|
|
void Blockchain::compressSaveFiles(const boost::system::error_code &error)
|
|
{
|
|
if (error) // operation aborted is the main one we expect here.
|
|
return;
|
|
std::unique_lock<std::recursive_mutex> lock(m_lock);
|
|
auto path = (m_basedir / "blockchain.part").string();
|
|
std::ofstream out(path);
|
|
Streaming::BufferPool pool;
|
|
for (size_t i = 1; i < m_longestChain.size(); ++i) {
|
|
const auto &header = m_longestChain.at(i);
|
|
auto cd = header.write(pool);
|
|
assert(cd.size() == 80);
|
|
out.write(cd.begin(), cd.size());
|
|
}
|
|
|
|
auto newPath(path);
|
|
newPath.resize(newPath.size() - 5);
|
|
out.flush();
|
|
out.close();
|
|
if (rename(path.c_str(), newPath.c_str()) != 0) {
|
|
logFatal() << "Rename of blockchain part to blockchain failed";
|
|
return;
|
|
}
|
|
for (int i = 1; i < m_fileCount; ++i) {
|
|
path = newPath + std::to_string(i);
|
|
if (remove(path.c_str()) != 0)
|
|
logFatal() << "Failed to remove blockchain file" << path;
|
|
path += ".info";
|
|
if (remove(path.c_str()) != 0)
|
|
logFatal() << "Failed to remove blockchain meta file" << path;
|
|
}
|
|
|
|
m_lastSavedHeader = blockHeight_priv();
|
|
m_saveTimerStarted = false;
|
|
m_fileCount = 1;
|
|
}
|
|
|
|
int Blockchain::skipBlocksOfFile(const BlockHeader &bh) const
|
|
{
|
|
// On finding the first block in the file, check how it relates to the existing blockheaders already
|
|
// known. Most importantly from the static data.
|
|
const uint256 blockhash = bh.createHash();
|
|
if (blockhash == m_longestChain.at(0).createHash()) {
|
|
// external file starts at genesis.
|
|
return blockHeight_priv() + 1;
|
|
}
|
|
// do we know the parent?
|
|
auto former = m_blockHeight.find(bh.hashPrevBlock);
|
|
if (former == m_blockHeight.end()) {
|
|
// the file doesn't append data. You may used to have a static file and no longer do.
|
|
return -1; // this will cause the caller to skip loading this file.
|
|
}
|
|
return blockHeight_priv() - former->second;
|
|
}
|
|
|
|
void Blockchain::loadSlow(std::ifstream &in, const std::string &meta)
|
|
{
|
|
std::ofstream out;
|
|
if (!meta.empty()) {
|
|
out.open(meta, std::ios_base::trunc | std::ios_base::binary);
|
|
if (!out.is_open())
|
|
logFatal() << "FAIL Could not write info file:" << meta;
|
|
char version = Version;
|
|
out.write(&version, 1);
|
|
}
|
|
|
|
int skipNumber = -1;
|
|
while (true) {
|
|
char headerData[80];
|
|
in.read(headerData, sizeof(headerData));
|
|
if (in.fail())
|
|
break;
|
|
|
|
const BlockHeader *bh = reinterpret_cast<const BlockHeader*>(headerData);
|
|
if (skipNumber == -1) {
|
|
skipNumber = skipBlocksOfFile(*bh);
|
|
if (skipNumber == -1) {
|
|
logWarning() << "Blockchain WARN: Loaded blocksdata do not match our chain" << meta;
|
|
logWarning() << " += starts at" << bh->createHash();
|
|
|
|
// remove not fully created info file.
|
|
std::error_code ec; // we ignore failures.
|
|
std::filesystem::remove(meta, ec);
|
|
return;
|
|
}
|
|
}
|
|
const uint256 blockhash = bh->createHash();
|
|
if (out.is_open())
|
|
out.write(reinterpret_cast<const char*>(blockhash.begin()), 32);
|
|
|
|
if (skipNumber > 0) {
|
|
--skipNumber;
|
|
continue;
|
|
}
|
|
|
|
m_tip.chainWork += bh->blockProof();
|
|
m_longestChain.push_back(*bh);
|
|
m_blockHeight.insert(std::make_pair(blockhash, blockHeight_priv()));
|
|
}
|
|
// finish with writing the cumulative work.
|
|
if (out.is_open()) {
|
|
out.write(reinterpret_cast<const char*>(&m_tip.chainWork), 32);
|
|
if (out.fail())
|
|
logFatal() << "Out of disk space";
|
|
out.close();
|
|
}
|
|
}
|
|
|
|
BlockHeader Blockchain::blockAt_priv(size_t i) const
|
|
{
|
|
if (i == 0)
|
|
return m_longestChain.at(0);
|
|
// this is an accessor for m_longestHeaders, static headers are not in there.
|
|
assert(m_numStaticHeaders >= 0);
|
|
assert(i > static_cast<size_t>(m_numStaticHeaders));
|
|
return m_longestChain.at(i - m_numStaticHeaders);
|
|
}
|
|
|
|
int Blockchain::blockHeight_priv() const
|
|
{
|
|
// m_longestChain has the genesis, followed by the non-static headers.
|
|
// the m_chainStartPoint variable holds the checkpoint we started at.
|
|
|
|
// the static headers repeat the genesis block, which we need to ignore.
|
|
int start = m_chainStartPoint;
|
|
if (m_numStaticHeaders) // if static blocks present:
|
|
start += m_numStaticHeaders - 1; // -1 to ignore the duplicate 'startPoint' block
|
|
|
|
// and then by tradition, the blockheight is counted without the genesis block,
|
|
// so we do a minus one.
|
|
return static_cast<int>(m_longestChain.size() + start - 1);
|
|
}
|