tomFlowee
6c9f1c4651
Instead of all places having their own copy and being wasteful in that way, we move ownership of an app wide version to the application singleton.
351 lines
11 KiB
C++
351 lines
11 KiB
C++
/*
|
|
* This file is part of the Flowee project
|
|
* Copyright (C) 2024-2025 Tom Zander <tom@flowee.org>
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
#include "BlockHeadersChecker.h"
|
|
#include "FloweePay.h"
|
|
|
|
#include <utils/Logger.h>
|
|
#include <utils/streaming/BufferPools.h>
|
|
#include <p2p/Blockchain.h>
|
|
#include <QTimer>
|
|
#include <QNetworkReply>
|
|
|
|
static constexpr const char * HEADERFILE = "https://flowee.org/products/pay/blockheaders";
|
|
|
|
#define TEMP_DOWNLOAD_FILENAME "module_blocks_newheaders"
|
|
#define PAY_STATIC_HEADERS "staticHeaders"
|
|
|
|
BlockHeadersChecker::BlockHeadersChecker(QObject *parent)
|
|
: QObject(parent)
|
|
{
|
|
connect (this, &BlockHeadersChecker::finishUp, this, [=](Status status) {
|
|
this->setStatus(status);
|
|
}, Qt::QueuedConnection);
|
|
}
|
|
|
|
int BlockHeadersChecker::wantedHeight() const
|
|
{
|
|
return m_wantedHeight;
|
|
}
|
|
|
|
void BlockHeadersChecker::setWantedHeight(int h)
|
|
{
|
|
if (m_wantedHeight == h)
|
|
return;
|
|
m_wantedHeight = h;
|
|
emit wantedHeightChanged();
|
|
|
|
QTimer::singleShot(0, this, SLOT(startChecking()));
|
|
}
|
|
|
|
void BlockHeadersChecker::startChecking()
|
|
{
|
|
logInfo() << "starting" << m_wantedHeight;
|
|
assert(m_checkpoint >= 0);
|
|
if (m_wantedHeight == 0 || FloweePay::instance()->p2pNet()->chain() != P2PNet::MainChain) {
|
|
setStatus(NoDownloadNeeded);
|
|
return;
|
|
}
|
|
setStatus(Checking);
|
|
const auto &blockchain = FloweePay::instance()->p2pNet()->blockchain();
|
|
bool have = false;
|
|
const auto sources = blockchain.dataSources();
|
|
assert(!sources.empty());
|
|
for (const auto &source : sources) {
|
|
if (source.startBlock <= m_wantedHeight)
|
|
have = true;
|
|
}
|
|
if (have) {
|
|
logDebug() << "Nothing to do, the blockchain has the needed headers";
|
|
setStatus(NoDownloadNeeded);
|
|
return;
|
|
}
|
|
|
|
int startHeight = 0;
|
|
for (const auto &cp : blockchain.checkpoints()) {
|
|
if (cp.height < m_wantedHeight)
|
|
startHeight = cp.height;
|
|
else
|
|
break; // checkpoints are ordered.
|
|
}
|
|
|
|
m_checkpoint = startHeight;
|
|
m_downloadTo = sources.front().startBlock;
|
|
|
|
// do a HEAD
|
|
if (m_headerReply == nullptr) {
|
|
QNetworkRequest headRequest((QUrl(HEADERFILE)));
|
|
headRequest.setTransferTimeout(6000);
|
|
m_headerReply = FloweePay::instance()->network()->head(headRequest);
|
|
connect(m_headerReply, SIGNAL(finished()), this, SLOT(headerReturned()));
|
|
}
|
|
}
|
|
|
|
void BlockHeadersChecker::headerReturned()
|
|
{
|
|
assert(m_headerReply);
|
|
auto reply = m_headerReply;
|
|
m_headerReply->deleteLater();
|
|
m_headerReply = nullptr;
|
|
|
|
if (m_newHeaders) {
|
|
logFatal() << "Already downloading...";
|
|
return;
|
|
}
|
|
if (reply->error() != QNetworkReply::NoError) {
|
|
logDebug() << reply->errorString();
|
|
setStatus(NetworkFailure);
|
|
return;
|
|
}
|
|
|
|
uint64_t length = reply->header(QNetworkRequest::ContentLengthHeader).toLongLong();
|
|
auto allowedRange = reply->rawHeader("Accept-Ranges");
|
|
|
|
if (allowedRange != "bytes") {
|
|
setStatus(NetworkFailure);
|
|
logFatal() << "Can't download partial files. Failing";
|
|
return;
|
|
}
|
|
|
|
// Check if this is a continuation download.
|
|
assert(m_downloadTo > m_checkpoint);
|
|
assert(m_checkpoint > 0);
|
|
m_newHeaders = new QFile(TEMP_DOWNLOAD_FILENAME, this);
|
|
bool isContinuation = false;
|
|
if (m_newHeaders->open(QIODevice::ReadOnly)) {
|
|
char bytes[80];
|
|
auto read = m_newHeaders->read(bytes, 80);
|
|
if (read == 80) {
|
|
BlockHeader *bh = reinterpret_cast<BlockHeader*>(bytes);
|
|
auto hash = bh->createHash();
|
|
auto &blockchain = FloweePay::instance()->p2pNet()->blockchain();
|
|
for (const auto &cp : blockchain.checkpoints()) {
|
|
if (cp.height == m_checkpoint) {
|
|
isContinuation = cp.hash == hash;
|
|
logCritical() << "Partial file found of" << m_newHeaders->size()
|
|
<< "bytes. Continuing!";
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
m_newHeaders->close();
|
|
}
|
|
|
|
// We're going to download the headers.
|
|
// On the remote server there is one file that has the genesis
|
|
// all the way up to a recent height=((length / 80) - 1)
|
|
|
|
// We use the 'range' feature of the webserver to only download
|
|
// what we need, so lets calculate the offsets here.
|
|
const int from = m_checkpoint * 80 + (isContinuation ? m_newHeaders->size() : 0);
|
|
const int to = m_downloadTo * 80;
|
|
if (to > static_cast<int>(length)) {
|
|
logCritical() << "Remote file does not have the data we want";
|
|
setStatus(NetworkFailure);
|
|
return;
|
|
}
|
|
|
|
QNetworkRequest downloadRequest((QUrl(HEADERFILE)));
|
|
downloadRequest.setTransferTimeout(30000);
|
|
QString range("bytes=%1-%2");
|
|
range = range.arg(from);
|
|
range = range.arg(to - 1); // -1 because ranges are inclusive.
|
|
downloadRequest.setRawHeader("Range", range.toLatin1());
|
|
|
|
if (!m_newHeaders->open(QIODevice::WriteOnly
|
|
| (isContinuation ? QIODevice::Append : QIODevice::Truncate))) {
|
|
logFatal() << "Can not open file...";
|
|
setStatus(DiskFailure);
|
|
return;
|
|
}
|
|
setTotalDownload(to - from);
|
|
setStatus(Downloading);
|
|
|
|
// actually start the download
|
|
m_downloadReply = FloweePay::instance()->network()->get(downloadRequest);
|
|
connect(m_downloadReply, SIGNAL(readyRead()), this, SLOT(onBytesDownloaded()));
|
|
connect(m_downloadReply, SIGNAL(finished()), this, SLOT(downloadFinished()));
|
|
}
|
|
|
|
void BlockHeadersChecker::downloadFinished()
|
|
{
|
|
assert(m_downloadReply);
|
|
auto reply = m_downloadReply;
|
|
m_downloadReply->deleteLater();
|
|
m_downloadReply = nullptr;
|
|
|
|
if (reply->error() != QNetworkReply::NoError) {
|
|
logCritical() << "headers download gave:" << reply->errorString();
|
|
setStatus(NetworkFailure);
|
|
m_newHeaders->close();
|
|
m_newHeaders->deleteLater();
|
|
m_newHeaders = nullptr;
|
|
return;
|
|
}
|
|
|
|
assert(m_newHeaders->isOpen());
|
|
logDebug() << "Wrote out:" << m_newHeaders->size() << "bytes" << (m_newHeaders->size() / 80);
|
|
// --- use the file:
|
|
// the new file is the headers I didn't have yet. We need to
|
|
// create a new file that has both the old static headers as well
|
|
// as the new ones concatenated in one.
|
|
// the file we put this in has to have an application-long lifetime.
|
|
// so lets give ownership to the main app.
|
|
|
|
QFile oldStaticHeaders(PAY_STATIC_HEADERS);
|
|
if (!oldStaticHeaders.open(QIODevice::ReadOnly)) {
|
|
logFatal() << "Failed to find old static headers";
|
|
setStatus(DiskFailure);
|
|
return;
|
|
}
|
|
logDebug() << " old ones:" << oldStaticHeaders.size() << "=>" << oldStaticHeaders.size() / 80;
|
|
setStatus(Verifying);
|
|
char buf[4096];
|
|
while (true) {
|
|
auto count = oldStaticHeaders.read(buf, sizeof(buf));
|
|
if (count <= 0)
|
|
break;
|
|
m_newHeaders->write(buf, count);
|
|
}
|
|
oldStaticHeaders.close();
|
|
m_newHeaders->close();
|
|
|
|
// make sure that the UI can update the changes we pushed here
|
|
// before we start the process new headers which is going to take
|
|
// some actual processing time, depending on how big our headers
|
|
// are now.
|
|
// keep the file object from getting garbage collected.
|
|
// the map() requires the QFile to stay alive.
|
|
m_newHeaders->setParent(FloweePay::instance());
|
|
FloweePay::instance()->ioService().post(std::bind(&BlockHeadersChecker::processNewHeaders, this));
|
|
}
|
|
|
|
void BlockHeadersChecker::processNewHeaders()
|
|
{
|
|
// the currenty in use static headers are in the location I want the new ones to be
|
|
// because when we restart, we open by path.
|
|
//
|
|
// After a successful attempt to replace staticChain we'll thus remove the old
|
|
// staticHeaders file and move the one on top of that.
|
|
// "staticHeaders_new" -> finished,
|
|
// static headers path...
|
|
// Knowing that this will never run on Windowz, lets just rename the open file and
|
|
// place the newly created file at the known path.
|
|
|
|
struct RAII {
|
|
explicit RAII(BlockHeadersChecker *dd) : d(dd) {}
|
|
~RAII() {
|
|
emit d->finishUp(status);
|
|
}
|
|
BlockHeadersChecker *d;
|
|
Status status = Success;
|
|
};
|
|
RAII raii(this);
|
|
|
|
if (!m_newHeaders->open(QIODevice::ReadOnly)) {
|
|
logFatal() << "Failed to re-open replacement static headers";
|
|
raii.status = DiskFailure;
|
|
return;
|
|
}
|
|
try {
|
|
logCritical() << "Prepared a new static headers chain. Size:" << m_newHeaders->size()
|
|
<< "Replacing the current one with this new file";
|
|
auto &blockchain = FloweePay::instance()->p2pNet()->blockchain();
|
|
blockchain.replaceStaticChain(m_newHeaders->map(0, m_newHeaders->size()),
|
|
m_newHeaders->size(), TEMP_DOWNLOAD_FILENAME ".info");
|
|
m_newHeaders->close();
|
|
|
|
// make sure we use the new ones after restart.
|
|
QFile::remove(PAY_STATIC_HEADERS);
|
|
QFile::remove(PAY_STATIC_HEADERS ".info");
|
|
QFile::rename(TEMP_DOWNLOAD_FILENAME ".info", PAY_STATIC_HEADERS ".info");
|
|
m_newHeaders->rename(PAY_STATIC_HEADERS);
|
|
} catch (const std::exception &e) {
|
|
logCritical() << "Replace static chain failed. Reason:" << e;
|
|
raii.status = NetworkFailure;
|
|
m_newHeaders->close();
|
|
m_newHeaders->remove();
|
|
m_newHeaders->deleteLater();
|
|
m_newHeaders = nullptr;
|
|
}
|
|
}
|
|
|
|
void BlockHeadersChecker::onBytesDownloaded()
|
|
{
|
|
assert(m_downloadReply);
|
|
assert(m_newHeaders);
|
|
assert(m_newHeaders->isOpen());
|
|
|
|
int downloaded = 0;
|
|
char buf[4096];
|
|
while (true) {
|
|
auto readCount = m_downloadReply->read(buf, sizeof(buf));
|
|
if (readCount <= 0)
|
|
break;
|
|
m_newHeaders->write(buf, readCount);
|
|
downloaded += readCount;
|
|
}
|
|
setBytesDownloaded(m_bytesDownloaded + downloaded);
|
|
}
|
|
|
|
BlockHeadersChecker::Status BlockHeadersChecker::status() const
|
|
{
|
|
return m_status;
|
|
}
|
|
|
|
void BlockHeadersChecker::setStatus(Status newStatus)
|
|
{
|
|
if (m_status == newStatus)
|
|
return;
|
|
m_status = newStatus;
|
|
emit statusChanged();
|
|
}
|
|
|
|
void BlockHeadersChecker::restart()
|
|
{
|
|
if (m_headerReply == nullptr
|
|
&& m_downloadReply == nullptr
|
|
&& m_newHeaders == nullptr)
|
|
startChecking();
|
|
}
|
|
|
|
int BlockHeadersChecker::totalDownload() const
|
|
{
|
|
return m_totalDownload;
|
|
}
|
|
|
|
void BlockHeadersChecker::setTotalDownload(int newTotalDownload)
|
|
{
|
|
if (m_totalDownload == newTotalDownload)
|
|
return;
|
|
m_totalDownload = newTotalDownload;
|
|
emit totalDownloadChanged();
|
|
}
|
|
|
|
void BlockHeadersChecker::setBytesDownloaded(int count)
|
|
{
|
|
if (m_bytesDownloaded == count)
|
|
return;
|
|
m_bytesDownloaded = count;
|
|
emit bytesDownloadedChanged();
|
|
}
|
|
|
|
int BlockHeadersChecker::bytesDownloaded()
|
|
{
|
|
return m_bytesDownloaded;
|
|
}
|