tomFlowee
1662f51193
We parse a date/time for headers now with a specific method that returns a QDateTime We added a partial (incremental) download feature to avoid the need to wait until completed. This also allows us to lower mem usage by spooling to disk as data comes in.
293 lines
11 KiB
C++
293 lines
11 KiB
C++
/*
|
|
* This file is part of the Flowee project
|
|
* Copyright (C) 2025 Tom Zander <tom@flowee.org>
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
#include "SimpleHttpClient.h"
|
|
|
|
#include <Logger.h>
|
|
#include <QCoreApplication>
|
|
#include <boost/asio/buffers_iterator.hpp>
|
|
#include <utils/streaming/BufferPools.h>
|
|
|
|
// static
|
|
std::shared_ptr<SimpleHttpClient> SimpleHttpClient::create(boost::asio::io_context &context,
|
|
boost::asio::ssl::context &sslContext,
|
|
const boost::beast::http::request<boost::beast::http::vector_body<char>> &request)
|
|
{
|
|
std::shared_ptr<SimpleHttpClient> rc;
|
|
rc.reset(new SimpleHttpClient(context, sslContext));
|
|
|
|
if (request.find(boost::beast::http::field::host) == request.end())
|
|
throw std::runtime_error("Missing hostname");
|
|
|
|
rc->m_request = request;
|
|
auto app = QCoreApplication::instance();
|
|
QString useragent = QString("%1%2/%3")
|
|
.arg(app->organizationName(),
|
|
app->applicationName(),
|
|
app->applicationVersion());
|
|
rc->m_request.set(boost::beast::http::field::user_agent,
|
|
useragent.toStdString());
|
|
auto size = QString::number(request.body().size());
|
|
rc->m_request.set(boost::beast::http::field::content_length, size.toStdString());
|
|
|
|
// kickstart the process.
|
|
boost::asio::post(context, std::bind(&SimpleHttpClient::resolve, rc));
|
|
return rc;
|
|
}
|
|
|
|
std::shared_ptr<SimpleHttpClient> SimpleHttpClient::createIncremental(boost::asio::io_context &context, boost::asio::ssl::context &sslContext, const boost::beast::http::request<boost::beast::http::vector_body<char> > &request)
|
|
{
|
|
auto p = create(context, sslContext, request);
|
|
p->m_incremental = true;
|
|
return p;
|
|
}
|
|
|
|
int SimpleHttpClient::httpErrorCode() const
|
|
{
|
|
if (m_response)
|
|
return m_response->result_int();
|
|
return m_headResponse.result_int();
|
|
}
|
|
|
|
std::string_view SimpleHttpClient::headerValue(const char *name) const
|
|
{
|
|
if (m_response)
|
|
return (*m_response)[name];
|
|
return m_headResponse[name];
|
|
}
|
|
|
|
std::string_view SimpleHttpClient::headerValue(boost::beast::http::field field) const
|
|
{
|
|
if (m_response)
|
|
return (*m_response)[field];
|
|
return m_headResponse[field];
|
|
}
|
|
|
|
QDateTime SimpleHttpClient::headerDate(boost::beast::http::field field) const
|
|
{
|
|
auto strView = headerValue(field);
|
|
// http date/time format example: "Fri, 14 Mar 2025 21:58:39 GMT"
|
|
return QDateTime::fromString(
|
|
QString::fromLatin1(strView.begin(), strView.size()),
|
|
"ddd, dd MMM yyyy hh:mm:ss t");
|
|
}
|
|
|
|
boost::beast::http::request<boost::beast::http::vector_body<char>> SimpleHttpClient::request() const
|
|
{
|
|
return m_request;
|
|
}
|
|
|
|
Streaming::ConstBuffer SimpleHttpClient::responseBody() const
|
|
{
|
|
return m_responseBody;
|
|
}
|
|
|
|
SimpleHttpClient::SimpleHttpClient(boost::asio::io_context &context, boost::asio::ssl::context &sslContext)
|
|
: m_resolver(context),
|
|
m_stream(context, sslContext)
|
|
{
|
|
}
|
|
|
|
void SimpleHttpClient::resolve()
|
|
{
|
|
// we always use https, which implies we always need to have a hostname
|
|
// as opposed to a raw IP.
|
|
// so we can assume that the user put a DNS-valid name in the request.
|
|
auto hostIter = m_request.find(boost::beast::http::field::host);
|
|
assert(hostIter != m_request.end());
|
|
std::string hostname = hostIter->value();
|
|
|
|
// Set SNI (Server Name Indication)
|
|
if (!SSL_set_tlsext_host_name(m_stream.native_handle(), hostname.c_str())) {
|
|
boost::beast::error_code ec{static_cast<int>(::ERR_get_error()), boost::asio::error::get_ssl_category()};
|
|
logCritical() << "SNI Error: " << ec.message();
|
|
setError(DnsIssues);
|
|
emit finished();
|
|
return;
|
|
}
|
|
|
|
// resolve and then move to the connect method
|
|
m_resolver.async_resolve(hostname, "443",
|
|
std::bind(&SimpleHttpClient::connect, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
|
|
}
|
|
|
|
void SimpleHttpClient::connect(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type results)
|
|
{
|
|
if (ec) {
|
|
logWarning() << "SimpleHttpClient hit error" << ec.message();
|
|
setError(DnsIssues);
|
|
emit finished();
|
|
return;
|
|
}
|
|
boost::beast::get_lowest_layer(m_stream).async_connect(results,
|
|
std::bind(&SimpleHttpClient::sslHandshake, shared_from_this(), std::placeholders::_1));
|
|
}
|
|
|
|
void SimpleHttpClient::sslHandshake(boost::beast::error_code ec)
|
|
{
|
|
if (ec) {
|
|
logWarning() << "SimpleHttpClient hit error" << ec.message();
|
|
setError(SslIssues);
|
|
emit finished();
|
|
return;
|
|
}
|
|
m_stream.async_handshake(boost::asio::ssl::stream_base::client,
|
|
std::bind(&SimpleHttpClient::sendRequest, shared_from_this(), std::placeholders::_1));
|
|
}
|
|
|
|
void SimpleHttpClient::sendRequest(boost::beast::error_code ec)
|
|
{
|
|
if (ec) {
|
|
logWarning() << "SimpleHttpClient hit error" << ec.message();
|
|
setError(SslIssues);
|
|
emit finished();
|
|
return;
|
|
}
|
|
boost::beast::http::async_write(m_stream, m_request,
|
|
std::bind(&SimpleHttpClient::requestSent, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
|
|
}
|
|
|
|
void SimpleHttpClient::requestSent(boost::beast::error_code ec, std::size_t bytesTransferred)
|
|
{
|
|
logInfo() << "Sent" << bytesTransferred << "bytes in request, waiting for reply";
|
|
if (ec) {
|
|
logWarning() << "SimpleHttpClient hit error" << ec.message();
|
|
setError(SendIssues);
|
|
emit finished();
|
|
return;
|
|
}
|
|
// Receive the HTTP response asynchronously
|
|
if (m_request.method() == boost::beast::http::verb::head) {
|
|
// this feels very dirty, but that's boost for you.
|
|
// there being no body is failing the normal flow, so needs special attention.
|
|
m_parser.reset(new boost::beast::http::response_parser<boost::beast::http::empty_body>());
|
|
m_parser->skip(true);
|
|
boost::beast::http::async_read(m_stream, m_readBuffer, *m_parser,
|
|
std::bind(&SimpleHttpClient::readHeaderReply, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
|
|
}
|
|
else if (m_incremental) {
|
|
m_partialParser.reset(new boost::beast::http::response_parser<boost::beast::http::dynamic_body>());
|
|
boost::beast::http::async_read_some(m_stream, m_readBuffer, *m_partialParser,
|
|
std::bind(&SimpleHttpClient::readPartialReply, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
|
|
} else {
|
|
m_response.reset(new boost::beast::http::response<boost::beast::http::dynamic_body>());
|
|
boost::beast::http::async_read(m_stream, m_readBuffer, *m_response,
|
|
std::bind(&SimpleHttpClient::readReply, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
|
|
}
|
|
}
|
|
|
|
void SimpleHttpClient::readHeaderReply(boost::beast::error_code ec, std::size_t bytesTransferred)
|
|
{
|
|
assert(m_parser.get());
|
|
logInfo() << "Reply HEAD received." << bytesTransferred << "bytes";
|
|
if (ec) {
|
|
logWarning() << "SimpleHttpClient hit error" << ec.message();
|
|
setError(ReceiveIssues);
|
|
}
|
|
else {
|
|
m_headResponse = m_parser->release();
|
|
if (httpErrorCode() != 200)
|
|
setError(PeerIssues);
|
|
}
|
|
|
|
m_stream.async_shutdown(
|
|
std::bind(&SimpleHttpClient::connectionShutdown, shared_from_this(), std::placeholders::_1));
|
|
}
|
|
|
|
void SimpleHttpClient::readReply(boost::beast::error_code ec, std::size_t bytesTransferred)
|
|
{
|
|
logInfo() << "Reply received." << bytesTransferred << "bytes";
|
|
assert(m_response.get());
|
|
if (ec) {
|
|
logWarning() << "SimpleHttpClient hit error" << ec.message();
|
|
setError(ReceiveIssues);
|
|
}
|
|
else {
|
|
assert(m_request.method() != boost::beast::http::verb::head);
|
|
if (httpErrorCode() != 200)
|
|
setError(PeerIssues);
|
|
const auto buf = m_response->body();
|
|
auto pool = Streaming::pool(buf.size());
|
|
std::string data = boost::beast::buffers_to_string(buf.data()); // misnomer, its just a stupid bytearray
|
|
pool->write(data.c_str(), data.size());
|
|
m_responseBody = pool->commit();
|
|
}
|
|
|
|
// Shutdown the SSL stream asynchronously
|
|
m_stream.async_shutdown(
|
|
std::bind(&SimpleHttpClient::connectionShutdown, shared_from_this(), std::placeholders::_1));
|
|
}
|
|
|
|
void SimpleHttpClient::readPartialReply(boost::beast::error_code ec, std::size_t bytesTransferred)
|
|
{
|
|
logInfo() << "Reply received." << bytesTransferred << "bytes";
|
|
assert(m_partialParser.get());
|
|
if (ec && ec != boost::beast::http::error::end_of_stream) {
|
|
logWarning() << "SimpleHttpClient hit error" << ec.message();
|
|
setError(ReceiveIssues);
|
|
return;
|
|
}
|
|
if (!m_partialParser->is_header_done()) { // need more!
|
|
boost::beast::http::async_read_some(m_stream, m_readBuffer, *m_partialParser,
|
|
std::bind(&SimpleHttpClient::readHeaderReply, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
|
|
return;
|
|
}
|
|
|
|
auto &body = m_partialParser->get().body();
|
|
auto data = body.data();
|
|
std::size_t available = boost::beast::buffer_bytes(data);
|
|
if (available > 0) {
|
|
auto pool = Streaming::pool(available);
|
|
|
|
auto *ptr = pool->data();
|
|
for (const auto &buf : data) {
|
|
ptr = std::copy(boost::asio::buffers_begin(buf), boost::asio::buffers_end(buf), ptr);
|
|
}
|
|
body.consume(available);
|
|
emit dataAvailable(pool->commit(available));
|
|
}
|
|
if (m_partialParser->is_done()) {
|
|
m_stream.async_shutdown(
|
|
std::bind(&SimpleHttpClient::connectionShutdown, shared_from_this(), std::placeholders::_1));
|
|
} else {
|
|
boost::beast::http::async_read_some(m_stream, m_readBuffer, *m_partialParser,
|
|
std::bind(&SimpleHttpClient::readPartialReply, shared_from_this(), std::placeholders::_1, std::placeholders::_2));
|
|
}
|
|
}
|
|
|
|
void SimpleHttpClient::connectionShutdown(boost::beast::error_code ec)
|
|
{
|
|
if (ec && ec != boost::asio::error::eof && ec != boost::asio::ssl::error::stream_truncated) {
|
|
logWarning() << "Shutdown error:" << ec.message();
|
|
}
|
|
emit finished();
|
|
}
|
|
|
|
SimpleHttpClient::ErrorType SimpleHttpClient::error() const
|
|
{
|
|
return m_error;
|
|
}
|
|
|
|
void SimpleHttpClient::setError(ErrorType newError)
|
|
{
|
|
if (m_error == newError)
|
|
return;
|
|
m_error = newError;
|
|
if (m_error != NoError)
|
|
emit errored(m_error);
|
|
}
|