Add support for streaming transfers (#353)

This change adds onChunkCallback to the request. If defined it will be
called repeatedly with the incoming data. This allows to process data on
the go or write it to disk instead of accumulating the data in memory.
This commit is contained in:
Martin Natano
2022-02-01 06:54:32 +01:00
committed by GitHub
parent c28b569535
commit db7057de69
8 changed files with 120 additions and 17 deletions

View File

@ -400,12 +400,14 @@ namespace ix
std::pair<bool, std::string> Socket::readBytes(
size_t length,
const OnProgressCallback& onProgressCallback,
const OnChunkCallback& onChunkCallback,
const CancellationRequest& isCancellationRequested)
{
std::array<uint8_t, 1 << 14> readBuffer;
std::vector<uint8_t> output;
while (output.size() != length)
size_t bytesRead = 0;
while (bytesRead != length)
{
if (isCancellationRequested && isCancellationRequested())
{
@ -413,12 +415,21 @@ namespace ix
return std::make_pair(false, errorMsg);
}
size_t size = std::min(readBuffer.size(), length - output.size());
size_t size = std::min(readBuffer.size(), length - bytesRead);
ssize_t ret = recv((char*) &readBuffer[0], size);
if (ret > 0)
{
output.insert(output.end(), readBuffer.begin(), readBuffer.begin() + ret);
if (onChunkCallback)
{
std::string chunk(readBuffer.begin(), readBuffer.begin() + ret);
onChunkCallback(chunk);
}
else
{
output.insert(output.end(), readBuffer.begin(), readBuffer.begin() + ret);
}
bytesRead += ret;
}
else if (ret <= 0 && !Socket::isWaitNeeded())
{
@ -426,7 +437,7 @@ namespace ix
return std::make_pair(false, errorMsg);
}
if (onProgressCallback) onProgressCallback((int) output.size(), (int) length);
if (onProgressCallback) onProgressCallback((int) bytesRead, (int) length);
// Wait with a 1ms timeout until the socket is ready to read.
// This way we are not busy looping