kapitan/IXWebSocket
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

(ws) add gzip and gunzip ws sub commands

Browse Source
This commit is contained in:
Benjamin Sergeant 2020-09-28 10:19:27 -07:00
parent 6077f86af8
commit 6f188a5131
11 changed files with 275 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -31,6 +31,7 @@ set( IXWEBSOCKET_SOURCES
ixwebsocket/IXDNSLookup.cpp
ixwebsocket/IXExponentialBackoff.cpp
ixwebsocket/IXGetFreePort.cpp
ixwebsocket/IXGzipCodec.cpp
ixwebsocket/IXHttp.cpp
ixwebsocket/IXHttpClient.cpp
ixwebsocket/IXHttpServer.cpp
@ -66,6 +67,7 @@ set( IXWEBSOCKET_HEADERS
ixwebsocket/IXDNSLookup.h
ixwebsocket/IXExponentialBackoff.h
ixwebsocket/IXGetFreePort.h
ixwebsocket/IXGzipCodec.h
ixwebsocket/IXHttp.h
ixwebsocket/IXHttpClient.h
ixwebsocket/IXHttpServer.h

4
docs/CHANGELOG.md
View File

@ -2,6 +2,10 @@
All changes to this project will be documented in this file.
## [10.4.7] - 2020-09-28
(ws) add gzip and gunzip ws sub commands
## [10.4.6] - 2020-09-26
(cmake) use FetchContent cmake module to retrieve jsoncpp third party dependency

12
ixcrypto/ixcrypto/IXHash.cpp
View File

@ -19,4 +19,16 @@ namespace ix
return hashAddress;
}
uint64_t djb2HashStr(const std::string& data)
{
uint64_t hashAddress = 5381;
for (size_t i = 0; i < data.size(); ++i)
{
hashAddress = ((hashAddress << 5) + hashAddress) + data[i];
}
return hashAddress;
}
} // namespace ix

2
ixcrypto/ixcrypto/IXHash.h
View File

@ -8,8 +8,10 @@
#include <cstdint>
#include <vector>
#include <string>
namespace ix
{
uint64_t djb2Hash(const std::vector<uint8_t>& data);
uint64_t djb2HashStr(const std::string& data);
}

104
ixwebsocket/IXGzipCodec.cpp Normal file
View File

@ -0,0 +1,104 @@
/*
* IXGzipCodec.cpp
* Author: Benjamin Sergeant
* Copyright (c) 2020 Machine Zone, Inc. All rights reserved.
*/
#include "IXGzipCodec.h"
#include <array>
#ifdef IXWEBSOCKET_USE_ZLIB
#include <zlib.h>
#endif
namespace ix
{
#ifdef IXWEBSOCKET_USE_ZLIB
std::string gzipCompress(const std::string& str)
{
z_stream zs; // z_stream is zlib's control structure
memset(&zs, 0, sizeof(zs));
// deflateInit2 configure the file format: request gzip instead of deflate
const int windowBits = 15;
const int GZIP_ENCODING = 16;
deflateInit2(&zs,
Z_DEFAULT_COMPRESSION,
Z_DEFLATED,
windowBits | GZIP_ENCODING,
8,
Z_DEFAULT_STRATEGY);
zs.next_in = (Bytef*) str.data();
zs.avail_in = (uInt) str.size(); // set the z_stream's input
int ret;
char outbuffer[32768];
std::string outstring;
// retrieve the compressed bytes blockwise
do
{
zs.next_out = reinterpret_cast<Bytef*>(outbuffer);
zs.avail_out = sizeof(outbuffer);
ret = deflate(&zs, Z_FINISH);
if (outstring.size() < zs.total_out)
{
// append the block to the output string
outstring.append(outbuffer, zs.total_out - outstring.size());
}
} while (ret == Z_OK);
deflateEnd(&zs);
return outstring;
}
bool gzipDecompress(const std::string& in, std::string& out)
{
z_stream inflateState;
std::memset(&inflateState, 0, sizeof(inflateState));
inflateState.zalloc = Z_NULL;
inflateState.zfree = Z_NULL;
inflateState.opaque = Z_NULL;
inflateState.avail_in = 0;
inflateState.next_in = Z_NULL;
if (inflateInit2(&inflateState, 16 + MAX_WBITS) != Z_OK)
{
return false;
}
inflateState.avail_in = (uInt) in.size();
inflateState.next_in = (unsigned char*) (const_cast<char*>(in.data()));
const int kBufferSize = 1 << 14;
std::array<unsigned char, kBufferSize> compressBuffer;
do
{
inflateState.avail_out = (uInt) kBufferSize;
inflateState.next_out = &compressBuffer.front();
int ret = inflate(&inflateState, Z_SYNC_FLUSH);
if (ret == Z_NEED_DICT || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR)
{
inflateEnd(&inflateState);
return false;
}
out.append(reinterpret_cast<char*>(&compressBuffer.front()),
kBufferSize - inflateState.avail_out);
} while (inflateState.avail_out == 0);
inflateEnd(&inflateState);
return true;
}
#endif
} // namespace ix

15
ixwebsocket/IXGzipCodec.h Normal file
View File

@ -0,0 +1,15 @@
/*
* IXGzipCodec.h
* Author: Benjamin Sergeant
* Copyright (c) 2020 Machine Zone, Inc. All rights reserved.
*/
#pragma once
#include <string>
namespace ix
{
std::string gzipCompress(const std::string& str);
bool gzipDecompress(const std::string& in, std::string& out);
} // namespace ix

67
ixwebsocket/IXHttpClient.cpp
View File

@ -6,11 +6,11 @@
#include "IXHttpClient.h"
#include "IXGzipCodec.h"
#include "IXSocketFactory.h"
#include "IXUrlParser.h"
#include "IXUserAgent.h"
#include "IXWebSocketHttpHeaders.h"
#include <array>
#include <assert.h>
#include <cstring>
#include <iomanip>
@ -18,10 +18,6 @@
#include <sstream>
#include <vector>
#ifdef IXWEBSOCKET_USE_ZLIB
#include <zlib.h>
#endif
namespace ix
{
const std::string HttpClient::kPost = "POST";
@ -501,12 +497,12 @@ namespace ix
downloadSize = payload.size();
#ifdef IXWEBSOCKET_USE_ZLIB
// If the content was compressed with gzip, decode it
if (headers["Content-Encoding"] == "gzip")
{
#ifdef IXWEBSOCKET_USE_ZLIB
std::string decompressedPayload;
if (!gzipInflate(payload, decompressedPayload))
if (!gzipDecompress(payload, decompressedPayload))
{
std::string errorMsg("Error decompressing payload");
return std::make_shared<HttpResponse>(code,
@ -519,8 +515,18 @@ namespace ix
downloadSize);
}
payload = decompressedPayload;
}
#else
std::string errorMsg("ixwebsocket was not compiled with gzip support on");
return std::make_shared<HttpResponse>(code,
description,
HttpErrorCode::Gzip,
headers,
payload,
errorMsg,
uploadSize,
downloadSize);
#endif
}
return std::make_shared<HttpResponse>(code,
description,
@ -680,51 +686,6 @@ namespace ix
return ss.str();
}
#ifdef IXWEBSOCKET_USE_ZLIB
bool HttpClient::gzipInflate(const std::string& in, std::string& out)
{
z_stream inflateState;
std::memset(&inflateState, 0, sizeof(inflateState));
inflateState.zalloc = Z_NULL;
inflateState.zfree = Z_NULL;
inflateState.opaque = Z_NULL;
inflateState.avail_in = 0;
inflateState.next_in = Z_NULL;
if (inflateInit2(&inflateState, 16 + MAX_WBITS) != Z_OK)
{
return false;
}
inflateState.avail_in = (uInt) in.size();
inflateState.next_in = (unsigned char*) (const_cast<char*>(in.data()));
const int kBufferSize = 1 << 14;
std::array<unsigned char, kBufferSize> compressBuffer;
do
{
inflateState.avail_out = (uInt) kBufferSize;
inflateState.next_out = &compressBuffer.front();
int ret = inflate(&inflateState, Z_SYNC_FLUSH);
if (ret == Z_NEED_DICT || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR)
{
inflateEnd(&inflateState);
return false;
}
out.append(reinterpret_cast<char*>(&compressBuffer.front()),
kBufferSize - inflateState.avail_out);
} while (inflateState.avail_out == 0);
inflateEnd(&inflateState);
return true;
}
#endif
void HttpClient::log(const std::string& msg, HttpRequestArgsPtr args)
{
if (args->logger)

4
ixwebsocket/IXHttpClient.h
View File

@ -90,10 +90,6 @@ namespace ix
private:
void log(const std::string& msg, HttpRequestArgsPtr args);
#ifdef IXWEBSOCKET_USE_ZLIB
bool gzipInflate(const std::string& in, std::string& out);
#endif
// Async API background thread runner
void run();
// Async API

50
ixwebsocket/IXHttpServer.cpp
View File

@ -6,6 +6,7 @@
#include "IXHttpServer.h"
#include "IXGzipCodec.h"
#include "IXNetSystem.h"
#include "IXSocketConnect.h"
#include "IXUserAgent.h"
@ -14,10 +15,6 @@
#include <sstream>
#include <vector>
#ifdef IXWEBSOCKET_USE_ZLIB
#include <zlib.h>
#endif
namespace
{
std::pair<bool, std::vector<uint8_t>> load(const std::string& path)
@ -43,51 +40,6 @@ namespace
auto vec = res.second;
return std::make_pair(res.first, std::string(vec.begin(), vec.end()));
}
#ifdef IXWEBSOCKET_USE_ZLIB
std::string gzipCompress(const std::string& str)
{
z_stream zs; // z_stream is zlib's control structure
memset(&zs, 0, sizeof(zs));
// deflateInit2 configure the file format: request gzip instead of deflate
const int windowBits = 15;
const int GZIP_ENCODING = 16;
deflateInit2(&zs,
Z_DEFAULT_COMPRESSION,
Z_DEFLATED,
windowBits | GZIP_ENCODING,
8,
Z_DEFAULT_STRATEGY);
zs.next_in = (Bytef*) str.data();
zs.avail_in = (uInt) str.size(); // set the z_stream's input
int ret;
char outbuffer[32768];
std::string outstring;
// retrieve the compressed bytes blockwise
do
{
zs.next_out = reinterpret_cast<Bytef*>(outbuffer);
zs.avail_out = sizeof(outbuffer);
ret = deflate(&zs, Z_FINISH);
if (outstring.size() < zs.total_out)
{
// append the block to the output string
outstring.append(outbuffer, zs.total_out - outstring.size());
}
} while (ret == Z_OK);
deflateEnd(&zs);
return outstring;
}
#endif
} // namespace
namespace ix

2
ixwebsocket/IXWebSocketVersion.h
View File

@ -6,4 +6,4 @@
#pragma once
#define IX_WEBSOCKET_VERSION "10.4.6"
#define IX_WEBSOCKET_VERSION "10.4.7"

136
ws/ws.cpp
View File

@ -33,6 +33,7 @@
#include <ixsentry/IXSentryClient.h>
#include <ixsnake/IXSnakeServer.h>
#include <ixwebsocket/IXDNSLookup.h>
#include <ixwebsocket/IXGzipCodec.h>
#include <ixwebsocket/IXHttpClient.h>
#include <ixwebsocket/IXHttpServer.h>
#include <ixwebsocket/IXNetSystem.h>
@ -127,6 +128,38 @@ namespace
std::ifstream infile(fileName);
return infile.good();
}
std::string extractFilename(const std::string& path)
{
std::string::size_type idx;
idx = path.rfind('/');
if (idx != std::string::npos)
{
std::string filename = path.substr(idx + 1);
return filename;
}
else
{
return path;
}
}
std::string removeExtension(const std::string& path)
{
std::string::size_type idx;
idx = path.rfind('.');
if (idx != std::string::npos)
{
std::string filename = path.substr(idx + 1);
return filename;
}
else
{
return path;
}
}
} // namespace
namespace ix
@ -1103,6 +1136,76 @@ namespace ix
return 0;
}
int ws_gzip(const std::string& filename)
{
auto res = readAsString(filename);
bool found = res.first;
if (!found)
{
spdlog::error("Cannot read content of {}", filename);
return 1;
}
spdlog::info("gzip input: {} cksum {}", filename, ix::djb2HashStr(res.second));
std::string compressedBytes;
{
Bench bench("compressing file");
compressedBytes = gzipCompress(res.second);
}
std::string outputFilename(filename);
outputFilename += ".gz";
std::ofstream f;
f.open(outputFilename);
f << compressedBytes;
f.close();
spdlog::info("gzip output: {} cksum {}", outputFilename, ix::djb2HashStr(compressedBytes));
return 0;
}
int ws_gunzip(const std::string& filename)
{
spdlog::info("filename to gunzip: {}", filename);
auto res = readAsString(filename);
bool found = res.first;
if (!found)
{
spdlog::error("Cannot read content of {}", filename);
return 1;
}
spdlog::info("gunzip input: {} cksum {}", filename, ix::djb2HashStr(res.second));
std::string decompressedBytes;
{
Bench bench("decompressing file");
if (!gzipDecompress(res.second, decompressedBytes))
{
spdlog::error("Cannot decompress content of {}", filename);
return 1;
}
}
std::string outputFilename(removeExtension(filename));
std::ofstream f;
f.open(outputFilename);
f << decompressedBytes;
f.close();
spdlog::info(
"gunzip output: {} cksum {}", outputFilename, ix::djb2HashStr(decompressedBytes));
return 0;
}
int ws_autoroute(const std::string& url,
bool disablePerMessageDeflate,
const ix::SocketTLSOptions& tlsOptions,
@ -1305,22 +1408,6 @@ namespace ix
return 0;
}
std::string extractFilename(const std::string& path)
{
std::string::size_type idx;
idx = path.rfind('/');
if (idx != std::string::npos)
{
std::string filename = path.substr(idx + 1);
return filename;
}
else
{
return path;
}
}
WebSocketHttpHeaders parseHeaders(const std::string& data)
{
WebSocketHttpHeaders headers;
@ -2848,6 +2935,7 @@ int main(int argc, char** argv)
std::string publisherRolename;
std::string publisherRolesecret;
std::string sendMsg("hello world");
std::string filename;
ix::SocketTLSOptions tlsOptions;
ix::CobraConfig cobraConfig;
ix::CobraBotConfig cobraBotConfig;
@ -3206,6 +3294,14 @@ int main(int argc, char** argv)
dnsLookupApp->fallthrough();
dnsLookupApp->add_option("host", hostname, "Hostname")->required();
CLI::App* gzipApp = app.add_subcommand("gzip", "Gzip compressor");
gzipApp->fallthrough();
gzipApp->add_option("filename", filename, "Filename")->required();
CLI::App* gunzipApp = app.add_subcommand("gunzip", "Gzip decompressor");
gunzipApp->fallthrough();
gunzipApp->add_option("filename", filename, "Filename")->required();
CLI11_PARSE(app, argc, argv);
// pid file handling
@ -3502,6 +3598,14 @@ int main(int argc, char** argv)
{
ret = ix::ws_dns_lookup(hostname);
}
else if (app.got_subcommand("gzip"))
{
ret = ix::ws_gzip(filename);
}
else if (app.got_subcommand("gunzip"))
{
ret = ix::ws_gunzip(filename);
}
else if (version)
{
std::cout << "ws " << ix::userAgent() << std::endl;