From 33a6648de9ee0af33c44518656b56b0a30b6c1ab Mon Sep 17 00:00:00 2001
From: suvendhu <suvendhu.hansa@iopsys.eu>
Date: Wed, 7 Sep 2022 17:48:35 +0530
Subject: [PATCH] compress content when method set to GZIP
---
Makefile | 2 +-
bbf_plugin/datamodel.c | 18 ++++----
src/utils.c | 97 ++++++++++++++++++++++++++++++++++++++++--
3 files changed, 103 insertions(+), 14 deletions(-)
diff --git a/Makefile b/Makefile
index c7be29f..678352b 100644
--- a/Makefile
+++ b/Makefile
@@ -6,7 +6,7 @@ SRC_DIR = src
OBJS = $(addprefix $(SRC_DIR)/, bulkdata.o config.o report.o buci.o utils.o )
PROG_CFLAGS = $(CFLAGS) -Wall -Wextra -Werror -fstrict-aliasing -fPIC
-PROG_LDFLAGS = $(LDFLAGS) -lubus -luci -lubox -ljson-c -lcurl -lblobmsg_json
+PROG_LDFLAGS = $(LDFLAGS) -lubus -luci -lubox -ljson-c -lcurl -lblobmsg_json -lz
.PHONY: all clean
diff --git a/bbf_plugin/datamodel.c b/bbf_plugin/datamodel.c
index ddcebd9..cfb95a4 100644
--- a/bbf_plugin/datamodel.c
+++ b/bbf_plugin/datamodel.c
@@ -24,6 +24,7 @@ DM_MAP_OBJ tDynamicObj[] = {
{0}
};
+char *compression_supported[] = {"None", "GZIP", NULL};
/*************************************************************
* ENTRY METHOD
*************************************************************/
@@ -867,7 +868,7 @@ static int set_BulkDataProfileHTTP_Password(char *refparam, struct dmctx *ctx, v
static int get_BulkDataProfileHTTP_CompressionsSupported(char *refparam, struct dmctx *ctx, void *data, char *instance, char **value)
{
- *value = "GZIP,Compress,Deflate";
+ *value = "GZIP";
return 0;
}
@@ -877,10 +878,9 @@ static int get_BulkDataProfileHTTP_Compression(char *refparam, struct dmctx *ctx
dmuci_get_value_by_section_string(((struct dmmap_dup *)data)->config_section, "http_compression", value);
if(strcmp(*value, "gzip") == 0)
*value = "GZIP";
- else if(strcmp(*value, "compress") == 0)
- *value = "Compress";
- else if(strcmp(*value, "deflate") == 0)
- *value = "Deflate";
+ else
+ *value = "None";
+
return 0;
}
@@ -888,16 +888,14 @@ static int set_BulkDataProfileHTTP_Compression(char *refparam, struct dmctx *ctx
{
switch (action) {
case VALUECHECK:
- if (dm_validate_string(value, -1, -1, NULL, NULL))
+ if (dm_validate_string(value, -1, -1, compression_supported, NULL))
return FAULT_9007;
break;
case VALUESET:
if(strcmp(value, "GZIP") == 0)
dmuci_set_value_by_section(((struct dmmap_dup *)data)->config_section, "http_compression", "gzip");
- else if(strcmp(value, "Compress") == 0)
- dmuci_set_value_by_section(((struct dmmap_dup *)data)->config_section, "http_compression", "compress");
- else if(strcmp(value, "Deflate") == 0)
- dmuci_set_value_by_section(((struct dmmap_dup *)data)->config_section, "http_compression", "deflate");
+ else
+ dmuci_set_value_by_section(((struct dmmap_dup *)data)->config_section, "http_compression", "none");
break;
}
return 0;
diff --git a/src/utils.c b/src/utils.c
index 5859cb2..2e6466a 100644
--- a/src/utils.c
+++ b/src/utils.c
@@ -10,6 +10,7 @@
*/
#include <curl/curl.h>
+#include <zlib.h>
#include <libubus.h>
#include <libubox/blob.h>
#include <libubox/blobmsg.h>
@@ -21,6 +22,12 @@
static unsigned char LogLevel = DEFAULT_LOG_LEVEL;
static bool g_usp_object_available = false;
+enum compress_type {
+ COMPRESS_TYPE_GZIP,
+ COMPRESS_TYPE_COMPRESS,
+ COMPRESS_TYPE_DEFLATE
+};
+
void set_log_level(unsigned char level)
{
LogLevel = level;
@@ -365,6 +372,77 @@ static void create_request_url(struct profile *profile, char *url, size_t len)
}
}
+static unsigned char *compress_report(char *input_buf, long input_len, long *content_len, int type)
+{
+ z_stream zlib_ctx;
+ int err;
+ int output_len;
+ unsigned char *output_buf;
+
+ if (content_len == NULL) {
+ return NULL;
+ }
+
+ // Exit if not GZIP compression - in this case we just don't compress the report
+ if (type != COMPRESS_TYPE_GZIP || input_buf == NULL) {
+ *content_len = input_len;
+ return (unsigned char *)input_buf;
+ }
+
+ // Initialise the zlib context
+ memset(&zlib_ctx, 0, sizeof(zlib_ctx));
+ zlib_ctx.zalloc = Z_NULL;
+ zlib_ctx.zfree = Z_NULL;
+ zlib_ctx.opaque = NULL;
+
+ // Exit if unable to start deflate
+ #define WINDOW_BITS (15+16) // Plus 16 to get a gzip wrapper, as suggested by the zlib documentation
+ #define MEM_LEVEL 8 // This is the default value, as suggested by the zlib documentation
+ err = deflateInit2(&zlib_ctx, Z_DEFAULT_COMPRESSION, Z_DEFLATED, WINDOW_BITS, MEM_LEVEL, Z_DEFAULT_STRATEGY);
+ if (err != Z_OK) {
+ WARNING("deflateInit2 returned %d. Falling back to sending uncompressed data", err);
+ *content_len = input_len;
+ return (unsigned char *)input_buf;
+ }
+
+ // Allocate a worst case buffer to hold the compressed data
+ output_len = (int)deflateBound(&zlib_ctx, input_len);
+ output_buf = malloc(output_len); // Use malloc because the uncompressed report was generated with malloc() and this needs to be consistent
+ if (output_buf == NULL) {
+ WARNING("malloc failed. Falling back to sending uncompressed data", err);
+ deflateEnd(&zlib_ctx);
+ *content_len = input_len;
+ return (unsigned char *)input_buf;
+ }
+
+ // Initialise the zlib context for this compression
+ zlib_ctx.next_in = (unsigned char *)input_buf;
+ zlib_ctx.avail_in = input_len;
+ zlib_ctx.next_out = output_buf;
+ zlib_ctx.avail_out= output_len;
+
+ // Exit if compression failed
+ err = deflate(&zlib_ctx, Z_FINISH);
+ if (err != Z_STREAM_END) {
+ WARNING("deflate failed (err=%d). Falling back to sending uncompressed data", err);
+ deflateEnd(&zlib_ctx);
+ free(output_buf);
+ *content_len = input_len;
+ return (unsigned char *)input_buf;
+ }
+
+ // Deallocate all compression state stored in the zlib context
+ // NOTE: We ignore errors from this and just log them
+ err = deflateEnd(&zlib_ctx);
+ if (err != Z_OK) {
+ WARNING("deflateEnd failed (err=%d, %s). Ignoring error", err, zlib_ctx.msg);
+ }
+
+ INFO("BulkDataReport(uncompressed size=%d, compressed size=%lu)", input_len, zlib_ctx.total_out);
+ *content_len = zlib_ctx.total_out;
+ return output_buf;
+}
+
int http_send_message(struct profile *profile, char *msg_out)
{
#define HTTP_TIMEOUT 20
@@ -427,23 +505,33 @@ int http_send_message(struct profile *profile, char *msg_out)
curl_easy_setopt(curl, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
curl_easy_setopt(curl, CURLOPT_NOBODY, 0);
+ unsigned char *content = NULL;
+ long content_len = 0;
+ long msg_out_len = msg_out ? strlen(msg_out) : 0;
if (strcasecmp(profile->http_compression, "gzip") == 0) {
curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "gzip");
header_list = curl_slist_append(header_list, "Content-Encoding: gzip");
+ content = compress_report(msg_out, msg_out_len, &content_len, COMPRESS_TYPE_GZIP);
} else if (strcasecmp(profile->http_compression, "compress") == 0) {
curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "compress");
header_list = curl_slist_append(header_list, "Content-Encoding: compress");
+ content = compress_report(msg_out, msg_out_len, &content_len, COMPRESS_TYPE_COMPRESS);
} else if (strcasecmp(profile->http_compression, "deflate") == 0) {
curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "deflate");
- header_list = curl_slist_append(header_list, "Content-Encoding: deflate");
+ header_list = curl_slist_append(header_list, "Content-Encoding: deflate");
+ content = compress_report(msg_out, msg_out_len, &content_len, COMPRESS_TYPE_DEFLATE);
+ } else {
+ /* no compression */
+ content = (unsigned char *)msg_out;
+ content_len = msg_out ? msg_out_len : 0;
}
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, header_list);
if (strcasecmp(profile->http_method, "put") == 0)
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
- curl_easy_setopt(curl, CURLOPT_POSTFIELDS, msg_out);
- curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, msg_out ? (long)strlen(msg_out) : 0);
+ curl_easy_setopt(curl, CURLOPT_POSTFIELDS, content);
+ curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, content_len);
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf);
@@ -460,6 +548,9 @@ int http_send_message(struct profile *profile, char *msg_out)
}
}
+ if (content != (unsigned char *)msg_out)
+ FREE(content);
+
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
if (http_code == 200)
INFO("Receive HTTP 200 OK from Bulkdata Collector");
--
GitLab