Mirrors
/
rspamd
mirror of https://github.com/rspamd/rspamd.git
3
0
Fork 0
Code Releases Activity
Browse Source

Merge pull request #5191 from rspamd/vstakhov-zstd-headers 

[Rework] Allow `Content-Encoding` standard header for zstd compression
pull/5193/head
Vsevolod Stakhov 1 year ago
committed by GitHub
parent
28302c84ef 973c12585f
commit
c9b804517d
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
6 changed files with 253 additions and 243 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      src/client/rspamdclient.c
  2. 481
      src/libserver/protocol.c
  3. 2
      src/libserver/protocol_internal.h
  4. 6
      src/libserver/task.c
  5. 3
      src/libserver/worker_util.c
  6. 3
      src/lua/lua_http.c

1
src/client/rspamdclient.c
View File

@ -441,6 +441,7 @@ rspamd_client_command(struct rspamd_client_connection *conn,
if (compressed) {
rspamd_http_message_add_header(req->msg, COMPRESSION_HEADER, "zstd");
rspamd_http_message_add_header(req->msg, CONTENT_ENCODING_HEADER, "zstd");
if (dict_id != 0) {
char dict_str[32];

481
src/libserver/protocol.c
View File

@ -490,271 +490,271 @@ rspamd_protocol_handle_headers(struct rspamd_task *task,
hv_tok->len = h->value.len;
switch (*hn_tok->begin) {
case 'd':
case 'D':
IF_HEADER(DELIVER_TO_HEADER)
{
task->deliver_to = rspamd_protocol_escape_braces(task, hv_tok);
msg_debug_protocol("read deliver-to header, value: %s",
task->deliver_to);
}
else
{
msg_debug_protocol("wrong header: %T", hn_tok);
}
break;
case 'h':
case 'H':
IF_HEADER(HELO_HEADER)
{
task->helo = rspamd_mempool_ftokdup(task->task_pool, hv_tok);
msg_debug_protocol("read helo header, value: %s", task->helo);
}
IF_HEADER(HOSTNAME_HEADER)
{
task->hostname = rspamd_mempool_ftokdup(task->task_pool,
hv_tok);
msg_debug_protocol("read hostname header, value: %s", task->hostname);
}
break;
case 'f':
case 'F':
IF_HEADER(FROM_HEADER)
{
if (hv_tok->len == 0) {
/* Replace '' with '<>' to fix parsing issue */
RSPAMD_FTOK_ASSIGN(hv_tok, "<>");
case 'd':
case 'D':
IF_HEADER(DELIVER_TO_HEADER)
{
task->deliver_to = rspamd_protocol_escape_braces(task, hv_tok);
msg_debug_protocol("read deliver-to header, value: %s",
task->deliver_to);
}
task->from_envelope = rspamd_email_address_from_smtp(
hv_tok->begin,
hv_tok->len);
msg_debug_protocol("read from header, value: %T", hv_tok);
if (!task->from_envelope) {
msg_err_protocol("bad from header: '%T'", hv_tok);
task->flags |= RSPAMD_TASK_FLAG_BROKEN_HEADERS;
else
{
msg_debug_protocol("wrong header: %T", hn_tok);
}
}
IF_HEADER(FILENAME_HEADER)
{
task->msg.fpath = rspamd_mempool_ftokdup(task->task_pool,
hv_tok);
msg_debug_protocol("read filename header, value: %s", task->msg.fpath);
}
IF_HEADER(FLAGS_HEADER)
{
msg_debug_protocol("read flags header, value: %T", hv_tok);
rspamd_protocol_process_flags(task, hv_tok);
}
break;
case 'q':
case 'Q':
IF_HEADER(QUEUE_ID_HEADER)
{
task->queue_id = rspamd_mempool_ftokdup(task->task_pool,
hv_tok);
msg_debug_protocol("read queue_id header, value: %s", task->queue_id);
}
else
{
msg_debug_protocol("wrong header: %T", hn_tok);
}
break;
case 'r':
case 'R':
IF_HEADER(RCPT_HEADER)
{
rspamd_protocol_process_recipients(task, hv_tok);
msg_debug_protocol("read rcpt header, value: %T", hv_tok);
}
IF_HEADER(RAW_DATA_HEADER)
{
srch.begin = "yes";
srch.len = 3;
msg_debug_protocol("read raw data header, value: %T", hv_tok);
break;
case 'h':
case 'H':
IF_HEADER(HELO_HEADER)
{
task->helo = rspamd_mempool_ftokdup(task->task_pool, hv_tok);
msg_debug_protocol("read helo header, value: %s", task->helo);
}
IF_HEADER(HOSTNAME_HEADER)
{
task->hostname = rspamd_mempool_ftokdup(task->task_pool,
hv_tok);
msg_debug_protocol("read hostname header, value: %s", task->hostname);
}
break;
case 'f':
case 'F':
IF_HEADER(FROM_HEADER)
{
if (hv_tok->len == 0) {
/* Replace '' with '<>' to fix parsing issue */
RSPAMD_FTOK_ASSIGN(hv_tok, "<>");
}
task->from_envelope = rspamd_email_address_from_smtp(
hv_tok->begin,
hv_tok->len);
msg_debug_protocol("read from header, value: %T", hv_tok);
if (rspamd_ftok_casecmp(hv_tok, &srch) == 0) {
task->flags &= ~RSPAMD_TASK_FLAG_MIME;
msg_debug_protocol("disable mime parsing");
if (!task->from_envelope) {
msg_err_protocol("bad from header: '%T'", hv_tok);
task->flags |= RSPAMD_TASK_FLAG_BROKEN_HEADERS;
}
}
}
break;
case 'i':
case 'I':
IF_HEADER(IP_ADDR_HEADER)
{
if (!rspamd_parse_inet_address(&task->from_addr,
hv_tok->begin, hv_tok->len,
RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
msg_err_protocol("bad ip header: '%T'", hv_tok);
IF_HEADER(FILENAME_HEADER)
{
task->msg.fpath = rspamd_mempool_ftokdup(task->task_pool,
hv_tok);
msg_debug_protocol("read filename header, value: %s", task->msg.fpath);
}
else {
msg_debug_protocol("read IP header, value: %T", hv_tok);
has_ip = TRUE;
IF_HEADER(FLAGS_HEADER)
{
msg_debug_protocol("read flags header, value: %T", hv_tok);
rspamd_protocol_process_flags(task, hv_tok);
}
}
else
{
msg_debug_protocol("wrong header: %T", hn_tok);
}
break;
case 'p':
case 'P':
IF_HEADER(PASS_HEADER)
{
srch.begin = "all";
srch.len = 3;
break;
case 'q':
case 'Q':
IF_HEADER(QUEUE_ID_HEADER)
{
task->queue_id = rspamd_mempool_ftokdup(task->task_pool,
hv_tok);
msg_debug_protocol("read queue_id header, value: %s", task->queue_id);
}
else
{
msg_debug_protocol("wrong header: %T", hn_tok);
}
break;
case 'r':
case 'R':
IF_HEADER(RCPT_HEADER)
{
rspamd_protocol_process_recipients(task, hv_tok);
msg_debug_protocol("read rcpt header, value: %T", hv_tok);
}
IF_HEADER(RAW_DATA_HEADER)
{
srch.begin = "yes";
srch.len = 3;
msg_debug_protocol("read pass header, value: %T", hv_tok);
msg_debug_protocol("read raw data header, value: %T", hv_tok);
if (rspamd_ftok_casecmp(hv_tok, &srch) == 0) {
task->flags |= RSPAMD_TASK_FLAG_PASS_ALL;
msg_debug_protocol("pass all filters");
if (rspamd_ftok_casecmp(hv_tok, &srch) == 0) {
task->flags &= ~RSPAMD_TASK_FLAG_MIME;
msg_debug_protocol("disable mime parsing");
}
}
}
IF_HEADER(PROFILE_HEADER)
{
msg_debug_protocol("read profile header, value: %T", hv_tok);
task->flags |= RSPAMD_TASK_FLAG_PROFILE;
}
break;
case 's':
case 'S':
IF_HEADER(SETTINGS_ID_HEADER)
{
msg_debug_protocol("read settings-id header, value: %T", hv_tok);
task->settings_elt = rspamd_config_find_settings_name_ref(
task->cfg, hv_tok->begin, hv_tok->len);
if (task->settings_elt == NULL) {
GString *known_ids = g_string_new(NULL);
struct rspamd_config_settings_elt *cur;
DL_FOREACH(task->cfg->setting_ids, cur)
{
rspamd_printf_gstring(known_ids, "%s(%ud);",
cur->name, cur->id);
break;
case 'i':
case 'I':
IF_HEADER(IP_ADDR_HEADER)
{
if (!rspamd_parse_inet_address(&task->from_addr,
hv_tok->begin, hv_tok->len,
RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) {
msg_err_protocol("bad ip header: '%T'", hv_tok);
}
else {
msg_debug_protocol("read IP header, value: %T", hv_tok);
has_ip = TRUE;
}
}
else
{
msg_debug_protocol("wrong header: %T", hn_tok);
}
break;
case 'p':
case 'P':
IF_HEADER(PASS_HEADER)
{
srch.begin = "all";
srch.len = 3;
msg_debug_protocol("read pass header, value: %T", hv_tok);
if (rspamd_ftok_casecmp(hv_tok, &srch) == 0) {
task->flags |= RSPAMD_TASK_FLAG_PASS_ALL;
msg_debug_protocol("pass all filters");
}
}
IF_HEADER(PROFILE_HEADER)
{
msg_debug_protocol("read profile header, value: %T", hv_tok);
task->flags |= RSPAMD_TASK_FLAG_PROFILE;
}
break;
case 's':
case 'S':
IF_HEADER(SETTINGS_ID_HEADER)
{
msg_debug_protocol("read settings-id header, value: %T", hv_tok);
task->settings_elt = rspamd_config_find_settings_name_ref(
task->cfg, hv_tok->begin, hv_tok->len);
if (task->settings_elt == NULL) {
GString *known_ids = g_string_new(NULL);
struct rspamd_config_settings_elt *cur;
DL_FOREACH(task->cfg->setting_ids, cur)
{
rspamd_printf_gstring(known_ids, "%s(%ud);",
cur->name, cur->id);
}
msg_warn_protocol("unknown settings id: %T(%d); known_ids: %v",
hv_tok,
rspamd_config_name_to_id(hv_tok->begin, hv_tok->len),
known_ids);
msg_warn_protocol("unknown settings id: %T(%d); known_ids: %v",
hv_tok,
rspamd_config_name_to_id(hv_tok->begin, hv_tok->len),
known_ids);
g_string_free(known_ids, TRUE);
g_string_free(known_ids, TRUE);
}
else {
msg_debug_protocol("applied settings id %T -> %ud", hv_tok,
task->settings_elt->id);
}
}
else {
msg_debug_protocol("applied settings id %T -> %ud", hv_tok,
task->settings_elt->id);
IF_HEADER(SETTINGS_HEADER)
{
msg_debug_protocol("read settings header, value: %T", hv_tok);
seen_settings_header = TRUE;
}
}
IF_HEADER(SETTINGS_HEADER)
{
msg_debug_protocol("read settings header, value: %T", hv_tok);
seen_settings_header = TRUE;
}
break;
case 'u':
case 'U':
IF_HEADER(USER_HEADER)
{
/*
break;
case 'u':
case 'U':
IF_HEADER(USER_HEADER)
{
/*
* We must ignore User header in case of spamc, as SA has
* different meaning of this header
*/
msg_debug_protocol("read user header, value: %T", hv_tok);
if (!RSPAMD_TASK_IS_SPAMC(task)) {
task->auth_user = rspamd_mempool_ftokdup(task->task_pool,
hv_tok);
}
else {
msg_info_protocol("ignore user header: legacy SA protocol");
msg_debug_protocol("read user header, value: %T", hv_tok);
if (!RSPAMD_TASK_IS_SPAMC(task)) {
task->auth_user = rspamd_mempool_ftokdup(task->task_pool,
hv_tok);
}
else {
msg_info_protocol("ignore user header: legacy SA protocol");
}
}
}
IF_HEADER(URLS_HEADER)
{
msg_debug_protocol("read urls header, value: %T", hv_tok);
IF_HEADER(URLS_HEADER)
{
msg_debug_protocol("read urls header, value: %T", hv_tok);
srch.begin = "extended";
srch.len = 8;
srch.begin = "extended";
srch.len = 8;
if (rspamd_ftok_casecmp(hv_tok, &srch) == 0) {
task->protocol_flags |= RSPAMD_TASK_PROTOCOL_FLAG_EXT_URLS;
msg_debug_protocol("extended urls information");
}
/* TODO: add more formats there */
}
IF_HEADER(USER_AGENT_HEADER)
{
msg_debug_protocol("read user-agent header, value: %T", hv_tok);
if (rspamd_ftok_casecmp(hv_tok, &srch) == 0) {
task->protocol_flags |= RSPAMD_TASK_PROTOCOL_FLAG_EXT_URLS;
msg_debug_protocol("extended urls information");
}
if (hv_tok->len == 6 &&
rspamd_lc_cmp(hv_tok->begin, "rspamc", 6) == 0) {
task->protocol_flags |= RSPAMD_TASK_PROTOCOL_FLAG_LOCAL_CLIENT;
/* TODO: add more formats there */
}
}
break;
case 'l':
case 'L':
IF_HEADER(NO_LOG_HEADER)
{
msg_debug_protocol("read log header, value: %T", hv_tok);
srch.begin = "no";
srch.len = 2;
IF_HEADER(USER_AGENT_HEADER)
{
msg_debug_protocol("read user-agent header, value: %T", hv_tok);
if (rspamd_ftok_casecmp(hv_tok, &srch) == 0) {
task->flags |= RSPAMD_TASK_FLAG_NO_LOG;
if (hv_tok->len == 6 &&
rspamd_lc_cmp(hv_tok->begin, "rspamc", 6) == 0) {
task->protocol_flags |= RSPAMD_TASK_PROTOCOL_FLAG_LOCAL_CLIENT;
}
}
}
IF_HEADER(LOG_TAG_HEADER)
{
msg_debug_protocol("read log-tag header, value: %T", hv_tok);
/* Ensure that a tag is valid */
if (rspamd_fast_utf8_validate(hv_tok->begin, hv_tok->len) == 0) {
memcpy(task->task_pool->tag.uid, hv_tok->begin,
MIN(hv_tok->len, sizeof(task->task_pool->tag.uid)));
break;
case 'l':
case 'L':
IF_HEADER(NO_LOG_HEADER)
{
msg_debug_protocol("read log header, value: %T", hv_tok);
srch.begin = "no";
srch.len = 2;
if (rspamd_ftok_casecmp(hv_tok, &srch) == 0) {
task->flags |= RSPAMD_TASK_FLAG_NO_LOG;
}
}
}
break;
case 'm':
case 'M':
IF_HEADER(MTA_TAG_HEADER)
{
char *mta_tag;
mta_tag = rspamd_mempool_ftokdup(task->task_pool, hv_tok);
rspamd_mempool_set_variable(task->task_pool,
RSPAMD_MEMPOOL_MTA_TAG,
mta_tag, NULL);
msg_debug_protocol("read MTA-Tag header, value: %s", mta_tag);
}
IF_HEADER(MTA_NAME_HEADER)
{
char *mta_name;
mta_name = rspamd_mempool_ftokdup(task->task_pool, hv_tok);
rspamd_mempool_set_variable(task->task_pool,
RSPAMD_MEMPOOL_MTA_NAME,
mta_name, NULL);
msg_debug_protocol("read MTA-Name header, value: %s", mta_name);
}
IF_HEADER(MILTER_HEADER)
{
task->protocol_flags |= RSPAMD_TASK_PROTOCOL_FLAG_MILTER;
msg_debug_protocol("read Milter header, value: %T", hv_tok);
}
break;
case 't':
case 'T':
IF_HEADER(TLS_CIPHER_HEADER)
{
task->flags |= RSPAMD_TASK_FLAG_SSL;
msg_debug_protocol("read TLS cipher header, value: %T", hv_tok);
}
break;
default:
msg_debug_protocol("generic header: %T", hn_tok);
break;
IF_HEADER(LOG_TAG_HEADER)
{
msg_debug_protocol("read log-tag header, value: %T", hv_tok);
/* Ensure that a tag is valid */
if (rspamd_fast_utf8_validate(hv_tok->begin, hv_tok->len) == 0) {
memcpy(task->task_pool->tag.uid, hv_tok->begin,
MIN(hv_tok->len, sizeof(task->task_pool->tag.uid)));
}
}
break;
case 'm':
case 'M':
IF_HEADER(MTA_TAG_HEADER)
{
char *mta_tag;
mta_tag = rspamd_mempool_ftokdup(task->task_pool, hv_tok);
rspamd_mempool_set_variable(task->task_pool,
RSPAMD_MEMPOOL_MTA_TAG,
mta_tag, NULL);
msg_debug_protocol("read MTA-Tag header, value: %s", mta_tag);
}
IF_HEADER(MTA_NAME_HEADER)
{
char *mta_name;
mta_name = rspamd_mempool_ftokdup(task->task_pool, hv_tok);
rspamd_mempool_set_variable(task->task_pool,
RSPAMD_MEMPOOL_MTA_NAME,
mta_name, NULL);
msg_debug_protocol("read MTA-Name header, value: %s", mta_name);
}
IF_HEADER(MILTER_HEADER)
{
task->protocol_flags |= RSPAMD_TASK_PROTOCOL_FLAG_MILTER;
msg_debug_protocol("read Milter header, value: %T", hv_tok);
}
break;
case 't':
case 'T':
IF_HEADER(TLS_CIPHER_HEADER)
{
task->flags |= RSPAMD_TASK_FLAG_SSL;
msg_debug_protocol("read TLS cipher header, value: %T", hv_tok);
}
break;
default:
msg_debug_protocol("generic header: %T", hn_tok);
break;
}
rspamd_task_add_request_header (task, hn_tok, hv_tok);
@ -1716,6 +1716,7 @@ void rspamd_protocol_http_reply(struct rspamd_http_message *msg,
rspamd_fstring_free(reply);
rspamd_http_message_set_body_from_fstring_steal(msg, compressed_reply);
rspamd_http_message_add_header(msg, COMPRESSION_HEADER, "zstd");
rspamd_http_message_add_header(msg, CONTENT_ENCODING_HEADER, "zstd");
if (task->cfg->libs_ctx->out_dict &&
task->cfg->libs_ctx->out_dict->id != 0) {

2
src/libserver/protocol_internal.h
View File

@ -93,6 +93,8 @@ extern "C" {
#define RAW_DATA_HEADER "Raw"
#define COMPRESSION_HEADER "Compression"
#define MESSAGE_OFFSET_HEADER "Message-Offset"
#define CONTENT_ENCODING_HEADER "Content-Encoding"
#define ACCEPT_ENCODING_HEADER "Accept-Enconding"
#ifdef __cplusplus
}

6
src/libserver/task.c
View File

@ -519,7 +519,11 @@ rspamd_task_load_message(struct rspamd_task *task,
debug_task("got input of length %z", task->msg.len);
/* Check compression */
tok = rspamd_task_get_request_header(task, "compression");
tok = rspamd_task_get_request_header(task, COMPRESSION_HEADER);
if (!tok) {
tok = rspamd_task_get_request_header(task, CONTENT_ENCODING_HEADER);
}
if (tok) {
/* Need to uncompress */

3
src/libserver/worker_util.c
View File

@ -57,6 +57,7 @@
#include "contrib/libev/ev.h"
#include "libstat/stat_api.h"
#include "libserver/protocol_internal.h"
struct rspamd_worker *rspamd_current_worker = NULL;
@ -600,7 +601,7 @@ rspamd_controller_maybe_compress(struct rspamd_http_connection_entry *entry,
{
if (entry->support_gzip) {
if (rspamd_fstring_gzip(&buf)) {
rspamd_http_message_add_header(msg, "Content-Encoding", "gzip");
rspamd_http_message_add_header(msg, CONTENT_ENCODING_HEADER, "gzip");
}
}

3
src/lua/lua_http.c
View File

@ -21,6 +21,7 @@
#include "unix-std.h"
#include "zlib.h"
#include "utlist.h"
#include "libserver/protocol_internal.h"
/***
* @module rspamd_http
@ -1107,7 +1108,7 @@ lua_http_request(lua_State *L)
if (body) {
if (gzip) {
if (rspamd_fstring_gzip(&body)) {
rspamd_http_message_add_header(msg, "Content-Encoding", "gzip");
rspamd_http_message_add_header(msg, CONTENT_ENCODING_HEADER, "gzip");
}
}

Write Preview
Loading…
Cancel
Save