[Rework] Finish rework for the rest of places that use HTTP

src/client/rspamc.c

@@ -14,8 +14,9 @@
  * limitations under the License.
  */
 #include "config.h"
-#include "util.h"
-#include "http.h"
+#include "libutil/util.h"
+#include "libutil/http.h"
+#include "libutil/http_private.h"
 #include "rspamdclient.h"
 #include "utlist.h"
 #include "unix-std.h"
@@ -950,12 +951,12 @@ rspamc_stat_output (FILE *out, ucl_object_t *obj)
 static void
 rspamc_output_headers (FILE *out, struct rspamd_http_message *msg)
 {
-	struct rspamd_http_header *h;
+	struct rspamd_http_header *h, *htmp;
 
-	LL_FOREACH (msg->headers, h)
-	{
+	HASH_ITER (hh, msg->headers, h, htmp) {
 		rspamd_fprintf (out, "%T: %T\n", h->name, h->value);
 	}
+
 	rspamd_fprintf (out, "\n");
 }
 
@@ -1223,6 +1224,8 @@ rspamc_client_cb (struct rspamd_client_connection *conn,
 	struct rspamc_command *cmd;
 	FILE *out = stdout;
 	gdouble finish = rspamd_get_ticks (), diff;
+	const gchar *body;
+	gsize body_len;
 
 	cmd = cbdata->cmd;
 	diff = finish - cbdata->start;
@@ -1275,9 +1278,13 @@ rspamc_client_cb (struct rspamd_client_connection *conn,
 			else if (err != NULL) {
 				rspamd_fprintf (out, "%s\n", err->message);
 
-				if (json && msg != NULL && msg->body != NULL) {
-					/* We can also output the resulting json */
-					rspamd_fprintf (out, "%V\n", msg->body);
+				if (json && msg != NULL) {
+					body = rspamd_http_message_get_body (msg, &body_len);
+
+					if (body) {
+						/* We can also output the resulting json */
+						rspamd_fprintf (out, "%*s\n", (gint)body_len, body);
+					}
 				}
 			}
 		}

src/client/rspamdclient.c

@@ -14,8 +14,9 @@
  * limitations under the License.
  */
 #include "rspamdclient.h"
-#include "util.h"
-#include "http.h"
+#include "libutil/util.h"
+#include "libutil/http.h"
+#include "libutil/http_private.h"
 #include "unix-std.h"
 
 #ifdef HAVE_FETCH_H
@@ -115,7 +116,7 @@ rspamd_client_finish_handler (struct rspamd_http_connection *conn,
 		return 0;
 	}
 	else {
-		if (msg->body == NULL || msg->body_buf.len == 0 || msg->code != 200) {
+		if (rspamd_http_message_get_body (msg, NULL) == NULL || msg->code != 200) {
 			err = g_error_new (RCLIENT_ERROR, msg->code, "HTTP error: %d, %.*s",
 					msg->code,
 					(gint)msg->status->len, msg->status->str);
@@ -205,6 +206,7 @@ rspamd_client_command (struct rspamd_client_connection *conn,
 	gsize remain, old_len;
 	GList *cur;
 	GString *input = NULL;
+	rspamd_fstring_t *body;
 
 	req = g_slice_alloc0 (sizeof (struct rspamd_client_request));
 	req->conn = conn;
@@ -243,11 +245,11 @@ rspamd_client_command (struct rspamd_client_connection *conn,
 			return FALSE;
 		}
 
-		req->msg->body = rspamd_fstring_new_init (input->str, input->len);
+		body = rspamd_fstring_new_init (input->str, input->len);
+		rspamd_http_message_set_body_from_fstring_steal (req->msg, body);
 		req->input = input;
 	}
 	else {
-		req->msg->body = NULL;
 		req->input = NULL;
 	}
 

src/controller.c

@@ -18,6 +18,7 @@
 #include "libutil/rrd.h"
 #include "libutil/map.h"
 #include "libutil/map_private.h"
+#include "libutil/http_private.h"
 #include "libstat/stat_api.h"
 #include "rspamd.h"
 #include "libserver/worker_util.h"
@@ -871,7 +872,6 @@ rspamd_controller_handle_get_map (struct rspamd_http_connection_entry *conn_ent,
 	gboolean found = FALSE;
 	struct rspamd_http_message *reply;
 
-
 	if (!rspamd_controller_check_password (conn_ent, session, msg, FALSE)) {
 		return 0;
 	}
@@ -919,10 +919,8 @@ rspamd_controller_handle_get_map (struct rspamd_http_connection_entry *conn_ent,
 	reply = rspamd_http_new_message (HTTP_RESPONSE);
 	reply->date = time (NULL);
 	reply->code = 200;
-	reply->body = rspamd_fstring_sized_new (st.st_size);
 
-	/* Read the whole buffer */
-	if (read (fd, reply->body->str, st.st_size) == -1) {
+	if (!rspamd_http_message_set_body_from_fd (reply, fd)) {
 		close (fd);
 		rspamd_http_message_free (reply);
 		msg_err_session ("cannot read map %s: %s", bk->uri, strerror (errno));
@@ -930,8 +928,6 @@ rspamd_controller_handle_get_map (struct rspamd_http_connection_entry *conn_ent,
 		return 0;
 	}
 
-	reply->body->len = st.st_size;
-
 	close (fd);
 
 	rspamd_http_connection_reset (conn_ent->conn);
@@ -1420,7 +1416,7 @@ rspamd_controller_handle_learn_common (
 		return 0;
 	}
 
-	if (msg->body == NULL || msg->body->len == 0) {
+	if (rspamd_http_message_get_body (msg, NULL) == NULL) {
 		msg_err_session ("got zero length body, cannot continue");
 		rspamd_controller_send_error (conn_ent,
 			400,
@@ -1521,7 +1517,7 @@ rspamd_controller_handle_scan (struct rspamd_http_connection_entry *conn_ent,
 		return 0;
 	}
 
-	if (msg->body == NULL || msg->body->len == 0) {
+	if (rspamd_http_message_get_body (msg, NULL) == NULL) {
 		msg_err_session ("got zero length body, cannot continue");
 		rspamd_controller_send_error (conn_ent,
 			400,
@@ -1595,7 +1591,7 @@ rspamd_controller_handle_saveactions (
 		return 0;
 	}
 
-	if (msg->body == NULL || msg->body->len == 0) {
+	if (rspamd_http_message_get_body (msg, NULL) == NULL) {
 		msg_err_session ("got zero length body, cannot continue");
 		rspamd_controller_send_error (conn_ent,
 			400,
@@ -1714,7 +1710,7 @@ rspamd_controller_handle_savesymbols (
 		return 0;
 	}
 
-	if (msg->body == NULL || msg->body->len == 0) {
+	if (rspamd_http_message_get_body (msg, NULL) == NULL) {
 		msg_err_session ("got zero length body, cannot continue");
 		rspamd_controller_send_error (conn_ent,
 			400,
@@ -1840,7 +1836,7 @@ rspamd_controller_handle_savemap (struct rspamd_http_connection_entry *conn_ent,
 		return 0;
 	}
 
-	if (msg->body == NULL || msg->body->len == 0) {
+	if (rspamd_http_message_get_body (msg, NULL) == NULL) {
 		msg_err_session ("got zero length body, cannot continue");
 		rspamd_controller_send_error (conn_ent,
 			400,
@@ -2162,7 +2158,7 @@ rspamd_controller_handle_custom (struct rspamd_http_connection_entry *conn_ent,
 		cmd->privilleged)) {
 		return 0;
 	}
-	if (cmd->require_message && (msg->body == NULL || msg->body->len == 0)) {
+	if (cmd->require_message && (rspamd_http_message_get_body (msg, NULL) == NULL)) {
 		msg_err_session ("got zero length body, cannot continue");
 		rspamd_controller_send_error (conn_ent,
 			400,

src/fuzzy_storage.c

@@ -33,6 +33,7 @@
 #include "ref.h"
 #include "xxhash.h"
 #include "libutil/hash.h"
+#include "libutil/http_private.h"
 #include "unix-std.h"
 
 /* This number is used as expire time in seconds for cache items  (2 days) */
@@ -260,6 +261,7 @@ fuzzy_mirror_updates_to_http (struct rspamd_fuzzy_storage_ctx *ctx,
 	gsize len;
 	guint32 rev;
 	const gchar *p;
+	rspamd_fstring_t *reply;
 
 	rev = rspamd_fuzzy_backend_version (ctx->backend, local_db_name);
 	rev = GUINT32_TO_LE (rev);
@@ -278,8 +280,8 @@ fuzzy_mirror_updates_to_http (struct rspamd_fuzzy_storage_ctx *ctx,
 		}
 	}
 
-	msg->body = rspamd_fstring_sized_new (len);
-	msg->body = rspamd_fstring_append (msg->body, (const char *)&rev,
+	reply = rspamd_fstring_sized_new (len);
+	reply = rspamd_fstring_append (reply, (const char *)&rev,
 			sizeof (rev));
 
 	for (cur = ctx->updates_pending->head; cur != NULL; cur = g_list_next (cur)) {
@@ -295,15 +297,14 @@ fuzzy_mirror_updates_to_http (struct rspamd_fuzzy_storage_ctx *ctx,
 		}
 
 		p = (const char *)io_cmd;
-		msg->body = rspamd_fstring_append (msg->body, (const char *)&len,
-					sizeof (len));
-		msg->body = rspamd_fstring_append (msg->body, p, len);
+		reply = rspamd_fstring_append (reply, (const char *)&len, sizeof (len));
+		reply = rspamd_fstring_append (reply, p, len);
 	}
 
 	/* Last chunk */
 	len = 0;
-	msg->body = rspamd_fstring_append (msg->body, (const char *)&len,
-			sizeof (len));
+	reply = rspamd_fstring_append (reply, (const char *)&len, sizeof (len));
+	rspamd_http_message_set_body_from_fstring_steal (msg, reply);
 }
 
 static void
@@ -936,7 +937,8 @@ rspamd_fuzzy_mirror_process_update (struct fuzzy_master_update_session *session,
 	} state = read_len;
 	GList *updates = NULL, *cur;
 
-	if (!msg->body || msg->body->len == 0 || !msg->url || msg->url->len == 0) {
+	if (!rspamd_http_message_get_body (msg, NULL) || !msg->url
+			|| msg->url->len == 0) {
 		msg_err ("empty update message, not processing");
 
 		return;
@@ -963,8 +965,7 @@ rspamd_fuzzy_mirror_process_update (struct fuzzy_master_update_session *session,
 	 * <0> - end of data
 	 * ... - ignored
 	 */
-	p = (const guchar *)msg->body->str;
-	remain = msg->body->len;
+	p = rspamd_http_message_get_body (msg, &remain);
 
 	if (remain > sizeof (guint32) * 2) {
 		memcpy (&revision, p, sizeof (guint32));

src/libserver/protocol.c

@@ -21,6 +21,7 @@
 #include "message.h"
 #include "utlist.h"
 #include "http.h"
+#include "http_private.h"
 #include "email_addr.h"
 #include "worker_private.h"
 #include "cryptobox.h"
@@ -270,11 +271,10 @@ rspamd_protocol_handle_headers (struct rspamd_task *task,
 	rspamd_fstring_t *hn, *hv;
 	rspamd_ftok_t *hn_tok, *hv_tok, srch;
 	gboolean fl, has_ip = FALSE;
-	struct rspamd_http_header *h;
+	struct rspamd_http_header *h, *htmp;
 	struct rspamd_email_address *addr;
 
-	LL_FOREACH (msg->headers, h)
-	{
+	HASH_ITER (hh, msg->headers, h, htmp) {
 		hn = rspamd_fstring_new_init (h->name->begin, h->name->len);
 		hv = rspamd_fstring_new_init (h->value->begin, h->value->len);
 		hn_tok = rspamd_ftok_map (hn);
@@ -1017,6 +1017,7 @@ rspamd_protocol_http_reply (struct rspamd_http_message *msg,
 	const struct rspamd_re_cache_stat *restat;
 	gpointer h, v;
 	ucl_object_t *top = NULL;
+	rspamd_fstring_t *reply;
 	gint action;
 
 	/* Write custom headers */
@@ -1050,21 +1051,22 @@ rspamd_protocol_http_reply (struct rspamd_http_message *msg,
 				restat->bytes_scanned);
 	}
 
-	msg->body = rspamd_fstring_sized_new (1000);
+	reply = rspamd_fstring_sized_new (1000);
 
 	if (msg->method < HTTP_SYMBOLS && !RSPAMD_TASK_IS_SPAMC (task)) {
-		rspamd_ucl_emit_fstring (top, UCL_EMIT_JSON_COMPACT, &msg->body);
+		rspamd_ucl_emit_fstring (top, UCL_EMIT_JSON_COMPACT, &reply);
 	}
 	else {
 		if (RSPAMD_TASK_IS_SPAMC (task)) {
-			rspamd_ucl_tospamc_output (top, &msg->body);
+			rspamd_ucl_tospamc_output (top, &reply);
 		}
 		else {
-			rspamd_ucl_torspamc_output (top, &msg->body);
+			rspamd_ucl_torspamc_output (top, &reply);
 		}
 	}
 
 	ucl_object_unref (top);
+	rspamd_http_message_set_body_from_fstring_steal (msg, reply);
 
 	if (!(task->flags & RSPAMD_TASK_FLAG_NO_STAT)) {
 		/* Update stat for default metric */
@@ -1187,6 +1189,7 @@ rspamd_protocol_write_reply (struct rspamd_task *task)
 	struct rspamd_http_message *msg;
 	const gchar *ctype = "application/json";
 	struct rspamd_abstract_worker_ctx *actx;
+	rspamd_fstring_t *reply;
 
 	msg = rspamd_http_new_message (HTTP_RESPONSE);
 
@@ -1218,9 +1221,10 @@ rspamd_protocol_write_reply (struct rspamd_task *task)
 		ucl_object_insert_key (top,
 			ucl_object_fromstring (g_quark_to_string (task->err->domain)),
 			"error_domain", 0, false);
-		msg->body = rspamd_fstring_sized_new (256);
-		rspamd_ucl_emit_fstring (top, UCL_EMIT_JSON_COMPACT, &msg->body);
+		reply = rspamd_fstring_sized_new (256);
+		rspamd_ucl_emit_fstring (top, UCL_EMIT_JSON_COMPACT, &reply);
 		ucl_object_unref (top);
+		rspamd_http_message_set_body_from_fstring_steal (msg, reply);
 	}
 	else {
 		msg->status = rspamd_fstring_new_init ("OK", 2);
@@ -1243,7 +1247,7 @@ rspamd_protocol_write_reply (struct rspamd_task *task)
 			}
 			break;
 		case CMD_PING:
-			msg->body = rspamd_fstring_new_init ("pong" CRLF, 6);
+			rspamd_http_message_set_body (msg, "pong" CRLF, 6);
 			ctype = "text/plain";
 			break;
 		case CMD_OTHER:

src/libserver/rspamd_control.c

@@ -16,7 +16,8 @@
 #include "config.h"
 #include "rspamd.h"
 #include "rspamd_control.h"
-#include "http.h"
+#include "libutil/http.h"
+#include "libutil/http_private.h"
 #include "unix-std.h"
 #include "utlist.h"
 
@@ -107,6 +108,7 @@ rspamd_control_send_error (struct rspamd_control_session *session,
 		gint code, const gchar *error_msg, ...)
 {
 	struct rspamd_http_message *msg;
+	rspamd_fstring_t *reply;
 	va_list args;
 
 	msg = rspamd_http_new_message (HTTP_RESPONSE);
@@ -118,8 +120,9 @@ rspamd_control_send_error (struct rspamd_control_session *session,
 
 	msg->date = time (NULL);
 	msg->code = code;
-	msg->body = rspamd_fstring_new ();
-	rspamd_printf_fstring (&msg->body, "{\"error\":\"%V\"}", msg->status);
+	reply = rspamd_fstring_sized_new (msg->status->len + 16);
+	rspamd_printf_fstring (&reply, "{\"error\":\"%V\"}", msg->status);
+	rspamd_http_message_set_body_from_fstring_steal (msg, reply);
 	rspamd_http_connection_reset (session->conn);
 	rspamd_http_connection_write_message (session->conn,
 			msg,
@@ -136,13 +139,15 @@ rspamd_control_send_ucl (struct rspamd_control_session *session,
 		ucl_object_t *obj)
 {
 	struct rspamd_http_message *msg;
+	rspamd_fstring_t *reply;
 
 	msg = rspamd_http_new_message (HTTP_RESPONSE);
 	msg->date = time (NULL);
 	msg->code = 200;
 	msg->status = rspamd_fstring_new_init ("OK", 2);
-	msg->body = rspamd_fstring_sized_new (BUFSIZ);
-	rspamd_ucl_emit_fstring (obj, UCL_EMIT_JSON_COMPACT, &msg->body);
+	reply = rspamd_fstring_sized_new (BUFSIZ);
+	rspamd_ucl_emit_fstring (obj, UCL_EMIT_JSON_COMPACT, &reply);
+	rspamd_http_message_set_body_from_fstring_steal (msg, reply);
 	rspamd_http_connection_reset (session->conn);
 	rspamd_http_connection_write_message (session->conn,
 			msg,

src/libserver/worker_util.c

@@ -24,6 +24,7 @@
 #include "rspamd_control.h"
 #include "libutil/map.h"
 #include "libutil/map_private.h"
+#include "libutil/http_private.h"
 
 #ifdef WITH_GPERF_TOOLS
 #include <gperftools/profiler.h>
@@ -352,6 +353,7 @@ rspamd_controller_send_error (struct rspamd_http_connection_entry *entry,
 {
 	struct rspamd_http_message *msg;
 	va_list args;
+	rspamd_fstring_t *reply;
 
 	msg = rspamd_http_new_message (HTTP_RESPONSE);
 
@@ -362,8 +364,9 @@ rspamd_controller_send_error (struct rspamd_http_connection_entry *entry,
 
 	msg->date = time (NULL);
 	msg->code = code;
-	msg->body = rspamd_fstring_new ();
-	rspamd_printf_fstring (&msg->body, "{\"error\":\"%V\"}", msg->status);
+	reply = rspamd_fstring_sized_new (msg->status->len + 16);
+	rspamd_printf_fstring (&reply, "{\"error\":\"%V\"}", msg->status);
+	rspamd_http_message_set_body_from_fstring_steal (msg, reply);
 	rspamd_http_connection_reset (entry->conn);
 	rspamd_http_connection_write_message (entry->conn,
 		msg,
@@ -381,12 +384,14 @@ rspamd_controller_send_string (struct rspamd_http_connection_entry *entry,
 	const gchar *str)
 {
 	struct rspamd_http_message *msg;
+	rspamd_fstring_t *reply;
 
 	msg = rspamd_http_new_message (HTTP_RESPONSE);
 	msg->date = time (NULL);
 	msg->code = 200;
 	msg->status = rspamd_fstring_new_init ("OK", 2);
-	msg->body = rspamd_fstring_new_init (str, strlen (str));
+	reply = rspamd_fstring_new_init (str, strlen (str));
+	rspamd_http_message_set_body_from_fstring_steal (msg, reply);
 	rspamd_http_connection_reset (entry->conn);
 	rspamd_http_connection_write_message (entry->conn,
 		msg,
@@ -404,13 +409,15 @@ rspamd_controller_send_ucl (struct rspamd_http_connection_entry *entry,
 	ucl_object_t *obj)
 {
 	struct rspamd_http_message *msg;
+	rspamd_fstring_t *reply;
 
 	msg = rspamd_http_new_message (HTTP_RESPONSE);
 	msg->date = time (NULL);
 	msg->code = 200;
 	msg->status = rspamd_fstring_new_init ("OK", 2);
-	msg->body = rspamd_fstring_sized_new (BUFSIZ);
-	rspamd_ucl_emit_fstring (obj, UCL_EMIT_JSON_COMPACT, &msg->body);
+	reply = rspamd_fstring_sized_new (BUFSIZ);
+	rspamd_ucl_emit_fstring (obj, UCL_EMIT_JSON_COMPACT, &reply);
+	rspamd_http_message_set_body_from_fstring_steal (msg, reply);
 	rspamd_http_connection_reset (entry->conn);
 	rspamd_http_connection_write_message (entry->conn,
 		msg,

src/libutil/map.c

@@ -20,6 +20,7 @@
 #include "map.h"
 #include "map_private.h"
 #include "http.h"
+#include "http_private.h"
 #include "rspamd.h"
 #include "cryptobox.h"
 #include "unix-std.h"

src/lua/lua_http.c

@@ -17,6 +17,7 @@
 #include "buffer.h"
 #include "dns.h"
 #include "http.h"
+#include "http_private.h"
 #include "utlist.h"
 #include "unix-std.h"
 
@@ -150,7 +151,9 @@ lua_http_finish_handler (struct rspamd_http_connection *conn,
 		struct rspamd_http_message *msg)
 {
 	struct lua_http_cbdata *cbd = (struct lua_http_cbdata *)conn->ud;
-	struct rspamd_http_header *h;
+	struct rspamd_http_header *h, *htmp;
+	const gchar *body;
+	gsize body_len;
 
 	lua_rawgeti (cbd->L, LUA_REGISTRYINDEX, cbd->cbref);
 	/* Error */
@@ -158,14 +161,23 @@ lua_http_finish_handler (struct rspamd_http_connection *conn,
 	/* Reply code */
 	lua_pushinteger (cbd->L, msg->code);
 	/* Body */
-	lua_pushlstring (cbd->L, msg->body->str, msg->body->len);
+	body = rspamd_http_message_get_body (msg, &body_len);
+
+	if (body_len > 0) {
+		lua_pushlstring (cbd->L, body, body_len);
+	}
+	else {
+		lua_pushnil (cbd->L);
+	}
 	/* Headers */
 	lua_newtable (cbd->L);
-	LL_FOREACH (msg->headers, h) {
+
+	HASH_ITER (hh, msg->headers, h, htmp) {
 		lua_pushlstring (cbd->L, h->name->begin, h->name->len);
 		lua_pushlstring (cbd->L, h->value->begin, h->value->len);
 		lua_settable (cbd->L, -3);
 	}
+
 	if (lua_pcall (cbd->L, 4, 0, 0) != 0) {
 		msg_info ("callback call failed: %s", lua_tostring (cbd->L, -1));
 		lua_pop (cbd->L, 1);
@@ -296,19 +308,23 @@ lua_http_request (lua_State *L)
 		else {
 			ev_base = NULL;
 		}
+
 		if (lua_gettop (L) >= 4 && rspamd_lua_check_udata (L, 4, "rspamd{resolver}")) {
 			resolver = *(struct rspamd_dns_resolver **)lua_touserdata (L, 4);
 		}
 		else {
 			resolver = lua_http_global_resolver (ev_base);
 		}
+
 		if (lua_gettop (L) >= 5 && rspamd_lua_check_udata (L, 5, "rspamd{session}")) {
 			session = *(struct rspamd_async_session **)lua_touserdata (L, 5);
 		}
 		else {
 			session = NULL;
 		}
+
 		msg = rspamd_http_message_from_url (url);
+
 		if (msg == NULL) {
 			lua_pushboolean (L, FALSE);
 			return 1;
@@ -403,13 +419,13 @@ lua_http_request (lua_State *L)
 		lua_gettable (L, -2);
 		if (lua_type (L, -1) == LUA_TSTRING) {
 			lua_body = lua_tolstring (L, -1, &bodylen);
-			msg->body = rspamd_fstring_new_init (lua_body, bodylen);
+			rspamd_http_message_set_body (msg, lua_body, bodylen);
 		}
 		else if (lua_type (L, -1) == LUA_TUSERDATA) {
 			t = lua_check_text (L, -1);
 			/* TODO: think about zero-copy possibilities */
 			if (t) {
-				msg->body = rspamd_fstring_new_init (t->start, t->len);
+				rspamd_http_message_set_body (msg, t->start, t->len);
 			}
 		}
 

src/plugins/fuzzy_check.c

@@ -42,6 +42,7 @@
 #include "keypair.h"
 #include "lua/lua_common.h"
 #include "unix-std.h"
+#include "libutil/http_private.h"
 #include <math.h>
 
 #define DEFAULT_SYMBOL "R_FUZZY_HASH"

src/plugins/surbl.c

@@ -40,6 +40,7 @@
 #include "surbl.h"
 #include "utlist.h"
 #include "libserver/html.h"
+#include "libutil/http_private.h"
 #include "unix-std.h"
 
 static struct surbl_ctx *surbl_module_ctx = NULL;

src/rspamadm/control.c

@@ -17,7 +17,8 @@
 #include "rspamadm.h"
 #include "cryptobox.h"
 #include "printf.h"
-#include "http.h"
+#include "libutil/http.h"
+#include "libutil/http_private.h"
 #include "addr.h"
 #include "unix-std.h"
 #include <event.h>
@@ -100,11 +101,14 @@ rspamd_control_finish_handler (struct rspamd_http_connection *conn,
 	struct ucl_parser *parser;
 	ucl_object_t *obj;
 	rspamd_fstring_t *out;
+	const gchar *body;
+	gsize body_len;
 	struct rspamadm_control_cbdata *cbdata = conn->ud;
 
+	body = rspamd_http_message_get_body (msg, &body_len);
 	parser = ucl_parser_new (0);
 
-	if (!ucl_parser_add_chunk (parser, msg->body->str, msg->body->len)) {
+	if (!body || !ucl_parser_add_chunk (parser, body, body_len)) {
 		rspamd_fprintf (stderr, "cannot parse server's reply: %s\n",
 				ucl_parser_get_error (parser));
 		ucl_parser_free (parser);

src/rspamadm/lua_repl.c

@@ -17,6 +17,8 @@
 #include "config.h"
 #include "rspamadm.h"
 #include "cryptobox.h"
+#include "libutil/http.h"
+#include "libutil/http_private.h"
 #include "printf.h"
 #include "lua/lua_common.h"
 #include "message.h"
@@ -533,11 +535,14 @@ rspamadm_lua_handle_exec (struct rspamd_http_connection_entry *conn_ent,
 	struct rspamadm_lua_repl_context *ctx;
 	struct rspamadm_lua_repl_session *session = conn_ent->ud;
 	ucl_object_t *obj, *elt;
+	const gchar *body;
+	gsize body_len;
 
 	ctx = session->ctx;
 	L = ctx->L;
+	body = rspamd_http_message_get_body (msg, &body_len);
 
-	if (msg->body == NULL || msg->body->len == 0) {
+	if (body == NULL) {
 		rspamd_controller_send_error (conn_ent, 400, "Empty lua script");
 
 		return 0;
@@ -547,8 +552,8 @@ rspamadm_lua_handle_exec (struct rspamd_http_connection_entry *conn_ent,
 	err_idx = lua_gettop (L);
 
 	/* First try return + input */
-	tb = g_string_sized_new (msg->body->len + sizeof ("return "));
-	rspamd_printf_gstring (tb, "return %V", msg->body);
+	tb = g_string_sized_new (body_len + sizeof ("return "));
+	rspamd_printf_gstring (tb, "return %*s", (gint)body_len, body);
 
 	if (luaL_loadstring (L, tb->str) != 0) {
 		/* Reset stack */
@@ -556,7 +561,7 @@ rspamadm_lua_handle_exec (struct rspamd_http_connection_entry *conn_ent,
 		lua_pushcfunction (L, &rspamd_lua_traceback);
 		err_idx = lua_gettop (L);
 		/* Try with no return */
-		if (luaL_loadbuffer (L, msg->body->str, msg->body->len, "http input") != 0) {
+		if (luaL_loadbuffer (L, body, body_len, "http input") != 0) {
 			rspamd_controller_send_error (conn_ent, 400, "Invalid lua script");
 
 			return 0;

src/rspamd_proxy.c

@@ -17,6 +17,8 @@
 #include "libutil/util.h"
 #include "libutil/map.h"
 #include "libutil/upstream.h"
+#include "libutil/http.h"
+#include "libutil/http_private.h"
 #include "libserver/protocol.h"
 #include "libserver/cfg_file.h"
 #include "libserver/url.h"
@@ -67,6 +69,7 @@ struct rspamd_http_upstream {
 	struct rspamd_cryptobox_pubkey *key;
 	gint parser_from_ref;
 	gint parser_to_ref;
+	gboolean local;
 };
 
 struct rspamd_http_mirror {
@@ -77,6 +80,7 @@ struct rspamd_http_mirror {
 	gdouble prob;
 	gint parser_from_ref;
 	gint parser_to_ref;
+	gboolean local;
 };
 
 static const guint64 rspamd_rspamd_proxy_magic = 0xcdeb4fd1fc351980ULL;
@@ -318,6 +322,11 @@ rspamd_proxy_parse_upstream (rspamd_mempool_t *pool,
 		ctx->default_upstream = up;
 	}
 
+	elt = ucl_object_lookup (obj, "local");
+	if (elt && ucl_object_toboolean (elt)) {
+		up->local = TRUE;
+	}
+
 	/*
 	 * Accept lua function here in form
 	 * fun :: String -> UCL
@@ -431,6 +440,11 @@ rspamd_proxy_parse_mirror (rspamd_mempool_t *pool,
 		up->prob = 1.0;
 	}
 
+	elt = ucl_object_lookup (obj, "local");
+	if (elt && ucl_object_toboolean (elt)) {
+		up->local = TRUE;
+	}
+
 	/*
 	 * Accept lua function here in form
 	 * fun :: String -> UCL
@@ -1065,6 +1079,7 @@ proxy_backend_master_finish_handler (struct rspamd_http_connection *conn,
 {
 	struct rspamd_proxy_backend_connection *bk_conn = conn->ud;
 	struct rspamd_proxy_session *session;
+	rspamd_fstring_t *reply;
 
 	session = bk_conn->s;
 	rspamd_http_connection_steal_msg (session->master_conn->backend_conn);
@@ -1082,8 +1097,9 @@ proxy_backend_master_finish_handler (struct rspamd_http_connection *conn,
 	if (session->is_spamc) {
 		/* We need to reformat ucl to fit with legacy spamc protocol */
 		if (bk_conn->results) {
-			rspamd_fstring_clear (msg->body);
-			rspamd_ucl_torspamc_output (bk_conn->results, &msg->body);
+			reply = rspamd_fstring_new ();
+			rspamd_ucl_torspamc_output (bk_conn->results, &reply);
+			rspamd_http_message_set_body_from_fstring_steal (msg, reply);
 		}
 		else {
 			msg_warn_session ("cannot parse results from the master backend, "

test/CMakeLists.txt

@@ -26,6 +26,7 @@ ENDIF(NOT CMAKE_SYSTEM_NAME STREQUAL "Darwin")
 TARGET_LINK_LIBRARIES(rspamd-test rspamd-cdb)
 TARGET_LINK_LIBRARIES(rspamd-test lcbtrie)
 TARGET_LINK_LIBRARIES(rspamd-test rspamd-http-parser)
+TARGET_LINK_LIBRARIES(rspamd-test rspamd-lpeg)
 TARGET_LINK_LIBRARIES(rspamd-test ${RSPAMD_REQUIRED_LIBRARIES})
 IF (ENABLE_SNOWBALL MATCHES "ON")
 	TARGET_LINK_LIBRARIES(rspamd-test stemmer)

test/rspamd_http_test.c

@@ -16,7 +16,8 @@
 #include "config.h"
 #include "rspamd.h"
 #include "util.h"
-#include "http.h"
+#include "libutil/http.h"
+#include "libutil/http_private.h"
 #include "tests.h"
 #include "ottery.h"
 #include "cryptobox.h"

utils/rspamd_http_bench.c

@@ -16,7 +16,8 @@
 #include "config.h"
 #include "rspamd.h"
 #include "util.h"
-#include "http.h"
+#include "libutil/http.h"
+#include "libutil/http_private.h"
 #include "ottery.h"
 #include "cryptobox.h"
 #include "unix-std.h"

utils/rspamd_http_server.c

@@ -16,7 +16,9 @@
 #include "config.h"
 #include "rspamd.h"
 #include "util.h"
-#include "http.h"
+#include "libutil/fstring.h"
+#include "libutil/http.h"
+#include "libutil/http_private.h"
 #include "ottery.h"
 #include "cryptobox.h"
 #include "keypair.h"
@@ -83,6 +85,7 @@ rspamd_server_finish (struct rspamd_http_connection *conn,
 	gulong size;
 	const gchar *url_str;
 	guint url_len;
+	rspamd_fstring_t *body;
 
 	if (!session->reply) {
 		session->reply = TRUE;
@@ -100,9 +103,11 @@ rspamd_server_finish (struct rspamd_http_connection *conn,
 
 			reply->code = 200;
 			reply->status = rspamd_fstring_new_init ("OK", 2);
-			reply->body = rspamd_fstring_sized_new (size);
-			reply->body->len = size;
-			memset (reply->body->str, 0, size);
+			body = rspamd_fstring_sized_new (size);
+			body->len = size;
+			memset (body->str, 0, size);
+			rspamd_http_message_set_body_from_fstring_steal (msg, body);
+
 		}
 		else {
 			reply->code = 404;