Mirrors
/
rspamd
mirror of https://github.com/rspamd/rspamd.git
3
0
Fork 0
Code Releases Activity
Browse Source

* Add configuration utils for kvstorage

rspamd-0.5
Vsevolod Stakhov 14 years ago
parent
df46d970cd
commit
bb783aa50c
10 changed files with 462 additions and 17 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      lib/CMakeLists.txt
  2. 2
      src/cfg_file.h
  3. 11
      src/cfg_utils.c
  4. 6
      src/cfg_xml.c
  5. 5
      src/cfg_xml.h
  6. 6
      src/classifiers/bayes.c
  7. 362
      src/kvstorage_config.c
  8. 81
      src/kvstorage_config.h
  9. 2
      src/plugins/regexp.c
  10. 3
      src/tokenizers/tokenizers.c

1
lib/CMakeLists.txt
View File

@ -48,6 +48,7 @@ SET(RSPAMDLIBSRC ../src/binlog.c
../src/html.c
../src/images.c
../src/kvstorage.c
../src/kvstorage_config.c
../src/lmtp_proto.c
../src/logger.c
../src/map.c

2
src/cfg_file.h
View File

@ -378,7 +378,7 @@ gchar* get_module_opt (struct config_file *cfg, gchar *module_name, gchar *opt_n
* @param limit string representation of limit (eg. 1M)
* @return numeric value of limit
*/
gsize parse_limit (const gchar *limit);
gsize parse_limit (const gchar *limit, guint len);
/**
* Parse time

11
src/cfg_utils.c
View File

@ -268,16 +268,17 @@ get_module_opt (struct config_file *cfg, gchar *module_name, gchar *opt_name)
}
gsize
parse_limit (const gchar *limit)
parse_limit (const gchar *limit, guint len)
{
gsize result = 0;
gchar *err_str;
const gchar *err_str;
if (!limit || *limit == '\0')
if (!limit || *limit == '\0' || len == 0) {
return 0;
}
errno = 0;
result = strtoul (limit, &err_str, 10);
result = strtoul (limit, (gchar **)&err_str, 10);
if (*err_str != '\0') {
/* Megabytes */
@ -292,7 +293,7 @@ parse_limit (const gchar *limit)
else if (*err_str == 'g' || *err_str == 'G') {
result *= 1073741824L;
}
else {
else if (len > 0 && err_str - limit != len) {
msg_warn ("invalid limit value '%s' at position '%s'", limit, err_str);
result = 0;
}

6
src/cfg_xml.c
View File

@ -1521,7 +1521,7 @@ xml_handle_size (struct config_file *cfg, struct rspamd_xml_userdata *ctx, GHash
gsize *dest;
dest = (gsize *)G_STRUCT_MEMBER_P (dest_struct, offset);
*dest = parse_limit (data);
*dest = parse_limit (data, -1);
return TRUE;
}
@ -1792,7 +1792,7 @@ if (g_ascii_strcasecmp (element_name, (x)) == 0) { \
else { \
res = FALSE; \
if ((required) == TRUE) { \
*error = g_error_new (xml_error_quark (), XML_UNMATCHED_TAG, "element %s is unexpected in this state, expected %s", element_name, (x)); \
if (*error == NULL) *error = g_error_new (xml_error_quark (), XML_UNMATCHED_TAG, "element %s is unexpected in this state, expected %s", element_name, (x)); \
ud->state = XML_ERROR; \
} \
} \
@ -2137,7 +2137,7 @@ check_module_option (const gchar *mname, const gchar *optname, const gchar *data
}
break;
case MODULE_OPT_TYPE_SIZE:
(void)parse_limit (data);
(void)parse_limit (data, -1);
if (errno != 0) {
msg_warn ("non-numeric data for option: '%s' for module: '%s': %s", optname, mname, strerror (errno));
return FALSE;

5
src/cfg_xml.h
View File

@ -167,7 +167,7 @@ void register_worker_opt (gint wtype, const gchar *optname, element_handler_func
void register_classifier_opt (const gchar *ctype, const gchar *optname);
/* Register new xml subparser */
void register_suparser (const gchar *tag, enum xml_read_state state, const GMarkupParser *parser, gpointer user_data);
void register_subparser (const gchar *tag, enum xml_read_state state, const GMarkupParser *parser, gpointer user_data);
/* Check validity of module option */
gboolean check_module_option (const gchar *mname, const gchar *optname, const gchar *data);
@ -175,4 +175,7 @@ gboolean check_module_option (const gchar *mname, const gchar *optname, const gc
/* Dumper functions */
gboolean xml_dump_config (struct config_file *cfg, const gchar *filename);
/* XML error quark for reporting errors */
GQuark xml_error_quark (void);
#endif

6
src/classifiers/bayes.c
View File

@ -221,7 +221,7 @@ bayes_classify (struct classifier_ctx* ctx, statfile_pool_t *pool, GTree *input,
data.learned_tokens = 0;
if (ctx->cfg->opts && (value = g_hash_table_lookup (ctx->cfg->opts, "max_tokens")) != NULL) {
minnodes = parse_limit (value);
minnodes = parse_limit (value, -1);
data.max_tokens = minnodes;
}
else {
@ -320,7 +320,7 @@ bayes_learn (struct classifier_ctx* ctx, statfile_pool_t *pool, const char *symb
data.learned_tokens = 0;
data.learned_tokens = 0;
if (ctx->cfg->opts && (value = g_hash_table_lookup (ctx->cfg->opts, "max_tokens")) != NULL) {
minnodes = parse_limit (value);
minnodes = parse_limit (value, -1);
data.max_tokens = minnodes;
}
else {
@ -425,7 +425,7 @@ bayes_learn_spam (struct classifier_ctx* ctx, statfile_pool_t *pool,
data.learned_tokens = 0;
if (ctx->cfg->opts && (value = g_hash_table_lookup (ctx->cfg->opts, "max_tokens")) != NULL) {
minnodes = parse_limit (value);
minnodes = parse_limit (value, -1);
data.max_tokens = minnodes;
}
else {

362
src/kvstorage_config.c
View File

@ -0,0 +1,362 @@
/* Copyright (c) 2010, Vsevolod Stakhov
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL Rambler BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "kvstorage_config.h"
#include "main.h"
#include "cfg_xml.h"
/* Global hash of storages indexed by id */
GHashTable *storages = NULL;
/* Last used id for explicit numbering */
gint last_id = 0;
struct kvstorage_config_parser {
enum {
KVSTORAGE_STATE_INIT,
KVSTORAGE_STATE_PARAM,
KVSTORAGE_STATE_BACKEND,
KVSTORAGE_STATE_EXPIRE,
KVSTORAGE_STATE_ID,
KVSTORAGE_STATE_NAME,
KVSTORAGE_STATE_CACHE_TYPE,
KVSTORAGE_STATE_CACHE_MAX_ELTS,
KVSTORAGE_STATE_CACHE_MAX_MEM,
KVSTORAGE_STATE_BACKEND_TYPE,
KVSTORAGE_STATE_EXPIRE_TYPE,
KVSTORAGE_STATE_ERROR
} state;
struct kvstorage_config *current_storage;
memory_pool_t *pool;
gchar *cur_elt;
};
static void
kvstorage_config_destroy (gpointer k)
{
struct kvstorage_config *kconf = k;
if (kconf->name) {
g_free (kconf->name);
}
g_free (kconf);
}
/* XML parse callbacks */
/* Called for open tags <foo bar="baz"> */
void kvstorage_xml_start_element (GMarkupParseContext *context,
const gchar *element_name,
const gchar **attribute_names,
const gchar **attribute_values,
gpointer user_data,
GError **error)
{
struct kvstorage_config_parser *kv_parser = user_data;
switch (kv_parser->state) {
case KVSTORAGE_STATE_INIT:
/* Make temporary pool */
if (kv_parser->pool != NULL) {
memory_pool_delete (kv_parser->pool);
}
kv_parser->pool = memory_pool_new (memory_pool_get_size ());
/* Create new kvstorage_config */
kv_parser->current_storage = g_malloc0 (sizeof (struct kvstorage_config));
kv_parser->current_storage->id = ++last_id;
break;
case KVSTORAGE_STATE_PARAM:
if (g_ascii_strcasecmp (element_name, "type") == 0) {
kv_parser->state = KVSTORAGE_STATE_CACHE_TYPE;
}
else if (g_ascii_strcasecmp (element_name, "max_elements") == 0) {
kv_parser->state = KVSTORAGE_STATE_CACHE_MAX_ELTS;
}
else if (g_ascii_strcasecmp (element_name, "max_memory") == 0) {
kv_parser->state = KVSTORAGE_STATE_CACHE_MAX_MEM;
}
else if (g_ascii_strcasecmp (element_name, "id") == 0) {
kv_parser->state = KVSTORAGE_STATE_ID;
}
else if (g_ascii_strcasecmp (element_name, "name") == 0) {
kv_parser->state = KVSTORAGE_STATE_NAME;
}
else if (g_ascii_strcasecmp (element_name, "backend") == 0) {
kv_parser->state = KVSTORAGE_STATE_BACKEND;
}
else if (g_ascii_strcasecmp (element_name, "expire") == 0) {
kv_parser->state = KVSTORAGE_STATE_EXPIRE;
}
else {
if (*error == NULL) {
*error = g_error_new (xml_error_quark (), XML_EXTRA_ELEMENT, "element %s is unexpected",
element_name);
}
kv_parser->state = KVSTORAGE_STATE_ERROR;
}
kv_parser->cur_elt = memory_pool_strdup (kv_parser->pool, element_name);
break;
case KVSTORAGE_STATE_BACKEND:
if (g_ascii_strcasecmp (element_name, "type") == 0) {
kv_parser->state = KVSTORAGE_STATE_BACKEND_TYPE;
}
else {
if (*error == NULL) {
*error = g_error_new (xml_error_quark (), XML_EXTRA_ELEMENT, "element %s is unexpected in backend definition",
element_name);
}
kv_parser->state = KVSTORAGE_STATE_ERROR;
}
break;
case KVSTORAGE_STATE_EXPIRE:
if (g_ascii_strcasecmp (element_name, "type") == 0) {
kv_parser->state = KVSTORAGE_STATE_EXPIRE_TYPE;
}
else {
if (*error == NULL) {
*error = g_error_new (xml_error_quark (), XML_EXTRA_ELEMENT, "element %s is unexpected in expire definition",
element_name);
}
kv_parser->state = KVSTORAGE_STATE_ERROR;
}
break;
default:
/* Do nothing at other states */
break;
}
}
#define CHECK_TAG(s) \
do { \
if (g_ascii_strcasecmp (element_name, kv_parser->cur_elt) == 0) { \
kv_parser->state = (s); \
} \
else { \
if (*error == NULL) *error = g_error_new (xml_error_quark (), XML_UNMATCHED_TAG, "element %s is unexpected in this state, expected %s", element_name, kv_parser->cur_elt); \
kv_parser->state = KVSTORAGE_STATE_ERROR; \
} \
} while (0)
/* Called for close tags </foo> */
void kvstorage_xml_end_element (GMarkupParseContext *context,
const gchar *element_name,
gpointer user_data,
GError **error)
{
struct kvstorage_config_parser *kv_parser = user_data;
switch (kv_parser->state) {
case KVSTORAGE_STATE_INIT:
case KVSTORAGE_STATE_PARAM:
if (g_ascii_strcasecmp (element_name, "keystorage") == 0) {
/* XXX: Init actual storage */
g_hash_table_insert (storages, &kv_parser->current_storage->id, kv_parser->current_storage);
kv_parser->state = KVSTORAGE_STATE_INIT;
g_markup_parse_context_pop (context);
return;
}
if (*error == NULL) {
*error = g_error_new (xml_error_quark (), XML_EXTRA_ELEMENT, "end element %s is unexpected, expected start element",
element_name);
}
kv_parser->state = KVSTORAGE_STATE_ERROR;
break;
case KVSTORAGE_STATE_ID:
case KVSTORAGE_STATE_NAME:
case KVSTORAGE_STATE_CACHE_TYPE:
case KVSTORAGE_STATE_CACHE_MAX_ELTS:
case KVSTORAGE_STATE_CACHE_MAX_MEM:
CHECK_TAG (KVSTORAGE_STATE_PARAM);
break;
case KVSTORAGE_STATE_BACKEND_TYPE:
CHECK_TAG (KVSTORAGE_STATE_BACKEND);
break;
case KVSTORAGE_STATE_EXPIRE_TYPE:
CHECK_TAG (KVSTORAGE_STATE_EXPIRE);
break;
case KVSTORAGE_STATE_BACKEND:
if (g_ascii_strcasecmp (element_name, "backend") == 0) {
kv_parser->state = KVSTORAGE_STATE_PARAM;
}
else {
if (*error == NULL) {
*error = g_error_new (xml_error_quark (), XML_EXTRA_ELEMENT, "element %s is unexpected",
element_name);
}
kv_parser->state = KVSTORAGE_STATE_ERROR;
}
break;
case KVSTORAGE_STATE_EXPIRE:
if (g_ascii_strcasecmp (element_name, "expire") == 0) {
kv_parser->state = KVSTORAGE_STATE_PARAM;
}
else {
if (*error == NULL) {
*error = g_error_new (xml_error_quark (), XML_EXTRA_ELEMENT, "element %s is unexpected",
element_name);
}
kv_parser->state = KVSTORAGE_STATE_ERROR;
}
break;
default:
/* Do nothing at other states */
break;
}
}
#undef CHECK_TAG
/* text is not nul-terminated */
void kvstorage_xml_text (GMarkupParseContext *context,
const gchar *text,
gsize text_len,
gpointer user_data,
GError **error)
{
struct kvstorage_config_parser *kv_parser = user_data;
gchar *err_str;
switch (kv_parser->state) {
case KVSTORAGE_STATE_INIT:
case KVSTORAGE_STATE_PARAM:
if (*error == NULL) {
*error = g_error_new (xml_error_quark (), XML_EXTRA_ELEMENT, "text is unexpected, expected start element");
}
kv_parser->state = KVSTORAGE_STATE_ERROR;
break;
case KVSTORAGE_STATE_ID:
kv_parser->current_storage->id = strtoul (text, &err_str, 10);
if ((gsize)(err_str - text) != text_len) {
if (*error == NULL) {
*error = g_error_new (xml_error_quark (), XML_EXTRA_ELEMENT, "invalid number: %*s", (int)text_len, text);
}
kv_parser->state = KVSTORAGE_STATE_ERROR;
}
else {
last_id = kv_parser->current_storage->id;
}
break;
case KVSTORAGE_STATE_NAME:
kv_parser->current_storage->name = g_malloc (text_len + 1);
rspamd_strlcpy (kv_parser->current_storage->name, text, text_len + 1);
break;
case KVSTORAGE_STATE_CACHE_MAX_ELTS:
kv_parser->current_storage->cache.max_elements = parse_limit (text, text_len);
break;
case KVSTORAGE_STATE_CACHE_MAX_MEM:
kv_parser->current_storage->cache.max_memory = parse_limit (text, text_len);
break;
case KVSTORAGE_STATE_CACHE_TYPE:
if (g_ascii_strncasecmp (text, "hash", MIN (text_len, sizeof ("hash") - 1)) == 0) {
kv_parser->current_storage->cache.type = KVSTORAGE_TYPE_CACHE_HASH;
}
else if (g_ascii_strncasecmp (text, "radix", MIN (text_len, sizeof ("radix") - 1)) == 0) {
kv_parser->current_storage->cache.type = KVSTORAGE_TYPE_CACHE_RADIX;
}
else {
if (*error == NULL) {
*error = g_error_new (xml_error_quark (), XML_EXTRA_ELEMENT, "invalid cache type: %*s", (int)text_len, text);
}
kv_parser->state = KVSTORAGE_STATE_ERROR;
}
break;
case KVSTORAGE_STATE_BACKEND_TYPE:
if (g_ascii_strncasecmp (text, "null", MIN (text_len, sizeof ("null") - 1)) == 0) {
kv_parser->current_storage->backend.type = KVSTORAGE_TYPE_BACKEND_NULL;
}
else {
if (*error == NULL) {
*error = g_error_new (xml_error_quark (), XML_EXTRA_ELEMENT, "invalid backend type: %*s", (int)text_len, text);
}
kv_parser->state = KVSTORAGE_STATE_ERROR;
}
break;
case KVSTORAGE_STATE_EXPIRE_TYPE:
if (g_ascii_strncasecmp (text, "lru", MIN (text_len, sizeof ("lru") - 1)) == 0) {
kv_parser->current_storage->expire.type = KVSTORAGE_TYPE_EXPIRE_LRU;
}
else {
if (*error == NULL) {
*error = g_error_new (xml_error_quark (), XML_EXTRA_ELEMENT, "invalid expire type: %*s", (int)text_len, text);
}
kv_parser->state = KVSTORAGE_STATE_ERROR;
}
break;
default:
/* Do nothing at other states */
break;
}
}
/* Called on error, including one set by other
* methods in the vtable. The GError should not be freed.
*/
void kvstorage_xml_error (GMarkupParseContext *context,
GError *error,
gpointer user_data)
{
msg_err ("kvstorage xml parser error: %s", error->message);
}
/** Public API */
/* Init subparser of kvstorage config */
void
init_kvstorage_config (void)
{
GMarkupParser *parser;
struct kvstorage_config_parser *kv_parser;
if (storages == NULL) {
storages = g_hash_table_new_full (g_int_hash, g_int_equal, NULL, kvstorage_config_destroy);
}
else {
/* Create new global table */
g_hash_table_destroy (storages);
storages = g_hash_table_new_full (g_int_hash, g_int_equal, NULL, kvstorage_config_destroy);
}
/* Create and register subparser */
parser = g_malloc0 (sizeof (GMarkupParser));
parser->start_element = kvstorage_xml_start_element;
parser->end_element = kvstorage_xml_end_element;
parser->error = kvstorage_xml_error;
parser->text = kvstorage_xml_text;
kv_parser = g_malloc0 (sizeof (struct kvstorage_config_parser));
kv_parser->state = KVSTORAGE_STATE_PARAM;
register_subparser ("keystorage", XML_READ_START, parser, kv_parser);
}
/* Get configuration for kvstorage with specified ID */
struct kvstorage_config*
get_kvstorage_config (gint id)
{
if (storages == NULL) {
return NULL;
}
return g_hash_table_lookup (storages, &id);
}

81
src/kvstorage_config.h
View File

@ -0,0 +1,81 @@
/* Copyright (c) 2010, Vsevolod Stakhov
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL AUTHOR BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef KVSTORAGE_CONFIG_H_
#define KVSTORAGE_CONFIG_H_
#include "config.h"
#include "kvstorage.h"
/* Type of kvstorage cache */
enum kvstorage_cache_type {
KVSTORAGE_TYPE_CACHE_HASH,
KVSTORAGE_TYPE_CACHE_RADIX
};
/* Type of kvstorage backend */
enum kvstorage_backend_type {
KVSTORAGE_TYPE_BACKEND_NULL = 0
};
/* Type of kvstorage expire */
enum kvstorage_expire_type {
KVSTORAGE_TYPE_EXPIRE_LRU
};
/* Cache config */
struct kvstorage_cache_config {
gsize max_elements;
gsize max_memory;
enum kvstorage_cache_type type;
};
/* Backend config */
struct kvstorage_backend_config {
enum kvstorage_backend_type type;
};
/* Expire config */
struct kvstorage_expire_config {
enum kvstorage_expire_type type;
};
/* The main keystorage config */
struct kvstorage_config {
gint id;
gchar *name;
struct kvstorage_cache_config cache;
struct kvstorage_backend_config backend;
struct kvstorage_expire_config expire;
struct rspamd_kv_storage *storage;
};
/* Init subparser of kvstorage config */
void init_kvstorage_config (void);
/* Get configuration for kvstorage with specified ID */
struct kvstorage_config* get_kvstorage_config (gint id);
#endif /* KVSTORAGE_CONFIG_H_ */

2
src/plugins/regexp.c
View File

@ -498,7 +498,7 @@ regexp_module_config (struct config_file *cfg)
regexp_module_ctx->statfile_prefix = DEFAULT_STATFILE_PREFIX;
}
if ((value = get_module_opt (cfg, "regexp", "max_size")) != NULL) {
regexp_module_ctx->max_size = parse_limit (value);
regexp_module_ctx->max_size = parse_limit (value, -1);
}
else {
regexp_module_ctx->max_size = 0;

3
src/tokenizers/tokenizers.c
View File

@ -256,7 +256,6 @@ tokenize_subject (struct worker_task *task, GTree ** tree)
{
f_str_t subject;
const gchar *sub;
token_node_t *new = NULL;
struct tokenizer *osb_tokenizer;
if (*tree == NULL) {
@ -268,13 +267,11 @@ tokenize_subject (struct worker_task *task, GTree ** tree)
/* Try to use pre-defined subject */
if (task->subject != NULL) {
new = memory_pool_alloc (task->task_pool, sizeof (token_node_t));
subject.begin = task->subject;
subject.len = strlen (task->subject);
osb_tokenizer->tokenize_func (osb_tokenizer, task->task_pool, &subject, tree, FALSE, TRUE, NULL);
}
if ((sub = g_mime_message_get_subject (task->message)) != NULL) {
new = memory_pool_alloc (task->task_pool, sizeof (token_node_t));
subject.begin = (gchar *)sub;
subject.len = strlen (sub);
osb_tokenizer->tokenize_func (osb_tokenizer, task->task_pool, &subject, tree, FALSE, TRUE, NULL);

Write Preview
Loading…
Cancel
Save