[Feature] ASN module; support matching ASN/country in multimap

- Also removed stuff from ip_score
 - Updated tests

src/plugins/lua/asn.lua

@@ -0,0 +1,100 @@
+--[[
+Copyright (c) 2011-2016, Vsevolod Stakhov <vsevolod@highsecure.ru>
+Copyright (c) 2016, Andrew Lewis <nerf@judo.za.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local rspamd_logger = require "rspamd_logger"
+local rspamd_regexp = require "rspamd_regexp"
+
+local options = {
+  provider_type = 'cymru',
+  provider_info = {
+    ip4 = 'origin.asn.cymru.com',
+    ip6 = 'origin6.asn.cymru.com',
+  },
+  symbol = 'ASN',
+}
+
+local cymru_re = rspamd_regexp.create_cached("[\\|\\s]")
+
+local function asn_check(task)
+
+  local function asn_set(asn, ipnet, country)
+    local descr_t = {}
+    if asn then
+      task:get_mempool():set_variable("asn", asn)
+      table.insert(descr_t, "asn:" .. asn)
+    end
+    if ipnet then
+      task:get_mempool():set_variable("ipnet", ipnet)
+      table.insert(descr_t, "ipnet:" .. ipnet)
+    end
+    if country then
+      task:get_mempool():set_variable("country", country)
+      table.insert(descr_t, "country:" .. country)
+    end
+    if options['symbol'] then
+      task:insert_result(options['symbol'], 0.0, table.concat(descr_t, ', '))
+    end
+  end
+
+  local asn_check_func = {}
+  function asn_check_func.cymru(ip)
+    local function cymru_dns_cb(resolver, to_resolve, results, err, key)
+      if not (results and results[1]) then return end
+      local parts = cymru_re:split(results[1])
+      -- "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8
+      asn_set(parts[1], parts[2], parts[3])
+    end
+    local dnsbl = options['provider_info']['ip' .. ip:get_version()]
+    local req_name = rspamd_logger.slog("%1.%2",
+        table.concat(ip:inversed_str_octets(), '.'), dnsbl)
+    task:get_resolver():resolve_txt(task:get_session(), task:get_mempool(),
+        req_name, cymru_dns_cb)
+  end
+
+  local ip = task:get_from_ip()
+  if not (ip and ip:is_valid()) then return end
+  asn_check_func[options['provider_type']](ip)
+end
+
+-- Configuration options
+local configure_asn_module = function()
+  local opts =  rspamd_config:get_all_opt('asn')
+  if opts then
+    for k,v in pairs(opts) do
+      options[k] = v
+    end
+  end
+  if options['provider_type'] == 'cymru' then
+    if not options['provider_info'] and options['provider_info']['ip4'] and
+        options['provider_info']['ip6'] then
+      rspamd_logger.errx("Missing required provider_info for cymru")
+      return false
+    end
+  else
+    rspamd_logger.errx("Unknown provider_type: %s", options['provider_type'])
+    return false
+  end
+  return true
+end
+
+if configure_asn_module() then
+  rspamd_config:register_symbol({
+    name = 'ASN_CHECK',
+    type = 'prefilter',
+    callback = asn_check,
+  })
+end

src/plugins/lua/ip_score.lua

@@ -28,8 +28,6 @@ local whitelist = nil
 local asn_cc_whitelist = nil
 
 local options = {
-  asn_provider = 'origin.asn.cymru.com', -- provider for ASN data
-  asn6_provider = 'origin6.asn.cymru.com', -- provider for ASN data
   actions = { -- how each action is treated in scoring
     ['reject'] = 1.0,
     ['add header'] = 0.25,
@@ -56,38 +54,6 @@ local options = {
 
 local asn_re = rspamd_regexp.create_cached("[\\|\\s]")
 
-local function asn_check(task)
-  local ip = task:get_from_ip()
-
-  local function asn_dns_cb(resolver, to_resolve, results, err, key)
-    if results and results[1] then
-      local parts = asn_re:split(results[1])
-      -- "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8
-      if parts[1] then
-        task:get_mempool():set_variable("asn", parts[1])
-      end
-      if parts[2] then
-        task:get_mempool():set_variable("ipnet", parts[2])
-      end
-      if parts[3] then
-        task:get_mempool():set_variable("country", parts[3])
-      end
-    end
-  end
-
-  if ip and ip:is_valid() then
-    local asn_provider = 'asn_provider'
-    if ip:get_version() == 6 then
-      asn_provider = 'asn6_provider'
-    end
-    local req_name = rspamd_logger.slog("%1.%2",
-      table.concat(ip:inversed_str_octets(), '.'), options[asn_provider])
-
-    task:get_resolver():resolve_txt(task:get_session(), task:get_mempool(),
-        req_name, asn_dns_cb)
-  end
-end
-
 local function ip_score_hash_key(asn, country, ipnet, ip)
   -- We use the most common attribute as hashing key
   if country then
@@ -364,13 +330,6 @@ end
 configure_ip_score_module()
 if redis_params then
   -- Register ip_score module
-  if options['asn_provider'] then
-    rspamd_config:register_symbol({
-      name = 'ASN_CHECK',
-      type = 'prefilter',
-      callback = asn_check,
-    })
-  end
   rspamd_config:register_symbol({
     name = 'IPSCORE_SAVE',
     type = 'postfilter',

src/plugins/lua/multimap.lua

@@ -509,6 +509,16 @@ local function multimap_callback(task, rule)
     if hostname and hostname ~= 'unknown' then
       match_hostname(rule, hostname)
     end
+  elseif rt == 'asn' then
+    local asn = task:get_mempool():get_variable('asn')
+    if asn then
+      match_rule(rule, asn)
+    end
+  elseif rt == 'country' then
+    local country = task:get_mempool():get_variable('country')
+    if country then
+      match_rule(rule, country)
+    end
   end
 end
 
@@ -595,7 +605,9 @@ local function add_multimap_rule(key, newrule)
         or newrule['type'] == 'filename'
         or newrule['type'] == 'url'
         or newrule['type'] == 'content'
-        or newrule['type'] == 'hostname' then
+        or newrule['type'] == 'hostname'
+        or newrule['type'] == 'asn'
+        or newrule['type'] == 'country' then
         if newrule['regexp'] then
           newrule['hash'] = rspamd_config:add_map ({
             url = newrule['map'],

test/functional/cases/102_multimap.robot

@@ -210,6 +210,24 @@ MAP - REDIS - URL NOFILTER - MISS
   ${result} =  Scan Message With Rspamc  ${URL4}
   Check Rspamc  ${result}  REDIS_URL_NOFILTER  inverse=1
 
+MAP - REDIS - ASN - HIT
+  Redis HSET  asn  15169  ${EMPTY}
+  ${result} =  Scan Message With Rspamc  ${MESSAGE}  -i  8.8.8.8
+  Check Rspamc  ${result}  REDIS_ASN
+
+MAP - REDIS - ASN - MISS
+  ${result} =  Scan Message With Rspamc  ${MESSAGE}  -i  46.228.47.114
+  Check Rspamc  ${result}  REDIS_ASN  inverse=1
+
+MAP - REDIS - CC - HIT
+  Redis HSET  cc  US  ${EMPTY}
+  ${result} =  Scan Message With Rspamc  ${MESSAGE}  -i  8.8.8.8
+  Check Rspamc  ${result}  REDIS_COUNTRY
+
+MAP - REDIS - ASN - MISS
+  ${result} =  Scan Message With Rspamc  ${MESSAGE}  -i  46.228.47.114
+  Check Rspamc  ${result}  REDIS_COUNTRY  inverse=1
+
 *** Keywords ***
 Multimap Setup
   ${PLUGIN_CONFIG} =  Get File  ${TESTDIR}/configs/multimap.conf

test/functional/configs/multimap.conf

@@ -1,3 +1,5 @@
+asn {
+}
 redis {
   servers = "${REDIS_ADDR}:${REDIS_PORT}";
 }
@@ -96,4 +98,12 @@ multimap {
     type = "url";
     map = "redis://urlnofilter";
   }
+  REDIS_COUNTRY {
+    type = "country";
+    map = "redis://cc";
+  }
+  REDIS_ASN {
+    type = "asn";
+    map = "redis://asn";
+  }
 }