Browse Source

[Feature] Improve LLM prompt and add sender frequency tracking

* Update default prompt to reduce false positives on legitimate emails
  - Explicitly recognize verification emails as legitimate
  - Require MULTIPLE red flags for phishing classification
  - Add guidance on known/frequent senders
* Add sender frequency detection in context
  - Classify senders as: new, occasional, known, frequent
  - Based on sender_counts from user context
  - Passed to LLM via context snippet
* Prompt instructs LLM to reduce phishing score for known senders
* Helps avoid false positives on transactional/verification emails
pull/5647/head
Vsevolod Stakhov 5 days ago
parent
commit
893ee87141
No known key found for this signature in database GPG Key ID: 7647B6790081437
  1. 21
      lualib/llm_context.lua
  2. 38
      src/plugins/lua/gpt.lua

21
lualib/llm_context.lua

@ -336,12 +336,28 @@ local function join_list(arr)
return table.concat(arr, ', ')
end
local function format_context_prompt(ctx)
local function format_context_prompt(ctx, task)
local bullets = to_bullets_recent(ctx.recent_messages or {}, 5)
local top_senders = join_list(ctx.top_senders or {})
local flagged = join_list(ctx.flagged_phrases or {})
local spam_types = join_list(ctx.last_spam_labels or {})
-- Check if current sender is known
local sender_frequency = 'new'
if task then
local from = ((task:get_from('smtp') or EMPTY)[1] or EMPTY)['addr']
if from and ctx.sender_counts and ctx.sender_counts[from] then
local count = ctx.sender_counts[from]
if count >= 10 then
sender_frequency = 'frequent'
elseif count >= 3 then
sender_frequency = 'known'
else
sender_frequency = 'occasional'
end
end
end
local parts = {}
table.insert(parts, 'User recent correspondence summary:')
if bullets ~= '' then
@ -356,6 +372,7 @@ local function format_context_prompt(ctx)
if spam_types ~= '' then
table.insert(parts, string.format('Last detected spam types: %s', spam_types))
end
table.insert(parts, string.format('Current sender: %s', sender_frequency))
return table.concat(parts, '\n')
end
@ -409,7 +426,7 @@ function M.fetch(task, redis_params, opts, callback, debug_module)
lua_util.debugm(N, task, 'context warm-up OK: %s messages, generating snippet',
tostring(msg_count))
local prompt_snippet = format_context_prompt(ctx)
local prompt_snippet = format_context_prompt(ctx, task)
callback(nil, ctx, prompt_snippet)
end

38
src/plugins/lua/gpt.lua

@ -1239,19 +1239,43 @@ if opts then
if not settings.prompt then
if settings.extra_symbols then
settings.prompt = "Analyze this email strictly as a spam detector given the email message, subject, " ..
"FROM and url domains. Evaluate spam probability (0-1). " ..
settings.prompt = "Analyze this email as a spam detector. Evaluate spam probability (0-1).\n\n" ..
"LEGITIMATE patterns to recognize:\n" ..
"- Verification emails with time-limited codes are NORMAL and legitimate\n" ..
"- Transactional emails (receipts, confirmations, password resets) from services\n" ..
"- 'Verify email' or 'confirmation code' is NOT automatically phishing\n" ..
"- Emails from frequent/known senders (see context) are more trustworthy\n\n" ..
"Flag as SPAM/PHISHING only with MULTIPLE red flags:\n" ..
"- Urgent threats or fear tactics (account closure, legal action)\n" ..
"- Domain impersonation or suspicious lookalikes\n" ..
"- Requests for passwords, SSN, credit card numbers\n" ..
"- Mismatched URLs pointing to different domains than sender\n" ..
"- Poor grammar/spelling in supposedly professional emails\n\n" ..
"IMPORTANT: If sender is 'frequent' or 'known', reduce phishing probability " ..
"unless there are strong contradictory signals.\n\n" ..
"Output ONLY 3 lines:\n" ..
"1. Numeric score (0.00-1.00)\n" ..
"2. One-sentence reason citing whether it is spam, the strongest red flag, or why it is ham\n" ..
"3. Empty line or mention ONLY the primary concern category if found from the list: " ..
"2. One-sentence reason citing the strongest indicator\n" ..
"3. Primary category if applicable: " ..
table.concat(lua_util.keys(categories_map), ', ')
else
settings.prompt = "Analyze this email strictly as a spam detector given the email message, subject, " ..
"FROM and url domains. Evaluate spam probability (0-1). " ..
settings.prompt = "Analyze this email as a spam detector. Evaluate spam probability (0-1).\n\n" ..
"LEGITIMATE patterns to recognize:\n" ..
"- Verification emails with time-limited codes are NORMAL and legitimate\n" ..
"- Transactional emails (receipts, confirmations, password resets) from services\n" ..
"- 'Verify email' or 'confirmation code' is NOT automatically phishing\n" ..
"- Emails from frequent/known senders (see context) are more trustworthy\n\n" ..
"Flag as SPAM/PHISHING only with MULTIPLE red flags:\n" ..
"- Urgent threats or fear tactics (account closure, legal action)\n" ..
"- Domain impersonation or suspicious lookalikes\n" ..
"- Requests for passwords, SSN, credit card numbers\n" ..
"- Mismatched URLs pointing to different domains than sender\n" ..
"- Poor grammar/spelling in supposedly professional emails\n\n" ..
"IMPORTANT: If sender is 'frequent' or 'known', reduce phishing probability " ..
"unless there are strong contradictory signals.\n\n" ..
"Output ONLY 2 lines:\n" ..
"1. Numeric score (0.00-1.00)\n" ..
"2. One-sentence reason citing whether it is spam, the strongest red flag, or why it is ham\n"
"2. One-sentence reason citing the strongest indicator"
end
end

Loading…
Cancel
Save