From 15f9671b667cf369255aaa27ee4257267990095c Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Tue, 6 Feb 2024 19:35:43 +0000 Subject: [PATCH] feat: privacy mode Adds support for hiding IP addresses & hostnames associated with clients sending authenticated mail in to Postal over SMTP and HTTP --- app/controllers/servers_controller.rb | 2 +- app/models/incoming_message_prototype.rb | 2 +- app/models/outgoing_message_prototype.rb | 10 +-- app/views/servers/advanced.html.haml | 6 +- ...40206173036_add_privacy_mode_to_servers.rb | 5 ++ db/schema.rb | 63 ++++++++++--------- lib/postal/received_header.rb | 43 +++++++++++++ lib/postal/smtp_server/client.rb | 21 ++----- spec/lib/postal/received_header_spec.rb | 48 ++++++++++++++ 9 files changed, 141 insertions(+), 59 deletions(-) create mode 100644 db/migrate/20240206173036_add_privacy_mode_to_servers.rb create mode 100644 lib/postal/received_header.rb create mode 100644 spec/lib/postal/received_header_spec.rb diff --git a/app/controllers/servers_controller.rb b/app/controllers/servers_controller.rb index 69f03850..8e9626b6 100644 --- a/app/controllers/servers_controller.rb +++ b/app/controllers/servers_controller.rb @@ -41,7 +41,7 @@ def create def update extra_params = [:spam_threshold, :spam_failure_threshold, :postmaster_address] - extra_params += [:send_limit, :allow_sender, :log_smtp_data, :outbound_spam_threshold, :message_retention_days, :raw_message_retention_days, :raw_message_retention_size] if current_user.admin? + extra_params += [:send_limit, :allow_sender, :privacy_mode, :log_smtp_data, :outbound_spam_threshold, :message_retention_days, :raw_message_retention_days, :raw_message_retention_size] if current_user.admin? if @server.update(safe_params(*extra_params)) redirect_to_with_json organization_server_path(organization, @server), notice: "Server settings have been updated" else diff --git a/app/models/incoming_message_prototype.rb b/app/models/incoming_message_prototype.rb index c832904f..8dee389c 100644 --- a/app/models/incoming_message_prototype.rb +++ b/app/models/incoming_message_prototype.rb @@ -91,7 +91,7 @@ def raw_message content: attachment[:data] } end - mail.header["Received"] = "from #{@source_type} (#{@ip} [#{@ip}]) by Postal with HTTP; #{Time.now.utc.rfc2822}" + mail.header["Received"] = Postal::ReceivedHeader.generate(@server, @source_type, @ip, :http) mail.to_s end end diff --git a/app/models/outgoing_message_prototype.rb b/app/models/outgoing_message_prototype.rb index 1cf1a1f7..c894917c 100644 --- a/app/models/outgoing_message_prototype.rb +++ b/app/models/outgoing_message_prototype.rb @@ -175,7 +175,7 @@ def raw_message content: attachment[:data] } end - mail.header["Received"] = "from #{@source_type} (#{resolved_hostname} [#{@ip}]) by Postal with HTTP; #{Time.now.utc.rfc2822}" + mail.header["Received"] = Postal::ReceivedHeader.generate(@server, @source_type, @ip, :http) mail.message_id = "<#{@message_id}>" mail.to_s end @@ -196,12 +196,4 @@ def create_message(address) { id: message.id, token: message.token } end - def resolved_hostname - @resolved_hostname ||= begin - Resolv.new.getname(@ip) - rescue StandardError - @ip - end - end - end diff --git a/app/views/servers/advanced.html.haml b/app/views/servers/advanced.html.haml index a4b429ff..7c8cdc2a 100644 --- a/app/views/servers/advanced.html.haml +++ b/app/views/servers/advanced.html.haml @@ -19,7 +19,11 @@ .fieldSet__input = f.select :allow_sender, [["No", false], ["Yes - can use Sender header", true]], {}, :class => 'input input--select' %p.fieldSet__text If enabled, outgoing messages can use any address in the From header as long as a Sender header is included with an authorized address. - + .fieldSet__field + = f.label :privacy_mode, "Privacy mode", :class => 'fieldSet__label' + .fieldSet__input + = f.select :privacy_mode, [["Disabled", false], ["Enabled", true]], {}, :class => 'input input--select' + %p.fieldSet__text If enabled, when Postal adds Received headers to e-mails it will not include IP or hostname information of the client submitting the message. .fieldSet__field = f.label :log_smtp_data, "Log SMTP data?", :class => 'fieldSet__label' .fieldSet__input diff --git a/db/migrate/20240206173036_add_privacy_mode_to_servers.rb b/db/migrate/20240206173036_add_privacy_mode_to_servers.rb new file mode 100644 index 00000000..77572193 --- /dev/null +++ b/db/migrate/20240206173036_add_privacy_mode_to_servers.rb @@ -0,0 +1,5 @@ +class AddPrivacyModeToServers < ActiveRecord::Migration[6.1] + def change + add_column :servers, :privacy_mode, :boolean, default: false + end +end diff --git a/db/schema.rb b/db/schema.rb index e815656c..a1b9e762 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -2,16 +2,17 @@ # of editing this file, please use the migrations feature of Active Record to # incrementally modify your database, and then regenerate this schema definition. # -# Note that this schema.rb definition is the authoritative source for your -# database schema. If you need to create the application database on another -# system, you should be using db:schema:load, not running all the migrations -# from scratch. The latter is a flawed and unsustainable approach (the more migrations -# you'll amass, the slower it'll run and the greater likelihood for issues). +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20_210_727_210_551) do - create_table "additional_route_endpoints", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| +ActiveRecord::Schema.define(version: 2024_02_06_173036) do + + create_table "additional_route_endpoints", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.integer "route_id" t.string "endpoint_type" t.integer "endpoint_id" @@ -19,7 +20,7 @@ t.datetime "updated_at", null: false end - create_table "address_endpoints", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "address_endpoints", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.integer "server_id" t.string "uuid" t.string "address" @@ -28,7 +29,7 @@ t.datetime "updated_at", null: false end - create_table "authie_sessions", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "authie_sessions", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "token" t.string "browser_id" t.integer "user_id" @@ -57,7 +58,7 @@ t.index ["user_id"], name: "index_authie_sessions_on_user_id" end - create_table "credentials", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "credentials", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.integer "server_id" t.string "key" t.string "type" @@ -70,7 +71,7 @@ t.string "uuid" end - create_table "domains", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "domains", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.integer "server_id" t.string "uuid" t.string "name" @@ -99,7 +100,7 @@ t.index ["uuid"], name: "index_domains_on_uuid", length: 8 end - create_table "http_endpoints", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "http_endpoints", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.integer "server_id" t.string "uuid" t.string "name" @@ -116,7 +117,7 @@ t.integer "timeout" end - create_table "ip_addresses", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "ip_addresses", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.integer "ip_pool_id" t.string "ipv4" t.string "ipv6" @@ -126,7 +127,7 @@ t.integer "priority" end - create_table "ip_pool_rules", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "ip_pool_rules", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "uuid" t.string "owner_type" t.integer "owner_id" @@ -137,7 +138,7 @@ t.datetime "updated_at", null: false end - create_table "ip_pools", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "ip_pools", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "name" t.string "uuid" t.datetime "created_at", precision: 6 @@ -146,14 +147,14 @@ t.index ["uuid"], name: "index_ip_pools_on_uuid", length: 8 end - create_table "organization_ip_pools", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "organization_ip_pools", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.integer "organization_id" t.integer "ip_pool_id" t.datetime "created_at", null: false t.datetime "updated_at", null: false end - create_table "organization_users", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "organization_users", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.integer "organization_id" t.integer "user_id" t.datetime "created_at", precision: 6 @@ -162,7 +163,7 @@ t.string "user_type" end - create_table "organizations", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "organizations", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "uuid" t.string "name" t.string "permalink" @@ -178,7 +179,7 @@ t.index ["uuid"], name: "index_organizations_on_uuid", length: 8 end - create_table "queued_messages", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "queued_messages", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.integer "server_id" t.integer "message_id" t.string "domain" @@ -197,7 +198,7 @@ t.index ["server_id"], name: "index_queued_messages_on_server_id" end - create_table "routes", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "routes", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "uuid" t.integer "server_id" t.integer "domain_id" @@ -212,7 +213,7 @@ t.index ["token"], name: "index_routes_on_token", length: 6 end - create_table "servers", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "servers", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.integer "organization_id" t.string "uuid" t.string "name" @@ -240,13 +241,14 @@ t.text "domains_not_to_click_track" t.string "suspension_reason" t.boolean "log_smtp_data", default: false + t.boolean "privacy_mode", default: false t.index ["organization_id"], name: "index_servers_on_organization_id" t.index ["permalink"], name: "index_servers_on_permalink", length: 6 t.index ["token"], name: "index_servers_on_token", length: 6 t.index ["uuid"], name: "index_servers_on_uuid", length: 8 end - create_table "smtp_endpoints", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "smtp_endpoints", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.integer "server_id" t.string "uuid" t.string "name" @@ -260,13 +262,13 @@ t.datetime "updated_at", precision: 6 end - create_table "statistics", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "statistics", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.bigint "total_messages", default: 0 t.bigint "total_outgoing", default: 0 t.bigint "total_incoming", default: 0 end - create_table "track_certificates", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "track_certificates", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "domain" t.text "certificate" t.text "intermediaries" @@ -280,7 +282,7 @@ t.index ["domain"], name: "index_track_certificates_on_domain", length: 8 end - create_table "track_domains", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "track_domains", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "uuid" t.integer "server_id" t.integer "domain_id" @@ -296,7 +298,7 @@ t.text "excluded_click_domains" end - create_table "user_invites", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "user_invites", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "uuid" t.string "email_address" t.datetime "expires_at", precision: 6 @@ -305,7 +307,7 @@ t.index ["uuid"], name: "index_user_invites_on_uuid", length: 12 end - create_table "users", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "users", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.string "uuid" t.string "first_name" t.string "last_name" @@ -323,14 +325,14 @@ t.index ["uuid"], name: "index_users_on_uuid", length: 8 end - create_table "webhook_events", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "webhook_events", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.integer "webhook_id" t.string "event" t.datetime "created_at", precision: 6 t.index ["webhook_id"], name: "index_webhook_events_on_webhook_id" end - create_table "webhook_requests", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "webhook_requests", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.integer "server_id" t.integer "webhook_id" t.string "url" @@ -343,7 +345,7 @@ t.datetime "created_at", precision: 6 end - create_table "webhooks", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", force: :cascade do |t| + create_table "webhooks", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.integer "server_id" t.string "uuid" t.string "name" @@ -356,4 +358,5 @@ t.datetime "updated_at", precision: 6 t.index ["server_id"], name: "index_webhooks_on_server_id" end + end diff --git a/lib/postal/received_header.rb b/lib/postal/received_header.rb new file mode 100644 index 00000000..4afbaf2d --- /dev/null +++ b/lib/postal/received_header.rb @@ -0,0 +1,43 @@ +module Postal + class ReceivedHeader + + OUR_HOSTNAMES = { + smtp: Postal.config.dns.smtp_server_hostname, + http: Postal.config.web.host + } + + class << self + + def generate(server, helo, ip_address, method) + our_hostname = OUR_HOSTNAMES[method] + if our_hostname.nil? + raise Error, "`method` is invalid (must be one of #{OUR_HOSTNAMES.join(', ')})" + end + + header = "by #{our_hostname} with #{method.to_s.upcase}; #{Time.now.utc.rfc2822}" + + if server.nil? || server.privacy_mode == false + hostname = resolve_hostname(ip_address) + header = "from #{helo} (#{hostname} [#{ip_address}]) #{header}" + end + + header + end + + private + + def resolve_hostname(ip_address) + Resolv::DNS.open do |dns| + dns.timeouts = [10, 5] + begin + dns.getname(ip_address) + rescue StandardError + ip_address + end + end + end + + end + + end +end diff --git a/lib/postal/smtp_server/client.rb b/lib/postal/smtp_server/client.rb index d6791856..5e3906da 100644 --- a/lib/postal/smtp_server/client.rb +++ b/lib/postal/smtp_server/client.rb @@ -103,17 +103,6 @@ def log(text) private - def resolve_hostname - Resolv::DNS.open do |dns| - dns.timeouts = [10, 5] - @hostname = begin - dns.getname(@ip_address) - rescue StandardError - @ip_address - end - end - end - def proxy(data) if m = data.match(/\APROXY (.+) (.+) (.+) (.+) (.+)\z/) @ip_address = m[2] @@ -143,7 +132,6 @@ def starttls end def ehlo(data) - resolve_hostname @helo_name = data.strip.split(" ", 2)[1] transaction_reset @state = :welcomed @@ -151,7 +139,6 @@ def ehlo(data) end def helo(data) - resolve_hostname @helo_name = data.strip.split(" ", 2)[1] transaction_reset @state = :welcomed @@ -377,10 +364,10 @@ def data(data) @headers = {} @receiving_headers = true - received_header_content = "from #{@helo_name} (#{@hostname} [#{@ip_address}]) by #{Postal.config.dns.smtp_server_hostname} with SMTP; #{Time.now.utc.rfc2822}".force_encoding("BINARY") - unless Postal.config.smtp_server.strip_received_headers? - @data << "Received: #{received_header_content}\r\n" - end + received_header = Postal::ReceivedHeader.generate(@credential&.server, @helo_name, @ip_address, :smtp) + .force_encoding("BINARY") + + @data << "Received: #{received_header_content}\r\n" @headers["received"] = [received_header_content] handler = proc do |data| diff --git a/spec/lib/postal/received_header_spec.rb b/spec/lib/postal/received_header_spec.rb new file mode 100644 index 00000000..541da807 --- /dev/null +++ b/spec/lib/postal/received_header_spec.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +require "rails_helper" + +describe Postal::ReceivedHeader do + before do + allow(Resolv::DNS).to receive(:open).and_return("hostname.com") + end + + describe ".generate" do + context "when server is nil" do + it "returns the correct string" do + result = described_class.generate(nil, "testhelo", "1.1.1.1", :smtp) + expect(result).to eq "from testhelo (hostname.com [1.1.1.1]) " \ + "by #{Postal.config.dns.smtp_server_hostname} " \ + "with SMTP; #{Time.now.utc.rfc2822}" + end + end + + context "when server is provided with privacy_mode=true" do + it "returns the correct string" do + server = Server.new(privacy_mode: true) + result = described_class.generate(server, "testhelo", "1.1.1.1", :smtp) + expect(result).to eq "by #{Postal.config.dns.smtp_server_hostname} " \ + "with SMTP; #{Time.now.utc.rfc2822}" + end + end + + context "when server is provided with privacy_mode=false" do + it "returns the correct string" do + server = Server.new(privacy_mode: false) + result = described_class.generate(server, "testhelo", "1.1.1.1", :smtp) + expect(result).to eq "from testhelo (hostname.com [1.1.1.1]) " \ + "by #{Postal.config.dns.smtp_server_hostname} " \ + "with SMTP; #{Time.now.utc.rfc2822}" + end + end + + context "when type is http" do + it "returns the correct string" do + result = described_class.generate(nil, "web-ui", "1.1.1.1", :http) + expect(result).to eq "from web-ui (hostname.com [1.1.1.1]) " \ + "by #{Postal.config.web.host} " \ + "with HTTP; #{Time.now.utc.rfc2822}" + end + end + end +end