app/models/outgoing_message.rb
# == Schema Information
# Schema version: 20230412084830
#
# Table name: outgoing_messages
#
# id :integer not null, primary key
# info_request_id :integer not null
# body :text not null
# status :string not null
# message_type :string not null
# created_at :datetime not null
# updated_at :datetime not null
# last_sent_at :datetime
# incoming_message_followup_id :integer
# what_doing :string not null
# prominence :string default("normal"), not null
# prominence_reason :text
# from_name :text
#
# models/outgoing_message.rb:
# A message, associated with a request, from the user of the site to somebody
# else. e.g. An initial request for information, or a complaint.
#
# Copyright (c) 2007 UK Citizens Online Democracy. All rights reserved.
# Email: hello@mysociety.org; WWW: http://www.mysociety.org/
class OutgoingMessage < ApplicationRecord
include MessageProminence
include Rails.application.routes.url_helpers
include LinkToHelper
include Taggable
STATUS_TYPES = %w(ready sent failed).freeze
MESSAGE_TYPES = %w(initial_request followup).freeze
WHAT_DOING_VALUES = %w(normal_sort
internal_review
external_review
new_information).freeze
# To override the default letter
attr_accessor :default_letter
before_validation :cache_from_name
validates_presence_of :info_request
validates_presence_of :from_name, unless: -> (m) { !m.info_request&.user }
validates_inclusion_of :status, in: STATUS_TYPES
validates_inclusion_of :message_type, in: MESSAGE_TYPES
validate :template_changed
validate :body_uses_mixed_capitals
validate :body_has_signature
validate :what_doing_value
belongs_to :info_request,
inverse_of: :outgoing_messages
belongs_to :incoming_message_followup,
inverse_of: :outgoing_message_followups,
foreign_key: 'incoming_message_followup_id',
class_name: 'IncomingMessage'
has_one :user,
inverse_of: :outgoing_messages,
through: :info_request
# can have many events, for items which were resent by site admin e.g. if
# contact address changed
has_many :info_request_events,
inverse_of: :outgoing_message,
dependent: :destroy
delegate :public_body, to: :info_request, private: true, allow_nil: true
after_initialize :set_default_letter
# reindex if body text is edited (e.g. by admin interface)
after_update :xapian_reindex_after_update
strip_attributes allow_empty: true
admin_columns include: [:to, :from, :subject]
default_url_options[:host] = AlaveteliConfiguration.domain
scope :followup, -> { where(message_type: 'followup') }
scope :is_searchable, -> { where(prominence: 'normal') }
def self.expected_send_errors
[ EOFError,
IOError,
Timeout::Error,
Errno::ECONNRESET,
Errno::ECONNABORTED,
Errno::EPIPE,
Errno::ETIMEDOUT,
Net::SMTPAuthenticationError,
Net::SMTPServerBusy,
Net::SMTPSyntaxError,
Net::SMTPUnknownError,
OpenSSL::SSL::SSLError ].concat(additional_send_errors)
end
def self.additional_send_errors
[]
end
def self.default_salutation(public_body)
_("Dear {{public_body_name}},", public_body_name: public_body.name)
end
def self.fill_in_salutation(text, public_body)
text.gsub(Template::BatchRequest.placeholder_salutation,
default_salutation(public_body))
end
def self.with_body(body)
# TODO: can add other databases here which have regexp_replace
if ActiveRecord::Base.connection.adapter_name == 'PostgreSQL'
# Exclude whitespace from the body comparison using regexp_replace
where("regexp_replace(outgoing_messages.body, '[[:space:]]', '', 'g') =
regexp_replace(?, '[[:space:]]', '', 'g')", body)
else
# For other databases (e.g. SQLite) not the end of the world being space-sensitive for this check
where(body: body)
end
end
def get_internal_review_insert_here_note
_("GIVE DETAILS ABOUT YOUR COMPLAINT HERE")
end
def get_default_message
letter_template.body(default_message_replacements)
end
def set_signature_name(name)
# We compare against raw_body as body strips linebreaks and applies
# censor rules
self.body = get_default_message + name if raw_body == get_default_message
end
def from_name
return info_request.external_user_name if info_request.is_external?
super || info_request.user_name
end
def safe_from_name
return info_request.external_user_name if info_request.is_external?
info_request.apply_censor_rules_to_text(from_name)
end
# Public: The value to be used in the From: header of an OutgoingMailer
# message.
#
# Returns a String
def from
info_request.incoming_name_and_email
end
# Public: The value to be used in the To: header of an OutgoingMailer message.
#
# Returns a String
def to
if replying_to_incoming_message?
# calling safe_from_name from so censor rules are run
MailHandler.address_from_name_and_email(incoming_message_followup.safe_from_name,
incoming_message_followup.from_email)
else
info_request.recipient_name_and_email
end
end
# Public: The value to be used in the Subject: header of an OutgoingMailer
# message.
#
# Returns a String
def subject
if message_type == 'followup'
if what_doing == 'internal_review'
_("Internal review of {{email_subject}}",
email_subject: info_request.email_subject_request(html: false))
else
info_request.
email_subject_followup(incoming_message: incoming_message_followup,
html: false)
end
else
info_request.email_subject_request(html: false)
end
end
# Public: The body text of the OutgoingMessage. The text is cleaned and
# CensorRules are applied.
#
# options - Hash of options
# :censor_rules - Array of CensorRules to apply. Defaults to the
# applicable_censor_rules of the associated
# InfoRequest. (optional)
#
# Returns a String
def body(options = {})
text = raw_body.dup
return text if text.nil?
text = clean_text(text)
# Use the given censor_rules; otherwise fetch them from the associated
# info_request
censor_rules = options.fetch(:censor_rules) do
info_request.try(:applicable_censor_rules) or []
end
censor_rules.reduce(text) { |t, rule| rule.apply_to_text(t) }
end
def raw_body
read_attribute(:body)
end
def apply_masks(text, content_type)
info_request.apply_masks(text, content_type)
end
# Used to give warnings when writing new messages
def contains_email?
MySociety::Validate.email_find_regexp.match(body)
end
def contains_postcode?
MySociety::Validate.contains_postcode?(body)
end
def is_owning_user?(user)
info_request.is_owning_user?(user)
end
# Without recording the send failure, parts of the public and admin
# interfaces for the request and authority may become inaccessible.
def record_email_failure(failure_reason)
self.last_sent_at = Time.zone.now
self.status = 'failed'
save!
info_request.log_event(
'send_error',
reason: failure_reason,
outgoing_message_id: id
)
set_info_request_described_state
end
def record_email_delivery(to_addrs, message_id, log_event_type = 'sent')
self.last_sent_at = Time.zone.now
self.status = 'sent'
save!
if message_type == 'followup'
log_event_type = "followup_#{ log_event_type }"
end
info_request.log_event(
log_event_type,
email: to_addrs,
outgoing_message_id: id,
smtp_message_id: message_id
)
set_info_request_described_state
end
def sendable?
if status == 'ready'
if message_type == 'initial_request'
true
elsif message_type == 'followup'
true
else
raise "Message id #{id} has type '#{message_type}' which cannot be sent"
end
elsif status == 'sent'
raise "Message id #{id} has already been sent"
else
raise "Message id #{id} not in state for sending"
end
end
# Public: Return logged Message-ID attributes for this OutgoingMessage.
# Note that these are not the MTA ID: https://en.wikipedia.org/wiki/Message-ID
#
# Returns an Array
def smtp_message_ids
info_request_events.
order(:created_at).
map { |event| event.params[:smtp_message_id] }.
compact.
map do |smtp_id|
smtp_id.match(/<(.*)>/) { |m| m.captures.first } || smtp_id
end
end
# Public: Return logged MTA IDs for this OutgoingMessage.
#
# Returns an Array
def mta_ids
case AlaveteliConfiguration.mta_log_type.to_sym
when :exim
exim_mta_ids
when :postfix
postfix_mta_ids
else
raise 'Unexpected MTA type'
end
end
# Public: Return the MTA logs for this message.
#
# Returns an Array.
def mail_server_logs
case AlaveteliConfiguration.mta_log_type.to_sym
when :exim
exim_mail_server_logs
when :postfix
postfix_mail_server_logs
else
raise 'Unexpected MTA type'
end
end
def delivery_status
# If the outgoing status is failed, we won't have mail logs, and know we can
# present a failed status to the end user.
if status == 'failed'
MailServerLog::DeliveryStatus.new(:failed)
else
mail_server_logs.map(&:delivery_status).compact.reject(&:unknown?).last ||
MailServerLog::DeliveryStatus.new(:unknown)
end
end
# An admin function
def prepare_message_for_resend
if MESSAGE_TYPES.include?(message_type) &&
(status == 'sent' || status == 'failed')
self.status = 'ready'
else
raise "Message id #{id} has type '#{message_type}' status " \
"'#{status}' which prepare_message_for_resend can't handle"
end
end
# Returns the text to quote the original message when sending this one
def quoted_part_to_append_to_email
if message_type == 'followup' && !incoming_message_followup.nil?
quoted = "\n\n-----Original Message-----\n\n"
quoted += incoming_message_followup.get_body_for_quoting
quoted += "\n"
else
""
end
end
# We hide emails from display in outgoing messages.
def remove_privacy_sensitive_things!(text)
text.gsub!(MySociety::Validate.email_find_regexp, "[email address]")
end
# Returns text for indexing / text display
def get_text_for_indexing(strip_salutation = true, opts = {})
if opts.empty?
text = body.strip
else
text = body(opts).strip
end
if strip_salutation && public_body
salutation = self.class.default_salutation(public_body)
text.sub!(/#{Regexp.escape(salutation)}\s*/, '')
end
# Remove email addresses from display/index etc.
remove_privacy_sensitive_things!(text)
text
end
# Return body for display as HTML
def get_body_for_html_display
text = body.strip
remove_privacy_sensitive_things!(text)
text = CGI.escapeHTML(text)
text = MySociety::Format.make_clickable(text, { contract: 1, nofollow: true })
text.gsub!(/\[(email address|mobile number)\]/, '[<a href="/help/officers#mobiles">\1</a>]')
text = ActionController::Base.helpers.simple_format(text)
text.html_safe
end
# Return body for display as text
def get_body_for_text_display
get_text_for_indexing(strip_salutation=false)
end
def xapian_reindex_after_update
return unless saved_change_to_attribute?(:body)
info_request_events.find_each(&:xapian_mark_needs_index)
end
def default_letter=(text)
original_default = get_default_message.clone
@default_letter = text
self.body = get_default_message if raw_body == original_default
end
private
def cache_from_name
return if read_attribute(:from_name)
self.from_name = info_request.user_name if info_request
end
def set_info_request_described_state
if status == 'failed'
info_request.set_described_state('error_message')
elsif message_type == 'initial_request'
info_request.set_described_state('waiting_response')
elsif message_type == 'followup'
if info_request.described_state == 'waiting_clarification'
info_request.set_described_state('waiting_response')
end
if what_doing == 'internal_review'
info_request.set_described_state('internal_review')
end
end
end
def set_default_letter
self.body = get_default_message if raw_body.nil?
end
def letter_template
@letter_template ||=
if what_doing == 'internal_review'
Template::InternalReview.new
elsif replying_to_incoming_message?
Template::IncomingMessageFollowup.new
else
Template::InitialRequest.new
end
end
def default_message_replacements
opts = {}
if info_request
opts[:url] = request_url(info_request) if info_request.url_title
opts[:info_request_title] = info_request.title if info_request.title
opts[:embargo] = true if info_request.embargo
end
opts[:public_body_name] =
if replying_to_incoming_message?
OutgoingMailer.
name_for_followup(info_request, incoming_message_followup)
else
public_body&.name
end
opts[:letter] = default_letter if default_letter
opts
end
def replying_to_incoming_message?
message_type == 'followup' &&
incoming_message_followup &&
incoming_message_followup.safe_from_name &&
incoming_message_followup.valid_to_reply_to?
end
def template_changed
if raw_body.empty? || HTMLEntities.new.decode(raw_body) =~
/\A#{template_regex(letter_template.body(default_message_replacements))}/
if message_type == 'followup'
if what_doing == 'internal_review'
errors.add(:body, _("Please give details explaining why you want a review"))
else
errors.add(:body, _("Please enter your follow up message"))
end
elsif
errors.add(:body, _("Please enter your letter requesting information"))
else
raise "Message id #{id} has type '#{message_type}' which validate can't handle"
end
end
end
def template_regex(template_text)
text = template_text.gsub("\r", "\n") # in case we have '\r\n' or even '\r's all the way down
# feels like this should need a gsub(/\//, '\/') but doesn't seem to
Regexp.escape(text.squeeze("\n")).
gsub("\\n", '\s*').
gsub('\ \s*', '\s*').
gsub('\s*\ ', '\s*')
end
def body_has_signature
if raw_body =~ /#{template_regex(letter_template.signoff(default_message_replacements))}\s*\Z/m
errors.add(:body, _("Please sign at the bottom with your name, or alter the \"{{signoff}}\" signature", signoff: letter_template.signoff(default_message_replacements)))
end
end
def body_uses_mixed_capitals
unless MySociety::Validate.uses_mixed_capitals(body)
errors.add(:body, _('Please write your message using a mixture of capital and lower case letters. This makes it easier for others to read.'))
end
end
def what_doing_value
if what_doing.nil? || !WHAT_DOING_VALUES.include?(what_doing)
errors.add(:what_doing_dummy, _('Please choose what sort of reply you are making.'))
end
end
def exim_mta_ids
lines = smtp_message_ids.map do |smtp_message_id|
info_request.
mail_server_logs.
where("line ILIKE :q", q: "%#{ smtp_message_id }%").
where("line ILIKE :marker", marker: "%<=%").
last.
try(:line)
end
lines.compact.map { |line| line[/\w{6}-\w{6}-\w{2}/].strip }.compact
end
def exim_mail_server_logs
logs = mta_ids.flat_map do |mta_id|
info_request.
mail_server_logs.
where('line ILIKE :mta_id', mta_id: "%#{ mta_id }%")
end
smarthost_mta_ids = logs.flat_map do |log|
line = log.line(decorate: true)
if line.delivery_status.try(:delivered?)
match = line.to_s.match(/C=".*?id=(?<message_id>\w+-\w+-\w+).*"/)
match[:message_id] if match
end
end
smarthost_mta_ids.compact!
smarthost_logs = smarthost_mta_ids.flat_map do |mta_id|
info_request.
mail_server_logs.
where('line ILIKE :mta_id', mta_id: "%#{ mta_id }%")
end
# Need to call #uniq because the more_logs query pulls out the initial
# delivery line
(logs + smarthost_logs).uniq
end
def postfix_mta_ids
lines = smtp_message_ids.map do |smtp_message_id|
info_request.
mail_server_logs.
where("line ILIKE :q", q: "%#{ smtp_message_id }%").
last.
try(:line)
end
lines.compact.map { |line| line.split(' ')[5].strip.chomp(':') }
end
def postfix_mail_server_logs
mta_ids.flat_map do |mta_id|
info_request.
mail_server_logs.
where('line ILIKE :mta_id', mta_id: "%#{ mta_id }%")
end
end
# remove excess linebreaks that unnecessarily space it out
def clean_text(text)
text.strip.gsub(/(?:\n\s*){2,}/, "\n\n")
end
end