mysociety/alaveteli

View on GitHub
app/models/info_request.rb

Summary

Maintainability
F
1 wk
Test Coverage
# -*- encoding : utf-8 -*-
# == Schema Information
#
# Table name: info_requests
#
#  id                                    :integer          not null, primary key
#  title                                 :text             not null
#  user_id                               :integer
#  public_body_id                        :integer          not null
#  created_at                            :datetime         not null
#  updated_at                            :datetime         not null
#  described_state                       :string(255)      not null
#  awaiting_description                  :boolean          default(FALSE), not null
#  prominence                            :string(255)      default("normal"), not null
#  url_title                             :text             not null
#  law_used                              :string(255)      default("foi"), not null
#  allow_new_responses_from              :string(255)      default("anybody"), not null
#  handle_rejected_responses             :string(255)      default("bounce"), not null
#  idhash                                :string(255)      not null
#  external_user_name                    :string(255)
#  external_url                          :string(255)
#  attention_requested                   :boolean          default(FALSE)
#  comments_allowed                      :boolean          default(TRUE), not null
#  info_request_batch_id                 :integer
#  last_public_response_at               :datetime
#  reject_incoming_at_mta                :boolean          default(FALSE), not null
#  rejected_incoming_count               :integer          default(0)
#  date_initial_request_last_sent_at     :date
#  date_response_required_by             :date
#  date_very_overdue_after               :date
#  last_event_forming_initial_request_id :integer
#  use_notifications                     :boolean
#  last_event_time                       :datetime
#

require 'digest/sha1'
require 'fileutils'

class InfoRequest < ActiveRecord::Base
  include AdminColumn
  include Rails.application.routes.url_helpers
  include AlaveteliPro::RequestSummaries
  include AlaveteliFeatures::Helpers

  @non_admin_columns = %w(title url_title)

  strip_attributes :allow_empty => true
  strip_attributes :only => [:title],
                   :replace_newlines => true, :collapse_spaces => true

  validates_presence_of :title, :message => N_("Please enter a summary of your request")
  validates_format_of :title, :with => /[[:alpha:]]/,
    :message => N_("Please write a summary with some text in it"),
    :unless => Proc.new { |info_request| info_request.title.blank? }
  validates :title, :length => {
    :maximum => 200,
    :message => _('Please keep the summary short, like in the subject of an ' \
                  'email. You can use a phrase, rather than a full sentence.')
  }
  validates :title, :length => {
    :minimum => 3,
    :message => _('Summary is too short. Please be a little more descriptive ' \
                  'about the information you are asking for.'),
    :unless => Proc.new { |info_request| info_request.title.blank? },
    :on => :create
  }

  belongs_to :user,
             :inverse_of => :info_requests,
             :counter_cache => true

  validate :must_be_internal_or_external

  belongs_to :public_body,
             :inverse_of => :info_requests,
             :counter_cache => true
  belongs_to :info_request_batch,
             :inverse_of => :info_requests

  validates_presence_of :public_body_id, :message => N_("Please select an authority"),
                                         :unless => Proc.new { |info_request| info_request.is_batch_request_template? }

  has_many :info_request_events,
           -> { order('created_at, id') },
           :inverse_of => :info_request,
           :dependent => :destroy
  has_many :outgoing_messages,
           -> { order('created_at') },
           :inverse_of => :info_request,
           :dependent => :destroy
  has_many :incoming_messages,
           -> { order('created_at') },
           :inverse_of => :info_request,
           :dependent => :destroy
  has_many :user_info_request_sent_alerts,
           :inverse_of => :info_request,
           :dependent => :destroy
  has_many :track_things,
           -> { order('created_at desc') },
           :inverse_of => :info_request,
           :dependent => :destroy
  has_many :widget_votes,
           :inverse_of => :info_request,
           :dependent => :destroy
  has_many :comments,
           -> { order('created_at') },
           :inverse_of => :info_request,
           :dependent => :destroy
  has_many :censor_rules,
           -> { order('created_at desc') },
           :inverse_of => :info_request,
           :dependent => :destroy
  has_many :mail_server_logs,
           -> { order('mail_server_log_done_id, "order"') },
           :inverse_of => :info_request,
           :dependent => :destroy
  has_one :embargo,
          :inverse_of => :info_request,
          :class_name => "AlaveteliPro::Embargo"

  attr_accessor :is_batch_request_template
  attr_reader :followup_bad_reason

  has_tag_string

  scope :pro, ProQuery.new
  scope :is_public, Prominence::PublicQuery.new
  scope :is_searchable, Prominence::SearchableQuery.new
  scope :embargoed, Prominence::EmbargoedQuery.new
  scope :not_embargoed, Prominence::NotEmbargoedQuery.new
  scope :embargo_expiring, Prominence::EmbargoExpiringQuery.new
  scope :embargo_expired_today, Prominence::EmbargoExpiredTodayQuery.new
  scope :visible_to_requester, Prominence::VisibleToRequesterQuery.new
  scope :been_published, Prominence::BeenPublishedQuery.new

  scope :awaiting_response, State::AwaitingResponseQuery.new
  scope :response_received, State::ResponseReceivedQuery.new
  scope :clarification_needed, State::ClarificationNeededQuery.new
  scope :complete, State::CompleteQuery.new
  scope :other, State::OtherQuery.new
  scope :overdue, State::OverdueQuery.new
  scope :very_overdue, State::VeryOverdueQuery.new

  class << self
    alias_method :in_progress, :awaiting_response
  end
  scope :action_needed, State::ActionNeededQuery.new
  scope :updated_before, ->(ts) { where('"info_requests"."updated_at" < ?', ts) }

  # user described state (also update in info_request_event, admin_request/edit.rhtml)
  validate :must_be_valid_state
  validates_inclusion_of :prominence, :in => Prominence::VALUES

  validates_inclusion_of :law_used, :in => [
    'foi', # Freedom of Information Act
    'eir', # Environmental Information Regulations
  ]

  # who can send new responses
  validates_inclusion_of :allow_new_responses_from, :in => [
    'anybody', # anyone who knows the request email address
    'authority_only', # only people from authority domains
    'nobody'
  ]
  # what to do with refused new responses
  validates_inclusion_of :handle_rejected_responses, :in => [
    'bounce', # return them to sender
    'holding_pen', # put them in the holding pen
    'blackhole' # just dump them
  ]

  # only check on create, so existing models with mixed case are allowed
  validate :title_formatting, :on => :create

  after_initialize :set_defaults
  after_save :update_counter_cache
  after_destroy :update_counter_cache
  after_update :reindex_some_request_events
  before_destroy :expire
  # make sure the url_title is unique but don't update
  # existing requests unless the title is being changed
  before_save :update_url_title,
    :if => Proc.new { |request| request.title_changed? }
  before_validation :compute_idhash
  before_create :set_use_notifications

  # Subset of states accepted via the API
  def self.allowed_incoming_states
    [
      'waiting_response',
      'rejected',
      'successful',
      'partially_successful'
    ]
  end

  # Possible reasons that a request could be reported for administrator attention
  def report_reasons
    [_("Contains defamatory material"),
     _("Not a valid request"),
     _("Request for personal information"),
     _("Contains personal information"),
     _("Vexatious"),
     _("Other")]
  end


  # Public: Overrides the ActiveRecord attribute accessor
  #
  # opts = Hash of options (default: {})
  #        :decorate - Wrap the string in a ProminenceCalculator decorator that
  #        has methods indicating whether the InfoRequest is public, searchable
  #        etc.
  # Returns a String or ProminenceCalculator
  def prominence(opts = {})
    decorate = opts.fetch(:decorate, false)
    if decorate
      Prominence::Calculator.new(self)
    else
      read_attribute(:prominence)
    end
  end

  # opts = Hash of options (default: {})
  # Returns a StateCalculator
  def state(opts = {})
    State::Calculator.new(self)
  end

  def must_be_valid_state
    unless State.all.include?(described_state)
      errors.add(:described_state, "is not a valid state")
    end
  end

  def is_batch_request_template?
    is_batch_request_template == true
  end

  def indexed_by_search?
    prominence(:decorate => true).is_searchable?
  end

  # The request must either be internal, in which case it has
  # a foreign key reference to a User object and no external_url or external_user_name,
  # or else be external in which case it has no user_id but does have an external_url,
  # and may optionally also have an external_user_name.
  #
  # External requests are requests that have been added using the API, whereas internal
  # requests are requests made using the site.
  def must_be_internal_or_external
    # We must permit user_id and external_user_name both to be nil, because the system
    # allows a request to be created by a non-logged-in user.
    if user_id
      errors.add(:external_user_name, "must be null for an internal request") unless external_user_name.nil?
      errors.add(:external_url, "must be null for an internal request") unless external_url.nil?
    end
  end

  def is_external?
    external_url.nil? ? false : true
  end

  def user_name
    is_external? ? external_user_name : user.name
  end

  def user_name_slug
    if is_external?
      if external_user_name.nil?
        fake_slug = "anonymous"
      else
        fake_slug = MySociety::Format.simplify_url_part(external_user_name, 'external_user', 32)
      end
      (public_body.url_name || "") + "_" + fake_slug
    else
      user.url_name
    end
  end

  def user_json_for_api
    is_external? ? { :name => user_name || _("Anonymous user") } : user.json_for_api
  end

  @@custom_states_loaded = false
  begin
    require 'customstates'
    include InfoRequestCustomStates
    @@custom_states_loaded = true
  rescue MissingSourceFile, NameError
  end

  def self.custom_states_loaded
    @@custom_states_loaded
  end

  OLD_AGE_IN_DAYS = 21.days

  # If the URL name has changed, then all request: queries will break unless
  # we update index for every event. Also reindex if prominence changes.
  def reindex_some_request_events
    if changes.include?('url_title') || changes.include?('prominence') || changes.include?('user_id')
      reindex_request_events
    end
  end

  def reindex_request_events
    for info_request_event in info_request_events
      info_request_event.xapian_mark_needs_index
    end
  end

  # Force reindex when tag string changes
  alias_method :orig_tag_string=, :tag_string=
  def tag_string=(tag_string)
    ret = self.orig_tag_string=(tag_string)
    reindex_request_events
    ret
  end

  def expire(options={})
    # Clear out cached entries, by removing files from disk (the built in
    # Rails fragment cache made doing this and other things too hard)
    foi_fragment_cache_directories.each{ |dir| FileUtils.rm_rf(dir) }

    # Remove any download zips
    FileUtils.rm_rf(download_zip_dir)

    # Remove the database caches of body / attachment text (the attachment text
    # one is after privacy rules are applied)
    clear_in_database_caches! unless options[:preserve_database_cache]

    # also force a search reindexing (so changed text reflected in search)
    reindex_request_events
  end

  # Removes anything cached about the object in the database, and saves
  def clear_in_database_caches!
    for incoming_message in incoming_messages
      incoming_message.clear_in_database_caches!
    end
  end

  # When name is changed, also change the url name
  def title=(title)
    write_attribute(:title, title)
    update_url_title
  end

  # Public: url_title attribute reader
  #
  # opts - Hash of options (default: {})
  #        :collapse - Set true to strip the numeric section. Use this to group
  #                    lots of similar requests by url_title.
  #
  # Returns a String
  def url_title(opts = {})
    _url_title = super()
    return _url_title.gsub(/[_0-9]+$/, "") if opts[:collapse]
    _url_title
  end

  def update_url_title
    return unless title
    url_title = MySociety::Format.simplify_url_part(title, 'request', 32)
    # For request with same title as others, add on arbitary numeric identifier
    unique_url_title = url_title
    suffix_num = 2 # as there's already one without numeric suffix
    conditions = id ? ["id <> ?", id] : []

    while InfoRequest.
      where(:url_title => unique_url_title).
        where(conditions).
          any? do
      unique_url_title = "#{url_title}_#{suffix_num}"
      suffix_num = suffix_num + 1
    end

    write_attribute(:url_title, unique_url_title)
  end

  def update_last_public_response_at
    last_public_event = get_last_public_response_event
    if last_public_event
      self.last_public_response_at = last_public_event.created_at
    else
      self.last_public_response_at = nil
    end
    save
  end

  # Remove spaces from ends (for when used in emails etc.)
  # Needed for legacy reasons, even though we call strip_attributes now
  def title
    _title = read_attribute(:title)
    if _title
      _title.strip!
    end
    _title
  end

  # Email which public body should use to respond to request. This is in
  # the format PREFIXrequest-ID-HASH@DOMAIN. Here ID is the id of the
  # FOI request, and HASH is a signature for that id.
  def incoming_email
    magic_email("request-")
  end

  def incoming_name_and_email
    MailHandler.address_from_name_and_email(user_name, incoming_email)
  end

  # Subject lines for emails about the request
  def email_subject_request(opts = {})
    html = opts.fetch(:html, true)
    _('{{law_used_full}} request - {{title}}',
      :law_used_full => law_used_human(:full),
      :title => (html ? title : title.html_safe))
  end

  def email_subject_followup(opts = {})
    incoming_message = opts.fetch(:incoming_message, nil)
    html = opts.fetch(:html, true)
    if incoming_message.nil? || !incoming_message.valid_to_reply_to? || !incoming_message.subject
      'Re: ' + email_subject_request(:html => html)
    else
      if incoming_message.subject.match(/^Re:/i)
        incoming_message.subject
      else
        'Re: ' + incoming_message.subject
      end
    end
  end

  def law_used_human(key = :full)
    begin
      applicable_law.fetch(key)
    rescue KeyError
      raise "Unknown key '#{key}' for '#{law_used}'"
    end
  end

  # Return info request corresponding to an incoming email address, or nil if
  # none found. Checks the hash to ensure the email came from the public body -
  # only they are sent the email address with the has in it. (We don't check
  # the prefix and domain, as sometimes those change, or might be elided by
  # copying an email, and that doesn't matter)
  def self.find_by_incoming_email(incoming_email)
    id, hash = InfoRequest._extract_id_hash_from_email(incoming_email)
    if hash_from_id(id) == hash
      # Not using find(id) because we don't exception raised if nothing found
      find_by_id(id)
    end
  end

  # Return list of info requests which *might* be right given email address
  # e.g. For the id-hash email addresses, don't match the hash.
  def self.guess_by_incoming_email(incoming_message)
    guesses = []
    # 1. Try to guess based on the email address(es)
    incoming_message.addresses.each do |address|
      id, hash = InfoRequest._extract_id_hash_from_email(address)
      guesses.push(InfoRequest.find_by_id(id))
      guesses.push(InfoRequest.find_by_idhash(hash))
    end
    guesses.compact.uniq
  end

  # Internal function used by find_by_magic_email and guess_by_incoming_email
  def self._extract_id_hash_from_email(incoming_email)
    # Match case insensitively, FOI officers often write Request with capital R.
    incoming_email = incoming_email.downcase

    # The optional bounce- dates from when we used to have separate emails for the envelope from.
    # (that was abandoned because councils would send hand written responses to them, not just
    # bounce messages)
    incoming_email =~ /request-(?:bounce-)?([a-z0-9]+)-([a-z0-9]+)/
    id = $1.to_i
    hash = $2

    if hash
      # Convert l to 1, and o to 0. FOI officers quite often retype the
      # email address and make this kind of error.
      hash.gsub!(/l/, "1")
      hash.gsub!(/o/, "0")
    end

    [id, hash]
  end

  # When constructing a new request, use this to check user hasn't double submitted.
  # TODO: could have a date range here, so say only check last month's worth of new requests. If somebody is making
  # repeated requests, say once a quarter for time information, then might need to do that.
  # TODO: this *should* also check outgoing message joined to is an initial
  # request (rather than follow up)
  def self.find_existing(title, public_body_id, body)
    conditions = { :title => title,
                   :public_body_id => public_body_id,
                   :outgoing_messages => { :body => body } }

    InfoRequest.
      includes(:outgoing_messages).
        where(conditions).
          references(:outgoing_messages).
            first
  end

  def find_existing_outgoing_message(body)
    # TODO: can add other databases here which have regexp_replace
    if ActiveRecord::Base.connection.adapter_name == "PostgreSQL"
      # Exclude spaces from the body comparison using regexp_replace
      outgoing_messages.where("regexp_replace(outgoing_messages.body,
                                              '[[:space:]]', '', 'g') =
                               regexp_replace(?, '[[:space:]]', '', 'g')",
                               body).first
    else
      # For other databases (e.g. SQLite) not the end of the world being space-sensitive for this check
      outgoing_messages.where("outgoing_messages.body = ?", body).first
    end
  end

  # Has this email already been received here? Based just on message id.
  def already_received?(email, raw_email_data)
    message_id = email.message_id
    if message_id.nil?
      raise "No message id for this message"
    end

    for im in incoming_messages
      if message_id == im.message_id
        return true
      end
    end

    false
  end

  def receive(email, raw_email_data, *args)
    defaults = { :override_stop_new_responses => false,
                 :rejected_reason => nil,
                 :source => :internal }

    opts = if args.first.is_a?(Hash)
      defaults.merge(args.shift)
    elsif args.empty?
      defaults
    else
      warn %q([DEPRECATION] InfoRequest#receive with these params will be
          removed in 0.31. It has been replaced a method which accepts
          an options hash as its third argument. :override_stop_new_responses
          and :rejected_reason are valid keys to that hash).squish
      arg_opts = {}
      arg_opts[:override_stop_new_responses] = args[0] unless args[0].nil?
      arg_opts[:rejected_reason] = args[1] unless args[1].nil?

      defaults.merge(arg_opts)
    end

    if receive_mail_from_source? opts[:source]
      # Is this request allowing responses?
      accepted =
        if opts[:override_stop_new_responses]
          true
        else
          accept_incoming?(email, raw_email_data)
        end

      if accepted
        incoming_message =
          create_response!(email, raw_email_data, opts[:rejected_reason])

        # Notify the user that a new response has been received, unless the
        # request is external
        unless is_external?
          if use_notifications?
            info_request_event = info_request_events.find_by(
              event_type: 'response',
              incoming_message_id: incoming_message.id
            )
            user.notify(info_request_event)
          else
            RequestMailer.new_response(self, incoming_message).deliver_now
          end
        end
      end
    end
  end

  # An annotation (comment) is made
  def add_comment(body, user)
    comment = Comment.new
    ActiveRecord::Base.transaction do
      comment.body = body
      comment.user = user
      comment.info_request = self
      comment.save!

      log_event("comment", { :comment_id => comment.id })
      save!
    end
    comment
  end

  # The "holding pen" is a special request which stores incoming emails whose
  # destination request is unknown.
  def self.holding_pen_request
    ir = InfoRequest.find_by_url_title("holding_pen")
    if ir.nil?
      ir = InfoRequest.new(
        :user => User.internal_admin_user,
        :public_body => PublicBody.internal_admin_body,
        :title => 'Holding pen',
        :described_state => 'waiting_response',
        :awaiting_description => false,
        :prominence  => 'hidden'
      )
      om = OutgoingMessage.new({
        :status => 'ready',
        :message_type => 'initial_request',
        :body => 'This is the holding pen request. It shows responses that were sent to invalid addresses, and need moving to the correct request by an adminstrator.',
        :last_sent_at => Time.zone.now,
        :what_doing => 'normal_sort'

      })
      ir.outgoing_messages << om
      om.info_request = ir
      ir.save!
      ir.log_event('sent', { :outgoing_message_id => om.id, :email => ir.public_body.request_email })
    end
    ir
  end

  # states which require administrator action (hence email administrators
  # when they are entered, and offer state change dialog to them)
  def self.requires_admin_states
    %w(requires_admin error_message attention_requested)
  end

  def requires_admin?
    self.class.requires_admin_states.include?(described_state)
  end

  # Report this request for administrator attention
  def report!(reason, message, user)
    ActiveRecord::Base.transaction do
      set_described_state('attention_requested', user, "Reason: #{reason}\n\n#{message}")
      self.attention_requested = true # tells us if attention has ever been requested
      save!
    end
  end

  # change status, including for last event for later historical purposes
  # described_state should always indicate the current state of the request, as described
  # by the request owner (or, in some other cases an admin or other user)
  def set_described_state(new_state, set_by = nil, message = "")
    old_described_state = described_state
    ActiveRecord::Base.transaction do
      self.awaiting_description = false
      last_event = info_request_events.last
      last_event.described_state = new_state

      self.described_state = new_state
      last_event.save!
      save!
    end

    calculate_event_states

    if requires_admin?
      # Check there is someone to send the message "from"
      if set_by && user
        RequestMailer.requires_admin(self, set_by, message).deliver_now
      end
    end

    unless set_by.nil? || is_actual_owning_user?(set_by) || described_state == 'attention_requested'
      RequestMailer.
        old_unclassified_updated(self).deliver_now unless is_external?
    end
  end

  # Work out what state to display for the request on the site. In addition to values of
  # self.described_state, can take these values:
  #   waiting_classification
  #   waiting_response_overdue
  #   waiting_response_very_overdue
  # (this method adds an assessment of overdueness with respect to the current time to 'waiting_response'
  # states, and will return 'waiting_classification' instead of the described_state if the
  # awaiting_description flag is set on the request).
  def calculate_status(cached_value_ok=false)
    if cached_value_ok && @cached_calculated_status
      return @cached_calculated_status
    end
    @cached_calculated_status = @@custom_states_loaded ? theme_calculate_status : base_calculate_status
  end

  def base_calculate_status
    return 'waiting_classification' if awaiting_description
    return described_state unless described_state == "waiting_response"
    # Compare by date, so only overdue on next day, not if 1 second late
    return 'waiting_response_very_overdue' if
    Time.zone.now.strftime("%Y-%m-%d") > date_very_overdue_after.strftime("%Y-%m-%d")
    return 'waiting_response_overdue' if
    Time.zone.now.strftime("%Y-%m-%d") > date_response_required_by.strftime("%Y-%m-%d")
    return 'waiting_response'
  end

  # 'described_state' can be populated on any info_request_event but is only
  # ever used in the process populating calculated_state on the
  # info_request_event (if it represents a response, outgoing message, edit
  # or status update), or previous response or outgoing message events for
  # the same request.

  # Fill in any missing event states for first response before a description
  # was made. i.e. We take the last described state in between two responses
  # (inclusive of earlier), and set it as calculated value for the earlier
  # response. Also set the calculated state for any initial outgoing message,
  # follow up, edit or status_update to the described state of that event.

  # Note that the calculated state of the latest info_request_event will
  # be used in latest_status based searches and should match the described_state
  # of the info_request.
  def calculate_event_states
    curr_state = nil
    for event in info_request_events.reverse
      event.xapian_mark_needs_index  # we need to reindex all events in order to update their latest_* terms
      if curr_state.nil?
        if event.described_state
          curr_state = event.described_state
        end
      end

      if curr_state && event.event_type == 'response'
        event.set_calculated_state!(curr_state)

        if event.last_described_at.nil? # TODO: actually maybe this isn't needed
          event.last_described_at = Time.zone.now
          event.save!
        end
        curr_state = nil
      elsif curr_state && (event.event_type == 'followup_sent' || event.event_type == 'sent') && event.described_state && (event.described_state == 'waiting_response' || event.described_state == 'internal_review')
        # Followups can set the status to waiting response / internal
        # review. Initial requests ('sent') set the status to waiting response.

        # We want to store that in calculated_state state so it gets
        # indexed.
        event.set_calculated_state!(event.described_state)

        # And we don't want to propagate it to the response itself,
        # as that might already be set to waiting_clarification / a
        # success status, which we want to know about.
        curr_state = nil
      elsif curr_state && (['edit', 'status_update'].include? event.event_type)
        # A status update or edit event should get the same calculated state as described state
        # so that the described state is always indexed (and will be the latest_status
        # for the request immediately after it has been described, regardless of what
        # other request events precede it). This means that request should be correctly included
        # in status searches for that status. These events allow the described state to propagate in
        # case there is a preceding response that the described state should be applied to.
        event.set_calculated_state!(event.described_state)
      end
    end
  end

  # Find last InfoRequestEvent which  was:
  # -- sent at all
  # -- OR the same message was resent
  # -- OR the public body requested clarification, and a follow up was sent
  def last_event_forming_initial_request
    info_request_event_id = read_attribute(:last_event_forming_initial_request_id)
    last_sent = if info_request_event_id
      InfoRequestEvent.find_by_id(info_request_event_id)
    else
      calculate_last_event_forming_initial_request
    end

    if last_sent.nil?
      raise "internal error, last_event_forming_initial_request gets nil for " \
            "request #{ id } outgoing messages count " \
            "#{ outgoing_messages.size } all events: " \
            "#{ info_request_events.to_yaml }"
    end

    last_sent
  end

  def calculate_last_event_forming_initial_request
    # TODO: This can be removed when last_event_forming_initial_request_id has
    # been populated for all requests

    expecting_clarification = false
    last_sent = nil
    info_request_events.each do |event|
      if event.described_state == 'waiting_clarification'
        expecting_clarification = true
      end

      if %w(sent resent followup_sent followup_resent).include?(event.event_type)
        if last_sent.nil?
          last_sent = event
        elsif event.event_type == 'resent'
          last_sent = event
        elsif expecting_clarification && event.event_type == 'followup_sent'
          # TODO: this needs to cope with followup_resent, which it doesn't.
          # Not really easy to do, and only affects cases where followups
          # were resent after a clarification.
          last_sent = event
          expecting_clarification = false
        end
      end
    end
    last_sent
  end

  # Log an event to the history of some things that have happened to this request
  def log_event(type, params, options = {})
    event = info_request_events.create!(:event_type => type, :params => params)
    set_due_dates(event) if event.resets_due_dates?
    if options[:created_at]
      event.update_column(:created_at, options[:created_at])
    end
    if !self.last_event_time || (event.created_at > self.last_event_time)
      self.update_column(:last_event_time, event.created_at)
    end
    event
  end

  def set_due_dates(sent_event)
    self.last_event_forming_initial_request_id = sent_event.id
    self.date_initial_request_last_sent_at = sent_event.created_at.to_date
    self.date_response_required_by = calculate_date_response_required_by
    self.date_very_overdue_after = calculate_date_very_overdue_after
    save!
  end

  # TODO: once date_initial_request_sent_at is populated for all
  # requests, this can be removed
  # The last time that the initial request was sent/resent
  def date_initial_request_last_sent_at
    date = read_attribute(:date_initial_request_last_sent_at)
    return date.to_date if date
    calculate_date_initial_request_last_sent_at
  end

  # TODO: once date_initial_request_sent_at is populated for all
  # requests, this can be removed
  def calculate_date_initial_request_last_sent_at
    last_sent = last_event_forming_initial_request
    last_sent.outgoing_message.last_sent_at.to_date
  end

  def late_calculator
    @late_calculator ||= DefaultLateCalculator.new
  end

  # TODO: once date_response_required_by is populated for all
  # requests, this can be removed
  def date_response_required_by
    date = read_attribute(:date_response_required_by)
    return date if date
    calculate_date_response_required_by
  end

  def calculate_date_response_required_by
    Holiday.due_date_from(date_initial_request_last_sent_at,
                          late_calculator.reply_late_after_days,
                          AlaveteliConfiguration.working_or_calendar_days)
  end

  # TODO: once date_very_overdue_after is populated for all
  # requests, this can be removed
  def date_very_overdue_after
    date = read_attribute(:date_very_overdue_after)
    return date if date
    calculate_date_very_overdue_after
  end

  def calculate_date_very_overdue_after
    Holiday.due_date_from(date_initial_request_last_sent_at,
                          late_calculator.reply_very_late_after_days,
                          AlaveteliConfiguration.working_or_calendar_days)
  end

  def last_embargo_set_event
    info_request_events.
      where(:event_type => 'set_embargo').
        reorder('created_at DESC').
          first
  end

  def last_embargo_expire_event
    info_request_events.
      where(:event_type => 'expire_embargo').
        reorder('created_at DESC').
          first
  end

  # Where the initial request is sent to
  def recipient_email
    public_body.request_email
  end

  def recipient_email_valid_for_followup?
    public_body.is_followupable?
  end

  def recipient_name_and_email
    MailHandler.address_from_name_and_email(
      # TRANSLATORS: Please don't use double quotes (") in this translation
      # or it will break the site's ability to send emails to authorities!
      _("{{law_used}} requests at {{public_body}}",
        :law_used => law_used_human(:short),
        :public_body => public_body.short_or_long_name),
        recipient_email)
  end


  def public_response_events
    condition = <<-SQL
        info_request_events.event_type = ?
        AND incoming_messages.prominence = ?
    SQL

    info_request_events.
      joins(:incoming_message).
      where(condition, 'response', 'normal')
  end

  # The last public response is the default one people might want to reply to
  def get_last_public_response_event_id
    get_last_public_response_event.id if get_last_public_response_event
  end

  def get_last_public_response_event
    public_response_events.last
  end

  def get_last_public_response
    get_last_public_response_event.incoming_message if get_last_public_response_event
  end

  def public_outgoing_events
    info_request_events.select{|e| e.outgoing? && e.outgoing_message.is_public? }
  end

  # The last public outgoing message
  def get_last_public_outgoing_event
    public_outgoing_events.last
  end

  # Text from the the initial request, for use in summary display
  def initial_request_text
    return '' if outgoing_messages.empty?
    body_opts = { :censor_rules => applicable_censor_rules }
    first_message = outgoing_messages.first
    first_message.is_public? ? first_message.get_text_for_indexing(true, body_opts) : ''
  end

  # Returns index of last event which is described or nil if none described.
  def index_of_last_described_event
    events = info_request_events
    events.each_index do |i|
      revi = events.size - 1 - i
      m = events[revi]
      return revi if m.described_state
    end
    nil
  end

  def last_event_id_needing_description
    last_event = events_needing_description[-1]
    last_event.nil? ? 0 : last_event.id
  end

  # Returns all the events which the user hasn't described yet - an empty array if all described.
  def events_needing_description
    events = info_request_events
    i = index_of_last_described_event
    if i.nil?
      events
    else
      events[i + 1, events.size]
    end
  end

  # Returns last event
  def get_last_event
    events = info_request_events
    if events.size == 0
      return nil
    else
      return events[-1]
    end
  end

  def last_update_hash
    Digest::SHA1.hexdigest(info_request_events.last.created_at.to_i.to_s + updated_at.to_i.to_s)
  end

  # Get previous email sent to
  def get_previous_email_sent_to(info_request_event)
    last_email = nil
    for e in info_request_events
      if ((info_request_event.is_sent_sort? && e.is_sent_sort?) || (info_request_event.is_followup_sort? && e.is_followup_sort?)) && e.outgoing_message_id == info_request_event.outgoing_message_id
        if e.id == info_request_event.id
          break
        end
        last_email = e.params[:email]
      end
    end
    last_email
  end

  # Display version of status
  def self.get_status_description(status)
    descriptions = {
      'waiting_classification'        => _("Awaiting classification."),
      'waiting_response'              => _("Awaiting response."),
      'waiting_response_overdue'      => _("Delayed."),
      'waiting_response_very_overdue' => _("Long overdue."),
      'not_held'                      => _("Information not held."),
      'rejected'                      => _("Refused."),
      'partially_successful'          => _("Partially successful."),
      'successful'                    => _("Successful."),
      'waiting_clarification'         => _("Waiting clarification."),
      'gone_postal'                   => _("Handled by post."),
      'internal_review'               => _("Awaiting internal review."),
      'error_message'                 => _("Delivery error"),
      'requires_admin'                => _("Unusual response."),
      'attention_requested'           => _("Reported for administrator attention."),
      'user_withdrawn'                => _("Withdrawn by the requester."),
      'vexatious'                     => _("Considered by administrators as vexatious and hidden from site."),
      'not_foi'                       => _("Considered by administrators as not an FOI request and hidden from site."),
    }
    if descriptions[status]
      descriptions[status]
    elsif respond_to?(:theme_display_status)
      theme_display_status(status)
    else
      raise _("unknown status {{status}}", :status => status)
    end
  end

  def display_status(cached_value_ok=false)
    InfoRequest.get_status_description(calculate_status(cached_value_ok))
  end

  # Called by incoming_email - and used to be called to generate separate
  # envelope from address until we abandoned it.
  def magic_email(prefix_part)
    raise "id required to create a magic email" if not id
    InfoRequest.magic_email_for_id(prefix_part, id)
  end

  def self.magic_email_for_id(prefix_part, id)
    magic_email = AlaveteliConfiguration::incoming_email_prefix
    magic_email += prefix_part + id.to_s
    magic_email += "-" + InfoRequest.hash_from_id(id)
    magic_email += "@" + AlaveteliConfiguration::incoming_email_domain
    magic_email
  end

  def compute_idhash
    self.idhash = InfoRequest.hash_from_id(id)
  end

  def self.create_from_attributes(info_request_atts, outgoing_message_atts, user=nil)
    info_request = new(info_request_atts)
    default_message_params = {
      :status => 'ready',
      :message_type => 'initial_request',
      :what_doing => 'normal_sort'
    }

    attrs = outgoing_message_atts.merge(default_message_params)

    if attrs.respond_to?(:permit)
      attrs.permit(:body, :what_doing, :status, :message_type, :what_doing)
    end

    outgoing_message = OutgoingMessage.new(attrs)
    info_request.outgoing_messages << outgoing_message
    outgoing_message.info_request = info_request
    info_request.user = user
    info_request
  end

  def self.from_draft(draft)
    info_request = new(:title => draft.title,
                       :user => draft.user,
                       :public_body => draft.public_body)
    info_request.outgoing_messages.new(:body => draft.body,
                                       :status => 'ready',
                                       :message_type => 'initial_request',
                                       :what_doing => 'normal_sort',
                                       :info_request => info_request)
    if draft.embargo_duration
      info_request.embargo = AlaveteliPro::Embargo.new(
        :embargo_duration => draft.embargo_duration,
        :info_request => info_request
      )
    end
    info_request
  end

  def self.hash_from_id(id)
    Digest::SHA1.hexdigest(id.to_s + AlaveteliConfiguration::incoming_email_secret)[0,8]
  end

  # Used to find when event last changed
  def self.last_event_time_clause(event_type=nil, join_table=nil, join_clause=nil)
    event_type_clause = ''
    event_type_clause = " AND info_request_events.event_type = '#{event_type}'" if event_type
    tables = ['info_request_events']
    tables << join_table if join_table
    join_clause = "AND #{join_clause}" if join_clause
    "(SELECT info_request_events.created_at
          FROM #{tables.join(', ')}
          WHERE info_request_events.info_request_id = info_requests.id
          #{event_type_clause}
          #{join_clause}
          ORDER BY created_at desc
          LIMIT 1)"
  end

  def self.where_old_unclassified(age_in_days=nil)
    age_in_days =
      if age_in_days
        age_in_days.days
      else
        OLD_AGE_IN_DAYS
      end

    where("awaiting_description = ?
          AND last_public_response_at < ?
          AND url_title != 'holding_pen'
          AND user_id IS NOT NULL",
          true, Time.zone.now - age_in_days)
  end

  def self.download_zip_dir
    File.join(Rails.root, "cache", "zips", "#{Rails.env}")
  end

  def foi_fragment_cache_directories
    # return stub path so admin can expire it
    directories = []
    path = File.join("request", request_dirs)
    foi_cache_path = File.expand_path(File.join(Rails.root, 'cache', 'views'))
    directories << File.join(foi_cache_path, path)
    AlaveteliLocalization.available_locales.each do |locale|
      directories << File.join(foi_cache_path, locale, path)
    end

    directories
  end

  def is_followupable?(incoming_message)
    if is_external?
      @followup_bad_reason = "external"
      false
    elsif !OutgoingMailer.is_followupable?(self, incoming_message)
      @followup_bad_reason = if public_body.is_requestable?
        "unexpected followupable inconsistency"
      else
        public_body.not_requestable_reason
      end
      false
    else
      @followup_bad_reason = nil
      true
    end
  end

  def postal_email
    if who_can_followup_to.size == 0
      public_body.request_email
    else
      who_can_followup_to[-1][1]
    end
  end

  def postal_email_name
    if who_can_followup_to.size == 0
      public_body.name
    else
      who_can_followup_to[-1][0]
    end
  end

  def request_dirs
    first_three_digits = id.to_s[0..2]
    File.join(first_three_digits.to_s, id.to_s)
  end

  def download_zip_dir
    File.join(InfoRequest.download_zip_dir, "download", request_dirs)
  end

  def make_zip_cache_path(user)
    # The zip file varies depending on user because it can include different
    # messages depending on whether the user can access hidden or
    # requester_only messages. We name it appropriately, so that every user
    # with the right permissions gets a file with only the right things in.
    cache_file_dir = File.join(InfoRequest.download_zip_dir,
                               "download",
                               request_dirs,
                               last_update_hash)
    cache_file_suffix = zip_cache_file_suffix(user)
    File.join(cache_file_dir, "#{url_title}#{cache_file_suffix}.zip")
  end

  def zip_cache_file_suffix(user)
    # Simple short circuit for requests where everything is public
    if all_correspondence_is_public?
      ""
    # If the user can view hidden things, they can view anything, so no need
    # to go any further
    elsif User.view_hidden?(user)
      "_hidden"
    # If the user can't view hidden things, but owns the request, they can
    # see more than the public, so they get requester_only
    elsif is_owning_user?(user)
      "_requester_only"
    # Everyone else can only see public stuff, which is the default case
    else
      ""
    end
  end

  def is_old_unclassified?
    !is_external? && awaiting_description && url_title != 'holding_pen' && get_last_public_response_event &&
      Time.zone.now > get_last_public_response_event.created_at + OLD_AGE_IN_DAYS
  end

  # List of incoming messages to followup, by unique email
  def who_can_followup_to(skip_message = nil)
    ret = []
    done = {}
    for incoming_message in incoming_messages.reverse
      if incoming_message == skip_message
        next
      end
      incoming_message.safe_mail_from

      next if ! incoming_message.is_public?

      email = OutgoingMailer.email_for_followup(self, incoming_message)
      name = OutgoingMailer.name_for_followup(self, incoming_message)

      if !done.include?(email.downcase)
        ret = ret + [[name, email, incoming_message.id]]
      end
      done[email.downcase] = 1
    end

    if !done.include?(public_body.request_email.downcase)
      ret = ret + [[public_body.name, public_body.request_email, nil]]
    end
    done[public_body.request_email.downcase] = 1

    ret.reverse
  end

  # Get the list of censor rules that apply to this request
  def applicable_censor_rules
    applicable_rules = [censor_rules, CensorRule.global]
    unless public_body.blank?
      applicable_rules << public_body.censor_rules
    end
    applicable_rules << user.censor_rules if user
    applicable_rules.flatten
  end

  def apply_censor_rules_to_text(text)
    applicable_censor_rules.
      reduce(text) { |text, rule| rule.apply_to_text(text) }
  end

  def apply_censor_rules_to_binary(text)
    applicable_censor_rules.
      reduce(text) { |text, rule| rule.apply_to_binary(text) }
  end

  def apply_masks(text, content_type)
    mask_options = { :censor_rules => applicable_censor_rules,
                     :masks => masks }
    AlaveteliTextMasker.apply_masks(text, content_type, mask_options)
  end

  # Masks we apply to text associated with this request convert email addresses
  # we know about into textual descriptions of them
  def masks
    masks = [{ :to_replace => incoming_email,
               :replacement =>  _('[FOI #{{request}} email]',
                                  :request => id.to_s) },
             { :to_replace => AlaveteliConfiguration::contact_email,
               :replacement => _("[{{site_name}} contact email]",
                                 :site_name => AlaveteliConfiguration::site_name)} ]
    if public_body.is_followupable?
      masks << { :to_replace => public_body.request_email,
                 :replacement => _("[{{public_body}} request email]",
                                   :public_body => public_body.short_or_long_name) }
    end
  end

  def is_owning_user?(user)
    return false unless user
    user.id == user_id || user.owns_every_request?
  end

  def is_actual_owning_user?(user)
    return false unless user
    user.id == user_id
  end

  def all_correspondence_is_public?
    prominence(:decorate => true).is_public? &&
      incoming_messages.all?{ |message| message.is_public? } &&
      outgoing_messages.all?{ |message| message.is_public? }
  end

  def self.reject_incoming_at_mta(options)
    query = InfoRequest.where(["updated_at < (now() -
                                interval ?)
                                AND allow_new_responses_from = 'nobody'
                                AND rejected_incoming_count >= ?
                                AND reject_incoming_at_mta = ?
                                AND url_title <> 'holding_pen'",
                                "#{options[:age_in_months]} months",
                                options[:rejection_threshold], false])
    yield query.pluck(:id) if block_given?

    if options[:dryrun]
      0
    else
      query.update_all(:reject_incoming_at_mta => true)
    end
  end

  # This is called from cron regularly.
  def self.stop_new_responses_on_old_requests
    old = AlaveteliConfiguration.restrict_new_responses_on_old_requests_after_months
    very_old = old * 2

    # 'old' months since last change to request, only allow new incoming
    # messages from authority domains
    InfoRequest
      .been_published
      .where(allow_new_responses_from: 'anybody')
      .where.not(url_title: 'holding_pen')
      .updated_before(old.months.ago.to_date)
      .update_all(allow_new_responses_from: 'authority_only')

    # 'very_old' months since last change to request, don't allow any new
    # incoming messages
    InfoRequest
      .been_published
      .where(allow_new_responses_from: %w[anybody authority_only])
      .where.not(url_title: 'holding_pen')
      .updated_before(very_old.months.ago.to_date)
      .update_all(allow_new_responses_from: 'nobody')
  end

  def json_for_api(deep)
    ret = {
      :id => id,
      :url_title => url_title,
      :title => title,
      :created_at => created_at,
      :updated_at => updated_at,
      :described_state => described_state,
      :display_status => display_status,
      :awaiting_description => awaiting_description,
      :prominence => prominence,
      :law_used => law_used,
      :tags => tag_array,

      # not sure we need to make these, mainly anti-spam, admin params public
      # allow_new_responses_from
      # handle_rejected_responses
    }

    if deep
      if user
        ret[:user] = user.json_for_api
      else
        ret[:user_name] = user_name
      end
      ret[:public_body] = public_body.json_for_api
      ret[:info_request_events] = info_request_events.map { |e| e.json_for_api(false) }
    end
    ret
  end

  # This method updates the count columns of the PublicBody that
  # store the number of "not held", "to some extent successful" and
  # "both visible and classified" requests when saving or destroying
  # an InfoRequest associated with the body:
  def update_counter_cache(body = public_body)
    success_states = ['successful', 'partially_successful']
    basic_params = {
      :public_body_id => body.id,
    }
    [['info_requests_not_held_count', { :awaiting_description => false,
                                        :described_state => 'not_held' }],
     ['info_requests_successful_count', { :awaiting_description => false,
                                          :described_state => success_states }],
     ['info_requests_visible_classified_count', { :awaiting_description => false }],
     ['info_requests_visible_count', {}]].each do |column, extra_params|
       params = basic_params.clone.update extra_params
       body.send "#{column}=", InfoRequest.where(params).is_searchable.count
     end
     body.without_revision do
       body.no_xapian_reindex = true
       body.save(validate: false)
     end
  end

  def similar_cache_key
    "request/similar/#{id}"
  end

  # Get requests that have similar important terms
  def similar_requests(limit=10)
    ids, more = similar_ids(limit)
    [InfoRequest.includes(:public_body => :translations).find(ids), more]
  end

  # Get the ids of similar requests, and whether there are more
  def similar_ids(limit=10)
    Rails.cache.fetch(similar_cache_key, expires_in: 3.days) do
      ids = []
      xapian_similar_more = false
      begin
        xapian_similar =
          ActsAsXapian::Similar.new([InfoRequestEvent],
                                    info_request_events,
                                    :limit => limit,
                                    :collapse_by_prefix => 'request_collapse')
        xapian_similar_more = (xapian_similar.matches_estimated > limit)
        ids = xapian_similar.results.map do |result|
          result[:model].info_request_id
        end
      rescue
      end
      [ids, xapian_similar_more]
    end
  end

  def self.request_list(filters, page, per_page, max_results)
    query = InfoRequestEvent.make_query_from_params(filters)
    search_options = {
      :limit => 25,
      :offset => (page - 1) * per_page,
      :collapse_by_prefix => 'request_collapse' }

    xapian_object = search_events(query, search_options)
    list_results = xapian_object.results.map { |r| r[:model] }
    matches_estimated = xapian_object.matches_estimated
    show_no_more_than = [matches_estimated, max_results].min
    return { :results => list_results,
             :matches_estimated => matches_estimated,
             :show_no_more_than => show_no_more_than }
  end

  def self.recent_requests
    request_events = []
    request_events_all_successful = false
    # Get some successful requests
    begin
      query = 'variety:response (status:successful OR status:partially_successful)'
      max_count = 5
      search_options = {
        :limit => max_count,
        :collapse_by_prefix => 'request_title_collapse' }

      xapian_object = search_events(query, search_options)
      xapian_object.results
      request_events = xapian_object.results.map { |r| r[:model] }

      # If there are not yet enough successful requests, fill out the list with
      # other requests
      if request_events.count < max_count
        query = 'variety:sent'
        search_options[:limit] = max_count-request_events.count
        xapian_object = search_events(query, search_options)
        xapian_object.results
        more_events = xapian_object.results.map { |r| r[:model] }
        request_events += more_events
        # Overall we still want the list sorted with the newest first
        request_events.sort!{|e1,e2| e2.created_at <=> e1.created_at}
      else
        request_events_all_successful = true
      end
    rescue
      request_events = []
    end

    [request_events, request_events_all_successful]
  end

  def self.find_in_state(state)
    where(:described_state => state).
      order('last_event_time')
  end

  def move_to_public_body(destination_public_body, opts = {})
    return nil unless destination_public_body.try(:persisted?)
    old_body = public_body
    editor = opts.fetch(:editor)

    attrs = { :public_body => destination_public_body }

    if destination_public_body
      attrs.merge!({
        :law_used => destination_public_body.law_only_short.downcase
      })
    end

    return_val = if update_attributes(attrs)
      log_event('move_request',
                :editor => editor,
                :public_body_url_name => public_body.url_name,
                :old_public_body_url_name => old_body.url_name)

      reindex_request_events

      public_body
    end

    # HACK: Manually reset counter caches
    # https://github.com/rails/rails/issues/10865
    old_body.class.reset_counters(old_body.id, :info_requests)
    update_counter_cache(old_body)
    public_body.class.reset_counters(public_body.id, :info_requests)
    update_counter_cache
    return_val
  end

  def move_to_user(destination_user, opts = {})
    return nil unless destination_user.try(:persisted?)
    old_user = user
    editor = opts.fetch(:editor)

    return_val = if update_attributes(:user => destination_user)
      log_event('move_request',
                :editor => editor,
                :user_url_name => user.url_name,
                :old_user_url_name => old_user.url_name)

      reindex_request_events

      user
    end

    # HACK: Manually reset counter caches
    # https://github.com/rails/rails/issues/10865
    old_user.class.reset_counters(old_user.id, :info_requests)
    user.class.reset_counters(user.id, :info_requests)
    return_val
  end

  def self.log_overdue_events
    log_overdue_event_type('overdue')
  end

  def self.log_very_overdue_events
    log_overdue_event_type('very_overdue')
  end

  # Is the attached embargo expiring soon?
  #
  # Returns boolean
  def embargo_expiring?
    embargo ? embargo.expiring_soon? : false
  end

  # Has a previously attached embargo expired?
  #
  # Returns boolean
  def embargo_expired?
    if !embargo && last_embargo_expire_event
      true
    else
      false
    end
  end

  # Is the attached embargo still present but has reached its publication date
  #
  # Returns boolean
  def embargo_pending_expiry?
    embargo ? embargo.expired? : false
  end

  # @see RequestSummaries#should_summarise?
  def should_summarise?
    self.info_request_batch_id.blank?
  end

  # Requests in a batch should update their parent batch request when they
  # are updated.
  #
  # @see RequestSummaries#should_update_parent_summary?
  def should_update_parent_summary?
    self.info_request_batch_id.present?
  end

  # @see RequestSummaries#request_summary_parent
  def request_summary_parent
    if self.info_request_batch_id.blank?
      nil
    else
      self.info_request_batch
    end
  end

  # @see RequestSummaries#request_summary_body
  def request_summary_body
    self.outgoing_messages.any? ? self.outgoing_messages.first.body : ""
  end

  # @see RequestSummaries#request_summary_public_body_names
  def request_summary_public_body_names
    self.public_body.name unless self.public_body.blank?
  end

  # @see RequestSummaries#request_summary_categories
  def request_summary_categories
    categories = []
    if self.embargo_expiring?
      categories << AlaveteliPro::RequestSummaryCategory.embargo_expiring
    end
    # A request with no events is in the process of being sent (probably
    # having been created within our tests rather than in real code) and will
    # error if we try to get the phase, skip it for now because it'll be saved
    # when it's sent and trigger this code again anyway.
    if self.last_event_forming_initial_request_id.present?
      phase_slug = self.state.phase.to_s
      phase = AlaveteliPro::RequestSummaryCategory.find_by(slug: phase_slug)
      categories << phase unless phase.blank?
    end
    categories
  end

  private

  def receive_mail_from_source?(source)
    if source == :internal
      true
    elsif feature_enabled?(:accept_mail_from_anywhere)
      true
    else
      if feature_enabled?(:accept_mail_from_poller, user)
        source == :poller
      else
        source == :mailin
      end
    end
  end

  def self.log_overdue_event_type(event_type)
    date_field = case event_type
    when 'overdue'
      'date_response_required_by'
    when 'very_overdue'
      'date_very_overdue_after'
    else
      raise ArgumentError("Event type #{event_type} not handled")
    end

    query =
      where(["awaiting_description = ?
              AND described_state = ?
              AND #{date_field} < ?
              AND (SELECT id
              FROM info_request_events
              WHERE info_request_id = info_requests.id
              AND event_type = ?
              AND created_at > info_requests.#{date_field})
              IS NULL",
              false,
              'waiting_response',
              Time.zone.today,
              event_type])

    query.find_each(:batch_size => 100) do |info_request|
      # Date to DateTime representing beginning of day
      created_at = info_request.send(date_field).beginning_of_day + 1.day
      event = info_request.log_event(event_type,
                                     { :event_created_at => Time.zone.now },
                                     { :created_at => created_at })
      if info_request.use_notifications?
        info_request.user.notify(event)
      end
    end

  end

  def accept_incoming?(email, raw_email_data)
    # See if new responses are prevented
    gatekeeper = ResponseGatekeeper.for(allow_new_responses_from, self)
    # Take action if the message looks like spam
    spam_checker = ResponseGatekeeper::SpamChecker.new

    # What rejected the email – the gatekeeper or the spam checker?
    response_rejector =
      if gatekeeper.allow?(email)
        if spam_checker.allow?(email)
          nil
        else
          spam_checker
        end
      else
        gatekeeper
      end

    # Figure out how to reject the mail if it was rejected
    response_rejection =
      if response_rejector
        ResponseRejection.
          for(response_rejector.rejection_action, self, email, raw_email_data)
      end

    will_be_rejected = (response_rejector && response_rejection) ? true : false
    if will_be_rejected && response_rejection.reject(response_rejector.reason)
      # update without changing the updated_at field
      self.update_column(:rejected_incoming_count, self.rejected_incoming_count.next)
      logger.info "Rejected incoming mail: #{ response_rejector.reason } request: #{ id }"
      false
    else
      true
    end
  end

  def create_response!(email, raw_email_data, rejected_reason = nil)
    incoming_message = incoming_messages.build

    # To avoid a deadlock when simultaneously dealing with two
    # incoming emails that refer to the same InfoRequest, we
    # lock the row for update.
    with_lock do
      # TODO: These are very tightly coupled
      raw_email = RawEmail.new
      incoming_message.raw_email = raw_email
      incoming_message.save!
      raw_email.data = raw_email_data
      raw_email.save!

      unless described_state == 'user_withdrawn'
        self.awaiting_description = true
      end

      params = { :incoming_message_id => incoming_message.id }
      params[:rejected_reason] = rejected_reason.to_s if rejected_reason
      log_event("response", params)

      save!
    end

    # for the "waiting_classification" index
    reindex_request_events

    incoming_message
  end

  def set_defaults
    begin
      if described_state.nil?
        self.described_state = 'waiting_response'
      end
    rescue ActiveModel::MissingAttributeError
      # this should only happen on Model.exists? call. It can be safely ignored.
      # See http://www.tatvartha.com/2011/03/activerecordmissingattributeerror-missing-attribute-a-bug-or-a-features/
    end

    # FOI or EIR?
    if new_record? && public_body && public_body.eir_only?
      self.law_used = 'eir'
    end
  end

  def set_use_notifications
    if use_notifications.nil?
      self.use_notifications = feature_enabled?(:notifications, user) && \
                               info_request_batch_id.present?
    end
    return true
  end

  def applicable_law
    begin
      TranslatedConstants.law_used_readable_data.fetch(law_used.to_sym)
    rescue KeyError
      raise "Unknown law used '#{law_used}'"
    end
  end

  def title_formatting
    return unless title
    unless MySociety::Validate.uses_mixed_capitals(title, 1) ||
      title_starts_with_number || title_is_acronym(6)
      errors.add(:title, _('Please write the summary using a mixture of capital and lower case letters. This makes it easier for others to read.'))
    end
    if title =~ /^(FOI|Freedom of Information)\s*requests?$/i
      errors.add(:title, _('Please describe more what the request is about in the subject. There is no need to say it is an FOI request, we add that on anyway.'))
    end
  end

  def title_is_acronym(max_length)
    title.upcase == title && title.length <= max_length && !title.include?(" ")
  end

  def title_starts_with_number
    title.include?(" ") && title.split(" ").first =~ /^\d+$/
  end

  def self.add_conditions_from_extra_params(params, extra_params)
    if extra_params[:conditions]
      condition_string = extra_params[:conditions].shift
      params[:conditions][0] += " AND #{condition_string}"
      params[:conditions] += extra_params[:conditions]
    end
  end
  private_class_method :add_conditions_from_extra_params

  def self.search_events(query, opts = {})
    defaults = {
      :offset => 0,
      :limit => 20,
      :sort_by_prefix => 'created_at',
      :sort_by_ascending => true
    }
    ActsAsXapian::Search.new([InfoRequestEvent], query, defaults.merge(opts))
  end
  private_class_method :search_events
end