bin/epafh
#!/usr/bin/env ruby
#require 'bundler/setup'
#Bundler.require
require 'pry'
require 'zlib'
require 'net/imap'
require 'pp'
require 'mechanize'
require 'yaml'
require 'hash_validator'
require 'uri'
require 'thor'
require 'json'
require 'mail'
require 'colorize'
#Net::IMAP.debug = true
class Hash
#take keys of hash and transform those to a symbols
def self.transform_keys_to_symbols(value)
return value if not value.is_a?(Hash)
hash = value.inject({}) do |memo,(k,v)|
memo[k.to_sym] = Hash.transform_keys_to_symbols(v); memo
end
return hash
end
end
module Epafh
EPAFI_CONFIG_FILE = File.join(ENV['HOME'],'.epafh','config.yml')
EPAFI_IGNORE_FILE = File.join(ENV['HOME'],'.epafh','ignore.yml')
class ContactManager
CRM_LOGIN_URL = '/login'
CRM_LEADS_URL = '/leads.json'
CRM_CONTACTS_URL = '/contacts.json'
def initialize config
@config = config
@browser = Mechanize.new { |agent|
agent.user_agent_alias = 'Mac Safari'
}
@ignore_list = Set.new
@keep_list = Set.new
## Load configuration file
#
unless File.exist? EPAFI_CONFIG_FILE then
raise "Unable to find configuration file #{EPAFI_CONFIG_FILE}"
end
@config = config
connect!
load_contacts
load_leads
load_ignore
#puts @keep_list.to_a
rescue RuntimeError => e
STDERR.puts e.message
end
def connect!
@browser.get(@config[:crm][:baseurl] + CRM_LOGIN_URL) do |page|
page.form_with(action: '/authentication') do |f|
f['authentication[username]'] = @config[:crm][:login]
f['authentication[password]'] = @config[:crm][:password]
end.click_button
end
rescue Mechanize::ResponseCodeError
raise "Authentication error. Verify your credentials."
end
def load_ignore
if File.exist? EPAFI_IGNORE_FILE
ignore_list = YAML.load_file(EPAFI_IGNORE_FILE)
ignore_list.each do |email|
@ignore_list << email.strip.downcase
end
end
end
def load_leads page=1
crm_leads_page = @browser.get(@config[:crm][:baseurl] + CRM_LEADS_URL + "?page=#{page}")
crm_leads = JSON.parse crm_leads_page.body
crm_leads.each do |lead_obj|
keep_contact lead_obj['lead']['email'].split(',')
keep_contact lead_obj['lead']['alt_email'].split(',')
end
if crm_leads.size > 0 then
load_leads (page + 1)
end
end
def load_contacts page=1
crm_contacts_page = @browser.get(@config[:crm][:baseurl] + CRM_CONTACTS_URL + "?page=#{page}")
crm_contacts = JSON.parse crm_contacts_page.body
crm_contacts.each do |contact_obj|
keep_contact contact_obj['contact']['email'].split(',')
keep_contact contact_obj['contact']['alt_email'].split(',')
end
if crm_contacts.size > 0 then
load_contacts (page + 1)
end
#contacts.to_a.sort.join(', ')
end
def keep_contact emails
emails = emails.to_a if emails.is_a? Set
[emails].flatten.each do |mail|
@keep_list << mail.strip.downcase
end
end
def ignore_contact emails
emails = emails.to_a if emails.is_a? Set
[emails].flatten.each do |mail|
@ignore_list << mail.strip.downcase
end
File.open(EPAFI_IGNORE_FILE, 'w') do |f|
f.write @ignore_list.to_a.to_yaml
end
end
def include? mail
return (
(@ignore_list.include? mail.strip.downcase) or
(@keep_list.include? mail.strip.downcase)
)
end
end
class CrawlerApp
attr_reader :imap
attr_reader :contacts
TMPMAIL_FILE = '.tmpmail'
def initialize config
@saved_key = 'RFC822'
@filter_headers = 'BODY[HEADER.FIELDS (FROM TO Subject)]'.upcase
@config = config
@imap = nil
@contact_manager = ContactManager.new config
end
def connect!
@imap = Net::IMAP.new(
@config[:imap][:server],
ssl: {verify_mode: OpenSSL::SSL::VERIFY_NONE},
port: 993
)
@imap.login(@config[:imap][:login], @config[:imap][:password])
#@imap.select(SOURCE_MAILBOX)
end
def disconnect!
imap.logout
imap.disconnect
end
MAIL_REGEXP = /\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}\b/
def examine_message message
m = Mail.read_from_string message.attr[@saved_key]
return if m.from.nil?
return if m.to.nil?
emails = Set.new
begin
emails.merge m.from
emails.merge [m.to].flatten if m.to
emails.merge [m.cc].flatten if m.cc
rescue => e
binding.pry
end
body_emails = Set.new
m.body.parts.each do |part|
next if part.content_type != 'text/plain'
#body_emails = m.body.decoded.scan MAIL_REGEXP
part_emails = part.decoded.scan MAIL_REGEXP
#pp body_emails
if not part_emails.empty? then
body_emails.merge part_emails
end
end
emails.merge body_emails
# puts emails.to_a.join(' , ')
remaining_emails = (
emails
.map{ |e| [e, (@contact_manager.include? e)] }
.select{ |e,t| !t }
)
seen_emails = (
remaining_emails
.empty?
)
# puts @contacts.to_a.join(', ')
if seen_emails then
print "."
return
else
puts ""
all_addr = {
from: (m.from || []),
to: (m.to || []),
cc: (m.cc || []),
body: (body_emails || [])
}
all_addr.each do |key, list|
list.each do |addr|
addr_str = if remaining_emails.map{|e,t| e}.include? addr then
addr.yellow.on_black
else addr
end
str = "%4s: %s" % [key.to_s.upcase, addr_str]
puts str
end
end
puts ""
#puts " ORIGINAL EMAILS: #{emails.to_a.join(', ')}"
#puts "REMAINING EMAILS: #{remaining_emails.map{|e,t| e}.join(', ')}".yellow.on_black
#puts " SEEN EMAILS: #{seen_emails}"
end
while true
begin
puts "\n### #{m.subject}"
print "#{m.from.join(',')} --> #{m.to.join(',')} "
puts "[Ignore/Add/Skip/Detail] ?"
i = STDIN.gets
case i.strip
when /^[iI]$/ then # ignore
@contact_manager.ignore_contact remaining_emails.map{|e,t| e}
break
when /^[aA]$/ then # add
@contact_manager.keep_contact remaining_emails.map{|e,t| e}
break
when /^[sS]$/ then #skip
break
when /^[dD]$/ then # decode
# puts m.body.decoded
File.open(TMPMAIL_FILE + ".2", 'w') do |f|
f.write message.attr[@saved_key]
end
system "formail < #{TMPMAIL_FILE}.2 > #{TMPMAIL_FILE}"
system "mutt -R -f #{TMPMAIL_FILE}"
end
rescue Encoding::ConverterNotFoundError
STDERR.puts "ERROR: encoding problem in email. Unable to convert."
end
end
return
end
def examine_all
@imap.list('', '*').each do |mailbox|
puts "\nMAILBOX #{mailbox.name}".yellow
next unless mailbox.name =~ /#{@config[:imap][:pattern]}/
@imap.examine mailbox.name
puts "Searching #{mailbox.name}"
messages_in_mailbox = @imap.responses['EXISTS'][0]
if not messages_in_mailbox then
say "#{mailbox.name} does not have any messages"
next
end
@imap.select mailbox.name #GYR: TEST
ids = @imap.search('SINCE 1-Jan-2001')
# NOT OR TO "@agilefant.org" CC "@agilefant.org"')
if ids.empty?
puts "\tFound no messages"
else
examine_message_list mailbox.name, ids
end
end
end
def examine_message_list mailbox_name, ids
ids.each do |id|
@imap.select mailbox_name #GYR: TEST
message = imap.fetch(id, [@saved_key])[0]
examine_message message
end
rescue IOError
# re-connect and try again
connect!
retry
end
end
class Crawler < Thor
CONFIG_FILE = 'config/secrey.yml'
include Thor::Actions
default_task :crawl
desc 'crawl', 'Crawls email to save mails'
def crawl
#saved_info = []
parse_configuration
## Run application
app = CrawlerApp.new @config
app.connect!
app.examine_all
#pp saved_info
app.disconnect!
end
def initialize *args
@config = {}
super
end
private
def parse_configuration
## Load configuration
@config.merge! Hash.transform_keys_to_symbols(
YAML::load( File.open( EPAFI_CONFIG_FILE ) )
)
## Validate configuration structure
validations = {
crm: {
baseurl: lambda { |url| url =~ URI::regexp },
login: 'string',
password: 'string'
},
imap: {
server: 'string',
login: 'string',
password: 'string'
}
}
validator = HashValidator.validate(@config, validations)
raise "Configuration is not valid: #{validator.errors.inspect}" unless validator.valid?
end
end
end
Epafh::Crawler.start