lib/google_safe_browsing/response_helper.rb
module GoogleSafeBrowsing
class ResponseHelper
def self.parse_full_hash_response(response)
f = StringIO.new(response)
full_hashes = []
while(! f.eof? )
hash = {}
meta = f.gets.chomp.split(':')
hash[:list] = meta[0]
hash[:add_chunk_num] = meta[1]
hash[:full_hash] = f.read(meta[2].to_i).unpack('H*')[0]
full_hashes << hash
end
full_hashes
end
def self.parse_data_response(response)
ret = {}
ret[:lists] = []
ret[:data_urls] = {}
current_list = ''
response.split("\n").each do |line|
vals = line.split(':')
case vals[0]
when 'n'
ret[:delay_seconds] = vals[1].to_i
@delay_seconds = ret[:delay_seconds].to_i
when 'i'
ret[:lists] << vals[1]
current_list = vals[1]
ret[:data_urls][current_list] = []
when 'u'
ret[:data_urls][current_list] << vals[1]
when 'r'
# reset (delete all data and try again)
when 'ad'
# vals[1] is a CHUNKLIST representing add chunks to delete
# we can also delete the associated Shavar Hashes
# we no longer have to report that we received these chunks
AddShavar.delete_chunks_from_list(current_list,
ChunkList.new(vals[1]))
when 'sd'
# vals[1] is a CHUNKLIST representing sub chunks to delete
# we no longer have to report that we received these chunks
SubShavar.delete_chunks_from_list(current_list,
ChunkList.new(vals[1]))
else
GoogleSafeBrowsing.logger.warn \
"Unknown data response directive: #{vals[0]}"
end
end
ret
end
def self.receive_data(url, list)
urls = url.split(',')
url = urls[0]
mac = urls[1]
open(url) do |f|
body = f.read
unless mac.blank? || HttpHelper.valid_mac?(body, mac)
raise InvalidMACVerification
end
f.rewind
while(line = f.gets)
line_actions = parse_data_line(line)
chunk = f.read(line_actions[:chunk_length])
# f iterator is now set for next chunk
add_attrs = { chunk_number: line_actions[:chunk_number],
list: list,
prefix: nil,
host_key: nil
}
case line_actions[:action]
when 'a'
if line_actions[:chunk_length] == 0
record_add_shavar_to_insert(add_attrs)
else
chunk_iterator = chunk.bytes.to_enum
counter = 0
begin
while true
add_attrs[:host_key] = BinaryHelper.read_bytes_as_hex(chunk_iterator, 4)
count = chunk_iterator.next
if count > 0
count.times do |i|
add_attrs[:prefix] = BinaryHelper.read_bytes_as_hex(chunk_iterator, line_actions[:hash_length])
record_add_shavar_to_insert(add_attrs)
end
else
add_attrs[:prefix] = add_attrs[:host_key]
record_add_shavar_to_insert(add_attrs)
end
counter += 1
end
rescue StopIteration
GoogleSafeBrowsing.logger.info <<-LOG.gsub(/\n\s*/, '')
Added #{counter} host_keys for add chunk number
#{line_actions[:chunk_number]}
LOG
end
end
when 's'
sub_attrs = add_attrs.merge({ add_chunk_number: nil })
if line_actions[:chunk_length] == 0
record_sub_shavar_to_insert(sub_attrs)
else
chunk_iterator = chunk.bytes.to_enum
counter = 0
begin
while true
sub_attrs[:host_key] = BinaryHelper.read_bytes_as_hex(chunk_iterator, 4)
count = chunk_iterator.next
if count > 0
count.times do |i|
sub_attrs[:add_chunk_number] = BinaryHelper.unpack_add_chunk_num(BinaryHelper.read_bytes_from(chunk_iterator, 4))
sub_attrs[:prefix] = BinaryHelper.read_bytes_as_hex(chunk_iterator, line_actions[:hash_length])
record_sub_shavar_to_insert(sub_attrs)
end
else
sub_attrs[:add_chunk_number] = BinaryHelper.unpack_add_chunk_num(BinaryHelper.read_bytes_from(chunk_iterator, 4))
sub_attrs[:prefix] = sub_attrs[:host_key]
record_sub_shavar_to_insert(sub_attrs)
end
counter += 1
end
rescue StopIteration
GoogleSafeBrowsing.logger.info <<-LOG.gsub(/\n\s*/, '')
Added #{counter} host_keys for sub chunk number
#{line_actions[:chunk_number]}
LOG
end
end
else
GoogleSafeBrowsing.logger.info <<-LOG.gsub(/\n\s*/, '')
neither a nor s =================================================
LOG
end
end
end
while @add_shavar_values && @add_shavar_values.any?
AddShavar.connection.execute <<-SQL
INSERT INTO gsb_add_shavars (prefix, host_key, chunk_number, list)
VALUES #{pop_and_join @add_shavar_values}
SQL
end
while @sub_shavar_values && @sub_shavar_values.any?
SubShavar.connection.execute <<-SQL
INSERT INTO gsb_sub_shavars (prefix,
host_key,
add_chunk_number,
chunk_number,
list)
VALUES #{pop_and_join @sub_shavar_values}
SQL
end
FullHash.delete_subbed
@add_shavar_values = []
@sub_shavar_values = []
end
def self.record_add_shavar_to_insert(h)
@add_shavar_values ||= []
values = [h[:prefix], h[:host_key], h[:chunk_number], h[:list]]
@add_shavar_values << "(#{escape_and_join values})"
end
def self.record_sub_shavar_to_insert(h)
@sub_shavar_values ||= []
values = [h[:prefix],
h[:host_key],
h[:add_chunk_number],
h[:chunk_number],
h[:list]]
@sub_shavar_values << "(#{escape_and_join values})"
end
def self.parse_data_line(line)
split_line = line.split(':')
ret = {}
ret[:action] = split_line[0]
ret[:chunk_number] = split_line[1].to_i
ret[:hash_length] = split_line[2].to_i
ret[:chunk_length] = split_line[3].to_i
ret
end
def self.escape_and_join(values)
values.map do |v|
ActiveRecord::Base::sanitize(v)
end.join(', ')
end
def self.pop_and_join(records)
records.pop(10_000).join(', ')
end
end
end