lib/msf/core/auxiliary/http_crawler.rb
# -*- coding: binary -*-
module Msf
###
#
# This module provides methods for implementing a web crawler
#
###
module Auxiliary::HttpCrawler
include ::Msf::Auxiliary::Report
def initialize(info = {})
super
register_options(
[
Opt::RHOST,
Opt::RPORT(80),
OptString.new('VHOST', [ false, "HTTP server virtual host" ]),
OptString.new('URI', [ true, "The starting page to crawl", "/"]),
Opt::Proxies,
OptInt.new('MAX_PAGES', [ true, 'The maximum number of pages to crawl per URL', 500]),
OptInt.new('MAX_MINUTES', [ true, 'The maximum number of minutes to spend on each URL', 5]),
OptInt.new('MAX_THREADS', [ true, 'The maximum number of concurrent requests', 4]),
OptString.new('HttpUsername', [false, 'The HTTP username to specify for authentication']),
OptString.new('HttpPassword', [false, 'The HTTP password to specify for authentication']),
OptString.new('DOMAIN', [ true, 'The domain to use for windows authentication', 'WORKSTATION']),
OptBool.new('SSL', [ false, 'Negotiate SSL/TLS for outgoing connections', false])
], self.class
)
register_advanced_options(
[
OptBool.new('DirBust', [ false, 'Bruteforce common URL paths', true]),
OptInt.new('RequestTimeout', [false, 'The maximum number of seconds to wait for a reply', 15]),
OptInt.new('RedirectLimit', [false, 'The maximum number of redirects for a single request', 5]),
OptInt.new('RetryLimit', [false, 'The maximum number of attempts for a single request', 5]),
OptString.new('UserAgent', [true, 'The User-Agent header to use for all requests',
"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
]),
OptString.new('BasicAuthUser', [false, 'The HTTP username to specify for basic authentication']),
OptString.new('BasicAuthPass', [false, 'The HTTP password to specify for basic authentication']),
OptString.new('HTTPAdditionalHeaders', [false, "A list of additional headers to send (separated by \\x01)"]),
OptString.new('HTTPCookie', [false, "A HTTP cookie header to send with each request"]),
Opt::SSLVersion
], self.class
)
register_autofilter_ports([ 80, 8080, 443, 8000, 8888, 8880, 8008, 3000, 8443 ])
register_autofilter_services(%W{ http https })
begin
require 'anemone'
@anemone_loaded = true
rescue ::Exception => e
@anemone_loaded = false
@anemone_error = e
end
end
def setup
raise RuntimeError, "Could not load Anemone/Nokogiri: #{@anemone_error}" if not @anemone_loaded
super
end
def cleanup
if @crawler
@crawler.shutdown rescue nil
@crawler = nil
end
super
end
##
#
# Crawler methods and accessors
#
##
# A target object for tracking URLs
class WebTarget < ::Hash
def to_url
proto = self[:ssl] ? "https" : "http"
host = self[:vhost] ? self[:vhost] : self[:host]
if Rex::Socket.is_ipv6?(host)
host = "[#{host}]"
end
"#{proto}://#{host}:#{self[:port]}#{self[:path]}"
end
end
# A custom error to signify we hit the page request cap
class MaximumPageCount < ::RuntimeError
end
# Some accessors for stat tracking
attr_accessor :targets
attr_accessor :url_count, :url_total, :form_count, :request_count
# Entry point for the crawler code
def run
self.request_count = 0
self.form_count = 0
self.url_count = 0
self.url_total = 1
path,query = datastore['URI'].split('?', 2)
query ||= ""
t = WebTarget.new
t.merge!({
:vhost => vhost,
:host => rhost,
:port => rport,
:ssl => ssl,
:path => path,
:query => query,
:info => ""
})
if datastore['HttpUsername'] and datastore['HttpUsername'] != ''
t[:username] = datastore['HttpUsername'].to_s
t[:password] = datastore['HttpPassword'].to_s
t[:domain] = datastore['DOMAIN'].to_s
end
if datastore['HTTPCookie']
t[:cookies] = {}
datastore['HTTPCookie'].to_s.split(';').each do |pair|
k,v = pair.strip.split('=', 2)
next if not v
t[:cookies][k] = v
end
end
if datastore['HTTPAdditionalHeaders']
t[:headers] = datastore['HTTPAdditionalHeaders'].to_s.split("\x01").select{|x| x.to_s.length > 0}
end
t[:site] = report_web_site(:wait => true, :host => t[:host], :port => t[:port], :vhost => t[:vhost], :ssl => t[:ssl])
print_status("Crawling #{t.to_url}...")
begin
@current_vhost = t[:vhost]
@current_site = t[:site]
::Timeout.timeout(max_crawl_time) { crawl_target(t) }
rescue ::Timeout::Error
print_error("Crawl of #{t.to_url} has reached the configured timeout")
ensure
@current_vhost = nil
end
print_status("Crawl of #{t.to_url} complete")
end
def get_connection_timeout
datastore['RequestTimeout']
end
def max_page_count
datastore['MAX_PAGES']
end
def max_crawl_time
datastore['MAX_MINUTES'] * 60.0
end
def max_crawl_threads
datastore['MAX_THREADS']
end
def dirbust?
datastore['DirBust']
end
# Scrub links that end in these extensions. If more or less is
# desired by a particular module, this should get redefined.
def get_link_filter
/\.(js|png|jpe?g|bmp|gif|swf|jar|zip|gz|bz2|rar|pdf|docx?|pptx?)$/i
end
def focus_crawl(page)
page.links
end
def crawl_target(t)
cnt = 0
opts = crawler_options(t)
url = t.to_url
@crawler = ::Anemone::Core.new([url], opts)
@crawler.on_every_page do |page|
cnt += 1
self.request_count += 1
# Extract any interesting data from the page
crawler_process_page(t, page, cnt)
# Blow up if we hit our maximum page count
if cnt >= max_page_count
print_error("Maximum page count reached for #{url}")
raise MaximumPageCount, "Maximum page count reached"
end
end
# Skip link processing based on a regular expression
@crawler.skip_links_like(
get_link_filter
)
# Focus our crawling on interesting, but not over-crawled links
@crawler.focus_crawl do |page|
focus_crawl(page)
end
begin
@crawler.run
rescue MaximumPageCount
# No need to print anything else
rescue ::Timeout::Error
# Bubble this up to the top-level handler
raise $!
rescue ::Exception => e
# Ridiculous f'ing anonymous timeout exception which I've no idea
# how it comes into existence.
if e.to_s =~ /execution expired/
raise ::Timeout::Error
else
print_error("Crawler Exception: #{url} #{e} #{e.backtrace}")
end
ensure
@crawler.shutdown rescue nil
@crawler = nil
end
end
# Specific module implementations should redefine this method
# with whatever is meaningful to them.
def crawler_process_page(t, page, cnt)
return if page.nil? # Skip over pages that don't contain any info aka page is nil. We can't process these types of pages since there is no data to process.
msg = "[#{"%.5d" % cnt}/#{"%.5d" % max_page_count}] #{page ? page.code || "ERR" : "ERR"} - #{@current_site.vhost} - #{page.url}"
case page.code
when 301,302
if page.headers and page.headers["location"]
print_status(msg + " -> " + page.headers["location"].to_s)
else
print_status(msg)
end
when 500...599
# XXX: Log the fact that we hit an error page
print_good(msg)
when 401,403
print_good(msg)
when 200
print_status(msg)
when 404
print_error(msg)
else
print_error(msg)
end
end
def crawler_options(t)
opts = {}
opts[:user_agent] = datastore['UserAgent']
opts[:verbose] = false
opts[:threads] = max_crawl_threads
opts[:obey_robots_txt] = false
opts[:redirect_limit] = datastore['RedirectLimit']
opts[:retry_limit] = datastore['RetryLimit']
opts[:accept_cookies] = true
opts[:depth_limit] = false
opts[:skip_query_strings] = false
opts[:discard_page_bodies] = true
opts[:framework] = framework
opts[:module] = self
opts[:timeout] = get_connection_timeout
opts[:dirbust] = dirbust?
if (t[:headers] and t[:headers].length > 0)
opts[:inject_headers] = t[:headers]
end
if t[:cookies]
opts[:cookies] = t[:cookies]
end
opts[:username] = t[:username] || ''
opts[:password] = t[:password] || ''
opts[:domain] = t[:domain] || 'WORKSTATION'
if ssl
opts[:ssl_version] = ssl_version
end
opts
end
##
#
# Wrappers for getters
#
##
#
# Returns the target host
#
def rhost
datastore['RHOST']
end
#
# Returns the remote port
#
def rport
datastore['RPORT']
end
#
# Returns the VHOST of the HTTP server.
#
def vhost
datastore['VHOST'] || datastore['RHOST']
end
#
# Returns the boolean indicating SSL
#
def ssl
((datastore.default?('SSL') and rport.to_i == 443) or datastore['SSL'])
end
#
# Returns the string indicating SSL version
#
def ssl_version
datastore['SSLVersion']
end
#
# Returns the configured proxy list
#
def proxies
datastore['Proxies']
end
end
end