master
canCrawl
func canCrawl(a, b *url.URL, d int, subdomains bool) (yes bool) { if a.Host != b.Host { if !subdomains { return false }