for link in filter(lambda x: x.url.endswith('/'), links):
            self.crawler_url.crawler.add_url(CrawlerUrl(self.crawler_url.crawler, link, 3, self.crawler_url,
                                                        type='directory', timeout=self.crawler_url.timeout))