urllib.error.HTTPError: HTTP Error 400: Bad Request

[PandaWhoCodes]

PySitemap/crawler.py

Line 26 in 61c728f

response = urllib.request.urlopen(url)

Putting this under a try block should stop the script from crashing

[binary-sort]

Including headers in request might help

[ideleon]

a fix for that issue :)

def crawl(self, url):

            try:
                response=urllib.request.urlopen(url)
            except urllib.error.HTTPError as e:
                    print('HTTPError: {}'.format(e.code))
            except urllib.error.URLError as e:
                    print('URLError: {}'.format(e.reason))
            else:

                if not self.no_verbose:
                        print("Parsing " + url)

                response = urllib.request.urlopen(url)
                page = str(response.read())

                pattern = '<a [^>]*href=[\'|"](.*?)[\'"].*?>'

                found_links = re.findall(pattern, page)
                links = []

                for link in found_links:
                        is_url = self.is_url(link)

                        if is_url:
                                is_internal = self.is_internal(link)

                                if is_internal:
                                        self.add_url(link, links, self.exclude)
                                        self.add_url(link, self.found_links, self.exclude)

                for link in links:
                        if link not in self.visited_links:
                                link = self.normalize(link)

                                self.visited_links.append(link)
                                self.crawl(urljoin(self.url, link))