webloader.py 文件源码

def load(self, url):
        """Load script from URL."""

        # Check for a redirect to get a final base_url where to start

        resp = requests.head(url, allow_redirects=True)
        if url != resp.url:
            # Followed a redirect
            url = resp.url

        fname = get_response_fname(resp)
        _, ext = os.path.splitext(fname)

        if ext == ".py":
            py_file = self.fetch_file(url, url)
            return py_file
        else:
            self.crawl(url, url)