spiders.py 文件源码

def parse_first(self, response):
        allowed = allowed_re(
            response.url, self.settings.getbool('HARD_URL_CONSTRAINT'))
        if allowed not in self.allowed:
            self.allowed.add(allowed)
            # Reset link extractors to pick up with the latest self.allowed regexps
            self._reset_link_extractors()
            self.logger.info('Updated allowed regexps: %s', self.allowed)
        yield from self.parse(response)