crawler.py 文件源码

def login(self, response):
        self.log('Logging in...')
        try:
            full_args, args, url, method, params = fill_login_form(response.url, response.body, self.login_user, self.login_pass)
            validated_url = self.url_valid(url, response.url)
            real_url = urlsplit(validated_url)
            result_db.add_to_result(method.upper(), real_url.scheme + "://" + real_url.hostname + real_url.path,
                                    list(dict(full_args).keys()))
            yield FormRequest(validated_url,
                               method=method,
                               formdata=args,
                               callback=self.confirm_login,
                               dont_filter=True)
        except Exception as e:
            print(e)
            self.log('Login failed')
            for start_url in self.start_urls:
                if (";" in start_url):
                    split_arr = start_url.split(';')
                    validated_url = split_arr[0]
                    yield Request(url=validated_url, dont_filter=True, callback=self.parse_res)
                    time.sleep(int(split_arr[1]))

                else:
                    validated_url = start_url
                    yield Request(url=validated_url, dont_filter=True, callback=self.parse_res)

                real_url = urlsplit(validated_url)
                if len(real_url.query) > 0 and self.get_ext(real_url.path) not in self.not_allowed:
                    # only add to result if have parameters
                    param_dict = parse_qs(real_url.query, keep_blank_values=True)
                    result_db.add_to_result("GET", real_url.scheme + "://" + real_url.hostname + real_url.path, list(param_dict.keys()))
                if self.ignore_params:
                    tag_url = real_url.scheme + "://" + real_url.hostname + real_url.path
                else:
                    tag_url = validated_url
                    for param in self.ignore_fields:
                        if param in real_url.query:
                            tag_url = real_url.path
                if tag_url not in self.urls_visited and self.get_ext(real_url.path) not in self.not_allowed:
                    self.urls_visited.append(tag_url)