pipelines.py 文件源码

def __getValue(self,url):
        isHaveManyQueryInUrl = False
        for value in self.valuedict:
            div_by_value = url.split(value.rstrip('\n'))
            mm = div_by_value[0]
            if mm in self.seen:
                raise DropItem('Duplicate link %s' % url)
            elif len(div_by_value) > 1 and not isHaveManyQueryInUrl:
                self.seen.add(mm)
                isHaveManyQueryInUrl = True
                line = url+'\n'
                print url
                self.file.write(line)