crawler.py 文件源码

def run(self, tags):
    rpc_list = [urlfetch.create_rpc() for x in xrange(len(tags))]
    for rpc, tag in zip(rpc_list, tags):
      self._fetch_async(rpc, tag)
      time.sleep(0.5) # prevent to be blocked
    return reduce(lambda s,rpc: s + self._parse_result(rpc.get_result()),
        rpc_list, [])