spider_runner.py 文件源码

def spider_start(self, kw):
        '''
        '''
        pp = SubProcessProtocol(self, self.factory.spiders, kw)

        args = [sys.executable] + kw['cmd'].split(' ')
        reactor.spawnProcess(pp, 
                             sys.executable, args=args,
                             env={
                                  'PYTHONPATH':'/srv/ataobao:$PYTHONPATH',
                                  'ENV':'TEST',
                                  },
                             path='/srv/ataobao/crawler')
#         for i in range(0, kw["process"]):
#             pp = SubProcessProtocol(self, self.factory.spiders, kw)
#             args = [sys.executable, "spider_tx.py", kw["spider"], str(kw["threads"])]
#             args = [sys.executable, "crawler/worker.py", "-w", "shop", "-p", "10"]
#             args = [sys.executable, "crawler/tbcat.py", "-c", "16"]
#             print args
#             reactor.spawnProcess(pp, sys.executable, args=args)