LaGouSpiderMain.py 文件源码

def LaGouSpiderWithKeyWord(position, city):
    # ??????
    pageCount = SearchPageCount(position, city)
    if pageCount == 0:
        print('???????????????????')
        return

    totaldata = DataFrame().T
    urls = []
    for i in range(0, pageCount):
        url = 'http://www.lagou.com/jobs/positionAjax.json?'
        params = {'city': city, 'kd': position, 'pn': i+1}
        url += parse.urlencode(params)
        urls.append(url)
    # ??work?
    pool = ThreadPool(processes=8)
    # ?????rdatas
    rdatas = pool.map(get_rdata, urls)
    for rdata in rdatas:
        totaldata = pd.concat([totaldata, rdata])
    totaldata.to_csv('lagou.csv')