transfermarkt_crawler.py 文件源码

def run_crawler(base_url, ua, start_date, end_date,
                google_username, google_password):

    temp_df = pd.DataFrame()

    dates = date_range(start_date, end_date)

    for d in dates:
        url = '{0}//transfers/transfertagedetail/statistik/top/land_id_zu/0/land_id_ab/0/leihe//datum/{1}/plus/1'.format(
            base_url, d)

        rqst = requests.get(url, headers={"User-Agent": ua})
        resp = TextResponse(url, body=rqst.content)

        players, nat, ages, positions, prev_clubs, next_clubs, mkt_values, trans_prices = get_data_lists(
            resp)

        df = get_df(players,
                    nat,
                    ages,
                    positions,
                    prev_clubs,
                    next_clubs,
                    mkt_values,
                    trans_prices,
                    d)

        trends_df = get_trends_data(google_username, google_password,
                                    players, d)

        df = pd.merge(df, trends_df, how='left', on='player')

        temp_df = pd.concat([temp_df, df])

    return temp_df