main.py 文件源码

def crawl(url):
    """
    ????URL?????????????????
    """
    try:
        html = requests.get(url)
    except:
        with open("log.log","a") as file:
            file.write("Http error on " + time.ctime())
        time.sleep(60)
        return None
    soup = BeautifulSoup(html.text, 'lxml')
    data_list = []
    for cont in soup.find_all("div", {"class":"content"}):
        raw_data = cont.get_text()
        data = raw_data.replace("\n","")
        data_list.append(data)
    return data_list