run_part4.py 文件源码

def print_time(source_type, source_name, document_item):
    today = datetime.datetime.now().strftime("%Y-%m-%d") + " 12:00:00"
    yestday = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d') + " 00:00:00"
    connect('yuqing', host=MONGODB_URI['host'], port=MONGODB_URI['port'],
            username=MONGODB_URI['username'], password=MONGODB_URI['password'])
    crawl_pages = document_item.objects(Q(insert_time__gte=yestday) & Q(insert_time__lte=today))
    new_pages = document_item.objects(Q(time__gte=yestday) & Q(time__lte=today)).count()
    ymonitor = YuqingSpiderMonitor()
    ymonitor.key = source_name
    logging.error(source_name)
    ymonitor.crawl_pages = str(len(crawl_pages))
    ymonitor.new_pages = str(new_pages)
    ymonitor.source_type = source_type
    ymonitor.date_stat = datetime.datetime.now().strftime("%Y-%m-%d") + " 00:00:00"
    if len(crawl_pages) > 0:
        date1 = crawl_pages.order_by('insert_time')[0].insert_time
        date2 = crawl_pages.order_by('-insert_time')[0].insert_time
        ymonitor.duration = str(TimeUtil.get_date_diff_seconds(date1, date2))
    else:
        ymonitor.duration = str(0)
    ymonitor._id = Md5Util.generate_md5(source_name+today)
    ymonitor.save()