LvzPoliceTickerDetailViewCrawler.java 文件源码

@Async
public Future<Iterable<PoliceTicker>> execute(final Iterable<String> detailURLs) {
    final Stopwatch watch = Stopwatch.createStarted();
    logger.info("Start crawling detail pages");
    final List<PoliceTicker> policeTickers = new ArrayList<>();
    for (final Iterator<String> iterator = detailURLs.iterator(); iterator.hasNext();) {
        final PoliceTicker ticker = crawl(iterator.next());
        if (ticker != null) {
            policeTickers.add(ticker);
        }
        if (iterator.hasNext()) {
            try {
                Thread.sleep(WAIT_BEFORE_EACH_ACCESS_TO_PREVENT_BANNING);
            } catch (final InterruptedException e) {
                logger.error(e.toString(), e);
            }
        }
    }
    watch.stop();
    logger.info("Finished crawling {} detail pages in {} ms", policeTickers.size(), watch.elapsed(TimeUnit.MILLISECONDS));
    return new AsyncResult<>(policeTickers);
}