def parse(self, response):
self.logger.info('parse: {}'.format(response))
is_no_update = False
# Collect list of news from current page
# Note: no next page button on cnnindonesia, all is loaded here
article_selectors = response.css('a.list_kontribusi');
if not article_selectors:
raise CloseSpider('article_selectors not found')
for article in article_selectors:
url_selectors = article.css('a::attr(href)')
if not url_selectors:
raise CloseSpider('url_selectors not found')
url = url_selectors.extract()[0]
# Example: Jumat, 23/09/2016 21:17
info_selectors = article.css('div.text > div > span.tanggal::text')
if not info_selectors:
raise CloseSpider('info_selectors not found')
info = info_selectors.extract()[0]
info_time = info.split(',')[1].strip()
# Parse date information
try:
# Example: 23/09/2016 21:17
published_at_wib = datetime.strptime(info_time, '%d/%m/%Y %H:%M')
except ValueError as err:
raise CloseSpider('cannot_parse_date: {}'.format(err))
published_at = wib_to_utc(published_at_wib)
if self.media['last_scraped_at'] >= published_at:
is_no_update = True
break
# For each url we create new scrapy Request
yield Request(url, callback=self.parse_news)
if is_no_update:
self.logger.info('Media have no update')
return
评论列表
文章目录