def spider_closed(self, spider, reason):
spider.logger.info('Spider closed: %s %s', spider.name, reason)
# if spider finished without error update last_scraped_at
if reason == 'finished':
try:
self.logger.info('Updating media last_scraped_at information')
self.cursor.execute(sql_update_media, [spider.name])
self.db.commit()
self.db.close()
except mysql.Error as err:
self.logger.error('Unable to update last_scraped_at: %s', err)
self.db.rollback()
self.db.close()
if self.is_slack:
error_msg = '{}: Unable to update last_scraped_at: {}'.format(
spider.name, err)
self.slack.chat.post_message('#rojak-pantau-errors', error_msg,
as_user=True)
else:
if self.is_slack:
# Send error to slack
error_msg = '{}: Spider fail because: {}'.format(
spider.name, reason)
self.slack.chat.post_message('#rojak-pantau-errors',
error_msg, as_user=True)
# subscibe to item_droped event
评论列表
文章目录