def mem_check(opts):
while True:
if opts['gc']:
try:
gc.collect()
except Exception as e:
logging.exception(repr(e) + ' while gc.collect()')
try:
rss = psutil.Process(os.getpid()).memory_info().rss
logging.info('current memory used: {rss}'.format(rss=rss))
if rss > opts['threshold']:
memory_dump(opts)
os.abort()
except Exception as e:
logging.exception(repr(e) + ' while checking memory usage')
finally:
time.sleep(opts['interval'])
python类exception()的实例源码
def runspider(name):
configure_logging(install_root_handler=False)
logging.basicConfig(
filename='log/%s.log' % name,
format='%(levelname)s %(asctime)s: %(message)s',
level=logging.DEBUG
)
process = CrawlerProcess(get_project_settings())
try:
logging.info('runspider start spider:%s' % name)
process.crawl(name)
process.start()
except Exception as e:
logging.exception('runspider spider:%s exception:%s' % (name, e))
logging.debug('finish this spider:%s\n\n' % name)
def do(self, function_name, data):
if function_name in dir(self.module):
if self.debug is True:
# this makes the plugin fail with stack trace in debug mode
eval("self.module." + function_name)(data)
else:
# otherwise we log the exception and carry on
try:
eval("self.module." + function_name)(data)
except Exception:
logging.exception("problem in module {} {}".format(function_name, data))
if "catch_all" in dir(self.module):
if self.debug is True:
# this makes the plugin fail with stack trace in debug mode
self.module.catch_all(data)
else:
try:
self.module.catch_all(data)
except Exception:
logging.exception("problem in catch all: {} {}".format(self.module, data))
def validate_rule(self, rule, resource):
"""
Validate a single rule on the values of a resource by eval the boolean value of rule's expression.
:param rule: rule dictionary
:param resource: resource values dictionary
:return: True if valid, otherwise False
"""
try:
if 'destroy' in resource and resource['destroy']:
return True # skip validation if resource is being destroyed
R = ResourceContainer(resource) # R is the identifier used in the rule expression
return eval(rule["expr"])
except Exception as e:
'''exceptions are suppressed here to let the validation continue. So that the user will
receive al complete report and be able to fix all errors at once.'''
logging.exception('[EXCEPTION] Exception occurred during rule expression evaluation')
return False
def destroy(self):
"""
Destroy the Manialink with it's handlers and references.
Will also hide the Manialink for all users!
"""
try:
SignalManager.get_signal('maniaplanet:manialink_answer').unregister(self.handle)
except Exception as e:
logging.exception(e)
try:
await self.manager.destroy(self)
except:
pass
self.receivers = dict()
self.data = None
self.player_data = None
def destroy_sync(self):
"""
Destroy the Manialink with it's handlers and references.
Will also hide the Manialink for all users!
This method is sync and will call a async method (destroying of the manialink at our players) async but will not
be executed at the same time. Be aware with this one!
"""
try:
SignalManager.get_signal('maniaplanet:manialink_answer').unregister(self.handle)
asyncio.ensure_future(self.manager.destroy(self))
except Exception as e:
logging.exception(e)
self.receivers = dict()
self.data = None
self.player_data = None
def get_player(self, login=None, pk=None, lock=True):
"""
Get player by login or primary key.
:param login: Login.
:param pk: Primary Key identifier.
:param lock: Lock for a sec when receiving.
:return: Player or exception if not found
:rtype: pyplanet.apps.core.maniaplanet.models.Player
"""
try:
if login:
return await Player.get_by_login(login)
elif pk:
return await Player.get(pk=pk)
else:
raise PlayerNotFound('Player not found.')
except DoesNotExist:
if lock:
await asyncio.sleep(4)
return await self.get_player(login=login, pk=pk, lock=False)
else:
raise PlayerNotFound('Player not found.')
def process_setup(self, seqid, iprot, oprot):
args = setup_args()
args.read(iprot)
iprot.readMessageEnd()
result = setup_result()
try:
self._handler.setup(args.name, args.description)
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except wpwithinpy.wpthrift_types.ttypes.Error as err:
msg_type = TMessageType.REPLY
result.err = err
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("setup", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_addService(self, seqid, iprot, oprot):
args = addService_args()
args.read(iprot)
iprot.readMessageEnd()
result = addService_result()
try:
self._handler.addService(args.svc)
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except wpwithinpy.wpthrift_types.ttypes.Error as err:
msg_type = TMessageType.REPLY
result.err = err
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("addService", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_removeService(self, seqid, iprot, oprot):
args = removeService_args()
args.read(iprot)
iprot.readMessageEnd()
result = removeService_result()
try:
self._handler.removeService(args.svc)
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except wpwithinpy.wpthrift_types.ttypes.Error as err:
msg_type = TMessageType.REPLY
result.err = err
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("removeService", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_initConsumer(self, seqid, iprot, oprot):
args = initConsumer_args()
args.read(iprot)
iprot.readMessageEnd()
result = initConsumer_result()
try:
self._handler.initConsumer(args.scheme, args.hostname, args.port, args.urlPrefix, args.clientID, args.hceCard, args.pspConfig)
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except wpwithinpy.wpthrift_types.ttypes.Error as err:
msg_type = TMessageType.REPLY
result.err = err
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("initConsumer", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_initProducer(self, seqid, iprot, oprot):
args = initProducer_args()
args.read(iprot)
iprot.readMessageEnd()
result = initProducer_result()
try:
self._handler.initProducer(args.pspConfig)
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except wpwithinpy.wpthrift_types.ttypes.Error as err:
msg_type = TMessageType.REPLY
result.err = err
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("initProducer", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_startServiceBroadcast(self, seqid, iprot, oprot):
args = startServiceBroadcast_args()
args.read(iprot)
iprot.readMessageEnd()
result = startServiceBroadcast_result()
try:
self._handler.startServiceBroadcast(args.timeoutMillis)
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except wpwithinpy.wpthrift_types.ttypes.Error as err:
msg_type = TMessageType.REPLY
result.err = err
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("startServiceBroadcast", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_stopServiceBroadcast(self, seqid, iprot, oprot):
args = stopServiceBroadcast_args()
args.read(iprot)
iprot.readMessageEnd()
result = stopServiceBroadcast_result()
try:
self._handler.stopServiceBroadcast()
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except wpwithinpy.wpthrift_types.ttypes.Error as err:
msg_type = TMessageType.REPLY
result.err = err
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("stopServiceBroadcast", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_deviceDiscovery(self, seqid, iprot, oprot):
args = deviceDiscovery_args()
args.read(iprot)
iprot.readMessageEnd()
result = deviceDiscovery_result()
try:
result.success = self._handler.deviceDiscovery(args.timeoutMillis)
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except wpwithinpy.wpthrift_types.ttypes.Error as err:
msg_type = TMessageType.REPLY
result.err = err
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("deviceDiscovery", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_searchForDevice(self, seqid, iprot, oprot):
args = searchForDevice_args()
args.read(iprot)
iprot.readMessageEnd()
result = searchForDevice_result()
try:
result.success = self._handler.searchForDevice(args.timeoutMillis, args.deviceName)
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except wpwithinpy.wpthrift_types.ttypes.Error as err:
msg_type = TMessageType.REPLY
result.err = err
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("searchForDevice", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_requestServices(self, seqid, iprot, oprot):
args = requestServices_args()
args.read(iprot)
iprot.readMessageEnd()
result = requestServices_result()
try:
result.success = self._handler.requestServices()
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except wpwithinpy.wpthrift_types.ttypes.Error as err:
msg_type = TMessageType.REPLY
result.err = err
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("requestServices", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_selectService(self, seqid, iprot, oprot):
args = selectService_args()
args.read(iprot)
iprot.readMessageEnd()
result = selectService_result()
try:
result.success = self._handler.selectService(args.serviceId, args.numberOfUnits, args.priceId)
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except wpwithinpy.wpthrift_types.ttypes.Error as err:
msg_type = TMessageType.REPLY
result.err = err
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("selectService", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_makePayment(self, seqid, iprot, oprot):
args = makePayment_args()
args.read(iprot)
iprot.readMessageEnd()
result = makePayment_result()
try:
result.success = self._handler.makePayment(args.request)
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except wpwithinpy.wpthrift_types.ttypes.Error as err:
msg_type = TMessageType.REPLY
result.err = err
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("makePayment", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_beginServiceDelivery(self, seqid, iprot, oprot):
args = beginServiceDelivery_args()
args.read(iprot)
iprot.readMessageEnd()
result = beginServiceDelivery_result()
try:
result.success = self._handler.beginServiceDelivery(args.serviceID, args.serviceDeliveryToken, args.unitsToSupply)
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except wpwithinpy.wpthrift_types.ttypes.Error as err:
msg_type = TMessageType.REPLY
result.err = err
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("beginServiceDelivery", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_endServiceDelivery(self, seqid, iprot, oprot):
args = endServiceDelivery_args()
args.read(iprot)
iprot.readMessageEnd()
result = endServiceDelivery_result()
try:
result.success = self._handler.endServiceDelivery(args.serviceID, args.serviceDeliveryToken, args.unitsReceived)
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except wpwithinpy.wpthrift_types.ttypes.Error as err:
msg_type = TMessageType.REPLY
result.err = err
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("endServiceDelivery", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_CloseRPCAgent(self, seqid, iprot, oprot):
args = CloseRPCAgent_args()
args.read(iprot)
iprot.readMessageEnd()
result = CloseRPCAgent_result()
try:
self._handler.CloseRPCAgent()
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("CloseRPCAgent", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
# HELPER FUNCTIONS AND STRUCTURES
def process_endServiceDelivery(self, seqid, iprot, oprot):
args = endServiceDelivery_args()
args.read(iprot)
iprot.readMessageEnd()
result = endServiceDelivery_result()
try:
self._handler.endServiceDelivery(args.serviceID, args.serviceDeliveryToken, args.unitsReceived)
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except wpwithinpy.wpthrift_types.ttypes.Error as err:
msg_type = TMessageType.REPLY
result.err = err
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("endServiceDelivery", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_makePaymentEvent(self, seqid, iprot, oprot):
args = makePaymentEvent_args()
args.read(iprot)
iprot.readMessageEnd()
result = makePaymentEvent_result()
try:
self._handler.makePaymentEvent(args.totalPrice, args.orderCurrency, args.clientToken, args.orderDescription, args.uuid)
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("makePaymentEvent", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_serviceDiscoveryEvent(self, seqid, iprot, oprot):
args = serviceDiscoveryEvent_args()
args.read(iprot)
iprot.readMessageEnd()
result = serviceDiscoveryEvent_result()
try:
self._handler.serviceDiscoveryEvent(args.remoteAddr)
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("serviceDiscoveryEvent", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_servicePricesEvent(self, seqid, iprot, oprot):
args = servicePricesEvent_args()
args.read(iprot)
iprot.readMessageEnd()
result = servicePricesEvent_result()
try:
self._handler.servicePricesEvent(args.remoteAddr, args.serviceId)
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("servicePricesEvent", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def process_serviceTotalPriceEvent(self, seqid, iprot, oprot):
args = serviceTotalPriceEvent_args()
args.read(iprot)
iprot.readMessageEnd()
result = serviceTotalPriceEvent_result()
try:
self._handler.serviceTotalPriceEvent(args.remoteAddr, args.serviceID, args.totalPriceResp)
msg_type = TMessageType.REPLY
except (TTransport.TTransportException, KeyboardInterrupt, SystemExit):
raise
except Exception as ex:
msg_type = TMessageType.EXCEPTION
logging.exception(ex)
result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error')
oprot.writeMessageBegin("serviceTotalPriceEvent", msg_type, seqid)
result.write(oprot)
oprot.writeMessageEnd()
oprot.trans.flush()
def find_by_cookie(cls, cookie_str):
if not cookie_str:
return None
try:
L = cookie_str.split('-')
if len(L) != 3:
return None
uid, expires, sha1 = L
if int(expires) < time.time():
return None
user = await cls.find(uid)
if not user:
return None
s = '%s-%s-%s-%s' % (uid, user.get('password'), expires, COOKIE_KEY)
if sha1 != hashlib.sha1(s.encode('utf-8')).hexdigest():
logging.info('invalid sha1')
return None
user.password = '******'
return user
except Exception as e:
logging.exception(e)
return None
# ?????
def execute(self, kintaro, force=False):
try:
project = kintaro.projects().rpcGetProject(body={
'project_id': self.project_id,
}).execute()
except errors.HttpError as e:
logging.exception('Error fetching -> {}'.format(self))
return
self.modified_by = project['mod_info'].get('updated_by')
self.modified = datetime.datetime.fromtimestamp(
int(project['mod_info']['updated_on_millis']) / 1000.0)
if force or self.last_run is None or self.modified > self.last_run:
if self.webhook_url:
self.run_webhook(project)
else:
logging.info('Skipping (no webhook) -> {}'.format(self))
else:
logging.info('Skipping (up-to-date) -> {}'.format(self))
self.last_run = datetime.datetime.now()
self.put()
def __init__(self, request=None, response=None):
"""Calls the constructor of the super and does the local setup."""
super(InitHandler, self).__init__(request, response)
self.client = pubsub_utils.get_client()
# self._setup_topic()
self._setup_subscription()
# def _setup_topic(self):
# """Creates a topic if it does not exist."""
# topic_name = pubsub_utils.get_full_topic_name()
# try:
# self.client.projects().topics().get(
# topic=topic_name).execute()
# except errors.HttpError as e:
# if e.resp.status == 404:
# self.client.projects().topics().create(
# name=topic_name, body={}).execute()
# else:
# logging.exception(e)
# raise
