def error_back(self, failure):
#request????error_back
#???????request?????????????????
#???????dns???????
#???????????redis???????
if failure.check(HttpError):
#?????200?????
response = failure.value.response
self.logger.error('[%s] GET [%d]', response.url, response.status)
if failure.check(DNSLookupError):
#dns????
request = failure.request
self.logger.error('[%s] DNSLookupError', request.url)
if failure.check(TimeoutError, TCPTimedOutError):
#????
request = failure.request
self.logger.error('[%s] TimeoutError', request.url)
python类TimeoutError()的实例源码
def testServerTimeout(self):
self.server.timeoutTest = True
self.client.timeout = 5 #seconds
self.selectedArgs = None
def login():
d = self.client.login('testuser', 'password-test')
d.addErrback(timedOut)
return d
def timedOut(failure):
self._cbStopClient(None)
failure.trap(error.TimeoutError)
d = self.connected.addCallback(strip(login))
d.addErrback(self._ebGeneral)
return defer.gatherResults([d, self.loopback()])
def connectionLost(self, reason):
if self.timeout > 0:
self.setTimeout(None)
if self._timedOut:
reason = error.TimeoutError()
elif self._greetingError:
reason = ServerErrorResponse(self._greetingError)
d = []
if self._waiting is not None:
d.append(self._waiting)
self._waiting = None
if self._blockedQueue is not None:
d.extend([deferred for (deferred, f, a) in self._blockedQueue])
self._blockedQueue = None
for w in d:
w.errback(reason)
def errback(self, failure):
if failure.check(DNSLookupError):
host = urlparse(failure.request.url).hostname
logger.error('DNSLookupError on host[%s]',
host)
elif failure.check(TimeoutError,
TCPTimedOutError):
request = failure.request
logger.error('TimeoutError on url[%s]',
request.url)
elif failure.check(HttpError):
response = failure.value.response
logger.error('HttpError on url[%s, status=%d]',
response.url,
response.status)
else:
logger.error('SpiderError on url[%s, error=%s',
failure.request.url,
repr(failure))
def testServerTimeout(self):
self.server.timeoutTest = True
self.client.timeout = 5 #seconds
self.selectedArgs = None
def login():
d = self.client.login('testuser', 'password-test')
d.addErrback(timedOut)
return d
def timedOut(failure):
self._cbStopClient(None)
failure.trap(error.TimeoutError)
d = self.connected.addCallback(strip(login))
d.addErrback(self._ebGeneral)
return defer.gatherResults([d, self.loopback()])
def connectionLost(self, reason):
if self.timeout > 0:
self.setTimeout(None)
if self._timedOut:
reason = error.TimeoutError()
elif self._greetingError:
reason = ServerErrorResponse(self._greetingError)
d = []
if self._waiting is not None:
d.append(self._waiting)
self._waiting = None
if self._blockedQueue is not None:
d.extend([deferred for (deferred, f, a) in self._blockedQueue])
self._blockedQueue = None
for w in d:
w.errback(reason)
def error_back(self, failure):
#request????error_back
#???????request?????????????????
#???????dns???????
#???????????redis???????
if failure.check(HttpError):
#?????200?????
response = failure.value.response
self.logger.error('[%s] GET [%d]', response.url, response.status)
if failure.check(DNSLookupError):
#dns????
request = failure.request
self.logger.error('[%s] DNSLookupError', request.url)
if failure.check(TimeoutError, TCPTimedOutError):
#????
request = failure.request
self.logger.error('[%s] TimeoutError', request.url)
def _collect_memory(self, netboxes):
memory = dict()
for mib in self._mibs_for_me(MEMORY_MIBS):
try:
mem = yield mib.get_memory_usage()
except (TimeoutError, defer.TimeoutError):
self._logger.debug("collect_memory: ignoring timeout in %s",
mib.mib['moduleName'])
else:
if mem:
self._logger.debug("Found memory values from %s: %r",
mib.mib['moduleName'], mem)
memory.update(mem)
timestamp = time.time()
result = []
for name, (used, free) in memory.items():
for netbox in netboxes:
prefix = metric_prefix_for_memory(netbox, name)
result.extend([
(prefix + '.used', (timestamp, used)),
(prefix + '.free', (timestamp, free)),
])
defer.returnValue(result)
def errback_httpbin(self, failure):
# log all failures
self.logger.error(repr(failure))
# in case you want to do something special for some errors,
# you may need the failure's type:
if failure.check(HttpError):
# these exceptions come from HttpError spider middleware
# you can get the non-200 response
response = failure.value.response
self.logger.error('HttpError on %s', response.url)
elif failure.check(DNSLookupError):
# this is the original request
request = failure.request
self.logger.error('DNSLookupError on %s', request.url)
elif failure.check(TimeoutError, TCPTimedOutError):
request = failure.request
self.logger.error('TimeoutError on %s', request.url)
def result_errmsg(result):
"""Return a useful error message string given a twisted errBack result."""
try:
from pywbem.cim_operations import CIMError
if result.type == ConnectionRefusedError:
return 'connection refused. Check IP and zWBEMPort'
elif result.type == TimeoutError:
return 'connection timeout. Check IP and zWBEMPort'
elif result.type == CIMError:
if '401' in result.value.args[1]:
return 'login failed. Check zWBEMUsername and zWBEMPassword'
else:
return result.value.args[1]
else:
return result.getErrorMessage()
except AttributeError:
pass
return str(result)
def run_command(self, command_line):
"""
Run commands in a remote shell like the winrs application on Windows.
Accepts multiple commands. Returns a dictionary with the following
structure:
CommandResponse
.stdout = [<non-empty, stripped line>, ...]
.stderr = [<non-empty, stripped line>, ...]
.exit_code = <int>
"""
shell_id = yield self._create_shell()
try:
cmd_response = yield self._run_command(shell_id, command_line)
except TimeoutError:
yield self._sender.close_connections()
yield self._delete_shell(shell_id)
yield self._sender.close_connections()
defer.returnValue(cmd_response)
def start(self, command_line):
log.debug("LongRunningCommand run_command: {0}".format(command_line))
elem = yield self._sender.send_request('create')
self._shell_id = _find_shell_id(elem)
command_line_elem = _build_command_line_elem(command_line)
log.debug('LongRunningCommand run_command: sending command request '
'(shell_id={0}, command_line_elem={1})'.format(
self._shell_id, command_line_elem))
try:
command_elem = yield self._sender.send_request(
'command', shell_id=self._shell_id,
command_line_elem=command_line_elem,
timeout=self._sender._sender._conn_info.timeout)
except TimeoutError:
yield self._sender.close_connections()
raise
self._command_id = _find_command_id(command_elem)
def onError(self, result, config):
prefix = 'failed collection - '
if isinstance(result, Failure):
result = result.value
if isinstance(result, error.TimeoutError):
result = 'Timeout while connecting to host'
prefix = ''
msg = 'WindowsServiceLog: {0}{1} {2}'.format(prefix, result, config)
log.error(msg)
data = self.new_data()
errorMsgCheck(config, data['events'], result.message)
if not data['events']:
data['events'].append({
'eventClass': "/Status/WinService",
'severity': ZenEventClasses.Error,
'eventClassKey': 'WindowsServiceCollectionStatus',
'eventKey': 'WindowsServiceCollection',
'summary': msg,
'device': config.id})
return data
def add_timeout(deferred, timeout):
'''
Raise TimeoutError on deferred after timeout seconds.
Returns original deferred.
'''
def timeout_deferred():
if not deferred.called:
deferred.errback(TimeoutError())
timeout_d = reactor.callLater(timeout, timeout_deferred)
def cancel_timeout_d(result):
if not timeout_d.called:
timeout_d.cancel()
return result
deferred.addBoth(cancel_timeout_d)
return deferred
def errback_httpbin(self, failure):
self.logger.error(repr(failure))
#if isinstance(failure.value, HttpError):
if failure.check(HttpError):
# you can get the response
response = failure.value.response
self.logger.error('HttpError on %s', response.url)
#elif isinstance(failure.value, DNSLookupError):
elif failure.check(DNSLookupError):
# this is the original request
request = failure.request
self.logger.error('DNSLookupError on %s', request.url)
#elif isinstance(failure.value, TimeoutError):
elif failure.check(TimeoutError):
request = failure.request
self.logger.error('TimeoutError on %s', request.url)
def handle_disconnected_connect(self):
self.state = "connecting"
if not self.factoryStarted:
self.factory.doStart()
self.factoryStarted = True
if self.timeout is not None:
self.timeoutID = self.reactor.callLater(self.timeout, self.connectionFailed, failure.Failure(error.TimeoutError()))
self.sub = _SubConnector(self)
self.sub.startConnecting()
self.factory.startedConnecting(self)
def connect(self):
"""Start connection to remote server."""
if self.state != "disconnected":
raise RuntimeError, "can't connect in this state"
self.state = "connecting"
if not self.factoryStarted:
self.factory.doStart()
self.factoryStarted = 1
self.transport = transport = self._makeTransport()
if self.timeout is not None:
self.timeoutID = self.reactor.callLater(self.timeout, transport.failIfNotConnected, error.TimeoutError())
self.factory.startedConnecting(self)
def programTimeout(err):
err.trap(error.TimeoutError)
proto.signalProcess('KILL')
return err
def queryUDP(self, queries, timeout = None):
"""
Make a number of DNS queries via UDP.
@type queries: A C{list} of C{dns.Query} instances
@param queries: The queries to make.
@type timeout: Sequence of C{int}
@param timeout: Number of seconds after which to reissue the query.
When the last timeout expires, the query is considered failed.
@rtype: C{Deferred}
@raise C{twisted.internet.defer.TimeoutError}: When the query times
out.
"""
if timeout is None:
timeout = self.timeout
addresses = self.servers + list(self.dynServers)
if not addresses:
return defer.fail(IOError("No domain name servers available"))
used = addresses.pop()
return self.protocol.query(used, queries, timeout[0]
).addErrback(self._reissue, addresses, [used], queries, timeout
)
def _reissue(self, reason, addressesLeft, addressesUsed, query, timeout):
reason.trap(dns.DNSQueryTimeoutError)
# If there are no servers left to be tried, adjust the timeout
# to the next longest timeout period and move all the
# "used" addresses back to the list of addresses to try.
if not addressesLeft:
addressesLeft = addressesUsed
addressesLeft.reverse()
addressesUsed = []
timeout = timeout[1:]
# If all timeout values have been used, or the protocol has no
# transport, this query has failed. Tell the protocol we're
# giving up on it and return a terminal timeout failure to our
# caller.
if not timeout or self.protocol.transport is None:
self.protocol.removeResend(reason.value.id)
return failure.Failure(defer.TimeoutError(query))
# Get an address to try. Take it out of the list of addresses
# to try and put it ino the list of already tried addresses.
address = addressesLeft.pop()
addressesUsed.append(address)
# Issue a query to a server. Use the current timeout. Add this
# function as a timeout errback in case another retry is required.
d = self.protocol.query(address, query, timeout[0], reason.value.id)
d.addErrback(self._reissue, addressesLeft, addressesUsed, query, timeout)
return d
def testTimeout(self):
def login():
d = self.client.login('test', 'twisted')
d.addCallback(loggedIn)
d.addErrback(timedOut)
return d
def loggedIn(result):
self.fail("Successfully logged in!? Impossible!")
def timedOut(failure):
failure.trap(error.TimeoutError)
self._cbStopClient(None)
def quit():
return self.client.quit()
self.client.timeout = 0.01
# Tell the server to not return a response to client. This
# will trigger a timeout.
pop3testserver.TIMEOUT_RESPONSE = True
methods = [login, quit]
map(self.connected.addCallback, map(strip, methods))
self.connected.addCallback(self._cbStopClient)
self.connected.addErrback(self._ebGeneral)
return self.loopback()
def failed(self, failure, job_id):
if failure.check(CancelledError):
self.job_failed("Response max size exceeded! job id: %s!" % job_id, job_id)
elif failure.check(InvalidResponseRetry):
ex = failure.value
if job_id in self.retry_counter and self.retry_counter[job_id] == self.max_retry:
self.job_failed("Max retry has been reached! job id: %s!" % job_id, job_id)
else:
self.job_failed_retry(ex.message, job_id)
elif failure.check(ResponseNeverReceived):
self.job_failed("No response from the server! job id: %s!" % job_id, job_id)
elif failure.check(ResponseFailed):
# @TODO add retry
self.job_failed("Connection to server failed, retry .... %s!" % job_id, job_id)
elif failure.check(NoResponseContent):
self.job_failed("Response has no content .... %s!" % job_id, job_id)
elif failure.check(TimeoutError):
if job_id in self.retry_counter and self.retry_counter[job_id] == self.max_retry:
self.job_failed("Max retry has been reached! job id: %s!" % job_id, job_id)
else:
self.job_failed_retry("Request timeout .... %s!" % job_id, job_id)
elif failure.check(ConnectionRefusedError):
if job_id in self.retry_counter and self.retry_counter[job_id] == self.max_retry:
self.job_failed("Max retry has been reached! job id: %s!" % job_id, job_id)
else:
self.job_failed_retry("Connection refused .... %s!" % job_id, job_id)
else:
ex = failure.value
self.job_failed("No proper failure found: %s, \n %s!" % (job_id, ex.message), job_id)
failure.printTraceback()
def handle_disconnected_connect(self):
self.state = "connecting"
if not self.factoryStarted:
self.factory.doStart()
self.factoryStarted = True
if self.timeout is not None:
self.timeoutID = self.reactor.callLater(self.timeout, self.connectionFailed, failure.Failure(error.TimeoutError()))
self.sub = _SubConnector(self)
self.sub.startConnecting()
self.factory.startedConnecting(self)
def connect(self):
"""Start connection to remote server."""
if self.state != "disconnected":
raise RuntimeError, "can't connect in this state"
self.state = "connecting"
if not self.factoryStarted:
self.factory.doStart()
self.factoryStarted = 1
self.transport = transport = self._makeTransport()
if self.timeout is not None:
self.timeoutID = self.reactor.callLater(self.timeout, transport.failIfNotConnected, error.TimeoutError())
self.factory.startedConnecting(self)
def programTimeout(err):
err.trap(error.TimeoutError)
proto.signalProcess('KILL')
return err
def queryUDP(self, queries, timeout = None):
"""
Make a number of DNS queries via UDP.
@type queries: A C{list} of C{dns.Query} instances
@param queries: The queries to make.
@type timeout: Sequence of C{int}
@param timeout: Number of seconds after which to reissue the query.
When the last timeout expires, the query is considered failed.
@rtype: C{Deferred}
@raise C{twisted.internet.defer.TimeoutError}: When the query times
out.
"""
if timeout is None:
timeout = self.timeout
addresses = self.servers + list(self.dynServers)
if not addresses:
return defer.fail(IOError("No domain name servers available"))
used = addresses.pop()
return self.protocol.query(used, queries, timeout[0]
).addErrback(self._reissue, addresses, [used], queries, timeout
)
def _reissue(self, reason, addressesLeft, addressesUsed, query, timeout):
reason.trap(dns.DNSQueryTimeoutError)
# If there are no servers left to be tried, adjust the timeout
# to the next longest timeout period and move all the
# "used" addresses back to the list of addresses to try.
if not addressesLeft:
addressesLeft = addressesUsed
addressesLeft.reverse()
addressesUsed = []
timeout = timeout[1:]
# If all timeout values have been used, or the protocol has no
# transport, this query has failed. Tell the protocol we're
# giving up on it and return a terminal timeout failure to our
# caller.
if not timeout or self.protocol.transport is None:
self.protocol.removeResend(reason.value.id)
return failure.Failure(defer.TimeoutError(query))
# Get an address to try. Take it out of the list of addresses
# to try and put it ino the list of already tried addresses.
address = addressesLeft.pop()
addressesUsed.append(address)
# Issue a query to a server. Use the current timeout. Add this
# function as a timeout errback in case another retry is required.
d = self.protocol.query(address, query, timeout[0], reason.value.id)
d.addErrback(self._reissue, addressesLeft, addressesUsed, query, timeout)
return d
def testTimeout(self):
def login():
d = self.client.login('test', 'twisted')
d.addCallback(loggedIn)
d.addErrback(timedOut)
return d
def loggedIn(result):
self.fail("Successfully logged in!? Impossible!")
def timedOut(failure):
failure.trap(error.TimeoutError)
self._cbStopClient(None)
def quit():
return self.client.quit()
self.client.timeout = 0.01
# Tell the server to not return a response to client. This
# will trigger a timeout.
pop3testserver.TIMEOUT_RESPONSE = True
methods = [login, quit]
map(self.connected.addCallback, map(strip, methods))
self.connected.addCallback(self._cbStopClient)
self.connected.addErrback(self._ebGeneral)
return self.loopback()
def connect(self, factory):
"""Connect with WebSocket over TCP or SSL."""
websocketsFactory = WebSocketsClientFactory()
websocketsFactory.setHandshake(self._handshake)
websocketsFactory.wrappedFactory = factory
endpoint = self._getEndpoint()
deferred = endpoint.connect(websocketsFactory)
protocolReference = [] # Trick to save a reference to the protocol
def onConnectFailure(failure):
# The connection failed (either due to an error or due to
# the low-level endpoint timeout). Let's cancel our own
# timeout and propagate the error.
call.cancel()
return failure
def onConnectSuccess(protocol):
# We're connected, now let's wait for the handshake
protocolReference.append(protocol)
return protocol.deferred.addBoth(onHandshakeFinished)
def onHandshakeFinished(value):
# The handshake has finished, either successfully or not. Unless
# this is a timeout failure itself, let's cancel the timeout call.
if not isinstance(value, Failure) or not value.check(TimeoutError):
call.cancel()
return value
def onTimeout():
# If we got here it means that the handshake has timed out, because
# if the connection times out we cancel our own timeout (see
# onConnectFailure). Let's abort the connection and errback.
[protocol] = protocolReference
protocol.abortHandshake(TimeoutError())
call = self._reactor.callLater(self._timeout, onTimeout)
deferred.addErrback(onConnectFailure)
deferred.addCallback(onConnectSuccess)
return deferred
def process_exception(self, request, exception, spider):
#???????????????????request??????
#???????????????
#????????????????????????????????????
if isinstance(exception, DNSLookupError):
self.crawler.signals.send_catch_log(dnslookuperror, spider=spider)
if isinstance(exception, (TimeoutError, TCPTimedOutError)):
self.crawler.signals.send_catch_log(timeouterror, spider=spider)
return request
