def openBrowserRecursively(total, idName, browser):
try:
for i in range(total):
iterator = i * 100
url = r"https://www.google.com/search?q={word}&newwindow=1&biw=300&bih=629&tbm=isch&ijn={times}&start={start}"
try:
browser.get(url.format(word= idName, start=iterator,times = i))
except SocketError as e:
if e.errno != errno.ECONNRESET:
raise # raise to reset the connection
pass
time.sleep(1.5) # 1.5 seconds is the tuned time for HKU service not to be monitored and closed
except:
if isWindows:
os.system("taskkill /im chrome.exe /F")
else :
os.system("kill " + str(os.getpid()))
openBrowserRecursively(total, idName, browser)
# basic session setup
python类ECONNRESET的实例源码
def _receiveWithTimeout(self, count):
chunk = str()
while len(chunk) < count:
try:
tmp = self.socket.recv(count - len(chunk))
except socket.timeout:
if not self._running:
print("CLIENT: Socket timed out and termination requested.")
return None
else:
continue
except socket.error as e:
if e[0] == errno.ECONNRESET:
print("CLIENT: Connection reset by peer.")
return None
else:
raise e
if not tmp or len(tmp) == 0:
print("CLIENT: Socket has been closed.")
return None
chunk = chunk + tmp
return chunk
def _transmitWithTimeout(self, data):
sent = 0
while sent < len(data):
try:
tmp = self.socket.send(data[sent:])
except socket.timeout:
if not self._running:
print("CLIENT: Socket timed out and termination requested.")
return False
else:
continue
except socket.error as e:
if e[0] == errno.ECONNRESET:
print("CLIENT: Connection reset by peer.")
return False
else:
raise e
if tmp == 0:
return False
sent += tmp
return True
def _receiveWithTimeout(self, count):
chunk = str()
while len(chunk) < count:
try:
tmp = self.socket.recv(count - len(chunk))
except socket.timeout:
if not self._running:
print "CLIENT: Socket timed out and termination requested."
return None
else:
continue
except socket.error, e:
if e[0] == errno.ECONNRESET:
print "CLIENT: Connection reset by peer."
return None
else:
raise e
if not tmp or len(tmp) == 0:
print "CLIENT: Socket has been closed."
return None
chunk = chunk + tmp
return chunk
def request(self, host, handler, request_body, verbose=0):
#retry request once if cached connection has gone cold
for i in (0, 1):
try:
return self.single_request(host, handler, request_body, verbose)
except socket.error, e:
if i or e.errno not in (errno.ECONNRESET, errno.ECONNABORTED, errno.EPIPE):
raise
except httplib.BadStatusLine: #close after we sent request
if i:
raise
##
# Send a complete request, and parse the response.
#
# @param host Target host.
# @param handler Target PRC handler.
# @param request_body XML-RPC request body.
# @param verbose Debugging flag.
# @return Parsed response.
def test_serialized_with_retry_socket_error_conn_reset(self):
exc = socket.error()
exc.errno = errno.ECONNRESET
plugin = 'glance'
fn = 'download_vhd'
num_retries = 1
callback = None
retry_cb = mock.Mock()
with mock.patch.object(self.session, 'call_plugin_serialized',
spec=True) as call_plugin_serialized:
call_plugin_serialized.side_effect = exc
self.assertRaises(
exception.PluginRetriesExceeded,
self.session.call_plugin_serialized_with_retry, plugin, fn,
num_retries, callback, retry_cb)
call_plugin_serialized.assert_called_with(plugin, fn)
self.assertEqual(2, call_plugin_serialized.call_count)
self.assertEqual(2, retry_cb.call_count)
def test_serialized_with_retry_socket_reset_reraised(self):
exc = socket.error()
exc.errno = errno.ECONNRESET
plugin = 'glance'
fn = 'download_vhd'
num_retries = 1
callback = None
retry_cb = mock.Mock()
with mock.patch.object(self.session, 'call_plugin_serialized',
spec=True) as call_plugin_serialized:
call_plugin_serialized.side_effect = exc
self.assertRaises(
exception.PluginRetriesExceeded,
self.session.call_plugin_serialized_with_retry, plugin, fn,
num_retries, callback, retry_cb)
call_plugin_serialized.assert_called_with(plugin, fn)
self.assertEqual(2, call_plugin_serialized.call_count)
def request(self, host, handler, request_body, verbose=0):
#retry request once if cached connection has gone cold
for i in (0, 1):
try:
return self.single_request(host, handler, request_body, verbose)
except socket.error, e:
if i or e.errno not in (errno.ECONNRESET, errno.ECONNABORTED, errno.EPIPE):
raise
except httplib.BadStatusLine: #close after we sent request
if i:
raise
##
# Send a complete request, and parse the response.
#
# @param host Target host.
# @param handler Target PRC handler.
# @param request_body XML-RPC request body.
# @param verbose Debugging flag.
# @return Parsed response.
def handle_one_request(self):
if not self.disable_transport_ssl and self.scheme == 'http':
leadbyte = self.connection.recv(1, socket.MSG_PEEK)
if leadbyte in ('\x80', '\x16'):
server_name = ''
if leadbyte == '\x16':
for _ in xrange(2):
leaddata = self.connection.recv(1024, socket.MSG_PEEK)
if is_clienthello(leaddata):
try:
server_name = extract_sni_name(leaddata)
finally:
break
try:
certfile = CertUtil.get_cert(server_name or 'www.google.com')
ssl_sock = ssl.wrap_socket(self.connection, ssl_version=self.ssl_version, keyfile=certfile, certfile=certfile, server_side=True)
except StandardError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
logging.exception('ssl.wrap_socket(self.connection=%r) failed: %s', self.connection, e)
return
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
self.scheme = 'https'
return BaseHTTPServer.BaseHTTPRequestHandler.handle_one_request(self)
def read(self, n):
buffer = []
while self.offset < self.file_size:
try:
data = self.stream.read(min(n, self.file_size - self.offset))
self.offset += len(data)
n -= len(data)
buffer.append(data)
if n == 0 or data:
break
except socket.timeout:
self._progress()
self._restart()
except socket.error as e:
if e.errno != errno.ECONNRESET:
raise
self._progress()
self._restart()
return "".join(buffer)
Wikipedia_Crawler.py 文件源码
项目:Entity-Triple-Entity-Extraction
作者: shubham0420
项目源码
文件源码
阅读 23
收藏 0
点赞 0
评论 0
def URL_to_HTML(URL):
try:
HTML = urllib2.urlopen(URL)
except urllib2.HTTPError as error:
print u'HTTPError: {0} ({1})'.format(URL, error.code)
except urllib2.URLError as error:
print u'URLError: {0} ({1})'.format(URL, error.reason)
except httplib.BadStatusLine as error:
print u'BadStatusLine: {}'.format(URL)
except SocketError as error:
if error.errno != errno.ECONNRESET:
raise
pass
else:
Charset = HTML.headers['content-type'][HTML.headers['content-type'].index('=') + 1:]
HTML = unicode(HTML.read(), Charset)
return HTML
def request(self, host, handler, request_body, verbose=0):
#retry request once if cached connection has gone cold
for i in (0, 1):
try:
return self.single_request(host, handler, request_body, verbose)
except socket.error, e:
if i or e.errno not in (errno.ECONNRESET, errno.ECONNABORTED, errno.EPIPE):
raise
except httplib.BadStatusLine: #close after we sent request
if i:
raise
##
# Send a complete request, and parse the response.
#
# @param host Target host.
# @param handler Target PRC handler.
# @param request_body XML-RPC request body.
# @param verbose Debugging flag.
# @return Parsed response.
def request(self, host, handler, request_body, verbose=0):
#retry request once if cached connection has gone cold
for i in (0, 1):
try:
return self.single_request(host, handler, request_body, verbose)
except socket.error, e:
if i or e.errno not in (errno.ECONNRESET, errno.ECONNABORTED, errno.EPIPE):
raise
except httplib.BadStatusLine: #close after we sent request
if i:
raise
##
# Send a complete request, and parse the response.
#
# @param host Target host.
# @param handler Target PRC handler.
# @param request_body XML-RPC request body.
# @param verbose Debugging flag.
# @return Parsed response.
def read(self, sz):
try:
buff = self.handle.recv(sz)
except socket.error, e:
if (e.args[0] == errno.ECONNRESET and
(sys.platform == 'darwin' or sys.platform.startswith('freebsd'))):
# freebsd and Mach don't follow POSIX semantic of recv
# and fail with ECONNRESET if peer performed shutdown.
# See corresponding comment and code in TSocket::read()
# in lib/cpp/src/transport/TSocket.cpp.
self.close()
# Trigger the check to raise the END_OF_FILE exception below.
buff = ''
else:
raise
if len(buff) == 0:
raise TTransportException(type=TTransportException.END_OF_FILE, message='TSocket read 0 bytes')
return buff
def request(self, host, handler, request_body, verbose=0):
#retry request once if cached connection has gone cold
for i in (0, 1):
try:
return self.single_request(host, handler, request_body, verbose)
except socket.error, e:
if i or e.errno not in (errno.ECONNRESET, errno.ECONNABORTED, errno.EPIPE):
raise
except httplib.BadStatusLine: #close after we sent request
if i:
raise
##
# Send a complete request, and parse the response.
#
# @param host Target host.
# @param handler Target PRC handler.
# @param request_body XML-RPC request body.
# @param verbose Debugging flag.
# @return Parsed response.
def pcl_can_query_retry(e):
"""
???????? ?????????? ??? ?????? pcl_query_retry ?? ??????????? ?????????? ???????
?????????:
e (Exception) -- ?????????? ?? pcl_query_retry
?????????:
None ??? ?????????????? ??????????
"""
if type(e) == pclError:
# https://docs.pcloud.com/errors/index.html
if e.errno >= 3000:
pass
# HTTP
elif (e.errno >= 500 and e.errno < 600) or e.errno == 401 or e.errno == 429:
pass
else:
raise e
elif type(e) == socket.error and not (e.errno == errno.ECONNRESET or e.errno == errno.ECONNREFUSED):
raise e
def sendfile_wrapper(self, sock, file, offset, nbytes, headers=[], trailers=[]):
"""A higher level wrapper representing how an application is
supposed to use sendfile().
"""
while 1:
try:
if self.SUPPORT_HEADERS_TRAILERS:
return os.sendfile(sock, file, offset, nbytes, headers,
trailers)
else:
return os.sendfile(sock, file, offset, nbytes)
except OSError as err:
if err.errno == errno.ECONNRESET:
# disconnected
raise
elif err.errno in (errno.EAGAIN, errno.EBUSY):
# we have to retry send data
continue
else:
raise
def _write(self, length):
# we need to write all the data but it's a non-blocking socket
# so loop until it's all written eating EAGAIN exceptions
data=self.netbuf_out_mv
written=0
while written < length:
try:
written += self.sock_write(data[written:length])
except OSError as e:
if len(e.args) > 0:
if e.args[0] == errno.EAGAIN:
# can't write yet, try again
pass
elif e.args[0] == errno.ECONNRESET: # connection closed
return # we are done: TODO: error?
else:
raise
else:
# something else...propagate the exception
raise
def _send_loop(self, send_method, data, *args):
if self.act_non_blocking:
return send_method(data, *args)
while 1:
try:
return send_method(data, *args)
except socket.error as e:
eno = get_errno(e)
if eno == errno.ENOTCONN or eno not in SOCKET_BLOCKING:
raise
try:
self._trampoline(self.fd, write=True, timeout=self.gettimeout(),
timeout_exc=socket.timeout("timed out"))
except IOClosed:
raise socket.error(errno.ECONNRESET, 'Connection closed by another thread')
def handle_one_request(self):
if not self.disable_transport_ssl and self.scheme == 'http':
leadbyte = self.connection.recv(1, socket.MSG_PEEK)
if leadbyte in ('\x80', '\x16'):
server_name = ''
if leadbyte == '\x16':
for _ in xrange(2):
leaddata = self.connection.recv(1024, socket.MSG_PEEK)
if is_clienthello(leaddata):
try:
server_name = extract_sni_name(leaddata)
finally:
break
try:
certfile = CertUtil.get_cert(server_name or 'www.google.com')
ssl_sock = ssl.wrap_socket(self.connection, ssl_version=self.ssl_version, keyfile=certfile, certfile=certfile, server_side=True)
except StandardError as e:
if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET):
logging.exception('ssl.wrap_socket(self.connection=%r) failed: %s', self.connection, e)
return
self.connection = ssl_sock
self.rfile = self.connection.makefile('rb', self.bufsize)
self.wfile = self.connection.makefile('wb', 0)
self.scheme = 'https'
return BaseHTTPServer.BaseHTTPRequestHandler.handle_one_request(self)
def _handle_bt_error(self, bt_error):
assert isinstance(bt_error, IOError)
#'timed out' is caused by the wait_for_connection loop
if isinstance(bt_error, socket.timeout):
pass
#'resource unavailable' is when data cannot be read because there is nothing in the buffer
elif bt_error.errno == errno.EAGAIN:
pass
#'connection reset' is caused when the client disconnects
elif bt_error.errno == errno.ECONNRESET:
self._client_connected = False
if self.when_client_disconnects:
self.when_client_disconnects()
#'conection timeout' is caused when the server can no longer connect to read from the client
# (perhaps the client has gone out of range)
elif bt_error.errno == errno.ETIMEDOUT:
self._client_connected = False
if self.when_client_disconnects:
self.when_client_disconnects()
else:
raise bt_error
def do_GET(self):
query = self.path.split("?",1)[-1]
filepath = urllib.unquote_plus(query)
self.suppress_socket_error_report = None
self.send_headers(filepath)
print "sending data"
try:
self.write_response(filepath)
except socket.error, e:
if isinstance(e.args, tuple):
if e[0] in (errno.EPIPE, errno.ECONNRESET):
print "disconnected"
self.suppress_socket_error_report = True
return
raise
def request(self, host, handler, request_body, verbose=0):
#retry request once if cached connection has gone cold
for i in (0, 1):
try:
return self.single_request(host, handler, request_body, verbose)
except socket.error, e:
if i or e.errno not in (errno.ECONNRESET, errno.ECONNABORTED, errno.EPIPE):
raise
except httplib.BadStatusLine: #close after we sent request
if i:
raise
##
# Send a complete request, and parse the response.
#
# @param host Target host.
# @param handler Target PRC handler.
# @param request_body XML-RPC request body.
# @param verbose Debugging flag.
# @return Parsed response.
def sendfile_wrapper(self, sock, file, offset, nbytes, headers=[], trailers=[]):
"""A higher level wrapper representing how an application is
supposed to use sendfile().
"""
while 1:
try:
if self.SUPPORT_HEADERS_TRAILERS:
return os.sendfile(sock, file, offset, nbytes, headers,
trailers)
else:
return os.sendfile(sock, file, offset, nbytes)
except OSError as err:
if err.errno == errno.ECONNRESET:
# disconnected
raise
elif err.errno in (errno.EAGAIN, errno.EBUSY):
# we have to retry send data
continue
else:
raise
def request(self, host, handler, request_body, verbose=0):
#retry request once if cached connection has gone cold
for i in (0, 1):
try:
return self.single_request(host, handler, request_body, verbose)
except socket.error, e:
if i or e.errno not in (errno.ECONNRESET, errno.ECONNABORTED, errno.EPIPE):
raise
except httplib.BadStatusLine: #close after we sent request
if i:
raise
##
# Send a complete request, and parse the response.
#
# @param host Target host.
# @param handler Target PRC handler.
# @param request_body XML-RPC request body.
# @param verbose Debugging flag.
# @return Parsed response.
def test_listen(self):
logger = self.useFixture(FakeLogger(format="%(asctime)s %(message)s"))
self.executable.listen(6666)
self.executable.sleep(1)
self.executable.spawn()
# Try to connect to the socket
sock = socket.socket()
transient = (errno.ECONNREFUSED, errno.ECONNRESET)
attempts = 10
for i in range(1, attempts + 1):
try:
sock.connect(("127.0.0.1", self.executable.port))
except socket.error as error: # pragma: no cover
if error.errno in transient and i != attempts:
time.sleep(0.05 * i)
continue
logging.error("connection attempt %d failed", i)
raise error
break
self.assertEqual("127.0.0.1", sock.getsockname()[0])
def request(self, host, handler, request_body, verbose=0):
#retry request once if cached connection has gone cold
for i in (0, 1):
try:
return self.single_request(host, handler, request_body, verbose)
except socket.error, e:
if i or e.errno not in (errno.ECONNRESET, errno.ECONNABORTED, errno.EPIPE):
raise
except httplib.BadStatusLine: #close after we sent request
if i:
raise
##
# Send a complete request, and parse the response.
#
# @param host Target host.
# @param handler Target PRC handler.
# @param request_body XML-RPC request body.
# @param verbose Debugging flag.
# @return Parsed response.
def _receiveWithTimeout(self, count):
chunk = str()
while len(chunk) < count:
try:
tmp = self.socket.recv(count - len(chunk))
except socket.timeout:
if not self._running:
print "CLIENT: Socket timed out and termination requested."
return None
else:
continue
except socket.error, e:
if e[0] == errno.ECONNRESET:
print "CLIENT: Connection reset by peer."
return None
else:
raise e
if not tmp or len(tmp) == 0:
print "CLIENT: Socket has been closed."
return None
chunk = chunk + tmp
return chunk
def _on_local_error(self):
if self._local_sock:
err = eventloop.get_sock_error(self._local_sock)
if err.errno not in [errno.ECONNRESET, errno.EPIPE]:
logging.error(err)
logging.error("local error, exception from %s:%d" % (self._client_address[0], self._client_address[1]))
self.destroy()
def _on_remote_error(self):
if self._remote_sock:
err = eventloop.get_sock_error(self._remote_sock)
if err.errno not in [errno.ECONNRESET]:
logging.error(err)
if self._remote_address:
logging.error("remote error, when connect to %s:%d" % (self._remote_address[0], self._remote_address[1]))
else:
logging.error("remote error, exception from %s:%d" % (self._client_address[0], self._client_address[1]))
self.destroy()