def monkey_patch():
"""Greening the world (eventlet)
The approach is to monkeypatch the Standard Library. For more details
see http://eventlet.net/doc/patching.html
"""
if os.name == 'nt':
# eventlet monkey patching the os and thread modules causes
# subprocess.Popen to fail on Windows when using pipes due
# to missing non-blocking IO support.
#
# bug report on eventlet:
# https://bitbucket.org/eventlet/eventlet/issue/132/
# eventletmonkey_patch-breaks
eventlet.monkey_patch(os=False, thread=False)
else:
eventlet.monkey_patch()
python类net()的实例源码
def producer(start_url):
"""Recursively crawl starting from *start_url*. Returns a set of
urls that were found."""
pool = eventlet.GreenPool()
seen = set()
q = eventlet.Queue()
q.put(start_url)
# keep looping if there are new urls, or workers that may produce more urls
while True:
while not q.empty():
url = q.get()
# limit requests to eventlet.net so we don't crash all over the internet
if url not in seen and 'eventlet.net' in url:
seen.add(url)
pool.spawn_n(fetch, url, q)
pool.waitall()
if q.empty():
break
return seen
def _yield_before_after():
# Yield to any other co-routines...
#
# See: http://eventlet.net/doc/modules/greenthread.html
# for how this zero sleep is really a cooperative yield to other potential
# co-routines...
eventlet.sleep(0)
try:
yield
finally:
eventlet.sleep(0)
def _yield_before_after():
# Yield to any other co-routines...
#
# See: http://eventlet.net/doc/modules/greenthread.html
# for how this zero sleep is really a cooperative yield to other potential
# co-routines...
eventlet.sleep(0)
try:
yield
finally:
eventlet.sleep(0)
def fetch(url, seen, pool):
"""Fetch a url, stick any found urls into the seen set, and
dispatch any new ones to the pool."""
print("fetching", url)
data = ''
with eventlet.Timeout(5, False):
data = urllib2.urlopen(url).read()
for url_match in url_regex.finditer(data):
new_url = url_match.group(0)
# only send requests to eventlet.net so as not to destroy the internet
if new_url not in seen and 'eventlet.net' in new_url:
seen.add(new_url)
# while this seems stack-recursive, it's actually not:
# spawned greenthreads start their own stacks
pool.spawn_n(fetch, new_url, seen, pool)
