def __init__(self, host="127.0.0.1", port=5672, username="", password="", **connection_options):
"""
Event transport via RabbitMQ server.
:param host: ipv4 or hostname
:param port: the port where the server listens
:param username: username used for authentication
:param password: password used for authentication
:param connection_options: extra arguments that will be used in
:py:class:`pika.BlockingConnection` initialization.
"""
if not pika:
raise RuntimeError("RabbitMqEventTransport requires 'pika' to run")
super(RabbitMqEventTransport, self).__init__()
self._handlers = {}
self.connection = pika.BlockingConnection(
pika.ConnectionParameters(
host=host, port=port,
credentials=pika.PlainCredentials(username=username, password=password),
**connection_options
)
)
self.channel = self.connection.channel()
python类ConnectionParameters()的实例源码
def get_connection_amqp():
try:
port = int(config.get('ckan.harvest.mq.port', PORT))
except ValueError:
port = PORT
userid = config.get('ckan.harvest.mq.user_id', USERID)
password = config.get('ckan.harvest.mq.password', PASSWORD)
hostname = config.get('ckan.harvest.mq.hostname', HOSTNAME)
virtual_host = config.get('ckan.harvest.mq.virtual_host', VIRTUAL_HOST)
credentials = pika.PlainCredentials(userid, password)
parameters = pika.ConnectionParameters(host=hostname,
port=port,
virtual_host=virtual_host,
credentials=credentials,
frame_max=10000)
log.debug("pika connection using %s" % parameters.__dict__)
return pika.BlockingConnection(parameters)
def _check_analysis_queue(queue_name, thread_id=0):
"""
Private static method whose create the queue_name queue as singleton
"""
# check if connection exists for the thread
if thread_id not in Queue.connections:
try:
Queue.connections[thread_id] = pika.BlockingConnection(
pika.ConnectionParameters(Queue.host))
except pika.exceptions.ConnectionClosed as e:
logging.error("Error with RMQ server, check it's started.")
os._exit(1)
Queue.consumers[thread_id] = True
# check if channel exists for the thread
if queue_name not in Queue.channels\
or Queue.channels[queue_name].is_closed:
Queue.channels[queue_name] = Queue.connections[thread_id].channel()
Queue.channels[queue_name].queue_declare(queue=queue_name)
def __init__(self, host: str = 'localhost', port: int = 5672, login: str = 'guest',
password: str = 'guest', virtual_host: str = '/',
ssl: bool = False, *, loop=None, **kwargs):
self.loop = loop if loop else asyncio.get_event_loop()
self.future_store = FutureStore(loop=self.loop)
self.__credentials = PlainCredentials(login, password) if login else None
self.__connection_parameters = ConnectionParameters(
host=host,
port=port,
credentials=self.__credentials,
virtual_host=virtual_host,
ssl=ssl,
**kwargs
)
self._channels = dict()
self._connection = None
self.__closing = None
self.__write_lock = asyncio.Lock(loop=self.loop)
def __init__(self, queue_name, serializer, rabbitmq_configs, *args, **kwargs):
self.queue_name = queue_name
self.serialize = serializer
super(RabbitMQRunner, self).__init__(*args, **kwargs)
self.log(logging.DEBUG, "RabbitMQ Runner is ready...")
def _create_pool():
connection_pool_configs = rabbitmq_configs.get('connection_pool_configs', {})
def create_connection():
self.log(logging.DEBUG, "Creating new rabbitmq connection")
con_params = pika.ConnectionParameters(**rabbitmq_configs.get('connection_parameters', {}))
return pika.BlockingConnection(con_params)
return pika_pool.QueuedPool(
create=create_connection,
**connection_pool_configs
)
self._pool = SimpleLazyObject(_create_pool)
def connect(self):
if self.connecting:
print('PikaClient: Already connecting to RabbitMQ')
return
print('PikaClient: Connecting to RabbitMQ on localhost:5672, Object: %s' % (self,))
self.connecting = True
# credentials = pika.PlainCredentials('guest', 'guest')
# param = pika.ConnectionParameters(host='localhost',
# port=5672,
# virtual_host="/"
# credentials=credentials
# )
param = pika.ConnectionParameters("localhost")
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
#Currently this will close tornado ioloop.
#self.connection.add_on_close_callback(self.on_closed)
def connect(self):
"""
Create blocking connection in RMQ
:return: pika.BlockingConnection
"""
return BlockingConnection(
parameters=ConnectionParameters(
host=self.host,
port=self.port,
virtual_host=self.vhost,
credentials=PlainCredentials(
username=self.user,
password=self.passwd,
)
)
)
def __init__(self, cb_server_address, rmq_username, rmq_password, routing_key):
self.q = Queue.Queue()
self.go = True
# in case the cb url is passed in (which is often required for API stuff),
# try to parse out the IP/DNS information.
# This could be cleaner and better.
cb_server_address = cb_server_address.lower()
if cb_server_address.startswith("https://"):
cb_server_address = cb_server_address[8:]
elif cb_server_address.startswith("http://"):
cb_server_address = cb_server_address[7:]
cb_server_address = cb_server_address.split('/')[0]
# Set the connection parameters to connect to rabbit-server1 on port 5672
# on the / virtual host using the username "guest" and password "guest"
credentials = pika.PlainCredentials(rmq_username, rmq_password)
parameters = pika.ConnectionParameters(cb_server_address,
5004,
'/',
credentials)
self.connection = pika.BlockingConnection(parameters)
self.channel = self.connection.channel()
queue_name = self.__generate_queue_name()
# make sure you use auto_delete so the queue isn't left filling
# with events when this program exists.
self.channel.queue_declare(queue=queue_name, auto_delete=True)
self.channel.queue_bind(exchange='api.events', queue=queue_name, routing_key=routing_key)
self.channel.basic_consume(self.__on_message, queue=queue_name)
threading.Thread.__init__(self)
def from_settings(settings):
"""
:param: settings object
:return: Channel object
"""
connection_type = settings.get('RABBITMQ_CONNECTION_TYPE', RABBITMQ_CONNECTION_TYPE)
connection_parameters = settings.get('RABBITMQ_CONNECTION_PARAMETERS', RABBITMQ_CONNECTION_PARAMETERS)
connection = {
'blocking': pika.BlockingConnection,
'libev': pika.LibevConnection,
'select': pika.SelectConnection,
'tornado': pika.TornadoConnection,
'twisted': pika.TwistedConnection
}[connection_type](pika.ConnectionParameters(**connection_parameters))
channel = connection.channel()
channel.basic_qos(prefetch_count=1)
url = settings.get('REDIS_URL', REDIS_URL)
host = settings.get('REDIS_HOST', REDIS_HOST)
port = settings.get('REDIS_PORT', REDIS_PORT)
# REDIS_URL takes precedence over host/port specification.
if url:
redis_server = redis.from_url(url)
else:
redis_server = redis.Redis(host=host, port=port)
return channel, redis_server
def __init__(self, host, port, user, password, vhost, queue_name):
credentials = pika.PlainCredentials(user, password)
parameters = pika.ConnectionParameters(
host=host, port=port,
virtual_host=vhost, credentials=credentials
)
self.queue_name = queue_name
self.connection = pika.BlockingConnection(parameters)
self.channel = self.connection.channel()
self.channel.confirm_delivery()
def connections(self, wait):
"""
wait for connections to both rabbitmq and elasticsearch to be made
before binding a routing key to a channel and sending messages to
elasticsearch
"""
while wait:
try:
params = pika.ConnectionParameters(host=self.rmq_host,
port=self.rmq_port)
connection = pika.BlockingConnection(params)
self.channel = connection.channel()
self.channel.exchange_declare(exchange='topic_recs',
exchange_type='topic')
result = self.channel.queue_declare(exclusive=True)
self.queue_name = result.method.queue
self.es_conn = Elasticsearch([{'host': self.es_host,
'port': self.es_port}])
wait = False
print("connected to rabbitmq...")
except Exception as e: # pragma: no cover
print(str(e))
print("waiting for connection to rabbitmq..." + str(e))
time.sleep(2)
wait = True
def _connect(self):
cp = pika.ConnectionParameters(host=self.host, port=self.port)
self._conn = pika.BlockingConnection(cp)
def getMqConnection(mqConf,vErrors,maxMsgTotal):
# try to connect via amqp
amqpLink = pika.BlockingConnection(
pika.ConnectionParameters(
mqConf["server"],
amqpPort,
mqConf.get("vhost",'/'),
pika.PlainCredentials(mqConf["user"], mqConf["pwd"])))
return amqpLink
def __init__(self):
app_conf = imp.load_source('app_conf', os.getenv('EAGLE_HOME', '..') + '/eagle_cfg.py')
cred = pika.credentials.PlainCredentials(app_conf.MQ_USERNAME, app_conf.MQ_PASSWORD)
parameter = pika.ConnectionParameters(host=app_conf.MQ_HOST, port=app_conf.MQ_PORT, credentials=cred)
self.connection = pika.BlockingConnection(parameters=parameter)
self.channel = self.connection.channel()
def __init__(self, parent):
self.parent = parent
self.cfg = parent.cfg
self.ae = parent.ae
self.fq = parent.fuzzing_queues
self.utils = parent.utils
self.fo = parent.fileops
self.cth = parent.cthulhu
try:
self.connection = pika.BlockingConnection(
pika.ConnectionParameters(host = 'localhost'))
self.ae.m_ok("Successfully connected to message queue (broker)")
except Exception:
self.ae.m_fatal("[!] Could not connect to the message queue!")
self.channel = self.connection.channel()
###########################################################
# Declare queue serving mutations to clients
###########################################################
self.channel.queue_declare(queue = 'rpc_mutations_queue')
self.channel.basic_qos(prefetch_count = 1)
self.channel.basic_consume(self.on_mutation_request,
queue = 'rpc_mutations_queue')
###########################################################
# Declare queue receiveing mutation objects from clients
###########################################################
self.channel.queue_declare(queue = 'rpc_evaluations_queue')
self.channel.basic_qos(prefetch_count = 1)
self.channel.basic_consume(self.on_evaluation_request,
queue = 'rpc_evaluations_queue')
def __init__(self):
print("Authenticating with rabbitmq server : %s" % HOSTURI)
credentials = pika.PlainCredentials('simplerpc', 'simplerpc')
connURI = pika.ConnectionParameters(host=HOSTURI,
credentials=credentials)
self.connection = pika.BlockingConnection(connURI)
self.channel = self.connection.channel()
result = self.channel.queue_declare(exclusive=True)
self.callback_queue = result.method.queue
self.channel.basic_consume(self.on_response, no_ack=True,
queue=self.callback_queue)
print("Authentication seems to be successful!")
def run(self):
while True:
try:
self.log(logging.DEBUG, "Running the RabbitMQ worker: {}".format(os.getpid()))
with pika.BlockingConnection(pika.ConnectionParameters(**self.connection_params)) as connection:
channel = connection.channel()
channel.queue_declare(queue=self.queue_name, durable=True)
channel.basic_qos(prefetch_count=1)
for message_object in channel.consume(queue=self.queue_name, inactivity_timeout=10):
if message_object is None:
connection.process_data_events(time_limit=5)
else:
self.callback(channel, *message_object)
except Exception as exp:
self.log(logging.ERROR, "Worker have issues while receiving: {}".format(exp))
def get_connection():
params = pika.ConnectionParameters(host=_RABBIT_MQ_HOST)
connection = pika.BlockingConnection(params)
return connection
def get_connection():
# ????
parameters = pika.ConnectionParameters(RABBIT_MQ_HOST, RABBIT_MQ_PORT)
# ????????
connection = pika.BlockingConnection(parameters)
return connection
def _get_pika_channel_connection(self):
""" Connect to pika server and return channel and connection"""
parameters = pika.ConnectionParameters(host=self.host, port=self.port)
connection = pika.BlockingConnection(parameters)
channel = connection.channel()
channel.exchange_declare(exchange=self.exchange)
channel.queue_declare(queue=self.queue_name, durable=True,
arguments={'x-message-ttl': self.expire_ms, })
channel.queue_bind(queue=self.queue_name,
exchange=self.exchange,
routing_key=self.exchange + '-' + self.queue_name)
return channel, connection
def connect(self):
"""
?rabbitmq ???????????????????????
:return:
"""
if self.connect_count <= 0 and self.connection is None or self.connection.is_closed:
credentials = pika.PlainCredentials(self.user, self.pwd)
parameters = pika.ConnectionParameters(self.host, self.port, '/', credentials)
self.connection = pika.BlockingConnection(parameters)
self.channel = self.connection.channel()
self.connect_count += 1
def connect(self):
if self.connecting:
return
self.connecting = True
param = pika.ConnectionParameters(host=settings.RABBITMQ_HOST)
self.connection = pika.adapters.tornado_connection.TornadoConnection(
param,
on_open_callback=self.on_connected,
on_open_error_callback=self.on_connection_error,
on_close_callback=self.close)
def __init__(self, addr):
threading.Thread.__init__(self)
ip, port = addr
self.url = ip + '/' + str(port)
creden = pika.PlainCredentials(RABBITU, RABBITP)
params = pika.ConnectionParameters(host=ip, credentials=creden)
self.connection = pika.BlockingConnection(params)
self.channel = self.connection.channel()
self.channel.queue_declare(queue=self.url)
# self.channel.basic_qos(prefetch_count=1)
def __init__(self, url):
aurl = urlparse(url)
address = aurl.netloc.split(':')
ip, port = address[0], int(address[1])
self.url = ip + '/' + str(port)
creden = pika.PlainCredentials(RABBITU, RABBITP)
params = pika.ConnectionParameters(host=ip, credentials=creden)
self.connection = pika.BlockingConnection(params)
self.channel = self.connection.channel()
def publish(body):
""" Publishing message.
Args:
body (str): message.
"""
connection = pika.BlockingConnection(
pika.ConnectionParameters(host=rpc_host))
channel = connection.channel()
channel.basic_publish(exchange='',
routing_key=CONF['os_info']['REGION_NAME'],
body=body)
def start():
""" Start cntl server. """
connection = pika.BlockingConnection(
pika.ConnectionParameters(host=CONF['rpc']['address']))
channel = connection.channel()
channel.queue_declare(queue=CONF['os_info']['REGION_NAME'])
channel.basic_consume(handle,
queue=CONF['os_info']['REGION_NAME'],
no_ack=True)
channel.start_consuming()
def connect(self):
if self.connecting:
return
self.connecting = True
cred = pika.PlainCredentials('guest', 'guest')
param = pika.ConnectionParameters(
host='localhost',
port=5672,
virtual_host='/',
credentials=cred
)
self.connection = TornadoConnection(param,
on_open_callback=self.on_connected)
def connection(server, user, password):
credentials = pika.PlainCredentials(user, password)
try:
connection = pika.BlockingConnection(
pika.ConnectionParameters(
host=server,
credentials=credentials,
)
)
return connection
except:
message = "Failed rabbit connection to server {}".format(server)
log.exception(message)
raise RabbitConnectionFailed(message)
connection.py 文件源码
项目:Distributed-Multi-User-Scrapy-System-with-a-Web-UI
作者: aaldaber
项目源码
文件源码
阅读 22
收藏 0
点赞 0
评论 0
def from_settings(settings, spider_name):
connection_type = settings.get('RABBITMQ_CONNECTION_TYPE',
RABBITMQ_CONNECTION_TYPE)
queue_name = "%s:requests" % spider_name
connection_host = settings.get('RABBITMQ_HOST')
connection_port = settings.get('RABBITMQ_PORT')
connection_username = settings.get('RABBITMQ_USERNAME')
connection_pass = settings.get('RABBITMQ_PASSWORD')
connection_attempts = 5
retry_delay = 3
credentials = pika.PlainCredentials(connection_username, connection_pass)
connection = {
'blocking': pika.BlockingConnection,
'libev': pika.LibevConnection,
'select': pika.SelectConnection,
'tornado': pika.TornadoConnection,
'twisted': pika.TwistedConnection
}[connection_type](pika.ConnectionParameters(host=connection_host,
port=connection_port, virtual_host='/',
credentials=credentials,
connection_attempts=connection_attempts,
retry_delay=retry_delay))
channel = connection.channel()
channel.queue_declare(queue=queue_name, durable=True)
return channel
def run(self, input_cmd):
body = input_cmd.split("run \"")[1].split("\" ")[0]
hosts = input_cmd.split("--hosts")[1].strip().split(" ")
for host_ip in hosts:
if self.judge_legal_ip(host_ip): # ??IP??????
'''
self.connections = {
host_ip1:{
'connection': pika.BlockingConnection(),
'channel': Channel1,
'callback_queue': result.method.queue
},
host_ip2:{
'connection': pika.BlockingConnection(),
'channel': Channel2,
'callback_queue': result.method.queue
}
}
'''
print("IP [%s] is Legal ip" % host)
conn_info = {}
conn_info['connection'] = pika.BlockingConnection(
pika.ConnectionParameters(host=host))
conn_info['channel'] = conn_info['connection'].channel(
)
result = self.channel.queue_declare(exclusive=True)
conn_info['callback_queue'] = result.method.queue
conn_info['channel'].basic_consume(
self.on_response, no_ack=True, queue=conn_info['callback_queue'])
conn_info['response'] = None
self.connections[host_ip] = conn_info
else:
print("IP [%s] is Illegal ip" % host_ip)