def configure_nginx_if_necessary():
nginx_config_path = os.path.join('/etc/nginx/sites-available', env.domain_name)
if exists(nginx_config_path):
print('nginx config found, not creating another one')
else:
nginx_config_variables = {
'source_dir': PROJECT_FOLDER,
'domain': env.domain_name,
'ssl_params_path': SSL_PARAMS_PATH,
'fullchain_path': os.path.join(env.letsencrypt_folder, 'fullchain.pem'),
'privkey_path': os.path.join(env.letsencrypt_folder, 'privkey.pem'),
'socket_path': SOCKET_PATH
}
upload_template(
filename='deploy_configs/nginx_config',
destination=nginx_config_path,
context=nginx_config_variables,
use_sudo=True
)
nginx_config_alias = os.path.join('/etc/nginx/sites-enabled', env.domain_name)
sudo('ln -sf %s %s' % (nginx_config_path, nginx_config_alias))
python类sudo()的实例源码
def setup_system():
"""Setup the system dependencies and repo.
"""
add_apt('fkrull/deadsnakes')
apt_install(
'emacs-nox',
'python3.6-dev',
'python3.6-gdbm',
'python3.6-venv',
'nginx',
'nginx-core',
'screen',
'gcc',
'libssl-dev',
)
ensure_venv('combine')
sudo('mkdir -p /tmp/gunicorn_run')
sudo('chmod 777 /tmp/gunicorn_run')
restart_nginx()
sudo('mkdir -p /var/run/watch-ip')
sudo('chmod 777 /var/run/watch-ip')
def prepare_apt():
"""
Download software from apt
Note, on a slower internet connection, this will take a while to finish,
because it has to download many packages, include latex and all its
dependencies.
"""
sudo("apt-get -qq update")
sudo("apt-get -y install git python3 make python-virtualenv zip python-dev")
# Needed to build the docs
sudo("apt-get -y install graphviz inkscape texlive texlive-xetex texlive-fonts-recommended texlive-latex-extra librsvg2-bin")
# Our Ubuntu is too old to include Python 3.3
sudo("apt-get -y install python-software-properties")
sudo("add-apt-repository -y ppa:fkrull/deadsnakes")
sudo("apt-get -y update")
sudo("apt-get -y install python3.3")
def deploy_consul_binary(self):
""" Install the consul software """
log_green('deploying consul binary...')
with settings(
hide('stdout', 'running'),
host_string=self.host_string,
private_key_filename=self.private_key
):
apt_install(packages=['unzip'])
with cd('/usr/local/bin'):
if 'consul' not in sudo('ls /usr/local/bin'):
sudo(
'wget -c https://releases.hashicorp.com/consul/%s'
'/consul_%s_linux_amd64.zip' % (self.version,
self.version)
)
sudo('unzip *zip')
sudo('rm -f *.zip')
add_usr_local_bin_to_path()
def download_consul_web_ui_files(self):
""" installs the consul web ui files """
log_green('install web ui for consul...')
with settings(
hide('stdout', 'running'),
host_string=self.host_string,
private_key_filename=self.private_key,
):
if 'dist' not in sudo('ls /home/consul/'):
with cd('/home/consul'):
sudo(
'wget -c '
'https://releases.hashicorp.com/consul/'
'%s/consul_%s_web_ui.zip' % (
self.version, self.version),
user='consul'
)
sudo('unzip -o *.zip', user='consul')
sudo('rm -f *.zip', user='consul')
def create_consul_server_init_script(self):
""" creates the consul server init file """
log_green('create consul server init script...')
with settings(
hide('stdout', 'running'),
host_string=self.host_string,
private_key_filename=self.private_key,
):
consul_init_file = '/etc/systemd/system/consul-server.service'
upload_template(filename='consul-init-server.j2',
template_dir='templates',
destination=consul_init_file,
use_sudo=True,
use_jinja=True,
backup=False,
context={'consul_interface': self.consul_interface,
'node_ip': self.consul_ip})
sudo('systemctl daemon-reload')
sudo('systemctl enable consul-server')
def create_consul_client_init_script(self, tinc_network_name):
""" creates the consul client init file """
log_green('create consul client init script ...')
with settings(
hide('stdout', 'running'),
host_string=self.host_string,
private_key_filename=self.private_key,
):
consul_init_file = '/etc/systemd/system/consul-client.service'
upload_template(filename='consul-init-client.j2',
template_dir='templates',
destination=consul_init_file,
use_sudo=True,
use_jinja=True,
backup=False,
context={'tinc_network_name': tinc_network_name,
'node_ip': self.tinc_ip})
sudo('systemctl daemon-reload')
sudo('systemctl enable consul-client')
def install_fsconsul(self):
""" installs fsconsul """
log_green('installing fsconsul ...')
with settings(
hide('stdout', 'running'),
host_string=self.host_string,
private_key_filename=self.private_key
):
add_usr_local_bin_to_path()
with cd('/usr/local/bin'):
if 'fsconsul' not in sudo('ls'):
sudo('wget -O fsconsul -c '
'https://bintray.com/cimpress-mcp/Go/download_file?'
'file_path=v0.6.5%2Flinux-amd64%2Ffsconsul')
sudo('chmod 755 fsconsul')
def test_that_patches_were_installed_on(node):
line = '0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded'
with settings(
hide('stdout', 'running'),
host_string=node.host_string,
private_key_filename=node.private_key
):
print(" running on %s" % node.host_string)
cmd = sudo('apt-get -u upgrade --assume-no')
try:
assert line in cmd.stdout
except Exception as detail:
raise Exception("%s %s" % (cmd.stdout, detail))
try:
assert cmd.return_code == 0
except Exception as detail:
raise Exception("%s %s" % (cmd.return_code, detail))
def test_that_cron_apt_is_installed_on(node):
line = 'cron-apt'
with settings(
hide('stdout', 'running'),
host_string=node.host_string,
private_key_filename=node.private_key
):
print(" running on %s" % node.host_string)
cmd = sudo('dpkg -l')
try:
assert line in cmd.stdout
except Exception as detail:
raise Exception("%s %s" % (cmd.stdout, detail))
try:
assert cmd.return_code == 0
except Exception as detail:
raise Exception("%s %s" % (cmd.return_code, detail))
def test_that_tinc_binaries_were_installed_on(node):
line = '/usr/sbin/tincd'
with settings(
hide('stdout', 'running'),
host_string=node.host_string,
private_key_filename=node.private_key
):
print(" running on %s" % node.host_string)
cmd = sudo('which tincd')
try:
assert line in cmd.stdout
except Exception as detail:
raise Exception("%s %s" % (cmd.stdout, detail))
def test_that_tinc_key_pairs_were_deployed_on(tinc_network):
for tinc_node in tinc_network.tinc_nodes:
with settings(
hide('stdout', 'running'),
host_string=tinc_node.host_string,
private_key_filename=tinc_node.private_key
):
tinc_network_name = tinc_network.tinc_network_name
print(" running on %s" % tinc_node.host_string)
cmd = sudo('ls -l /etc/tinc/%s' % tinc_network_name)
try:
assert 'rsa_key.priv' in cmd.stdout
except Exception as detail:
raise Exception("%s %s" % (cmd.stdout, detail))
try:
assert 'rsa_key.pub' in cmd.stdout
except Exception as detail:
raise Exception("%s %s" % (cmd.stdout, detail))
def test_that_tinc_conf_files_were_deployed_on(tinc_network):
for tinc_node in tinc_network.tinc_nodes:
with settings(
hide('stdout', 'running'),
host_string=tinc_node.host_string,
private_key_filename=tinc_node.private_key
):
tinc_network_name = tinc_network.tinc_network_name
print(" running on %s" % tinc_node.host_string)
cmd = sudo('ls -l /etc/tinc/%s' % tinc_network_name)
try:
assert 'tinc.conf' in cmd.stdout
except Exception as detail:
raise Exception("%s %s" % (cmd.stdout, detail))
def test_that_tinc_nets_boot_files_were_deployed_on(tinc_network):
for tinc_node in tinc_network.tinc_nodes:
with settings(
hide('stdout', 'running'),
host_string=tinc_node.host_string,
private_key_filename=tinc_node.private_key
):
print(" running on %s" % tinc_node.host_string)
cmd = sudo('ls -l /etc/tinc/')
try:
assert 'nets.boot' in cmd.stdout
except Exception as detail:
raise Exception("%s %s" % (cmd.stdout, detail))
def test_that_tinc_peers_host_files_were_deployed_on(tinc_network):
for tinc_node in tinc_network.tinc_nodes:
with settings(
hide('stdout', 'running'),
host_string=tinc_node.host_string,
private_key_filename=tinc_node.private_key
):
tinc_network_name = tinc_network.tinc_network_name
print(" running on %s" % tinc_node.host_string)
cmd = sudo('ls -l /etc/tinc/%s/hosts' % tinc_network_name)
for tinc_peer in tinc_node.tinc_peers:
try:
assert tinc_peer.tinc_name in cmd.stdout
except Exception as detail:
raise Exception("%s %s" % (cmd.stdout, detail))
def test_that_tinc_is_running_on(node):
with settings(
hide('stdout', 'running'),
host_string=node.host_string,
private_key_filename=node.private_key
):
print(" running on %s" % node.host_string)
cmd = sudo('COLUMNS=1000 ps -edalf | grep tincd | grep -v grep ')
try:
assert 'tincd' in cmd.stdout
except Exception as detail:
raise Exception("%s %s" % (cmd.stdout, detail))
cmd = sudo('systemctl is-active tinc')
try:
assert 'active' in cmd.stdout
except Exception as detail:
raise Exception("%s %s" % (cmd.stdout, detail))
def test_that_fail2ban_is_running_on(node):
with settings(
hide('stdout', 'running'),
host_string=node.host_string,
private_key_filename=node.private_key
):
print(" running on %s" % node.host_string)
cmd = sudo('COLUMNS=1000 ps -edalf | grep fail2ban | grep -v grep ')
try:
assert 'fail2ban' in cmd.stdout
except Exception as detail:
raise Exception("%s %s" % (cmd.stdout, detail))
cmd = sudo('systemctl is-active fail2ban')
try:
assert 'active' in cmd.stdout
except Exception as detail:
raise Exception("%s %s" % (cmd.stdout, detail))
def test_that_tinc_peers_are_pingable_on(tinc_network):
for tinc_node in tinc_network.tinc_nodes:
with settings(
hide('stdout', 'running'),
host_string=tinc_node.host_string,
private_key_filename=tinc_node.private_key
):
print(" running on %s" % tinc_node.host_string)
for tinc_peer in tinc_node.tinc_peers:
cmd = sudo('ping -c 1 %s' % tinc_peer.tinc_ip)
try:
assert cmd.return_code == 0
except Exception as detail:
raise Exception("%s %s" % (cmd.return_code, detail))
def test_that_consul_server_is_running_on(consul_node):
with settings(
hide('stdout', 'running'),
host_string=consul_node.host_string,
private_key_filename=consul_node.private_key
):
print(" running on %s" % consul_node.host_string)
cmd = sudo('systemctl is-active consul-server')
try:
assert 'active' in cmd.stdout
except Exception as detail:
raise Exception("%s %s" % (cmd.stdout, detail))
line = 'consul agent -config-dir /etc/consul.d/server'
cmd = sudo('COLUMNS=1000 ps -edalf')
try:
assert line in cmd.stdout
except Exception as detail:
raise Exception("%s %s" % (cmd.stdout, detail))
def test_that_consul_client_is_running_on(consul_node):
with settings(
hide('stdout', 'running'),
host_string=consul_node.host_string,
private_key_filename=consul_node.private_key
):
print(" running on %s" % consul_node.host_string)
line = 'consul agent -config-dir /etc/consul.d/client'
cmd = sudo('COLUMNS=1000 ps -edalf')
try:
assert line in cmd.stdout
except Exception as detail:
raise Exception("%s %s" % (cmd.stdout, detail))
cmd = sudo('systemctl is-active consul-client')
try:
assert 'active' in cmd.stdout
except Exception as detail:
raise Exception("%s %s" % (cmd.stdout, detail))
def test_that_fsconsul_service_is_running_on(consul_node):
with settings(
hide('stdout', 'running'),
host_string=consul_node.host_string,
private_key_filename=consul_node.private_key
):
print(" running on %s" % consul_node.host_string)
line = 'fsconsul -configFile=/etc/fsconsul.json'
cmd = sudo('COLUMNS=1000 ps -edalf')
try:
assert line in cmd.stdout
except Exception as detail:
raise Exception("%s %s" % (cmd.stdout, detail))
cmd = sudo('systemctl is-active fsconsul')
try:
assert 'active' in cmd.stdout
except Exception as detail:
raise Exception("%s %s" % (cmd.stdout, detail))
def deploy():
test()
with cd('/home/deploy/webapp'):
run("git pull")
run("pip install -r requirements.txt")
sudo("cp supervisord.conf /etc/supervisor/conf.d/webapp.conf")
sudo("cp nginx.conf /etc/nginx/sites-available/your_domain")
sudo("ln -sf /etc/nginx/sites-available/your_domain "
"/etc/nginx/sites-enabled/your_domain")
sudo("cp apache.conf /etc/apache2/sites-available/your_domain")
sudo("ln -sf /etc/apache2/sites-available/your_domain "
"/etc/apache2/sites-enabled/your_domain")
sudo("service nginx restart")
sudo("service apache2 restart")
def setup_supervisor():
# We use supervisord to keep Crestify running in the background
# Recover from crashes, and to start automatically on bootup
# Also, using more than 1 gunicorn worker resulted in socket not being released, so only 1 worker will be used
sudo('apt-get -y install supervisor')
sudo('mkdir /var/log/crestify/')
sudo(
'cd /home/crestify/crestify && ../crestifyenv/bin/honcho export -s /bin/sh -a crestify supervisord /etc/supervisor/conf.d')
fd = StringIO()
get('/etc/supervisor/conf.d/crestify.conf', fd)
content = fd.getvalue().splitlines()
for n, i in enumerate(content):
if i.startswith("environment="):
content[n] = i + ",PATH=/home/crestify/crestifyenv/bin:%(ENV_PATH)s"
if i.startswith("user="):
content[n] = "user=crestify"
if i.startswith("stopsignal="):
content[n] = "stopsignal=TERM" # Both Gunicorn and Celery use SIGTERM for graceful shutdown
content = StringIO("\n".join(content))
put(content, "/etc/supervisor/conf.d/crestify.conf", use_sudo=True)
sudo('supervisorctl reread')
sudo('supervisorctl update')
def setupServers():
sudo('yes '' | add-apt-repository ppa:fkrull/deadsnakes-python2.7 -y')
sudo('apt-get -y update')
sudo('apt-get -y install python2.7')
sudo('apt-get -y dist-upgrade')
sudo('apt-get -y install python-pip python-dev build-essential')
sudo('apt-get -y install libssl-dev libffi-dev git-all')
sudo('yes | pip install --upgrade pip')
sudo('yes | pip install --upgrade virtualenv')
sudo('yes | pip install --upgrade petlib')
sudo('yes | pip install twisted==16.6.0')
sudo('yes | pip install numpy')
sudo('yes | pip install service_identity')
sudo('yes | pip install sphinxmix')
sudo('apt-get -y install htop')
#sudo('apt-get -y install tshark')
if fabric.contrib.files.exists("loopix"):
with cd("loopix"):
run("git pull")
run("git checkout %s" % BRANCH)
else:
run("git clone https://github.com/UCL-InfoSec/loopix.git")
def mount_ebs_volumes(host_config):
env.host_string = helper.get_env_host_string(host_config)
env.user = helper.get_env_user(host_config)
env.key_filename = helper.get_env_key_filename(host_config)
sudo("apt-get -y install xfsprogs")
for ebs in host_config['ec2-mounts']:
device = ebs['device']
mount = ebs['mount']
sudo("mkdir -p {}".format(mount))
sudo("mv /etc/fstab /etc/fstab.old")
sudo("touch /etc/fstab")
if sudo('mkfs.xfs -f {0}'.format(device), warn_only=True):
run("echo '{0}\t{1}\txfs\tdefaults\t0\t0' | sudo tee -a /etc/fstab".format(device, mount))
sudo('sudo mount -a')
logger.info("EBS volume {} : {} mounted.".format(device, mount))
def sync_src():
get_vars()
with fab.lcd('..'):
destination = '/home/%s/senic-hub' % AV['build_user']
fab.sudo('mkdir -p %s' % destination, user=AV['build_user'])
rsync(
'-rlptvD',
'--exclude', '.*',
'--exclude', '*.egg-info',
'--exclude', '__pycache__',
'--exclude', 'node_modules',
'--exclude', '/build',
'--exclude', '/development',
'--exclude', '/dist',
'--exclude', '/docs',
'--exclude', '/venv',
'.',
'{host_string}:%s' % destination)
def init_localdb(flag='all'):
with settings(warn_only=True):
# clear rabbitmq
# if flag == 'all' or flag == 'rabbitmq':
# sudo(" echo 'clear the rabbitmq data' ")
# sudo("rabbitmqctl stop_app")
# sudo("rabbitmqctl reset")
# sudo("rabbitmqctl stop")
# sudo("rabbitmqctl start_app")
# clear leveldb
if flag == 'all' or flag == 'leveldb':
sudo(" echo 'clear the leveldb data only' ")
sudo("rm -rf /localdb/{bigchain,votes,header}/*")
# Install localdb
def install_localdb():
# leveldb & plyvel install
with settings(warn_only=True):
user_group = env.user
sudo(" echo 'leveldb & plyvel install' ")
sudo("mkdir -p /localdb/{bigchain,votes,header}")
sudo("chown -R " + user_group + ':' + user_group + ' /localdb')
sudo('pip3 install leveldb==0.194')
sudo('apt-get install libleveldb1 libleveldb-dev libsnappy1 libsnappy-dev')
sudo('apt-get -y -f install')
sudo('pip3 install plyvel==0.9')
# ramq & pika install
sudo(" echo 'ramq & pika install' ")
sudo('apt-get -y install rabbitmq-server')
sudo('pip3 install pika==0.10.0')
#sudo('rabbitmq-server restart')
# Install RethinkDB
def install_newrelic():
newrelic_license_key = environ.get('NEWRELIC_KEY')
if newrelic_license_key is None:
sys.exit('The NEWRELIC_KEY environment variable is not set')
else:
# Andreas had this "with settings(..." line, but I'm not sure why:
# with settings(warn_only=True):
# Use the installation instructions from NewRelic:
# http://tinyurl.com/q9kyrud
# ...with some modifications
sudo("echo 'deb http://apt.newrelic.com/debian/ newrelic non-free' >> "
"/etc/apt/sources.list.d/newrelic.list")
sudo('wget -O- https://download.newrelic.com/548C16BF.gpg | '
'apt-key add -')
sudo('apt-get update')
sudo('apt-get -y --force-yes install newrelic-sysmond')
sudo('nrsysmond-config --set license_key=' + newrelic_license_key)
sudo('/etc/init.d/newrelic-sysmond start')
###########################
# Security / Firewall Stuff
###########################
def set_fw():
# snmp
sudo('iptables -A INPUT -p tcp --dport 161 -j ACCEPT')
sudo('iptables -A INPUT -p udp --dport 161 -j ACCEPT')
# dns
sudo('iptables -A OUTPUT -p udp -o eth0 --dport 53 -j ACCEPT')
sudo('iptables -A INPUT -p udp -i eth0 --sport 53 -j ACCEPT')
# rethinkdb
sudo('iptables -A INPUT -p tcp --dport 28015 -j ACCEPT')
sudo('iptables -A INPUT -p udp --dport 28015 -j ACCEPT')
sudo('iptables -A INPUT -p tcp --dport 29015 -j ACCEPT')
sudo('iptables -A INPUT -p udp --dport 29015 -j ACCEPT')
sudo('iptables -A INPUT -p tcp --dport 8080 -j ACCEPT')
sudo('iptables -A INPUT -i eth0 -p tcp --dport 8080 -j DROP')
sudo('iptables -I INPUT -i eth0 -s 127.0.0.1 -p tcp --dport 8080 -j ACCEPT')
# save rules
sudo('iptables-save > /etc/sysconfig/iptables')
#########################################################
# Some helper-functions to handle bad behavior of cluster
#########################################################
#
#read blockchain-nodes and set all nodes
