def start(self, verbose=False):
if verbose:
self.process = subprocess.Popen(filter(None, self.cmd.split(" ")),
stdin=None,
stdout=None,
stderr=None)
else:
self.process = subprocess.Popen(filter(None, self.cmd.split(" ")),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
self.stat_fd = open("/proc/" + str(self.process.pid) + "/stat")
self.init()
try:
self.set_init_state()
except:
return False
self.initial_mem_usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
#time.sleep(1)
self.kafl_shm.seek(0x0)
self.kafl_shm.write(self.virgin_bitmap)
self.kafl_shm.flush()
return True
python类getrusage()的实例源码
def get_used_memory():
""" Return the used memory in MB """
if platform.system() == 'Linux':
for line in open('/proc/self/status'):
if line.startswith('VmRSS:'):
return int(line.split()[1]) >> 10
else:
warnings.warn("Please install psutil to have better "
"support with spilling")
if platform.system() == "Darwin":
import resource
rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
return rss >> 20
# TODO: support windows
return 0
def memory_usage():
# If we are on linux
if platform == "linux" or platform == "linux2":
kilobytes = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss # peak memory usage (bytes on OS X, kilobytes on Linux)
gigabytes = kilobytes * 1e-6
return gigabytes
# If we are on Mac OS X
elif platform == "darwin":
kilobytes = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss # peak memory usage (bytes on OS X, kilobytes on Linux)
gigabytes = kilobytes * 1e-9
return gigabytes
# We don't support Windows
elif platform == "win32": raise EnvironmentError("The Windows operating system is not supported")
# Unrecognized platform
else: raise EnvironmentError("Unrecognized platform")
# -----------------------------------------------------------------
def report_resources(agent):
ru = resource.getrusage(resource.RUSAGE_SELF)
out = {"utime": ru.ru_utime,
"stime": ru.ru_stime,
"maxrss": ru.ru_maxrss,
"ixrss": ru.ru_ixrss,
"idrss": ru.ru_idrss,
"isrss": ru.ru_isrss,
"minflt": ru.ru_minflt,
"majflt": ru.ru_majflt,
"nswap": ru.ru_nswap,
"inblock": ru.ru_inblock,
"oublock": ru.ru_oublock,
"msgsnd": ru.ru_msgsnd,
"msgrcv": ru.ru_msgrcv,
"nsignals": ru.ru_nsignals,
"nvcsw": ru.ru_nvcsw,
"nivcsw": ru.ru_nivcsw
}
return out
def test_lots_of_queries(self):
import resource
import objgraph
class LoadTest(Model):
k = columns.Integer(primary_key=True)
v = columns.Integer()
sync_table(LoadTest)
gc.collect()
objgraph.show_most_common_types()
print("Starting...")
for i in range(1000000):
if i % 25000 == 0:
# print memory statistic
print("Memory usage: %s" % (resource.getrusage(resource.RUSAGE_SELF).ru_maxrss))
LoadTest.create(k=i, v=i)
objgraph.show_most_common_types()
raise Exception("you shouldn't be here")
def get_resource_usage_report():
r = resource.getrusage(0)
key_to_description = {
"ru_utime": "time in user mode (float)",
"ru_stime": "time in system mode (float)",
"ru_maxrss": "maximum resident set size",
"ru_ixrss": "shared memory size",
"ru_idrss": "unshared memory size",
"ru_isrss": "unshared stack size",
"ru_minflt": "page faults not requiring I/O",
"ru_majflt": "page faults requiring I/O",
"ru_nswap": "number of swap outs",
"ru_inblock": "block input operations",
"ru_oublock": "block output operations",
"ru_msgsnd": "messages sent",
"ru_msgrcv": "messages received",
"ru_nsignals": "signals received",
"ru_nvcsw": "voluntary context switches",
"ru_nivcsw": "involuntary context switches",
}
return dict([(v, getattr(r, k)) for k, v in key_to_description.items()])
def memory_fitness(threshold=2e9, maximum=3e9):
"""
Returns a penalty for using too much memory. Add this to your fitness
function. This measures the current processes maximum resident set (maxrss)
which is the all time peak memory usage for the calling process.
Argument threshold is where the this penalty begins.
Argument maximum is where this penalty becomes an error (ValueError).
Returns in the range [0, 1] where 0 is no penalty and 1 is the maximum
memory usage. Linear ramp from threshold to maximum.
"""
rsc = resource.getrusage(resource.RUSAGE_SELF)
size = rsc.ru_maxrss * 1024
fit = (size - threshold) / (maximum - threshold)
if fit > 1:
raise ValueError("Individual exceded memory limit (size %d bytes, maximum %d)."%(size, maximum))
return max(0, fit)
def test_gcr_memory(self):
self.request = FakeRequest()
count = 0
current = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024.0 / 1024.0
while True:
count += 1
get_current_request()
if count % 1000000 == 0:
break
if count % 100000 == 0:
gc.collect()
new = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024.0 / 1024.0
if new - current > 10: # memory leak, this shouldn't happen
assert new == current
def ingest_daemon(self):
x = 0
while True:
node_work_directory = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(
os.path.abspath(__file__)
))),
'VEDA_WORKING'
)
FD = FileDiscovery(
node_work_directory=node_work_directory
)
FD.discover_studio_ingested_videos()
FD.about_video_ingest()
reset_queries()
x += 1
if x >= 100:
print 'Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
x = 0
def youtube_daemon(self):
x = 0
while True:
self.course_list = generate_course_list()
for course in self.course_list:
print "%s%s: Callback" % (course.institution, course.edx_classid)
callfunction(course)
x += 1
if x >= 100:
print 'Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
x = 0
reset_queries()
self.course_list = []
time.sleep(10)
def test_gcr_memory(self):
self.request = get_mocked_request()
count = 0
current = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024.0 / 1024.0
while True:
count += 1
utils.get_current_request()
if count % 1000000 == 0:
break
if count % 100000 == 0:
gc.collect()
new = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024.0 / 1024.0
if new - current > 10: # memory leak, this shouldn't happen
assert new == current
def clock():
"""clock() -> floating point number
Return the *TOTAL USER+SYSTEM* CPU time in seconds since the start of
the process. This is done via a call to resource.getrusage, so it
avoids the wraparound problems in time.clock()."""
u,s = resource.getrusage(resource.RUSAGE_SELF)[:2]
return u+s
def memory_usage():
# If we are on linux
if platform == "linux" or platform == "linux2":
kilobytes = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss # peak memory usage (bytes on OS X, kilobytes on Linux)
gigabytes = kilobytes * 1e-6
return gigabytes
# If we are on Mac OS X
elif platform == "darwin":
kilobytes = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss # peak memory usage (bytes on OS X, kilobytes on Linux)
gigabytes = kilobytes * 1e-9
return gigabytes
# We don't support Windows
elif platform == "win32": raise EnvironmentError("The Windows operating system is not supported")
# Unrecognized platform
else: raise EnvironmentError("Unrecognized platform")
# -----------------------------------------------------------------
def get_ram(self):
"""Get the bot's RAM usage info."""
if have_psutil: # yay!
mu = psutil.Process(os.getpid()).memory_info().rss
return (True, mu / 1000000, mu / 1048576)
else: # aww
raw_musage = 0
got_conversion = False
musage_dec = 0
musage_hex = 0
if sys.platform.startswith('linux'): # Linux & Windows report in kilobytes
raw_musage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
got_conversion = True
musage_dec = raw_musage / 1000
musage_hex = raw_musage / 1024
elif sys.platform == 'darwin': # Mac reports in bytes
raw_musage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
got_conversion = True
musage_dec = raw_musage / 1000000 # 1 million. 1000 * 1000
musage_hex = raw_musage / 1048576 # 1024 * 1024
if got_conversion:
return (got_conversion, musage_dec, musage_hex)
else:
return (got_conversion,)
def main():
parse_command_line()
print("Loading markov chains...")
markov_chain.load_chains()
# Print memory usage for the server when all chains are loaded
memory_usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1000.0
memory_usage = humanfriendly.format_size(memory_usage)
print("Markov chain server loaded, memory usage %s" % memory_usage)
application = tornado.web.Application([
(r"/", MainHandler),
], debug=False)
application.listen(5666, address='127.0.0.1')
tornado.ioloop.IOLoop.instance().start()
def Debug():
try:
global debugCount
debugCount = debugCount + 1
resUsage=getrusage(RUSAGE_SELF)
size=resUsage.ru_maxrss
info("Memory usage : " + str(debugCount) + " size: " + str(size))
info("Resouce usage info: " + str(resUsage))
# Memory leaks display currently commented out
# show_growth()
# obj=get_leaking_objects()
# warn('Leaking objects size='+str(len(obj)))
# filepathdebug='/var/log/myDebug'+str(debugCount)
# with open(filepathdebug, "w+") as f: #replace filepath & filename
# f.write('Debug resouce iteration: ' + str(debugCount) + " size: " + str(size))
# f.write('Leaking objects size='+str(len(obj)) + '\n')
# f.write('Leaking objects size='+str(typestats()) + '\n')
# f.write('Leaking objects'+str(obj) + '\n')
except Exception as e:
error('failed to track memory: ' + str(e))
def get_avg_cpu_load(self):
"""Returns the average cpu load since the last call
Returns the user and system time fraction per second as tuple or None
"""
if not self.start_wall_time:
rusage = resource.getrusage(resource.RUSAGE_SELF)
self.start_wall_time = time.time()
self.start_cpu_user_time = rusage.ru_utime
self.start_cpu_sys_time = rusage.ru_stime
return None
else:
now = time.time()
rusage = resource.getrusage(resource.RUSAGE_SELF)
time_delta = now-self.start_wall_time
avg_user_time = (rusage.ru_utime-self.start_cpu_user_time)/time_delta
avg_sys_time = (rusage.ru_stime-self.start_cpu_sys_time)/time_delta
self.start_wall_time = now
self.start_cpu_user_time = rusage.ru_utime
self.start_cpu_sys_time = rusage.ru_stime
return avg_user_time, avg_sys_time
def _runtest (infile):
global g_tmpfile
assert (g_tmpfile)
parser = DDSMTParser()
smtformula = parser.parse(infile)
_log (1, "parser: done")
_log (2, "parser: maxrss: {} MiB".format(
resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1000))
_dump (smtformula, g_tmpfile)
_log (1, "dumper: done")
_log (1)
if not _cmp (infile, g_tmpfile):
bugfile = "bug-ddsmtparsertest-" + os.path.basename(infile)
shutil.copyfile(g_tmpfile, bugfile)
nbugs += 1
_log (1, "bug: " + bugfile)
def get_used_memory():
""" Return the used memory in MB """
if platform.system() == 'Linux':
for line in open('/proc/self/status'):
if line.startswith('VmRSS:'):
return int(line.split()[1]) >> 10
else:
warnings.warn("Please install psutil to have better "
"support with spilling")
if platform.system() == "Darwin":
import resource
rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
return rss >> 20
# TODO: support windows
return 0
def runCommand(cmd, rUsage=False):
"""
Run system command and get output, error, process handle and resource usage handle
:param cmd:
:param rUsage:
:return:
"""
import sys
from subprocess import Popen, PIPE, STDOUT
from resource import getrusage,RUSAGE_SELF,RUSAGE_CHILDREN
res = None
out = None
err = None
try:
print "Start to run command: %s" % cmd
p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
# p.wait()
except OSError:
print (err)
sys.exit(p.returncode)
(out, err) = p.communicate()
if rUsage:
res = getrusage(RUSAGE_CHILDREN)
return (out, err, p, res)
def runCommand(cmd, rUsage=False):
"""
Run system command and get output, error, process handle and resource usage handle
:param cmd:
:param rUsage:
:return:
"""
import sys
from subprocess import Popen, PIPE, STDOUT
from resource import getrusage,RUSAGE_SELF,RUSAGE_CHILDREN
res = None
out = None
err = None
try:
logger.info("Start to run command: %s" % cmd)
p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
# p.wait()
except OSError:
print (err)
sys.exit(p.returncode)
(out, err) = p.communicate()
if rUsage:
res = getrusage(RUSAGE_CHILDREN)
return (out, err, p, res)
def check_idle_cpu_usage(duration, allowed_part):
if resource is None:
# TODO: use https://code.google.com/p/psutil/
from nose.plugins.skip import SkipTest
raise SkipTest('CPU usage testing not supported (`import resource` failed)')
r1 = resource.getrusage(resource.RUSAGE_SELF)
eventlet.sleep(duration)
r2 = resource.getrusage(resource.RUSAGE_SELF)
utime = r2.ru_utime - r1.ru_utime
stime = r2.ru_stime - r1.ru_stime
# This check is reliably unreliable on Travis, presumably because of CPU
# resources being quite restricted by the build environment. The workaround
# is to apply an arbitrary factor that should be enough to make it work nicely.
if os.environ.get('TRAVIS') == 'true':
allowed_part *= 1.2
assert utime + stime < duration * allowed_part, \
"CPU usage over limit: user %.0f%% sys %.0f%% allowed %.0f%%" % (
utime / duration * 100, stime / duration * 100,
allowed_part * 100)
def make_post_exec_msg(start_time: datetime=None, comment: str=None) -> str:
"""Build Post-Execution Message with information about RAM and Time."""
use, al, msg = 0, 0, ""
if sys.platform.startswith(("win", "darwin")):
msg = "No information about RAM usage available on non-Linux systems."
elif sys.platform.startswith("linux"):
use = int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss *
resource.getpagesize() if resource else 0)
al = int(os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
if hasattr(os, "sysconf") else 0)
msg += f"""Total Max Memory Used: ~{use / al:.2%} Percent.
{ bytes2human(use) } ({ use } bytes) of
{ bytes2human(al) } ({ al } bytes) of total RAM Memory.\n"""
if start_time:
_t = datetime.now() - start_time
msg += f"Total Working Time: ~{ timedelta2human(_t) } ({ _t }).\n"
if comment:
msg += str(comment).strip()
log.debug("Preparing Simple Post-Execution Messages.")
atexit.register(log.info, msg)
return msg
def test_lots_of_queries(self):
import resource
import objgraph
class LoadTest(Model):
k = columns.Integer(primary_key=True)
v = columns.Integer()
sync_table(LoadTest)
gc.collect()
objgraph.show_most_common_types()
print("Starting...")
for i in range(1000000):
if i % 25000 == 0:
# print memory statistic
print("Memory usage: %s" % (resource.getrusage(resource.RUSAGE_SELF).ru_maxrss))
LoadTest.create(k=i, v=i)
objgraph.show_most_common_types()
raise Exception("you shouldn't be here")
def _performance_log(func):
""" Logs information for performance measurement """
def wrapper(*arg):
""" wrapper """
start = datetime.datetime.now()
# Code execution
res = func(*arg)
if _log_performance:
usage = resource.getrusage(resource.RUSAGE_SELF)
memory_process = (usage[2])/1000
delta = datetime.datetime.now() - start
delta_milliseconds = int(delta.total_seconds() * 1000)
_logger.info("PERFORMANCE - {0} - milliseconds |{1:>8,}| - memory MB |{2:>8,}|"
.format(func.__name__,
delta_milliseconds,
memory_process))
return res
return wrapper
def _performance_log(func):
""" Logs information for performance measurement """
def wrapper(*arg):
""" wrapper """
start = datetime.datetime.now()
# Code execution
res = func(*arg)
if _log_performance:
usage = resource.getrusage(resource.RUSAGE_SELF)
process_memory = usage.ru_maxrss / 1000
delta = datetime.datetime.now() - start
delta_milliseconds = int(delta.total_seconds() * 1000)
_LOGGER.info("PERFORMANCE - {0} - milliseconds |{1:>8,}| - memory MB |{2:>8,}|"
.format(func.__name__,
delta_milliseconds,
process_memory))
return res
return wrapper
def main(arguments=None):
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
parser = HelpfulArgumentParser(description=__doc__, prog='igdiscover')
parser.add_argument('--profile', default=False, action='store_true',
help='Save profiling information to igdiscover.prof')
parser.add_argument('--version', action='version', version='%(prog)s ' + __version__)
subparsers = parser.add_subparsers()
for command_name in COMMANDS:
module = importlib.import_module('.' + command_name, 'igdiscover')
subparser = subparsers.add_parser(command_name,
help=module.__doc__.split('\n')[1], description=module.__doc__)
subparser.set_defaults(func=module.main)
module.add_arguments(subparser)
args = parser.parse_args(arguments)
if not hasattr(args, 'func'):
parser.error('Please provide the name of a subcommand to run')
elif args.profile:
import cProfile as profile
profile.runctx('args.func(args)', globals(), locals(), filename='igdiscover.prof')
logger.info('Wrote profiling data to igdiscover.prof')
else:
args.func(args)
if sys.platform == 'linux':
rself = resource.getrusage(resource.RUSAGE_SELF)
rchildren = resource.getrusage(resource.RUSAGE_CHILDREN)
memory_kb = rself.ru_maxrss + rchildren.ru_maxrss
cpu_time = rself.ru_utime + rself.ru_stime + rchildren.ru_utime + rchildren.ru_stime
cpu_time_s = format_duration(cpu_time)
logger.info('CPU time {}. Maximum memory usage {:.3f} GB'.format(
cpu_time_s, memory_kb / 1E6))
def main(args):
try:
config = Config.from_default_path()
except FileNotFoundError as e:
sys.exit("Pipeline configuration file {!r} not found. Please create it!".format(e.filename))
print('IgDiscover version {} with Python {}. Configuration:'.format(__version__,
platform.python_version()))
for k, v in sorted(vars(config).items()):
# TODO the following line is only necessary for non-YAML configurations
if k.startswith('_'):
continue
print(' ', k, ': ', repr(v), sep='')
# snakemake sets up its own logging and this cannot be easily changed
# (setting keep_logger=True crashes), so remove our own log handler
# for now
logger.root.handlers = []
snakefile_path = pkg_resources.resource_filename('igdiscover', 'Snakefile')
success = snakemake(snakefile_path,
snakemakepath='snakemake', # Needed in snakemake 3.9.0
dryrun=args.dryrun,
cores=args.cores,
keepgoing=args.keepgoing,
printshellcmds=args.print_commands,
targets=args.targets if args.targets else None,
)
if sys.platform == 'linux' and not args.dryrun:
cputime = resource.getrusage(resource.RUSAGE_SELF).ru_utime
cputime += resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime
h = int(cputime // 3600)
m = (cputime - h * 3600) / 60
print('Total CPU time: {}h {:.2f}m'.format(h, m))
sys.exit(0 if success else 1)
def get_cpu_time():
"""Return CPU time used by process and children"""
if sys.platform != 'linux':
return None
rs = resource.getrusage(resource.RUSAGE_SELF)
rc = resource.getrusage(resource.RUSAGE_CHILDREN)
return rs.ru_utime + rs.ru_stime + rc.ru_utime + rc.ru_stime
def __enter__(self):
self.start = time.time()
print '%s\tself_premaxrss_mb\t%0.1f' % (self.note, resource.getrusage(resource.RUSAGE_SELF)[2]/1e3)
print '%s\tchildren_premaxrss_mb\t%0.1f' % (self.note, resource.getrusage(resource.RUSAGE_SELF)[2]/1e3)
sys.stdout.flush()
