def __init__(self, n_walkers, n_workers=None, gpu_indices=None):
if gpu_indices is not None:
self.gpu_indices = gpu_indices
self.n_workers = len(gpu_indices)
else:
assert n_workers, "If gpu_indices are not given the n_workers must be given"
self.n_workers = n_workers
self.gpu_indices = range(n_workers)
# make a Queue for free workers, when one is being used it is
# popped off and locked
self.free_workers = mulproc.Queue()
# the semaphore provides the locks on the workers
self.lock = mulproc.Semaphore(self.n_workers)
# initialize a list to put results in
self.results_list = mulproc.Manager().list()
for i in range(n_walkers):
self.results_list.append(None)
# add the free worker indices (not device/gpu indices) to the
# free workers queue
for i in range(self.n_workers):
self.free_workers.put(i)
python类Semaphore()的实例源码
def test_kill_process_tree(self):
"""Spin up a process that can't be killed by SIGTERM and make sure it gets killed anyway."""
child_process_killed = multiprocessing.Value('i', 0)
process_done = multiprocessing.Semaphore(0)
child_pid = multiprocessing.Value('i', 0)
setup_done = multiprocessing.Semaphore(0)
args = [child_process_killed, child_pid, process_done, setup_done]
child = multiprocessing.Process(target=TestHelpers._parent_of_ignores_sigterm, args=args)
try:
child.start()
self.assertTrue(process_done.acquire(timeout=5.0))
self.assertEqual(1, child_process_killed.value)
finally:
try:
os.kill(child_pid.value, signal.SIGKILL) # terminate doesnt work here
except OSError:
pass
def is_multiprocessing_available():
if (pycam.Utils.get_platform() == pycam.Utils.OSPlatform.WINDOWS) and \
hasattr(sys, "frozen") and sys.frozen:
return False
try:
import multiprocessing
# try to initialize a semaphore - this can trigger shm access failures
# (e.g. on Debian Lenny with Python 2.6.6)
multiprocessing.Semaphore()
return True
except ImportError:
if "missing_module" not in __issued_warnings:
log.info("Python's multiprocessing module is missing: disabling parallel processing")
__issued_warnings.append("missing_module")
except OSError:
if "shm_access_failed" not in __issued_warnings:
log.info("Python's multiprocessing module failed to acquire read/write access to "
"shared memory (shm) - disabling parallel processing")
__issued_warnings.append("shm_access_failed")
return False
def test_timeout(self):
if self.TYPE != 'processes':
self.skipTest('test not appropriate for {}'.format(self.TYPE))
sem = self.Semaphore(0)
acquire = TimingWrapper(sem.acquire)
self.assertEqual(acquire(False), False)
self.assertTimingAlmostEqual(acquire.elapsed, 0.0)
self.assertEqual(acquire(False, None), False)
self.assertTimingAlmostEqual(acquire.elapsed, 0.0)
self.assertEqual(acquire(False, TIMEOUT1), False)
self.assertTimingAlmostEqual(acquire.elapsed, 0)
self.assertEqual(acquire(True, TIMEOUT2), False)
self.assertTimingAlmostEqual(acquire.elapsed, TIMEOUT2)
self.assertEqual(acquire(timeout=TIMEOUT3), False)
self.assertTimingAlmostEqual(acquire.elapsed, TIMEOUT3)
def test_waitfor_timeout(self):
# based on test in test/lock_tests.py
cond = self.Condition()
state = self.Value('i', 0)
success = self.Value('i', False)
sem = self.Semaphore(0)
p = self.Process(target=self._test_waitfor_timeout_f,
args=(cond, state, success, sem))
p.daemon = True
p.start()
self.assertTrue(sem.acquire(timeout=10))
# Only increment 3 times, so state == 4 is never reached.
for i in range(3):
time.sleep(0.01)
with cond:
state.value += 1
cond.notify()
p.join(5)
self.assertTrue(success.value)
def __init__(self, num_processes=1, tasks_per_requests=1, bitmap_size=(64 << 10)):
self.to_update_queue = multiprocessing.Queue()
self.to_master_queue = multiprocessing.Queue()
self.to_master_from_mapserver_queue = multiprocessing.Queue()
self.to_master_from_slave_queue = multiprocessing.Queue()
self.to_mapserver_queue = multiprocessing.Queue()
self.to_slave_queues = []
for i in range(num_processes):
self.to_slave_queues.append(multiprocessing.Queue())
self.slave_locks_A = []
self.slave_locks_B = []
for i in range(num_processes):
self.slave_locks_A.append(multiprocessing.Lock())
self.slave_locks_B.append(multiprocessing.Lock())
self.slave_locks_B[i].acquire()
self.reload_semaphore = multiprocessing.Semaphore(multiprocessing.cpu_count()/2)
self.num_processes = num_processes
self.tasks_per_requests = tasks_per_requests
self.stage_abortion_notifier = multiprocessing.Value('b', False)
self.slave_termination = multiprocessing.Value('b', False, lock=False)
self.sampling_failed_notifier = multiprocessing.Value('b', False)
self.effector_mode = multiprocessing.Value('b', False)
self.files = ["/dev/shm/kafl_fuzzer_master_", "/dev/shm/kafl_fuzzer_mapserver_", "/dev/shm/kafl_fuzzer_bitmap_"]
self.sizes = [(65 << 10), (65 << 10), bitmap_size]
self.tmp_shm = [{}, {}, {}]
def __init__(self, opt, world):
super().__init__(opt)
self.inner_world = world
self.numthreads = opt['numthreads']
self.sync = { # syncronization primitives
# semaphores for counting queued examples
'queued_sem': Semaphore(0), # counts num exs to be processed
'threads_sem': Semaphore(0), # counts threads
'reset_sem': Semaphore(0), # allows threads to reset
# flags for communicating with threads
'reset_flag': Value('b', False), # threads should reset
'term_flag': Value('b', False), # threads should terminate
# counters
'epoch_done_ctr': Value('i', 0), # number of done threads
'total_parleys': Value('l', 0), # number of parleys in threads
}
# don't let threads create more threads!
self.threads = []
for i in range(self.numthreads):
self.threads.append(HogwildProcess(i, opt, world, self.sync))
for t in self.threads:
t.start()
for _ in self.threads:
self.sync['threads_sem'].acquire()
def clopure_pmap(self, *args, local_vars):
if len(args) <= 1:
raise ClopureRuntimeError("pmap takes at least 2 arguments")
seqs = [self.evaluate(arg, local_vars=local_vars) for arg in args[1:]]
p = Pool(self.procs)
s = Semaphore(self.queue_size)
input_iter = (((args[0],) + x, local_vars) for x in input_semaphore_hook(zip(*seqs), s))
return output_semaphore_hook(p.imap(self.mp_evaluate_wrapper, input_iter), s)
def clopure_pmap_unord(self, *args, local_vars):
if len(args) <= 1:
raise ClopureRuntimeError("pmap-unord takes at least 2 arguments")
seqs = [self.evaluate(arg, local_vars=local_vars) for arg in args[1:]]
p = Pool(self.procs)
s = Semaphore(self.queue_size)
input_iter = (((args[0],) + x, local_vars) for x in input_semaphore_hook(zip(*seqs), s))
return output_semaphore_hook(p.imap_unordered(self.mp_evaluate_wrapper, input_iter), s)
def clopure_iter_mp_split_unord(self, fn, local_vars):
def iter_split_generator(*g):
q_in = Queue()
q_out = Queue()
exit_input_thread = False
semaphore = Semaphore(self.queue_size)
ps = [Process(target=self.iter_split_evaluate_wrapper, args=(fn, local_vars, len(g), q_in, q_out)) for i in range(self.procs)]
for p in ps:
p.start()
def input_thread():
try:
for i, item in enumerate(zip(*g)):
semaphore.acquire()
if exit_input_thread:
return
q_in.put((i, item))
except BaseException:
traceback.print_exc(file=sys.stdout)
for i in range(self.procs):
q_in.put((0, EOFMessage))
t = Thread(target=input_thread)
t.start()
n_working_procs = self.procs
while True:
k, data = q_out.get()
if data is EOFMessage:
n_working_procs -= 1
if n_working_procs == 0:
break
continue
yield data
semaphore.release()
for p in ps:
p.join()
exit_input_thread = True
semaphore.release()
return iter_split_generator
def test_kill_using_shell(self):
"""Test when no process exists."""
child_pid = multiprocessing.Value('i', 0)
setup_done = multiprocessing.Semaphore(0)
args = [child_pid, setup_done]
child = multiprocessing.Process(target=TestHelpers._ignores_sigterm, args=args)
child.start()
self.assertTrue(setup_done.acquire(timeout=1.0))
pid_to_kill = child_pid.value
self.assertTrue(helpers.kill_using_shell(logging.getLogger(), pid_to_kill,
signal=signal.SIGKILL))
child.join() # remove orphan process
self.assertFalse(helpers.kill_using_shell(logging.getLogger(), pid_to_kill,
signal=signal.SIGKILL))
def __init__(self, n):
self.n = n
self.counter = SharedCounter(0)
self.barrier = Semaphore(0)
def __init__(self, forks_number):
self.forks_number = forks_number
self.semaphore = Semaphore(self.forks_number)
def test_semaphore(self):
sem = self.Semaphore(2)
self._test_semaphore(sem)
self.assertEqual(sem.release(), None)
self.assertReturnsIfImplemented(3, get_value, sem)
self.assertEqual(sem.release(), None)
self.assertReturnsIfImplemented(4, get_value, sem)
def __init__(self, n):
self.n = n
self.counter = SharedCounter(0)
self.barrier = Semaphore(0)
def clopure_iter_mp_split(self, fn, local_vars):
def iter_split_generator(*g):
q_in = Queue()
q_out = Queue()
exit_input_thread = False
semaphore = Semaphore(self.queue_size)
ps = [Process(target=self.iter_split_evaluate_wrapper, args=(fn, local_vars, len(g), q_in, q_out)) for i in range(self.procs)]
for p in ps:
p.start()
def input_thread():
try:
for i, item in enumerate(zip(*g)):
semaphore.acquire()
if exit_input_thread:
return
q_in.put((i, item))
except BaseException:
traceback.print_exc(file=sys.stdout)
for i in range(self.procs):
q_in.put((0, EOFMessage))
t = Thread(target=input_thread)
t.start()
cur = 0
n_working_procs = self.procs
l = [None] * self.queue_size
while True:
k, data = q_out.get()
if data is EOFMessage:
n_working_procs -= 1
if n_working_procs == 0:
break
continue
l[k - cur] = (k, data)
while l[0]:
yield l.pop(0)[1]
l.append(None)
cur += 1
semaphore.release()
exit_input_thread = True
semaphore.release()
return iter_split_generator
def test_notify(self):
cond = self.Condition()
sleeping = self.Semaphore(0)
woken = self.Semaphore(0)
p = self.Process(target=self.f, args=(cond, sleeping, woken))
p.daemon = True
p.start()
p = threading.Thread(target=self.f, args=(cond, sleeping, woken))
p.daemon = True
p.start()
# wait for both children to start sleeping
sleeping.acquire()
sleeping.acquire()
# check no process/thread has woken up
time.sleep(DELTA)
self.assertReturnsIfImplemented(0, get_value, woken)
# wake up one process/thread
cond.acquire()
cond.notify()
cond.release()
# check one process/thread has woken up
time.sleep(DELTA)
self.assertReturnsIfImplemented(1, get_value, woken)
# wake up another
cond.acquire()
cond.notify()
cond.release()
# check other has woken up
time.sleep(DELTA)
self.assertReturnsIfImplemented(2, get_value, woken)
# check state is not mucked up
self.check_invariant(cond)
p.join()
def test_wait_integer(self):
from multiprocessing.connection import wait
expected = 3
sorted_ = lambda l: sorted(l, key=lambda x: id(x))
sem = multiprocessing.Semaphore(0)
a, b = multiprocessing.Pipe()
p = multiprocessing.Process(target=self.signal_and_sleep,
args=(sem, expected))
p.start()
self.assertIsInstance(p.sentinel, int)
self.assertTrue(sem.acquire(timeout=20))
start = time.time()
res = wait([a, p.sentinel, b], expected + 20)
delta = time.time() - start
self.assertEqual(res, [p.sentinel])
self.assertLess(delta, expected + 2)
self.assertGreater(delta, expected - 2)
a.send(None)
start = time.time()
res = wait([a, p.sentinel, b], 20)
delta = time.time() - start
self.assertEqual(sorted_(res), sorted_([p.sentinel, b]))
self.assertLess(delta, 0.4)
b.send(None)
start = time.time()
res = wait([a, p.sentinel, b], 20)
delta = time.time() - start
self.assertEqual(sorted_(res), sorted_([a, p.sentinel, b]))
self.assertLess(delta, 0.4)
p.terminate()
p.join()
