def test_various_ops(self):
# This takes about n/3 seconds to run (about n/3 clumps of tasks,
# times about 1 second per clump).
NUMTASKS = 10
# no more than 3 of the 10 can run at once
sema = threading.BoundedSemaphore(value=3)
mutex = threading.RLock()
numrunning = Counter()
threads = []
for i in range(NUMTASKS):
t = TestThread("<thread %d>"%i, self, sema, mutex, numrunning)
threads.append(t)
self.assertEqual(t.ident, None)
self.assertTrue(re.match('<TestThread\(.*, initial\)>', repr(t)))
t.start()
if verbose:
print('waiting for all tasks to complete')
for t in threads:
t.join(NUMTASKS)
self.assertTrue(not t.is_alive())
self.assertNotEqual(t.ident, 0)
self.assertFalse(t.ident is None)
self.assertTrue(re.match('<TestThread\(.*, stopped -?\d+\)>',
repr(t)))
if verbose:
print('all tasks done')
self.assertEqual(numrunning.get(), 0)
python类BoundedSemaphore()的实例源码
def testWithBoundedSemaphore(self):
lock = threading.BoundedSemaphore()
def locked():
if lock.acquire(False):
lock.release()
return False
else:
return True
self.boilerPlate(lock, locked)
def __init__(self, domain, subdomains=None, q=None, lock=threading.Lock()):
subdomains = subdomains or []
self.base_url = 'https://dnsdumpster.com/'
self.domain = urlparse.urlparse(domain).netloc
self.subdomains = []
self.live_subdomains = []
self.session = requests.Session()
self.engine_name = "DNSdumpster"
multiprocessing.Process.__init__(self)
self.threads = 70
self.lock = threading.BoundedSemaphore(value=self.threads)
self.q = q
self.timeout = 25
self.print_banner()
return
def __init__(self,subdomains,ports):
self.subdomains = subdomains
self.ports = ports
self.threads = 20
self.lock = threading.BoundedSemaphore(value=self.threads)
def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
subdomains = subdomains or []
base_url = 'https://dnsdumpster.com/'
self.live_subdomains = []
self.engine_name = "DNSdumpster"
self.threads = 70
self.lock = threading.BoundedSemaphore(value=self.threads)
self.q = q
super(DNSdumpster, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
return
def __init__(self, subdomains, ports):
self.subdomains = subdomains
self.ports = ports
self.threads = 20
self.lock = threading.BoundedSemaphore(value=self.threads)
def __init__(self, initial=1, period=1, amount=1):
super(RateLimitBucket, self).__init__()
self.semaphore = threading.BoundedSemaphore(initial)
self.amount = amount
self.period = period
self._stop_event = threading.Event()
def test_various_ops(self):
# This takes about n/3 seconds to run (about n/3 clumps of tasks,
# times about 1 second per clump).
NUMTASKS = 10
# no more than 3 of the 10 can run at once
sema = threading.BoundedSemaphore(value=3)
mutex = threading.RLock()
numrunning = Counter()
threads = []
for i in range(NUMTASKS):
t = TestThread("<thread %d>"%i, self, sema, mutex, numrunning)
threads.append(t)
self.assertEqual(t.ident, None)
self.assertTrue(re.match('<TestThread\(.*, initial\)>', repr(t)))
t.start()
if verbose:
print 'waiting for all tasks to complete'
for t in threads:
t.join(NUMTASKS)
self.assertTrue(not t.is_alive())
self.assertNotEqual(t.ident, 0)
self.assertFalse(t.ident is None)
self.assertTrue(re.match('<TestThread\(.*, \w+ -?\d+\)>', repr(t)))
if verbose:
print 'all tasks done'
self.assertEqual(numrunning.get(), 0)
def testWithBoundedSemaphore(self):
lock = threading.BoundedSemaphore()
def locked():
if lock.acquire(False):
lock.release()
return False
else:
return True
self.boilerPlate(lock, locked)
# This is needed to make the test actually run under regrtest.py!
gce_firewall_enforcer_test.py 文件源码
项目：forseti-security
作者: GoogleCloudPlatform
项目源码
文件源码
阅读 21
收藏 0
点赞 0
评论 0
def test_apply_change_lots_of_rules(self):
"""Changing more rules than permitted by the operation semaphore works.
Setup:
* Create a new bounded semaphore with a limit of 2 operations.
* Create a list of 10 rules to insert.
* Run _apply_change.
Expected Results:
* All rules end up in the successes list.
"""
insert_function = self.firewall_api.insert_firewall_rule
self.enforcer.operation_sema = threading.BoundedSemaphore(value=2)
test_rule_name = 'test-network-allow-internal-0'
test_rule = constants.EXPECTED_FIREWALL_RULES[test_rule_name]
test_rules = []
for i in xrange(10):
rule = copy.deepcopy(test_rule)
rule['name'] = '%s-%i' % (test_rule_name, i)
test_rules.append(rule)
(successes, failures, change_errors) = self.enforcer._apply_change(
insert_function, test_rules)
self.assertSameStructure(test_rules, successes)
self.assertListEqual([], failures)
self.assertListEqual([], change_errors)
def __init__(self,
global_configs=None,
dry_run=False,
concurrent_workers=1,
project_sema=None,
max_running_operations=0):
"""Initialize.
Args:
global_configs (dict): Global configurations.
dry_run (bool): If True, will simply log what action would have been
taken without actually applying any modifications.
concurrent_workers (int): The number of parallel enforcement threads
to execute.
project_sema (threading.BoundedSemaphore): An optional semaphore
object, used to limit the number of concurrent projects getting
written to.
max_running_operations (int): Used to limit the number of concurrent
write operations on a single project's firewall rules. Set to 0 to
allow unlimited in flight asynchronous operations.
"""
self.global_configs = global_configs
self.enforcement_log = enforcer_log_pb2.EnforcerLog()
self._dry_run = dry_run
self._concurrent_workers = concurrent_workers
self._project_sema = project_sema
self._max_running_operations = max_running_operations
self._local = LOCAL_THREAD
def initialize_batch_enforcer(global_configs, concurrent_threads,
max_write_threads, max_running_operations,
dry_run):
"""Initialize and return a BatchFirewallEnforcer object.
Args:
global_configs (dict): Global configurations.
concurrent_threads: The number of parallel enforcement threads to execute.
max_write_threads: The maximum number of enforcement threads that can be
actively updating project firewalls.
max_running_operations: The maximum number of write operations per
enforcement thread.
dry_run: If True, will simply log what action would have been taken
without actually applying any modifications.
Returns:
A BatchFirewallEnforcer instance.
"""
if max_write_threads:
project_sema = threading.BoundedSemaphore(value=max_write_threads)
else:
project_sema = None
enforcer = batch_enforcer.BatchFirewallEnforcer(
global_configs=global_configs,
dry_run=dry_run,
concurrent_workers=concurrent_threads,
project_sema=project_sema,
max_running_operations=max_running_operations)
return enforcer
def test_various_ops(self):
# This takes about n/3 seconds to run (about n/3 clumps of tasks,
# times about 1 second per clump).
NUMTASKS = 10
# no more than 3 of the 10 can run at once
sema = threading.BoundedSemaphore(value=3)
mutex = threading.RLock()
numrunning = Counter()
threads = []
for i in range(NUMTASKS):
t = TestThread("<thread %d>"%i, self, sema, mutex, numrunning)
threads.append(t)
self.assertEqual(t.ident, None)
self.assertTrue(re.match('<TestThread\(.*, initial\)>', repr(t)))
t.start()
if verbose:
print('waiting for all tasks to complete')
for t in threads:
t.join()
self.assertTrue(not t.is_alive())
self.assertNotEqual(t.ident, 0)
self.assertFalse(t.ident is None)
self.assertTrue(re.match('<TestThread\(.*, stopped -?\d+\)>',
repr(t)))
if verbose:
print('all tasks done')
self.assertEqual(numrunning.get(), 0)
def testWithBoundedSemaphore(self):
lock = threading.BoundedSemaphore()
def locked():
if lock.acquire(False):
lock.release()
return False
else:
return True
self.boilerPlate(lock, locked)
def test_various_ops(self):
# This takes about n/3 seconds to run (about n/3 clumps of tasks,
# times about 1 second per clump).
NUMTASKS = 10
# no more than 3 of the 10 can run at once
sema = threading.BoundedSemaphore(value=3)
mutex = threading.RLock()
numrunning = Counter()
threads = []
for i in range(NUMTASKS):
t = TestThread("<thread %d>"%i, self, sema, mutex, numrunning)
threads.append(t)
self.assertEqual(t.ident, None)
self.assertTrue(re.match('<TestThread\(.*, initial\)>', repr(t)))
t.start()
if verbose:
print 'waiting for all tasks to complete'
for t in threads:
t.join(NUMTASKS)
self.assertTrue(not t.is_alive())
self.assertNotEqual(t.ident, 0)
self.assertFalse(t.ident is None)
self.assertTrue(re.match('<TestThread\(.*, \w+ -?\d+\)>', repr(t)))
if verbose:
print 'all tasks done'
self.assertEqual(numrunning.get(), 0)
def testWithBoundedSemaphore(self):
lock = threading.BoundedSemaphore()
def locked():
if lock.acquire(False):
lock.release()
return False
else:
return True
self.boilerPlate(lock, locked)
# This is needed to make the test actually run under regrtest.py!
def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
subdomains = subdomains or []
base_url = 'https://dnsdumpster.com/'
self.live_subdomains = []
self.engine_name = "DNSdumpster"
self.threads = 70
self.lock = threading.BoundedSemaphore(value=self.threads)
self.q = q
super(DNSdumpster, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
return
def __init__(self, subdomains, ports):
self.subdomains = subdomains
self.ports = ports
self.threads = 20
self.lock = threading.BoundedSemaphore(value=self.threads)
def __init__(self, max_threads=3):
# priority queue picks things with a lesser value first
self._submission_queue = queue.PriorityQueue()
self._submission_set = set()
self._threads_semaphore = threading.BoundedSemaphore(max_threads)
self._thread = threading.Thread(None, self.__run, 'ExecutorQueue.__run')
self._thread.daemon = True
self._thread.start()
def test_various_ops(self):
# This takes about n/3 seconds to run (about n/3 clumps of tasks,
# times about 1 second per clump).
NUMTASKS = 10
# no more than 3 of the 10 can run at once
sema = threading.BoundedSemaphore(value=3)
mutex = threading.RLock()
numrunning = Counter()
threads = []
for i in range(NUMTASKS):
t = TestThread("<thread %d>"%i, self, sema, mutex, numrunning)
threads.append(t)
self.assertEqual(t.ident, None)
self.assertTrue(re.match('<TestThread\(.*, initial\)>', repr(t)))
t.start()
if verbose:
print('waiting for all tasks to complete')
for t in threads:
t.join()
self.assertTrue(not t.is_alive())
self.assertNotEqual(t.ident, 0)
self.assertFalse(t.ident is None)
self.assertTrue(re.match('<TestThread\(.*, stopped -?\d+\)>',
repr(t)))
if verbose:
print('all tasks done')
self.assertEqual(numrunning.get(), 0)
