def __init__(self, *, slot_bytes, slot_count):
"""Initializer.
Args:
slot_bytes: The maximum size of slots in the buffer.
slot_count: How many slots should be in the buffer.
"""
self.slot_count = slot_count
self.array = SlotArray(slot_bytes=slot_bytes, slot_count=slot_count)
self.lock = ReadersWriterLock()
# Each reading process may modify its own Pointer while the read
# lock is being held. Each reading process can also load the position
# of the writer, but not load any other readers. Each reading process
# can also load the value of the 'active' count.
self.readers = []
# The writer can load and store the Pointer of all the reader Pointers
# or the writer Pointer while the write lock is held. It can also load
# and store the value of the 'active' acount.
self.writer = Pointer(self.slot_count)
self.active = multiprocessing.RawValue(ctypes.c_uint, 0)
python类RawValue()的实例源码
def __init__(self, params, opt_type=None, lr=0, step=0):
self.var_shapes = [
var.get_shape().as_list()
for var in params]
self.size = sum([np.prod(shape) for shape in self.var_shapes])
self.step = RawValue(ctypes.c_int, step)
if opt_type == 'adam':
self.ms = self.malloc_contiguous(self.size)
self.vs = self.malloc_contiguous(self.size)
self.lr = RawValue(ctypes.c_float, lr)
elif opt_type == 'adamax':
self.ms = self.malloc_contiguous(self.size)
self.vs = self.malloc_contiguous(self.size)
self.lr = RawValue(ctypes.c_float, lr)
elif opt_type == 'rmsprop':
self.vars = self.malloc_contiguous(self.size, np.ones(self.size, dtype=np.float))
elif opt_type == 'momentum':
self.vars = self.malloc_contiguous(self.size)
else:
self.vars = self.malloc_contiguous(self.size)
def test_value(self, raw=False):
if raw:
values = [self.RawValue(code, value)
for code, value, _ in self.codes_values]
else:
values = [self.Value(code, value)
for code, value, _ in self.codes_values]
for sv, cv in zip(values, self.codes_values):
self.assertEqual(sv.value, cv[1])
proc = self.Process(target=self._test, args=(values,))
proc.daemon = True
proc.start()
proc.join()
for sv, cv in zip(values, self.codes_values):
self.assertEqual(sv.value, cv[2])
def test_getobj_getlock(self):
val1 = self.Value('i', 5)
lock1 = val1.get_lock()
obj1 = val1.get_obj()
val2 = self.Value('i', 5, lock=None)
lock2 = val2.get_lock()
obj2 = val2.get_obj()
lock = self.Lock()
val3 = self.Value('i', 5, lock=lock)
lock3 = val3.get_lock()
obj3 = val3.get_obj()
self.assertEqual(lock, lock3)
arr4 = self.Value('i', 5, lock=False)
self.assertFalse(hasattr(arr4, 'get_lock'))
self.assertFalse(hasattr(arr4, 'get_obj'))
self.assertRaises(AttributeError, self.Value, 'i', 5, lock='navalue')
arr5 = self.RawValue('i', 5)
self.assertFalse(hasattr(arr5, 'get_lock'))
self.assertFalse(hasattr(arr5, 'get_obj'))
worker.py 文件源码
项目：lustre_task_driven_monitoring_framework
作者: GSI-HPC
项目源码
文件源码
阅读 31
收藏 0
点赞 0
评论 0
def __init__(self):
# # RETURNS STDOUT: self._state = "TEXT" + str(NUMBER)
# # RETURNS BAD VALUE: self._timestamp.value = 1234567890.99
# self._state = multiprocessing.RawValue(ctypes.c_char_p)
# self._ost_name = multiprocessing.RawValue(ctypes.c_char_p)
# self._timestamp = multiprocessing.RawValue(ctypes.c_float)
self._state = multiprocessing.RawValue(ctypes.c_int, WorkerState.NOT_READY)
self._ost_name = multiprocessing.RawArray('c', 64)
self._timestamp = multiprocessing.RawValue(ctypes.c_uint, 0)
def __init__(self, slot_count, *, start=None):
default = start if start is not None else 0
self.counter = multiprocessing.RawValue(ctypes.c_longlong, default)
self.position = Position(slot_count)
def __init__(self):
self.lock = multiprocessing.Lock()
self.readers_condition = multiprocessing.Condition(self.lock)
self.writer_condition = multiprocessing.Condition(self.lock)
self.readers = multiprocessing.RawValue(ctypes.c_uint, 0)
self.writer = multiprocessing.RawValue(ctypes.c_bool, False)
def __init__(self, ndata, nprocs):
self._ndata = mp.RawValue(ctypes.c_int, ndata)
self._start = mp.RawValue(ctypes.c_int, 0)
self._lock = mp.Lock()
min_chunk = ndata // nprocs
min_chunk = ndata if min_chunk <= 2 else min_chunk
self._chunk = min_chunk
def __init__(self, initval=0):
self.val = RawValue('i', initval)
self.last_step_update_target = RawValue('i', initval)
self.lock = Lock()
def __init__(self, initval=0):
self.val = RawValue('i', initval)
self.last_step_update_target = RawValue('i', initval)
self.lock = Lock()
def get_n_gpu():
detected_n_gpu = mp.RawValue('i', 0)
p = mp.Process(target=n_gpu_subprocess, args=(detected_n_gpu,))
p.start()
p.join()
n_gpu = int(detected_n_gpu.value)
if n_gpu == -1:
raise ImportError("Must be able to import pygpu to use GPUs.")
return n_gpu
def __init__(self, num_actions, alg_type, opt_type = None, lr = 0):
# Net
if alg_type in ['q', 'sarsa']:
self.var_shapes = [(8, 8, 4, 16),
(16),
(4, 4, 16, 32),
(32),
(2592, 256), #(3872, 256) if PADDING = "SAME"
(256),
(256, num_actions),
(num_actions)]
self.size = 0
for shape in self.var_shapes:
self.size += np.prod(shape)
if opt_type == "adam":
self.ms = self.malloc_contiguous(self.size)
self.vs = self.malloc_contiguous(self.size)
self.lr = RawValue(ctypes.c_float, lr)
elif opt_type == "rmsprop":
self.vars = self.malloc_contiguous(self.size, np.ones(self.size, dtype=np.float))
else: #momentum
self.vars = self.malloc_contiguous(self.size)
else:
# no lstm
self.var_shapes = [(8, 8, 4, 16),
(16),
(4, 4, 16, 32),
(32),
(2592, 256), #(3872, 256)
(256),
(256, num_actions),
(num_actions),
(256, 1),
(1)]
self.size = 0
for shape in self.var_shapes:
self.size += np.prod(shape)
if opt_type == "adam":
self.ms = self.malloc_contiguous(self.size)
self.vs = self.malloc_contiguous(self.size)
self.lr = RawValue(ctypes.c_float, lr)
if opt_type == "rmsprop":
self.vars = self.malloc_contiguous(self.size, np.ones(self.size, dtype=np.float))
elif opt_type == "momentum":
self.vars = self.malloc_contiguous(self.size)
else:
self.vars = self.malloc_contiguous(self.size)
