python类cpu_count()的实例源码

def cpu_count_physical():
    """Return the number of physical cores in the system."""
    mapping = {}
    current_info = {}
    with open_binary('%s/cpuinfo' % get_procfs_path()) as f:
        for line in f:
            line = line.strip().lower()
            if not line:
                # new section
                if (b'physical id' in current_info and
                        b'cpu cores' in current_info):
                    mapping[current_info[b'physical id']] = \
                        current_info[b'cpu cores']
                current_info = {}
            else:
                # ongoing section
                if (line.startswith(b'physical id') or
                        line.startswith(b'cpu cores')):
                    key, value = line.split(b'\t:', 1)
                    current_info[key] = int(value)

    # mimic os.cpu_count()
    return sum(mapping.values()) or None

def cpu_count(logical=True):
    """Return the number of logical CPUs in the system (same as
    os.cpu_count() in Python 3.4).

    If logical is False return the number of physical cores only
    (e.g. hyper thread CPUs are excluded).

    Return None if undetermined.

    The return value is cached after first call.
    If desired cache can be cleared like this:

    >>> psutil.cpu_count.cache_clear()
    """
    if logical:
        return _psplatform.cpu_count_logical()
    else:
        return _psplatform.cpu_count_physical()

def cpu_count_physical():
    """Return the number of physical cores in the system."""
    mapping = {}
    current_info = {}
    with open_binary('%s/cpuinfo' % get_procfs_path()) as f:
        for line in f:
            line = line.strip().lower()
            if not line:
                # new section
                if (b'physical id' in current_info and
                        b'cpu cores' in current_info):
                    mapping[current_info[b'physical id']] = \
                        current_info[b'cpu cores']
                current_info = {}
            else:
                # ongoing section
                if (line.startswith(b'physical id') or
                        line.startswith(b'cpu cores')):
                    key, value = line.split(b'\t:', 1)
                    current_info[key] = int(value)

    # mimic os.cpu_count()
    return sum(mapping.values()) or None

def cpu_count(logical=True):
    """Return the number of logical CPUs in the system (same as
    os.cpu_count() in Python 3.4).

    If logical is False return the number of physical cores only
    (e.g. hyper thread CPUs are excluded).

    Return None if undetermined.

    The return value is cached after first call.
    If desired cache can be cleared like this:

    >>> psutil.cpu_count.cache_clear()
    """
    if logical:
        return _psplatform.cpu_count_logical()
    else:
        return _psplatform.cpu_count_physical()

def startEvents(self):
        """ Process events
        """
        if self.state != QueueStatus.un_init:
            raise RuntimeError("At present the queue can only be started once!")

        self._thread_parent = Thread(target = self._worker, args = ( self.queue_parent,))
        self._thread_parent.start()

        self._child_worker_count = 1 if cpu_count() <= 1 else cpu_count() - 1
        self._thread_children = [
                Thread(target = self._worker, args = ( self.queue_child,)) \
                        for i in range(self._child_worker_count) ]
        [ x.start() for x in self._thread_children ]

        self._thread_feedback = Thread(target = self._worker, args = ( self.queue_feedback,))
        self._thread_feedback.start()

        self._thread_final = Thread(target = self._worker, args = ( self.queue_final,))
        self._thread_final.start()

        self._put_parent()
        self.on_started()

def wait_cgroup(sock, execute_task, time_limit_ns, memory_limit_bytes, process_limit):
    cgroup = CGroup()
    try:
        cgroup.memory_limit_bytes = memory_limit_bytes
        cgroup.pids_max = process_limit
        await cgroup.accept(sock)
        start_idle = _get_idle()

        while True:
            cpu_usage_ns = cgroup.cpu_usage_ns
            idle_usage_ns = int((_get_idle() - start_idle) / cpu_count() * 1e9)
            time_usage_ns = max(cpu_usage_ns, idle_usage_ns)
            time_remain_ns = time_limit_ns - time_usage_ns
            if time_remain_ns <= 0:
                return time_usage_ns, cgroup.memory_usage_bytes
            try:
                await wait_for(shield(execute_task), (time_remain_ns + WAIT_JITTER_NS) / 1e9)
                return cgroup.cpu_usage_ns, cgroup.memory_usage_bytes
            except TimeoutError:
                pass
    finally:
        while cgroup.kill():
            await sleep(.001)
        cgroup.close()

def cpu_count_physical():
    """Return the number of physical cores in the system."""
    mapping = {}
    current_info = {}
    with open_binary('%s/cpuinfo' % get_procfs_path()) as f:
        for line in f:
            line = line.strip().lower()
            if not line:
                # new section
                if (b'physical id' in current_info and
                        b'cpu cores' in current_info):
                    mapping[current_info[b'physical id']] = \
                        current_info[b'cpu cores']
                current_info = {}
            else:
                # ongoing section
                if (line.startswith(b'physical id') or
                        line.startswith(b'cpu cores')):
                    key, value = line.split(b'\t:', 1)
                    current_info[key] = int(value)

    # mimic os.cpu_count()
    return sum(mapping.values()) or None

def cpu_count(logical=True):
    """Return the number of logical CPUs in the system (same as
    os.cpu_count() in Python 3.4).

    If logical is False return the number of physical cores only
    (e.g. hyper thread CPUs are excluded).

    Return None if undetermined.

    The return value is cached after first call.
    If desired cache can be cleared like this:

    >>> psutil.cpu_count.cache_clear()
    """
    if logical:
        return _psplatform.cpu_count_logical()
    else:
        return _psplatform.cpu_count_physical()

def cpu_count_physical():
    """Return the number of physical cores in the system."""
    mapping = {}
    current_info = {}
    with open_binary('%s/cpuinfo' % get_procfs_path()) as f:
        for line in f:
            line = line.strip().lower()
            if not line:
                # new section
                if (b'physical id' in current_info and
                        b'cpu cores' in current_info):
                    mapping[current_info[b'physical id']] = \
                        current_info[b'cpu cores']
                current_info = {}
            else:
                # ongoing section
                if (line.startswith(b'physical id') or
                        line.startswith(b'cpu cores')):
                    key, value = line.split(b'\t:', 1)
                    current_info[key] = int(value)

    # mimic os.cpu_count()
    return sum(mapping.values()) or None

def cpu_count(logical=True):
    """Return the number of logical CPUs in the system (same as
    os.cpu_count() in Python 3.4).

    If logical is False return the number of physical cores only
    (e.g. hyper thread CPUs are excluded).

    Return None if undetermined.

    The return value is cached after first call.
    If desired cache can be cleared like this:

    >>> psutil.cpu_count.cache_clear()
    """
    if logical:
        return _psplatform.cpu_count_logical()
    else:
        return _psplatform.cpu_count_physical()

def info():
    uname = os.uname()
    cpu = lscpu()
    mem = meminfo()
    return {
        'arch': uname.machine,
        'byte_order': sys.byteorder,
        'cpu_cache_L1d': parse_size(cpu.get('L1d cache')),
        'cpu_cache_L1i': parse_size(cpu.get('L1i cache')),
        'cpu_cache_L2': parse_size(cpu.get('L2 cache')),
        'cpu_cache_L3': parse_size(cpu.get('L3 cache')),
        'cpu_count': os.cpu_count(),
        'cpu_mhz': parse_float(cpu.get('CPU MHz')),
        'cpu_name': cpu.get('Model name'),
        'kernel': uname.sysname,
        'kernel_release': uname.release,
        'kernel_version': uname.version,
        'memory': parse_size(mem.get('MemTotal'))
    }

def oparallel(std_data, final_meas, groups=None, cfg=None):
    """Calculate the hospital group scores for each LVM."""
    if cfg is not None:
        groups = cfg.GROUPS

    cpus = os.cpu_count()
    # nproc = 1 if cpus is None else cpus - 1 or 1  # leave one CPU unused
    nproc = 1 if cpus is None else cpus  # use all CPUs

    pool = multiprocessing.Pool(nproc)
    group_data = zip(
        [std_data for _ in groups],
        [final_meas[g] for g in groups],
        groups,
        [cfg for _ in groups]
        )
    r = pool.map(worker, group_data)
    return zip(*r)

def idle_cpu_count(mincpu=1):
    """Estimate number of idle CPUs, for use by multiprocessing code
    needing to determine how many processes can be run without excessive
    load. This function uses :func:`os.getloadavg` which is only available
    under a Unix OS.

    Parameters
    ----------
    mincpu : int
      Minimum number of CPUs to report, independent of actual estimate

    Returns
    -------
    idle : int
      Estimate of number of idle CPUs
    """

    if PY2:
        ncpu = mp.cpu_count()
    else:
        ncpu = os.cpu_count()
    idle = int(ncpu - np.floor(os.getloadavg()[0]))
    return max(mincpu, idle)

def cpu_count_physical():
    """Return the number of physical cores in the system."""
    mapping = {}
    current_info = {}
    with open_binary('%s/cpuinfo' % get_procfs_path()) as f:
        for line in f:
            line = line.strip().lower()
            if not line:
                # new section
                if (b'physical id' in current_info and
                        b'cpu cores' in current_info):
                    mapping[current_info[b'physical id']] = \
                        current_info[b'cpu cores']
                current_info = {}
            else:
                # ongoing section
                if (line.startswith(b'physical id') or
                        line.startswith(b'cpu cores')):
                    key, value = line.split(b'\t:', 1)
                    current_info[key] = int(value)

    # mimic os.cpu_count()
    return sum(mapping.values()) or None

def cpu_count(logical=True):
    """Return the number of logical CPUs in the system (same as
    os.cpu_count() in Python 3.4).

    If logical is False return the number of physical cores only
    (e.g. hyper thread CPUs are excluded).

    Return None if undetermined.

    The return value is cached after first call.
    If desired cache can be cleared like this:

    >>> psutil.cpu_count.cache_clear()
    """
    if logical:
        return _psplatform.cpu_count_logical()
    else:
        return _psplatform.cpu_count_physical()

def get_torch_num_workers(num_workers: int):
    """turn an int into a useful number of workers for a pytorch DataLoader.
    -1 means "use all CPU's", -2, means "use all but 1 CPU", etc.
    Note: 0 is interpreted by pytorch as doing data loading in the main process, while any positive number spawns a
    new process. We do not allow more processes to spawn than there are CPU's."""
    num_cpu = cpu_count()
    if num_workers < 0:
        n_workers = num_cpu + 1 + num_workers
        if n_workers < 0:
            print("Warning: {} fewer workers than the number of CPU's were specified, but there are only {} CPU's; "
                  "running data loading in the main process (num_workers = 0).".format(num_workers + 1, num_cpu))
        num_workers = max(0, n_workers)
    if num_workers > num_cpu:
        print("Warning, `num_workers` is {} but only {} CPU's are available; "
              "using this number instead".format(num_workers, num_cpu))
    return min(num_workers, num_cpu)


#####################################################################
# Iterator utils                                                    #
#####################################################################

def __init__(self, j=os.cpu_count(), max_utilization=100):
        """
        Initialize a parallel loop object.

        :param j: The maximum number of parallel jobs.
        :param max_utilization: The maximum CPU utilization. Above this no more new jobs
                                will be started.
        """
        self._j = j
        self._max_utilization = max_utilization
        # This gets initialized to 0, may be set to 1 anytime, but must not be reset to 0 ever;
        # thus, no locking is needed when accessing
        self._break = multiprocessing.sharedctypes.Value('i', 0, lock=False)
        self._lock = multiprocessing.Condition()
        self._slots = multiprocessing.sharedctypes.Array('i', j, lock=False)
        psutil.cpu_percent(None)

    # Beware! this is running in a new process now. state is shared with fork,
    # but only changes to shared objects will be visible in parent.

def __init__(self, test, *, cache=None, split=config_splitters.zeller,
                 proc_num=os.cpu_count(), max_utilization=100,
                 config_iterator=config_iterators.forward):
        """
        Initialize a CombinedParallelDD object.

        :param test: A callable tester object.
        :param cache: Cache object to use.
        :param split: Splitter method to break a configuration up to n part.
        :param proc_num: The level of parallelization.
        :param max_utilization: The maximum CPU utilization accepted.
        :param config_iterator: Reference to a generator function that provides config indices in an arbitrary order.
        """
        AbstractParallelDD.__init__(self, test, split, proc_num, max_utilization, cache=cache)

        self._config_iterator = config_iterator

def __init__(self, test, *, cache=None, split=config_splitters.zeller,
                 proc_num=os.cpu_count(), max_utilization=100,
                 subset_first=True, subset_iterator=config_iterators.forward, complement_iterator=config_iterators.forward):
        """
        Initialize a ParallelDD object.

        :param test: A callable tester object.
        :param cache: Cache object to use.
        :param split: Splitter method to break a configuration up to n part.
        :param proc_num: The level of parallelization.
        :param max_utilization: The maximum CPU utilization accepted.
        :param subset_first: Boolean value denoting whether the reduce has to start with the subset based approach or not.
        :param subset_iterator: Reference to a generator function that provides config indices in an arbitrary order.
        :param complement_iterator: Reference to a generator function that provides config indices in an arbitrary order.
        """
        AbstractParallelDD.__init__(self, test, split, proc_num, max_utilization, cache=cache)

        self._subset_first = subset_first
        self._subset_iterator = subset_iterator
        self._complement_iterator = complement_iterator

def get_logical_cpu_count():
    if psutil is not None:
        # Number of logical CPUs
        cpu_count = psutil.cpu_count()
    elif hasattr(os, 'cpu_count'):
        # Python 3.4
        cpu_count = os.cpu_count()
    else:
        cpu_count = None
        try:
            import multiprocessing
        except ImportError:
            pass
        else:
            try:
                cpu_count = multiprocessing.cpu_count()
            except NotImplementedError:
                pass

    if cpu_count is not None and cpu_count < 1:
        return None

    return cpu_count

def cpu_count_physical():
    """Return the number of physical cores in the system."""
    mapping = {}
    current_info = {}
    with open_binary('%s/cpuinfo' % get_procfs_path()) as f:
        for line in f:
            line = line.strip().lower()
            if not line:
                # new section
                if (b'physical id' in current_info and
                        b'cpu cores' in current_info):
                    mapping[current_info[b'physical id']] = \
                        current_info[b'cpu cores']
                current_info = {}
            else:
                # ongoing section
                if (line.startswith(b'physical id') or
                        line.startswith(b'cpu cores')):
                    key, value = line.split(b'\t:', 1)
                    current_info[key] = int(value)

    # mimic os.cpu_count()
    return sum(mapping.values()) or None

def cpu_count(logical=True):
    """Return the number of logical CPUs in the system (same as
    os.cpu_count() in Python 3.4).

    If logical is False return the number of physical cores only
    (e.g. hyper thread CPUs are excluded).

    Return None if undetermined.

    The return value is cached after first call.
    If desired cache can be cleared like this:

    >>> psutil.cpu_count.cache_clear()
    """
    if logical:
        return _psplatform.cpu_count_logical()
    else:
        return _psplatform.cpu_count_physical()

def cpu_count(logical=True):
    """Return the number of logical CPUs in the system (same as
    os.cpu_count() in Python 3.4).

    If logical is False return the number of physical cores only
    (e.g. hyper thread CPUs are excluded).

    Return None if undetermined.

    The return value is cached after first call.
    If desired cache can be cleared like this:

    >>> psutil.cpu_count.cache_clear()
    """
    if logical:
        return _psplatform.cpu_count_logical()
    else:
        return _psplatform.cpu_count_physical()

def cpu_count_physical():
    """Return the number of physical cores in the system."""
    mapping = {}
    current_info = {}
    with open_binary('%s/cpuinfo' % get_procfs_path()) as f:
        for line in f:
            line = line.strip().lower()
            if not line:
                # new section
                if (b'physical id' in current_info and
                        b'cpu cores' in current_info):
                    mapping[current_info[b'physical id']] = \
                        current_info[b'cpu cores']
                current_info = {}
            else:
                # ongoing section
                if (line.startswith(b'physical id') or
                        line.startswith(b'cpu cores')):
                    key, value = line.split(b'\t:', 1)
                    current_info[key] = int(value)

    # mimic os.cpu_count()
    return sum(mapping.values()) or None

def cpu_count(logical=True):
    """Return the number of logical CPUs in the system (same as
    os.cpu_count() in Python 3.4).

    If logical is False return the number of physical cores only
    (e.g. hyper thread CPUs are excluded).

    Return None if undetermined.

    The return value is cached after first call.
    If desired cache can be cleared like this:

    >>> psutil.cpu_count.cache_clear()
    """
    if logical:
        return _psplatform.cpu_count_logical()
    else:
        return _psplatform.cpu_count_physical()

def cpu_count_physical():
    """Return the number of physical cores in the system."""
    mapping = {}
    current_info = {}
    with open_binary('%s/cpuinfo' % get_procfs_path()) as f:
        for line in f:
            line = line.strip().lower()
            if not line:
                # new section
                if (b'physical id' in current_info and
                        b'cpu cores' in current_info):
                    mapping[current_info[b'physical id']] = \
                        current_info[b'cpu cores']
                current_info = {}
            else:
                # ongoing section
                if (line.startswith(b'physical id') or
                        line.startswith(b'cpu cores')):
                    key, value = line.split(b'\t:', 1)
                    current_info[key] = int(value)

    # mimic os.cpu_count()
    return sum(mapping.values()) or None

def cpu_count(logical=True):
    """Return the number of logical CPUs in the system (same as
    os.cpu_count() in Python 3.4).

    If logical is False return the number of physical cores only
    (e.g. hyper thread CPUs are excluded).

    Return None if undetermined.

    The return value is cached after first call.
    If desired cache can be cleared like this:

    >>> psutil.cpu_count.cache_clear()
    """
    if logical:
        return _psplatform.cpu_count_logical()
    else:
        return _psplatform.cpu_count_physical()

def workers(master_host, master_port, relay_socket_path, num_workers):
    # Start the relay
    master_redis_cfg = {'host': master_host, 'port': master_port}
    relay_redis_cfg = {'unix_socket_path': relay_socket_path}
    if os.fork() == 0:
        RelayClient(master_redis_cfg, relay_redis_cfg).run()
        return
    # Start the workers
    noise = SharedNoiseTable()  # Workers share the same noise
    num_workers = num_workers if num_workers else os.cpu_count()
    logging.info('Spawning {} workers'.format(num_workers))
    for _ in range(num_workers):
        if os.fork() == 0:
            run_worker(relay_redis_cfg, noise=noise)
            return
    os.wait()

def _get_thread_pool(self):
        # lazily initialized
        if not self._thread_pool:
            self._thread_pool = ThreadPool(os.cpu_count())
        return self._thread_pool

def main():
  print("Main Process PID: {}".format(multiprocessing.current_process().pid))
  myProcess = MyProcess()
  myProcess.start()
  myProcess.join()

  processes = []

  for i in range(os.cpu_count()):
    process = MyProcess()
    processes.append(process)
    process.start()

  for process in processes:
    process.join()