Skip to content

resources

pykmhelpers.core.resources

Helpers for inspecting and adjusting process resource limits (ulimits).

get_ulimit(resource_type=resource.RLIMIT_NOFILE)

Return the (soft, hard) ulimit values for the given resource type.

Source code in pykmhelpers/core/resources.py 
 9
10
11
def get_ulimit(resource_type=resource.RLIMIT_NOFILE):
    """Return the (soft, hard) ulimit values for the given resource type."""
    return resource.getrlimit(resource_type)

get_available_ram(safety_margin=1.0)

Return usable RAM in bytes: currently available memory scaled down by safety_margin.

Source code in pykmhelpers/core/resources.py 
14
15
16
17
18
19
20
21
def get_available_ram(safety_margin: float = 1.0) -> int:
    """Return usable RAM in bytes: currently available memory scaled down by `safety_margin`."""
    rss_limit, _ = get_ulimit(resource.RLIMIT_RSS)
    available_ram = psutil.virtual_memory().available
    usable_ram = rss_limit if 0 < rss_limit < available_ram else available_ram
    if usable_ram <= 0:
        raise ValueError("could not determine available RAM")
    return int(usable_ram * safety_margin)

get_available_threads(safety_margin=1.0)

Return a safe thread count: CPUs available to this process, scaled down by safety_margin.

Source code in pykmhelpers/core/resources.py 
24
25
26
27
28
29
30
31
def get_available_threads(safety_margin: float = 1.0) -> int:
    """Return a safe thread count: CPUs available to this process, scaled down by `safety_margin`."""
    try:
        cpu_count = len(os.sched_getaffinity(0))
    except AttributeError:
        # sched_getaffinity is Linux-only
        cpu_count = os.cpu_count() or 1
    return max(1, int(cpu_count * safety_margin))

get_max_open_files(safety_margin=1.0)

Return a safe open-files ceiling: the soft RLIMIT_NOFILE scaled down by safety_margin.

Source code in pykmhelpers/core/resources.py 
34
35
36
37
def get_max_open_files(safety_margin: float = 1.0) -> int:
    """Return a safe open-files ceiling: the soft RLIMIT_NOFILE scaled down by `safety_margin`."""
    soft, _ = get_ulimit(resource.RLIMIT_NOFILE)
    return max(1, int(soft * safety_margin))

maximize_nofile()

Raise the open-files (RLIMIT_NOFILE) limit to the highest allowed value.

Source code in pykmhelpers/core/resources.py 
40
41
42
43
44
45
46
47
48
49
def maximize_nofile():
    """Raise the open-files (RLIMIT_NOFILE) limit to the highest allowed value."""
    soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
    try:
        resource.setrlimit(
            resource.RLIMIT_NOFILE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY)
        )
    except ValueError:
        # Not root: raise soft to hard limit only
        resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))