Verstack Local
841a95c56a
Save the tested release workflows, RAM media processing, program discovery, FLIRT library, and shared-object maintenance. Add Git exclusions, file attributes, and instructions for a later push using a forwarded SSH agent.
48 lines
2.1 KiB
Python
48 lines
2.1 KiB
Python
"""CPU and RAM bounds for independent work inside one extraction job."""
|
|
import math
|
|
import os
|
|
import pathlib
|
|
|
|
MIB=1024**2
|
|
|
|
def cgroup_dirs():
|
|
root=pathlib.Path('/sys/fs/cgroup')
|
|
try:
|
|
group=next(line.split(':',2)[2] for line in pathlib.Path('/proc/self/cgroup').read_text().splitlines() if line.startswith('0::'))
|
|
path=(root/group.lstrip('/')).resolve()
|
|
while path.is_relative_to(root) and path!=root:
|
|
yield path
|
|
path=path.parent
|
|
except (OSError,ValueError,StopIteration):pass
|
|
yield root
|
|
|
|
def available_cpus():
|
|
count=len(os.sched_getaffinity(0)) if hasattr(os,'sched_getaffinity') else os.cpu_count() or 1
|
|
# Respect cgroup v2 quotas as well as the process affinity mask.
|
|
for path in cgroup_dirs():
|
|
try:
|
|
quota,period=(path/'cpu.max').read_text().split()
|
|
if quota!='max':count=min(count,max(1,math.ceil(int(quota)/int(period))))
|
|
except (OSError,ValueError,ZeroDivisionError):pass
|
|
return max(1,count)
|
|
|
|
def cpu_threads(request):
|
|
available=available_cpus()
|
|
requested=request.get('cpu_threads',0)
|
|
if not isinstance(requested,int) or requested<0:raise ValueError('cpu_threads must be a nonnegative integer')
|
|
# Leave two logical CPUs for the workbench and archive service on larger hosts.
|
|
return min(available,requested) if requested else max(1,available-2)
|
|
|
|
def worker_count(request,jobs,per_worker_bytes=256*MIB):
|
|
memory=int(request.get('workspace_bytes',per_worker_bytes*4))//4
|
|
try:
|
|
available=next(int(line.split()[1])*1024 for line in pathlib.Path('/proc/meminfo').read_text().splitlines() if line.startswith('MemAvailable:'))
|
|
memory=min(memory,available//4)
|
|
except (OSError,ValueError,StopIteration):pass
|
|
for path in cgroup_dirs():
|
|
try:
|
|
limit=(path/'memory.max').read_text().strip()
|
|
if limit!='max':memory=min(memory,max(0,int(limit)-int((path/'memory.current').read_text()))//4)
|
|
except (OSError,ValueError):pass
|
|
return max(1,min(cpu_threads(request),max(1,jobs),max(1,memory//per_worker_bytes)))
|