Browse Source

Added hotfixes and removed --dummy

pull/11/head
Egon Rijpkema 3 years ago
parent
commit
e35ad2e97d
  1. 37
      roles/kill_memory_hogs/files/kill_hoggs.py
  2. 2
      roles/kill_memory_hogs/tasks/main.yml

37
roles/kill_memory_hogs/files/kill_hoggs.py

@ -17,7 +17,7 @@ def post_to_slack(message: str): @@ -17,7 +17,7 @@ def post_to_slack(message: str):
Args:
message (str): Message to post
"""
with open('/opt/kill_hogs/kill_hogs.yml', 'r') as f:
with open('/opt/kill_hoggs/kill_hoggs.yml', 'r') as f:
config = yaml.load(f.read())
slack_url = config['slack_url']
data = json.dumps({
@ -79,8 +79,8 @@ def kill_hogs(memory_threshold, @@ -79,8 +79,8 @@ def kill_hogs(memory_threshold,
for proc in procs:
try:
proc.cpu_percent()
except Exception as e:
logging.exception(e)
except (psutil.NoSuchProcess, FileNotFoundError) as e:
pass
time.sleep(interval)
for proc in procs:
@ -89,23 +89,24 @@ def kill_hogs(memory_threshold, @@ -89,23 +89,24 @@ def kill_hogs(memory_threshold,
# see https://psutil.readthedocs.io/en/latest/
proc.cached_cpu_percent = proc.cpu_percent()
proc.cached_memory_percent = proc.memory_percent()
except Exception as e:
logging.exception(e)
proc_cpu_percent = 0
if proc.uids().real == 0 or (proc.cached_memory_percent < .1
and proc.cached_cpu_percent < 1):
continue # do not kill root processes.
# Check username here. It is somewhat expensive.
username = proc.username()
if not ((username[0] in ('s', 'p', 'f') and username[1:].isdigit())
or username[:5] == 'umcg-'):
continue # we only kill processes of p, s and f accounts
users[username]['memory_percent'] += proc.cached_memory_percent
users[username]['cpu_percent'] += proc.cached_cpu_percent
users[username]['processes'].append(proc)
if proc.uids().real == 0 or (proc.cached_memory_percent < .1
and proc.cached_cpu_percent < 1):
continue # do not kill root processes.
# Check username here. It is somewhat expensive.
username = proc.username()
if not ((username[0] in ('s', 'p', 'f') and username[1:].isdigit())
or username[:5] == 'umcg-'):
continue # we only kill processes of p, s and f accounts
users[username]['memory_percent'] += proc.cached_memory_percent
users[username]['cpu_percent'] += proc.cached_cpu_percent
users[username]['processes'].append(proc)
except (psutil.NoSuchProcess, FileNotFoundError) as e:
pass
#logging.exception(e)
for username, data in users.items():
if data['memory_percent'] > memory_threshold or data['cpu_percent'] > cpu_threshold:

2
roles/kill_memory_hogs/tasks/main.yml

@ -31,7 +31,7 @@ @@ -31,7 +31,7 @@
name: Kill jobs that are using excessive memory.
minute: '/10'
user: root
job: '/usr/bin/python36 /opt/kill_hoggs/kill_hoggs.py --slack --dummy'
job: '/usr/bin/python36 /opt/kill_hoggs/kill_hoggs.py --slack'
cron_file: kill_memory_hogs
- cronvar:

Loading…
Cancel
Save