From 185114a01e55ad3c27d68f6ed2943b2d72e50106 Mon Sep 17 00:00:00 2001
From: Matthijs van der Wild <matthijs.van-der-wild@durham.ac.uk>
Date: Wed, 28 May 2025 17:31:54 +0000
Subject: [PATCH] move python logic get_memory_fraction to separate library
---
pyproject.toml | 1 +
scripts/query_memory.py | 32 ++++++++++++++++++++++++++++++++
steps/get_memory_fraction.cwl | 22 +++-------------------
3 files changed, 36 insertions(+), 19 deletions(-)
create mode 100755 scripts/query_memory.py
diff --git a/pyproject.toml b/pyproject.toml
index bdb2b267..030dc7b1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,6 +7,7 @@ dependencies = [
"losoto",
"numpy",
"pandas",
+ "psutil",
]
[project.optional-dependencies]
diff --git a/scripts/query_memory.py b/scripts/query_memory.py
new file mode 100755
index 00000000..df920c67
--- /dev/null
+++ b/scripts/query_memory.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import psutil
+import sys
+
+
+def main(memory_percentage):
+ """
+ Computes memory_percentage percent of the available virtual memory.
+ Writes the amount of available memory to a dictionary.
+ """
+
+ # psutil outputs the memory in bytes. This is converted into
+ # mebibytes (1 MiB = 2^20 B) to match CWL's ResourceRequirement input.
+ required_memory = int(
+ psutil.virtual_memory().available / 2**20 * memory_percentage / 100
+ )
+
+ result = {"memory": required_memory}
+
+ with open("./memory.json", "w") as fp:
+ json.dump(result, fp)
+
+ return result
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("memory_percentage", type=int)
+ main(parser.parse_args().memory_percentage)
diff --git a/steps/get_memory_fraction.cwl b/steps/get_memory_fraction.cwl
index f96d348a..c47b844b 100644
--- a/steps/get_memory_fraction.cwl
+++ b/steps/get_memory_fraction.cwl
@@ -7,15 +7,15 @@ doc: |
This is used to ensure that flagging jobs don't exceed the
node's resources.
-baseCommand:
- - python3
- - query_memory.py
+baseCommand: query_memory.py
stdout: memory.txt
inputs:
- id: fraction
type: int
+ inputBinding:
+ position: 1
doc: |
The required fraction of the node's
available memory for a flagging job.
@@ -37,20 +37,4 @@ hints:
requirements:
- class: InlineJavascriptRequirement
- - class: InitialWorkDirRequirement
- listing:
- - entryname: query_memory.py
- entry: |
- import json
- import psutil
- # psuitil outputs the memory in bytes.
- # This is converted into mebibytes (1 MiB = 2^20 B)
- # to match CWL's ResourceRequirement input.
- required_memory = int(psutil.virtual_memory().available / 2**20
- * $(inputs.fraction) / 100)
- print("The available memory is ", required_memory)
- result = {"memory" : required_memory}
-
- with open('./memory.json', 'w') as fp:
- json.dump(result, fp)
--
GitLab