faster attribute computation

rcm/attributes.py

@@ -13,27 +13,60 @@ def printd(*args, **kargs):
 
 def rcmattr(func):
     '''Function decorator. Checks and sets arguments'''
-    def wrapper(Aggr, *args, **kwargs):
-        if type(Aggr) != Aggregator:
-            raise TypeError("argument of attribute should be Aggregator")
+    def wrapper(*args, **kwargs):
         printd(" - evaluation of the attribute {:s} - ".format(func.__name__))
-        res = func(Aggr, *args, **kwargs)
+        res = func(*args, **kwargs)
         printd(" - finished the attribute {:s} ({}) - ".format(func.__name__, res))
         return res
+
+    wrapper.decorator = rcmattr
+    wrapper.__name__ = func.__name__
+    wrapper.__doc__ = func.__doc__
+
     return wrapper
 
 class Attributes:
+    _aggr = None
+    attr_values = {}
+
+    def __init__(self, Aggr):
+        # init function
+        if type(Aggr) != Aggregator:
+            raise TypeError("argument for init should be Aggregator")
+        self._aggr = Aggr
+        return
+
+    def calculate_attrs(self):
+        attr_func = self.get_attribute_funcs()
+
+        for func in attr_func:
+            self.attr_values[func.__name__] = func()
+
+        return self.attr_values
+
+    def get_attribute_funcs(self):
+        """
+            Returns all methods with attribute decorator
+        """
+        methods = []
+        for maybeDecorated in dir(self):
+            cur_attr = self.__getattribute__(maybeDecorated)
+            if hasattr(cur_attr, 'decorator'):
+                if cur_attr.decorator is rcmattr:
+                    methods.append(cur_attr)
+        return methods
+
 
     @rcmattr
-    def cpu_usage_total_max(Aggr):
+    def cpu_usage_total_max(self):
         cpu_usage = 0
         cpu_usage_sum = 0
 
         alloc_cu_sum = 0
 
-        cpu_requested = Aggr.job.requested_cu
+        cpu_requested = self._aggr.job.requested_cu
 
-        for node in Aggr.nodes:
+        for node in self._aggr.nodes:
             thr_cnt = node.virt_thr_core + node.phys_thr_core
 
             cpu_usage_sum += node.proc.cpu_usage
@@ -47,45 +80,42 @@ class Attributes:
         res = "NORM"
         if cpu_usage < 50 and alloc_cu_sum <= 8:
             res = "LOW"
-            break
         elif cpu_usage < 80 and alloc_cu_sum > 8:
             res = "LOW"
-            break
 
         return res
 
     @rcmattr
-    def mem_usage_total(Aggr):
-        MEMDIFF_HIGH = 1024*1024*1024
+    def mem_usage_total(self):
+        MEM_LOW_BOUND = 32*1024*1024*1024 # 32 GiB
         mem_req_total = 0
         mem_used_max = 0
 
-        for node in Aggr.nodes:
-            mem_per_core = node.main_mem / (node.sockets * node.cores_per_socket)
-            mem_req_total += mem_per_core * node.alloc_cu
+        for node in self._aggr.nodes:
+            thr_cnt = node.virt_thr_core + node.phys_thr_core
+            mem_per_core = node.main_mem / (node.sockets * node.cores_per_socket * thr_cnt)
 
+            mem_req_total += mem_per_core * node.alloc_cu
             mem_used_max += node.proc.mem_rss_max
 
         mem_ratio = mem_used_max / mem_req_total
 
-        mem_diff = abs(mem_used_max - mem_req_total)
-
         printd("mem total = {}, mem used {}, ratio {}".
             format(mem_req_total, mem_used_max, mem_ratio))
 
-        res = "MED"
-        if mem_ratio > 1 and mem_diff > MEMDIFF_HIGH:
+        res = "NORM"
+        if mem_used_max > MEM_LOW_BOUND and mem_ratio > 1:
             res = "HIGH"
-        elif mem_ratio < 0.5 and mem_diff > MEMDIFF_HIGH:
+        elif mem_used_max > MEM_LOW_BOUND and mem_ratio < 0.3:
             res = "LOW"
 
         return res
 
     @rcmattr
-    def node_cpu_usage_max(Aggr):
+    def node_cpu_usage_max(self):
         res = ["ZERO", "LOW", "NORM", "HIGH"]
         max_res = res.index("ZERO")
-        for node in Aggr.nodes:
+        for node in self._aggr.nodes:
             thr_cnt = node.virt_thr_core + node.phys_thr_core
             node_proc_ratio = node.proc.cpu_usage / (node.alloc_cu / thr_cnt)
 
@@ -107,10 +137,11 @@ class Attributes:
 
         return res[max_res]
 
-    def node_cpu_usage_min(Aggr):
+    @rcmattr
+    def node_cpu_usage_min(self):
         res = ["ZERO", "LOW", "NORM", "HIGH"]
         min_res = res.index("HIGH")
-        for node in Aggr.nodes:
+        for node in self._aggr.nodes:
             thr_cnt = node.virt_thr_core + node.phys_thr_core
             node_proc_ratio = node.proc.cpu_usage / (node.alloc_cu / thr_cnt)
 
@@ -128,20 +159,20 @@ class Attributes:
             elif node_proc_ratio > 80 and node.alloc_cu > 8:
                 node_res = res.index("NORM")
 
-            min_res = min(node_res, max_res)
+            min_res = min(node_res, min_res)
 
         return res[min_res]
 
     @rcmattr
-    def req_walltime(Aggr):
+    def req_walltime(self):
         MIN_R = 4
         SHIFT_R = 6
 
-        rtime = Aggr.job.run_time
-        utime = Aggr.job.requested_time
+        rtime = self._aggr.job.run_time
+        utime = self._aggr.job.requested_time
 
         res = "UNKNOWN"
-        if rtime > MIN_R && utime < rtime/2 && utime < (rtime - SHIFT_R):
+        if rtime > MIN_R and utime < rtime/2 and utime < (rtime - SHIFT_R):
             res = "HIGH"
         else:
             res = "NORM"
@@ -149,9 +180,9 @@ class Attributes:
         return res
 
     @rcmattr
-    def overloaded_node_exists(Aggr):
+    def overloaded_node_exists(self):
         exists = False
-        for node in Aggr.nodes:
+        for node in self._aggr.nodes:
             if node.proc.cpu_usage > 90:
                 continue
 
@@ -162,8 +193,8 @@ class Attributes:
         return exists
 
     @rcmattr
-    def job_nodes_amount(Aggr):
-        nnodes = count(Aggr.nodes)
+    def job_nodes_amount(self):
+        nnodes = len(self._aggr.nodes)
 
         res = "ERR"
         if nnodes == 1:
@@ -172,3 +203,13 @@ class Attributes:
             res = "MULT"
 
         return res
+
+    @rcmattr
+    def mem_swap_used(self):
+        swap_used = False
+
+        for node in self._aggr.nodes:
+            if node.proc.mem_swap_max > 0:
+                swap_used = True
+
+        return swap_used

rcm/rcm.py

 from .rules import RULES
 from rcm import attributes
 
-attributes.DEBUG = True
+attributes.DEBUG = False
 
-def attributes_match(attrs, Aggr):
+def attributes_match(attrs, values):
     matches = True
-    for rule_attr, attr_value in attrs.items():
-        try:
-            attr_func = getattr(attributes.Attributes, rule_attr)
-        except AttributeError:
-            raise AttributeError("there is no attribute " + rule_attr)
+    for attr_name, attr_expected in attrs.items():
+        if attr_name not in values:
+            raise ValueError("Attribute is not found", attr_name)
 
+        attr_value = values[attr_name]
         expected_vals = []
-        if isinstance(attr_value, list):
-            expected_vals.extend(attr_value)
+
+        if isinstance(attr_expected, list):
+            expected_vals.extend(attr_expected)
         else:
-            expected_vals.append(attr_value)
+            expected_vals.append(attr_expected)
 
-        if attr_func(Aggr) not in expected_vals:
+        if attr_value not in expected_vals:
             matches = False
             break
 
@@ -25,9 +25,13 @@ def attributes_match(attrs, Aggr):
 
 
 def get_recommendations(Aggr):
+    Attrs = attributes.Attributes(Aggr)
+
+    attr_values = Attrs.calculate_attrs()
+
     rcms = []
     for rule in RULES:
-        if attributes_match(rule["attrs"], Aggr):
+        if attributes_match(rule["attrs"], attr_values):
             rcms.append(rule["msg"])
 
     return rcms