added average GPU utilization value

db/aggrstruct.py

@@ -38,6 +38,7 @@ class GPUData:
     temp_max = None
     power_max = None
     usage_max = None
+    usage_avg = None
     cpu_usage_max = None
     cpu_mem_rss_max = None
     cpu_proc_total = None

db/influx/influxdb_fetchgpu.py

@@ -183,6 +183,8 @@ def get_gpu_queries(job_id, t_start, t_end, gpus, delta):
             q = get_query_gpu_last(m, par["dbname"], t_start, t_end, gpus)
         elif par["query"] is metrics.QType.MAX_PER_DEVICE:
             q = get_query_aggr_per_device(m, par["dbname"], t_start, t_end, gpus, "max")
+        elif par["query"] is metrics.QType.AVG_PER_DEVICE:
+            q = get_query_aggr_per_device(m, par["dbname"], t_start, t_end, gpus, "mean")
         elif par["query"] is metrics.QType.AVG_PER_DEVICE_INT:
             q = get_query_aggr_per_device_seq(m, par["dbname"], t_start, t_end, delta, gpus, "mean")
         elif par["query"] is metrics.QType.MAX_SUM_PER_INT:

db/influx/influxdb_wrapper.py

@@ -115,6 +115,7 @@ def get_aggregator(job_id, type="text"):
                     newbus.temp_max = busd["gpu_temperature_max"]
                     newbus.power_max = busd["gpu_power_max"]
                     newbus.usage_max = busd["gpu_utilization_max"]
+                    newbus.usage_avg = busd["gpu_utilization_avg"]
                     newbus.cpu_usage_max = busd["gpu_cpu_usage_max"]
                     newbus.cpu_mem_rss_max = busd["gpu_cpu_mem_max"]
                     newbus.cpu_proc_total = busd["gpu_cpu_proc_count_total"]

db/influx/metrics.py

@@ -288,6 +288,12 @@ metrics_gpu = {
         "query": QType.MAX_PER_DEVICE,
         "measurement": conf.measurements["gpu_node"],
     },
+    "gpu_utilization_avg": {
+        "dbname": "utilization.gpu",
+        "type": MType.INT,
+        "query": QType.AVG_PER_DEVICE,
+        "measurement": conf.measurements["gpu_node"],
+    },
     "gpu_memory_avg_seq": {
         "dbname": "memory.used",
         "type": MType.FLT,

format/formats_json.py

@@ -55,6 +55,7 @@ class OutputSchemes:
                             "temp_max": "temp_max",
                             "power_max": "power_max",
                             "usage_max": "usage_max",
+                            "usage_avg": "usage_avg",
                             "cpu_usage_max": "cpu_usage_max",
                             "cpu_mem_rss_max": "cpu_mem_rss_max",
                             "cpu_proc_total": "cpu_proc_total",
@@ -121,6 +122,7 @@ class OutputSchemes:
                             "temp_max": "temp_max",
                             "power_max": "power_max",
                             "usage_max": "usage_max",
+                            "usage_avg": "usage_avg",
                             "cpu_usage_max": "cpu_usage_max",
                             "cpu_mem_rss_max": "cpu_mem_rss_max",
                             "cpu_proc_total": "cpu_proc_total",