Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
ProfiT-HPC
aggregator
Commits
ccbb4aef
Commit
ccbb4aef
authored
Jan 13, 2020
by
Azat Khuziyakhmetov
Browse files
added average GPU utilization value
parent
46ed308b
Changes
5
Hide whitespace changes
Inline
Side-by-side
db/aggrstruct.py
View file @
ccbb4aef
...
...
@@ -38,6 +38,7 @@ class GPUData:
temp_max
=
None
power_max
=
None
usage_max
=
None
usage_avg
=
None
cpu_usage_max
=
None
cpu_mem_rss_max
=
None
cpu_proc_total
=
None
...
...
db/influx/influxdb_fetchgpu.py
View file @
ccbb4aef
...
...
@@ -183,6 +183,8 @@ def get_gpu_queries(job_id, t_start, t_end, gpus, delta):
q
=
get_query_gpu_last
(
m
,
par
[
"dbname"
],
t_start
,
t_end
,
gpus
)
elif
par
[
"query"
]
is
metrics
.
QType
.
MAX_PER_DEVICE
:
q
=
get_query_aggr_per_device
(
m
,
par
[
"dbname"
],
t_start
,
t_end
,
gpus
,
"max"
)
elif
par
[
"query"
]
is
metrics
.
QType
.
AVG_PER_DEVICE
:
q
=
get_query_aggr_per_device
(
m
,
par
[
"dbname"
],
t_start
,
t_end
,
gpus
,
"mean"
)
elif
par
[
"query"
]
is
metrics
.
QType
.
AVG_PER_DEVICE_INT
:
q
=
get_query_aggr_per_device_seq
(
m
,
par
[
"dbname"
],
t_start
,
t_end
,
delta
,
gpus
,
"mean"
)
elif
par
[
"query"
]
is
metrics
.
QType
.
MAX_SUM_PER_INT
:
...
...
db/influx/influxdb_wrapper.py
View file @
ccbb4aef
...
...
@@ -115,6 +115,7 @@ def get_aggregator(job_id, type="text"):
newbus
.
temp_max
=
busd
[
"gpu_temperature_max"
]
newbus
.
power_max
=
busd
[
"gpu_power_max"
]
newbus
.
usage_max
=
busd
[
"gpu_utilization_max"
]
newbus
.
usage_avg
=
busd
[
"gpu_utilization_avg"
]
newbus
.
cpu_usage_max
=
busd
[
"gpu_cpu_usage_max"
]
newbus
.
cpu_mem_rss_max
=
busd
[
"gpu_cpu_mem_max"
]
newbus
.
cpu_proc_total
=
busd
[
"gpu_cpu_proc_count_total"
]
...
...
db/influx/metrics.py
View file @
ccbb4aef
...
...
@@ -288,6 +288,12 @@ metrics_gpu = {
"query"
:
QType
.
MAX_PER_DEVICE
,
"measurement"
:
conf
.
measurements
[
"gpu_node"
],
},
"gpu_utilization_avg"
:
{
"dbname"
:
"utilization.gpu"
,
"type"
:
MType
.
INT
,
"query"
:
QType
.
AVG_PER_DEVICE
,
"measurement"
:
conf
.
measurements
[
"gpu_node"
],
},
"gpu_memory_avg_seq"
:
{
"dbname"
:
"memory.used"
,
"type"
:
MType
.
FLT
,
...
...
format/formats_json.py
View file @
ccbb4aef
...
...
@@ -55,6 +55,7 @@ class OutputSchemes:
"temp_max"
:
"temp_max"
,
"power_max"
:
"power_max"
,
"usage_max"
:
"usage_max"
,
"usage_avg"
:
"usage_avg"
,
"cpu_usage_max"
:
"cpu_usage_max"
,
"cpu_mem_rss_max"
:
"cpu_mem_rss_max"
,
"cpu_proc_total"
:
"cpu_proc_total"
,
...
...
@@ -121,6 +122,7 @@ class OutputSchemes:
"temp_max"
:
"temp_max"
,
"power_max"
:
"power_max"
,
"usage_max"
:
"usage_max"
,
"usage_avg"
:
"usage_avg"
,
"cpu_usage_max"
:
"cpu_usage_max"
,
"cpu_mem_rss_max"
:
"cpu_mem_rss_max"
,
"cpu_proc_total"
:
"cpu_proc_total"
,
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment