From d0ab9ce668c94d1aae5808d1c0b02bf4f722ab2a Mon Sep 17 00:00:00 2001 From: Sabrina Krakau Date: Tue, 2 May 2023 09:36:51 +0200 Subject: [PATCH 1/7] Add CPU model name to trace_file and TraceRecord Signed-off-by: Sabrina Krakau --- .../src/main/groovy/nextflow/trace/TraceRecord.groovy | 5 +++++ .../src/main/resources/nextflow/executor/command-trace.txt | 2 ++ 2 files changed, 7 insertions(+) diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy index 96e0f372c7..6fab169ac7 100644 --- a/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy @@ -87,6 +87,7 @@ class TraceRecord implements Serializable { syscw: 'num', // -- /proc/$pid/io read_bytes: 'mem', // -- /proc/$pid/io write_bytes:'mem', // -- /proc/$pid/io + cpu_model: 'str', // -- /proc/cpuinfo field 'model name' attempt: 'num', workdir: 'str', script: 'str', @@ -448,6 +449,10 @@ class TraceRecord implements Serializable { this.put(name, val) break + case 'cpu_model': + this.put(name, value) + break + default: def val = parseLong(value, file, name) this.put(name, val) diff --git a/modules/nextflow/src/main/resources/nextflow/executor/command-trace.txt b/modules/nextflow/src/main/resources/nextflow/executor/command-trace.txt index 9cf1292962..eafb4a4eef 100644 --- a/modules/nextflow/src/main/resources/nextflow/executor/command-trace.txt +++ b/modules/nextflow/src/main/resources/nextflow/executor/command-trace.txt @@ -163,6 +163,7 @@ nxf_trace_linux() { ## https://stackoverflow.com/questions/27508531/calculate-cpu-per-process/27514562##27514562 ## https://stackoverflow.com/questions/16726779/how-do-i-get-the-total-cpu-usage-of-an-application-from-proc-pid-stat local num_cpus=$(< /proc/cpuinfo grep '^processor' -c) + local cpu_model=$(< /proc/cpuinfo grep '^model name' | head -n 1 | awk 'BEGIN{FS="\t: "} { print $2 }') local tot_time0=$(grep '^cpu ' /proc/stat | awk '{sum=$2+$3+$4+$5+$6+$7+$8+$9; printf "%.0f",sum}') local cpu_time0=$(2> /dev/null < /proc/$pid/stat awk '{printf "%.0f", ($16+$17)*10 }' || echo -n 'X') local io_stat0=($(2> /dev/null < /proc/$pid/io sed 's/^.*:\s*//' | head -n 6 | tr '\n' ' ' || echo -n '0 0 0 0 0 0')) @@ -199,6 +200,7 @@ nxf_trace_linux() { echo "nextflow.trace/v2" > $trace_file echo "realtime=$wall_time" >> $trace_file echo "%cpu=$ucpu" >> $trace_file + echo "cpu_model=$cpu_model" >> $trace_file echo "rchar=${io_stat1[0]}" >> $trace_file echo "wchar=${io_stat1[1]}" >> $trace_file echo "syscr=${io_stat1[2]}" >> $trace_file From 236229d4e034f8d5129893e23dbd37213de5a2a1 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 5 May 2023 10:26:30 +0200 Subject: [PATCH 2/7] add cpu_model to nxf_trace_mac Signed-off-by: mirpedrol --- .../src/main/resources/nextflow/executor/command-trace.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/nextflow/src/main/resources/nextflow/executor/command-trace.txt b/modules/nextflow/src/main/resources/nextflow/executor/command-trace.txt index eafb4a4eef..62da5a46ce 100644 --- a/modules/nextflow/src/main/resources/nextflow/executor/command-trace.txt +++ b/modules/nextflow/src/main/resources/nextflow/executor/command-trace.txt @@ -127,6 +127,7 @@ nxf_write_trace() { echo "nextflow.trace/v2" > $trace_file echo "realtime=$wall_time" >> $trace_file echo "%cpu=$ucpu" >> $trace_file + echo "cpu_model=$cpu_model" >> $trace_file echo "rchar=${io_stat1[0]}" >> $trace_file echo "wchar=${io_stat1[1]}" >> $trace_file echo "syscr=${io_stat1[2]}" >> $trace_file @@ -144,6 +145,7 @@ nxf_trace_mac() { local end_millis=$(nxf_date) local wall_time=$((end_millis-start_millis)) local ucpu='' + local cpu_model='' local io_stat1=('' '' '' '' '' '') nxf_write_trace } From 2817cf26ff9bfcf06151deef1a1f2db102e35dd0 Mon Sep 17 00:00:00 2001 From: Sabrina Krakau Date: Mon, 19 Jun 2023 14:38:56 +0200 Subject: [PATCH 3/7] Add changes for CPU model info to test-bash-wrapper-with-trace.txt Signed-off-by: Sabrina Krakau --- .../nextflow/executor/test-bash-wrapper-with-trace.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt index f97c04e61b..49ea12e675 100644 --- a/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt +++ b/modules/nextflow/src/test/resources/nextflow/executor/test-bash-wrapper-with-trace.txt @@ -109,6 +109,7 @@ nxf_write_trace() { echo "nextflow.trace/v2" > $trace_file echo "realtime=$wall_time" >> $trace_file echo "%cpu=$ucpu" >> $trace_file + echo "cpu_model=$cpu_model" >> $trace_file echo "rchar=${io_stat1[0]}" >> $trace_file echo "wchar=${io_stat1[1]}" >> $trace_file echo "syscr=${io_stat1[2]}" >> $trace_file @@ -125,6 +126,7 @@ nxf_trace_mac() { local end_millis=$(nxf_date) local wall_time=$((end_millis-start_millis)) local ucpu='' + local cpu_model='' local io_stat1=('' '' '' '' '' '') nxf_write_trace } @@ -139,6 +141,7 @@ nxf_trace_linux() { local pid=$$ command -v ps &>/dev/null || { >&2 echo "Command 'ps' required by nextflow to collect task metrics cannot be found"; exit 1; } local num_cpus=$(< /proc/cpuinfo grep '^processor' -c) + local cpu_model=$(< /proc/cpuinfo grep '^model name' | head -n 1 | awk 'BEGIN{FS="\t: "} { print $2 }') local tot_time0=$(grep '^cpu ' /proc/stat | awk '{sum=$2+$3+$4+$5+$6+$7+$8+$9; printf "%.0f",sum}') local cpu_time0=$(2> /dev/null < /proc/$pid/stat awk '{printf "%.0f", ($16+$17)*10 }' || echo -n 'X') local io_stat0=($(2> /dev/null < /proc/$pid/io sed 's/^.*:\s*//' | head -n 6 | tr '\n' ' ' || echo -n '0 0 0 0 0 0')) @@ -171,6 +174,7 @@ nxf_trace_linux() { echo "nextflow.trace/v2" > $trace_file echo "realtime=$wall_time" >> $trace_file echo "%cpu=$ucpu" >> $trace_file + echo "cpu_model=$cpu_model" >> $trace_file echo "rchar=${io_stat1[0]}" >> $trace_file echo "wchar=${io_stat1[1]}" >> $trace_file echo "syscr=${io_stat1[2]}" >> $trace_file From 6c524aacaca686f42c69169b73ca9d8b4039f355 Mon Sep 17 00:00:00 2001 From: Sabrina Krakau Date: Mon, 19 Jun 2023 14:45:21 +0200 Subject: [PATCH 4/7] Move cpu_model traceRecord field Signed-off-by: Sabrina Krakau --- .../src/main/groovy/nextflow/trace/TraceRecord.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy index 6fab169ac7..43efcb0ce3 100644 --- a/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy @@ -87,7 +87,6 @@ class TraceRecord implements Serializable { syscw: 'num', // -- /proc/$pid/io read_bytes: 'mem', // -- /proc/$pid/io write_bytes:'mem', // -- /proc/$pid/io - cpu_model: 'str', // -- /proc/cpuinfo field 'model name' attempt: 'num', workdir: 'str', script: 'str', @@ -101,7 +100,8 @@ class TraceRecord implements Serializable { error_action:'str', vol_ctxt: 'num', inv_ctxt: 'num', - hostname: 'str' + hostname: 'str', + cpu_model: 'str', // -- /proc/cpuinfo field 'model name' ] static public Map> FORMATTER = [ From 6cc161b45057ea20927eacd31d24afd0e5b5bd64 Mon Sep 17 00:00:00 2001 From: Sabrina Krakau Date: Mon, 19 Jun 2023 15:21:27 +0200 Subject: [PATCH 5/7] Add cpu_model traceRecord field to docs Signed-off-by: Sabrina Krakau --- docs/tracing.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/tracing.md b/docs/tracing.md index b53cbf19b4..5b940a77f9 100644 --- a/docs/tracing.md +++ b/docs/tracing.md @@ -307,6 +307,9 @@ The following table shows the fields that can be included in the execution repor ::: : The host on which the task was executed. Supported only for the Kubernetes executor yet. Activate with `k8s.fetchNodeName = true` in the Nextflow config file. +`cpu_model` +: The name of the CPU model which was used to execute the task. This data is read from file `/proc/cpuinfo`. + :::{note} These metrics provide an estimation of the resources used by running tasks. They are not an alternative to low-level performance analysis tools, and they may not be completely accurate, especially for very short-lived tasks (running for less than a few seconds). ::: From cd48e7390fad6a17ca60cfb3af22ff57d8ef8f8d Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 5 Jul 2023 10:51:12 +0200 Subject: [PATCH 6/7] Update modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy [ci fast] Signed-off-by: Paolo Di Tommaso --- .../nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy b/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy index 43efcb0ce3..0662c8fb20 100644 --- a/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/trace/TraceRecord.groovy @@ -101,7 +101,7 @@ class TraceRecord implements Serializable { vol_ctxt: 'num', inv_ctxt: 'num', hostname: 'str', - cpu_model: 'str', // -- /proc/cpuinfo field 'model name' + cpu_model: 'str' ] static public Map> FORMATTER = [ From fd6ca60cb8d8d5853d40c1cf9fd1f9515fe04aa6 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Wed, 5 Jul 2023 10:51:25 +0200 Subject: [PATCH 7/7] Update docs/tracing.md [ci skip] Signed-off-by: Paolo Di Tommaso --- docs/tracing.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/tracing.md b/docs/tracing.md index 5b940a77f9..98cdb167b4 100644 --- a/docs/tracing.md +++ b/docs/tracing.md @@ -308,7 +308,9 @@ The following table shows the fields that can be included in the execution repor : The host on which the task was executed. Supported only for the Kubernetes executor yet. Activate with `k8s.fetchNodeName = true` in the Nextflow config file. `cpu_model` -: The name of the CPU model which was used to execute the task. This data is read from file `/proc/cpuinfo`. +: :::{versionadded} 22.07.0-edge + ::: +: The name of the CPU model used to execute the task. This data is read from file `/proc/cpuinfo`. :::{note} These metrics provide an estimation of the resources used by running tasks. They are not an alternative to low-level performance analysis tools, and they may not be completely accurate, especially for very short-lived tasks (running for less than a few seconds).