Skip to content

Commit

Permalink
perf test: JSON format checking
Browse files Browse the repository at this point in the history 
Add field checking tests for perf stat JSON output.

Sanity checks the expected number of fields are present, that the
expected keys are present and they have the correct values.

Committer notes:

Had to fix this:

  -               $(INSTALL) tests/shell/lib/*.sh '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/lib' \
  +               $(INSTALL) tests/shell/lib/*.sh '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/lib'; \

Committer testing:

  [root@quaco ~]# perf test json
   90: perf stat JSON output linter                                    : Ok
  [root@quaco ~]# set -o vi
  [root@quaco ~]# perf test -v json
   90: perf stat JSON output linter                                    :
  --- start ---
  test child forked, pid 560794
  Checking json output: no args [Success]
  Checking json output: system wide [Success]
  Checking json output: system wide Checking json output: system wide no aggregation [Success]
  Checking json output: interval [Success]
  Checking json output: event [Success]
  Checking json output: per core [Success]
  Checking json output: per thread [Success]
  Checking json output: per die [Success]
  Checking json output: per node [Success]
  Checking json output: per socket [Success]
  test child finished with 0
  ---- end ----
  perf stat JSON output linter: Ok
  [root@quaco ~]#

Signed-off-by: Claire Jensen <cjense@google.com>
Acked-by: Namhyung Kim <namhyung@kernel.org>
Tested-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Alexander Shishkin <alexander.shishkin@linux.intel.com>
Cc: Alyssa Ross <hi@alyssa.is>
Cc: Claire Jensen <clairej735@gmail.com>
Cc: Florian Fischer <florian.fischer@muhq.space>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: James Clark <james.clark@arm.com>
Cc: Jiri Olsa <jolsa@kernel.org>
Cc: Kan Liang <kan.liang@linux.intel.com>
Cc: Like Xu <likexu@tencent.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Sandipan Das <sandipan.das@amd.com>
Cc: Stephane Eranian <eranian@google.com>
Cc: Xing Zhengjun <zhengjun.xing@linux.intel.com>
Link: https://lore.kernel.org/r/20220805200105.2020995-3-irogers@google.com
Signed-off-by: Ian Rogers <irogers@google.com>
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
  • Loading branch information
@acmel
Claire Jensen authored and acmel committed Aug 10, 2022
1 parent df936ca commit 0c343af
Show file tree
Hide file tree
Showing 3 changed files with 245 additions and 1 deletion.
3 changes: 2 additions & 1 deletion tools/perf/Makefile.perf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1005,7 +1005,8 @@ install-tests: all install-gtk
$(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell'; \
$(INSTALL) tests/shell/*.sh '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell'; \
$(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/lib'; \
$(INSTALL) tests/shell/lib/*.sh '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/lib'
$(INSTALL) tests/shell/lib/*.sh '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/lib'; \
$(INSTALL) tests/shell/lib/*.py '$(DESTDIR_SQ)$(perfexec_instdir_SQ)/tests/shell/lib'

install-bin: install-tools install-tests install-traceevent-plugins

Expand Down
96 changes: 96 additions & 0 deletions tools/perf/tests/shell/lib/perf_json_output_lint.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
#!/usr/bin/python
# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
# Basic sanity check of perf JSON output as specified in the man page.

import argparse
import sys
import json

ap = argparse.ArgumentParser()
ap.add_argument('--no-args', action='store_true')
ap.add_argument('--interval', action='store_true')
ap.add_argument('--system-wide-no-aggr', action='store_true')
ap.add_argument('--system-wide', action='store_true')
ap.add_argument('--event', action='store_true')
ap.add_argument('--per-core', action='store_true')
ap.add_argument('--per-thread', action='store_true')
ap.add_argument('--per-die', action='store_true')
ap.add_argument('--per-node', action='store_true')
ap.add_argument('--per-socket', action='store_true')
args = ap.parse_args()

Lines = sys.stdin.readlines()

def isfloat(num):
try:
float(num)
return True
except ValueError:
return False


def isint(num):
try:
int(num)
return True
except ValueError:
return False

def is_counter_value(num):
return isfloat(num) or num == '<not counted>' or num == '<not supported>'

def check_json_output(expected_items):
if expected_items != -1:
for line in Lines:
if 'failed' not in line:
count = 0
count = line.count(',')
if count != expected_items and count >= 1 and count <= 3 and 'metric-value' in line:
# Events that generate >1 metric may have isolated metric
# values and possibly other prefixes like interval, core and
# aggregate-number.
continue
if count != expected_items:
raise RuntimeError(f'wrong number of fields. counted {count} expected {expected_items}'
f' in \'{line}\'')
checks = {
'aggregate-number': lambda x: isfloat(x),
'core': lambda x: True,
'counter-value': lambda x: is_counter_value(x),
'cgroup': lambda x: True,
'cpu': lambda x: isint(x),
'die': lambda x: True,
'event': lambda x: True,
'event-runtime': lambda x: isfloat(x),
'interval': lambda x: isfloat(x),
'metric-unit': lambda x: True,
'metric-value': lambda x: isfloat(x),
'node': lambda x: True,
'pcnt-running': lambda x: isfloat(x),
'socket': lambda x: True,
'thread': lambda x: True,
'unit': lambda x: True,
}
input = '[\n' + ','.join(Lines) + '\n]'
for item in json.loads(input):
for key, value in item.items():
if key not in checks:
raise RuntimeError(f'Unexpected key: key={key} value={value}')
if not checks[key](value):
raise RuntimeError(f'Check failed for: key={key} value={value}')


try:
if args.no_args or args.system_wide or args.event:
expected_items = 6
elif args.interval or args.per_thread or args.system_wide_no_aggr:
expected_items = 7
elif args.per_core or args.per_socket or args.per_node or args.per_die:
expected_items = 8
else:
# If no option is specified, don't check the number of items.
expected_items = -1
check_json_output(expected_items)
except:
print('Test failed for input:\n' + '\n'.join(Lines))
raise
147 changes: 147 additions & 0 deletions tools/perf/tests/shell/stat+json_output.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
#!/bin/bash
# perf stat JSON output linter
# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
# Checks various perf stat JSON output commands for the
# correct number of fields.

set -e

pythonchecker=$(dirname $0)/lib/perf_json_output_lint.py
if [ "x$PYTHON" == "x" ]
then
if which python3 > /dev/null
then
PYTHON=python3
elif which python > /dev/null
then
PYTHON=python
else
echo Skipping test, python not detected please set environment variable PYTHON.
exit 2
fi
fi

# Return true if perf_event_paranoid is > $1 and not running as root.
function ParanoidAndNotRoot()
{
[ $(id -u) != 0 ] && [ $(cat /proc/sys/kernel/perf_event_paranoid) -gt $1 ]
}

check_no_args()
{
echo -n "Checking json output: no args "
perf stat -j true 2>&1 | $PYTHON $pythonchecker --no-args
echo "[Success]"
}

check_system_wide()
{
echo -n "Checking json output: system wide "
if ParanoidAndNotRoot 0
then
echo "[Skip] paranoia and not root"
return
fi
perf stat -j -a true 2>&1 | $PYTHON $pythonchecker --system-wide
echo "[Success]"
}

check_system_wide_no_aggr()
{
echo -n "Checking json output: system wide "
if ParanoidAndNotRoot 0
then
echo "[Skip] paranoia and not root"
return
fi
echo -n "Checking json output: system wide no aggregation "
perf stat -j -A -a --no-merge true 2>&1 | $PYTHON $pythonchecker --system-wide-no-aggr
echo "[Success]"
}

check_interval()
{
echo -n "Checking json output: interval "
perf stat -j -I 1000 true 2>&1 | $PYTHON $pythonchecker --interval
echo "[Success]"
}


check_event()
{
echo -n "Checking json output: event "
perf stat -j -e cpu-clock true 2>&1 | $PYTHON $pythonchecker --event
echo "[Success]"
}

check_per_core()
{
echo -n "Checking json output: per core "
if ParanoidAndNotRoot 0
then
echo "[Skip] paranoia and not root"
return
fi
perf stat -j --per-core -a true 2>&1 | $PYTHON $pythonchecker --per-core
echo "[Success]"
}

check_per_thread()
{
echo -n "Checking json output: per thread "
if ParanoidAndNotRoot 0
then
echo "[Skip] paranoia and not root"
return
fi
perf stat -j --per-thread -a true 2>&1 | $PYTHON $pythonchecker --per-thread
echo "[Success]"
}

check_per_die()
{
echo -n "Checking json output: per die "
if ParanoidAndNotRoot 0
then
echo "[Skip] paranoia and not root"
return
fi
perf stat -j --per-die -a true 2>&1 | $PYTHON $pythonchecker --per-die
echo "[Success]"
}

check_per_node()
{
echo -n "Checking json output: per node "
if ParanoidAndNotRoot 0
then
echo "[Skip] paranoia and not root"
return
fi
perf stat -j --per-node -a true 2>&1 | $PYTHON $pythonchecker --per-node
echo "[Success]"
}

check_per_socket()
{
echo -n "Checking json output: per socket "
if ParanoidAndNotRoot 0
then
echo "[Skip] paranoia and not root"
return
fi
perf stat -j --per-socket -a true 2>&1 | $PYTHON $pythonchecker --per-socket
echo "[Success]"
}

check_no_args
check_system_wide
check_system_wide_no_aggr
check_interval
check_event
check_per_core
check_per_thread
check_per_die
check_per_node
check_per_socket
exit 0

0 comments on commit 0c343af

Please sign in to comment.