From 4d95245b66a06225fc1016813d74886a659c579e Mon Sep 17 00:00:00 2001 From: tengli Date: Mon, 16 Jan 2023 10:25:22 +0800 Subject: [PATCH 1/9] import valprod for CI test --- .github/workflows/main.yml | 13 +- Utilities/CMakeLists.txt | 1 + Utilities/valprod/CMakeLists.txt | 3 + Utilities/valprod/bin/flamegraph.pl | 1252 +++++++++++++++++ Utilities/valprod/bin/jMonitor | 76 + Utilities/valprod/bin/stackcollapse-perf.pl | 435 ++++++ Utilities/valprod/valprod/README.md | 95 ++ Utilities/valprod/valprod/UnitTest.py | 110 ++ Utilities/valprod/valprod/__init__.py | 0 Utilities/valprod/valprod/test/stress.sh | 5 + .../valprod/test/sub_process_stress.sh | 6 + Utilities/valprod/valprod/utils/Parser.py | 68 + Utilities/valprod/valprod/utils/PlotTester.py | 170 +++ Utilities/valprod/valprod/utils/TestConfig.py | 43 + Utilities/valprod/valprod/utils/__init__.py | 0 Utilities/valprod/valprod/utils/histUtil.py | 25 + Utilities/valprod/valprod/utils/monitors.py | 346 +++++ Utilities/valprod/valprod/utils/shellUtil.py | 96 ++ .../valprod/workflow/MonitoredProcess.py | 72 + Utilities/valprod/valprod/workflow/Process.py | 201 +++ .../valprod/valprod/workflow/TestWrapper.py | 42 + .../valprod/valprod/workflow/Workflow.py | 32 + .../valprod/valprod/workflow/__init__.py | 0 23 files changed, 3089 insertions(+), 2 deletions(-) create mode 100644 Utilities/valprod/CMakeLists.txt create mode 100755 Utilities/valprod/bin/flamegraph.pl create mode 100755 Utilities/valprod/bin/jMonitor create mode 100755 Utilities/valprod/bin/stackcollapse-perf.pl create mode 100644 Utilities/valprod/valprod/README.md create mode 100644 Utilities/valprod/valprod/UnitTest.py create mode 100644 Utilities/valprod/valprod/__init__.py create mode 100644 Utilities/valprod/valprod/test/stress.sh create mode 100644 Utilities/valprod/valprod/test/sub_process_stress.sh create mode 100644 Utilities/valprod/valprod/utils/Parser.py create mode 100644 Utilities/valprod/valprod/utils/PlotTester.py create mode 100644 Utilities/valprod/valprod/utils/TestConfig.py create mode 100644 Utilities/valprod/valprod/utils/__init__.py create mode 100644 Utilities/valprod/valprod/utils/histUtil.py create mode 100644 Utilities/valprod/valprod/utils/monitors.py create mode 100644 Utilities/valprod/valprod/utils/shellUtil.py create mode 100644 Utilities/valprod/valprod/workflow/MonitoredProcess.py create mode 100644 Utilities/valprod/valprod/workflow/Process.py create mode 100644 Utilities/valprod/valprod/workflow/TestWrapper.py create mode 100644 Utilities/valprod/valprod/workflow/Workflow.py create mode 100644 Utilities/valprod/valprod/workflow/__init__.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 395f4809a..a4ce0b10e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,9 +6,9 @@ name: CI on: # Triggers the workflow on push or pull request events but only for the master branch push: - branches: [ master ] + branches: [ master, CI-test ] pull_request: - branches: [ master ] + branches: [ master, CI-test ] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -47,3 +47,12 @@ jobs: env: LCG_RELEASE: ${{matrix.LCG_RELEASE}} CEPCSW_BLDTOOL: ${{matrix.CEPCSW_BLDTOOL}} + + - name: Run the simulation example + run: | + pwd + source setup.sh + jMonitor --gen-log -n SimTest --enable-io-profile --enable-perf gaudirun.py Examples/options/tut_detsim.py + env: + LCG_RELEASE: ${{matrix.LCG_RELEASE}} + CEPCSW_BLDTOOL: ${{matrix.CEPCSW_BLDTOOL}} diff --git a/Utilities/CMakeLists.txt b/Utilities/CMakeLists.txt index 7558d9c2f..360bad75c 100644 --- a/Utilities/CMakeLists.txt +++ b/Utilities/CMakeLists.txt @@ -2,4 +2,5 @@ add_subdirectory(DataHelper) add_subdirectory(KalTest) add_subdirectory(KalDet) add_subdirectory(KiTrack) +add_subdirectory(valprod) diff --git a/Utilities/valprod/CMakeLists.txt b/Utilities/valprod/CMakeLists.txt new file mode 100644 index 000000000..57d883f2d --- /dev/null +++ b/Utilities/valprod/CMakeLists.txt @@ -0,0 +1,3 @@ +install(DIRECTORY valprod DESTINATION python) +file(GLOB binaries bin/*) +install(FILES ${binaries} TYPE BIN PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) diff --git a/Utilities/valprod/bin/flamegraph.pl b/Utilities/valprod/bin/flamegraph.pl new file mode 100755 index 000000000..d2172b616 --- /dev/null +++ b/Utilities/valprod/bin/flamegraph.pl @@ -0,0 +1,1252 @@ +#!/usr/bin/perl -w +# +# flamegraph.pl flame stack grapher. +# +# This takes stack samples and renders a call graph, allowing hot functions +# and codepaths to be quickly identified. Stack samples can be generated using +# tools such as DTrace, perf, SystemTap, and Instruments. +# +# USAGE: ./flamegraph.pl [options] input.txt > graph.svg +# +# grep funcA input.txt | ./flamegraph.pl [options] > graph.svg +# +# Then open the resulting .svg in a web browser, for interactivity: mouse-over +# frames for info, click to zoom, and ctrl-F to search. +# +# Options are listed in the usage message (--help). +# +# The input is stack frames and sample counts formatted as single lines. Each +# frame in the stack is semicolon separated, with a space and count at the end +# of the line. These can be generated for Linux perf script output using +# stackcollapse-perf.pl, for DTrace using stackcollapse.pl, and for other tools +# using the other stackcollapse programs. Example input: +# +# swapper;start_kernel;rest_init;cpu_idle;default_idle;native_safe_halt 1 +# +# An optional extra column of counts can be provided to generate a differential +# flame graph of the counts, colored red for more, and blue for less. This +# can be useful when using flame graphs for non-regression testing. +# See the header comment in the difffolded.pl program for instructions. +# +# The input functions can optionally have annotations at the end of each +# function name, following a precedent by some tools (Linux perf's _[k]): +# _[k] for kernel +# _[i] for inlined +# _[j] for jit +# _[w] for waker +# Some of the stackcollapse programs support adding these annotations, eg, +# stackcollapse-perf.pl --kernel --jit. They are used merely for colors by +# some palettes, eg, flamegraph.pl --color=java. +# +# The output flame graph shows relative presence of functions in stack samples. +# The ordering on the x-axis has no meaning; since the data is samples, time +# order of events is not known. The order used sorts function names +# alphabetically. +# +# While intended to process stack samples, this can also process stack traces. +# For example, tracing stacks for memory allocation, or resource usage. You +# can use --title to set the title to reflect the content, and --countname +# to change "samples" to "bytes" etc. +# +# There are a few different palettes, selectable using --color. By default, +# the colors are selected at random (except for differentials). Functions +# called "-" will be printed gray, which can be used for stack separators (eg, +# between user and kernel stacks). +# +# HISTORY +# +# This was inspired by Neelakanth Nadgir's excellent function_call_graph.rb +# program, which visualized function entry and return trace events. As Neel +# wrote: "The output displayed is inspired by Roch's CallStackAnalyzer which +# was in turn inspired by the work on vftrace by Jan Boerhout". See: +# https://blogs.oracle.com/realneel/entry/visualizing_callstacks_via_dtrace_and +# +# Copyright 2016 Netflix, Inc. +# Copyright 2011 Joyent, Inc. All rights reserved. +# Copyright 2011 Brendan Gregg. All rights reserved. +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at docs/cddl1.txt or +# http://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at docs/cddl1.txt. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# 11-Oct-2014 Adrien Mahieux Added zoom. +# 21-Nov-2013 Shawn Sterling Added consistent palette file option +# 17-Mar-2013 Tim Bunce Added options and more tunables. +# 15-Dec-2011 Dave Pacheco Support for frames with whitespace. +# 10-Sep-2011 Brendan Gregg Created this. + +use strict; + +use Getopt::Long; + +use open qw(:std :utf8); + +# tunables +my $encoding; +my $fonttype = "Verdana"; +my $imagewidth = 1200; # max width, pixels +my $frameheight = 16; # max height is dynamic +my $fontsize = 12; # base text size +my $fontwidth = 0.59; # avg width relative to fontsize +my $minwidth = 0.1; # min function width, pixels +my $nametype = "Function:"; # what are the names in the data? +my $countname = "samples"; # what are the counts in the data? +my $colors = "hot"; # color theme +my $bgcolors = ""; # background color theme +my $nameattrfile; # file holding function attributes +my $timemax; # (override the) sum of the counts +my $factor = 1; # factor to scale counts by +my $hash = 0; # color by function name +my $palette = 0; # if we use consistent palettes (default off) +my %palette_map; # palette map hash +my $pal_file = "palette.map"; # palette map file name +my $stackreverse = 0; # reverse stack order, switching merge end +my $inverted = 0; # icicle graph +my $flamechart = 0; # produce a flame chart (sort by time, do not merge stacks) +my $negate = 0; # switch differential hues +my $titletext = ""; # centered heading +my $titledefault = "Flame Graph"; # overwritten by --title +my $titleinverted = "Icicle Graph"; # " " +my $searchcolor = "rgb(230,0,230)"; # color for search highlighting +my $notestext = ""; # embedded notes in SVG +my $subtitletext = ""; # second level title (optional) +my $help = 0; + +sub usage { + die < outfile.svg\n + --title TEXT # change title text + --subtitle TEXT # second level title (optional) + --width NUM # width of image (default 1200) + --height NUM # height of each frame (default 16) + --minwidth NUM # omit smaller functions (default 0.1 pixels) + --fonttype FONT # font type (default "Verdana") + --fontsize NUM # font size (default 12) + --countname TEXT # count type label (default "samples") + --nametype TEXT # name type label (default "Function:") + --colors PALETTE # set color palette. choices are: hot (default), mem, + # io, wakeup, chain, java, js, perl, red, green, blue, + # aqua, yellow, purple, orange + --bgcolors COLOR # set background colors. gradient choices are yellow + # (default), blue, green, grey; flat colors use "#rrggbb" + --hash # colors are keyed by function name hash + --cp # use consistent palette (palette.map) + --reverse # generate stack-reversed flame graph + --inverted # icicle graph + --flamechart # produce a flame chart (sort by time, do not merge stacks) + --negate # switch differential hues (blue<->red) + --notes TEXT # add notes comment in SVG (for debugging) + --help # this message + + eg, + $0 --title="Flame Graph: malloc()" trace.txt > graph.svg +USAGE_END +} + +GetOptions( + 'fonttype=s' => \$fonttype, + 'width=i' => \$imagewidth, + 'height=i' => \$frameheight, + 'encoding=s' => \$encoding, + 'fontsize=f' => \$fontsize, + 'fontwidth=f' => \$fontwidth, + 'minwidth=f' => \$minwidth, + 'title=s' => \$titletext, + 'subtitle=s' => \$subtitletext, + 'nametype=s' => \$nametype, + 'countname=s' => \$countname, + 'nameattr=s' => \$nameattrfile, + 'total=s' => \$timemax, + 'factor=f' => \$factor, + 'colors=s' => \$colors, + 'bgcolors=s' => \$bgcolors, + 'hash' => \$hash, + 'cp' => \$palette, + 'reverse' => \$stackreverse, + 'inverted' => \$inverted, + 'flamechart' => \$flamechart, + 'negate' => \$negate, + 'notes=s' => \$notestext, + 'help' => \$help, +) or usage(); +$help && usage(); + +# internals +my $ypad1 = $fontsize * 3; # pad top, include title +my $ypad2 = $fontsize * 2 + 10; # pad bottom, include labels +my $ypad3 = $fontsize * 2; # pad top, include subtitle (optional) +my $xpad = 10; # pad lefm and right +my $framepad = 1; # vertical padding for frames +my $depthmax = 0; +my %Events; +my %nameattr; + +if ($flamechart && $titletext eq "") { + $titletext = "Flame Chart"; +} + +if ($titletext eq "") { + unless ($inverted) { + $titletext = $titledefault; + } else { + $titletext = $titleinverted; + } +} + +if ($nameattrfile) { + # The name-attribute file format is a function name followed by a tab then + # a sequence of tab separated name=value pairs. + open my $attrfh, $nameattrfile or die "Can't read $nameattrfile: $!\n"; + while (<$attrfh>) { + chomp; + my ($funcname, $attrstr) = split /\t/, $_, 2; + die "Invalid format in $nameattrfile" unless defined $attrstr; + $nameattr{$funcname} = { map { split /=/, $_, 2 } split /\t/, $attrstr }; + } +} + +if ($notestext =~ /[<>]/) { + die "Notes string can't contain < or >" +} + +# background colors: +# - yellow gradient: default (hot, java, js, perl) +# - green gradient: mem +# - blue gradient: io, wakeup, chain +# - gray gradient: flat colors (red, green, blue, ...) +if ($bgcolors eq "") { + # choose a default + if ($colors eq "mem") { + $bgcolors = "green"; + } elsif ($colors =~ /^(io|wakeup|chain)$/) { + $bgcolors = "blue"; + } elsif ($colors =~ /^(red|green|blue|aqua|yellow|purple|orange)$/) { + $bgcolors = "grey"; + } else { + $bgcolors = "yellow"; + } +} +my ($bgcolor1, $bgcolor2); +if ($bgcolors eq "yellow") { + $bgcolor1 = "#eeeeee"; # background color gradient start + $bgcolor2 = "#eeeeb0"; # background color gradient stop +} elsif ($bgcolors eq "blue") { + $bgcolor1 = "#eeeeee"; $bgcolor2 = "#e0e0ff"; +} elsif ($bgcolors eq "green") { + $bgcolor1 = "#eef2ee"; $bgcolor2 = "#e0ffe0"; +} elsif ($bgcolors eq "grey") { + $bgcolor1 = "#f8f8f8"; $bgcolor2 = "#e8e8e8"; +} elsif ($bgcolors =~ /^#......$/) { + $bgcolor1 = $bgcolor2 = $bgcolors; +} else { + die "Unrecognized bgcolor option \"$bgcolors\"" +} + +# SVG functions +{ package SVG; + sub new { + my $class = shift; + my $self = {}; + bless ($self, $class); + return $self; + } + + sub header { + my ($self, $w, $h) = @_; + my $enc_attr = ''; + if (defined $encoding) { + $enc_attr = qq{ encoding="$encoding"}; + } + $self->{svg} .= < + + + + +SVG + } + + sub include { + my ($self, $content) = @_; + $self->{svg} .= $content; + } + + sub colorAllocate { + my ($self, $r, $g, $b) = @_; + return "rgb($r,$g,$b)"; + } + + sub group_start { + my ($self, $attr) = @_; + + my @g_attr = map { + exists $attr->{$_} ? sprintf(qq/$_="%s"/, $attr->{$_}) : () + } qw(id class); + push @g_attr, $attr->{g_extra} if $attr->{g_extra}; + if ($attr->{href}) { + my @a_attr; + push @a_attr, sprintf qq/xlink:href="%s"/, $attr->{href} if $attr->{href}; + # default target=_top else links will open within SVG + push @a_attr, sprintf qq/target="%s"/, $attr->{target} || "_top"; + push @a_attr, $attr->{a_extra} if $attr->{a_extra}; + $self->{svg} .= sprintf qq/\n/, join(' ', (@a_attr, @g_attr)); + } else { + $self->{svg} .= sprintf qq/\n/, join(' ', @g_attr); + } + + $self->{svg} .= sprintf qq/%s<\/title>/, $attr->{title} + if $attr->{title}; # should be first element within g container + } + + sub group_end { + my ($self, $attr) = @_; + $self->{svg} .= $attr->{href} ? qq/<\/a>\n/ : qq/<\/g>\n/; + } + + sub filledRectangle { + my ($self, $x1, $y1, $x2, $y2, $fill, $extra) = @_; + $x1 = sprintf "%0.1f", $x1; + $x2 = sprintf "%0.1f", $x2; + my $w = sprintf "%0.1f", $x2 - $x1; + my $h = sprintf "%0.1f", $y2 - $y1; + $extra = defined $extra ? $extra : ""; + $self->{svg} .= qq/\n/; + } + + sub stringTTF { + my ($self, $id, $x, $y, $str, $extra) = @_; + $x = sprintf "%0.2f", $x; + $id = defined $id ? qq/id="$id"/ : ""; + $extra ||= ""; + $self->{svg} .= qq/$str<\/text>\n/; + } + + sub svg { + my $self = shift; + return "$self->{svg}\n"; + } + 1; +} + +sub namehash { + # Generate a vector hash for the name string, weighting early over + # later characters. We want to pick the same colors for function + # names across different flame graphs. + my $name = shift; + my $vector = 0; + my $weight = 1; + my $max = 1; + my $mod = 10; + # if module name present, trunc to 1st char + $name =~ s/.(.*?)`//; + foreach my $c (split //, $name) { + my $i = (ord $c) % $mod; + $vector += ($i / ($mod++ - 1)) * $weight; + $max += 1 * $weight; + $weight *= 0.70; + last if $mod > 12; + } + return (1 - $vector / $max) +} + +sub color { + my ($type, $hash, $name) = @_; + my ($v1, $v2, $v3); + + if ($hash) { + $v1 = namehash($name); + $v2 = $v3 = namehash(scalar reverse $name); + } else { + $v1 = rand(1); + $v2 = rand(1); + $v3 = rand(1); + } + + # theme palettes + if (defined $type and $type eq "hot") { + my $r = 205 + int(50 * $v3); + my $g = 0 + int(230 * $v1); + my $b = 0 + int(55 * $v2); + return "rgb($r,$g,$b)"; + } + if (defined $type and $type eq "mem") { + my $r = 0; + my $g = 190 + int(50 * $v2); + my $b = 0 + int(210 * $v1); + return "rgb($r,$g,$b)"; + } + if (defined $type and $type eq "io") { + my $r = 80 + int(60 * $v1); + my $g = $r; + my $b = 190 + int(55 * $v2); + return "rgb($r,$g,$b)"; + } + + # multi palettes + if (defined $type and $type eq "java") { + # Handle both annotations (_[j], _[i], ...; which are + # accurate), as well as input that lacks any annotations, as + # best as possible. Without annotations, we get a little hacky + # and match on java|org|com, etc. + if ($name =~ m:_\[j\]$:) { # jit annotation + $type = "green"; + } elsif ($name =~ m:_\[i\]$:) { # inline annotation + $type = "aqua"; + } elsif ($name =~ m:^L?(java|javax|jdk|net|org|com|io|sun)/:) { # Java + $type = "green"; + } elsif ($name =~ /:::/) { # Java, typical perf-map-agent method separator + $type = "green"; + } elsif ($name =~ /::/) { # C++ + $type = "yellow"; + } elsif ($name =~ m:_\[k\]$:) { # kernel annotation + $type = "orange"; + } elsif ($name =~ /::/) { # C++ + $type = "yellow"; + } else { # system + $type = "red"; + } + # fall-through to color palettes + } + if (defined $type and $type eq "perl") { + if ($name =~ /::/) { # C++ + $type = "yellow"; + } elsif ($name =~ m:Perl: or $name =~ m:\.pl:) { # Perl + $type = "green"; + } elsif ($name =~ m:_\[k\]$:) { # kernel + $type = "orange"; + } else { # system + $type = "red"; + } + # fall-through to color palettes + } + if (defined $type and $type eq "js") { + # Handle both annotations (_[j], _[i], ...; which are + # accurate), as well as input that lacks any annotations, as + # best as possible. Without annotations, we get a little hacky, + # and match on a "/" with a ".js", etc. + if ($name =~ m:_\[j\]$:) { # jit annotation + if ($name =~ m:/:) { + $type = "green"; # source + } else { + $type = "aqua"; # builtin + } + } elsif ($name =~ /::/) { # C++ + $type = "yellow"; + } elsif ($name =~ m:/.*\.js:) { # JavaScript (match "/" in path) + $type = "green"; + } elsif ($name =~ m/:/) { # JavaScript (match ":" in builtin) + $type = "aqua"; + } elsif ($name =~ m/^ $/) { # Missing symbol + $type = "green"; + } elsif ($name =~ m:_\[k\]:) { # kernel + $type = "orange"; + } else { # system + $type = "red"; + } + # fall-through to color palettes + } + if (defined $type and $type eq "wakeup") { + $type = "aqua"; + # fall-through to color palettes + } + if (defined $type and $type eq "chain") { + if ($name =~ m:_\[w\]:) { # waker + $type = "aqua" + } else { # off-CPU + $type = "blue"; + } + # fall-through to color palettes + } + + # color palettes + if (defined $type and $type eq "red") { + my $r = 200 + int(55 * $v1); + my $x = 50 + int(80 * $v1); + return "rgb($r,$x,$x)"; + } + if (defined $type and $type eq "green") { + my $g = 200 + int(55 * $v1); + my $x = 50 + int(60 * $v1); + return "rgb($x,$g,$x)"; + } + if (defined $type and $type eq "blue") { + my $b = 205 + int(50 * $v1); + my $x = 80 + int(60 * $v1); + return "rgb($x,$x,$b)"; + } + if (defined $type and $type eq "yellow") { + my $x = 175 + int(55 * $v1); + my $b = 50 + int(20 * $v1); + return "rgb($x,$x,$b)"; + } + if (defined $type and $type eq "purple") { + my $x = 190 + int(65 * $v1); + my $g = 80 + int(60 * $v1); + return "rgb($x,$g,$x)"; + } + if (defined $type and $type eq "aqua") { + my $r = 50 + int(60 * $v1); + my $g = 165 + int(55 * $v1); + my $b = 165 + int(55 * $v1); + return "rgb($r,$g,$b)"; + } + if (defined $type and $type eq "orange") { + my $r = 190 + int(65 * $v1); + my $g = 90 + int(65 * $v1); + return "rgb($r,$g,0)"; + } + + return "rgb(0,0,0)"; +} + +sub color_scale { + my ($value, $max) = @_; + my ($r, $g, $b) = (255, 255, 255); + $value = -$value if $negate; + if ($value > 0) { + $g = $b = int(210 * ($max - $value) / $max); + } elsif ($value < 0) { + $r = $g = int(210 * ($max + $value) / $max); + } + return "rgb($r,$g,$b)"; +} + +sub color_map { + my ($colors, $func) = @_; + if (exists $palette_map{$func}) { + return $palette_map{$func}; + } else { + $palette_map{$func} = color($colors, $hash, $func); + return $palette_map{$func}; + } +} + +sub write_palette { + open(FILE, ">$pal_file"); + foreach my $key (sort keys %palette_map) { + print FILE $key."->".$palette_map{$key}."\n"; + } + close(FILE); +} + +sub read_palette { + if (-e $pal_file) { + open(FILE, $pal_file) or die "can't open file $pal_file: $!"; + while ( my $line = ) { + chomp($line); + (my $key, my $value) = split("->",$line); + $palette_map{$key}=$value; + } + close(FILE) + } +} + +my %Node; # Hash of merged frame data +my %Tmp; + +# flow() merges two stacks, storing the merged frames and value data in %Node. +sub flow { + my ($last, $this, $v, $d) = @_; + + my $len_a = @$last - 1; + my $len_b = @$this - 1; + + my $i = 0; + my $len_same; + for (; $i <= $len_a; $i++) { + last if $i > $len_b; + last if $last->[$i] ne $this->[$i]; + } + $len_same = $i; + + for ($i = $len_a; $i >= $len_same; $i--) { + my $k = "$last->[$i];$i"; + # a unique ID is constructed from "func;depth;etime"; + # func-depth isn't unique, it may be repeated later. + $Node{"$k;$v"}->{stime} = delete $Tmp{$k}->{stime}; + if (defined $Tmp{$k}->{delta}) { + $Node{"$k;$v"}->{delta} = delete $Tmp{$k}->{delta}; + } + delete $Tmp{$k}; + } + + for ($i = $len_same; $i <= $len_b; $i++) { + my $k = "$this->[$i];$i"; + $Tmp{$k}->{stime} = $v; + if (defined $d) { + $Tmp{$k}->{delta} += $i == $len_b ? $d : 0; + } + } + + return $this; +} + +# parse input +my @Data; +my @SortedData; +my $last = []; +my $time = 0; +my $delta = undef; +my $ignored = 0; +my $line; +my $maxdelta = 1; + +# reverse if needed +foreach (<>) { + chomp; + $line = $_; + if ($stackreverse) { + # there may be an extra samples column for differentials + # XXX todo: redo these REs as one. It's repeated below. + my($stack, $samples) = (/^(.*)\s+?(\d+(?:\.\d*)?)$/); + my $samples2 = undef; + if ($stack =~ /^(.*)\s+?(\d+(?:\.\d*)?)$/) { + $samples2 = $samples; + ($stack, $samples) = $stack =~ (/^(.*)\s+?(\d+(?:\.\d*)?)$/); + unshift @Data, join(";", reverse split(";", $stack)) . " $samples $samples2"; + } else { + unshift @Data, join(";", reverse split(";", $stack)) . " $samples"; + } + } else { + unshift @Data, $line; + } +} + +if ($flamechart) { + # In flame chart mode, just reverse the data so time moves from left to right. + @SortedData = reverse @Data; +} else { + @SortedData = sort @Data; +} + +# process and merge frames +foreach (@SortedData) { + chomp; + # process: folded_stack count + # eg: func_a;func_b;func_c 31 + my ($stack, $samples) = (/^(.*)\s+?(\d+(?:\.\d*)?)$/); + unless (defined $samples and defined $stack) { + ++$ignored; + next; + } + + # there may be an extra samples column for differentials: + my $samples2 = undef; + if ($stack =~ /^(.*)\s+?(\d+(?:\.\d*)?)$/) { + $samples2 = $samples; + ($stack, $samples) = $stack =~ (/^(.*)\s+?(\d+(?:\.\d*)?)$/); + } + $delta = undef; + if (defined $samples2) { + $delta = $samples2 - $samples; + $maxdelta = abs($delta) if abs($delta) > $maxdelta; + } + + # for chain graphs, annotate waker frames with "_[w]", for later + # coloring. This is a hack, but has a precedent ("_[k]" from perf). + if ($colors eq "chain") { + my @parts = split ";--;", $stack; + my @newparts = (); + $stack = shift @parts; + $stack .= ";--;"; + foreach my $part (@parts) { + $part =~ s/;/_[w];/g; + $part .= "_[w]"; + push @newparts, $part; + } + $stack .= join ";--;", @parts; + } + + # merge frames and populate %Node: + $last = flow($last, [ '', split ";", $stack ], $time, $delta); + + if (defined $samples2) { + $time += $samples2; + } else { + $time += $samples; + } +} +flow($last, [], $time, $delta); + +warn "Ignored $ignored lines with invalid format\n" if $ignored; +unless ($time) { + warn "ERROR: No stack counts found\n"; + my $im = SVG->new(); + # emit an error message SVG, for tools automating flamegraph use + my $imageheight = $fontsize * 5; + $im->header($imagewidth, $imageheight); + $im->stringTTF(undef, int($imagewidth / 2), $fontsize * 2, + "ERROR: No valid input provided to flamegraph.pl."); + print $im->svg; + exit 2; +} +if ($timemax and $timemax < $time) { + warn "Specified --total $timemax is less than actual total $time, so ignored\n" + if $timemax/$time > 0.02; # only warn is significant (e.g., not rounding etc) + undef $timemax; +} +$timemax ||= $time; + +my $widthpertime = ($imagewidth - 2 * $xpad) / $timemax; +my $minwidth_time = $minwidth / $widthpertime; + +# prune blocks that are too narrow and determine max depth +while (my ($id, $node) = each %Node) { + my ($func, $depth, $etime) = split ";", $id; + my $stime = $node->{stime}; + die "missing start for $id" if not defined $stime; + + if (($etime-$stime) < $minwidth_time) { + delete $Node{$id}; + next; + } + $depthmax = $depth if $depth > $depthmax; +} + +# draw canvas, and embed interactive JavaScript program +my $imageheight = (($depthmax + 1) * $frameheight) + $ypad1 + $ypad2; +$imageheight += $ypad3 if $subtitletext ne ""; +my $titlesize = $fontsize + 5; +my $im = SVG->new(); +my ($black, $vdgrey, $dgrey) = ( + $im->colorAllocate(0, 0, 0), + $im->colorAllocate(160, 160, 160), + $im->colorAllocate(200, 200, 200), + ); +$im->header($imagewidth, $imageheight); +my $inc = < + + + + + + + +INC +$im->include($inc); +$im->filledRectangle(0, 0, $imagewidth, $imageheight, 'url(#background)'); +$im->stringTTF("title", int($imagewidth / 2), $fontsize * 2, $titletext); +$im->stringTTF("subtitle", int($imagewidth / 2), $fontsize * 4, $subtitletext) if $subtitletext ne ""; +$im->stringTTF("details", $xpad, $imageheight - ($ypad2 / 2), " "); +$im->stringTTF("unzoom", $xpad, $fontsize * 2, "Reset Zoom", 'class="hide"'); +$im->stringTTF("search", $imagewidth - $xpad - 100, $fontsize * 2, "Search"); +$im->stringTTF("ignorecase", $imagewidth - $xpad - 16, $fontsize * 2, "ic"); +$im->stringTTF("matched", $imagewidth - $xpad - 100, $imageheight - ($ypad2 / 2), " "); + +if ($palette) { + read_palette(); +} + +# draw frames +$im->group_start({id => "frames"}); +while (my ($id, $node) = each %Node) { + my ($func, $depth, $etime) = split ";", $id; + my $stime = $node->{stime}; + my $delta = $node->{delta}; + + $etime = $timemax if $func eq "" and $depth == 0; + + my $x1 = $xpad + $stime * $widthpertime; + my $x2 = $xpad + $etime * $widthpertime; + my ($y1, $y2); + unless ($inverted) { + $y1 = $imageheight - $ypad2 - ($depth + 1) * $frameheight + $framepad; + $y2 = $imageheight - $ypad2 - $depth * $frameheight; + } else { + $y1 = $ypad1 + $depth * $frameheight; + $y2 = $ypad1 + ($depth + 1) * $frameheight - $framepad; + } + + my $samples = sprintf "%.0f", ($etime - $stime) * $factor; + (my $samples_txt = $samples) # add commas per perlfaq5 + =~ s/(^[-+]?\d+?(?=(?>(?:\d{3})+)(?!\d))|\G\d{3}(?=\d))/$1,/g; + + my $info; + if ($func eq "" and $depth == 0) { + $info = "all ($samples_txt $countname, 100%)"; + } else { + my $pct = sprintf "%.2f", ((100 * $samples) / ($timemax * $factor)); + my $escaped_func = $func; + # clean up SVG breaking characters: + $escaped_func =~ s/&/&/g; + $escaped_func =~ s//>/g; + $escaped_func =~ s/"/"/g; + $escaped_func =~ s/_\[[kwij]\]$//; # strip any annotation + unless (defined $delta) { + $info = "$escaped_func ($samples_txt $countname, $pct%)"; + } else { + my $d = $negate ? -$delta : $delta; + my $deltapct = sprintf "%.2f", ((100 * $d) / ($timemax * $factor)); + $deltapct = $d > 0 ? "+$deltapct" : $deltapct; + $info = "$escaped_func ($samples_txt $countname, $pct%; $deltapct%)"; + } + } + + my $nameattr = { %{ $nameattr{$func}||{} } }; # shallow clone + $nameattr->{title} ||= $info; + $im->group_start($nameattr); + + my $color; + if ($func eq "--") { + $color = $vdgrey; + } elsif ($func eq "-") { + $color = $dgrey; + } elsif (defined $delta) { + $color = color_scale($delta, $maxdelta); + } elsif ($palette) { + $color = color_map($colors, $func); + } else { + $color = color($colors, $hash, $func); + } + $im->filledRectangle($x1, $y1, $x2, $y2, $color, 'rx="2" ry="2"'); + + my $chars = int( ($x2 - $x1) / ($fontsize * $fontwidth)); + my $text = ""; + if ($chars >= 3) { # room for one char plus two dots + $func =~ s/_\[[kwij]\]$//; # strip any annotation + $text = substr $func, 0, $chars; + substr($text, -2, 2) = ".." if $chars < length $func; + $text =~ s/&/&/g; + $text =~ s//>/g; + } + $im->stringTTF(undef, $x1 + 3, 3 + ($y1 + $y2) / 2, $text); + + $im->group_end($nameattr); +} +$im->group_end(); + +print $im->svg; + +if ($palette) { + write_palette(); +} + +# vim: ts=8 sts=8 sw=8 noexpandtab diff --git a/Utilities/valprod/bin/jMonitor b/Utilities/valprod/bin/jMonitor new file mode 100755 index 000000000..e9eab9246 --- /dev/null +++ b/Utilities/valprod/bin/jMonitor @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +from valprod.utils.TestConfig import * +from valprod.workflow.TestWrapper import TestWrapper + +def getParser(): + import argparse + + argParser = argparse.ArgumentParser() + # job limit + argParser.add_argument('-T', '--time-limit', dest='maxTime', type=int, help='Time limit of process (s), if exceeded, process will be killed') + argParser.add_argument('-M', '--memory-limit', dest='maxVIR', type=float, help='Memory limit of process (Mb), if exceeded, process will be killed') + # log parser related + argParser.add_argument('--enable-parser', dest='parser', action='store_true', help='If enabled, log of the process will be parsed') + argParser.add_argument('-p', '--pattern', dest='fatalPattern', help='Python re patterns for the parser') + # monitor related + argParser.add_argument('--enable-monitor', dest='monitor', action='store_true', help='If enabled, process will be monitored (memory, cpu)') + argParser.add_argument('--enable-perf', dest='perf', action='store_true', help='If enabled, process will profiled via perf') + argParser.add_argument('--enable-io-profile', dest='io', action='store_true', help='If enabled, process disk IO will profiled') + argParser.add_argument('-i', '--interval', dest='interval', type=float, default='5', help='Time interval for monitor') + argParser.add_argument('-l', '--plotting-backend', dest='plotting_backend', default='root', choices=['root', 'matplotlib'], help='Backend for plotting monitoring figures') + argParser.add_argument('-b', '--monitor-backend', dest='monitor_backend', default='ps', choices=['ps', 'prmon'], help='Backend performance profiling') + # plot reference + argParser.add_argument('--enable-plotref', dest='plotref', action='store_true', help='If enabled, results of the process will be compared') + argParser.add_argument('-f', '--plotref-files', dest='plotreffile', help='reference file for plot testing') + argParser.add_argument('-o', '--plotref-output', dest='plotrefoutput', default='plotcmp.root', help='output root file for plot comparison') + argParser.add_argument('-m', '--histtest-method', dest='histtestmeth', choices=['Kolmogorov', 'Chi2'], default='Kolmogorov', help='Method of histogram testing') + argParser.add_argument('-c', '--histtest-cut', dest='histtestcut', type=float, default='0.9', help='P-Value cut for histogram testing') + # cmd + argParser.add_argument('command', nargs='+', help='job to be monitored') + argParser.add_argument('--gen-log', dest='log', action='store_true', help='whether to generate log file') + argParser.add_argument('-n', '--name', dest='name', default="None", help='name of the job') + + return argParser + +def configureArgs(args): + # Decide the output names for log, plot etc. + name = args.name + if args.name == 'None': + from random import randrange + print(args.command) + name = "_".join(args.command) + str(randrange(1000)) + + globalConfig.setAttr('maxTime', args.maxTime) + globalConfig.setAttr('maxMEM', args.maxVIR) + globalConfig.setAttr('parser', args.parser) + if args.parser and args.fatalPattern: + globalConfig.setAttr('fatalPattern', args.fatalPattern) + if args.perf: args.monitor = True + if args.monitor: + globalConfig.setAttr('monitorBackend', args.monitor_backend) + globalConfig.setAttr('plottingBackend', args.plotting_backend) + if args.interval: + globalConfig.setAttr('timeInterval', args.interval) + if args.perf: + globalConfig.setAttr('perf', True) + if args.io: + globalConfig.setAttr('profileIO', True) + if args.plotref and args.plotreffile: + globalConfig.setAttr('plotRef', args.plotreffile) + globalConfig.setAttr('plotOutput', args.plotrefoutput) + globalConfig.setAttr('histTestMeth', args.histtestmeth) + globalConfig.setAttr('histTestCut', args.histtestcut) + if args.log: + logname = name + ".log" + globalConfig.setAttr('genLog', True) + globalConfig.setAttr('logName', logname) + +if __name__ == '__main__': + parser = getParser() + args = parser.parse_args() + + configureArgs(args) + runner = TestWrapper(args.name, args.command) + ok, summary = runner.run() + print(summary) + print('--------------------------------') diff --git a/Utilities/valprod/bin/stackcollapse-perf.pl b/Utilities/valprod/bin/stackcollapse-perf.pl new file mode 100755 index 000000000..652b113e3 --- /dev/null +++ b/Utilities/valprod/bin/stackcollapse-perf.pl @@ -0,0 +1,435 @@ +#!/usr/bin/perl -w +# +# stackcollapse-perf.pl collapse perf samples into single lines. +# +# Parses a list of multiline stacks generated by "perf script", and +# outputs a semicolon separated stack followed by a space and a count. +# If memory addresses (+0xd) are present, they are stripped, and resulting +# identical stacks are colased with their counts summed. +# +# USAGE: ./stackcollapse-perf.pl [options] infile > outfile +# +# Run "./stackcollapse-perf.pl -h" to list options. +# +# Example input: +# +# swapper 0 [000] 158665.570607: cpu-clock: +# ffffffff8103ce3b native_safe_halt ([kernel.kallsyms]) +# ffffffff8101c6a3 default_idle ([kernel.kallsyms]) +# ffffffff81013236 cpu_idle ([kernel.kallsyms]) +# ffffffff815bf03e rest_init ([kernel.kallsyms]) +# ffffffff81aebbfe start_kernel ([kernel.kallsyms].init.text) +# [...] +# +# Example output: +# +# swapper;start_kernel;rest_init;cpu_idle;default_idle;native_safe_halt 1 +# +# Input may be created and processed using: +# +# perf record -a -g -F 997 sleep 60 +# perf script | ./stackcollapse-perf.pl > out.stacks-folded +# +# The output of "perf script" should include stack traces. If these are missing +# for you, try manually selecting the perf script output; eg: +# +# perf script -f comm,pid,tid,cpu,time,event,ip,sym,dso,trace | ... +# +# This is also required for the --pid or --tid options, so that the output has +# both the PID and TID. +# +# Copyright 2012 Joyent, Inc. All rights reserved. +# Copyright 2012 Brendan Gregg. All rights reserved. +# +# CDDL HEADER START +# +# The contents of this file are subject to the terms of the +# Common Development and Distribution License (the "License"). +# You may not use this file except in compliance with the License. +# +# You can obtain a copy of the license at docs/cddl1.txt or +# http://opensource.org/licenses/CDDL-1.0. +# See the License for the specific language governing permissions +# and limitations under the License. +# +# When distributing Covered Code, include this CDDL HEADER in each +# file and include the License file at docs/cddl1.txt. +# If applicable, add the following below this CDDL HEADER, with the +# fields enclosed by brackets "[]" replaced with your own identifying +# information: Portions Copyright [yyyy] [name of copyright owner] +# +# CDDL HEADER END +# +# 02-Mar-2012 Brendan Gregg Created this. +# 02-Jul-2014 " " Added process name to stacks. + +use strict; +use Getopt::Long; + +my %collapsed; + +sub remember_stack { + my ($stack, $count) = @_; + $collapsed{$stack} += $count; +} +my $annotate_kernel = 0; # put an annotation on kernel function +my $annotate_jit = 0; # put an annotation on jit symbols +my $annotate_all = 0; # enale all annotations +my $include_pname = 1; # include process names in stacks +my $include_pid = 0; # include process ID with process name +my $include_tid = 0; # include process & thread ID with process name +my $include_addrs = 0; # include raw address where a symbol can't be found +my $tidy_java = 1; # condense Java signatures +my $tidy_generic = 1; # clean up function names a little +my $target_pname; # target process name from perf invocation +my $event_filter = ""; # event type filter, defaults to first encountered event +my $event_defaulted = 0; # whether we defaulted to an event (none provided) +my $event_warning = 0; # if we printed a warning for the event + +my $show_inline = 0; +my $show_context = 0; + +my $srcline_in_input = 0; # if there are extra lines with source location (perf script -F+srcline) +GetOptions('inline' => \$show_inline, + 'context' => \$show_context, + 'srcline' => \$srcline_in_input, + 'pid' => \$include_pid, + 'kernel' => \$annotate_kernel, + 'jit' => \$annotate_jit, + 'all' => \$annotate_all, + 'tid' => \$include_tid, + 'addrs' => \$include_addrs, + 'event-filter=s' => \$event_filter) +or die < outfile\n + --pid # include PID with process names [1] + --tid # include TID and PID with process names [1] + --inline # un-inline using addr2line + --all # all annotations (--kernel --jit) + --kernel # annotate kernel functions with a _[k] + --jit # annotate jit functions with a _[j] + --context # adds source context to --inline + --srcline # parses output of 'perf script -F+srcline' and adds source context + --addrs # include raw addresses where symbols can't be found + --event-filter=EVENT # event name filter\n +[1] perf script must emit both PID and TIDs for these to work; eg, Linux < 4.1: + perf script -f comm,pid,tid,cpu,time,event,ip,sym,dso,trace + for Linux >= 4.1: + perf script -F comm,pid,tid,cpu,time,event,ip,sym,dso,trace + If you save this output add --header on Linux >= 3.14 to include perf info. +USAGE_END + +if ($annotate_all) { + $annotate_kernel = $annotate_jit = 1; +} + +my %inlineCache; + +my %nmCache; + +sub inlineCacheAdd { + my ($pc, $mod, $result) = @_; + if (defined($inlineCache{$pc})) { + $inlineCache{$pc}{$mod} = $result; + } else { + $inlineCache{$pc} = {$mod => $result}; + } +} + +# for the --inline option +sub inline { + my ($pc, $rawfunc, $mod) = @_; + + return $inlineCache{$pc}{$mod} if defined($inlineCache{$pc}{$mod}); + + # capture addr2line output + my $a2l_output = `addr2line -a $pc -e $mod -i -f -s -C`; + + # remove first line + $a2l_output =~ s/^(.*\n){1}//; + + if ($a2l_output =~ /\?\?\n\?\?:0/) { + # if addr2line fails and rawfunc is func+offset, then fall back to it + if ($rawfunc =~ /^(.+)\+0x([0-9a-f]+)$/) { + my $func = $1; + my $addr = hex $2; + + $nmCache{$mod}=`nm $mod` unless defined $nmCache{$mod}; + + if ($nmCache{$mod} =~ /^([0-9a-f]+) . \Q$func\E$/m) { + my $base = hex $1; + my $newPc = sprintf "0x%x", $base+$addr; + my $result = inline($newPc, '', $mod); + inlineCacheAdd($pc, $mod, $result); + return $result; + } + } + } + + my @fullfunc; + my $one_item = ""; + for (split /^/, $a2l_output) { + chomp $_; + + # remove discriminator info if exists + $_ =~ s/ \(discriminator \S+\)//; + + if ($one_item eq "") { + $one_item = $_; + } else { + if ($show_context == 1) { + unshift @fullfunc, $one_item . ":$_"; + } else { + unshift @fullfunc, $one_item; + } + $one_item = ""; + } + } + + my $result = join ";" , @fullfunc; + + inlineCacheAdd($pc, $mod, $result); + + return $result; +} + +my @stack; +my $pname; +my $m_pid; +my $m_tid; +my $m_period; + +# +# Main loop +# +while (defined($_ = <>)) { + + # find the name of the process launched by perf, by stepping backwards + # over the args to find the first non-option (no dash): + if (/^# cmdline/) { + my @args = split ' ', $_; + foreach my $arg (reverse @args) { + if ($arg !~ /^-/) { + $target_pname = $arg; + $target_pname =~ s:.*/::; # strip pathname + last; + } + } + } + + # skip remaining comments + next if m/^#/; + chomp; + + # end of stack. save cached data. + if (m/^$/) { + # ignore filtered samples + next if not $pname; + + if ($include_pname) { + if (defined $pname) { + unshift @stack, $pname; + } else { + unshift @stack, ""; + } + } + remember_stack(join(";", @stack), $m_period) if @stack; + undef @stack; + undef $pname; + next; + } + + # + # event record start + # + if (/^(\S.+?)\s+(\d+)\/*(\d+)*\s+/) { + # default "perf script" output has TID but not PID + # eg, "java 25607 4794564.109216: 1 cycles:" + # eg, "java 12688 [002] 6544038.708352: 235 cpu-clock:" + # eg, "V8 WorkerThread 25607 4794564.109216: 104345 cycles:" + # eg, "java 24636/25607 [000] 4794564.109216: 1 cycles:" + # eg, "java 12688/12764 6544038.708352: 10309278 cpu-clock:" + # eg, "V8 WorkerThread 24636/25607 [000] 94564.109216: 100 cycles:" + # other combinations possible + my ($comm, $pid, $tid, $period) = ($1, $2, $3, ""); + if (not $tid) { + $tid = $pid; + $pid = "?"; + } + + if (/:\s*(\d+)*\s+(\S+):\s*$/) { + $period = $1; + my $event = $2; + + if ($event_filter eq "") { + # By default only show events of the first encountered + # event type. Merging together different types, such as + # instructions and cycles, produces misleading results. + $event_filter = $event; + $event_defaulted = 1; + } elsif ($event ne $event_filter) { + if ($event_defaulted and $event_warning == 0) { + # only print this warning if necessary: + # when we defaulted and there was + # multiple event types. + print STDERR "Filtering for events of type: $event\n"; + $event_warning = 1; + } + next; + } + } + + if (not $period) { + $period = 1 + } + ($m_pid, $m_tid, $m_period) = ($pid, $tid, $period); + + if ($include_tid) { + $pname = "$comm-$m_pid/$m_tid"; + } elsif ($include_pid) { + $pname = "$comm-$m_pid"; + } else { + $pname = "$comm"; + } + $pname =~ tr/ /_/; + + # + # stack line + # + } elsif (/^\s*(\w+)\s*(.+) \((\S*)\)/) { + # ignore filtered samples + next if not $pname; + + my ($pc, $rawfunc, $mod) = ($1, $2, $3); + + if ($show_inline == 1 && $mod !~ m/(perf-\d+.map|kernel\.|\[[^\]]+\])/) { + my $inlineRes = inline($pc, $rawfunc, $mod); + # - empty result this happens e.g., when $mod does not exist or is a path to a compressed kernel module + # if this happens, the user will see error message from addr2line written to stderr + # - if addr2line results in "??" , then it's much more sane to fall back than produce a '??' in graph + if($inlineRes ne "" and $inlineRes ne "??" and $inlineRes ne "??:??:0" ) { + unshift @stack, $inlineRes; + next; + } + } + + # Linux 4.8 included symbol offsets in perf script output by default, eg: + # 7fffb84c9afc cpu_startup_entry+0x800047c022ec ([kernel.kallsyms]) + # strip these off: + $rawfunc =~ s/\+0x[\da-f]+$//; + + next if $rawfunc =~ /^\(/; # skip process names + + my $is_unknown=0; + my @inline; + for (split /\->/, $rawfunc) { + my $func = $_; + + if ($func eq "[unknown]") { + if ($mod ne "[unknown]") { # use module name instead, if known + $func = $mod; + $func =~ s/.*\///; + } else { + $func = "unknown"; + $is_unknown=1; + } + + if ($include_addrs) { + $func = "\[$func \<$pc\>\]"; + } else { + $func = "\[$func\]"; + } + } + + if ($tidy_generic) { + $func =~ s/;/:/g; + if ($func !~ m/\.\(.*\)\./) { + # This doesn't look like a Go method name (such as + # "net/http.(*Client).Do"), so everything after the first open + # paren (that is not part of an "(anonymous namespace)") is + # just noise. + $func =~ s/\((?!anonymous namespace\)).*//; + } + # now tidy this horrible thing: + # 13a80b608e0a RegExp:[&<>\"\'] (/tmp/perf-7539.map) + $func =~ tr/"\'//d; + # fall through to $tidy_java + } + + if ($tidy_java and $pname eq "java") { + # along with $tidy_generic, converts the following: + # Lorg/mozilla/javascript/ContextFactory;.call(Lorg/mozilla/javascript/ContextAction;)Ljava/lang/Object; + # Lorg/mozilla/javascript/ContextFactory;.call(Lorg/mozilla/javascript/C + # Lorg/mozilla/javascript/MemberBox;.(Ljava/lang/reflect/Method;)V + # into: + # org/mozilla/javascript/ContextFactory:.call + # org/mozilla/javascript/ContextFactory:.call + # org/mozilla/javascript/MemberBox:.init + $func =~ s/^L// if $func =~ m:/:; + } + + # + # Annotations + # + # detect inlined from the @inline array + # detect kernel from the module name; eg, frames to parse include: + # ffffffff8103ce3b native_safe_halt ([kernel.kallsyms]) + # 8c3453 tcp_sendmsg (/lib/modules/4.3.0-rc1-virtual/build/vmlinux) + # 7d8 ipv4_conntrack_local+0x7f8f80b8 ([nf_conntrack_ipv4]) + # detect jit from the module name; eg: + # 7f722d142778 Ljava/io/PrintStream;::print (/tmp/perf-19982.map) + if (scalar(@inline) > 0) { + $func .= "_[i]"; # inlined + } elsif ($annotate_kernel == 1 && $mod =~ m/(^\[|vmlinux$)/ && $mod !~ /unknown/) { + $func .= "_[k]"; # kernel + } elsif ($annotate_jit == 1 && $mod =~ m:/tmp/perf-\d+\.map:) { + $func .= "_[j]"; # jitted + } + + # + # Source lines + # + # + # Sample outputs: + # | a.out 35081 252436.005167: 667783 cycles: + # | 408ebb some_method_name+0x8b (/full/path/to/a.out) + # | uniform_int_dist.h:300 + # | 4069f5 main+0x935 (/full/path/to/a.out) + # | file.cpp:137 + # | 7f6d2148eb25 __libc_start_main+0xd5 (/lib64/libc-2.33.so) + # | libc-2.33.so[27b25] + # + # | a.out 35081 252435.738165: 306459 cycles: + # | 7f6d213c2750 [unknown] (/usr/lib64/libkmod.so.2.3.6) + # | libkmod.so.2.3.6[6750] + # + # | a.out 35081 252435.738373: 315813 cycles: + # | 7f6d215ca51b __strlen_avx2+0x4b (/lib64/libc-2.33.so) + # | libc-2.33.so[16351b] + # | 7ffc71ee9580 [unknown] ([unknown]) + # | + # + # | a.out 35081 252435.718940: 247984 cycles: + # | ffffffff814f9302 up_write+0x32 ([kernel.kallsyms]) + # | [kernel.kallsyms][ffffffff814f9302] + if($srcline_in_input and not $is_unknown){ + $_ = <>; + chomp; + s/\[.*?\]//g; + s/^\s*//g; + s/\s*$//g; + $func.=':'.$_ unless $_ eq ""; + } + + push @inline, $func; + } + + unshift @stack, @inline; + } else { + warn "Unrecognized line: $_"; + } +} + +foreach my $k (sort { $a cmp $b } keys %collapsed) { + print "$k $collapsed{$k}\n"; +} diff --git a/Utilities/valprod/valprod/README.md b/Utilities/valprod/valprod/README.md new file mode 100644 index 000000000..80ff11f90 --- /dev/null +++ b/Utilities/valprod/valprod/README.md @@ -0,0 +1,95 @@ +## Use valprod to build unit test cases + +### Via the jMonitor command + +The jMonitor command monitors the executable by adding a thin shell on top of it. +Example: + +`jMonitor runSimulation.exe` + +The detailed usage: +``` +usage: jMonitor [-h] [-T MAXTIME] [-WT WALLTIME] [-M MAXVIR] [--enable-parser] + [-p FATALPATTERN] [--enable-monitor] [-i INTERVAL] + [-b {root,matplotlib}] [--enable-plotref] [-f PLOTREFFILE] + [-o PLOTREFOUTPUT] [-m {Kolmogorov,Chi2}] [-c HISTTESTCUT] + [--gen-log] [-n NAME] + command + +positional arguments: + command job to be monitored + +optional arguments: + -h, --help show this help message and exit + -T MAXTIME, --max-time MAXTIME + Time limit of process (s), if exceeded, process will + be killed + -WT WALLTIME, --walltime WALLTIME + Time limit of process (hh:mm:ss) for PBS, if exceeded, + process will be killed + -M MAXVIR, --max-memory MAXVIR + Memory limit of process (Mb), if exceeded, process + will be killed + --enable-parser If enabled, log of the process will be parsed + -p FATALPATTERN, --pattern FATALPATTERN + Python re patterns for the parser + --enable-monitor If enabled, process will be monitored (memory, cpu) + -i INTERVAL, --interval INTERVAL + Time interval for monitor + -b {root,matplotlib}, --monitor-backend {root,matplotlib} + Backend for drawing monitoring figures + --enable-plotref If enabled, results of the process will be compared + -f PLOTREFFILE, --plotref-files PLOTREFFILE + reference file for plot testing + -o PLOTREFOUTPUT, --plotref-output PLOTREFOUTPUT + output root file for plot comparison + -m {Kolmogorov,Chi2}, --histtest-method {Kolmogorov,Chi2} + Method of histogram testing + -c HISTTESTCUT, --histtest-cut HISTTESTCUT + P-Value cut for histogram testing + --gen-log whether to generate log file + -n NAME, --name NAME name of the job +``` + +### Via the API + +The UnitTest module. + +Example: +``` + mytest = UnitTest() + mytest.addCase('test1', 'python run.py') + mytest.run() +``` + + 1. overall options: + + setTimeLimit: + Time limit (s) of the test case. If running time is exceeded, test case will be killed. (default: None) + setVIRtLimit: + Virtual memory limit (mb) of the test case. If exceeded, test case will be killed. (default: None) + enableCPUMonitor/enableVIRMonitor/enableRESMonitor: + If CPU monitor/ virtual memory monitor/ resident memory monitor is enabled, the real time CPU rate/ virtual memory/ resident memory of the test case will be recorded. + setFatalPattern: + If any of the re pattern is detected, in the stdout and stderr of the test case, it will be killed. + default: {} + + 2. local options: + options of one single test case, example: `mytest.addCase('test2', 'python run.py', timeLimit=900, genLog=True, RESMonitor=True)` + + genLog: + If genLog is set to True, the stdout and stderr of the test case will be recorded. (default: False) + maxTime: + Time limit (s) of the test case. If running time is exceeded, test case will be killed. (default: None) + maxVIR: + Virtual memory limit (kb) of the test case. If exceeded, test case will be killed. (default: None) + CPUMonitor/RESMonitor/VIRMonitor: + If CPU monitor/ virtual memory monitor/ resident memory monitor is enabled, the real time CPU rate/ virtual memory/ resident memory of the test case will be recorded. (default: False) + timeInterval: + Monitoring interval (s) of the test case. (default: 0.5, range: 0.3-10) + plotRef: + The plot reference file. If set, the plot output of the test case will be compared. (default: None) + histTestMeth: + Method of the histogram testing. (default: Kolmogorov, Kolmogorov or Chi2) + histTestCut: + The P-Value cut of the histogram test. If result is smaller than this cut, the histogram will be treated as inconsistent. (default: 0.9) diff --git a/Utilities/valprod/valprod/UnitTest.py b/Utilities/valprod/valprod/UnitTest.py new file mode 100644 index 000000000..592c7dc84 --- /dev/null +++ b/Utilities/valprod/valprod/UnitTest.py @@ -0,0 +1,110 @@ +''' + example: + mytest = UnitTest() + mytest.addCase('test1', 'python run.py') + mytest.run() + + 1. overall options: + + setTimeLimit: + Time limit (s) of the test case. If running time is exceeded, test case will be killed. (default: None) + setVIRtLimit: + Virtual memory limit (mb) of the test case. If exceeded, test case will be killed. (default: None) + enableCPUMonitor/enableVIRMonitor/enableRESMonitor: + If CPU monitor/ virtual memory monitor/ resident memory monitor is enabled, the real time CPU rate/ virtual memory/ resident memory of the test case will be recorded. + setFatalPattern: + If any of the re pattern is detected, in the stdout and stderr of the test case, it will be killed. + default: {} + + 2. local options: + options of one single test case, example: mytest.addCase('test2', 'python run.py', timeLimit=900, genLog=True, RESMonitor=True) + + genLog: + If genLog is set to True, the stdout and stderr of the test case will be recorded. (default: False) + maxTime: + Time limit (s) of the test case. If running time is exceeded, test case will be killed. (default: None) + maxVIR: + Virtual memory limit (kb) of the test case. If exceeded, test case will be killed. (default: None) + CPUMonitor/RESMonitor/VIRMonitor: + If CPU monitor/ virtual memory monitor/ resident memory monitor is enabled, the real time CPU rate/ virtual memory/ resident memory of the test case will be recorded. (default: False) + timeInterval: + Monitoring interval (s) of the test case. (default: 0.5, range: 0.3-10) + plotRef: + The plot reference file. If set, the plot output of the test case will be compared. (default: None) + histTestMeth: + Method of the histogram testing. (default: Kolmogorov, Kolmogorov or Chi2) + histTestCut: + The P-Value cut of the histogram test. If result is smaller than this cut, the histogram will be treated as inconsistent. (default: 0.9) + + 3. workflow: +''' + +import unittest +from valprod.workflow.TestWrapper import TestWrapper +from valprod.utils.TestConfig import TestConfig + +class UnitTest: + + def __init__(self): + self.defaultCFG = TestConfig() + self.case = ValprodTestCase + self.suite = unittest.TestSuite() + self.runner = unittest.TextTestRunner() + self.workflowID = 0 + + def addCase(self, name, cmd, **kwa): + setattr(self.case, "test_" + name, genTestCase(name, cmd, self.defaultCFG, **kwa)) + self.suite.addTest(self.case("test_" + name)) + + def addWorkflow(self, workflow): + self.workflowID += 1 + wfName = "test_workflow%d" % self.workflowID + setattr(self.case, wfName, genWorkflow(workflow, self.defaultCFG)) + self.suite.addTest(self.case(wfName)) + + def run(self): + self.runner.run(self.suite) + + def setTimeLimit(self, value): + self.defaultCFG.setAttr("maxTime", value) + + def setVIRLimit(self, value): + self.defaultCFG.setAttr("maxVIR", value) + + def enableProfile(self): + self.defaultCFG.setAttr("profile", True) + + def enablePerf(self): + self.defaultCFG.setAttr("perf", True) + + def enableIOProfile(self): + self.defaultCFG.setAttr("profileIO", True) + + def setFatalPattern(self, value): + self.defaultCFG.setAttr("fatalPattern", value) + + def enableGenLog(self): + self.defaultCFG.setAttr("genLog", True) + + +class ValprodTestCase(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + +def genTestCase(name, cmd, cfg, **kwa): + def case(self): + run = TestWrapper(name, cmd, cfg, **kwa) + ok, what = run.run() + self.assertTrue(ok, what) + return case + +def genWorkflow(workflow, cfg): + def wkCase(self): + workflow.setOverallCFG(cfg) + ok, what = workflow.run() + self.assertTrue(ok, what) + return wkCase diff --git a/Utilities/valprod/valprod/__init__.py b/Utilities/valprod/valprod/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/Utilities/valprod/valprod/test/stress.sh b/Utilities/valprod/valprod/test/stress.sh new file mode 100644 index 000000000..10ea4f210 --- /dev/null +++ b/Utilities/valprod/valprod/test/stress.sh @@ -0,0 +1,5 @@ +#!/bin/bash +for((i=1;i<=1000000000;i++)); +do + a=100*100 +done diff --git a/Utilities/valprod/valprod/test/sub_process_stress.sh b/Utilities/valprod/valprod/test/sub_process_stress.sh new file mode 100644 index 000000000..5d301c42d --- /dev/null +++ b/Utilities/valprod/valprod/test/sub_process_stress.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +./stress.sh & + ./stress.sh & +./stress.sh & +./stress.sh diff --git a/Utilities/valprod/valprod/utils/Parser.py b/Utilities/valprod/valprod/utils/Parser.py new file mode 100644 index 000000000..f3d39ef3d --- /dev/null +++ b/Utilities/valprod/valprod/utils/Parser.py @@ -0,0 +1,68 @@ +import re, os, copy + +class Parser: + + fatalPattern = [ + # C + '.*Segmentation fault', + # SNiPER + '.*ERROR:', + '.*FATAL:', + # Python + '.*IOError', + '.*ImportError', + '.*TypeError', + '.*MemoryError', + '.*SyntaxError', + '.*NameError', + '.*RuntimeError', + # Other + '.*\*\*\* Break \*\*\* segmentation violation', + '.*Warning in : macro .* not found', + ] + + successPattern = [] + + def __init__(self, cfg): + self.fatal = [] + self.success = {} + fs = cfg.getAttr('fatalPattern') or Parser.fatalPattern + for fatal in fs: + self.fatal.append(re.compile(fatal)) + sp = cfg.getAttr('successPattern') or Parser.successPattern + for s in sp: + self.success[s] = re.compile(s) + + def parse(self, word): + if not word: + return True, None + wordl = word.split('\n') + for pattern in self.fatal: + for w in wordl: + match = pattern.match(word) + if match: + return False, "Fatal line: %s" %w + return True, None + + def parseFile(self, file): + + if not os.path.exists(file): + return False, "Can not find log file: %s" % file + + s = copy.copy(self.success) + f = open(file, 'r') + for line in f: + w = line.strip('\n') + for pattern in self.fatal: + match = pattern.match(w) + if match: + f.close() + return False, "Fatal line: %s detected in log file: %s" % (w,file) + for ps, pattern in s.items(): + match = pattern.match(w) + if match: + del s[ps] + f.close() + if len(s): + return False, "Key words not found in %s:\n %s" % (file,'\n'.join(s.keys())) + return True, None diff --git a/Utilities/valprod/valprod/utils/PlotTester.py b/Utilities/valprod/valprod/utils/PlotTester.py new file mode 100644 index 000000000..ec77f570a --- /dev/null +++ b/Utilities/valprod/valprod/utils/PlotTester.py @@ -0,0 +1,170 @@ +import os +import ROOT + +class PlotTester: + + TH1Types = ['TH1D', 'TH1F'] + TH2Types = ['TH2D', 'TH2F'] + GraphTypes = ['TGraph'] + testMeth = ['Chi2', 'Kolmogorov'] + + def __init__(self, cfg, plotref, datadir = None): + self.cfg = cfg + self.cut = cfg.getAttr('histTestCut') + self.testName = self.cfg.getAttr('histTestMeth') + assert self.testName in PlotTester.testMeth, 'ERROR: unknown test method ' + self.testName + self.refFileNames = (type(plotref) == list and (plotref,) or ([plotref],))[0] + self.datadir = datadir + #print('@PlotTester: refFileNames=%s'%self.refFileNames) + self.fileNames = [] + for fn in self.refFileNames: + # We assume that the file has the same name with the reference file + if self.datadir: + self.fileNames.append(datadir+'/'+os.path.basename(fn)) + else: + self.fileNames.append(os.path.basename(fn)) + self.jointFile = None + self.curPath = '' + self.tempPlots = [] + self.okay = True + self.testResult = {} + + def _extractPlot(self, file, refFile): + # Get all plots in the file recursively, and find the target ones + self._recursiveGet(file) + for v in self.tempPlots: + v.append(refFile.Get(v[0])) + + def _recursiveGet(self, dir): + for k in dir.GetListOfKeys(): + clName = k.GetClassName() + objName = k.GetName() + if clName == 'TDirectoryFile': + tempPath = self.curPath + self.curPath += objName + '/' + self._recursiveGet(k.ReadObj()) + self.curPath = tempPath + elif clName in PlotTester.TH1Types + PlotTester.TH2Types + PlotTester.GraphTypes: + self.tempPlots.append([self.curPath + objName, k.ReadObj()]) + + def _drawTH1CMP(self, path, plot, refPlot, testResult): + canvas = self._mkCanvas("compare_"+plot.GetName()) + plot.SetLineColor(ROOT.kRed) + plot.SetStats(False) + refPlot.SetStats(False) + plot.Draw() + refPlot.Draw("same") + leg = ROOT.TLegend(0.13, 0.77, 0.3, 0.87) + leg.AddEntry(plot, "new_" + plot.GetName(), "lp") + leg.AddEntry(refPlot, "old_" + plot.GetName(), "lp") + leg.Draw("same") + if testResult: + text = ROOT.TPaveText(0.67, 0.74, 0.83, 0.87, "NB NDC") + text.SetFillColor(0) + text.SetFillStyle(0) + text.SetBorderSize(0) + text.AddText(self.testName + ': ' + str(testResult)) + text.SetTextSize(0.04) + if testResult < self.cut: + text.AddText('INCONSISTENT') + text.SetTextColor(ROOT.kRed) + else: + text.AddText('CONSISTENT') + text.SetTextColor(ROOT.kGreen) + text.Draw() + canvas.Write() + + def _drawTH2CMP(self, path, plot, refPlot, testResult): + canvas = self._mkCanvas("compare_"+plot.GetName()) + canvas.Divide(2,1) + canvas.GetPad(1).cd() + plot.SetTitle(plot.GetTitle() + ' (new)') + plot.Draw('Colz') + if testResult: + text = ROOT.TPaveText(0.13, 0.77, 0.3, 0.87, "NB NDC") + text.SetFillColor(0) + text.SetFillStyle(0) + text.SetBorderSize(0) + text.AddText(self.testName + ': ' + str(testResult)) + text.SetTextSize(0.04) + if testResult < self.cut: + text.AddText('INCONSISTENT') + text.SetTextColor(ROOT.kRed) + else: + text.AddText('CONSISTENT') + text.SetTextColor(ROOT.kGreen) + text.Draw() + canvas.GetPad(2).cd() + refPlot.SetTitle(refPlot.GetTitle() + ' (old)') + refPlot.Draw('Colz') + canvas.Write() + + def _drawGraphCMP(self, path, plot, refPlot): + canvas = self._mkCanvas("compare_"+plot.GetTitle()) + tmg = ROOT.TMultiGraph() + plot.SetLineColor(ROOT.kRed) + refPlot.SetLineColor(ROOT.kBlue) + tmg.Add(plot) + tmg.Add(refPlot) + tmg.Draw('APL') + leg = ROOT.TLegend(0.13, 0.77, 0.3, 0.87) + leg.AddEntry(plot, "new_" + plot.GetTitle(), "lp") + leg.AddEntry(refPlot, "old_" + plot.GetTitle(), "lp") + leg.Draw("same") + canvas.Write() + ''' + canvas = self._mkCanvas("compare_"+plot.GetTitle()) + canvas.Divide(2,1) + canvas.GetPad(1).cd() + plot.SetTitle(plot.GetTitle() + ' (new)') + plot.Draw() + canvas.GetPad(2).cd() + refPlot.SetTitle(refPlot.GetTitle() + ' (old)') + refPlot.Draw() + canvas.Write() + ''' + + def _mkCanvas(self, name): + if not self.jointFile: + jointName = self.cfg.getAttr('cmpOutput') + self.jointFile = ROOT.TFile(jointName, 'recreate') + self.jointFile.cd() + #makeROOTDir(self.jointFile, v[0]) + canvas = ROOT.TCanvas(name) + return canvas + + def _test(self, h1, h2): + if self.testName == 'Kolmogorov': + return h1.KolmogorovTest(h2) + elif self.testName == 'Chi2': + return h1.Chi2Test(h2) + + def run(self): + print("@PlotTester: self.fileNames=%s"%self.fileNames) + print("@PlotTester: self.refFileNames=%s"%self.refFileNames) + for i in range(len(self.refFileNames)): + assert os.path.exists(self.fileNames[i]), '%s does not exsist!' % self.fileNames[i] + assert os.path.exists(self.refFileNames[i]), '%s does not exsist!' % self.refFileNames[i] + file = ROOT.TFile.Open(self.fileNames[i]) + refFile = ROOT.TFile.Open(self.refFileNames[i]) + self._extractPlot(file, refFile) + # plots: [path, plot, plotref] + for plot in self.tempPlots: + clName = plot[1].ClassName() + if clName in PlotTester.TH1Types + PlotTester.TH2Types: + value = self._test(plot[1], plot[2]) + self.okay = self.okay and value > self.cut + self.testResult[plot[0]] = value + if clName in PlotTester.TH1Types: + self._drawTH1CMP(plot[0], plot[1], plot[2], value) + elif clName in PlotTester.TH2Types: + self._drawTH2CMP(plot[0], plot[1], plot[2], value) + elif clName in PlotTester.GraphTypes: + self._drawGraphCMP(plot[0], plot[1], plot[2]) + file.Close() + refFile.Close() + self.tempPlots = [] + if self.jointFile: + self.jointFile.Close() + + return self.okay, self.testResult diff --git a/Utilities/valprod/valprod/utils/TestConfig.py b/Utilities/valprod/valprod/utils/TestConfig.py new file mode 100644 index 000000000..66a456b41 --- /dev/null +++ b/Utilities/valprod/valprod/utils/TestConfig.py @@ -0,0 +1,43 @@ +class TestConfig: + + defaults = { + 'verbose' : False, + 'genLog' : True, + 'logFileName' : None, + 'maxTime' : None, + 'walltime' : None, + 'maxVIR' : None, + 'parser' : False, + 'profile' : False, + 'timeInterval' : 5, + 'monitorBackend': None, # ps or prmon + 'plottingBackend': 'root', # root or matplotlib + 'perf': False, + 'profileIO': False, + 'fatalPattern' : None, + 'plotRef' : None, + 'cmpOutput' : 'plotcmp.root', + 'histTestMeth' : 'Kolmogorov', + 'histTestCut' : 0.9, + 'logName' : None + } + + def __init__(self): + self.config = dict([(k,v) for (k,v) in TestConfig.defaults.items()]) + + def update(self, **kwa): + self.config.update(kwa) + + def setAttr(self, name, value): + assert type(name) == str, "ERROR: attribute must be of String type!" + self.config[name] = value + + def getAttr(self, name): + if name in self.config: + return self.config[name] + return None + + def getConfig(self): + return self.config + +globalConfig = TestConfig() diff --git a/Utilities/valprod/valprod/utils/__init__.py b/Utilities/valprod/valprod/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/Utilities/valprod/valprod/utils/histUtil.py b/Utilities/valprod/valprod/utils/histUtil.py new file mode 100644 index 000000000..c3b5894e4 --- /dev/null +++ b/Utilities/valprod/valprod/utils/histUtil.py @@ -0,0 +1,25 @@ +import ROOT + +def writeMHist(hist, rootfile): + hist.SetMarkerColor(ROOT.kRed) + hist.SetLineColor(ROOT.kRed) + hist.SetLineWidth(2) + hist.SetStats(0) + + ROOT.gROOT.SetBatch() + canv = ROOT.TCanvas("c_"+hist.GetName()) + hist.Draw() + + rootfile.cd() + canv.Write() + +def writeMHistPng(hist, file): + hist.SetMarkerColor(ROOT.kRed) + hist.SetLineColor(ROOT.kRed) + hist.SetLineWidth(2) + hist.SetStats(0) + + ROOT.gROOT.SetBatch() + canv = ROOT.TCanvas("c_"+hist.GetName()) + hist.Draw() + canv.SaveAs(file) diff --git a/Utilities/valprod/valprod/utils/monitors.py b/Utilities/valprod/valprod/utils/monitors.py new file mode 100644 index 000000000..9243ca7e9 --- /dev/null +++ b/Utilities/valprod/valprod/utils/monitors.py @@ -0,0 +1,346 @@ +from valprod.utils.shellUtil import * +import subprocess + +class PSMonitor(): + def __init__(self, name, interval, pid, plotBackend): + self._sub_monitors = [] + self._sub_monitors.append(VirtMonitor(name,interval,pid,plotBackend)) + self._sub_monitors.append(ResMonitor(name,interval,pid,plotBackend)) + self._sub_monitors.append(CpuMonitor(name,interval,pid,plotBackend)) + + def initialize(self): + for monitor in self._sub_monitors: + monitor.initialize() + + def execute(self): + for monitor in self._sub_monitors: + monitor.execute() + + def finalize(self): + for monitor in self._sub_monitors: + monitor.finalize() + +class PrmonMonitor(): + def __init__(self, name, interval, pid): + self._name = name + self._pid = pid + self._valid = True + self._interval = interval + + def initialize(self): + # Use prmon to monitor the process + prmon_output = self._name + '.prmon' + prmon_cmd = ['prmon', '--interval', str(self._interval), '--pid', str(self._pid), '--filename', prmon_output] + try: + self.prmon_subprocess = subprocess.Popen(args = prmon_cmd) + except: + self._valid = False + print("prmon execution failed. Switching off monitor") + + def execute(self): + pass + + def finalize(self): + if self._valid: + prmon_rc = self.prmon_subprocess.wait() + if 0==prmon_rc: + # Draw figures using the prmon_plot.py tool + os.system("prmon_plot.py --input %s --xvar wtime --yvar vmem,pss,rss,swap --yunit GB" % prmon_output) + os.system("prmon_plot.py --input %s --xvar wtime --yvar vmem,pss,rss,swap --diff --yunit MB" % prmon_output) + os.system("prmon_plot.py --input %s --xvar wtime --yvar utime,stime --yunit SEC --diff --stacked" % prmon_output) + else: + print("prmon process failed.") + +class PerfMonitor(): + def __init__(self, name, pid): + self._name = name + self._perfout = name+'.perfdata' + self._pid = pid + self._valid = True + + def initialize(self): + perf_cmd = ['perf', 'record', '-F 99', '-g', '-p', str(self._pid), '-o', self._perfout] + try: + self.perf_subprocess = subprocess.Popen(args = perf_cmd) + except: + self._valid = False + print("perf execution failed. Switching off perf") + + def execute(self): + pass + + def finalize(self): + if self._valid: + perf_rc = self.perf_subprocess.wait() + if 0==perf_rc: + ret = os.system("perf script -i %s | stackcollapse-perf.pl > %s.perf-folded" % (self._perfout, self._name)) + if ret: + print("perf profiling failed.") + return + ret = os.system("flamegraph.pl %s.perf-folded > %s_perf.svg" % (self._name, self._name)) + if ret: + print("perf profiling failed.") + return + else: + print("perf process failed.") + os.remove(self._perfout) + os.remove("%s.perf-folded"%self._name) + +class PidMonitor(): + def __init__(self, name, interval, pid, backend): + self.interval = interval + self.backend = backend + self.pid = pid + self.test_name = name + self.min = 0. + self.max = 1e15 + self.results = [] + self.hist = None + + def initialize(self): + pass + + def execute(self): + sum_value = 0.0 + + for pid in self.pid: + value = eval(self.fun + "(%s)" % pid) + if value < self.min: + self.min = value + if value > self.max: + self.max = value + sum_value = sum_value + value + self.results.append(sum_value) + + def finalize(self): + if self.backend == 'matplotlib': + import matplotlib.pyplot as plt + import numpy as np + ntime = len(self.results) + time_seq = np.linspace(0, ntime * self.interval, ntime) + plt.clf() + plt.plot(time_seq, self.results) + plt.xlabel(self.xtitle) + plt.ylabel(self.ytitle) + plt.title(self.title) + plt.savefig(self.test_name + '_' + self.monitor_name + '.png') + + if self.backend == 'root': + from ROOT import TCanvas, TH1F, TImage + ntime = len(self.results) + canvas = TCanvas() + hist = TH1F(self.test_name, self.title, ntime + 1, 0., ntime * self.interval) + hist.GetXaxis().SetTitle(self.xtitle) + hist.GetYaxis().SetTitle(self.ytitle) + for i in range(ntime): + hist.SetBinContent(i, self.results[i]) + hist.SetStats(0) + hist.Draw() + img = TImage.Create() + img.FromPad(canvas) + img.WriteImage(self.test_name + '_' + self.monitor_name + '.png') + +class VirtMonitor(PidMonitor): + + def __init__(self, name, interval, pid, plotBackend): + self.title = "%s Virtual Memory Usage" %name + self.xtitle = "Time [s]" + self.ytitle = "Virtual Memory Usage [MB]" + self.fun = "GetVirUse" + self.monitor_name = "VirMem" + PidMonitor.__init__(self, name, interval, pid, plotBackend) + + +class ResMonitor(PidMonitor): + + def __init__(self, name, interval, pid, plotBackend): + self.title = "%s Resident Memory Usage" %name + self.xtitle = "Time [s]" + self.ytitle = "Resident Memory Usage [MB]" + self.fun = "GetMemUse" + self.monitor_name = "ResMem" + PidMonitor.__init__(self, name, interval, pid, plotBackend) + + +class CpuMonitor(PidMonitor): + + def __init__(self, name, interval, pid, plotBackend): + self.title = "%s CPU Utilization" % name + self.xtitle = "Time [s]" + self.ytitle = "CPU Utilization [/%]" + self.fun = "GetCpuRate" + self.monitor_name = "CPURate" + PidMonitor.__init__(self, name, interval, pid, plotBackend) + +class DiskIOMonitor: + + def __init__(self, name, interval, pid, plotBackend): + self._valid = True + self._name = name + self._interval = interval + self._pid = pid + self._backend = plotBackend + self._read_count = [] + self._write_count = [] + self._read_bytes = [] + self._write_bytes = [] + + def initialize(self): + try: + import psutil + except: + print("Cannot import psutil. Switching off Disk IO Monitor") + self._valid = False + try: + self._process = psutil.Process(self._pid[0]) + except: + # process not created or already exit + self._valid = False + + def execute(self): + if not self._valid: + return + try: + io_counter = self._process.io_counters() + except: + # process already existed, or permission error + return + self._read_count.append(float(io_counter.read_count)) + self._write_count.append(float(io_counter.write_count)) + self._read_bytes.append(float(io_counter.read_bytes)/1000) # kb + self._write_bytes.append(float(io_counter.write_bytes)/1000) # kb + + def finalize(self): + if not self._valid: + return + + # Get per-interval data from accumulated data + for this_result in [self._read_count, self._write_count, self._read_bytes, self._write_bytes]: + length = len(this_result) + if length <= 1: return # too short + index = length - 1 + while index > 0: + this_result[index] = this_result[index] - this_result[index-1] + index = index - 1 + + if self._backend == 'matplotlib': + try: + import matplotlib.pyplot as plt + import numpy as np + except: + print("failed to import matplotlib.") + return + ntime = len( self._read_count) + time_seq = np.linspace(0, ntime * self._interval, ntime) + + # counters + plt.clf() + plt.fill_between(time_seq, self._read_count, label="Read", color="red", alpha=0.5) + plt.fill_between(time_seq, self._read_count, label="Write", color="blue", alpha=0.5) + plt.legend() + plt.xlabel("Time [s]") + plt.ylabel("Count") + plt.title("%s IO Operations" %self._name) + plt.savefig(self._name + '_IOCount.png') + + plt.clf() + plt.fill_between(time_seq, self._read_bytes, label="Read", color="red", alpha=0.5) + plt.fill_between(time_seq, self._write_bytes, label="Write", color="blue", alpha=0.5) + plt.legend() + plt.xlabel("Time [s]") + plt.ylabel("KB") + plt.title("%s IO Throughput" %self._name) + plt.savefig(self._name + '_IOThroughput.png') + + elif self._backend == 'root': + try: + from ROOT import TCanvas, TGraph, TImage, TLegend + from array import array + except: + print("failed to import ROOT") + return + ntime = len(self._read_count) + time_seq = list(range(0, ntime * self._interval, self._interval)) + + # Decide which + ts = array('d') + rc, wc, rb, wb= array('d'), array('d'), array('d'), array('d') + for i in time_seq: + ts.append(i) + for i in self._read_count: + rc.append(i) + for i in self._write_count: + wc.append(i) + for i in self._read_bytes: + rb.append(i) + for i in self._write_bytes: + wb.append(i) + + canvas = TCanvas() + graph1 = TGraph(ntime, ts, rc) + graph2 = TGraph(ntime, ts, wc) + graph1.SetLineColor(2) + graph1.SetLineWidth(3) + graph2.SetLineColor(4) + graph2.SetLineWidth(3) + leg = TLegend(.73,.88,.97,.78) + leg.SetBorderSize(0) + leg.SetFillColor(0) + leg.SetFillStyle(0) + leg.SetTextFont(42) + leg.SetTextSize(0.035) + leg.AddEntry(graph1,"Read","L") + leg.AddEntry(graph2,"Write","L") + if max(rc) > max(wc): + graph1.SetTitle("%s IO Operations" %self._name) + graph1.GetXaxis().SetTitle("Time [s]") + graph1.GetYaxis().SetTitle("Count") + graph1.Draw("ACP") + graph2.Draw("CP") + else: + graph2.SetTitle("%s IO Operations" %self._name) + graph2.GetXaxis().SetTitle("Time [s]") + graph2.GetYaxis().SetTitle("Count") + graph2.Draw("ACP") + graph1.Draw("CP") + leg.Draw() + img = TImage.Create() + img.FromPad(canvas) + img.WriteImage(self._name + '_IOCount.png') + + canvas = TCanvas() + graph1 = TGraph(ntime, ts, rb) + graph2 = TGraph(ntime, ts, wb) + graph1.SetLineColor(2) + graph1.SetLineWidth(3) + graph2.SetLineColor(4) + graph2.SetLineWidth(3) + graph1.SetTitle("%s IO Operations" %self._name) + graph1.GetXaxis().SetTitle("Time [s]") + graph1.GetYaxis().SetTitle("KB") + leg = TLegend(.73,.88,.97,.78) + leg.SetBorderSize(0) + leg.SetFillColor(0) + leg.SetFillStyle(0) + leg.SetTextFont(42) + leg.SetTextSize(0.035) + leg.AddEntry(graph1,"Read","L") + leg.AddEntry(graph2,"Write","L") + if max(rb) > max(wb): + graph1.SetTitle("%s IO Operations" %self._name) + graph1.GetXaxis().SetTitle("Time [s]") + graph1.GetYaxis().SetTitle("KB") + graph1.Draw("ACP") + graph2.Draw("CP") + else: + graph2.SetTitle("%s IO Operations" %self._name) + graph2.GetXaxis().SetTitle("Time [s]") + graph2.GetYaxis().SetTitle("KB") + graph2.Draw("ACP") + graph1.Draw("CP") + leg.Draw() + img = TImage.Create() + img.FromPad(canvas) + img.WriteImage(self._name + '_IOOperation.png') + else: + print("Invalid plotting backend: %s" % self._backend) diff --git a/Utilities/valprod/valprod/utils/shellUtil.py b/Utilities/valprod/valprod/utils/shellUtil.py new file mode 100644 index 000000000..5cc53f232 --- /dev/null +++ b/Utilities/valprod/valprod/utils/shellUtil.py @@ -0,0 +1,96 @@ +# -*- coding:utf-8 -*- +# author liteng + +import time +import os,sys +import subprocess + +def GetTotalSwap(): + value = os.popen("free | grep Swap | awk '{print $2}'").read() + return int(value) / 1024 ## convert from kB to MB + +def GetTotalMem(): + value = os.popen("free | grep Mem | awk '{print $2}'").read() + return int(value) / 1024 + +def GetTotalCpuRate(): + value = os.popen("vmstat|grep -v procs|grep -v swpd|awk '{print $13}'").read() + return int(value) + +def GetTotalMemUse(): + value = os.popen("free | grep Mem | awk '{print $3}'").read() + return int(value) / 1024 + +def GetVirUse(pid): + # Mac system will disregard 'h' and print header? + value = os.popen("ps uh %s | grep %s" % (pid,pid)).read().split() + if value: + return int(value[4]) /1024 + return 0 + +def GetMemUse(pid): + value = os.popen("ps uh %s | grep %s" % (pid,pid)).read().split() + if value: + return int(value[5]) /1024 + return 0 + +def GetCpuRate(pid): + value = os.popen("ps uh %s | grep %s" % (pid,pid)).read().split() + if value: + return float(value[2]) + return 0 + +def GetUserName(): + return os.popen("whoami").read().strip() + +def GetCondorJobStat(schedd): + user = GetUserName() + #return os.popen("condor_q %s -name %s -format \"%%d.\" ClusterId -format \"%%d \" ProcId -format \"%%d\n\" JobStatus" % (user, shedd)).read().strip() + + cmd = ["condor_q", user, "-name", schedd, "-format", '%d.', "ClusterId", "-format", "%d ", "ProcId", "-format", "%d\n", "JobStatus"] + + #print(cmd) + + output = "" + for _ in range(100): + try: + output = subprocess.check_output(cmd).strip() + break + except subprocess.CalledProcessError: + print("Call '%s' failed. Retry. Waiting 10s."%cmd) + time.sleep(10) + + #print(output) + + return output + +def SubmitCondorJob(script): + return os.popen("condor_submit %s -verbose | grep \"** Proc\" | awk \'{print $3}\'" % script).read().strip().strip(':') + +def GetLSFJobStat(): + user = GetUserName() + return os.popen("bjobs -u %s" % user).read() + +def SubmitLSFJob(script, log, queue, memoryLimit): + # Job is submitted to queue . + #return os.popen("bsub -q %s -o %s %s" % (queue, log, script)).read().split()[1].strip("<>") + return os.popen("bsub -M %d -q %s %s" % (memoryLimit, queue, script)).read().split()[1].strip("<>") + +def GetPBSJobStat(): + user = GetUserName() + return os.popen("qstat -u %s" % user).read() + +def SubmitPBSJob(script, log, walltime): + # Job is submitted to queue . + #return os.popen("bsub -q %s -o %s %s" % (queue, log, script)).read().split()[1].strip("<>") + return os.popen("qsub -l walltime=%s %s" % (walltime, script)).read().split(".")[0].strip("<>") + #return os.popen("qsub -joe %s" % (script)).read().split(".")[0].strip("<>") + +def MakeAndCD(dir): + if not os.path.isdir(dir): + try: + os.makedirs(dir) + except OSError: + print('Failed to make dir: ' + dir) + sys.exit(-1) + os.chdir(dir) diff --git a/Utilities/valprod/valprod/workflow/MonitoredProcess.py b/Utilities/valprod/valprod/workflow/MonitoredProcess.py new file mode 100644 index 000000000..13e751a57 --- /dev/null +++ b/Utilities/valprod/valprod/workflow/MonitoredProcess.py @@ -0,0 +1,72 @@ +#!/usr/bin/python +# -*- coding:utf-8 -*- + +import subprocess +import time, datetime, os +from valprod.utils.monitors import * +from valprod.workflow.Process import Process,status + +class MonitoredProcess(Process): + + def __init__(self, name, cmd, cfg): + Process.__init__(self, name, cmd, cfg) + self.pid = None + + # Set up monitoring time interval + self.interval = self.cfg.getAttr('timeInterval') + assert type(self.interval) == int or type(self.interval) == float, 'attribute time interval must be a number' + assert self.interval <= 60 and self.interval >= 0.3, 'attribute time interval must be a number between 0.3 and 60' + + if self.cfg.getAttr('monitorBackend') not in ['ps','prmon']: + assert False, 'Unknown monitor backend: %s' % self.cfg.getAttr('monitorBackend') + + self.stdout = self.stderr = open(self.logFileName, 'wb+') + + def run(self): + self._start_process() + + # Create monitors + all_pids = [self.parent_pid] + self.child_pids + monitorBackend = self.cfg.getAttr('monitorBackend') + if monitorBackend == 'ps': + self._monitor = PSMonitor(self.name, self.interval, all_pids, self.cfg.getAttr('plottingBackend')) + elif monitorBackend == 'prmon': + self._monitor = PrmonMonitor(self.name, self.interval, self.parent_pid) + if self.cfg.getAttr('perf'): + self._perf_monitor = PerfMonitor(self.name, self.parent_pid) + self._perf_monitor.initialize() + if self.cfg.getAttr('profileIO'): + self._io_monitor = DiskIOMonitor(self.name, self.interval, all_pids, self.cfg.getAttr('plottingBackend')) + self._io_monitor.initialize() + self._monitor.initialize() + + # The main loop. Wait until the process finishes or the limit is reached + while True: + time.sleep(self.interval) + if not self.process.poll() == None: + break + self._monitor.execute() + if self.cfg.getAttr('profileIO'): + self._io_monitor.execute() + if not self._checkLimit(): + break + + self._monitor.finalize() + if self.cfg.getAttr('perf'): + self._perf_monitor.finalize() + if self.cfg.getAttr('profileIO'): + self._io_monitor.finalize() + + ## Check if the process ends successfully + self._burnProcess() + if self.status == status.SUCCESS and self.name: + self.stdout.close() + self._parseLogFile() + if not self.genLog: + os.remove(self.logFileName) + + def _parseLogFile(self): + if self.logParser: + result, self.fatalLine = self.logParser.parseFile(self.logFileName) + if not result: + self.status = status.FAIL diff --git a/Utilities/valprod/valprod/workflow/Process.py b/Utilities/valprod/valprod/workflow/Process.py new file mode 100644 index 000000000..b46df7746 --- /dev/null +++ b/Utilities/valprod/valprod/workflow/Process.py @@ -0,0 +1,201 @@ +import datetime +import os,sys +import select +import subprocess +from valprod.utils.shellUtil import GetMemUse +from valprod.utils.Parser import Parser + +class status: + (SUCCESS, FAIL, TIMEOUT, OVERFLOW, ANR) = range(0, 5) + Description = { + FAIL: 'Return code is not zero', + TIMEOUT: 'Run time exceeded', + OVERFLOW: 'Memory overflow', + ANR: 'Not responding' + } + @staticmethod + def describe(stat): + if stat in status.Description: + return status.Description[stat] + + +class Process: + + def __init__(self, name, exe, cfg): + + print('') + self.cfg = cfg + self.name = name or 'test' + self.logParser = cfg.getAttr('parser') and Parser(cfg) or None + self.executable = exe + self.genLog = self.cfg.getAttr('genLog') + + # Log file name depends on what we are running + logname = self.cfg.getAttr('logName') + if logname: + self.logFileName = logname + else: + self.logFileName = self.name + '.log' + if self.cfg.getAttr('step'): + self.logFileName = self.cfg.getAttr('step')+'.log' + + # Merge stdout and stderr + self.stdout = self.stderr = subprocess.PIPE + self.process = None + self.parent_pid = None + self.child_pids = [] + self.returncode = None + self.status = None + self.memory_peak = 0 + self.timeout = self.cfg.getAttr('timeout') + self.timeLimit = self.cfg.getAttr('maxTime') + self.memoryLimit = self.cfg.getAttr('maxMEM') + self.duration = None + self.start = None + self.killed = None + self.fatalLine = None + + def _start_process(self): + print('Running test: %s' % self.name) + print(self.executable) + self.start = datetime.datetime.now() + self.process = subprocess.Popen(args = self.executable, stdout = self.stdout, stderr = subprocess.STDOUT) + self.parent_pid = self.process.pid + + ## Get children pids recursively. This is important in case the parent process + ## is just a thin wrapper. + try: + import psutil + except: + return + try: + parent = psutil.Process(self.parent_pid) + children = parent.children(recursive=True) + for child in children: + self.child_pids.append(child.pid) + except psutil.NoSuchProcess: + ## Parent process is not created or has already exited. Do nothing here + pass + + def run(self): + self._start_process() + if self.genLog: + # TODO + # * allow specify log directory + # * should support directory structure + + logDir = os.path.dirname(self.logFileName) + if len(logDir) and not os.path.exists(logDir): + os.makedirs(logDir) + logFile = open(self.logFileName, 'w') + stdout_wait_time = 0 + while True: + stdout_start = datetime.datetime.now() + fs = select.select([self.process.stdout], [], [], 10) + stdout_end = datetime.datetime.now() + if not fs[0]: + # No response + stdout_wait_time = stdout_wait_time + (stdout_end - stdout_start).seconds + if self.timeout and stdout_wait_time > self.timeout: + self.status = status.ANR + self._kill() + break + else: + stdout_wait_time = 0 + if self.process.stdout in fs[0]: + # Incoming message to parse + data = os.read(self.process.stdout.fileno(), 1024).decode() + if not data: + break + # If it is called in analysis step, we print the log info to screen + if self.cfg.getAttr("step"): + for l in data.splitlines(): print("[%d]: "%self.parent_pid, l) + if self.genLog: + logFile.write(str(data)+"\n\n") + if self.logParser: + if not self._parseLog(data): + self.status = status.FAIL + self._kill() + break + if not self._checkLimit(): + break + self._burnProcess() + self._checkLimit() + if self.genLog: + logFile.close() + + def getDuration(self): + return self.duration + + def _checkLimit(self): + self.duration = (datetime.datetime.now() - self.start).seconds + if self.timeLimit and (self.duration >= self.timeLimit): + # Time out + self.status = status.TIMEOUT + self._kill() + return False + if self.memoryLimit and (self._getMem() >= self.memoryLimit): + # Memory overflow + self.status = status.OVERFLOW + self._kill() + return False + return True + + def _kill(self): + if not self.process: + return + import signal + try: + os.kill(self.parent_pid, signal.SIGKILL) + os.waitpid(-1, os.WNOHANG) + except: + pass + for child_pid in self.child_pids: + try: + os.kill(child_pid, signal.SIGKILL) + except: + pass + + def _parseLog(self, data): + result, self.fatalLine = self.logParser.parse(data) + return result + + def _burnProcess(self): + self.returncode = self.process.wait() + if self.status: + return + if 0 == self.returncode: + self.status = status.SUCCESS + else: + self.status = status.FAIL + #FIXME: it seems that root macro process won't give a 0 return code + if type(self.executable) == list and self.executable[0] == 'root': + self.status = status.SUCCESS + if type(self.executable) == str and self.executable.startswith('root'): + self.status = status.SUCCESS + + def _getMem(self): + if not self.parent_pid: + return 0 + else: + mem_now = GetMemUse(self.parent_pid) + for child_pid in self.child_pids: + mem_now = mem_now + GetMemUse(child_pid) + if mem_now > self.memory_peak: + self.memory_peak = mem_now + return mem_now + + def outcome(self): + summary = '' + if self.duration: + summary = 'Running time: %ds. ' % self.duration + if self.memory_peak: + summary = summary + 'Peak memory: %dMb' % self.memory_peak + if self.status == status.SUCCESS: + summary = 'Successful. \n' + summary + return True, summary + if self.fatalLine: + summary = 'Failed. \n' + 'FatalLine: ' + self.fatalLine + '\n' + summary + return False, summary + summary = 'Failed. \n' + status.describe(self.status) + '\n' + summary + return False, summary diff --git a/Utilities/valprod/valprod/workflow/TestWrapper.py b/Utilities/valprod/valprod/workflow/TestWrapper.py new file mode 100644 index 000000000..a41284aa5 --- /dev/null +++ b/Utilities/valprod/valprod/workflow/TestWrapper.py @@ -0,0 +1,42 @@ +import copy +import sys +from valprod.workflow.MonitoredProcess import MonitoredProcess +from valprod.workflow.Process import Process +from valprod.utils.TestConfig import * + +class TestWrapper: + + def __init__(self, name, cmd, cfg=None, **kwa): + self.subprocess = None + if cfg: + self.cfg = cfg + else: + self.cfg = copy.deepcopy(globalConfig) + self.cfg.update(**kwa) + + if self.cfg.getAttr('profile') and (not self.cfg.getAttr('monitorBackend')): + # Ues ps by default + self.cfg.setAttr('monitorBackend', 'ps') + + + # Setup sub-process + if self.cfg.getAttr('monitorBackend'): + self.subprocess = MonitoredProcess(name, cmd, self.cfg) + else: + self.subprocess = Process(name, cmd, self.cfg) + + # Setup plot reference + self.plotTester = None + plotRef = self.cfg.getAttr('plotRef') + if plotRef: + assert type(plotRef) == str or type(plotRef) == list + from valprod.utils.PlotTester import PlotTester + self.plotTester = PlotTester(self.cfg, plotRef) + + def run(self): + self.subprocess.run() + ok, summary = self.subprocess.outcome() + # If process ends succefully, invoke plotTester, if there's one + if ok and self.plotTester: + ok, dec = self.plotTester.run() + return ok, summary diff --git a/Utilities/valprod/valprod/workflow/Workflow.py b/Utilities/valprod/valprod/workflow/Workflow.py new file mode 100644 index 000000000..eaa31cb57 --- /dev/null +++ b/Utilities/valprod/valprod/workflow/Workflow.py @@ -0,0 +1,32 @@ +from valprod.workflow.TestWrapper import TestWrapper +import copy + +class Workflow: + ''' + A workflow is a set of test cases that run in a certain sequence. + The creation of the TestWrapper objects will be delayed to the moment before the test cases are executed. + ''' + + def __init__(self): + self.overallCFG = None + self.stepCFG = [] + self.steps = [] + + def addStep(self, name, cmd, **kwa): + # Creation of the TestWrapper objects will be delayed + self.stepCFG.append([name, cmd.split(), kwa]) + + def setOverallCFG(self, cfg): + self.overallCFG = cfg + + def run(self): + for sc in self.stepCFG: + cfg = copy.deepcopy(self.overallCFG) + for k,v in list(sc[2].items()): + cfg.setAttr(k,v) + self.steps.append(TestWrapper(sc[0], sc[1], cfg)) + for step in self.steps: + ok, what = step.run() + if not ok: + return False, what + return True, '' diff --git a/Utilities/valprod/valprod/workflow/__init__.py b/Utilities/valprod/valprod/workflow/__init__.py new file mode 100644 index 000000000..e69de29bb From 4f520b1e1ba6b6c2987642ecfebdf229ed4d808e Mon Sep 17 00:00:00 2001 From: Li Teng Date: Mon, 16 Jan 2023 02:44:28 +0000 Subject: [PATCH 2/9] Convert step to int value --- Utilities/valprod/valprod/utils/monitors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Utilities/valprod/valprod/utils/monitors.py b/Utilities/valprod/valprod/utils/monitors.py index 9243ca7e9..bde4319c8 100644 --- a/Utilities/valprod/valprod/utils/monitors.py +++ b/Utilities/valprod/valprod/utils/monitors.py @@ -260,7 +260,7 @@ def finalize(self): print("failed to import ROOT") return ntime = len(self._read_count) - time_seq = list(range(0, ntime * self._interval, self._interval)) + time_seq = list(range(0, int(ntime * self._interval), int(self._interval))) # Decide which ts = array('d') From cd8691098d998adf3e7cb3133aa34813af3fecff Mon Sep 17 00:00:00 2001 From: Li Teng Date: Mon, 16 Jan 2023 03:04:15 +0000 Subject: [PATCH 3/9] Set EvtMax to 1000 --- Examples/options/tut_detsim.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Examples/options/tut_detsim.py b/Examples/options/tut_detsim.py index ededb9965..4fd2d3bc6 100644 --- a/Examples/options/tut_detsim.py +++ b/Examples/options/tut_detsim.py @@ -134,6 +134,6 @@ from Configurables import ApplicationMgr ApplicationMgr( TopAlg = [genalg, detsimalg, out], EvtSel = 'NONE', - EvtMax = 10, + EvtMax = 1000, ExtSvc = [rndmengine, rndmgensvc, dsvc, geosvc], ) From 3fb83bee1956fae66c07a4dbf5b0881b95ceb40d Mon Sep 17 00:00:00 2001 From: tengli Date: Mon, 16 Jan 2023 11:32:07 +0800 Subject: [PATCH 4/9] simplfy the workflow for a quicker test --- .github/workflows/main.yml | 24 ++---------------------- Examples/options/tut_detsim.py | 2 +- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a4ce0b10e..914329e81 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,40 +19,20 @@ jobs: build: # The type of runner that the job will run on runs-on: self-hosted - strategy: - matrix: - LCG_RELEASE: [LCG_101, KEY4HEP_STACK] - # CEPCSW_BLDTOOL: [make, ninja] - CEPCSW_BLDTOOL: [ninja] # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 - # Runs a single command using the runners shell - - name: Run a one-line script - run: echo Hello, world! - - # Runs a set of commands using the runners shell - - name: Run a multi-line script - run: | - echo Add other actions to build, - echo test, and deploy your project. - - name: Run the build script run: | pwd - bash ./.build.ci.sh - env: - LCG_RELEASE: ${{matrix.LCG_RELEASE}} - CEPCSW_BLDTOOL: ${{matrix.CEPCSW_BLDTOOL}} + bash build.sh - name: Run the simulation example run: | pwd source setup.sh + perf -h jMonitor --gen-log -n SimTest --enable-io-profile --enable-perf gaudirun.py Examples/options/tut_detsim.py - env: - LCG_RELEASE: ${{matrix.LCG_RELEASE}} - CEPCSW_BLDTOOL: ${{matrix.CEPCSW_BLDTOOL}} diff --git a/Examples/options/tut_detsim.py b/Examples/options/tut_detsim.py index 4fd2d3bc6..8ca7f7f61 100644 --- a/Examples/options/tut_detsim.py +++ b/Examples/options/tut_detsim.py @@ -134,6 +134,6 @@ from Configurables import ApplicationMgr ApplicationMgr( TopAlg = [genalg, detsimalg, out], EvtSel = 'NONE', - EvtMax = 1000, + EvtMax = 100, ExtSvc = [rndmengine, rndmgensvc, dsvc, geosvc], ) From 9e72a75be14cfa9cac9ed1974d56fbcd00cb2e70 Mon Sep 17 00:00:00 2001 From: tengli Date: Mon, 16 Jan 2023 11:34:16 +0800 Subject: [PATCH 5/9] set up env first --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 914329e81..a76fc1eba 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,6 +28,7 @@ jobs: - name: Run the build script run: | pwd + source setup.sh bash build.sh - name: Run the simulation example From 7fe92a979b67d8a52569cb1a1eed0ea0ad76ad07 Mon Sep 17 00:00:00 2001 From: tengli Date: Mon, 16 Jan 2023 11:54:59 +0800 Subject: [PATCH 6/9] upload profile figures --- .github/workflows/main.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a76fc1eba..d0bd06716 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,7 +29,7 @@ jobs: run: | pwd source setup.sh - bash build.sh + ./build.sh - name: Run the simulation example run: | @@ -37,3 +37,12 @@ jobs: source setup.sh perf -h jMonitor --gen-log -n SimTest --enable-io-profile --enable-perf gaudirun.py Examples/options/tut_detsim.py + ls -la + + - name: Archive profile artifacts + uses: actions/upload-artifact@v3 + with: + name: profile-figures + path: | + *.png + *.svg From 41cb730fdf4b9c54144ef1727328456ff4c4302d Mon Sep 17 00:00:00 2001 From: tengli Date: Mon, 16 Jan 2023 12:07:47 +0800 Subject: [PATCH 7/9] use build-ci --- .github/workflows/main.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d0bd06716..fa669b7c8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,6 +19,12 @@ jobs: build: # The type of runner that the job will run on runs-on: self-hosted + strategy: + matrix: + #LCG_RELEASE: [LCG_101, KEY4HEP_STACK] + LCG_RELEASE: [LCG_101] + # CEPCSW_BLDTOOL: [make, ninja] + CEPCSW_BLDTOOL: [ninja] # Steps represent a sequence of tasks that will be executed as part of the job steps: @@ -28,8 +34,10 @@ jobs: - name: Run the build script run: | pwd - source setup.sh - ./build.sh + bash ./.build.ci.sh + env: + LCG_RELEASE: ${{matrix.LCG_RELEASE}} + CEPCSW_BLDTOOL: ${{matrix.CEPCSW_BLDTOOL}} - name: Run the simulation example run: | From f1e621a117a625e7823b59d0b0f39fd3d1cec84d Mon Sep 17 00:00:00 2001 From: tengli Date: Mon, 16 Jan 2023 12:16:17 +0800 Subject: [PATCH 8/9] try install perf --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fa669b7c8..31454f482 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,7 +43,7 @@ jobs: run: | pwd source setup.sh - perf -h + yum install -y perf jMonitor --gen-log -n SimTest --enable-io-profile --enable-perf gaudirun.py Examples/options/tut_detsim.py ls -la From 772f71ae37d50b73e6d6b2ade9f0f4767957541b Mon Sep 17 00:00:00 2001 From: tengli Date: Mon, 16 Jan 2023 12:24:50 +0800 Subject: [PATCH 9/9] remove install perf --- .github/workflows/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 31454f482..6d866007b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,7 +43,6 @@ jobs: run: | pwd source setup.sh - yum install -y perf jMonitor --gen-log -n SimTest --enable-io-profile --enable-perf gaudirun.py Examples/options/tut_detsim.py ls -la