cpu

#!/usr/bin/perl -w
# -*- perl -*-

=head1 NAME

cpu - Plugin to monitor CPU usage and frequencies.

=head1 APPLICABLE SYSTEMS

All Linux systems, but read below section

=head1 CONFIGURATION

The plugin automatically selects which graphics drawing.
Charts related to the frequencies of processors depends on the settings of the kernel:
Power management and ACPI options -> CPU Frequency scaling -> CPU frequency translation statistics -> Advanced statistics

=head2 WARNING AND CRITICAL SETTINGS

You can set warning and critical levels for each of the data
series the plugin reports.  The following environment variables are
used as default for all fields:

  env.warning
  env.critical

But each field (system user nice etc...) can be controlled separately:
For example:

  env.system_warning 70
  env.user_warning 70
  env.idle_critical 1

Also each field of each cpu can be controlled separately
For example:

  env.cpu1_system_warning 70
  env.cpu0_user_warning 70
  env.cpu0_idle_critical 1

Algoritm is easy: for example current graph limit is env.cpu0_idle_critical if defined env.cpu0_idle_critical or env.idle_critical if defined env.idle_critical
or env.critical if defined env.critical. Or no limit

=head1 INTERPRETATION

The plugin shows each cpu usage in percent, shows the CPU frequency, 
frequency shift frequencies, the percentage of use of frequencies

=head1 MAGIC MARKERS

  #%# family=auto
  #%# capabilities=autoconf


=head1 VERSION

  2.0

=head1 BUGS

none known

=head1 AUTHOR

Gorlow Maxim aka Sheridan <sheridan@sheridan-home.ru> (email and jabber)

=head1 LICENSE

GPLv2

=cut

use strict;
use warnings;
use Munin::Plugin;
use Data::Dumper;
my @cnames = qw(user nice system idle iowait irq softirq steal guest);
my $stat_file = "/proc/stat";
my $freq_path = "/sys/devices/system/cpu";
my $limits = {};
my $cpuinfo = {}; $cpuinfo->{'cpu_count'} = 0;
my @stat_file_content = ();
my $freq_mul = 1000; # CPU frequency multiplier from kHz to Hz



my $graphs =
{
  'cpu_utilisation' => # multigraph
  {
    'title'  => 'CPU:t: utilisation',
    'args'   =>  '--base 1000 -r --lower-limit 0 --upper-limit 100',
    'vlabel' => '%',
    'scale'  => 'no',
    'info'   => 'This graph shows how CPU:t: time is spent :i:'
  },
  'cpu_all' => # single
  {
    'title'    => 'All CPU utilisation',
    'args'     =>  '--base 1000 -r --lower-limit 0 --upper-limit 100',
    'vlabel'   => '%',
    'scale'    => 'no',
    'info'     => 'This graph shows how CPU time is spent on each processor',
    'category' => 'cpu'
  },
  'cpu_freq_trans' => # multi
  {
    'title'    => 'CPU frequency transitions',
    'args'     =>  '--base 1000',
    'vlabel'   => 'count',
    'scale'    => 'no',
    'info'     => 'This graph shows CPU transitions of each processor',
  },
  'cpu_freq' => # child of cpu_freq_trans
  {
    'title'    => 'CPU:t: frequency (total)',
    'args'     =>  '--base 1000 -r --lower-limit 0 --upper-limit 100',
    'vlabel'   => '% of total',
    'info'     => 'This graph shows CPU:t: frequency :i:',
    'category' => 'cpu'
  },
  'cpu_freq_ps' => # child of cpu_freq_trans
  {
    'title'    => 'CPU:t: frequency (per secund)',
    'args'     =>  '--base 1000 -r --lower-limit 0 --upper-limit 100',
    'vlabel'   => '% per secund',
    'info'     => 'This graph shows CPU:t: frequency per secund from last update :i:',
    'category' => 'cpu'
  },
  'cpu_freq_trans_table' => # child of cpu_freq_trans
  {
    'title'    => 'CPU:t: frequency switches (total)',
    'args'     =>  '--base 1000 -r --lower-limit 0 --upper-limit 100',
    'vlabel'   => '% of total',
    'scale'    => 'no',
    'info'     => 'This graph shows CPU:t: frequency switches :i:',
  },
  'cpu_freq_trans_table_ps' => # child of cpu_freq_trans
  {
    'title'    => 'CPU:t: frequency switches (per secund)',
    'args'     =>  '--base 1000 -r --lower-limit 0 --upper-limit 100',
    'vlabel'   => '% per secund',
    'scale'    => 'no',
    'info'     => 'This graph shows CPU:t: frequency switches per secund from last update :i:',
  }
};

my $transparent = 'CC';
my $fields =
{
  'user' =>
  {
    'label' => 'User',
    'info'  => 'Normal processes executing in user mode',
    'type'  => 'GAUGE',
    'draw'  => 'AREASTACK',
    'min'   => 0,
    'max'   => 100
  },
  'nice' =>
  {
    'label' => 'Nice',
    'info'  => 'Niced processes executing in user mode',
    'type'  => 'GAUGE',
    'draw'  => 'AREASTACK',
    'min'   => 0,
    'max'   => 100
  },
  'system' =>
  {
    'label' => 'System',
    'info'  => 'Processes executing in kernel mode',
    'type'  => 'GAUGE',
    'draw'  => 'AREASTACK',
    'min'   => 0,
    'max'   => 100
  },
  'idle' =>
  {
    'label' => 'Idle',
    'info'  => 'Twiddling thumbs',
    'type'  => 'GAUGE',
    'draw'  => 'AREASTACK',
    'colour'=> 'FFFFDD'.$transparent,
    'min'   => 0,
    'max'   => 100
  },
  'iowait' =>
  {
    'label' => 'I/O wait',
    'info'  => 'Waiting for I/O to complete',
    'type'  => 'GAUGE',
    'draw'  => 'AREASTACK',
    'min'   => 0,
    'max'   => 100
  },
  'irq' =>
  {
    'label' => 'IRQ',
    'info'  => 'Servicing interrupts',
    'type'  => 'GAUGE',
    'draw'  => 'AREASTACK',
    'min'   => 0,
    'max'   => 100
  },
  'softirq' =>
  {
    'label' => 'Software IRQ',
    'info'  => 'Servicing software interrupts',
    'type'  => 'GAUGE',
    'draw'  => 'AREASTACK',
    'min'   => 0,
    'max'   => 100
  },
  'steal' =>
  {
    'label' => 'Steal',
    'info'  => 'Involuntary wait',
    'type'  => 'GAUGE',
    'draw'  => 'AREASTACK',
    'min'   => 0,
    'max'   => 100
  },
  'guest' =>
  {
    'label' => 'Guest',
    'info'  => 'Running a guest',
    'type'  => 'GAUGE',
    'draw'  => 'AREASTACK',
    'min'   => 0,
    'max'   => 100
  },
  'cpu_util' =>
  {
    'label' => ':t:',
    'info'  => ':t: utilisation',
    'type'  => 'GAUGE',
    'draw'  => 'LINE0.5',
    'min'   => 0,
    'max'   => 100
  },
  'freq_percent' =>
  {
    'label' => 'CPU:t: frequency',
    'info'  => 'CPU:t: frequency percent',
    'type'  => 'GAUGE',
    'draw'  => 'LINE0.5',
    'min'   => 0,
    'max'   => 100
  },
  'freq_hz' =>
  {
    'label' => ':t:',
    'info'  => 'CPU :t: frequency',
    'type'  => 'GAUGE',
    'draw'  => 'AREASTACK',
    'min'   => 0,
    'max'   => 100
  },
  'freq_trans' =>
  {
    'label' => ':t:',
    'info'  => ':t: frequency transitions',
    'type'  => 'GAUGE',
    'draw'  => 'LINE0.5',
    'min'   => 0
  },
  'freq_trans_table' =>
  {
    'label' => ':t:',
    'info'  => ':t: frequency switch',
    'type'  => 'GAUGE',
    'draw'  => 'AREASTACK',
    'min'   => 0,
    'max'   => 100
  }
  
};



# ----------------- main ----------------
load_cpuinfo();
need_multigraph();
remove_unavialabled_counters();

if (defined($ARGV[0]) and ($ARGV[0] eq 'autoconf')) 
{
  printf("%s\n", -e $stat_file ? "yes" : "no (stats not exists)");
  exit (0);
}

if (defined($ARGV[0]) and ($ARGV[0] eq 'config')) 
{
  print_config();
  exit (0);
}

print_values();
#print Dumper prepare_graphs_fields();
exit(0);

# ----------------- sub's ----------------

# ----------------------- trim whitespace at begin and end of string ------------
sub trim
{
  my($string)=@_;
  for ($string) { s/^\s+//; s/\s+$//; }
  return $string;
}


my $items_exists = {};
sub check_exists
{
  my ($t, $cpu_num) = @_[0..1];
  if (defined($cpu_num))
  {
    unless (exists($items_exists->{$t}{$cpu_num}))
    {
      if ($t eq 'freq_hz')        { $items_exists->{$t}{$cpu_num} = ( -e sprintf("%s/cpu%s/cpufreq/scaling_min_freq"   , $freq_path, $cpu_num) and
                                                                      -e sprintf("%s/cpu%s/cpufreq/scaling_cur_freq"   , $freq_path, $cpu_num) and
                                                                      -e sprintf("%s/cpu%s/cpufreq/scaling_max_freq"   , $freq_path, $cpu_num));  }
      elsif ($t eq 'freq_trans')  { $items_exists->{$t}{$cpu_num} =   -e sprintf("%s/cpu%s/cpufreq/stats/time_in_state", $freq_path, $cpu_num);   }
      elsif ($t eq 'freq_times')  { $items_exists->{$t}{$cpu_num} =   -e sprintf("%s/cpu%s/cpufreq/stats/total_trans"  , $freq_path, $cpu_num);   }
      elsif ($t eq 'freq_ttable') { $items_exists->{$t}{$cpu_num} =   -e sprintf("%s/cpu%s/cpufreq/stats/trans_table"  , $freq_path, $cpu_num);   }
    }
    return $items_exists->{$t}{$cpu_num};
  }
  else
  {
    unless(exists($items_exists->{$t}{'total'}))
    {
      my $c = 0;
      for (my $i=0; $i < $cpuinfo->{'cpu_count'}; $i++) { $c++ if (check_exists($t, $i)); }
      $items_exists->{$t}{'total'} = $c > 0;
    }
    return $items_exists->{$t}{'total'};
  }
}

# ------------------------- remove unavialable fields from graph --------------------------
sub remove_unavialabled_counters
{
  my @cpu = split(/\s+/, (grep(/cpu\s/, get_stat_file_content()))[0]);
  my $counters_count = scalar(@cpu) - 3;
  @cnames = @cnames[0..$counters_count];
}

# ----------------------- get sysfs file content ----------------
my $fcontents = {};
sub get_sys_file_content
{
  my $file = $_[0];
  return 'nan' if (-z $file);
  unless (exists($fcontents->{$file}))
  {
    open (FH, '<', $file) or die "$! $file \n";
    $fcontents->{$file} = <FH>;
    close (FH);
    chomp $fcontents->{$file};
  }
  return $fcontents->{$file};
}

# -------------------------- loading cpu info ---------------------------------
sub load_cpuinfo
{
  my $file = "/proc/cpuinfo";
  open (FH, '<', $file) or die "$! $file \n";
  my $cpu_num = -1;
  for my $line (<FH>)
  {
    chomp $line;
    $cpu_num++ if $line =~ m/^processor\s+:/;
    $cpuinfo->{$cpu_num}{'name'}     = trim((split(/:/,$line))[1]) if $line =~ m/^model name\s+:/;
    $cpuinfo->{$cpu_num}{'bogomips'} = trim((split(/:/,$line))[1]) if $line =~ m/^bogomips\s+:/;
    if (not exists($cpuinfo->{$cpu_num}{'info'}) and exists($cpuinfo->{$cpu_num}{'name'}) and exists($cpuinfo->{$cpu_num}{'bogomips'}))
    {
      $cpuinfo->{$cpu_num}{'info'} = sprintf("[%s (%s bogomips)]", $cpuinfo->{$cpu_num}{'name'}, $cpuinfo->{$cpu_num}{'bogomips'}) ;
    }
  }
  close (FH);
  $cpuinfo->{'cpu_count'} = $cpu_num+1;
}


# -------------------------- loading stat file lines ---------------------------------
sub get_stat_file_content
{
  if(scalar(@stat_file_content) == 0)
  {
    open (FH, '<', $stat_file) or die "$! $stat_file \n";
    for (<FH>)
    {
      next unless $_ =~ m/cpu/;
      chomp $_;
      push(@stat_file_content, $_);
    }
    close (FH);
  }
  return @stat_file_content;
}

# -------------------------------- replacing strings ------------------------
sub replace_template
{
  my ($string, $needle, $replacement) = @_[0..2];
  $string =~ s/$needle/$replacement/g;
  return $string;
}

sub replace_templates
{
  my ($src, $replacement_t, $replacement_i) = @_[0..2];
  my $dst = {};
  for my $key ( keys %{$src} )
  {
    $dst->{$key} = $src->{$key};
    if($key =~ m/label|info|title/) { $dst->{$key} = replace_template($dst->{$key}, ':t:', $replacement_t); }
    if($key =~ m/info|title/)       { $dst->{$key} = replace_template($dst->{$key}, ':i:', $replacement_i); }
  }
  return $dst;
}

sub append_order
{
  my ($pg, $graph_name, $field_name) = @_[0..2];
  $pg->{$graph_name}{'graph'}{'order'} = exists($pg->{$graph_name}{'graph'}{'order'}) ? sprintf("%s %s", $pg->{$graph_name}{'graph'}{'order'}, $field_name) : $field_name;
}

sub append_field
{
  my ($pg, $graph_name, $field_name, $field_src, $replacement_t, $replacement_i) = @_[0..5];
  $pg->{$graph_name}{'fields'}{$field_name} = replace_templates($fields->{$field_src}, $replacement_t, $replacement_i);
  append_order($pg, $graph_name, $field_name);
}

sub append_graph
{
  my ($pg, $graph_name, $category, $graph_src, $replacement_t, $replacement_i) = @_[0..5];
  $pg->{$graph_name}{'graph'} = replace_templates($graphs->{$graph_src}, $replacement_t, $replacement_i);
  $pg->{$graph_name}{'graph'}{'category'} = $category;
}

# ---------------------------------------- preparing data for graphs ------------------------------------
sub prepare_graphs_fields
{
  my $pg = {};
  # ------------------ cpu_utilisation -----------------------------------
  # ---------- general ----------------------
  append_graph($pg, 'cpu_utilisation', 'cpu', 'cpu_utilisation', '', '');
  for my $cname (@cnames) { append_field($pg, 'cpu_utilisation', $cname, $cname, '', ''); append_utilisation_limits($pg, 'cpu_utilisation', $cname, undef); }
  if(check_exists('freq_hz')) { for (my $i = 0; $i < $cpuinfo->{'cpu_count'}; $i++) { append_field($pg, 'cpu_utilisation', sprintf("fp_%s", $i), 'freq_percent', $i, ''); } }
  # ---------------- childs -------------------
  if ($cpuinfo->{'cpu_count'} > 1)
  {
    for (my $i=0; $i < $cpuinfo->{'cpu_count'}; $i++)
    {
      my $graph_name = sprintf("cpu_utilisation.cpu%s", $i);
      append_graph($pg, $graph_name, sprintf("CPU %s", $i), 'cpu_utilisation', $i, $cpuinfo->{$i}{'info'});
      for my $cname (@cnames) { append_field($pg, $graph_name, $cname, $cname, '', ''); append_utilisation_limits($pg, $graph_name, $cname, $i); }
      if(check_exists('freq_hz', $i)) { append_field($pg, $graph_name, 'fp', 'freq_percent', '', ''); }
    }
  }
  
  
  # --------------- cpu_frequency --------------------------------------------
  if(check_exists('freq_trans'))
  { 
    # ------------ general --------------------
    # - cpu frequency transitions -
    append_graph($pg, 'cpu_frequency', 'cpu', 'cpu_freq_trans', '', '');
    for (my $i = 0; $i < $cpuinfo->{'cpu_count'}; $i++) { if (check_exists('freq_trans', $i)) { append_field($pg, 'cpu_frequency', sprintf("cpu_%s", $i), 'freq_trans', sprintf("CPU%s", $i), ''); } }
    append_field($pg, 'cpu_frequency', 'total', 'freq_trans', 'Total', '');
    # ---------------- childs -------------------
    if(check_exists('freq_times'))
    {
      my $frequences = get_frequency_times();
      for (my $i=0; $i < $cpuinfo->{'cpu_count'}; $i++)
      {
        if(check_exists('freq_times', $i))
        {
          # - cpu frequencyes -
          my $graph_name = sprintf("cpu_frequency.percent_cpu%s", $i);
          append_graph($pg, $graph_name, sprintf("CPU %s", $i), 'cpu_freq', $i, $cpuinfo->{$i}{'info'});
          for my $freq (@{$frequences->{'names'}{$i}}) { append_field($pg, $graph_name, sprintf("hz_%s", $freq), 'freq_hz', scaleNumber($freq, 'Hz'), ''); }
          # - cpu frequencyes per secund -
          $graph_name = sprintf("cpu_frequency.percent_ps_cpu%s", $i);
          append_graph($pg, $graph_name, sprintf("CPU %s", $i), 'cpu_freq_ps', $i, $cpuinfo->{$i}{'info'});
          for my $freq (@{$frequences->{'names'}{$i}}) { append_field($pg, $graph_name, sprintf("hz_%s", $freq), 'freq_hz', scaleNumber($freq, 'Hz'), ''); }
        }
      }
    }
    if(check_exists('freq_ttable'))
    {
      my $f_table = get_frequency_trans_table();
      for (my $i=0; $i < $cpuinfo->{'cpu_count'}; $i++)
      {
        if(check_exists('freq_ttable', $i))
        {
          # - cpu frequencyes table -
          my $graph_name = sprintf("cpu_frequency.trans_table_cpu%s", $i);
          append_graph($pg, $graph_name, sprintf("CPU %s", $i), 'cpu_freq_trans_table', $i, $cpuinfo->{$i}{'info'});
          for my $from (sort keys %{$f_table->{'values'}{$i}})
          {
            for my $to (sort keys %{$f_table->{'values'}{$i}{$from}})
            {
              next if ($from eq $to);
              append_field($pg, $graph_name, sprintf("f_%s_t_%s", $from, $to), 'freq_trans_table', sprintf(". %9s -> %s", scaleNumber($from, 'Hz'), scaleNumber($to, 'Hz')), '');
            }
          }
          # - cpu frequencyes table per secund -
          $graph_name = sprintf("cpu_frequency.trans_table_ps_cpu%s", $i);
          append_graph($pg, $graph_name, sprintf("CPU %s", $i), 'cpu_freq_trans_table_ps', $i, $cpuinfo->{$i}{'info'});
          for my $from (sort keys %{$f_table->{'values'}{$i}})
          {
            for my $to (sort keys %{$f_table->{'values'}{$i}{$from}})
            {
              next if ($from eq $to);
              append_field($pg, $graph_name, sprintf("f_%s_t_%s", $from, $to), 'freq_trans_table', sprintf(". %9s -> %s", scaleNumber($from, 'Hz'), scaleNumber($to, 'Hz')), '');
            }
          }
        }
      }
    }
  }
  
  
  # --------------- cpu_all -----------------------------------------
  if ($cpuinfo->{'cpu_count'} > 1)
  {
    append_graph($pg, 'cpu_all', 'cpu', 'cpu_all', '', '');
    for (my $i = 0; $i < $cpuinfo->{'cpu_count'}; $i++) { append_field($pg, 'cpu_all', sprintf("cpu_%s", $i), 'cpu_util', sprintf("CPU%s", $i)); }
    append_field($pg, 'cpu_all', 'total', 'cpu_util', 'Combined');
  }
  return $pg;
}


# ------------------------------------ printing limits (for utilisation graphs) ----------------
sub append_utilisation_limits
{
  my ($pg, $graph_name, $cname, $i) = @_[0..3];
  for my $type (qw(warning critical))
  {
    my $field = sprintf("%s_%s", $cname, $type);
    my $cpu_field = defined($i) ? sprintf("cpu%s_%s_%s", $i, $cname, $type) : undef;
    my $limit = (defined($i) and defined($limits->{'utilisation'}{$cpu_field})) ? 
                $limits->{'utilisation'}{$cpu_field} : 
                (
                  defined($limits->{'utilisation'}{$field}) ? 
                  $limits->{'utilisation'}{$field} : 
                  ( 
                    defined($limits->{'utilisation'}{$type})  ? 
                    $limits->{'utilisation'}{$type}  : 
                    undef 
                  ) 
                );
    if(defined($limit)) { $pg->{$graph_name}{'fields'}{$cname}{$type} = $limit; }
  }
}

# ---------------- loading limits -------------
sub load_limits
{
  $limits->{'utilisation'}{'warning'}  = $ENV{warning}  || undef;
  $limits->{'utilisation'}{'critical'} = $ENV{critical} || undef;
  for my $cname (@cnames)
  {
    for my $t (qw(warning critical))
    {
      my $name = sprintf("%s_%s", $cname, $t);
      $limits->{'utilisation'}{$name} = $ENV{$name} || undef;
      for (my $i = 0; $i <= $cpuinfo->{'cpu_count'}; $i++)
      {
        $name = sprintf("cpu%s_%s_%s",$i, $cname, $t);
        $limits->{'utilisation'}{$name} = $ENV{$name} || undef;
      }
    }
  }
}

# --------------------------------- graph configs ----------------------------
sub print_config
{
  load_limits();
  my $config = prepare_graphs_fields();
  for my $g (sort keys %{$config})
  {
    printf("multigraph %s\n", $g);
    for my $go (sort keys %{$config->{$g}{'graph'}}) { printf("graph_%s %s\n", $go, $config->{$g}{'graph'}{$go}); }
    for my $f (sort keys %{$config->{$g}{'fields'}}) { for my $fo (sort keys %{$config->{$g}{'fields'}{$f}}) { printf("%s.%s %s\n", $f, $fo, $config->{$g}{'fields'}{$f}{$fo}); } }
    print "\n";
  }
}

# ----------------------------------- saving state data using munin --------------------
sub save_state_data
{
  my $data = $_[0];
  my $d = Data::Dumper->new([$data]);
  $d->Indent(0);
  save_state($d->Dump);
}

# -------------------------------- loading previous state data using munin -------------------
sub restore_state_data
{
  my $VAR1;
  my $states = (restore_state())[0];
  eval $states if defined $states;
  return $VAR1;
}

sub load_stats
{
  my $stats = {};
  # need to store --------------------
  $stats->{'timestamp'} = time();
  $stats->{'cpu_util'}  = get_cpu_utilisation_stats()                                ;
  $stats->{'f_trans'}   = get_frequency_transitions()  if check_exists('freq_trans') ;
  $stats->{'f_times'}   = get_frequency_times()        if check_exists('freq_times') ;
  $stats->{'f_ttable'}  = get_frequency_trans_table()  if check_exists('freq_ttable');

  save_state_data($stats);

  # no need to store --------------------
  $stats->{'f_minmax'}  = get_cpu_curr_max_freqences() if check_exists('freq_hz')    ;
  

  #print Dumper $stats;
  return $stats;
}

# ---------------------------------- loading cpu stats from loaded lines ------------------------
sub get_cpu_utilisation_stats
{
  my $stats = {};
  for (my $i = 0; $i <= $cpuinfo->{'cpu_count'}; $i++)
  {
    my $rx = $i == $cpuinfo->{'cpu_count'} ? 'cpu' : sprintf ("cpu%s", $i);
    my $cn = $i == $cpuinfo->{'cpu_count'} ? 'total' : $i;
    my @tmp = split(/\s+/, (grep(/$rx\s/, get_stat_file_content()))[0]);
    my $j = 1;
    for my $cname (@cnames)
    {
      $stats->{$cn}{$cname} = $tmp[$j];
      $j++;
    }
  }
  return $stats;
}
# ------------------ loading frequency transitions for each cpu ----------------------------
sub get_frequency_transitions
{
  my $stats = {};
  for (my $i = 0; $i < $cpuinfo->{'cpu_count'}; $i++)
  {
    next unless (check_exists('freq_trans', $i));
    $stats->{$i} = get_sys_file_content(sprintf("%s/cpu%s/cpufreq/stats/total_trans", $freq_path, $i));
  }
  return $stats;
}

# ------------------ loading frequency times for each cpu ----------------------------
sub get_frequency_times
{
  my $stat = {};
  for (my $i = 0; $i < $cpuinfo->{'cpu_count'}; $i++)
  {
    next unless (check_exists('freq_times', $i));
    my $total = 0;
    my $file = sprintf("%s/cpu%s/cpufreq/stats/time_in_state", $freq_path, $i);
    open (FH, '<', $file) or die "$! $file \n";
    for my $line (<FH>)
    {
      chomp $line;
      my ($hz, $count) = split(/\s+/, $line);
      $hz = $hz*$freq_mul;
      $stat->{'values'}{$i}{$hz} = $count;
      push(@{$stat->{'names'}{$i}}, $hz);
      $total += $count;
    }
    close (FH);
    $stat->{'total'}{$i} = $total;
  }
  return $stat;
}

# ------------------ loading current and max frequency for each cpu ----------------------------
sub get_cpu_curr_max_freqences
{
  my $freq = {};
  for (my $i = 0; $i < $cpuinfo->{'cpu_count'}; $i++)
  {
    next unless (check_exists('freq_hz', $i));
    my $cpu_path = sprintf("%s/cpu%s/cpufreq", $freq_path, $i);
    $freq->{'cur'}{$i} = get_sys_file_content(sprintf("%s/scaling_cur_freq", $cpu_path))*$freq_mul;
    $freq->{'max'}{$i} = get_sys_file_content(sprintf("%s/scaling_max_freq", $cpu_path))*$freq_mul;
    $freq->{'min'}{$i} = get_sys_file_content(sprintf("%s/scaling_min_freq", $cpu_path))*$freq_mul;
  }
  return $freq;
}

sub get_frequency_trans_table
{
  my $tbl = {};
  for (my $i = 0; $i < $cpuinfo->{'cpu_count'}; $i++)
  {
    next unless (check_exists('freq_ttable', $i));
    my @frequences;
    my $fcount = 0;
    my $total = 0;
    my $file = sprintf("%s/cpu%s/cpufreq/stats/trans_table", $freq_path, $i);
    open (FH, '<', $file) or die "$! $file \n";
    for my $line (<FH>)
    {
      chomp $line;
      my ($left, $right) = split(/:/, $line);
      next if($left =~ m/From/);
      if($left =~ m/\d+/)
      {
        my $frequence = trim($left)*$freq_mul;
        my @counters = split(/\s+/, trim($right));
        for (my $j = 0; $j<$fcount; $j++)
        {
          $tbl->{'values'}{$i}{$frequence}{$frequences[$j]*$freq_mul} = $counters[$j];
          $total += $counters[$j];
        }
      }
      else
      {
        @frequences = split(/\s+/, trim($right));
        $fcount = scalar(@frequences);
      }
    }
    $tbl->{'total'}{$i} = $total;
    close (FH);
  }
  return $tbl;
}

sub one_second_part
{
  my ($curr, $prev, $timediff) = @_[0..2];
  #print "$prev, $curr, $timediff\n";
  return 'NaN' if ($curr < $prev or $timediff < 0);
  return $curr - $prev if $timediff == 0;
  return ($curr - $prev)/$timediff;
}

sub divide
{
  my ($divider, $divident) = @_[0..1];
  return 'NaN' if $divident == 0;
  return $divider/$divident;
}

# -------------------------------- calculating fields values ------------------------------
sub calculate
{
  my ($pstats, $cstats) = @_[0..1];
  my $result = {};
  my $timediff = $cstats->{'timestamp'} - $pstats->{'timestamp'};
  # --- cpu utilisation ----
  for my $cpu (keys %{$cstats->{'cpu_util'}})
  {
    # ------ calculating 1%
    $result->{'cpu_util'}{'1%'}{$cpu} = 0;
    for my $cname (@cnames)
    {
      $result->{'cpu_util'}{'diff'}{$cpu}{$cname} = one_second_part($cstats->{'cpu_util'}{$cpu}{$cname}, $pstats->{'cpu_util'}{$cpu}{$cname}, $timediff);
      $result->{'cpu_util'}{'1%'}{$cpu} += $result->{'cpu_util'}{'diff'}{$cpu}{$cname} if $result->{'cpu_util'}{'diff'}{$cpu}{$cname} ne 'NaN';
    }
    $result->{'cpu_util'}{'1%'}{$cpu} = $result->{'cpu_util'}{'1%'}{$cpu}/100;
    # ------ calculating used percents
    $result->{'cpu_util'}{'used'}{$cpu} = 0;
    for my $cname (@cnames)
    {
      $result->{'cpu_util'}{'%'}{$cpu}{$cname} = divide($result->{'cpu_util'}{'diff'}{$cpu}{$cname}, $result->{'cpu_util'}{'1%'}{$cpu});
      next if $cname eq 'idle';
      $result->{'cpu_util'}{'used'}{$cpu} += $result->{'cpu_util'}{'%'}{$cpu}{$cname} if $result->{'cpu_util'}{'%'}{$cpu}{$cname} ne 'NaN';
    }
  }
  # ------ freq min max ----
  if (check_exists('freq_hz'))
  {
    for (my $i = 0; $i < $cpuinfo->{'cpu_count'}; $i++)
    {
      $result->{'f_minmax'}{'%'}{$i} =     divide($cstats->{'f_minmax'}{'cur'}{$i} - $cstats->{'f_minmax'}{'min'}{$i},
                                                (($cstats->{'f_minmax'}{'max'}{$i} - $cstats->{'f_minmax'}{'min'}{$i}) / 100 )) if (check_exists('freq_hz', $i));
    }
  }
  # ---- freq trans ----
  if (check_exists('freq_trans'))
  {
    $result->{'f_trans'}{'total'} = 0;
    for (my $i = 0; $i < $cpuinfo->{'cpu_count'}; $i++)
    {
      if(check_exists('freq_trans', $i))
      {
        $result->{'f_trans'}{$i} = one_second_part($cstats->{'f_trans'}{$i}, $pstats->{'f_trans'}{$i}, $timediff);
        $result->{'f_trans'}{'total'} += $result->{'f_trans'}{$i} if $result->{'f_trans'}{$i}  ne 'NaN';
      }
    }
  }
  # -- freq times ---
  if (check_exists('freq_times'))
  {
    for (my $i = 0; $i < $cpuinfo->{'cpu_count'}; $i++)
    {
      if (check_exists('freq_times', $i))
      {
        my $oneprc = $cstats->{'f_times'}{'total'}{$i}/100;
        my $ps_total = 0;
        for my $hz (@{$cstats->{'f_times'}{'names'}{$i}})
        {
          $result->{'f_times'}{$i}{$hz} = divide($cstats->{'f_times'}{'values'}{$i}{$hz}, $oneprc);
          $cstats->{'f_times'}{'%_ps'}{$i}{$hz} = one_second_part($cstats->{'f_times'}{'values'}{$i}{$hz}, $pstats->{'f_times'}{'values'}{$i}{$hz}, $timediff);
          $ps_total += $cstats->{'f_times'}{'%_ps'}{$i}{$hz};
        }
        $ps_total = $ps_total/100;
        for my $hz (@{$cstats->{'f_times'}{'names'}{$i}})
        {
          $result->{'f_times_ps'}{$i}{$hz} = divide($cstats->{'f_times'}{'%_ps'}{$i}{$hz}, $ps_total);
        }
      }
    }
  }
  # ------- freq trans table ---
  if (check_exists('freq_ttable'))
  {
    for (my $i = 0; $i < $cpuinfo->{'cpu_count'}; $i++)
    {
      if (check_exists('freq_ttable', $i))
      {
        my $oneprc = $cstats->{'f_ttable'}{'total'}{$i}/100;
        my $ps_total = 0;
        for my $from (keys %{$cstats->{'f_ttable'}{'values'}{$i}})
        {
          for my $to (keys %{$cstats->{'f_ttable'}{'values'}{$i}{$from}})
          {
            next if ($from eq $to);
            $result->{'f_ttable'}{$i}{$from}{$to} = divide($cstats->{'f_ttable'}{'values'}{$i}{$from}{$to}, $oneprc);
            $cstats->{'f_ttable'}{'%_ps'}{$i}{$from}{$to} = one_second_part($cstats->{'f_ttable'}{'values'}{$i}{$from}{$to}, $pstats->{'f_ttable'}{'values'}{$i}{$from}{$to}, $timediff);
            $ps_total += $cstats->{'f_ttable'}{'%_ps'}{$i}{$from}{$to};
          }
        }
        $ps_total = $ps_total/100;
        for my $from (keys %{$cstats->{'f_ttable'}{'values'}{$i}})
        {
          for my $to (keys %{$cstats->{'f_ttable'}{'values'}{$i}{$from}})
          {
            next if ($from eq $to);
            $result->{'f_ttable_ps'}{$i}{$from}{$to} = divide($cstats->{'f_ttable'}{'%_ps'}{$i}{$from}{$to}, $ps_total);
          }
        }
      }
    }
  }
  #print Dumper $result;
  return $result;
}

# ---------------------------------------- preparing values ------------------------------------
sub prepare_graphs_values
{
  my ($data) = $_[0];
  my $pg = {};
  # ------------------ cpu_utilisation -----------------------------------
  # ---------- general ----------------------
  for my $cname (@cnames) { $pg->{'cpu_utilisation'}{$cname} = $data->{'cpu_util'}{'%'}{'total'}{$cname}; }
  if(check_exists('freq_hz')) { for (my $i = 0; $i < $cpuinfo->{'cpu_count'}; $i++) { $pg->{'cpu_utilisation'}{sprintf("fp_%s", $i)} = $data->{'f_minmax'}{'%'}{$i}; } }
  # ---------------- childs -------------------
  if ($cpuinfo->{'cpu_count'} > 1)
  {
    for (my $i=0; $i < $cpuinfo->{'cpu_count'}; $i++)
    {
      my $graph_name = sprintf("cpu_utilisation.cpu%s", $i);
      for my $cname (@cnames) { $pg->{$graph_name}{$cname} = $data->{'cpu_util'}{'%'}{$i}{$cname}; }
      if(check_exists('freq_hz', $i)) { $pg->{$graph_name}{'fp'} = $data->{'f_minmax'}{'%'}{$i}; }
    }
  }
  
  
  # --------------- cpu_frequency --------------------------------------------
  if(check_exists('freq_trans'))
  { 
    # ------------ general --------------------
    for (my $i = 0; $i < $cpuinfo->{'cpu_count'}; $i++) { $pg->{'cpu_frequency'}{sprintf("cpu_%s", $i)} = $data->{'f_trans'}{$i} if (check_exists('freq_trans', $i)); }
    $pg->{'cpu_frequency'}{'total'} = $data->{'f_trans'}{'total'};
    # ---------------- childs -------------------
    if(check_exists('freq_times'))
    {
      for (my $i=0; $i < $cpuinfo->{'cpu_count'}; $i++)
      {
        if(check_exists('freq_times', $i))
        {
          my $graph_name = sprintf("cpu_frequency.percent_cpu%s", $i);
          for my $freq (keys %{$data->{'f_times'}{$i}}) { $pg->{$graph_name}{sprintf("hz_%s", $freq)} = $data->{'f_times'}{$i}{$freq}; }
          $graph_name = sprintf("cpu_frequency.percent_ps_cpu%s", $i);
          for my $freq (keys %{$data->{'f_times_ps'}{$i}}) { $pg->{$graph_name}{sprintf("hz_%s", $freq)} = $data->{'f_times_ps'}{$i}{$freq}; }
        }
      }
    }
    if(check_exists('freq_ttable'))
    {
      for (my $i=0; $i < $cpuinfo->{'cpu_count'}; $i++)
      {
        if(check_exists('freq_ttable', $i))
        {
          my $graph_name = sprintf("cpu_frequency.trans_table_cpu%s", $i);
          for my $from (keys %{$data->{'f_ttable'}{$i}})
          {
            for my $to (keys %{$data->{'f_ttable'}{$i}{$from}})
            {
              $pg->{$graph_name}{sprintf("f_%s_t_%s", $from, $to)} = $data->{'f_ttable'}{$i}{$from}{$to};
            }
          }
          $graph_name = sprintf("cpu_frequency.trans_table_ps_cpu%s", $i);
          for my $from (keys %{$data->{'f_ttable_ps'}{$i}})
          {
            for my $to (keys %{$data->{'f_ttable_ps'}{$i}{$from}})
            {
              $pg->{$graph_name}{sprintf("f_%s_t_%s", $from, $to)} = $data->{'f_ttable_ps'}{$i}{$from}{$to};
            }
          }
        }
      }
    }
  }

  # --------------- cpu_all --------------------------------------------
  if ($cpuinfo->{'cpu_count'} > 1)
  {
    for (my $i = 0; $i < $cpuinfo->{'cpu_count'}; $i++) { $pg->{'cpu_all'}{sprintf("cpu_%s", $i)} = $data->{'cpu_util'}{'used'}{$i}; }
    $pg->{'cpu_all'}{'total'} = $data->{'cpu_util'}{'used'}{'total'};
  }
  return $pg;
}



# -------------------------------- printing values -----------------------------------
sub print_values
{
  my $pstats = restore_state_data();
  my $cstats = load_stats();
  if (exists ($pstats->{'timestamp'}))
  {
    my $values = prepare_graphs_values(calculate($pstats, $cstats));
    #print Dumper $values;
    for my $g (sort keys %{$values})
    {
      printf("multigraph %s\n", $g);
      for my $f (sort keys %{$values->{$g}}) { printf("%s.value %s\n", $f, $values->{$g}{$f}); }
      print "\n";
    }
  }
}

__END__

- user: normal processes executing in user mode
- nice: niced processes executing in user mode
- system: processes executing in kernel mode
- idle: twiddling thumbs
- iowait: waiting for I/O to complete
- irq: servicing interrupts
- softirq: servicing softirqs
- steal: involuntary wait
- guest: running a guest