#! /usr/bin/perl

###
### check4spikes - Takes a total count of all events for each host,
### then waits for 1 second, performs the same check again. then
### calculates the difference in time to provide a list and count
### of the top 10 hosts that are spiking splunk with the largest
### number of events per second
###
### by Kirk Waingrow 2009
###
$AUTH="-auth admin:admin";
$STATDIR="/home/splunk/status";
$SPLUNK="/opt/splunk/bin/splunk";
$TMP="/tmp";


$LCK="/tmp/check4spikes.lck";
if ( -f "$LCK" ) { exit; }
`touch $LCK`;

sub collect_events
{
  @events=`$SPLUNK dispatch '| metadata type=hosts | sort totalCount d | fields host,totalCount' $AUTH`;
  $x=0;
  for $e (@events)
  {
    chomp($e);
    $num= int( ( split ' ', $events[$x])[1] );
    $strg= ( split ' ', $events[$x])[0] ;
    if ($num != 0) 
    { push(@node, "$strg"); push(@count, "$num"); 
      #print "$strg - $num\n"; 
    }
    $x++;
  }
}



@node=""; @count="";
collect_events; @OLDNODE=@node; @OLDCOUNT=@count;
$stime=`date +%s`; chomp($stime);

@node=""; @count="";
`sleep 1`;
collect_events; 
$etime=`date +%s`; chomp($stime);
$ltime=$etime-$stime;

$x=0;
open(OUT, "> /tmp/spikes.txt");
for $e (@node)
{ 
  if ( "$node[$x]" eq "$OLDNODE[$x]" )
  {
    $total=int ( ($count[$x] - $OLDCOUNT[$x])/$ltime);
    print OUT "$total:$node[$x]:q=host%3d\"$node[$x]\"%20minutesago%3d2\n";
  }
  $x++;
}

`sort -rn /tmp/spikes.txt | head -10 > $STATDIR/splunk_event_spikes.txt`;

`rm  /tmp/spikes.txt $LCK`;