Attention: All pages of this wiki depend on the pages that come before it, in order as they are listed on the Main Page. Please check for Dependencies.
Please also look at What You Need to Know Before Using This Wiki

/root/scripts/bwrpt.pl

From COCNM
Revision as of 15:37, 27 April 2014 by Bob (talk | contribs)
Jump to navigation Jump to search
New Script - Please read the notes at the top as things have changed since bwmon.sh
#!/usr/bin/perl

# Written by Bob Miller - bob@computerisms.ca
# Converted from Bash Script Feb 2014 and
# modified to account for >4GB/hour internet connections

## This should cover any packages required on a standard debian box:
## apt-get install libfile-tail-perl libtext-trim-perl

## This script depends on configuration of the iptaccount module available in xtables-addons
## It assumes each subnet matches one physical interface which matches one iptaccount table name
## Some iptables rules like this should work:
## iptables -t filter -I FORWARD -j ACCOUNT --addr 192.168.25.0/24 --tname lan
## iptables -t mangle -I POSTROUTING -j ACCOUNT --addr 0/0 --tname wan

## Set cron to run this often enough that the $wrap variable is not exceeded in one run
## For example, on slow connections, this can be run once per hour, on fast connections every
## 10 minutes.  It needs to be run offset so that the reporting fires before midnight, something
## Like so will do (note busy systems may need to run 2 minutes before midnight to give the 
## script time to run instead of one minute):
## 9-59/10 * * * * PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin; /usr/bin/perl /root/scripts/bwrpt.pl

#bwrpt.pl
use strict;
use warnings;
use Time::Piece;
use Time::Seconds;
use File::Path qw(make_path);
use File::Tail;
use Text::Trim;
use Net::SMTP;

####################################################################################
####################################################################################
##                                                                                ##
##                          Configuration Options                                 ##
##                                                                                ##
####################################################################################
####################################################################################

## Configure WAN ehternet port and matching --tname from iptables
our $wanface = "eth0";
our $wanipt = "wan";

## From your iptables, list all --tname values here except the wan
our @iptname = ("computerisms", "computerisms-pub", "computerisms2remote");

## WISHLIST ITEM
## Enable full detail for per-host reporting (set to 0 to disable, set to 1 to enable)
## List each --tname you have in @iptname and enable/disable each one
#our %iptdetail = (
#	"computerisms" => 1,
#	"computerisms-pub" => 0,
#	"computerisms2remote" => 0,
#);

## Set the base directory to store daily data
our $base = "/root/scripts/bwdata";

## Set 32-bit system or 64-bit system
our $wrap = 4294967295; ## Uncomment this for 32-bit
#our $wrap = 18446744073709551615; ## Uncomment this for 64-bit

## Set these values as per the cronjob to the last run of the day to generate and send a report.
our $rpthour = "23";
our $rptmin = "59";

## Set Mega or Mibi
our $mb = "1000"; ## Uncomment this line for Mega
#our $mb = "1024"; ## Uncomment this line for Mibi

## Set this to the max bandwidth in GB(GiB) before incurring overage charges
our $maxbw = "90";

## Configure mail settings
our $smtpserver = "mail.computerisms.com";
our $mailfrom = "firewall\@computerisms.com";
our $mailto = "bob.miller\@computerisms.com";
our $mailhello = "firewall.computerisms.com";
our $smtpauthname = "smtpauth\@computerisms.com";
our $smtpauthpass = "xxxxxxxxx";


####################################################################################
####################################################################################
##                                                                                ##
##           You should not need to change anything below here                    ##
##                                                                                ##
####################################################################################
####################################################################################

## Populate time and date variables

our $date = Time::Piece->new;
our $time = $date->time;
our $hour = $date->hour;
our $min = $date->min;
our $month = $date->strftime('%Y/%m');
our $thisday = $date->strftime('%d');
our $today = $date->ymd("/");
our $yesterday = ($date - ONE_DAY)->ymd("/");

## Test if Today's directory exists, if not create it

if (!-d "$base"."/"."$today"."/") {
	make_path("$base"."/"."$today");
}

## Set System Executables

our $doipt = qx(which iptaccount) or die "cannot find iptaccount program $!";
our $donbt = qx(which nbtscan) or die "cannot find nbtscan program $!";
#our $doarp = qx(which arp) or die "cannot find arp program $!";
chomp ($doipt, $donbt);

## Now Collect iptaccount data for internal subnets

for my $curiptname (@iptname) {
	if (!-d "$base"."/"."$today"."/"."$curiptname"."/") {
		make_path("$base"."/"."$today"."/"."$curiptname");
	}
	my @ipt = qx($doipt -s -l $curiptname);
	my ($lastrun, $hostname);
	foreach my $i (@ipt) {
		next if $i =~ /^\D/;
		chomp $i;
		my @ip = split(';', $i);
		my $curfile="$base"."/"."$today"."/"."$curiptname"."/"."$ip[0]";
		if (!-e $curfile) {
#			open IP, ">> $curfile" or die $!;
#			print IP "timestamp iptin iptout insince outsince runttlin runttlout runttl wrapin wrapout hostname hostchange\n";
			if (-e "$base"."/"."$yesterday"."/"."$curiptname"."/"."$ip[0]") {
				my $lastfile = File::Tail->new(name => "$base"."/"."$yesterday"."/"."$curiptname"."/"."$ip[0]", tail => 1);
				$lastrun = $lastfile -> read;
				my @resetday = split (/ /, $lastrun);
				@resetday[5..9,11] = (0) x 6;
				$lastrun = "@resetday";
				chomp $lastrun;
			} else {
				$lastrun = "0 0 0 0 0 0 0 0 0 0 undef 0 ";
				chomp $lastrun;
			}
		} else {
			my $lastfile = File::Tail->new(name => "$base"."/"."$today"."/"."$curiptname"."/"."$ip[0]", tail => 1) or die $!;
			$lastrun = $lastfile -> read;
			chomp $lastrun;
		}
		my @lastrun = split(/ /, $lastrun);
		my ($sincelastin, $curttlin, $sincelastout, $curttlout);
		if ($lastrun[1] > $ip[2]) {
			$lastrun[8]++;
#			$sincelastin = $lastrun[8]*$wrap+$ip[2]-$lastrun[5];
			$sincelastin = $wrap+$ip[2]-$lastrun[1];
			$curttlin = $lastrun[5]+$sincelastin;
		} else {
			$sincelastin = $ip[2]-$lastrun[1];
			$curttlin = $lastrun[5]+$sincelastin;
		}
		if ($lastrun[2] > $ip[4]) {
			$lastrun[9]++;
#			$sincelastout = $lastrun[9]*$wrap+$ip[4]-$lastrun[6];
			$sincelastout = $wrap+$ip[4]-$lastrun[2];
			$curttlout = $lastrun[6]+$sincelastout;
		} else {
			$sincelastout = $ip[4]-$lastrun[2];
			$curttlout = $lastrun[6]+$sincelastout;
		}
		my $runttl = $curttlin+$curttlout;
		next if ( $sincelastin == 0 && $sincelastout == 0 );
		my @hostlook;
		my $lookup = qx($donbt -s : $ip[0]);
		if ( $lookup ne '' ) {
			my @hostlook = split (/:/, $lookup);
			$hostname = $hostlook[1];
			$hostname = trim($hostname);
			chomp $hostname;
		} else {
			$lookup = qx(grep $ip[0] /proc/net/arp);
			if ( $lookup ne '' ) {
				@hostlook = split (' ', $lookup);
				$hostname = $hostlook[3];
				$hostname = trim($hostname);
				chomp $hostname;
				$hostname=$lastrun[10] if ( $hostname =~ /incomplete/ ); 
				$hostname=$lastrun[10] if ( $hostname =~ /00:00:00:00:00:00/ ); 
			} else { 
				$hostname = "NOT DEFINABLE";
				chomp $hostname;
			}
		}
		if ( $hostname ne "$lastrun[10]" ) {
			$lastrun[11]++;
		}
		open IP, ">> $curfile" or die $!;
		if ( -z $curfile) {
			print IP "timestamp iptin iptout insince outsince runttlin runttlout runttl wrapin wrapout hostname hostchange\n";
		}
		print IP "$time $ip[2] $ip[4] $sincelastin $sincelastout $curttlin $curttlout $runttl $lastrun[8] $lastrun[9] $hostname $lastrun[11]\n";
	}
}

## Now do data for WAN port

my $curfile="$base"."/"."$today"."/"."$wanface";
my ($lastrun, $curwanipt, $sincelastipt, $ttlipt, $insince, $runttlin, $outsince, $runttlout);
if (!-e $curfile) {
	open WAN, ">> $curfile" or die $!;
	print WAN "timestamp ipt sysin sysout iptsince runipt insince outsince runttlin runttlout runttl wrapipt wrapin wrapout\n";
	if (-e "$base"."/"."$yesterday"."/"."$wanface") {
		my $lastfile = File::Tail->new(name => "$base"."/"."$yesterday"."/"."$wanface", tail => 1) or die $!;
		$lastrun = $lastfile -> read;
		my @resetday = split (/ /, $lastrun);
		@resetday[4..13] = (0) x 10;
		$lastrun = "@resetday";
		chomp $lastrun;
	} else {
		$lastrun = "0 0 0 0 0 0 0 0 0 0 0 0 0 0";
		chomp $lastrun;
	}
} else {
	my $lastfile = File::Tail->new(name => "$base"."/"."$today"."/"."$wanface", tail => 1) or die $!;
	$lastrun = $lastfile -> read;
	chomp $lastrun;
}
my @ipt = qx($doipt -s -l $wanipt);
foreach my $j (@ipt) {
	if ($j =~ /^\d/) {
		chomp $j;
		my @ipt = split(';', $j);
		$curwanipt =$ipt[2];
		last;
	}
}
open RX, "/sys/class/net/$wanface/statistics/rx_bytes" or die $!;
open TX, "/sys/class/net/$wanface/statistics/tx_bytes" or die $!;
my $sysin = <RX>;
my $sysout = <TX>;
chomp ($sysin, $sysout);
my @lastrun = split(/ /, $lastrun);
if ($lastrun[1] > $curwanipt) {
	$lastrun[11]++;
#	$sincelastipt = $lastrun[11]*$wrap+$curwanipt-$lastrun[5];
	$sincelastipt = $wrap+$curwanipt-$lastrun[1];
	$ttlipt = $lastrun[5]+$sincelastipt;
} else {
	$sincelastipt = $curwanipt-$lastrun[1];
	$ttlipt = $sincelastipt+$lastrun[5];
}
if ($lastrun[2] > $sysin) {
	$lastrun[12]++;
#	$insince = $lastrun[12]*$wrap+$sysin-$lastrun[8];
	$insince = $wrap+$sysin-$lastrun[2];
	$runttlin = $lastrun[8]+$insince;
} else {
	$insince = $sysin-$lastrun[2];
	$runttlin = $insince+$lastrun[8];
}
if ($lastrun[3] > $sysout) {
	$lastrun[13]++;
#	$outsince = $lastrun[13]*$wrap+$sysout-$lastrun[9];
	$outsince = $wrap+$sysout-$lastrun[3];
	$runttlout = $lastrun[9]+$outsince;
} else {
	$outsince = $sysout-$lastrun[3];
	$runttlout = $outsince+$lastrun[9];
}
my $runttl = $runttlin+$runttlout;
open WAN, ">>$curfile" or die $!;
print WAN "$time $curwanipt $sysin $sysout $sincelastipt $ttlipt $insince $outsince $runttlin $runttlout $runttl $lastrun[11] $lastrun[12] $lastrun[13]\n";
close WAN;

## We should have a pretty comprehensive data set now, let's generate a report:

if ( $hour == $rpthour && $min == $rptmin) {
	my ($wanfile, $mtdwan, $mtdlan, $runipt, $runsysin, $runsysout, $runsysttl, $mtdreadwan, $mtdreadlan, $day, $daysip,  $todayipt, $todaysysin, $todaysysout, $todaysysttl, $mtdttlin, $mtdttlout, $mtdttl);
	my @mtdreadwan;
	my @mtdreadlan;
	my %daily=();
	my $rptfile = "$base"."/"."$today"."/"."Daily.rpt";
	opendir (DAYS,"$base"."/"."$month") or die "Can't open days $!";
	while (my $day = readdir(DAYS)) {
		next if $day =~ /^\D/;
		my $wanfile = File::Tail->new(name => "$base"."/"."$month"."/"."$day"."/"."$wanface", tail => 1);
		$mtdwan = $wanfile -> read;	
		@mtdreadwan = split (/ /, $mtdwan);
		$runipt += $mtdreadwan[5];
		$runsysin += $mtdreadwan[8];
		$runsysout += $mtdreadwan[9];
		$runsysttl += $mtdreadwan[10];
		if ($day == $thisday) {
			$todayipt = $mtdreadwan[5];
			$todaysysin = $mtdreadwan[8];
			$todaysysout = $mtdreadwan[9];
			$todaysysttl = $mtdreadwan[10];
		}
		foreach my $i (@iptname) {
			next if (!-e "$base"."/"."$month"."/"."$day"."/"."$i");
			opendir (IPT, "$base"."/"."$month"."/"."$day"."/"."$i") or die "can't open $i days-iptname $!";
			while (my $daysip = readdir(IPT)) {
				next if $daysip =~ /^\D/;
				my $lanfile = File::Tail->new(name => "$base"."/"."$month"."/"."$day"."/"."$i"."/"."$daysip", tail => 1);
				$mtdlan = $lanfile -> read;
				@mtdreadlan = split (/ /, $mtdlan);
				$daily{$day}{$i}{$daysip}{ttlin} = $mtdreadlan[5];
				$daily{$day}{$i}{$daysip}{ttlout} = $mtdreadlan[6];
				$daily{$day}{$i}{$daysip}{ttl} = $mtdreadlan[7];
				$daily{$i}{$daysip}{mtdttlin} += $mtdreadlan[5];
				$daily{$i}{$daysip}{mtdttlout} += $mtdreadlan[6];
				$daily{$i}{$daysip}{mtdttl} += $mtdreadlan[7];
				$daily{$day}{$i}{$daysip}{devname} = $mtdreadlan[10];	
				$daily{$day}{$i}{$daysip}{devchange} = $mtdreadlan[11];	
				$daily{$i}{mtdin} += $daily{$day}{$i}{$daysip}{ttlin};
				$daily{$i}{mtdout} += $daily{$day}{$i}{$daysip}{ttlout};
				$daily{$i}{mtd} += $daily{$day}{$i}{$daysip}{ttl};
				if ( $day == $thisday ) {
					$daily{$i}{dailyttlin} += $mtdreadlan[5];
					$daily{$i}{dailyttlout} += $mtdreadlan[6];
					$daily{$i}{dailyttl} += $mtdreadlan[7];
				}
			}		
		}
	}
	my $percentage = int($runsysttl/($maxbw*$mb*$mb*$mb)*100);
	open RPT, ">> $rptfile" or die $!;
	print RPT "****************************************************************************\n";
	print RPT "****************************************************************************\n";
	print RPT "**                                                                        **\n";
	print RPT "**                     DAILY BANDWIDTH REPORT                             **\n";
	print RPT "**                  For day ending: $today                            **\n";
	print RPT "**                Explanations and Documentation:                         **\n";
	print RPT "**  http://help.computerisms.ca/index.php?title=Daily_Bandwidth_Report    **\n";
	print RPT "**                                                                        **\n";
	print RPT "****************************************************************************\n";
	print RPT "****************************************************************************\n";
	print RPT "\n";
	print RPT "\n";
	print RPT "============================================================================\n";
	print RPT "SUMMARY\n";
	print RPT "============================================================================\n";
	print RPT "\n";
	print RPT "                       Percentage of Allowed Bandwidth Used This Month:  $percentage%\n";
	print RPT "\n";
	printf (RPT "%-36s%17s%3s%17s%3s\n", "", "Today", "", "This Month", "");
	printf (RPT "%-36s%17s%3s%17s%3s\n", "Raw Data on External Port", int($todaysysttl/$mb/$mb), "MB", int($runsysttl/$mb/$mb), "MB");
	printf (RPT "%-36s%17s%3s%17s%3s\n", "Accounted Data on External Port", int($todayipt/$mb/$mb), "MB", int($runipt/$mb/$mb), "MB");
	print RPT "\n";
	foreach my $i (@iptname) {
		if ( ! defined $daily{$i}{mtd} ) { 
			$daily{$i}{mtd} = 0;
		}
		if ( ! defined $daily{$i}{dailyttl} ) {
			$daily{$i}{dailyttl} = 0;
		}
		printf (RPT "%-36s%17s%3s%17s%3s\n", "Accounted Data on $i", int($daily{$i}{dailyttl}/$mb/$mb), "MB", int($daily{$i}{mtd}/$mb/$mb), "MB");
	}
	print RPT "\n";
	print RPT "============================================================================\n";
	print RPT "INTERNET USAGE - DETAILED REPORT\n";
	print RPT "============================================================================\n";
	print RPT "\n";
	printf (RPT "%-36s%17s%3s%17s%3s\n", "", "Today", "", "This Month", "");
	printf (RPT "%-36s%17s%3s%17s%3s\n", "Raw Data - Download ", int($todaysysin/$mb/$mb), "MB", int($runsysin/$mb/$mb), "MB");
	printf (RPT "%-36s%17s%3s%17s%3s\n", "Raw Data - Upload", int($todaysysout/$mb/$mb), "MB", int($runsysout/$mb/$mb), "MB");
	printf (RPT "%-36s%17s%3s%17s%3s\n", "Raw Data - Total Usage", int($todaysysttl/$mb/$mb), "MB", int($runsysttl/$mb/$mb), "MB");
	printf (RPT "%-36s%17s%3s%17s%3s\n", "Accounted Data - Total Usage", int($todayipt/$mb/$mb), "MB", int($runipt/$mb/$mb), "MB");
	print RPT "\n";
	foreach my $i (@iptname) {
		my $uci = uc $i;
		print RPT "============================================================================\n";
		print RPT "$uci USAGE - DETAILED REPORT\n";
		print RPT "============================================================================\n";
		print RPT "\n";
		printf (RPT "%-36s%17s%3s%17s%3s\n", "", "Today", "", "This Month", "");
		for my $def ( $daily{$i}{dailyttlin}, $daily{$i}{mtdin}, $daily{$i}{dailyttlout}, $daily{$i}{mtdout}, $daily{$i}{dailyttl}, $daily{$i}{mtd} ) {
			if ( ! defined $def ) { 
				$def = 0;
			}
		}
		printf (RPT "%-36s%17s%3s%17s%3s\n", "Accounted Data - Download ", int($daily{$i}{dailyttlout}/$mb/$mb), "MB", int($daily{$i}{mtdout}/$mb/$mb), "MB");
		printf (RPT "%-36s%17s%3s%17s%3s\n", "Accounted Data - Upload ", int($daily{$i}{dailyttlin}/$mb/$mb), "MB", int($daily{$i}{mtdin}/$mb/$mb), "MB");
		printf (RPT "%-36s%17s%3s%17s%3s\n", "Accounted Data - Total Usage ", int($daily{$i}{dailyttl}/$mb/$mb), "MB", int($daily{$i}{mtd}/$mb/$mb), "MB");
		print RPT "\n";
	}
	foreach my $i ( keys %{$daily{$thisday}} ) {
		my $uci = uc $i;
		print RPT "============================================================================\n";
		print RPT "$uci HOSTS - SUMMARY REPORT - VALUES IN MB\n";
		print RPT "============================================================================\n";
		print RPT "\n";
		printf (RPT "%-15s%6s%6s%6s%6s%6s%6s%19s%6s\n", "IP ADDR", "DAYDN", "DAYUP", "DAYTL", "MTDDN", "MTDUP", "MTDTL", "Last Seen Host  ", "Host #");
#		if ( keys %{$daily{$thisday}{$i}} ) {
			foreach my $ips (sort ( keys %{$daily{$thisday}{$i}} ) ) {
				printf (RPT "%-15s%6s%6s%6s%6s%6s%6s%19s%6s\n", $ips, int($daily{$thisday}{$i}{$ips}{ttlout}/$mb/$mb), int($daily{$thisday}{$i}{$ips}{ttlin}/$mb/$mb), int($daily{$thisday}{$i}{$ips}{ttl}/$mb/$mb), int($daily{$i}{$ips}{mtdttlout}/$mb/$mb), int($daily{$i}{$ips}{mtdttlin}/$mb/$mb), int($daily{$i}{$ips}{mtdttl}/$mb/$mb), $daily{$thisday}{$i}{$ips}{devname}, $daily{$thisday}{$i}{$ips}{devchange});
			}
#		}
	}
	close (RPT);
## Send the report

	open DRPT, "< $rptfile";
	my $dailyreport = do { local $/; <DRPT> };

	my $smtp = Net::SMTP->new ( "$smtpserver", Hello => "$mailhello" );
	$smtp->auth ("$smtpauthname","$smtpauthpass");
	$smtp->mail ("$mailfrom");
	$smtp->recipient ("$mailto", "$mailto2");
	$smtp->data();
	$smtp->datasend("From: $mailfrom\n");
	$smtp->datasend("To: $mailto, $mailto2\n");
	$smtp->datasend("Subject: Daily BWRprt - Monthly usage is $percentage %\n");
	$smtp->datasend("Date: $date\n");
	$smtp->datasend("\n");
	$smtp->datasend("$dailyreport\n");
	$smtp->dataend();
	$smtp->quit;
}