#!/usr/bin/perl -w
use strict;
use Data::Dumper;
use Getopt::Long;
use YAML qw( Dump LoadFile);
use Try::Tiny;
use IPC::Open3;
use Sys::Hostname;
use JSON;

#we don't run at the same time as ourselves
use Proc::PID::File;
if (Proc::PID::File->running()){
  print "$0 is already running!\n";
  exit();
}

my $jobsfile = "/var/lib/backup/backup.jobs";
my $cachefile = "/opt/eig_linux/data/backupstate.data";
my $backupdatecache = "/opt/eig_linux/data/backupdates.yaml";
my $quick = 0;
my $full = 0;
my $fix = 0;
my $verbose = 0;
my $submit = 0;
my $updatecache = 0;
my $cache = {};
GetOptions(
        "quick"     =>      \$quick,
        "full"      =>      \$full,
        "fix"       =>      \$fix,
        "verbose"   =>      \$verbose,
        "submit"    =>      \$submit,
        "updatecache"       =>      \$updatecache,
);
my $systemdata = {};

resumeifboxisrebooted();

if ( -f "/opt/eig_linux/data/backups/dirs.files.lastrun" ) {
  checklastmodified(\$systemdata,'dirs.files.lastrun-date','/opt/eig_linux/data/backups/dirs.files.lastrun');
  checklastmodified(\$systemdata,'dirs.files.lastrun-epoch','/opt/eig_linux/data/backups/dirs.files.lastrun');
} else {
  checklastmodified(\$systemdata,'dirs.files.lastrun-date','/usr/sec/data/backups/dirs.files.lastrun');
  checklastmodified(\$systemdata,'dirs.files.lastrun-epoch','/usr/sec/data/backups/dirs.files.lastrun');
}
checklastmodified(\$systemdata,'postbackup.log-date','/var/log/postbackup.log');
checklastmodified(\$systemdata,'postbackup.log-epoch','/var/log/postbackup.log');
if ( -f "/opt/eig_linux/data/backups/backupmysql.lastrun" ) {
  checklastmodified(\$systemdata,'mysql-backup.lastrun-date','/opt/eig_linux/data/backups/backupmysql.lastrun');
  checklastmodified(\$systemdata,'mysql-backup.lastrun-epoch','/opt/eig_linux/data/backups/backupmysql.lastrun');
} else {
  checklastmodified(\$systemdata,'mysql-backup.lastrun-date','/usr/sec/data/backups/backupmysql.lastrun');
  checklastmodified(\$systemdata,'mysql-backup.lastrun-epoch','/usr/sec/data/backups/backupmysql.lastrun');
}

check_fc(\$systemdata);

my $userdata = {};
open(PASSWD, "</etc/passwd") or die("failed to open passwd: $!");
while(<PASSWD>){
  my @ent = split(':', $_);
  if($ent[5] =~ m#^/home(\d*)#){
    $userdata->{$ent[0]}->{homedir} = $ent[5];
    if($1 eq ""){
      $userdata->{$ent[0]}->{homenum} = "";
    }else{
      $userdata->{$ent[0]}->{homenum} = $1;
    }
  }
}
close(PASSWD);

my %users;
my %cpanelusers;
$systemdata->{"cpanel-accounts-total"} = 0;
open(my $DAT, '<', '/etc/trueuserdomains');
while(<$DAT>) {
 chomp();
  if ( /^.*:\s(.*)$/ ) {
	my $user = $1;
    next if $user eq 'root';
    next if $user eq 'system';
    next if ( -e "/var/cpanel/suspended/$user" );
    $systemdata->{"cpanel-accounts-total"}++ if (! $users{$user}); 
    $users{$user} = 1;
    $cpanelusers{$user} = 1;
  }
}
close($DAT);
checkrpmversion(\$systemdata);

checkbackuphelperlastuser(\$systemdata);

$systemdata->{"backup-dirs-completed"} = 0;
$systemdata->{"backup-dirs-uncompleted"} = 0;

systemstatus(\$systemdata);
systeminfo(\$systemdata);
print_data($systemdata) if $quick;
exit if $quick;

checkbackupmounts(\$systemdata);

checkpkgacctrunner(\%cpanelusers,\$userdata,\$systemdata);

partitiondata(\$systemdata);

#print the data we have collected
if($submit){
  submitdata(\$systemdata);
}else{
  print_data($systemdata);
}
update_cache($cachefile, $systemdata);
exit unless $full;

foreach my $user (keys %users){
  checkbackup($user, \$userdata);
}
printbackupstats(\$userdata,\%users);


#### only subs below this line
sub print_data {
  my $systemdata = shift;
  foreach my $key (sort keys %$systemdata){
    if($key =~ m/date$/){
      my $date = localtime($systemdata->{$key});
      printf("%-40s %-40s\n",$key,$date);
    }else{
      printf("%-40s %-40s\n",$key,$systemdata->{$key});
    }
  }
}

sub check_fc {
  my $response_ref = shift;
  if( -e "/sys/class/fc_host"){
    my $has_non_soft_raid = 0;
    my $has_old = 0;
    open(MNTS, "</proc/mounts");
    while(<MNTS>){
      my @p = split('\s+');
      if($p[1] =~ m#^/backup#){
        if(substr($p[1],-4,4) eq ".old"){
          $has_old = 1;
          next;
        }
        if(substr($p[0],0,7) ne "/dev/md"){
          $has_non_soft_raid = 1;
        }
      }
    }
    close(MNTS);
    if($has_non_soft_raid){
      $$response_ref->{'has-fc'} = 'part';
    }else{
      $$response_ref->{'has-fc'} = 'yes';
    }
    if($has_old){
       $$response_ref->{'has-fc'} .= '.o';
    }
  }else{
    $$response_ref->{'has-fc'} = 'no';
  }
}

sub submitdata {
  my $response_ref = shift;
  my $hostname = `hostname`;
  chomp($hostname);
  $$response_ref->{hostname} = $hostname;
  my $message;
  foreach my $key (sort keys %{$$response_ref}){
    $message .= "$key,".$$response_ref->{$key}."\n";
  }
  require LWP::UserAgent;
  my $ua = LWP::UserAgent->new;
  $ua->agent("backupstate/0.1 ");

  my $req = HTTP::Request->new(POST => 'http://74.220.198.194/cgi-bin/backupstate.cgi');
  $req->content_type('application/x-www-form-urlencoded');
  $req->content("data=$message");

  my $res = $ua->request($req);
  my $response = $res->content;
  print "response from server: $response\n" if $verbose;
  unless($response =~ m/report accepted/){
    print "backupmaster didnt accept my report!\n" if $verbose;
  }
}

sub partitiondata {
  my $response_ref = shift;
  my @space = `/bin/df -P`;
  my $tmp = {};
  foreach(@space){
    next unless $_ =~ m#/(home|backup)(\d)#;
    my $type = $1;
    my $part = $2;
    my @p = split('\s+');
    $tmp->{$part}->{$type}->{'size'} = $p[1];
    $tmp->{$part}->{$type}->{'used'} = $p[2];
    if($type eq "home" and -e "/admin/mig$part/df.output"){
      open(DF, "</admin/mig$part/df.output");
      <DF>;
      my $sourcespace = <DF>;
      close(DF);
      my @s = split('\s+', $sourcespace);
      $tmp->{$part}->{$type}->{'used'} = $s[2] if ($s[2] > $p[2]);
    }
  }
  foreach my $key (keys %$tmp){
    $tmp->{$key}->{'home'}->{'size'} = 0 unless $tmp->{$key}->{'home'}->{'size'};
    $tmp->{$key}->{'home'}->{'used'} = 0 unless $tmp->{$key}->{'home'}->{'used'};
    $tmp->{$key}->{'backup'}->{'size'} = 0 unless $tmp->{$key}->{'backup'}->{'size'};
    $tmp->{$key}->{'backup'}->{'used'} = 0 unless $tmp->{$key}->{'backup'}->{'used'};
    $$response_ref->{"partition-data-$key"} = join('-',($tmp->{$key}->{'home'}->{'size'},$tmp->{$key}->{'home'}->{'used'},$tmp->{$key}->{'backup'}->{'size'},$tmp->{$key}->{'backup'}->{'used'}));
  }
}

sub checklastmodified {
  my $response_ref = shift;
  my $label = shift;
  my $file = shift;
  my @stats = stat($file);
  @stats = stat("$file.1") if( -e "$file.1" and $stats[7] == 0);
  @stats = stat("$file.2") if( -e "$file.2" and $stats[7] == 0);
  @stats = stat("$file.3") if( -e "$file.3" and $stats[7] == 0);
  @stats = stat("$file.4") if( -e "$file.4" and $stats[7] == 0);
  $$response_ref->{$label} = 0;
  $$response_ref->{$label} = $stats[9] if $stats[9];
}

sub systemstatus {
  my $response_ref = shift;
  $$response_ref->{"backup-status"} = "";
  $$response_ref->{"backup-stage"} = "";

  opendir my $dh, "/var/spool/eig_backup";
  my @spool = grep { ! m/\.skip$/ && !  m/^\./ } readdir($dh);
  closedir $dh;
  my $jobs_in_queue = @spool; 
  my $failed_count = 0;
  for my $user (@spool){
      open(my $fh, '<', "/var/spool/eig_backup/$user") or next;
      if (defined (my $count = <$fh>)){
        chomp($count);
        $failed_count++ if $count >= 5;
      }
      close $fh;
  }
  my $failed_string = $failed_count ? " Failed:$failed_count" : "";

  if ( `ps fauwwx | grep datacycl[e]` ) {
    $$response_ref->{"backup-status"} = "Datacycle running";
  }
  if ( -f "/var/run/queuer.pid" ) {
    $$response_ref->{"backup-stage"} = "Active: $jobs_in_queue jobs running$failed_string";
  } elsif ( $jobs_in_queue > 0 ) {
    $$response_ref->{"backup-stage"} = "Pending: $jobs_in_queue jobs queued$failed_string";
  } else {
    $$response_ref->{"backup-stage"} = "No jobs.";
  }
  if ( -f "/var/run/backup_cpanel.pid" ) {
    $$response_ref->{"backup-stage"} = "Active: backup_cpanel";
  } elsif ( -f "/var/run/backup-non-user-mysql" ) {
    $$response_ref->{"backup-stage"} = "Active: non-user-mysql";
  }
  if ( -f "/etc/disable_backups" ) {
    $$response_ref->{"backup-stage"} = "Backups Disabled";
  }
  if ( check_complete() && $$response_ref->{"backup-stage"} !~ m/^Active/ && !-f "/etc/disable_backups") {
    $$response_ref->{"backup-stage"} = "Incomplete";
  }
}

sub systeminfo {
  my $response_ref = shift;

  chomp(my $nproc = qx|nproc|);

  my $conf_file = q|/opt/eig_linux/etc/backup_server_info.conf|;
  my $fstab = q|/etc/fstab|;
  my $backup_boxes = [];
  try {
    open my $FILE, '<', $conf_file or die;
    while (my $line = <$FILE>) {
      chomp $line;
      if ($line =~ 'backup_hostname') {
        push @$backup_boxes, (split /: /, $line)[1];
      }
    }
  } catch {
    open my $FILE, '<', $fstab or die;
    while (my $line = <$FILE>) {
      chomp $line;
      if ($line =~ 'nfs4') {
        push @$backup_boxes, (split /:/, $line)[0];
        last;
      }
    }
  };

  my $meminfo = q|/proc/meminfo|;
  my $ram;
  try {
    open my $FILE, '<', $meminfo or die;
    while (my $line = <$FILE>) {
      chomp $line;
      if ($line =~ 'MemTotal') {
        $ram = (split /\s+/, $line)[1];
        last;
      }
    }
  } catch {
    chomp ($ram = qx(free -k | awk '/Mem:/ {print \$2}'));
  };

  $$response_ref->{nproc} = $nproc;
  $$response_ref->{'target-box'} = join ';', @$backup_boxes;
  $$response_ref->{ram} = $ram;
}

sub checkrpmversion {
  my $response_ref = shift;
  my $version = $1 if ( `rpm -q eig_backup` =~ m/(eig_backup-.*)\.el\d+\.noarch/ );
  chomp $version;
  $$response_ref->{"rpm-version"} = $version;
}

sub checkbackuphelperlastuser {
  my $response_ref = shift;
  if ( -f "/var/log/eig_backup/backuphelper_last_command" ) {
    my ($lastuser,$lastcmd) = ($1,$2) if ( `tail -n 1 /var/log/eig_backup/backuphelper_last_command` =~ m/^(\w+)\:(\w+)\:.*$/ );
    $lastuser = '' unless ( $lastuser );
    $lastcmd = '' unless ( $lastcmd );
    chomp $lastuser;
    chomp $lastcmd;
    $$response_ref->{"backuphelper-lastuser"} = $lastuser;
    $$response_ref->{"backuphelper-lastcmd"} = $lastcmd;
  }
}

sub checkpkgacctrunner {
  my $users_ref = shift;
  my $userdata_ref = shift;
  my $response_ref = shift;
  my $now = time;
  my $twodays = $now - (60 * 60 * 24 * 2);
  my $oneweek = $now - (60 * 60 * 24 * 7);
  my @days = qw(one-week six-days five-days four-days three-days two-days one-day);
  $$response_ref->{"cpanel-accounts-$_-behind"} = 0 foreach @days; 
  $$response_ref->{"cpanel-accounts-with-backups"} = 0;
  $$response_ref->{"cpanel-accounts-not-backuped-up"} = 0;
  $$response_ref->{"backups-daily"} = 0;
  $$response_ref->{"backups-weekly"} = 0;
  $$response_ref->{"backups-monthly"} = 0;
  my $backupdates = {};
  if (! $updatecache && -f "$backupdatecache" && ((stat "$backupdatecache")[9]) >= time - (60*60*13)){
    print "Loading backup dates from cache\n" if $verbose;
    $backupdates = LoadFile("$backupdatecache") || getbackupdates();
  }else{
    $backupdates = getbackupdates();
  }
  foreach my $user ( keys %{$users_ref}){
    my $verbose_messaged = 0;
    my $mostrecent = getmostrecentbackupdate($user,$response_ref,$backupdates);
    if($mostrecent == 0 || ! defined $backupdates->{$user}->{"dc_check"} ){
      print "$user does not have cpanel account backup\n" if ($verbose && $mostrecent == 0);
      print "$user does not have datacycle check file\n" if ($verbose && ! defined $backupdates->{$user}->{"dc_check"});
      $$response_ref->{"cpanel-accounts-not-backuped-up"}++; 
    }else{
      my $dc_check = ($backupdates->{$user}->{"dc_check"} + (60*60*12)); #buffer by 12 hours as cron only runs twice a day that sets them
      my $oldness = time - (60 * 60 * 24 * 7);
      foreach my $day (@days){
        if ($mostrecent < $oldness || $dc_check < $oldness){
          $$response_ref->{"cpanel-accounts-$day-behind"}++;
          if ($verbose && ! $verbose_messaged){
             print "$user $day behind on cpanel account\n" if ($mostrecent < $oldness);
             print "$user $day behind on datacycle check\n" if ($dc_check < $oldness);
             $verbose_messaged = 1; 
          }
        }
        $oldness += (60 * 60 * 24);
      }
      $$response_ref->{"cpanel-accounts-with-backups"}++;
    }
  }
}

sub getbackupdates{
    my $script=<<'EOF';
#!/usr/bin/env perl
use JSON;
EOF
    $script .= 'my $home = "/home/' . hostname . "\";\n";
    $script .=<<'EOF';
my $return = {};
opendir my $dh, $home;
my @backupdirs = grep {-d "$home/$_" && m/^backup/}  readdir($dh);
closedir $dh;
foreach my $backupdir (@backupdirs) {
    foreach my $ltype ( "daily", "weekly", "monthly" ) {
    	opendir $dh, "$home/$backupdir/cpbackup/$ltype";
    	foreach my $user (grep {-d "$home/$backupdir/cpbackup/$ltype/$_" && not m/^\./} readdir($dh)){
		if ($ltype eq "daily"){
		    $return->{"$user"}->{"dc_check"} = (stat("$home/$backupdir/cpbackup/$ltype/$user/homedir/.cpanel/backup_check"))[9];
		}
        	$return->{"$user"}->{"$ltype"} = (stat("$home/$backupdir/cpbackup/$ltype/$user/lastgenerated"))[9];
	}
    }
}
my $json=eval { encode_json($return) } || {};
print "$json";
EOF
    my $dates =  ssh_script($script);
    open (FH, ">$backupdatecache" ) or die "Can't open $backupdatecache : $! \n";
    print FH Dump( $dates );
    close FH;
    return $dates; 
}

sub ssh_script{
    my $script = shift;
    my $backup_server_config = LoadFile( '/opt/eig_linux/etc/backup_server_info.conf');
    die "Could not find backup host!" unless $backup_server_config->{backup_ip};
    my $backupkey = (-f '/admin/hgbackupdir/backupkey') ? '/admin/hgbackupdir/backupkey' : '/home/hgbackupdir/backupkey';
    my $proc="ssh -i $backupkey  -q -o LogLevel=QUIET -o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=15 -l root $backup_server_config->{backup_ip} 'cat | perl'";
    my ($stdout, $stderr, $exit_code);
    my $pid=open3(\*IN, \*OUT, \*ERR, $proc);
    print IN $script;
    close IN;
    chomp ($stdout = join '', <OUT>);
    close OUT;
    chomp ($stderr = join '', <ERR>);
    close ERR;
    waitpid($pid, 0);
    $exit_code=$? >> 8;
    die "SSH failed: $stderr\n" if ($exit_code);
    my $stat=eval { decode_json($stdout) } || {};
    return $stat;
}

sub getmostrecentbackupdate {
  my $user = shift;
  my $data_ref = shift;
  my $backupdates = shift;
  my $mostrecent = 0;
  foreach my $type (qw{daily weekly monthly}){
    if ( $type =~ m/^daily$/ and my $moddate = $backupdates->{"$user"}->{"$type"} ||  ((stat "/opt/eig_linux/data/lastgenerated/$user" )[9])) {
      $mostrecent = $moddate if($moddate > $mostrecent);
      $$data_ref->{"backups-$type"}++;
    } else {
      if (my $moddate = $backupdates->{"$user"}->{"$type"} || ((stat "/backup/cpbackup/$type/$user/lastgenerated")[9])){
        $mostrecent = $moddate if ($moddate > $mostrecent);
        $$data_ref->{"backups-$type"}++;
      }
    }
  }
  return $mostrecent;
}

sub checkbackupmounts {
  my $response_ref = shift;
  my $mountpointproblems = 0;
  my @homefstab = `grep /home /etc/fstab`;
  my %homes;
  foreach(@homefstab){
    chomp;
    next if m/^\s*#/;
    next if m/swap/;
    my @parts = split('\s+');
    $homes{$parts[1]} = 1;
  }
  my @fstab = `grep ' /backup' /etc/fstab`;
  my %should;
  foreach(@fstab){
    chomp;
    next if m/^\s*#/;
    my @parts = split('\s+');
    $should{$parts[1]} = 1;
  }
  foreach my $home (keys %homes){
    my $h = $home;
    $h =~ s#/home#/backup#;
    unless ($should{$h}){
      print "mountpoint: There is no $h mount for $home\n";
      $mountpointproblems++;
    }
  }
  my @mounts = `grep " /backup" /proc/mounts |grep -v fake`;
  if(@mounts != 0){
    foreach(@mounts){
      chomp;
      my @parts = split('\s+');
      if($should{$parts[1]} >= 1){
	$should{$parts[1]} = 2;
      }else{
	print "mountpoint: ".$parts[1]." should not be mounted?\n";
	$mountpointproblems++;
      }
    }
  }
  foreach(keys %should){
    if($should{$_} == 1){
      print "mountpoint: $_ not mounted\n";
      $mountpointproblems++;
    }elsif($should{$_} == 2){
      #this is the expected state
    }else{
      print "mountpoint: $_ uncaught state, let robert know about this!\n";
      $mountpointproblems++;
    }
  }
  if(@mounts == 0){
    unless($mountpointproblems){
      print "mountpoint: no backups mounted and no /backup in fstab\n";
      $mountpointproblems = 1;
    }
  }
  $$response_ref->{"mountpoint-problems"} = $mountpointproblems;
}

sub printbackupstats {
  my $userdata_ref = shift;
  my $users_ref = shift;
  foreach my $username (keys %$users_ref){
    foreach(qw{ numcpbackups numhomedirbackups numbackups} ){
      if(defined($$userdata_ref->{$username}->{$_})){
        $$userdata_ref->{TOTALS}->{"userswith-".$$userdata_ref->{$username}->{$_}."-$_"}++;
      }else{
        $$userdata_ref->{TOTALS}->{"userswith-0-$_"}++;
      }
    }
  }
  #  foreach my $num (0 ... 4){
  foreach my $type (qw{ numcpbackups numhomedirbackups numbackups} ){
    foreach my $num (0 ... 4){
      my $key = "userswith-$num-$type";
      print "$num $type ".$$userdata_ref->{TOTALS}->{$key}."\n" if defined($$userdata_ref->{TOTALS}->{$key});
    }
  }
}

sub checkbackup {
  my $username = shift;
  my $userdata_ref = shift;

#  checkbackuptype($username, $userdata_ref, "daily");
  foreach my $type ("daily", "weekly", "monthly"){
#    print "\t$type\n";
    checkbackuptype($username, $userdata_ref, $type);
  }
}

sub checkbackuptype {
  my $username = shift;
  my $userdata_ref = shift;
  my $type = shift;
  unless(-e "/backup".$$userdata_ref->{$username}->{homenum}."/cpbackup/$type/$username"){
    return 0;#    print "$username does not have a $type backup\n";
  }
  my @bstat = stat("/backup/cpbackup/$type/$username");
  if($$userdata_ref->{$username}->{homenum} ne ""){
    my @bnumstat = stat("/backup".$$userdata_ref->{$username}->{homenum}."/cpbackup/$type/$username");
    unless($bstat[0] == $bnumstat[0] and $bstat[1] == $bnumstat[1]){
      print "$username: /backup does not link properly to /backup".$$userdata_ref->{$username}->{homenum}."\n";
    }
  }
  $$userdata_ref->{$username}->{"cpbackup$type"} = 1 if( -d "/backup".$$userdata_ref->{$username}->{homenum}."/cpbackup/$type/$username/mysql");
  $$userdata_ref->{$username}->{numcpbackups}++ if( $$userdata_ref->{$username}->{"cpbackup$type"} );
  $$userdata_ref->{$username}->{"homedirbackup$type"} = 1 if( -d "/backup".$$userdata_ref->{$username}->{homenum}."/cpbackup/$type/$username/homedir");
  $$userdata_ref->{$username}->{numhomedirbackups}++ if( $$userdata_ref->{$username}->{"homedirbackup$type"} );
  $$userdata_ref->{$username}->{"date$type"} =  $bstat[9] if $$userdata_ref->{$username}->{"cpbackup$type"} and $$userdata_ref->{$username}->{"newbackup$type"};
  $$userdata_ref->{$username}->{numbackups}++;
}

sub resumeifboxisrebooted {
  my $response = `cat /proc/uptime`;
  my @p = split('\s+', $response);
  if($p[0] > 900 and $p[1] < 1800){
    print "resuming backups\n";
    system("/usr/sbin/backupctl --resume");
  }
}

sub check_complete {
  system("/opt/eig_linux/bin/check_backups");
  return "Incomplete Backups" if ( $? != 0 );
  return 0;
}

sub update_cache {
    my $cache_file = shift;
    my $userdata_ref = shift;
    open (FH, ">$cache_file" ) or die "Can't open $cachefile : $! \n";         
    print FH Dump( $userdata_ref );
    close FH; 
}
