#!/usr/bin/perl -w
use strict;
use Getopt::Long;
use YAML qw( Dump LoadFile);
use Try::Tiny;
use Sys::Hostname;
use JSON::XS;
use IO::Socket::INET;

my $eig_backup_config_file = '/opt/eig_linux/etc/eig_backup.conf';

my %eig_backup_config;
my $hgbackupdir = "/admin/hgbackupdir";
open my $config, '<', $eig_backup_config_file
    or die "Unable to open EIG Backup config file $eig_backup_config_file for reading: $!";
while (my $ol = <$config>) {
    chomp($ol);
    $eig_backup_config{$1} = $2 if $ol =~ m/^([^=]+?)=(.+)$/
}
close $config;

my $backupmaster_uri = $eig_backup_config{backupmaster_uri} ? $eig_backup_config{backupmaster_uri} : 'https://backupmaster.bluehost.com:1443/api/';

#we don't run at the same time as ourselves
use Proc::PID::File;
if (Proc::PID::File->running()){
    print "$0 is already running!\n";
    exit();
}

my $jobsfile = "/var/lib/backup/backup.jobs";
my $cachefile = "/opt/eig_linux/data/backupstate.data";
my $backupdatecache = "/opt/eig_linux/data/backupdates.yaml";
my $quick = 0;
my $full = 0;
my $fix = 0;
my $verbose = 0;
my $submit = 0;
my $updatecache = 0;
my $hostname = hostname;
my $cache = -f $cachefile ? LoadFile($cachefile) : {};
GetOptions(
        "quick"     =>      \$quick,
        "full"      =>      \$full,
        "fix"       =>      \$fix,
        "verbose"   =>      \$verbose,
        "submit"    =>      \$submit,
        "updatecache"       =>      \$updatecache,
);
my $systemdata = {};

resumeifboxisrebooted();

if ( -f "/opt/eig_linux/data/backups/dirs.files.lastrun" ) {
    checklastmodified(\$systemdata,'dirs.files.lastrun-date','/opt/eig_linux/data/backups/dirs.files.lastrun');
    checklastmodified(\$systemdata,'dirs.files.lastrun-epoch','/opt/eig_linux/data/backups/dirs.files.lastrun');
} else {
    checklastmodified(\$systemdata,'dirs.files.lastrun-date','/usr/sec/data/backups/dirs.files.lastrun');
    checklastmodified(\$systemdata,'dirs.files.lastrun-epoch','/usr/sec/data/backups/dirs.files.lastrun');
}
checklastmodified(\$systemdata,'postbackup.log-date','/var/log/postbackup.log');
checklastmodified(\$systemdata,'postbackup.log-epoch','/var/log/postbackup.log');
if ( -f "/opt/eig_linux/data/backups/backupmysql.lastrun" ) {
    checklastmodified(\$systemdata,'mysql-backup.lastrun-date','/opt/eig_linux/data/backups/backupmysql.lastrun');
    checklastmodified(\$systemdata,'mysql-backup.lastrun-epoch','/opt/eig_linux/data/backups/backupmysql.lastrun');
} else {
    checklastmodified(\$systemdata,'mysql-backup.lastrun-date','/usr/sec/data/backups/backupmysql.lastrun');
    checklastmodified(\$systemdata,'mysql-backup.lastrun-epoch','/usr/sec/data/backups/backupmysql.lastrun');
}

my $userdata = {};
open(PASSWD, "</etc/passwd") or die("failed to open passwd: $!");
while(<PASSWD>){
    my @ent = split(':', $_);
    if($ent[5] =~ m#^/home(\d*)#){
        $userdata->{$ent[0]}->{homedir} = $ent[5];
        if($1 eq ""){
            $userdata->{$ent[0]}->{homenum} = "";
        }else{
            $userdata->{$ent[0]}->{homenum} = $1;
        }
    }
}
close(PASSWD);

my %users;
my %cpanelusers;
$systemdata->{"cpanel-accounts-total"} = 0;
open(my $DAT, '<', '/etc/trueuserdomains');
while(<$DAT>) {
    chomp();
    if ( /^.*:\s(.*)$/ ) {
        my $user = $1;
        next if $user eq 'root';
        next if $user eq 'system';
        next if (! -d (getpwnam $user)[7]);
        next if ( -e "/var/cpanel/suspended/$user" );
        next if is_new_user($user);
        $systemdata->{"cpanel-accounts-total"}++ if (! $users{$user});
        $users{$user} = 1;
        $cpanelusers{$user} = 1;
    }
}
close($DAT);
my $backup_server_config = LoadFile('/opt/eig_linux/etc/backup_server_info.conf');
die "Could not find backup host!" unless $backup_server_config->{backup_ip};

checkrpmversion(\$systemdata);
checkbackuphelperlastuser(\$systemdata);

$systemdata->{"backup-dirs-completed"} = 0;
$systemdata->{"backup-dirs-uncompleted"} = 0;

systemstatus(\$systemdata);
systeminfo(\$systemdata);
print_data($systemdata) if $quick;
exit if $quick;

checkbackupmounts(\$systemdata);
checkpkgacctrunner(\%cpanelusers,\$userdata,\$systemdata);
checkdrstats(\$systemdata);
partitiondata(\$systemdata);

#print the data we have collected
if($submit){
  submitdata(\$systemdata);
}else{
  print_data($systemdata);
}
update_cache($cachefile, $systemdata);
exit unless $full;

foreach my $user (keys %users){
  checkbackup($user, \$userdata);
}
printbackupstats(\$userdata,\%users);


#### only subs below this line
sub print_data {
    my $systemdata = shift;
    foreach my $key (sort keys %$systemdata){
        if($key =~ m/date$/){
            my $date = localtime($systemdata->{$key});
            printf("%-40s %-40s\n",$key,$date);
        }else{
            printf("%-40s %-40s\n",$key,$systemdata->{$key});
        }
    }
}

sub submitdata {
    my $response_ref = shift;
    $$response_ref->{hostname} = `hostname`;
    chomp($$response_ref->{hostname});

    # lets send the graphite data out first
    if( -e '/etc/collectd.conf'){
        open my $fh, '<', '/etc/collectd.conf' or die "Unable to open file\n";
        my $content = join("\n", <$fh>);
        close $fh;
        my $now = time();
        my $statname = $$response_ref->{hostname};
        $statname =~ s/\./_/g;
        $content =~ m/(\w\.\w+\.\w+\.\w+\.)/xms;
        my $data = $1;
        $content =~ m/(50\.\d{1,3}\.\d{1,3}\.\d{1,3})/xms;
        my $ip = $1;
        my $prefix = sprintf("%s%s.backups-dc",
                            $data,
                            $statname
                            );
        my $reportdata = {};
        foreach my $key (sort keys %{$$response_ref}){
            my $mkey = $key;
            if($key =~ m/^(backup\-dirs|backups\-|cpanel\-accounts|mountpoint\-problems|backup\-queue|nproc|ram)/){
                $mkey =~ s/[\-\.]/_/g;
                $mkey =~ s/_/-/;
                $reportdata->{$mkey} = $$response_ref->{$key};
            } elsif($key =~ m/epoch/){
                my $diff = sprintf("%d",($now - $$response_ref->{$key}) / 60);
                $mkey =~ s/[\-\.]/_/g;
                $mkey =~ s/_epoch/-age/;
                $reportdata->{$mkey} = $diff;
            } elsif($key =~ m/^partition\-data\-(\d+)/){
                my $num = $1;
                my @dat = split('\-',$$response_ref->{$key});
                $reportdata->{"home${num}_size"} = sprintf("%.2f",($dat[0] / 1073741824));
                $reportdata->{"hometotal_size"} += sprintf("%.2f",($dat[0] / 1073741824));
                $reportdata->{"home${num}_used"} = sprintf("%.2f",($dat[1] / 1073741824));
                $reportdata->{"hometotal_used"} += sprintf("%.2f",($dat[1] / 1073741824));
                $reportdata->{"backup${num}_size"} = sprintf("%.2f",($dat[2] / 1073741824));
                $reportdata->{"backuptotal_size"} += sprintf("%.2f",($dat[2] / 1073741824));
                $reportdata->{"backup${num}_used"} = sprintf("%.2f",($dat[3] / 1073741824));
                $reportdata->{"backuptotal_used"} += sprintf("%.2f",($dat[3] / 1073741824));
            }
        }
        if(defined($reportdata->{ram})){
            $reportdata->{ram} = sprintf("%.2f",($$response_ref->{ram} / 1048576));
        }
        #we need to send it someplace
        my $socket = new IO::Socket::INET (
                PeerHost => $ip,
                PeerPort => '2003',
                Proto => 'udp',
        );

        #send the data
        my $buffer = '';
        my $limit = 1024;
        foreach my $key (sort keys %$reportdata){
            my $nextline = sprintf("%s.%s %.2f %d\n",$prefix,$key, $reportdata->{$key},$now);
            #see if this message puts us over the limit
            if( length($buffer) + length($nextline) > $limit ){
                $socket->send($buffer);
                $buffer = '';
            }
            $buffer .= $nextline;
        }
        if(length($buffer) > 0){
            $socket->send($buffer);
        }
    }

    my $message;
    foreach my $key (sort keys %{$$response_ref}){
        $message .= "$key,".$$response_ref->{$key}."\n";
    }
    require LWP::UserAgent;
    my $ua = LWP::UserAgent->new;
    $ua->agent("backupstate/0.1 ");

    my $req = HTTP::Request->new(POST => "$backupmaster_uri/backupstate.cgi");
    $req->content_type('application/x-www-form-urlencoded');
    $req->content("data=$message");

    my $res = $ua->request($req);
    my $response = $res->content;
    print "response from server: $response\n" if $verbose;
    unless($response =~ m/report accepted/){
        print "backupmaster didnt accept my report!\n" if $verbose;
    }
}

sub partitiondata {
    my $response_ref = shift;
    my @space = `/bin/df -P`;
    my $tmp = {};
    foreach(@space){
        next unless $_ =~ m#/(home|backup)(\d)#;
        my $type = $1;
        my $part = $2;
        my @p = split('\s+');
        $tmp->{$part}->{$type}->{'size'} = $p[1];
        $tmp->{$part}->{$type}->{'used'} = $p[2];
        if($type eq "home" and -e "/admin/mig$part/df.output"){
            open(DF, "</admin/mig$part/df.output");
            <DF>;
            my $sourcespace = <DF>;
            close(DF);
            my @s = split('\s+', $sourcespace);
            $tmp->{$part}->{$type}->{'used'} = $s[2] if ($s[2] > $p[2]);
        }
    }
    foreach my $key (keys %$tmp){
        $tmp->{$key}->{'home'}->{'size'} = 0 unless $tmp->{$key}->{'home'}->{'size'};
        $tmp->{$key}->{'home'}->{'used'} = 0 unless $tmp->{$key}->{'home'}->{'used'};
        $tmp->{$key}->{'backup'}->{'size'} = 0 unless $tmp->{$key}->{'backup'}->{'size'};
        $tmp->{$key}->{'backup'}->{'used'} = 0 unless $tmp->{$key}->{'backup'}->{'used'};
        $$response_ref->{"partition-data-$key"} = join('-',($tmp->{$key}->{'home'}->{'size'},$tmp->{$key}->{'home'}->{'used'},$tmp->{$key}->{'backup'}->{'size'},$tmp->{$key}->{'backup'}->{'used'}));
    }
}

sub checklastmodified {
    my $response_ref = shift;
    my $label = shift;
    my $file = shift;
    my @stats = stat($file);
    @stats = stat("$file.1") if( -e "$file.1" and $stats[7] == 0);
    @stats = stat("$file.2") if( -e "$file.2" and $stats[7] == 0);
    @stats = stat("$file.3") if( -e "$file.3" and $stats[7] == 0);
    @stats = stat("$file.4") if( -e "$file.4" and $stats[7] == 0);
    $$response_ref->{$label} = 0;
    $$response_ref->{$label} = $stats[9] if $stats[9];
}

sub systemstatus {
    my $response_ref = shift;
    $$response_ref->{"backup-status"} = "";
    $$response_ref->{"backup-stage"} = "";

    opendir my $dh, "/var/spool/eig_backup";
    my @spool = grep { ! m/\.skip$/ && !  m/^\./ } readdir($dh);
    closedir $dh;
    my $jobs_in_queue = @spool;
    $$response_ref->{"backup-queue_jobs"} = $jobs_in_queue;
    my $failed_count = 0;
    for my $user (@spool){
        open(my $fh, '<', "/var/spool/eig_backup/$user") or next;
        if (defined (my $count = <$fh>)){
            chomp($count);
            $failed_count++ if $count >= 5;
        }
        close $fh;
    }
    my $failed_string = $failed_count ? " Failed:$failed_count" : "";
    $$response_ref->{"backup-queue_failures"} = $failed_count;

    if ( `ps fauwwx | grep datacycl[e]` ) {
        $$response_ref->{"backup-status"} = "Datacycle running";
    }
    if ( -f "/var/run/queuer.pid" ) {
        $$response_ref->{"backup-stage"} = "Active: $jobs_in_queue jobs running$failed_string";
    } elsif ( $jobs_in_queue > 0 ) {
        $$response_ref->{"backup-stage"} = "Pending: $jobs_in_queue jobs queued$failed_string";
    } else {
        $$response_ref->{"backup-stage"} = "No jobs.";
    }
    if ( -f "/var/run/backup_cpanel.pid" ) {
        $$response_ref->{"backup-stage"} = "Active: backup_cpanel";
    } elsif ( -f "/var/run/backup-non-user-mysql" ) {
        $$response_ref->{"backup-stage"} = "Active: non-user-mysql";
    }
    if ( -f "/etc/disable_backups" ) {
        $$response_ref->{"backup-stage"} = "Backups Disabled";
    }
    if ( check_complete() && $$response_ref->{"backup-stage"} !~ m/^Active/ && !-f "/etc/disable_backups") {
        $$response_ref->{"backup-stage"} = "Incomplete";
    }
}

sub systeminfo {
    my $response_ref = shift;
    chomp(my $nproc = (-x '/usr/bin/nproc') ? qx|nproc| : qx|egrep -c '^processor' /proc/cpuinfo|);
    my $meminfo = q|/proc/meminfo|;
    my $ram;
    try {
        open my $FILE, '<', $meminfo or die;
        while (my $line = <$FILE>) {
            chomp $line;
            if ($line =~ 'MemTotal') {
                $ram = (split /\s+/, $line)[1];
                last;
            }
        }
    } catch {
        chomp ($ram = qx(free -k | awk '/Mem:/ {print \$2}'));
    };

    $$response_ref->{nproc} = $nproc;
    $$response_ref->{'target-box'} = $backup_server_config->{'backup_hostname'};
    $$response_ref->{ram} = $ram;
}

sub checkrpmversion {
    my $response_ref = shift;
    my $version = $1 if ( `rpm -q eig_backup` =~ m/(eig_backup-.*)\.el\d+\.noarch/ );
    chomp $version;
    $$response_ref->{"rpm-version"} = $version;
}

sub checkbackuphelperlastuser {
    my $response_ref = shift;
    if ( -f "/var/log/eig_backup/backuphelper_last_command" ) {
        my ($lastuser,$lastcmd) = ($1,$2) if ( `tail -n 1 /var/log/eig_backup/backuphelper_last_command` =~ m/^(\w+)\:(\w+)\:.*$/ );
        $lastuser = '' unless ( $lastuser );
        $lastcmd = '' unless ( $lastcmd );
        chomp $lastuser;
        chomp $lastcmd;
        $$response_ref->{"backuphelper-lastuser"} = $lastuser;
        $$response_ref->{"backuphelper-lastcmd"} = $lastcmd;
    }
}

sub checkpkgacctrunner {
    my $users_ref = shift;
    my $userdata_ref = shift;
    my $response_ref = shift;
    my $now = time;
    my $twodays = $now - (60 * 60 * 24 * 2);
    my $oneweek = $now - (60 * 60 * 24 * 7);
    my @days = qw(one-week six-days five-days four-days three-days two-days one-day);
    $$response_ref->{"cpanel-accounts-$_-behind"} = 0 foreach @days;
    $$response_ref->{"cpanel-accounts-with-backups"} = 0;
    $$response_ref->{"cpanel-accounts-not-backuped-up"} = 0;
    $$response_ref->{"backups-daily"} = 0;
    $$response_ref->{"backups-weekly"} = 0;
    $$response_ref->{"backups-monthly"} = 0;
    my $backupdates = {};
    if (! $updatecache && -f "$backupdatecache" && ((stat "$backupdatecache")[9]) >= time - (60*60*13)){
        print "Loading backup dates from cache\n" if $verbose;
        $backupdates = LoadFile("$backupdatecache") || getbackupdates();
    }else{
        $backupdates = getbackupdates();
    }
    foreach my $user ( keys %{$users_ref}){
        my $verbose_messaged = 0;
        my $mostrecent = getmostrecentbackupdate($user,$response_ref,$backupdates);
        if ($mostrecent == 0 || ! defined $backupdates->{$user}->{"dc_check"}){
        print "$user does not have cpanel account backup\n" if ($verbose && $mostrecent == 0);
        print "$user does not have datacycle check file\n" if ($verbose && ! defined $backupdates->{$user}->{"dc_check"});
        $$response_ref->{"cpanel-accounts-not-backuped-up"}++;
        }else{
        my $dc_check = ($backupdates->{$user}->{"dc_check"} + (60*60*12)); #buffer by 12 hours as cron only runs twice a day that sets them
        my $oldness = time - (60 * 60 * 24 * 7);
        foreach my $day (@days){
            if ($mostrecent < $oldness || $dc_check < $oldness){
            $$response_ref->{"cpanel-accounts-$day-behind"}++;
                if ($verbose && ! $verbose_messaged){
                    print "$user $day behind on cpanel account\n" if ($mostrecent < $oldness);
                    print "$user $day behind on datacycle check\n" if ($dc_check < $oldness);
                    $verbose_messaged = 1;
                }
            }
            $oldness += (60 * 60 * 24);
        }
        $$response_ref->{"cpanel-accounts-with-backups"}++;
        }
    }
}

sub is_new_user{
    my $user = shift;
    my $is_new = 0;
    open (my $fh, '<', "/var/cpanel/users/$user") or return 0;
    while (my $line = <$fh>) {
        next unless ($line =~ m/^STARTDATE=(\d+)/);
        $is_new = 1 if ($1 >= time() - (48*60*60));
        print "$user is a new cpanel account, skipping...\n" if ($verbose && $is_new);
        last;
    }
    close $fh;
    return $is_new;
}

sub getbackupdates{
    my $cmh = $eig_backup_config{'cmh'} ? '--cmh' : '';
    my $dates =  ssh_command("/opt/eig_linux/bin/get_backup_dates --server $hostname $cmh");
    open (FH, ">$backupdatecache" ) or die "Can't open $backupdatecache : $! \n";
    print FH Dump( $dates );
    close FH;
    return $dates;
}

sub ssh_command{
    my $cmd = shift;
    my $args = shift;
    $args->{'format'} //= 'JSON';
    my $backupkey = (-f '/admin/hgbackupdir/backupkey') ? '/admin/hgbackupdir/backupkey' : '/home/hgbackupdir/backupkey';
    my $proc="ssh -i $backupkey  -q -o LogLevel=QUIET -o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=15 -l root $backup_server_config->{backup_ip} $cmd";
    my $stdout = qx($proc);
    my $data;
    if ($args->{'format'} eq 'YAML') {
        $data=eval { Load($stdout) } || {};
    } else {
        $data=eval { decode_json($stdout) } || {};
    }
    return $data
}

sub getmostrecentbackupdate {
    my $user = shift;
    my $homedir = (getpwnam($user))[7];
    my $backupnum = $1 if $homedir =~ m|/home(\d*)/|;
    my $data_ref = shift;
    my $backupdates = shift;
    my $mostrecent = 0;
    foreach my $type (qw{daily weekly monthly}){
        my $moddate = $backupdates->{$user}->{'backup-dates'}->{"/backup$backupnum"}->{'cpbackup'}->{$type} || 0;
        $mostrecent = $moddate if($moddate > $mostrecent);
        $$data_ref->{"backups-$type"}++;
    }
    return $mostrecent;
}

sub checkbackupmounts {
    my $response_ref = shift;
    $$response_ref->{"mountpoint-problems"} = 0;

    #Get what backup mounts we should have from backupmaster
    my $mount_cache = '/opt/eig_linux/data/backup_mount_cache.json';
    my $expected_mounts;
    if ( -f $mount_cache && ((stat $mount_cache)[9]) >= time - (60*60*48) ){
        open(my $fh, "<", "$mount_cache");
        $expected_mounts  = decode_json(<$fh>);
        close $fh;
    }else{
        my $hostname = hostname;
        my $res=`wget -O - -q "$backupmaster_uri/cboxinfo?hostname=$hostname&type=mounts"`;
        $expected_mounts = eval {decode_json($res)};
        if ($expected_mounts){
            open(my $fh, ">", $mount_cache);
            print $fh $res;
            close($fh);
        } else {
            print "Failed to get backup mounts from backupmaster\n" if $verbose;
            $$response_ref->{"mountpoint-problems"} = 1;
            return;
        }
    }

    #Get the active mounts on the box
    my $mounts;
    my $number_of_homes = 0;
    open(my $fh, "<", "/proc/mounts");
    while (my $line = <$fh>){
        my ($device,$mountpoint) = split(/\s+/,$line);
        $number_of_homes++ if ($mountpoint =~  m|^/(home)\d*$| && $device =~ m|^/dev|);
        next unless $mountpoint =~ m|^/backup\d*$|;
        $mounts->{$mountpoint}->{"device"} = $device;
    }
    close($fh);

    #grab mounts in fstab, currently we are skipping any commented entries
    my $fstab;
    open( $fh, "<", "/etc/fstab");
    while (my $line = <$fh>){
        my ($device,$mountpoint) = split(/\s+/,$line);
        next unless $mountpoint =~ m|^/backup\d*$|;
        next if $device =~ m/^#/;
        $fstab->{$mountpoint}->{"device"} = $device;
    }
    close($fh);

    #we have all the info we need, lets check the mountpoints now
    foreach my $wanted_mountpoint (keys %$expected_mounts){
        if (defined $mounts->{$wanted_mountpoint}){
            unless ($mounts->{$wanted_mountpoint}->{'device'} =~ m/^$backup_server_config->{'backup_ip'}:$wanted_mountpoint/){
                print "$wanted_mountpoint is mounted but to the wrong IP!\n" if $verbose;
                $$response_ref->{"mountpoint-problems"}++;
            }
        }else{
            print "$wanted_mountpoint is not mounted!\n" if $verbose;
            $$response_ref->{"mountpoint-problems"}++;
        }

        if (defined $fstab->{$wanted_mountpoint}){
            unless ($fstab->{$wanted_mountpoint}->{'device'} =~ m/^$backup_server_config->{'backup_ip'}:$wanted_mountpoint/){
                print "$wanted_mountpoint is in fstab but with the wrong IP!\n" if $verbose;
                $$response_ref->{"mountpoint-problems"}++;
            }
        }else{
            print "$wanted_mountpoint is not in fstab or is commented!\n" if $verbose;
            $$response_ref->{"mountpoint-problems"}++;
        }
    }
    my $number_of_backup_mounts = keys %$expected_mounts;
    if ($number_of_backup_mounts < $number_of_homes || $number_of_backup_mounts == 0){
        print "Not enough backup mounts found for the number of homes!\n" if $verbose;
        $$response_ref->{"mountpoint-problems"}++;
    }
}

sub printbackupstats {
    my $userdata_ref = shift;
    my $users_ref = shift;
    foreach my $username (keys %$users_ref){
        foreach(qw{ numcpbackups numhomedirbackups numbackups} ){
            if(defined($$userdata_ref->{$username}->{$_})){
                $$userdata_ref->{TOTALS}->{"userswith-".$$userdata_ref->{$username}->{$_}."-$_"}++;
            }else{
                $$userdata_ref->{TOTALS}->{"userswith-0-$_"}++;
            }
        }
    }
    #  foreach my $num (0 ... 4){
    foreach my $type (qw{ numcpbackups numhomedirbackups numbackups} ){
        foreach my $num (0 ... 4){
            my $key = "userswith-$num-$type";
            print "$num $type ".$$userdata_ref->{TOTALS}->{$key}."\n" if defined($$userdata_ref->{TOTALS}->{$key});
        }
    }
}

sub checkbackup {
    my $username = shift;
    my $userdata_ref = shift;

#  checkbackuptype($username, $userdata_ref, "daily");
    foreach my $type ("daily", "weekly", "monthly"){
        checkbackuptype($username, $userdata_ref, $type);
    }
}

sub checkbackuptype {
    my $username = shift;
    my $userdata_ref = shift;
    my $type = shift;
    unless(-e "/backup".$$userdata_ref->{$username}->{homenum}."/cpbackup/$type/$username"){
        return 0;#    print "$username does not have a $type backup\n";
    }
    my @bstat = stat("/backup/cpbackup/$type/$username");
    if($$userdata_ref->{$username}->{homenum} ne ""){
        my @bnumstat = stat("/backup".$$userdata_ref->{$username}->{homenum}."/cpbackup/$type/$username");
        unless($bstat[0] == $bnumstat[0] and $bstat[1] == $bnumstat[1]){
            print "$username: /backup does not link properly to /backup".$$userdata_ref->{$username}->{homenum}."\n";
        }
    }
    $$userdata_ref->{$username}->{"cpbackup$type"} = 1 if( -d "/backup".$$userdata_ref->{$username}->{homenum}."/cpbackup/$type/$username/mysql");
    $$userdata_ref->{$username}->{numcpbackups}++ if( $$userdata_ref->{$username}->{"cpbackup$type"} );
    $$userdata_ref->{$username}->{"homedirbackup$type"} = 1 if( -d "/backup".$$userdata_ref->{$username}->{homenum}."/cpbackup/$type/$username/homedir");
    $$userdata_ref->{$username}->{numhomedirbackups}++ if( $$userdata_ref->{$username}->{"homedirbackup$type"} );
    $$userdata_ref->{$username}->{"date$type"} =  $bstat[9] if $$userdata_ref->{$username}->{"cpbackup$type"} and $$userdata_ref->{$username}->{"newbackup$type"};
    $$userdata_ref->{$username}->{numbackups}++;
}

sub resumeifboxisrebooted {
    my $response = `cat /proc/uptime`;
    my @p = split('\s+', $response);
    if($p[0] > 900 and $p[1] < 1800){
        print "resuming backups\n";
        system("/usr/sbin/backupctl --resume");
    }
}

sub check_complete {
    system("/opt/eig_linux/bin/check_backups");
    return "Incomplete Backups" if ( $? != 0 );
    return 0;
}

sub update_cache {
    my $cache_file = shift;
    my $userdata_ref = shift;
    open (FH, ">$cache_file" ) or die "Can't open $cachefile : $! \n";
    print FH Dump( $userdata_ref );
    close FH;
}

sub checkdrstats {
    my $data = shift;
    if (defined $eig_backup_config{'dr_enabled'} && $eig_backup_config{'dr_enabled'} == 1) {
        $$data->{"dr-enabled"} = 1;
        my $dr_stats = ssh_command("cat /home/$hostname/dr/dr-status.yaml", {'format' => 'YAML'});
        if (defined $dr_stats->{'last_boot'}) {
            $$data->{"last-dr-boot-epoch"} = $dr_stats->{'last_boot'};
        }
        if (defined $dr_stats->{'last_sync'}) {
            $$data->{"last-dr-sync-epoch"} = $dr_stats->{'last_sync'};
        }
    } else {
        $$data->{"dr-enabled"} = 0;
    }
    $$data->{"last-dr-sync-epoch"} ||= defined $cache->{"last-dr-sync-epoch"} ? $cache->{"last-dr-sync-epoch"} : 0;
    $$data->{"last-dr-boot-epoch"} ||= defined $cache->{"last-dr-boot-epoch"} ? $cache->{"last-dr-boot-epoch"} : 0;
    $$data->{"last-dr-sync-date"} = $$data->{"last-dr-sync-epoch"};
    $$data->{"last-dr-boot-date"} = $$data->{"last-dr-boot-epoch"};
}
