#!/usr/bin/env perl

use POSIX qw(mktime strftime);
use Time::Local;
use Term::ANSIColor qw(:constants);
use Sys::Hostname;
use LWP::Simple;

#use version; our $VERSION = qv('1.0');
$VERSION = '1.0';

use Pod::Usage;
#use Readonly;
use File::Basename;
my $prog = basename(__FILE__);

use Getopt::Long qw(VersionMessage HelpMessage);
Getopt::Long::Configure(
    qw(
        bundling
        no_auto_abbrev
        no_ignore_case_always
    )
);

$\ = "\n";

sub success {
    my $message = shift;
    if (not $time_line_flag) {
        printf "%s%s%s %s%s\n", GREEN, '[!]', RESET, $message, RESET;
    }
}

sub failure {
    my $message = shift;
    if (not $time_line_flag) {
        printf "%s%s%s %s%s\n", RED, '[X]', RESET, $message, RESET;
    }
}

# perhaps these functions will be replaced by using a date module

sub ts_conv {
    my ($timestamp) = @_;
    return strftime "%a %b %e %H:%M:%S %Z %Y", localtime $timestamp;
}

sub date_conv {
    my $date = shift;
    return if not $date;
    my ($count, $weekday, $month, $day, $time, $zone, $year);
    $count = @fields = split " ", $date;
    if ($count == 6) {
        ($weekday, $month, $day, $time, $zone, $year) = split " ", $date;
    } elsif ($count == 5) {
        ($weekday, $month, $day, $time, $year) = split " ", $date;
    } else {
        die "Invalid date format: $date";
    }
    my ($hours, $minutes, $seconds) = split ":", $time;
    return timelocal($seconds, $minutes, $hours, $day, $months{$month}, $year);
}

sub date_diff {
    my ($end, $start) = @_;
    return date_conv($end) - date_conv($start);
}

sub setup {
    print;

    get_options();

    if (-d '/var/log/eig_backup') {
        $hgbackupdir = '/var/log/eig_backup';
    } elsif (-d '/admin/hgbackupdir') {
        $hgbackupdir = '/admin/hgbackupdir';
    } else {
        $hgbackupdir = '/home/hgbackupdir';
    }

    if (-d '/admin/hgbackupdir') {
        $hgkeydir = '/admin/hgbackupdir';
    } else {
        $hgkeydir = '/home/hgbackupdir';
    }
    $hostname = hostname;

    $lcol = 31;
    $mcol = 48;
    $rcol = 20;
    $gap  = 6;

    $indent = ' ' x 4;

    $oneweek    =  7 * 24 * 60 * 60;
    $fourweeks  = $oneweek * 4;
    $threedays  =  3 * 24 * 60 * 60;
    $fivemins   =  5 * 60;
    $onehour    = 60 * 60;
    $threehours = 60 * 60 * 3;
    $current    = $fivemins;

    $backuptype = get_flag();

    my @monthnames = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec');
    my @monthnums  = 0 .. scalar(@monthnames);
    @months{@monthnames} = @monthnums;

    setup_colors();

    if ($backuptype eq 'filesys') {
        @logtypes = (['Filesystem', 'backup_verbose.log', 'backup_verbose.log', $backupcolor]);
        @backuptypes = qw(filesys);
    } elsif ($backuptype eq 'vps' || $hostname =~ m{^vps}msx) {
        @logtypes = (['VPS', 'backup_verbose.log', 'backup_verbose.log', $vpscolor]);
        @backuptypes = qw(vps);
    } elsif ($backuptype eq 'full' || $backuptype eq '') {
        @logtypes = (['Full', 'backup_verbose.log', 'backup_verbose.log', $backupcolor], ['Innodb', 'backup_verbose_innodb.log', 'backup_verbose_innodb.log', $innodbcolor]);
        @backuptypes = qw(full innodb);
    } else {
        failure "Unknown backup type: $backuptype";
    }
}

sub get_options {
    my %options = ();
    ($shorter_out_flag, $time_line_flag) = (0, 0);
    GetOptions( \%options,
        'timeline'   . '|t',
        'short'      . '|s',
        'help|?'       => sub {
                pod2usage(-sections => 'NAME', -verbose => 99, -exitval => 'NOEXIT');
                pod2usage(-verbose => 1, -exitval => 1 );
        },
        'man'         => sub { pod2usage(-verbose => 2, -exitval => 1 ); },
        'version'     => sub { printf "%s %s %s\n",  $prog, 'version', $VERSION; exit 1; }
    ) or pod2usage();

    if (exists $options{short}) {
        $shorter_out_flag = 1;
        delete $options{short};
    }

    if (exists $options{timeline}) {
        $time_line_flag = 1;
        printf "%s\r", 'Generating timeline...';
        delete $options{timeline};
    }

    my $flagcount = keys %options;

    if (scalar @ARGV) { pod2usage($prog . ': invalid argument.') }
}

sub setup_colors {
    $cancelcolor  = RED;
    $filesyscolor = YELLOW;
    $vpscolor     = CYAN;
    $fullcolor    = BLUE;

    $innodbcolor = MAGENTA;

    if ($backuptype eq 'filesys') {
        $backupcolor = $filesyscolor;
    } elsif ($backuptype eq 'vps') {
        $backupcolor = $vpscolor;
    } elsif ($backuptype eq 'full') {
        $backupcolor = $fullcolor;
    } else {
        $backupcolor = '';
    }

    %type_colors = (
        full    => $fullcolor,
        innodb  => $innodbcolor,
        filesys => $filesyscolor,
        vps     => $vpscolor
    );

    %status_colors = (
        Started => YELLOW,
        Completed => GREEN,
        Failed => RED,
        Canceled => $cancelcolor
    )
}

sub color {
    my ($color, $text) = @_;
    return sprintf '%s%s%s', $color, $text, RESET;
}

sub save_time_line {
    my $c1offset = 0;
    my $c2offset = 0;

    my $timestamp = shift;
    if ($timestamp) {
        my ($color1, $column1, $color2, $column2, $color3, @column3) = @_;
        if (ref($column1) eq "ARRAY") {
            ($column1, $c1offset) = @$column1;
        }
        if (ref($column2) eq "ARRAY") {
            ($column2, $c2offset) = @$column2;
        }
        push @time_line, [$timestamp, sprintf "%s%-*s%s%s%-*s%s%s%-*s%s", $color1, $lcol + $c1offset, $column1, RESET, $color2, $mcol + $c2offset, $column2, RESET, $color3, $rcol, "@column3", RESET];
    }
}

sub show_time_line {
    printf "%s %s\n", 'Backup timeline for', "$hostname" . get_cust_count();
    for my $event (sort {$a -> [0] <=> $b -> [0]} @time_line) {
        print @$event[1];
    }
}

sub hr {
    if (not $time_line_flag) {
        my $lrule = '- ' x (($lcol - $gap)/2) . '-';
        my $mrule = '- ' x (($mcol - $gap)/2) . '-';
        my $rrule = '- ' x (($rcol - $gap)/2) . '-';
        printf "%s%-*s%-*s%-*s\n", $indent, $lcol, $lrule, $mcol, $mrule, $rcol, $rrule;
    }
}

sub col {
    my $c1offset = 0;
    my $c2offset = 0;

    if (not $time_line_flag) {
        shift;
        my ($color1, $column1, $color2, $column2, $color3, @column3) = @_;
        if (ref($column1) eq "ARRAY") {
            ($column1, $c1offset) = @$column1;
        }
        if (ref($column2) eq "ARRAY") {
            ($column2, $c2offset) = @$column2;
        }
        printf "%s%s%-*s%s%s%-*s%s%s%-*s%s\n", $indent, $color1, $lcol + $c1offset, $column1, RESET, $color2, $mcol + $c2offset, $column2, RESET, $color3, $rcol, "@column3", RESET;

    } else {
        save_time_line(@_);
    }
}

sub commify {
    my ( $sign, $int, $frac ) = ( $_[0] =~ /^([+-]?)(\d*)(.*)/ );
    my $commified = (
        reverse scalar join ',',
        unpack '(A3)*',
        scalar reverse $int
    );
    return $sign . $commified . $frac;
}

sub tail {
    # from http://www.perlmonks.org/?node_id=195768
    my $filename = shift or die "Usage: $0 file numlines\n";
    my $numlines = shift;
    my $byte;
    open my $handle, '<', $filename or die "Couldn't open $filename: $!";
    # Rewind from the end of the file until count of eol 's
    seek $handle, -1, 2;  #get past last eol
    my $count=0;
    while (1){
        seek $handle, -1, 1;
        read $handle, $byte, 1;
        if (ord($byte) == 10 ) {$count++; if($count == $numlines) {last}}
        seek $handle, -1, 1;
        if (tell $handle == 0) {last}
    }
    my $tail = <$handle>;
    close $handle;
    return $tail;
}

sub get_flag {
    my $file = '/var/spool/cron/root';
    my $pattern = '(?<=\./backup\.pl[[:space:]])(--(full|filesys|vps))';
    my $flag;
    open my $handle, '<', $file;
    while (my $line = <$handle>) {
        if ($line =~ qr{$pattern}msx ) {
            $flag = $2;
        }
    }
    close $handle;
    return $flag;
}

sub get_cust_count {
    if (not $shorter_out_flag) {
        my $count, @vps_users;
        if ($backuptype eq 'full') {
            open FILE, '/etc/trueuserdomains';
            @_ = <FILE>;
            $count = ' (' . $. . ' accts) ';
            close FILE;
        } elsif ($backuptype eq 'vps') {
            @vps_users = `/usr/sbin/vzlist --all --no-header --output ctid`;
            $count = @vps_users;
            $count = ' (' . $count . ' containers)';
        }
        return $count; # returns empty string for --filesys, etc.
    }
}

sub today {
    my @tm = localtime;
    $todayts = timelocal @tm;
    ($thismonth, $thisyear, $today_dow) = @tm[4, 5, 6];
    $today = ts_conv($todayts);
    col($todayts, BOLD, "Today's Date:", BOLD, $today);
    hr();
}

sub server {
    my ($cores, $corecolor);
    my $fh;
    open $fh, '<', "/proc/cpuinfo" or die "Unable to open /proc/cpuinfo";
    while (<$fh>) {
        $cores++ if m{processor}msx;
    }
    close $fh;
    if ($cores < 8) {
        $corecolor = YELLOW;
    } elsif ($cores == 8) {
        $corecolor = RED;
    }
    col('', '', 'Server:', '', $hostname . get_cust_count(), $corecolor, $cores . " cores");
    hr();
}

sub nas_info {
    my $keyfile;
    my $nas_info = get "http://nas.hostgator.com/assign.php?server=$hostname";
    my ($nas_ip, $nas_dir) = split "\n", $nas_info;
    if (-f "$hgkeydir/naskey") {
        $keyfile = "$hgkeydir/naskey";
    } else {
        $keyfile = "$hgkeydir/backupkey";
    }
    chomp($nas_name = qx{ssh -n -i $keyfile root\@$nas_ip hostname});
    col('', '', 'NAS Info:', '', "$nas_name $nas_ip", '', $nas_dir);
    hr();
}

sub pids {
    my @types = (
        ['Backup', '[b]ackup\.(pl|exe)'],
        ['ssh',    "^ssh.*$hgbackupdir"],
        ['rsync',  "^$hgbackupdir/rsync"]
    );
    my ($type, $pattern, @procs, $color);
    local $, = " ";
    for $type (@types) {
        ($typename, $pattern) = @$type;
        chomp(@procs = qx{pgrep -f "$pattern"});
        if ($typename eq "Backup" && scalar(@procs)) {
            $running = 1;
        }
        my $procstring = join(' ', @procs);
        if (scalar(@procs)) {
            $color = GREEN;
        } else {
            $color = '';
            $procstring = 'N/A';
        }
        col('', '', "$typename PIDs:", $color, $procstring);
    }
    hr();
}

sub screen_sessions {
    my $screens = qx{screen -ls | grep -Es 'backup|rsync'};
    if ($screens) {
        for $s (split "\n", $screens) {
            col('', '', 'Screen:', GREEN, $s);
        }
    } else {
        col('', '', 'Screens:', '', 'N/A');
    }
    hr();
}

sub bytes_transferred {
    my $file = shift;
    my $total = 0;
    my $pattern = '^sent[ ].*bytes';
    open my $handle, '<', $file;
    while ( my $line = <$handle> ) {
        if ( $line =~ qr{$pattern}msx ) {
            $total += (split(" ", $line))[1];
        }
    }
    close $handle;
    return $total;
}

sub count_files {
    my ($file) = @_;
    my $count = 0;
    my $pattern = '^:[ ]/';
    if (-f $file) {
        open my $handle, '<', $file;
        while ( my $line = <$handle> ) {
            if ( $line =~ qr{$pattern}msx ) {
                $count++;
            }
        }
        close $handle;
    }
    return $count;
}

sub backup_verbose {
    my ($logname, $logtype, $logfile, $logcolor, $file, $ts, $verbose, $verbose_updts, $interval, $mins, $rate, $comment);
    for $log (@logtypes) {
        ($logname, $logtype, $logfile, $logcolor) = @$log;
        if (-f "$hgbackupdir/$logfile") {
            $comment = '';
            $file = readlink "$hgbackupdir/$logfile";
            $ts = (split(/\./, $file))[-1];
            if ($ts > $recent_verbose) {
                $recent_verbose = $ts;
            }
            $verbose = ts_conv($ts);
            col($ts, '', [$logcolor . $logname . RESET . " Verbose Log:", length($logcolor . RESET)], '', $verbose, '', $ts);

            $verbose_updts = (stat("$hgbackupdir/$logfile"))[9];
            $interval = $todayts - $verbose_updts;
            if ($interval <= $current) {
                $color = GREEN;
                $comment = 'Current';
            } elsif ($interval <= $onehour) {
                $color = YELLOW;
                $comment = 'Recent';
            }
            if (tail("$hgbackupdir/$logfile", 1) =~ "Too many backups running") {
                $color = YELLOW;
                $comment = 'Waiting';
            }

            col($verbose_updts, '', 'Verbose Log Updated:', '', ts_conv($verbose_updts), $color, $comment);

# TODO: add --detail flag required to print these counts and totals
            if (not $time_line_flag) {
                $mins = int(($verbose_updts - $ts) / 60);
                col('', '', 'Duration (minutes)', '', $mins);

                if (not $shorter_out_flag) {
                    my $bytes = bytes_transferred($file);
                    col('', '', 'Bytes Transferred', '', commify($bytes));

                    $rate = 0;
                    if ($mins != 0) {
                        $rate = int(($bytes / 1024) / ($mins * 60));
                    }
                    col('', '', 'KB / Second', '', $rate);

                    my $files = commify(count_files("$hgbackupdir/logs/$logtype.$ts"));
                    col('', '', 'Files Transferred', '', $files);
                }
            }
            hr
        }
    }
}

sub previous_backup_verbose {
    my ($log, $logname, $logtype, $logfile, $logcolor, $file, $ts, $verbose, $verbose_updts, $interval, $mins, $rate, $comment);
    for $log (@logtypes) {
        ($logname, $logtype, $logfile, $logcolor) = @$log;
        $comment = '';
        $file = (sort {-M $a <=> -M $b} glob("$hgbackupdir/logs/$logfile.*"))[1];
        if (-f $file) {
            $ts = (split(/\./, $file))[-1];
            $verbose = ts_conv($ts);
            col($ts, '', ["Previous " . $logcolor . $logname . RESET . " Log:", length($logcolor . RESET)], '', $verbose, '', $ts);

            $verbose_updts = (stat("$file"))[9];
            $interval = $todayts - $verbose_updts;
            if ($interval <= $current) {
                $color = GREEN;
                $comment = 'Current';
            } elsif ($interval <= $onehour) {
                $color = YELLOW;
                $comment = 'Recent';
            }

            col($verbose_updts, '', 'Verbose Log Updated:', '', ts_conv($verbose_updts), $color, $comment);

            if (not $time_line_flag) {
                $mins = int(($verbose_updts - $ts) / 60);
                col('', '', 'Duration (minutes)', '', $mins);

                if (not $shorter_out_flag) {
                    my $bytes = bytes_transferred($file);
                    col('', '', 'Bytes Transferred', '', commify($bytes));

                    $rate = 0;
                    if ($mins != 0) {
                        $rate = int(($bytes / 1024) / ($mins * 60));
                    }
                    col('', '', 'KB / Second', '', $rate);

                    my $files = commify(count_files("$hgbackupdir/logs/$logtype.$ts"));
                    col('', '', 'Files Transferred', '', $files);
                }
            }
            hr
        }
    }
}

sub progress {
    my ($log, $line, $logname, $logtype, $logfile, $logcolor, $progress, $progressdt, $progresspct, $progressts);
    my $pattern = 'PERCENTAGE';
    for $log (@logtypes) {
        ($logname, $logtype, $logfile, $logcolor) = @$log;
        if (-e "$hgbackupdir/$logfile") {
            open my $handle, '<', "$hgbackupdir/$logfile";
            while (my $line = <$handle>) {
                if ($line =~ qr{$pattern}msx ) {
                    $progress = $line;
                }
            }
            close $handle;
            if ($progress) {
                ($progressdt) = split " : ", $progress;
                $progresspct = (split " ", $progress)[-1] . '%';
                $progressts = date_conv($progressdt);
            } else {
                $progressdt = 'N/A';
            }
            col($progressts, '', [$logcolor . $logname . RESET . " Progress:", length($logcolor . RESET)], '', $progressdt, '', $progresspct);
            hr();
        }
    }
}

sub vps_progress {
    my ($found, $i, $msg, @progress, $entries);
    my $pattern = '^--vps[ ]user|^Container:';

    open my $handle, '<', "$hgbackupdir/backup_verbose.log";
    while (my $line = <$handle>) {
        if ($line =~ qr{$pattern}msx ) {
            push @progress, $line;
        }
    }
    close $handle;
    $entries = @progress;
    for ($i = $entries; $i > 0; $i--) {
        if ($progress[$i] =~ qr{Container}msx) {
            $found = 1;
            last;
        }
    }
    if ($found) {
        $msg = "Container " . ($entries - $i - 1) . " of " . ($i + 1);
    } else {
        $msg = 'N/A';
    }
    col('', '', [$vpscolor . 'VPS' . RESET . ' Progress:', length($vpscolor . RESET)], '', $msg, '');
    hr();
}

sub backup_status {
    my ($date, $status, $type, %logtimes, $hr, $completed, $started);
    my @statuses = qw(Started Completed Failed Canceled);
    open my $handle, '<', "$hgbackupdir/backup.log";
    while (my $line = <$handle>) {
        chomp $line;
        ($date, $status, $type) = split qr{[ ]?:[ ]|[ ]-[ ]+|\.$}msx, $line;
        $status =~ s/.*(start|complet|fail|cancel)(ed|ing).*/\L\u$1ed/i;
        $logtimes{"$type|$status"} = $date;
    }
    close $handle;
    for $type (@backuptypes) {
        $hr = 0;
        for $status (@statuses) {
            $date = $logtimes{"$type|$status"};
            $ts = date_conv($date);
            if ($status eq 'Started') {
                $started = $date;
            } elsif ($status eq 'Completed') {
                $completed = $date;
                if ($started and date_diff($completed, $started) < 0 and not $time_line_flag) {
                    next;
                }
            }
            if ($status eq 'Failed' or $status eq 'Canceled') {
                if ($date) {
                    if ($completed and date_diff($completed, $date) > 0 and not $time_line_flag) {
                        next;
                    }
                    my $diff = date_diff($today, $date);
                    if ($diff > $fourweeks) {
                        next;
                    } elsif ($diff > $oneweek) {
                        $date = 'old';
                        $ts = '';
                        $color = '';
                    }
                }
            }

            if ($date) {
                col($ts, $status_colors{$status}, "Backup $status", '', $date, $type_colors{$type}, $type);
                $hr = 1;
            }
        }
        if ($hr) {
            if ($started and $completed and date_diff($completed, $started) >= 0) {
                success("Backup completed - " . $type_colors{$type} . $type);
            } else {
                failure("Backup not completed - " . $type_colors{$type} . $type);
            }
            hr();
        }
    }
}

sub backup_errors {
    my ($line, $date, $msg, $ts, $errts, $width, $head, $i);
    my $file = "$hgbackupdir/backup_error.log";
    open my $handle, '<', $file;
    while ($line = <$handle>) { # get last line
        $saved = $line;
    }
    close $handle;
    chomp $saved;
    my ($date, $msg) = split qr{:[ ]}msx, $saved, 2;
    $ts = $errts = date_conv($date);
    if ($todayts - $ts > $oneweek) {
        $date = 'old';
        $ts = '';
    }
    col($ts, '', 'Backup Error:', '', $date, '', $ts);
    $ts = (stat($file))[9];
    $date = ts_conv($ts);
    if ($todayts - $ts > $oneweek) {
        $date = 'old';
        $ts = '';
    }
    col($ts, '', 'Backup Error File:', '', $date, '', $ts);
    if ($date ne 'old') {
        $width = $mcol - $gap;
        $head = 'Backup Error Msg:';
        for ($i = 0; $i < length($msg); $i += $width) {
            col('', '', $head, '', substr $msg, $i, $width);
            $head = '';
        }
        my @errtypes = (
            ['too many rsync failures', 'Too many rsync failures', ''],
            ['firewall', 'Possible firewall issue', ''],
            ['Failure to get disk/partition stats on NAS server.', '', 'statfail'],
            ['Could not list.*starttime', '', 'nolist'],
            ['appears to be full', 'Too little available space on NAS box destination directory', '']
        );
        my %subtypes = (
            nolist =>
                ['Read-only file system', 'Read-only filesystem on nas box'],
                ['Connection refused', 'Connection refused'],
                ['No space left on device', 'No space left on NAS box destination directory'],
            statfail =>
                ['Connection refused', 'Connection refused']
        );
        local $/ = undef;
        open my $handle, '<', "$hgbackupdir/logs/backup_verbose.log.$errts";
        my @contents = split "\n", <$handle>;
        close $handle;
        for my $errtype (@errtypes) {
            my ($pattern, $failmsg, $subtype) = @$errtype;
            if ($msg =~ $pattern) {
                if ($failmsg and not $subtype) {
                    failure($failmsg);
                } elsif ($subtype and not $failmsg) {
                    for $type (@subtypes{$subtype}) {
                        if (grep m{\Q@$type[0]}msx, @contents) {
                            failure @$type[1];
                        }
                     }
                } else {
                    die 'Internal error: unkown errtype structure';
                }
            }
        }
    }
    hr();
}

sub rsync_log {
    my ($color, $comment, $count);
    my $file = (sort {-M $a <=> -M $b} glob("$hgbackupdir/rsync_errors_log.*"))[0];
    my $ts = (split(/\./, $file))[-1];
    my $date = ts_conv($ts);
    if (not $file) {
        $date = 'N/A';
        $ts = '';
    } elsif ($todayts - $ts > $oneweek) {
        $date = 'old';
        $ts = '';
    }
    col($ts, '', 'Rsync Error Log:', '', $date, '', $ts);
    $ts = (stat($file))[9];
    $date = ts_conv($ts);
    if (not $file) {
        $date = 'N/A';
        $ts = '';
    } elsif ($todayts - $ts > $oneweek) {
        $date = 'old';
        $ts = '';
    } elsif ($date and $ts >= $recent_verbose) {
        $color = YELLOW;
        $comment = 'Current';
    }
    col($ts, '', 'Rsync Log Updated:', '', $date, $color, $comment);
    if ($ts and $date ne 'old') {
        open my $handle, '<', $file;
        while (<$handle>) {
            if (not /^$/) {
                $count++
            }
        }
        close $handle;
        col('', '', 'Rsync Error Count:', '', $count);
    }
    hr();
}

sub starts_completes {
    my $entries = 16;
    for my $type (@backuptypes) {
        my $type_color = $type_colors{$type};
        my $cstart = 0;
        my $ccomp = 0;
        my ($c, @starts, $start, $firststart, @comps, $comp, $ccompcolor, $i);

        open my $handle, '<', "$hgbackupdir/backup.log";
        while (my $line = <$handle>) {
            chomp $line;
            if ($line =~ $type) {
                $c++;
                if ($line =~ qr{Start}msx) {
                    @starts[++$start] = $c;
                }
                if ($line =~ qr{completed}msx) {
                    @comps[++$comp] = $c;
                }
            }
        }
        close $handle;
        for ($i = 1; $i <= $start; $i++) {
            if ($starts[$i] > $c - $entries) {
                $cstart = $start - $i + 1;
                last;
            }
        }
        $firststart = $i;
        for ($i = 1; $i <= $comp; $i++) {
            if ($comps[$i] > $c - $entries) {
                $ccomp = $comp - $i + 1;
                last;
            }
        }
        if ($starts[$firststart] > $comps[$i] && $comps[$i] > 0) {$ccomp--};
        if ($cstart != $ccomp) {
            $ccompcolor = RED;
        } else {
            $ccompcolor = '';
        }

        col('', $type_color, 'Recent Starts/Completes:', '', [$cstart . " / " . color($ccompcolor, $ccomp), length($ccompcolor . RESET)], $type_color, $type);
    }
    hr();
}

sub show_crons {
    # Date::Manip would be perfect, but it's not available on VPS nodes, so we'll spawn date(1)
    my $pattern = 'backup\.(pl|exe)';
    my ($status, $status_color);
    my @dow = qw(Sun Mon Tue Wed Thu Fri Sat Sun);
    open my $handle, '<', '/var/spool/cron/root';
    while (my $line = <$handle>) {
        if ($line =~ qr{$pattern}msx ) {
            chomp $line;
            if ($line =~ s{^\#}{}msx) {
                $status = "(disabled)";
                $status_color = RED;
            }
            ($minute, $hour, $dow, $type) = (split qr{[ ]|--}msx, $line)[0, 1, 4, -1];
            if ($today_dow == $dow) {
                $when = '';
            } else {
                $when = 'last';
            }
            chomp(($date, $ts) = split /\|/, qx{date -d "$hour:$minute $when @dow[$dow]" '+%a %b %e %T %Z %Y|%s'});
            col($ts, '', 'Recent scheduled cron:', '', $date, $type_colors{$type}, $type . RED . " $status");
            chomp(($date, $ts) = split /\|/, qx{date -d "$hour:$minute $when @dow[$dow] + 1 week" '+%a %b %e %T %Z %Y|%s'});
            col($ts, '', 'Next scheduled cron:', '', $date, $type_colors{$type}, $type . RED . " $status");
            $crons{$type}++;
            $croncount++
        }
    }
    close $handle;
    if ($croncount > 0) {
        for $type (@backuptypes) {
            if (not $crons{$type}) {
                col('', '', 'Crontab:', RED, 'crontab backup entry missing', $type_colors{$type}, $type);
            }
        }
    } else {
        col('', '', 'Crontab:', RED, 'crontab backup entries missing', RED, 'ALL');
    }
    @files = sort {-M $b <=> -M $a} glob("/var/log/cron*");
    $filenames = join(" ", @files);
    @entries = `zgrep --text --no-filename "[ /]backup\.pl" $filenames`;
    my $count = @entries;
    my @recent;
    if ($count == 0) {
            @recent = ();
    } elsif ($count == 1) {
            @recent = @entries;
    } elsif ($count > 1) {
            @recent = @entries[-2, -1];
    }
    if (not $shorter_out_flag) {
        $when = 'Prev';
        for my $entry (@recent) {
            chomp $entry;
            my ($month, $day, $time, $type) = (split qr{[ ]+|--|[)]$}msx, $entry)[0, 1, 2, -1];
            my ($hours, $minutes, $seconds) = split qr{:}, $time;
            if ($months{$month} <= $thismonth) {
                $year = $thisyear;
            } else {
                $year = $thisyear - 1;
            }
            $ts = timelocal($seconds, $minutes, $hours, $day, $months{$month}, $year);
            $date = ts_conv($ts);
            col($ts, '', "$when Logged cron:", '', $date, $type_colors{$type}, $type);
            $when = 'Last';
        }
    }
    hr();
}

sub interrupted {
    if (-e "$hgbackupdir/.backup_running" && not $running) {
        col('', RED, 'Backup Not Running', RED, 'Interrupted by reboot');
        hr();
    }
}

sub recent_boot {
    my ($comment, $color);
    local $/ = undef;
    open my $handle, '<', '/proc/uptime';
    my $contents = <$handle>;
    close $handle;
    my ($uptime) = split " ", $contents;
    my @tm = localtime;
    my $ts = timelocal @tm;

    my $lastboot = $ts - $uptime;
    if ($uptime > $oneweek) {
        $comment = 'not recent';
    } elsif ($uptime < $threedays) {
        $color = RED;
        $comment = 'Very Recent';
    } else {
        $color = YELLOW;
        $comment = 'Recent';
    }
    col($lastboot, '', 'Last boot (approx):', '',  ts_conv($lastboot), $color, $comment);
    hr()
}

setup();
today();
server();
nas_info();
pids();
screen_sessions();
backup_verbose();
previous_backup_verbose() if not $shorter_out_flag;
progress();
vps_progress if $hostname =~ qr{^vps}msx and not $shorter_out_flag;
backup_status();
backup_errors();
rsync_log();
starts_completes() if not $shorter_out_flag;
show_crons();
interrupted();
recent_boot();
show_time_line if $time_line_flag;

__END__

=head1 NAME

 backup-result - Show backup results

=head1 DESCRIPTION

 This script will display information useful in diagnosing backup problems.

=head1 SYNOPSIS

 backup-result [-s|-t]

=head1 OPTIONS

 -s, --short       Shorter, faster output - omit previous backups
 -t, --timeline    Output events in chronological order, may be used with -s
 -?, --help        Show this message

=cut
