#!/usr/bin/perl
use strict;
use warnings;

use File::Path;
use File::Slurp;
use Data::Dumper;
use Getopt::Long;
use Sys::Hostname;
use Socket;
use YAML qw (LoadFile Load Dump DumpFile);

use FindBin qw($Bin);
use lib "$Bin/../lib";
use lib "/opt/eig_linux/lib";
use BackupServer;
$ENV{'PATH'} .= ':/usr/bin:/usr/sbin';

my ($userbox, $plabel, $job, $job_file);
my $source = hostname;
my $rsync_threshold = '.98';
my $wanted_resize_percent = '.80';
my $minimumresize = '.15';
my $force = 0;
my $reset_failures = 0;
my $usage = 0;
my $cron = 0;
my $delete = 0;
my $list = 0;
my $find = 0;
my $background = 0;
my $create = 0;
my $run_jobs = 0;
my $cleanup = 0;
my $status = 0;

GetOptions(
    "plabel|p=s"        => \$plabel,
    "force"             => \$force,
    "cron"              => \$cron,
    "min|minimum-resize=f"  => \$minimumresize,
    "reset-failures"    => \$reset_failures,
    "delete"            => \$delete,
    "usage|help|h"      => \$usage,
    "list|l"            => \$list,
    "find"              => \$find,
    "background"        => \$background,
    "create"            => \$create,
    "run-jobs"          => \$run_jobs,
    "cleanup"           => \$cleanup,
    "status"           => \$status,
) || usage("Invalid options given");

usage() if $usage;
list() if $list;
find() if $find;
run_jobs() if $run_jobs;
cleanup() if $cleanup;
usage("Please specify plabel") unless $plabel;
die "Please run this script from a screen!" unless ($ENV{STY} || $cron || $delete || $reset_failures || $background || $create || $run_jobs || $status);


#get which partitions we need to migrate.
my $bk = new BackupServer;
my $partitions = $bk->get_partitions();
my $partition;
foreach my $id (keys %$partitions){
    next unless (defined $partitions->{$id}->{'plabel'} && $partitions->{$id}->{'plabel'} eq $plabel);
    $partition = $partitions->{$id};
    $userbox = $partition->{'chostname'};
    last;
}
die "No $plabel found on the server" if (not defined $partition);
status() if $status;

##setup or import from the job file and handle job management flags
$job_file = "/var/spool/backup_shrinks/$plabel";
if ( -f $job_file ){
    $job = LoadFile($job_file);
    $minimumresize = $job->{'minimumresize'};
    $force = $job->{'force'} || $force;;
} else {
    $job->{'hostname'} = $userbox;
    $job->{'minimumresize'} = $minimumresize;
}
$job->{'status'} = 'queued';
$job->{'force'} =  $force;
$job->{'error'} = "";
$job->{'failures'} = 0 if (not defined $job->{'failures'} or $reset_failures == 1);

update_job(qw(status error));
if ($delete){
    logger("This will ONLY clean up the job and not any mounts or created partitions",'INFO');
    run("/bin/rm -v $job_file");
}
exit if ($reset_failures || $delete || $create);
die "Refusing to run due to high amount of failures ($job->{'failures'})\n" if $job->{'failures'} >= 5;

if ($background){
    my $alive;
    if ($job->{'pid'}) {
        $alive = (kill 0, $job->{'pid'});
        logger("$userbox has an active migraiton as pid $job->{'pid'}\n",'FATAL') if ( $alive );
    }
    my $pid;
    $pid = fork();
    if ($pid == 0) {
        exec("/opt/eig_linux/bin/partition_shrink --plabel $plabel --cron 2>&1 > /dev/null");
    }
    exit;
}

my $mountpoint = "/home/$userbox$partition->{'cmount'}";
my $shrink_plabel = "$plabel.s"; #Keeping shorter plabel so we dont' run into the 16 char limit
my $shrink_mountpoint = "/home/$userbox$partition->{'cmount'}.shrink/";
my @active = qx(ps auxwww | grep '$mountpoint');
if (@active > 2){
    logger("Open procs found on $mountpoint, not starting the job for it.",'INFO');
    exit;
}
make_pid();

#setup the partition if it needs it
logger("$plabel.old already exists, Manual intervention required, the shrink likely failed to finalize",'FATAL')if (-e "/dev/backup/$plabel.old");
logger("$mountpoint is not a mountpoint!",'FATAL') unless check_mount($mountpoint);
if (! check_mount($shrink_mountpoint)){
    logger("Partition is not set up yet. Setting it up.",'INFO');
    my (undef, undef, $used) = split /\s++/, qx(df -P $mountpoint | sed 1d);
    my $percent_adjusted_homesize = ($used/($wanted_resize_percent * 0.97)/1024/1024/1024);
    my $minimum_size = $partition->{'homesize'} * $minimumresize;
    $partition->{'homesize'} = $minimum_size >= $percent_adjusted_homesize ? $minimum_size : $percent_adjusted_homesize;
    $job->{'newsize'} = sprintf "%.03f", ($partition->{'homesize'});
    logger("Wanted lv size of $partition->{'cmount'} will be $partition->{'homesize'}",'INFO');
    chomp(my $space_available = qx(vgs backup --noheadings -o vg_free --units t));
    logger("$space_available space available, we need $partition->{'homesize'}",'INFO');
    if ($space_available =~  m/([0-9]+\.[0-9]+)t/ && !$force){
        logger("Not enough space available for new partitions.  If you wish to proceed anyways please add the --force flag",'FATAL') if ($1 - $partition->{'homesize'}  <= 3);
    }
    run("lvcreate --name $shrink_plabel --size $partition->{'homesize'}t backup");
    run("mkfs.ext4 -L $shrink_plabel /dev/backup/$shrink_plabel && tune2fs  -i 0 -c 0 -m1 /dev/backup/$shrink_plabel");
    mkpath("$shrink_mountpoint") unless ( -d "$shrink_mountpoint");
    run("mount /dev/backup/$shrink_plabel $shrink_mountpoint");
}
logger("Mount failed to setup!",'FATAL') unless check_mount($shrink_mountpoint);

#Time to run the rsyncs
logger("Rsyncs kicking off for $plabel",'INFO');
$job->{'status'} = 'running';
update_job(qw(status));
my $rsync_tries = 0;
until  ( check_usage() || ($force && $rsync_tries)){
    logger("3 rsyncs and still not matching!\nPlease check manually",'FATAL') if ($rsync_tries >= 3);
    my $rsync = qq(rsync -aHXA --delete $mountpoint/ $shrink_mountpoint/);
    logger("Starting rsync\n$rsync",'INFO');
    my $start = time;
    run($rsync);
    my $elapsed = time - $start;
    logger("Finished rsync for $mountpoint with rc $?\nElapsed Time: $elapsed",'INFO');
    sleep 300 if $rsync_tries >=1;
    $rsync_tries++;
}
logger("Rsyncs finished successfully",'INFO');

#Lets flip and finalize
my $userbox_ip = inet_ntoa(inet_aton($userbox));
run("maint enable $userbox_ip",{'allow-errors' => 1});
sleep 180;
run("exportfs -u $userbox_ip:$mountpoint",{'allow-errors' => 1});
umount($mountpoint);
umount($shrink_mountpoint);
run("tune2fs -L $plabel.old /dev/backup/$plabel");
run("lvrename /dev/backup/$plabel $plabel.old");
run("tune2fs -L $plabel /dev/backup/$shrink_plabel");
run("lvrename /dev/backup/$shrink_plabel $plabel");
run("systemctl daemon-reload");
run("mount $mountpoint");
run("exportfs -ra");
run("maint disable $userbox_ip",{'allow-errors' => 1});
$job->{'status'} = 'complete';
$job->{'pid'} = undef;
update_job();
mkdir "/var/log/backup_shrinks/completed" unless -d "/var/log/backup_shrinks/completed";
run("mv -v $job_file /var/log/backup_shrinks/completed/");
run("lvremove /dev/backup/$plabel.old -y");
#thar be subs
sub check_mount{
    my $mountpoint = shift;
    my $res = qx(/bin/mountpoint $mountpoint);
    return 0 unless $res =~ m/is a mountpoint/;
    return 1;
}
sub umount {
    my $mount = shift;
    my $tries = 0;
    my $backoff = 60;
    do {
        logger("Failed to unmount $mount",'FATAL')  if ($tries >= 5);
        sleep ($tries * $backoff);
        run("umount $mount",{'allow-errors' => 1});
        $tries++;
    } while (check_mount($mount));
}
sub logger {
    my $msg = shift;
    my $level = shift;
    mkpath('/var/log/backup_shrinks/server_logs') unless ( -d '/var/log/backup_shrinks/server_logs' );
    my $log_file = defined $plabel ? "/var/log/backup_shrinks/server_logs/$plabel.log" : "/var/log/backup_shrinks/queuer.log";
    open(my $LOG, '>>', "$log_file");
    if ($level =~ m/fatal/i){
        $job->{'error'} = $msg;
        $job->{'status'} = 'failed';
        $job->{'failures'}++;
        delete $job->{'pid'};
        update_job(qw(status error));
        print $LOG localtime(time) . ":FATAL: $msg\n";
        die $msg;
    } else {
        print $LOG localtime(time) . ":$level: $msg\n";
        print localtime(time) . ": $msg\n";
    }
    close($LOG);
}
sub update_job {
    my @fields = @_;
    mkdir ('/var/spool/backup_shrinks') unless ( -d '/var/spool/backup_shrinks' );
    DumpFile($job_file,$job) or logger ("Unable to dump to $job_file",'FATAL');
}
sub check_usage {
    my $rc = 1;
    my $usage = (split (/\s++/, qx(df -P $mountpoint | tail -1)))[2];
    my $shrink_usage = (split (/\s++/, qx(df -P $shrink_mountpoint | tail -1)))[2];
    my $percent = $shrink_usage/$usage;
    my $msg = sprintf "%s at %.02f of source\ $usage/$shrink_usage", $partition->{'cmount'}, $percent;
    logger($msg,'INFO');
    $rc = 0 if ($percent <= $rsync_threshold);
    return $rc;
}
sub usage {
    my $msg = shift // "";
    my $help  = q(
Usage:
        --plabel=PLABEL         The plabel we are shrinking. Required.
        --delete                Deletes the job. Will not clean up any made partitions.
        --min=.8                Decimal percent for the minimum we will resize a drive to in relation to the homesize of the userbox.
                                This is .15 (15%) by default.  Boxes that are not expecting growth can resize more aggressively then those that are not.
        --force                 If this is set we will finalize the migration regardles of the QA check after one rsync
        --reset-failures        Resets the failure count in the job
        --list                  Lists active and completed jobs
        --find                  Generates a list of the best partititions to resize
        --help                  Prints this message.
        --background            Kick off the shrink in the background
        --run-jobs              Process the queue and make sure we have jobs running if needed.
        --cleanup               Removes all old lvs (interactive)
        --status                Print the status  of a job.

);
    print "$help" . "$msg\n";
    exit;
}
sub make_pid {
    my $alive;
    if ($job->{'pid'}) {
        $alive = (kill 0, $job->{'pid'});
        logger("$userbox has an active migraiton as pid $job->{'pid'}\n",'FATAL') if ( $alive );
    }
    $job->{'pid'} = $$;
    update_job();
}
sub run {
    my $cmd = shift;
    my $options = shift;
    logger("Running: $cmd",'INFO');
    my $res = qx($cmd);
    logger("$res",'INFO');
    my $fail = ($options->{'allow-errors'}) ? 'INFO' : 'FAIL';
    logger("Failed to run '$cmd':$!",$fail) if ($? != 0);
}
sub list {
    my $jobs =  get_all_jobs();
    if (keys %$jobs){
        print "Active Jobs:\n";
        foreach my $job (keys %$jobs){
        if (defined $jobs->{$job}->{'pid'}) {
                print "$job\n";
                print "\tError: $jobs->{$job}->{'error'}\n" if $jobs->{$job}->{'error'};
            }
        }
        print "Queued Jobs:\n";
        foreach my $job (keys %$jobs){
            if (! defined $jobs->{$job}->{'pid'}  &&  $jobs->{$job}->{'failures'} < 5 ){
                print "$job\n";
                print "\tError: $jobs->{$job}->{'error'}\n" if $jobs->{$job}->{'error'};
            }
        }
        print "Failed Jobs:\n";
        foreach my $job (keys %$jobs){
            if (! defined $jobs->{$job}->{'pid'}  &&  $jobs->{$job}->{'failures'} >= 5 ){
                print "$job\n";
                print "\tError: $jobs->{$job}->{'error'}\n" if $jobs->{$job}->{'error'};
            }
        }
    }
    my @lvs = qx(lvs);
    my @cleanups;
    foreach my $line (@lvs){
        my @label = split /\s+/, $line;
        next unless $label[1] =~ m/^\s*([a-z0-9\.]+)\.old$/;
        next if (defined $jobs->{$1});
        push @cleanups, $1;
    }
    if (@cleanups){
        print "Completed Jobs to cleanup:\n";
        printf "%s", join("\n",@cleanups);
    }
    print "\n";;
    exit;
}
sub get_all_jobs {
    my $job_folder = "/var/spool/backup_shrinks/";
    my $jobs;
    mkdir $job_folder unless -d $job_folder;
    opendir (my $dh, $job_folder);
    while (my $file = readdir $dh){
        next if ($file =~ m/^\./);
        $jobs->{$file} = LoadFile("$job_folder/$file");
    }

    #Find out which jobs are still running.
    for my $job (keys %$jobs){
        if (defined $jobs->{$job}->{'pid'}){
            if (kill 0, $jobs->{$job}->{'pid'}){
                #logger("Found $job with an active pid",'INFO');
            } else {
                logger("Found $job with a dead pid",'INFO');
                delete $jobs->{$job}->{'pid'};
                DumpFile("$job_folder/$job",$jobs->{$job});
            }
        }
    }
    return $jobs;
}
sub find {
    my $bk = new BackupServer;
    my $partitions = $bk->get_partitions();
    my $mounts;
    foreach my $id (keys %$partitions){
        $mounts->{$partitions->{$id}->{'plabel'}} =  $partitions->{$id};
    }
    my @df = qx(df -P);
    foreach my $line (@df){
        my ($dev, $size, $used) = split /\s++/, $line;
        next unless $dev =~ m|^/dev/mapper/backup-(\S++)|;
        $plabel = $1;
        next unless defined $mounts->{$plabel};
        $mounts->{$plabel}->{'reclaimed-space'} = sprintf "%.03f", ($size - $used/.8)/1024/1024/1024;
        my $min = $minimumresize;
        $min = .85 if  ($mounts->{$plabel}->{'chostname'} =~ m/(((box|host|just|fast)(2\d{3}|5[1-9]\d{2}))|hostgator)/);
        my $reclaimed = $size/1024/1024/1024 - $mounts->{$plabel}->{'homesize'} * $min ;
        $mounts->{$plabel}->{'reclaimed-space'} = sprintf ("%.03f",$reclaimed) if $reclaimed < $mounts->{$plabel}->{'reclaimed-space'};
    }
    print "Plabels we can free up .5T or more:\n";
    my $total = 0;
    my @mounted_plabels;
    foreach my $plabel (keys %$mounts){
        push @mounted_plabels, $plabel if defined $mounts->{$plabel}->{'reclaimed-space'}
    }
    foreach my $plabel ( sort { $mounts->{$b}->{'reclaimed-space'} <=> $mounts->{$a}->{'reclaimed-space'} } @mounted_plabels ){
        last if $mounts->{$plabel}->{'reclaimed-space'} <= .5;
        $total += $mounts->{$plabel}->{'reclaimed-space'};
        print "$plabel: $mounts->{$plabel}->{'reclaimed-space'}\n";
    }
    print "Estimated total free space to free up: $total\n";
    exit;
}
sub run_jobs {
    my $jobs =  get_all_jobs();
    my $active = 0;
    for my $job (keys %$jobs){
        $active++ if defined $jobs->{$job}->{'pid'};
    }
    my $threads = 4;
    #kick off a couple threads if we can
    for my $job (sort { $jobs->{$a}->{'failures'} <=> $jobs->{$b}->{'failures'} }  keys %$jobs){
        exit if $active >= $threads;
        next if defined $jobs->{$job}->{'pid'};
        next if $jobs->{$job}->{'failures'} >= 5;
        logger("Kicking off backup_shrink for $job",'INFO');
        my $pid;
        $pid = fork();
        if ($pid == 0) {
            exec("/opt/eig_linux/bin/partition_shrink --plabel=$job --background > /dev/null");
        } else {
            sleep 20;
            $active++;
        }
    }
    exit;
}
sub cleanup {
    my $jobs = get_all_jobs();
    my @lvs = qx(lvs);
    foreach my $line (@lvs){
        my @label = split /\s+/, $line;
        next unless $label[1] =~ m/^\s*([a-z0-9\.]+)\.old$/;
        next if (defined $jobs->{$1});
        system("lvremove /dev/backup/$1.old") if -e "/dev/backup/$1.old";
    }
    exit;
}
sub status {
    $job_file = "/var/spool/backup_shrinks/$plabel";
    die("No job found for plabel $plabel\n") unless( -f $job_file );
    $job = LoadFile($job_file);
    print "Status of $plabel:\n";
    print "Job status:\n";
    foreach my $key (sort keys %$job){
        print "\t$key: $job->{$key}\n";
    }
    print "\nVolume status:\n";
    my @df = qx(df -h | grep $plabel);
    print "\t" . join("\t",@df);
    my @lvs = qx(lvs | grep $plabel);
    print "\t" . join("\t",@lvs);
    my $userbox_ip = inet_ntoa(inet_aton($userbox));
    my $maint = -f "/etc/sysconfig/maint-$userbox_ip" ? "ENABLED" : "DISABLED";
    print "\nMaintenance is $maint\n";
    print "\nLog file is /var/log/backup_shrinks/server_logs/$plabel.log:\n";
    print "Last 10 lines of log:\n";
    my @logs = qx(tail -n 10 /var/log/backup_shrinks/server_logs/$plabel.log);
    print "\t" . join("\t",@logs);
    exit;
}
