#!/usr/bin/perl
#
# used to pull down objects out of backups and hydrate them onto the disk somewhere
#

use strict;
use File::Slurp;
use Fcntl qw(:flock);
use YAML qw(LoadFile DumpFile);
use Sys::Hostname;
use Data::Dumper;
$Data::Dumper::Sortkeys = 1;
$Data::Dumper::Indent = 2;
use FindBin qw($Bin);



my $args = {@ARGV};
my $stats = { time_start => time() };

# no point going onward if gsutil isn't configured
if(! -e "/root/.config/gcloud/active_config"){
  #$bk->logger('LOG_ERR',"gcloud doesn't appear to be configured properly");
  exit;
}

# some default config entries
my $conf = {
  state_dir => '/opt/eig_linux/data/obj-backups',
  backup_set_size => 30,
  bucketsuffix => '.prod.drshared.gcp.techops.tools',
  state_age_limit => 7,
};

# some defaults for gsutils

# merge on whatever you specify as arguments
foreach my $k (keys %$args){
  $conf->{$k} = $args->{$k};
}
if(! defined($conf->{server})){
  show_possible_servers();
}
if(! defined($conf->{user})){
  show_possible_users($conf->{server});
}

my $ud = load_user_state($conf->{server},$conf->{user});

print Dumper($ud) if defined($conf->{dump_userdata});
dump_uris($ud) if defined($conf->{dump_uris});

if(! defined($conf->{restore_point})){
  print "\nmissing \"restore_point\"\n";
  show_restore_points($ud)
}
$conf->{restore_point} = $ud->{latest_complete} if $conf->{restore_point} eq 'latest';
if(! defined($ud->{restore_points}->{$conf->{restore_point}})){
  print "\ninvalid restore_point of \"$conf->{restore_point}\"\n";
  show_restore_points($ud);
}
if(! defined($conf->{restore_dir}) or ! -e $conf->{restore_dir}){
  print "\nspecify an existing directory to restore into using restore_dir argument\n";
  show_usage();
  exit;
}

print "here we do restore of restore_point $conf->{restore_point}\n";

my $log_file = sprintf("/tmp/%s_%s_%s",$conf->{server},$conf->{user},$conf->{restore_point});
restore_files($ud,$conf->{restore_point},$conf->{restore_dir},$log_file);
if(defined($conf->{show_stats})){
  $stats->{time_end} = time();
  $stats->{time_diff} = $stats->{time_end} - $stats->{time_start};
  print Dumper($stats);
}
exit;

sub show_usage {
  print "\nusage:\n";
  print "$0 server <servername> user <username> restore_point <rp> restore_dir <dir>\n\n";
}

sub restore_files {
  my $ud = shift;
  my $rp = shift;
  my $rd = shift;
  my $lf = shift;

  my $file_num = 0;
  foreach my $uri (@{$ud->{restore_points}->{$rp}->{uris}}){
    $stats->{tarballs}++;
    my $tbs;
    my $log_file = $lf.'.'.$file_num;
    print "download and untar $uri into $rd log_file: $log_file\n";

    open(my $logfh,'>',$log_file);
    $file_num++;
    my $cmd = sprintf("/usr/bin/gsutil cp %s - |/usr/bin/lz4 -d |/usr/bin/tar --totals -C %s --strip-components=1 --verbose --verbose --numeric-owner --extract 2>&1",$uri,$rd);
    print "$cmd\n"; # if defined($args->{debug_steps});
    $stats->{tar_downloaded}++;
    open(my $tarfh,'-|',$cmd);
    while(my $tarline = <$tarfh>){
      #print $tarline if $tarline =~ m/delete.log/; 
      #print $tarline if defined($conf->{debug_restore});
      print $logfh $tarline;
      my @parts = split('\s+',$tarline);
      if(substr($tarline,0,16) eq 'Total bytes read'){
        #print "total: $tarline";
        $tbs->{tar_read_bytes} += $parts[3];
      }else{
        #print $tarline;
        $tbs->{extracted_files}++;
        $tbs->{extracted_bytes} += $parts[2];
      }
    }
    my $delete_file = "$rd/delete.log";
    unlink($delete_file) if -z $delete_file;
    if(-e $delete_file){
      print "handling $delete_file\n";# if defined($conf->{debug_restore});
      $stats->{delete_logs}++;
      foreach my $file (read_file($delete_file)){
        chomp($file);
        #my $slash_pos = index($file,'/');
        my $realfile = "$rd".substr($file,index($file,'/'));
        if( -f $realfile){
          $tbs->{delete_bytes} += -s _;
          $tbs->{delete_files}++;
          print $logfh "deleted $realfile\n";
          unlink($realfile);
        }elsif(-d _ ){
          my @dir_files = read_dir($realfile);
          my $num_files = scalar(@dir_files);
          if($num_files == 0){
            $tbs->{delete_dir}++;
            rmdir($realfile);
          }else{
            $tbs->{delete_dir_fail}++;
            print $logfh "cowardly refusing to unlink directory $file with $num_files files\n";
          }
        }else{
          $tbs->{delete_404}++;
          print $logfh "WARNING delete.log:\n";
          print $logfh "  from log:  $file\n";
          print $logfh "  not found: $realfile\n";
        }
      }
      unlink($delete_file);
    }
    close($logfh);
    close($tarfh);
    print Dumper($tbs) if defined($conf->{debug_restore});
    foreach my $k (keys %$tbs){
      $stats->{$k} += $tbs->{$k};
    }
  }
}


sub dump_uris {
  my $ud = shift;
  $Data::Dumper::Indent = 0;
  foreach my $uri (sort keys %{$ud->{uris}}){
    my @ol;
    push(@ol,$ud->{uris}->{$uri}->{uri});
    delete($ud->{uris}->{$uri}->{uri});
    foreach my $k (sort keys %{$ud->{uris}->{$uri}}){
      push(@ol,"$k=>$ud->{uris}->{$uri}->{$k}");
    }
    print join(',',@ol)."\n";
  }
  exit;
}

sub show_restore_points {
  my $ud = shift;

  show_usage();
  print "\nyou must specify one of these restore_points:\n";
  
  foreach my $time (sort { $a <=> $b } keys %{$ud->{restore_points}}){
    my $timestr = localtime($time);
    my $num_objects = scalar(@{$ud->{restore_points}->{$time}->{uris}});
    printf("%-12s status: %12s date: %24s Num_Obj: %2d\n",
           $time,
           $ud->{restore_points}->{$time}->{status},
           $timestr,
           $num_objects,
           );
    print Dumper($ud->{restore_points}->{$time}) if defined($conf->{debug_restore_options});
  }
  print "\nyou can also specify \"latest\" to pick the latest complete backup: $ud->{latest_complete}\n";
  print "\n";
  exit;
}

sub load_user_state {
  my $server = shift;
  my $user = shift;
  my $state_file = "$conf->{state_dir}/$server/$user.yaml";
  if(! -e $state_file){
    print "server $server user $user has no valid statefile at $state_file\n";
    exit;
  }
  my $age = -M _;
  if( $age > $conf->{state_age_limit}){
    print "\nWARNING WARNING WARNING WARNING WARNING WARNING WARNING\n";
    print " user $user has a state file that is $age days old\n";
    print "\nWARNING WARNING WARNING WARNING WARNING WARNING WARNING\n";
  }

  
  open(my $lfh,'<',"$state_file.lock");
  flock ($lfh, LOCK_SH) || die "flock: $!";
  my $state = LoadFile($state_file);
  close($lfh);
  #print Dumper($state);
  my $unknown_bs_num = 0;
  my $bad_bs_num = 0;
  my $restore_points = {};
  my $backup_sets = {};
  #ensure required pieces of information exist and create backup_sets hash
  foreach my $uri(sort {$state->{uris}->{$a}->{time} <=> $state->{uris}->{$b}->{time}} keys %{$state->{uris}}){
    if(! defined($state->{uris}->{$uri}->{backup_set})){
      $state->{uris}->{$uri}->{backup_set} = "unknown$unknown_bs_num";
      $unknown_bs_num++;
      $state->{uris}->{$uri}->{backup_set_pos} = 0;
    }
    if($state->{uris}->{$uri}->{result} ne 'success'){
      $state->{uris}->{$uri}->{backup_set} = "failure$bad_bs_num";
      $bad_bs_num++;
      $state->{uris}->{$uri}->{backup_set_pos} = 0;
    }
    $backup_sets->{$state->{uris}->{$uri}->{backup_set}}->{$state->{uris}->{$uri}->{backup_set_pos}} = $uri;
  }
  my $latest_complete = 0;
  # check to see if each backup is valid (needs backup_sets populated)
  foreach my $uri(sort {$state->{uris}->{$a}->{time} <=> $state->{uris}->{$b}->{time}} keys %{$state->{uris}}){
    my $missing = 0;
    for(my $pos = 0; $pos <= $state->{uris}->{$uri}->{backup_set_pos}; $pos++){
      if(defined($backup_sets->{$state->{uris}->{$uri}->{backup_set}}->{$pos})){
        push(@{$restore_points->{$state->{uris}->{$uri}->{time}}->{uris}},$backup_sets->{$state->{uris}->{$uri}->{backup_set}}->{$pos});
      }else{
        push(@{$state->{uris}->{$uri}->{restore_chain}},'missing');
        $missing++;
      }
    }
    $restore_points->{$state->{uris}->{$uri}->{time}}->{missing} = $missing;
    if($missing){
      $restore_points->{$state->{uris}->{$uri}->{time}}->{status} = 'incomplete';
    }elsif($state->{uris}->{$uri}->{backup_set} =~ m/^failure/){
      $restore_points->{$state->{uris}->{$uri}->{time}}->{status} = 'questionable';
    }elsif($state->{uris}->{$uri}->{backup_set} =~ m/^unknown/){
      $restore_points->{$state->{uris}->{$uri}->{time}}->{status} = 'questionable';
    }else{
      $latest_complete = $state->{uris}->{$uri}->{time} if $state->{uris}->{$uri}->{time} > $latest_complete;
      $restore_points->{$state->{uris}->{$uri}->{time}}->{status} = 'COMPLETE';
    }
    $restore_points->{$state->{uris}->{$uri}->{time}}->{backup_set} = $state->{uris}->{$uri}->{backup_set};
    printf("time %s bs: %s bsp %s uri: %s\n",
             $state->{uris}->{$uri}->{time},
             $state->{uris}->{$uri}->{backup_set},
             $state->{uris}->{$uri}->{backup_set_pos},
             $uri,
          ) if defined($conf->{debug_load_user_state});
    #print Dumper($state->{uris}->{$uri});

  }
  #print "latest_complete $latest_complete\n";
  #print Dumper($restore_points);
  return { latest_complete => $latest_complete, uris => $state->{uris}, restore_points => $restore_points};
}

sub show_possible_servers {
  print "missing \"server\"\n";
  show_usage();
  my @servers = read_dir($conf->{state_dir});
  if(@servers == 0){
    print "  looks like we don't have any server data directories;\n";
    print "  do you need to run obj-backup-pull-state?\n";
    print "  then call this script again?\n";
    print "$0 server ".hostname()."\n";
  }else{
    print "\nyou must specify one of these servers:\n\n";
    foreach my $server (sort @servers){
      print "$server\n";
    }
  }
  print "\n";
  exit;
}

sub show_possible_users {
  my $server = shift;
  my $server_dir = "$conf->{state_dir}/$server";
  if(! -e $server_dir){
    print "server '$server' is invalid, state_dir $server_dir does not exist\n";
    show_possible_servers();
    exit;
  }
  print "missing \"user\"\n";
  show_usage();
  my @state_files = read_dir($server_dir);
  my @users;
  print "\nyou must specify one of these users:\n";
  print "users not listed => no state file for user\n\n";
  foreach my $file (sort @state_files){
    next if substr($file,-5) ne '.yaml';
    print substr($file,0,-5)."\n";
  }
  print "\n";
  exit;
}

