#!/usr/bin/perl
#
# used to download old state files and merge their state files into the current state directory
# this allows tools that use the state files to see what old backups are available
#


use strict;
use File::Slurp;
use Fcntl qw(:flock);
use YAML qw(LoadFile DumpFile);
use Sys::Hostname;
use Data::Dumper;
$Data::Dumper::Sortkeys = 1;
use FindBin qw($Bin);



my $args = {@ARGV};
my $stats = {};

# no point going onward if gsutil isn't configured
if(! -e "/root/.config/gcloud/active_config"){
  #$bk->logger('LOG_ERR',"gcloud doesn't appear to be configured properly");
  exit;
}

# some default config entries
my $conf = {
  state_dir => '/opt/eig_linux/data/obj-backups',
  backup_set_size => 30,
  bucketsuffix => '.prod.drshared.gcp.techops.tools',
  sufficient_size_diff => 1024,
};

# merge on whatever you specify as arguments
foreach my $k (keys %$args){
  $conf->{$k} = $args->{$k};
}

if(defined($conf->{server})){
  process_server($conf->{server});
}elsif(-e "$Bin/../lib/BackupServer.pm"){
  use lib "$Bin/../lib";
  require BackupServer;
  my $bk = BackupServer->new($args);
  my $dr = $bk->get_gcp_dr_clients();
  foreach my $server (sort keys %$dr){
    process_server($server);
  }
}else{
  # otherwise, attempt to use ourselves as the hostname
  process_server(hostname());
}
print Dumper($stats);
exit;

sub process_server {
  my $server = shift;
  print "processing server: $server\n";

  my @results = qx|gsutil ls -l gs://${server}.prod.drshared.gcp.techops.tools/state/|;
  my $r = {};
  chomp(@results);
  foreach my $ol (@results){
    $ol =~ s/^ +//g;
    my @s = split(' +',$ol);
    if($s[2] =~ m/obj-state\.([0-9]+)\.tar\.bz2$/){
      $r->{$1}->{uri} = $s[2];
      $r->{$1}->{timestr} = $s[1];
      $r->{$1}->{size} = $s[0];
    }
  }
  my $size = 0;
  my $changed_files = 0;
  # go through them in reverse order
  foreach my $date (sort { $b <=> $a } keys %$r){
    $stats->{process_server}->{state_files_considered}++;
    #print Dumper($r->{$date});
    # is the next oldest backup larger than the last one we looked at, should probably merge it
    if($r->{$date}->{size} > ($size + $conf->{sufficient_size_diff})){
      print "$stats->{process_server}->{files_considered} proccess $r->{$date}->{uri} cause $size is smaller than $r->{$date}->{size} ".Dumper($r->{$date}) if defined($conf->{debug_server});
      $changed_files += process_server_state($date,$server,$r->{$date}->{uri});
      $stats->{process_server}->{state_files_processed}++;
    }
    $size = $r->{$date}->{size};
  }
  print "done with $server, changed $changed_files files\n";
}

sub process_server_state {
  my $date = shift;
  my $server = shift;
  my $uri = shift;
  print "  processing stateball $server $uri\n";
  my $working_dir = "/tmp/obj-backup.$date.$server";
  system("mkdir $working_dir") if ! -e $working_dir;
  if ( ! -e "$working_dir/$server/"){
    my $cmd = "gsutil cat $uri|tar -j -C $working_dir/ -x";
    print "    running $cmd\n";
    system($cmd);
  }

  my $src_dir = "$working_dir/$server";
  my $dst_dir = "$conf->{state_dir}/$server";
  system("mkdir -p $conf->{state_dir}/$server") if ! -e "$conf->{state_dir}/$server";
  # we cannot just copy from src to dest because the system will think it can do an incremental from the last backup
  # there are many cases where that would be fine, but if the old backup server had a system in queue
  # and had events but didn't get to them before the migration, that would be bad
  # hence we merge everything..

  # no need to iterate through destination folder, if we don't have anything in source, there is nothing to merge
  my $changed_files = 0;
  foreach my $file (read_dir($src_dir)){
    next if substr($file,-5) eq '.lock';
    if(defined($conf->{user})){
      next if substr($file,0,length($conf->{user})) ne $conf->{user};
    }
    print "merging $file\n" if defined($conf->{debug_merge});;
    $changed_files += merge_files("$src_dir/$file","$dst_dir/$file");
  }
  print "  made changes to $changed_files files\n";
  return $changed_files;
}


sub merge_files {
  my $src_file = shift;
  my $dst_file = shift;

  if( ! -e $src_file){
    #shouldn't happen but let's be safe anyway
    $stats->{merge_files}->{src_nonexistant}++;
    return 0;
  }

  print "    loading $src_file\n" if defined($conf->{debug_merge});
  my $src_data = {};
  my $src_data = LoadFile($src_file);
  print "SRC: ".Dumper($src_data) if defined($conf->{debug_merge});

  print "    loading $dst_file\n" if defined($conf->{debug_merge});
  my $dst_data = {};
  open(my $lfh,'>>',"$dst_file.lock");
  flock ($lfh, LOCK_EX) || die "flock: $!";
  if( -e $dst_file){
    $dst_data = LoadFile($dst_file);
  }
  #DumpFile($state_file,$state_data)
  print "DST: ".Dumper($dst_data) if defined($conf->{debug_merge});
  # merge URIs

  my $changes = 0;

  # convert backup state to use variables imbedded in the uris values instead of previous and current backup set
  # this is temporary
  foreach my $key (qw( previous_backup_set current_backup_set )){
    if(defined($src_data->{$key})){
      my $cnt = 0;
      my $set = 0;
      foreach my $backup (sort { $a <=> $b } keys %{$src_data->{$key}}){
        $set = $backup if $set == 0;
        my $uri = $src_data->{$key}->{$backup}->{uri};
        $src_data->{uris}->{$uri}->{backup_set} = $set;
        $src_data->{uris}->{$uri}->{backup_set_pos} = $cnt;
        $cnt++;
      }
      delete($src_data->{$key});
    }
  }

  # convert backup state to use variables imbedded in the uris values instead of previous and current backup set
  # this is temporary
  foreach my $key (qw( previous_backup_set current_backup_set )){
    if(defined($dst_data->{$key})){
      my $cnt = 0;
      my $set = 0;
      foreach my $backup (sort { $a <=> $b } keys %{$dst_data->{$key}}){
        $set = $backup if $set == 0;
        my $uri = $dst_data->{$key}->{$backup}->{uri};
        $dst_data->{uris}->{$uri}->{backup_set} = $set;
        $dst_data->{uris}->{$uri}->{backup_set_pos} = $cnt;
        $cnt++;
      }
      delete($dst_data->{$key});
      $changes++;
    }
  }

  foreach my $uri (keys %{$src_data->{uris}}){
    # if dst doesn't know about that uri, it's easy
    if(! defined($dst_data->{uris}->{$uri})){
      $dst_data->{uris}->{$uri} = $src_data->{uris}->{$uri};
    # is the timestamp on the src newer (greater) than the dst?
      $changes++;
    }elsif($dst_data->{uris}->{$uri}->{time} < $src_data->{uris}->{$uri}->{time}){
      $dst_data->{uris}->{$uri} = $src_data->{uris}->{$uri};
      $changes++;
    }
  }
  if($changes){
    $stats->{merge_files}->{files_changed}++;
    print "RESULTS: ".Dumper($dst_data) if defined($conf->{debug_merge});
    if(defined($conf->{skip_write})){
      print "would have written $dst_file except skip_write was set\n";
    }else{
      print "changes made from $src_file ; saving $dst_file\n" if defined($conf->{debug_merge});
      DumpFile("$dst_file",$dst_data) if ! defined($conf->{skip_write});
    }
  }else{
    $stats->{merge_files}->{files_unchanged}++;
    print "no changes, not saving $dst_file\n" if defined($conf->{debug_merge});
  }

  close($lfh);
  return 1 if $changes;
}



