#!/usr/bin/perl
use strict;
use Data::Dumper;
$Data::Dumper::Sortkeys = 1;
$Data::Dumper::Indent = 0;
use File::Path qw(make_path);
use File::Slurp;
use Proc::Daemon;
use Sys::Syslog;
use Fcntl qw(:flock);
use YAML qw(LoadFile DumpFile);
use Sys::Hostname;
use LWP::UserAgent;

use FindBin qw($Bin);
use lib "$Bin/../lib";
use BackupServer;

# perl obj-backup-worker host box2188.bluehost.com partition_number 2 user tesaaa1234


#####################################
# Initialization and info gathering
#####################################

#set this so syslog has a nice name
$0 = "obj-backup-worker";

my $args = {@ARGV};

# basic arg validation, more comprehensive is done below
foreach my $k ( qw( host partition_number user )){
  if(! defined($args->{$k})){
    print "you must specify $k as an argument\n";
    exit 1;
  }
}

# create variable to store information about this backup run
my $backup_run = {};
$backup_run->{time} = time();
$backup_run->{timestr} = localtime($backup_run->{time});
$backup_run->{server} = hostname();

# create backupserver object
my $bs_args = {};
$bs_args->{debug_stdout} = 1 if defined($args->{stdout});
my $bk = BackupServer->new($bs_args);

# some defaults for gsutils
my $GSUTIL_ARGS = "";
$GSUTIL_ARGS .= "-q" if ! defined($args->{stdout});
my $GSUTILS_LOG_FILE = "/tmp/$args->{user}.$backup_run->{time}.log";
my $GSUTIL_CP_ARGS = "-L $GSUTILS_LOG_FILE";

# no point going onward if gsutil isn't configured
if(! -e "/root/.config/gcloud/active_config"){
  $bk->logger('LOG_ERR',"gcloud doesn't appear to be configured properly");
  exit;
}

# some default config entries
my $conf = {
  state_dir => '/opt/eig_linux/data/obj-backups/',
  backup_set_size => 30,
  max_full_age => 20,
  min_full_age => 14,
  max_user_size => 15,
};
foreach my $k (keys %$conf){
  my $val = $bk->get_config_value("obj_${k}");
  if(defined($val)){
    $conf->{$k} = $val;
  }
}

if(! is_gcp_up()){
  $bk->logger('LOG_ERR',"unable to connect to storage.googleapis.com");
  exit;
}


#######################################################
# change pid name and move to background if asked for
#######################################################

# disconnect from calling process
if(defined($args->{background})){
  $bk->logger('LOG_NOTICE',"backgrounding for user: $args->{user}");
  Proc::Daemon::Init();
}

$bk->logger('LOG_NOTICE',sprintf("starting work on %s %s %s",$args->{host},$args->{"partition_number"},$args->{user}));

# create pid name
my $server_part_user_str = sprintf("%s %s %s",$args->{host},$args->{"partition_number"},$args->{user});
$0 = "obj-backup-worker $server_part_user_str";

# ensure no other process is working on this account right now
# must have this after modification of $0 otherwise we may not be using a unique pidfile
use Proc::PID::File;
if(Proc::PID::File->running( name => "$server_part_user_str")){
  $bk->logger('LOG_NOTICE',"Already running!");
  exit 1;
}

#####################################################################
# after this point any failures will get reported in the state file
#####################################################################

# create server state directory if it doesn't exist
my $server_state_dir = $conf->{state_dir}."/".$args->{host};
if( ! -e $server_state_dir){
  make_path($server_state_dir);
}
my $state_file = "${server_state_dir}/$args->{user}.yaml";
# load past state data if exists
my $state_data = load_state_file($state_file);

# check on the size of the use
# that is if we aren't running on a special system "user"
if($args->{user} =~ m/(dirs.files|non-user-mysql)/){
  $bk->logger('LOG_NOTICE',"system user $args->{user}");
}else{
  my $userinfo = get_userinfo($args->{host},$args->{user});
  if(! defined($userinfo->{diskusage})){
    $bk->logger('LOG_NOTICE',"user has no diskusage in userinfo file, not backing the user up");
    $backup_run->{result} = "skipped: no diskusage found in userinfo file";
    update_state_file($state_file,$state_data,$backup_run);
    exit;
  }
  $backup_run->{diskusage} = $userinfo->{diskusage};
  if($userinfo->{diskusage} > $conf->{max_user_size}){
    $bk->logger('LOG_ERR',"user over max size $userinfo->{diskusage} > $conf->{max_user_size}");
    $backup_run->{result} = "skipped: user diskusage over max size $userinfo->{diskusage} > $conf->{max_user_size}";
    update_state_file($state_file,$state_data,$backup_run);
    exit;
  }
}

# setup some working directory paths we need for later
my $prefix_dir =  sprintf("/home/%s/backup%s/cpbackup/daily",$args->{host},$args->{"partition_number"});
my $user_dir = $args->{user};
if($args->{user} eq 'dirs_files'){
  $user_dir = "dirs.files";
  $prefix_dir = sprintf("/home/%s/backup%s",$args->{host},$args->{"partition_number"});
}elsif($args->{user} eq 'non-user-mysql'){
  $prefix_dir = sprintf("/home/%s/backup%s",$args->{host},$args->{"partition_number"});
}

if(! -d "$prefix_dir/$user_dir"){
  $bk->logger('LOG_ERR',"$prefix_dir/$user_dir does not exist");
  $backup_run->{result} = "skipped: $prefix_dir/$user_dir does not exist";
  update_state_file($state_file,$state_data,$backup_run);
  exit 1;
}

$backup_run->{uri} = get_bucket_uri($args->{host},$args->{user});

# start by assuming we are doing a full backup, it will change later if appropriate
$backup_run->{type} = "full";

# have we done a backup into the same slot recently??
if(defined($state_data->{last_backup})){
  my $last_backup_age = ($backup_run->{time} - $state_data->{last_backup}->{time}) / 86400;
  # if our last backup uses the space we were intending to do a backup
  # then don't do a backup right now
  if($backup_run->{uri} eq $state_data->{last_backup}->{uri} and $last_backup_age < 2){
    $bk->logger('LOG_NOTICE',sprintf("we attemped to backup into uri %s only %.2f days ago", $backup_run->{uri}, $last_backup_age));
    if(! defined($args->{force})){
      $bk->logger('LOG_NOTICE',"QUITTING: try again later");
      $backup_run->{result} = sprintf("skipped: %.2f is too soon to backup into %s",$last_backup_age, $backup_run->{uri});
      update_state_file($state_file,$state_data,$backup_run);
      exit;
    }
  }
  $bk->logger('LOG_NOTICE',sprintf("last backup was to %s and is %.2f days old",$state_data->{last_backup}->{uri},$last_backup_age));
}

# have we done a full backup before and recently?
if(defined($state_data->{last_full_backup})){
  my $full_backup_age = ($backup_run->{time} - $state_data->{last_full_backup}->{time}) / 86400;
  # do an incremental backup if the full backup is too young to spinup another
  my $prob = 0;
  if ($full_backup_age >= $conf->{max_full_age}) {
    $prob = 1;
  } elsif ($full_backup_age > $conf->{min_full_age} ) {
    $prob = 1 / ($conf->{max_full_age} - $full_backup_age);
  }
  if ($prob < rand()) {
    $bk->logger('LOG_NOTICE',sprintf("last full backup was %.2f days old, doing incremental",$full_backup_age));
    $backup_run->{type} = 'incremental';
  }else{
    $bk->logger('LOG_NOTICE',sprintf("last full backup was %.2f days old, doing full backup",$full_backup_age));
  }
}
# non-user-mysql is special and gets a "full" backup every day
$backup_run->{type} = 'full' if $args->{user} eq 'non-user-mysql';

# array of log files we need to delete if things go well
my $event_files = get_event_files($args->{host},$args->{partition_number},$args->{user});
$bk->logger('LOG_DEBUG',Dumper($event_files)) if defined($args->{debug_event_selection});

$bk->logger('LOG_NOTICE',"backup type: $backup_run->{type}");

#####################
# full backup
#####################
if($backup_run->{type} eq "full"){

  my $tar_exclude = "";
  if($args->{user} eq 'dirs_files'){
    $tar_exclude = "--exclude monthly --exclude weekly --exclude logs --exclude new --exclude sessions";
  }

  # "-s ARCHIVE" is SUPER IMPORTANT, without it the object will cost 16x more if it is left out and the backup is stored as a STANDARD object
  my $cmd = "/usr/bin/tar --create --sparse --xattrs $tar_exclude -C $prefix_dir/ $user_dir |/usr/bin/lz4 |/usr/bin/gsutil $GSUTIL_ARGS cp $GSUTIL_CP_ARGS -s ARCHIVE - $backup_run->{uri}";
  if($args->{user} eq 'non-user-mysql'){
    $cmd = "/usr/bin/tar --create --sparse --xattrs $tar_exclude -C $prefix_dir/ $user_dir/mysql.1.sql |/usr/bin/lz4 |/usr/bin/gsutil $GSUTIL_ARGS cp $GSUTIL_CP_ARGS -s ARCHIVE - $backup_run->{uri}";
  }

  #add some magic flag files into the tarball
  $bk->logger('LOG_INFO',"create file $user_dir/backup-type");
  write_file("$prefix_dir/$user_dir/backup-type","full\n");
  $bk->logger('LOG_INFO', "create file $user_dir/backup-is-full");
  write_file("$prefix_dir/$user_dir/backup-is-full","$backup_run->{time}\n");

  $bk->logger('LOG_NOTICE',"full backup command: $cmd");
  system($cmd) if ! defined($args->{skip_tar});

  #clean up after ourselves
  unlink("$prefix_dir/$user_dir/backup-type");
  unlink("$prefix_dir/$user_dir/backup-is-full");

  # check status
  my $response = check_gsutil_status($GSUTILS_LOG_FILE,$backup_run->{uri});
  if($response eq 'OK'){
    delete_event_files();
    $backup_run->{result} = 'success';
  }else{
    $backup_run->{result} = "failure: gsutil response $response";
  }


#####################
# INCREMENTAL BACKUP
#####################
}elsif($backup_run->{type} eq 'incremental'){

  # no event files, no point in doing incremental
  my @event_keys = keys %$event_files;
  if(@event_keys == 0){
    $bk->logger('LOG_ERR', "QUITTING: no point doing an incremental if we don't have any valid event files");
    $backup_run->{result} = sprintf("skipped: no event files");
    update_state_file($state_file,$state_data,$backup_run);
    exit;
  }

  # list of files to delete after restoring the tarball
  my $deletelogfile = "$prefix_dir/$user_dir/delete.log";

  # command to generate tarball
  # "-s ARCHIVE" is SUPER IMPORTANT, without it the object will cost 16x more if stored as a STANDARD class of object
  my $tarcmd = "/usr/bin/tar --create --sparse --xattrs -C $prefix_dir/ --files-from=- |/usr/bin/lz4 |/usr/bin/gsutil $GSUTIL_ARGS cp $GSUTIL_CP_ARGS -s ARCHIVE - $backup_run->{uri}";

  # start list of files to delete after tarball is restored
  open(my $delfh, '>', $deletelogfile) or die("couldn't open deletefile: $deletelogfile $!");

  # run tar command piped into obj-storage command
  $bk->logger('LOG_NOTICE', "incremental backup command: $tarcmd");
  open(my $tarfh, '|-', $tarcmd) or die("unable to start tar: $!") if ! defined($args->{skip_tar});


  $bk->logger('LOG_NOTICE', "create and insert file $user_dir/backup-type");
  # create magic backup_type_file and populate
  write_file("$prefix_dir/$user_dir/backup-type","incremental\n");
  # have tar include that file
  print $tarfh "$user_dir/backup-type\n" if ! defined($args->{skip_tar});

  # do the same for a backup-is-incremental file
  $bk->logger('LOG_INFO', "create and insert $user_dir/backup-is-incremental");
  $bk->logger('LOG_NOTICE',"last full backup $state_data->{last_full_backup}->{timestr}");
  write_file("$prefix_dir/$user_dir/backup-is-incremental","last full backup: $state_data->{last_full_backup}->{uri} $state_data->{last_full_backup}->{time} $state_data->{last_full_backup}->{timestr}\n");
  print $tarfh "$user_dir/backup-is-incremental\n" if ! defined($args->{skip_tar});

  # tar will gladly tar the same file repeatedly
  # event files may have repeat file entries
  # we don't want to tar them multiple times
  my $seen_recv_file = {};

  # go through each event file
  foreach my $ek (sort {$a <=> $b } keys %$event_files){
    foreach my $et (sort keys %{$event_files->{$ek}}){
      $backup_run->{event_files}++;
      my $event_file = $event_files->{$ek}->{$et};
      #save log file for later
      $bk->logger('LOG_NOTICE', "processing event file $event_file");

      #read log file
      open(my $rsynclogfh, '<', $event_file) or die("logfile open failure $event_file $!");
      my $skip_deletes = 0;
      $skip_deletes = 1 if $et eq 'pkgacct';

      while(my $ol = <$rsynclogfh>){
        chomp($ol);
        my $op_pos = index($ol,']') + 2;
        my $file_op = substr($ol,$op_pos,5);
        my $file_pos = $op_pos + 5;
        my $filename = $user_dir.'/'.substr($ol,$file_pos);

        #delete
        if($file_op eq 'del._'){
          if($skip_deletes == 0){
            $bk->logger('LOG_INFO',"Deleting: $filename");
            $backup_run->{files_deleted}++;
            print $delfh $filename."\n";
          }
        }elsif($file_op eq 'recv_'){
          if(defined($seen_recv_file->{$filename})){
            $backup_run->{files_duplicate}++;
          }else{
            $seen_recv_file->{$filename}++;
            if($filename =~ m#/homedir/tmp/#){
              # NO-OP
              $backup_run->{files_skip_tmp}++;
            }elsif($filename =~ m#homedir/\.trash#){
              # NO-OP
              $backup_run->{files_skip_trash}++;
            }elsif(-d "$prefix_dir/$filename"){
              $bk->logger('LOG_DEBUG', "will not tar directories $filename");
              $backup_run->{files_skip_directory}++;
            }elsif(-e "$prefix_dir/$filename"){ #only include things that exist
              $backup_run->{files_added_bytes} += -s _;
              $backup_run->{files_added}++;
              $bk->logger('LOG_INFO', "adding file to tarball: $filename");
              print $tarfh $filename."\n" if ! defined($args->{skip_tar});
            }
          }
        }elsif(defined($args->{debug_parsing})){
          $bk->logger('LOG_INFO', "OTHER $ol");
          $bk->logger('LOG_INFO', "op: =$file_op=");
          $bk->logger('LOG_INFO', "filename: =$filename=");
        }
      }
      close($rsynclogfh);
    }
  }
  $bk->logger('LOG_NOTICE', "last of all, adding delete file to tarball: $user_dir/delete.log");
  close($delfh);
  print $tarfh $user_dir."/delete.log\n" if ! defined($args->{skip_tar});
  close($tarfh) if ! defined($args->{skip_tar});

  # remove files we added, writing to tar can be buffered, don't assume the file is synced as soon as we print to tarfh
  unlink "$prefix_dir/$user_dir/backup-type";
  unlink "$prefix_dir/$user_dir/backup-is-incremental";
  unlink("$prefix_dir/$user_dir/delete.log");

  my $response = check_gsutil_status($GSUTILS_LOG_FILE,$backup_run->{uri});
  my $endtime = time();
  $backup_run->{time_duration} = $endtime - $backup_run->{time};
  if($response eq 'OK'){
    delete_event_files();
    $backup_run->{result} = 'success';
  }else{
    $backup_run->{result} = "failure: gsutil response $response";
  }
}

update_state_file($state_file,$state_data,$backup_run);


#### THE END #########################
## Only subroutines below this line ##
######################################

sub load_state_file {
  my $state_file = shift;
  if(! -e $state_file){
    return;
  }
  open(my $lfh,'<',"$state_file.lock");
  flock ($lfh, LOCK_SH) || die "flock: $!";
  my $state = LoadFile($state_file);
  close($lfh);
  return $state;
}
sub update_state_file {
  my $state_file = shift;
  my $state = shift;
  my $backup_run = shift;

  my $endtime = time();
  $backup_run->{time_duration} = $endtime - $backup_run->{time};

  # cleanup bad URI entries
  if(defined($state_data->{uris})){
    foreach my $key (keys %{$state_data->{uris}}){
      delete $state_data->{uris}->{$key} if $key eq '';
      delete $state_data->{uris}->{$key} if substr($key,0,4) eq 'HASH';
    }
  }

  # convert backup state to use variables embedded in the uris values instead of previous and current backup set
  foreach my $key (qw( previous_backup_set current_backup_set )){
    if(defined($state_data->{$key})){
      my $cnt = 0;
      my $set = 0;
      foreach my $backup (sort { $a <=> $b } keys %{$state_data->{$key}}){
        $set = $backup if $set == 0;
        my $uri = $state_data->{$key}->{$backup}->{uri};
        $state_data->{uris}->{$uri}->{backup_set} = $set;
        $state_data->{uris}->{$uri}->{backup_set_pos} = $cnt;
        $cnt++;
      }
      delete($state_data->{$key});
    }
  }

  # full backups start new backup sets
  # always position 0
  if($backup_run->{type} eq 'full'){
    $backup_run->{backup_set} = $backup_run->{time};
    $backup_run->{backup_set_pos} = 0;
  }else{
    # this would be an incremental on the last_full_backup
    $backup_run->{backup_set} = $state_data->{last_full_backup}->{time};
    my $cnt = 0;
    foreach my $uri (keys %{$state_data->{uris}}){
      if(defined($state_data->{uris}->{$uri}->{backup_set})){
        if($state_data->{uris}->{$uri}->{backup_set} eq $state_data->{last_full_backup}->{time} and $state_data->{uris}->{$uri}->{result} eq 'success'){
          $cnt++;
        }
      }
    }
    # and would be this position in the backup_set
    $backup_run->{backup_set_pos} = $cnt;
  }

  if($backup_run->{result} eq 'success'){
    # update last_full_backup if we had a successful full backup
    if($backup_run->{type} eq 'full'){
      # replace last_full_backup with this run
      delete($state_data->{last_full_backup}) if defined($state_data->{last_full_backup});
      foreach my $k (qw( uri time timestr )){
        $state_data->{last_full_backup}->{$k} = $backup_run->{$k};
      }
    }

    # replace last_backup with this run data if we were successful
    delete($state_data->{last_backup}) if defined($state_data->{last_backup});
    foreach my $k (qw( time timestr uri backup_set backup_set_pos)){
      $state_data->{last_backup}->{$k} = $backup_run->{$k};
    }

  }

  # we will replace whatever was in this uri slot with updated information
  # when the new state is not "skipped"
  # even if failure as we may have corrupted whatever was in that uri
  if($backup_run->{uri} ne '' and substr($backup_run->{result},0,7) ne 'skipped'){
    delete($state_data->{uris}->{$backup_run->{uri}}) if defined($state_data->{uris}->{$backup_run->{uri}});
    foreach my $k (keys %$backup_run){
      $state_data->{uris}->{$backup_run->{uri}}->{$k} = $backup_run->{$k};
    }
  }

  # generic run history of past run attempts, including failures
  foreach my $k (keys %$backup_run){
    $state_data->{run_history}->{$backup_run->{time}}->{$k} = $backup_run->{$k};
  }
  # only keep 10;
  my @rhk = sort { $a <=> $b } keys %{$state_data->{run_history}};
  while(@rhk > 10){
    my $rdate = shift(@rhk);
    delete($state_data->{run_history}->{$rdate});
  }

  open(my $lfh,'>>',"$state_file.lock");
  flock ($lfh, LOCK_EX) || die "flock: $!";
  DumpFile($state_file,$state_data);
  close($lfh);

  # one final line before we quit
  foreach my $k (qw( event_files files_added files_added_bytes files_deleted time_duration gsutils_bytes )){
    $backup_run->{$k} = -1 if ! defined($backup_run->{$k});
  }
  $bk->logger('LOG_NOTICE', "backup status user: $args->{user} type: $backup_run->{type} ef: $backup_run->{event_files} add: $backup_run->{files_added} add_bytes: $backup_run->{files_added_bytes} del: $backup_run->{files_deleted} seconds: $backup_run->{time_duration} gsb: $backup_run->{gsutils_bytes}");
}

sub delete_event_files {
  foreach my $ek (sort keys %$event_files){
    foreach my $et (sort keys %{$event_files->{$ek}}){
      my $event_file = $event_files->{$ek}->{$et};
      $bk->logger('LOG_INFO', "deleting consumed eventfile $event_file");
      unlink($event_file) if (-e $event_file);
    }
  }
}

sub get_event_files {
  my $host = shift;
  my $partition_number = shift;
  my $username = shift;
  my $results = {};

  my $eventsdir = sprintf("/home/%s/backup%s/events",$host,$partition_number);

  # All the event files in the event directory
  my @all_event_files = read_dir($eventsdir);

  my $userlength = length($username) + 1;

  foreach my $event_file ( @all_event_files ){
    if(substr($event_file,0,$userlength) eq $username.'.'){
      my @p = split('\.',$event_file);
      $results->{$p[2]}->{$p[1]} = "$eventsdir/$event_file";
    }
  }

  return $results;
}


sub check_gsutil_status {
  my $log_file = shift;
  my $uri = shift;
  if(defined($args->{skip_tar})){
    return "SKIP_TAR_IS_SET";
  }
  my @logfile = read_file("$log_file");
  foreach my $ol (@logfile){
    next if $ol !~ m/$uri/;
    $ol =~ s/[\n\r]$//;
    #chomp($ol);
    my @attrib = split(',',$ol);
    $bk->logger('LOG_NOTICE', "gsutil status on file is: $ol");
    if($attrib[7] =~ m/^[0-9]+$/){
      $backup_run->{gsutils_bytes} += $attrib[7];
    }
    if($attrib[8] ne 'OK'){
      $bk->logger('LOG_WARNING', "failure $backup_run->{type} $uri $attrib[8]");
      $bk->logger('LOG_WARNING',Dumper($args));
      $bk->logger('LOG_WARNING',"review $GSUTILS_LOG_FILE for more details");
      return $ol;
    }
    unlink($GSUTILS_LOG_FILE);
    return($attrib[8]);
  }
}

sub get_bucket_uri {
  my $host = shift;
  my $user = shift;

  my $dr_clients = $bk->get_gcp_dr_clients();
  if(! defined($dr_clients->{$host}->{bucket_name})){
    $bk->logger('LOG_ERR',"quitting... no bucket found for $host");
    $backup_run->{result} = sprintf("error: no bucket uri");
    update_state_file($state_file,$state_data,$backup_run);
    exit;
  }
  my $bucket_name = $dr_clients->{$host}->{bucket_name};

  my $time = time();
  # int number of days since epoc began divided by $conf->{backup_set_size} with the remainder being the backup number
  my $backup_number = int($time / 86400 ) % $conf->{backup_set_size};
  my $bucket_uri = lc(sprintf("gs://%s/users/%s.%02d.tarball",$bucket_name,$user,$backup_number));
  return $bucket_uri;
}

sub get_userinfo {
  my $host = shift;
  my $user = shift;
  # /home/box2188.bluehost.com/backup1/dirs.files/cpbackup/daily/userinfo
  if( ! -d "/home/$host"){
    $bk->logger('LOG_ERR', "quitting... missing /home/$host");
    exit;
  }
  my $userinfofile = "/home/$host/backup1/dirs.files/cpbackup/daily/userinfo";
  if( ! -e $userinfofile ){
    $userinfofile = '';
    my @dirs = read_dir("/home/$host");
    foreach my $dir (sort @dirs){
      next if ! -e "/home/$host/$dir/dirs.files/cpbackup/daily/userinfo";
      $userinfofile = "/home/$host/$dir/dirs.files/cpbackup/daily/userinfo";
      last;
    }
    if($userinfofile == ''){
      $bk->logger('LOG_ERR', "quitting... unable to find userinfo file for $host");
      $backup_run->{result} = sprintf("error: no userinfo file found");
      update_state_file($state_file,$state_data,$backup_run);
      exit;
    }
  }

  my $userinfo = LoadFile($userinfofile);
  if(defined($userinfo->{$user})){
    return($userinfo->{$user});
  }
  return;
}

sub is_gcp_up {
  # check to see if we can connect to storage.googleapis.com
  my $ua = LWP::UserAgent->new(timeout => 10);
  $ua->agent("backup_server/1.0");
  my $req = HTTP::Request->new(GET => "https://storage.googleapis.com/storage/v1/b/some-non-existant-bucket-name/o");
  my $res = $ua->request($req);
  my $response_code = $res->code();
  if(length($response_code) == 3 and substr($response_code,0,1) ne '5'){
    return 1;
  }
  return 0;
}
