#!/usr/bin/perl
# massfin v3.0
# massively breaking shit across the server farm since '10
# THIS IS A SYMLINK. UPDATE THE REPO. /root/massfin.repo
#
# patrick p - jlavoy - dbarker
# hostgator llc - endurance international group
# "I wanna go fast" - ricky bobby
use strict;
use warnings;
use POSIX 'strftime';
use Parallel::ForkManager;
use Net::OpenSSH;
use Getopt::Long qw(:config gnu_getopt);
use Date::Format qw(!strftime);
use JSON::XS;
use YAML::XS;
use LWP::UserAgent;
use MIME::Base64;
use Time::HiRes qw(usleep);
use IO::Socket::INET;
use Fcntl qw(:DEFAULT :flock);
use sigtrap 'handler' => \&finish, 'INT';
require RPC::XML;
require RPC::XML::Client;
use MIME::Lite;
use URI;
use Capture::Tiny 'capture';
use Text::Wrap;
use Cwd 'abs_path';
use List::Util qw(shuffle);
use File::Copy qw(copy);
use File::Path qw(make_path remove_tree);
use lib '/opt/eig-perl';
use Progress;
use Data::Dumper;

# fintypes are now located in new configuration file: /root/massfin/massfin3.yaml

my $configuration_file = '/etc/massfin3.yaml';
if ( -f $ENV{"HOME"} . '/.massfin/massfin3.yaml' ) {
    $configuration_file = $ENV{"HOME"} . '/.massfin/massfin3.yaml';
}
my ( $progress, $force_uppertier ) = ( 0, 0 );
my ( $count,    $count_failures )  = ( 0, 0 );
our $massfin_pid = $$;
my $date                     = time();
my $pushdate                 = time();
my $massfin_error_file       = "/tmp/massfin.errors.$date";
my $massfin_error_file_retry = $massfin_error_file . ".retry";
my $terminal_width           = `tput cols`;
my $whoami                   = `logname`;
my $zeeroh                   = $0 . " " . ( join( " ", @ARGV ) );
my $logprefix                = 'MASSFIN - ' . time2str( "%c", time ) . ' - ';
our $ostype = 'linux';
my (
    @runlist,  @runlist_retry, @serverlist,  @commands,     @manual_groups, $conf,        $commandfile, $reason,         $max_children, @customlist,
    $quiet,    $verbose,       $noredir,     $skip,         $nolog,         $delay,       $random,      @_manual_groups, @environments, @_environments,
    $username, @_classes,      @classes,     @_fact_values, @fact_values,   @runlist_tmp, $tree_folder, $search_query,   @_runlist_tmp, @_customlist,
    $nowrap,   $dryrun,        $instruction, $help,         $retry,         $retry_sleep, $scp,         $debug
);
my $globalexcludes =
    'sitestudio|supra|asp.websitewelcome|gatorasp|sitebuilder|whmcs|minidedi|staff'
  . '|scripts|gator.hostgator|server.websitewelcome|suspended|toolbox|japan|basekit'
  . '|laruetac|secure.hostgator.com|br01.hostgator.com.br|in1.hostgator.in'
  . '|dev.davetha|pvamaster.hostgator|gator8001.hostgator.com|gatordev.hostgator.com'
  . '|gator4999.hostgator.com';

my $henson_url;

chomp($whoami) if ($whoami);

# Load configuration
our %conf = %{ YAML::XS::Load( slurp($configuration_file) ) };
$random = $conf{massfin_opts}->{randomize_list} if ( $conf{massfin_opts}->{randomize_list} );
my $log_path = $conf{massfin_opts}->{log_path};

# Read command line arguments
usage()
  if (
    !GetOptions(
        'c|command=s'     => \$commandfile,
        'r|reason=s'      => \$reason,
        'm|max=i'         => \$max_children,
        'l|list=s'        => \@_customlist,
        'q|quiet'         => \$quiet,
        'v|verbose'       => \$verbose,
        'n|noredir'       => \$noredir,
        's|skip'          => \$skip,
        'p|progress'      => \$progress,
        'b|nolog'         => \$nolog,
        'U|uppertier'     => \$force_uppertier,
        'd|delay=i'       => \$delay,
        'g|group=s'       => \@_manual_groups,
        'e|environment=s' => \@_environments,
        'C|class=s'       => \@_classes,
        'f|fact=s'        => \@_fact_values,
        't|tree=s'        => \$tree_folder,
        'S|search=s'      => \$search_query,
        'I|server=s'      => \@_runlist_tmp,
        'W|nowrap'        => \$nowrap,
        'D|dryrun'        => \$dryrun,
        'i|instruction=s' => \$instruction,
        'R|random!'       => \$random,
        'u|user=s'        => \$username,
        'w|windows|win'   => sub { $ostype = 'windows' },
        'T|timeout=i'     => \$conf{ssh_opts}->{ssh_timeout},
        'h|help'          => \$help,
        'retry=i'         => \$retry,
        'retrysleep=i'    => \$retry_sleep,
        'scp=s'           => \$scp,
        'debug'           => \$debug
    )
  );

$username = $conf{ssh_opts}->{$ostype}->{user} if ( !$username );
if (@_manual_groups) {
    foreach my $_tmp (@_manual_groups) {
        push( @manual_groups, split( /[,: ]+/, $_tmp ) );
    }
}
if (@_environments) {
    foreach my $_tmp (@_environments) {
        push( @environments, split( /[,: ]+/, $_tmp ) );
    }
}
if (@_classes) {
    foreach my $_tmp (@_classes) {
        push( @classes, split( /[, ]+/, $_tmp ) );
    }
}
if (@_fact_values) {
    foreach my $_tmp (@_fact_values) {
        push( @fact_values, split( /[, ]+/, $_tmp ) );
    }
}
if (@_runlist_tmp) {
    foreach my $_tmp (@_runlist_tmp) {
        push( @runlist_tmp, split( /[, ]+/, $_tmp ) );
    }
}
if (@_customlist) {
    foreach my $_tmp (@_customlist) {
        push( @customlist, split( /[ ]+/, $_tmp ) );
    }
}

my @fintype_list = @ARGV;
if (@fintype_list) {
    foreach (@fintype_list) {
        if ( !exists( $conf{fintypes}->{$_} ) ) {
            print( STDERR "[-] Unknown fintype \"$_\"\n" );
            usage();
        }
    }
}

if ( defined($tree_folder) ) {
    if ( !-d $tree_folder ) {
        print( STDERR "[-] \"$tree_folder\" does not exist.\n" );
        exit(1);
    }
}

$max_children = $conf{ssh_opts}->{max_children}    if ( !defined($max_children) );
$retry        = $conf{massfin_opts}->{retry}       if ( !$retry );
$retry_sleep  = $conf{massfin_opts}->{retry_sleep} if ( !$retry_sleep );

# Test for sane option combinations
usage()
  if (!( @fintype_list || @manual_groups || @environments || @classes || @fact_values || $search_query || @runlist_tmp || @customlist )
    || ( !$dryrun && ( defined($commandfile) || ( defined($instruction) ) ) && !defined($reason) )
    || ( !$dryrun && !defined($commandfile) && !defined($instruction) )
    || ($help) );

open( STDERR, '>/dev/null' ) if ($noredir);

# Test for files existing and readable
foreach my $customlist_file (@customlist) {
    if ( defined($customlist_file) ) {
        if ( !-f $customlist_file ) {
            senderror("[!] Unable to find server list file: $customlist_file");
        }
        else {
            $customlist_file = abs_path($customlist_file);
        }
    }
}
if ($commandfile) {
    if ( -f $commandfile ) {
        $commandfile = abs_path($commandfile);
        open( my $DAT, '<', $commandfile ) || senderror("[!] Unable to read in command file: $commandfile");
        @commands = <$DAT>;
        close($DAT);
    }
    else {
        senderror("[!] Unable to read in command file: $commandfile");
    }
}
elsif ($instruction) {
    senderror( "[!] Instruction too long. (" . length($instruction) . "/" . $conf{massfin_opts}->{instruct_len} . ") Use a command file: \"$instruction\"" )
      if ( length($instruction) > $conf{massfin_opts}->{instruct_len} );
    $commands[0] = $instruction;
}
elsif ($dryrun) {
    print( STDERR "[!] No commandfile or instruction found\n" );
}
else {
    senderror("[!] No commandfile or instruction found");
}

# Generate server list from Foreman.
print( STDERR show_info( "*** DRY-RUN ***", "" ) ) if ($dryrun);

my ( @match, @match_tmp );

print( STDERR "[*] Generating server list..." );
push( @match_tmp, @manual_groups );
foreach my $fintype (@fintype_list) {
    push( @match_tmp, @{ $conf{fintypes}->{$fintype}->{hostgroups} } );
    $globalexcludes .= '|pvanode|basekit|dev.davetha' if ( $fintype eq 'hgvps' or 'latamvps' );
}

# Foreman auth stuff
my $auth = "Basic " . MIME::Base64::encode( $conf{henson}->{login} . ":" . $conf{henson}->{password} );
my $ua   = LWP::UserAgent->new;
$ua->ssl_opts(
    verify_hostname => 0,
    SSL_use_cert    => 0,

    #    SSL_verify_mode => SSL_VERIFY_NONE,     # This breaks on wiz2, but makes work on wiz3
);
$ua->default_header( 'Authorization' => $auth );
$ua->default_header( 'Content-Type'  => 'application/json' );

# Get list of hostgroups from Foreman for class
foreach my $class (@classes) {
    my $henson_url = URI->new( $conf{henson}->{host} . "/api/puppetclasses/$class" );
    $henson_url->query_form( per_page => 10000, );
    my $henson_response      = $ua->get($henson_url)->content();
    my $class_hostgroup_list = decode_json($henson_response);
    if ( !exists( $class_hostgroup_list->{hostgroups} ) || scalar( @{ $class_hostgroup_list->{hostgroups} } ) == 0 ) {
        print( STDERR "\n[!] Unknown class \"$class\"\n" );
        exit;
    }
    foreach my $class_hostgroup ( @{ $class_hostgroup_list->{hostgroups} } ) {

        push( @match_tmp, $class_hostgroup->{name} );
    }
    print( STDERR "." );
}

# Get list of hostgroups from Foreman for environment
$henson_url = URI->new( $conf{henson}->{host} . '/api/v1/hostgroups' );
foreach my $environment (@environments) {
    $henson_url->query_form(
        per_page => 10000,
        search   => "environment=$environment"
    );
    my $henson_response    = $ua->get($henson_url)->content();
    my $env_hostgroup_list = decode_json($henson_response);
    if ( scalar( @{$env_hostgroup_list} ) == 0 ) {
        print( STDERR "\n[!] Unknown environment \"$environment\"\n" );
        exit;
    }
    foreach my $env_hostgroup ( @{$env_hostgroup_list} ) {
        push( @match_tmp, $env_hostgroup->{hostgroup}->{name} );
    }
    print( STDERR "." );
}

# Deduplicate our match list (hostgroups specified multiple times or exist in massfin group
@match = do {
    my %seen;
    grep { !$seen{$_}++ } @match_tmp;
};

# Get list of servers from Foreman for hostroup
$henson_url = URI->new( $conf{henson}->{host} . '/api/v1/hosts' );
foreach my $host_group (@match) {
    $henson_url->query_form(
        per_page => 10000,
        search   => "hostgroup=$host_group"
    );
    my $henson_response = $ua->get($henson_url)->content();
    my $server_list     = decode_json($henson_response);
    ### Commenting out because presently cannot tell difference between non-existent and empty hostgroup
    #if(scalar(@{$server_list}) == 0){
    #    print(STDERR "\n[!] Unknown hostgroup \"$host_group\"\n");
    #    exit;
    #}
    foreach my $server ( @{$server_list} ) {
        if ( $server->{host}->{name} !~ /$globalexcludes/ ) {
            if ( grep { $_ eq $host_group } @{ $conf{fintypes}->{hostgroups}->{'aso-legacy'} } or $host_group eq 'NAS800' ) {
                push( @runlist_tmp, $server->{host}->{name} . ':' . $conf{ssh_opts}->{$ostype}->{aso_ssh_port} );
            }
            else {
                push( @runlist_tmp, $server->{host}->{name} );
            }
        }
    }
    print( STDERR "." );
}

# Get list of servers from Foreman for fact match
$henson_url = URI->new( $conf{henson}->{host} . '/api/v1/hosts' );
while ( scalar(@fact_values) > 0 ) {
    my ( $_fact_name, $_fact_op, $_fact_val ) = ( shift(@fact_values), shift(@fact_values), shift(@fact_values) );
    $henson_url->query_form(
        per_page => 10000,
        search   => "facts.${_fact_name}${_fact_op}${_fact_val}"
    );
    my $henson_response = $ua->get($henson_url)->content();
    my $server_list     = decode_json($henson_response);
    foreach my $server ( @{$server_list} ) {
        if ( $server->{host}->{name} !~ /$globalexcludes/ ) {
            push( @runlist_tmp, $server->{host}->{name} );
        }
    }
    print( STDERR "." );
}

# Get list of servers from Foreman for search query
$henson_url = URI->new( $conf{henson}->{host} . '/api/v1/hosts' );
if ($search_query) {
    $henson_url->query_form(
        per_page => 10000,
        search   => "$search_query"
    );
    my $henson_response = $ua->get($henson_url)->content();
    my $server_list     = decode_json($henson_response);
    foreach my $server ( @{$server_list} ) {
        if ( $server->{host}->{name} !~ /$globalexcludes/ ) {
            push( @runlist_tmp, $server->{host}->{name} );
        }
    }
    print( STDERR "." );
}

# Direct lists from command line
foreach my $customlist_file (@customlist) {
    my $linenum = 0;
    open( my $DAT, '<', $customlist_file ) or die "[!] Unable to open custom list file \"$customlist_file\"!\n";
    while (<$DAT>) {
        chomp($_);
        if (/^\s*([a-zA-Z0-9-.]{1,63}(?::\d+)?)?\s*(\#.*)?$/) {
            push( @runlist_tmp, $1 ) if ($1);
        }
        else {
            die("[!] Unexpected data in custom list \"$customlist_file\", line $linenum: \"$_\"\n");
        }
    }
    close($DAT);
}
print( STDERR "Done!\n" );

# Deduplicate our runlist as fact checks + hostgroups could cause duplicate servers
@runlist = do {
    my %seen;
    grep { !$seen{$_}++ } @runlist_tmp;
};
@runlist = shuffle(@runlist) if ($random);

# Show some information before we run
print( STDERR show_info( "reason",         $reason ) )        if ($reason);
print( STDERR show_info( "random",         $random ) )        if ($random);
print( STDERR show_info( "ostype",         $ostype ) )        if ($ostype);
print( STDERR show_info( "command file",   $commandfile ) )   if ($commandfile);
print( STDERR show_info( "instruction",    $instruction ) )   if ($instruction);
print( STDERR show_info( "username",       $username ) )      if ($username);
print( STDERR show_info( "massfin type",   \@fintype_list ) ) if (@fintype_list);
print( STDERR show_info( "lists",          \@customlist ) )   if (@customlist);
print( STDERR show_info( "environments",   \@environments ) ) if (@environments);
print( STDERR show_info( "classes",        \@classes ) )      if (@classes);
print( STDERR show_info( "facts",          \@fact_values ) )  if (@fact_values);
print( STDERR show_info( "foreman search", $search_query ) )  if ($search_query);
print( STDERR show_info( "hostgroups",     \@match ) )        if (@match);
print( STDERR show_info( "servers",        \@runlist ) )      if ( $verbose && @runlist );
print( STDERR show_info( "server count",   scalar(@runlist) ) );
print( STDERR show_info( "max children",   $max_children ) );
print( STDERR show_info( "output tree",    $tree_folder ) ) if ( defined($tree_folder) );
print( STDERR show_info( "process id",     $massfin_pid ) );
print( STDERR show_info( "retry",          $retry ) )       if ($retry);
print( STDERR show_info( "retrysleep",     $retry_sleep ) ) if ( $retry && $retry_sleep );
print( STDERR show_info( "scp",            $scp ) )         if ($scp);

if ( !-d "$log_path" ) {
    make_path( "$log_path", { mode => 0770 } ) or die("Cannot create logging directory \"$log_path\". Error: $?\n");
}

if ( scalar(@runlist) < 1 ) {

    # Abort if no servers
    senderror("No servers found");
}
elsif ( !$skip ) {

    # Prompt to continue
    $| = 1;
    print( STDERR "[*] We're about to" . ( $dryrun ? ' dry' : ' ' ) . "run on " . scalar(@runlist) . " boxes" . ( $dryrun ? "\n" : ", press enter key to continue: " ) );
    my $read = <STDIN> if ( !$dryrun );
}

if ($scp) {
    if ( !-f abs_path($scp) ) {
        print abs_path($scp);
        senderror("[!] $scp file not found");
    }
}

# Affix username to reason for logging
$reason = "(" . $whoami . ") " . $reason if ( $whoami && $reason );

# Log to massfin.labs and send snitch emails only after we've commited to actually doing it
if ( !$dryrun ) {
    if ( !$nolog ) {
        $ENV{'PERL_LWP_SSL_VERIFY_HOSTNAME'} = 0;
        my $cli = RPC::XML::Client->new(
            'https://192.185.178.7/rpc',
            ssl_opts => {
                verify_hostname => 0,
                SSL_use_cert    => 0,
                SSL_verify_mode => 'SSL_VERIFY_NONE'
            },
        );
        my $response = $cli->send_request(
            'postlog',
            {
                serverlist  => "@fintype_list",
                command     => ( $commandfile ? $commandfile : '(instruction)' ),
                commandblob => join( "\n", @commands ),
                reason      => $reason,
            }
        );
    }
    else {
        my $message = MIME::Lite->new(
            From    => 'root@wizard2.hostgator.com',
            To      => 'level3s@hostgator.com',
            Subject => 'Massfin Logging Disabled Notification',
            Type    => 'text/plain',
            Data    => "Type: @fintype_list\nManual Groups: @manual_groups\nCommand File: " . ( $commandfile ? $commandfile : '(instruction)' ) . "\nReason: $reason\nCommands:\n @commands\n",
        );
        $message->send();
    }
    my $time    = localtime();
    my $message = MIME::Lite->new(
        From    => 'root@wizard2.hostgator.com',
        To      => 'directors@hostgator.com',
        Subject => 'Massfin Log ' . $time,
        Type    => 'text/plain',
        Data    => "Type: @fintype_list\nManual Groups: @manual_groups\nCommand File: " . ( $commandfile ? $commandfile : '(instruction)' ) . "\nReason: $reason\nCommands:\n @commands\n",
    );
    $message->send();
}

# Set-up progress stuff
if ($progress) {
    $progress = Progress->new( $count, scalar(@runlist) );
    $progress->start();
}

# Create lockfile
if ( !-e '/tmp/massfin-' . $massfin_pid . '.lock' ) {
    open( my $FH, '>', '/tmp/massfin-' . $massfin_pid . '.lock' ) or die('could not create lock file for I/O');
    close($FH);
}

$pushdate = time();

# Loop through servers
if ( !$dryrun ) {
    run_loop( $massfin_error_file, 0, @runlist );
    copy( $massfin_error_file, $massfin_error_file_retry );
    if ( -f $massfin_error_file && $retry ) {
        for ( my $i = 1 ; $i <= $retry ; $i++ ) {
            if ( -f $massfin_error_file_retry ) {
                open( my $retry_fh, '<', $massfin_error_file_retry );
                chomp( my @runlist_retry = <$retry_fh> );
                close($retry_fh);
                unlink($massfin_error_file_retry);
                if ( scalar(@runlist_retry) > 0 ) {
                    print( STDERR "\n[*] Retry $i/$retry with " . scalar(@runlist_retry) . " failed servers in $retry_sleep seconds\n" );
                    sleep($retry_sleep);
                    run_loop( $massfin_error_file_retry, $i, @runlist_retry );
                }
            }
        }
        if ( -f $massfin_error_file_retry ) {
            unlink($massfin_error_file);
            rename( $massfin_error_file_retry, $massfin_error_file );
        }
    }
}
else {
    print( STDERR show_info( "*** DRY-RUN ***", "" ) );
}
finish();

############################## ###  ##   #    Subs    #   ##  ### ##############################

# Adjust max_children on the fly
sub set_max_children {
    my $pm   = shift;
    my $path = $conf{massfin_opts}->{option_path};
    make_path( $path, { mode => 0777 } ) if ( !-d $path );
    make_path( $path . "/$massfin_pid" ) if ( !-e $path . "/$massfin_pid" );
    if ( -e $path . "/$massfin_pid/max_children" ) {
        chomp( my $procs = slurp( $path . "/$massfin_pid/max_children" ) );
        $pm->set_max_procs($procs) if ( $procs =~ /^\d+$/ );
    }
}

# Return informational output, like pre-run data for confirmation
sub show_info {
    my $decoration = "[*]";
    $Text::Wrap::columns = ( $terminal_width - ( length( $decoration . " " . $_[0] . ": " ) + 5 ) );
    if ( ref( $_[1] ) eq 'ARRAY' ) {
        return ( $decoration . " " . $_[0] . ": " . ( $nowrap ? "@{$_[1]}" : wrap( "", $decoration . ( " " x ( length( $_[0] ) + 3 ) ), @{ $_[1] } ) ) . "\n" );
    }
    else {
        return ( $decoration . " " . $_[0] . ": " . $_[1] . "\n" );
    }
}

# Write and log error message
sub senderror {
    my $errormsg = shift;
    print $logprefix . $errormsg . "\n";
    open( my $WAT, '>>', 'error_log' );
    print $WAT $logprefix . $errormsg . "\n";
    close($WAT);
    exit();
}

# Test port
sub check_port {
    if ( IO::Socket::INET->new( PeerAddr => $_[0], PeerPort => $_[1], Timeout => 3, Proto => 'tcp' ) ) {
        return 1;
    }
    else {
        return 0;
    }
}

# Show usage and die
sub usage {
    print(
"\n[!] Usage: $0 <types|-l <listfile>|-g <henson group>|-e <henson environment>|-C <henson class>|-f <fact comparison>|-S <search query>|-I <server>> < -c <command file>|-i <instruction> > -r <reason> [-m <integer>] [-q|--quiet] [-v|--verbose] [-p|--progress] [-n|--noredir] [-d|--delay <integer>] [-t|--tree <output path>] [-W|--nowrap] [-D|--dryrun] [-R|--(no)random] [-u|--user] [-w|--windows] [-T|--timeout] [-h|--help]\n\n"
    );
    print("\tSee \"--help\" for further help\n") if ( !$help );
    show_help()                                  if ($help);
    print("\n");
    exit(1);
}

# Show full help output
sub show_help {
    my $help_message = <<"END_HELP";
        OPTION				TYPE	DESCRIPTION
	-c|--command <filename>		cmd	Command file to run on target servers. Cannot be ran with --instruction
	-C|--class <classname>		list	Foreman class to add to server list. Can be specified more than once
	-d|--delay <seconds>			Pause between runs
	-D|--dryrun				Perform a dry-run. Show what it would do, but to not run on servers
	-e|--environment <environment>	list	Foreman environment to add to server list. Can be specified more than once
	-f|--fact <fact>		list	Puppet fact to add to server list. Can be specified more than once
	-g|--group <group>		list	Foreman group to add to server list. Can be specified more than once
	-h|--help				Show this help
	-i|--instruction <commands>	cmd	A short command to run on server. Cannot be ran with --command
	-I|--server <server>		list	Server to connect to. Can be specified more than once
	-l|--list <filename>		list	List of servers to connect to. Can be specified more than once
	-m|--max <num>				Maximum number of children to conenct to simultaneously. Default $max_children
	-n|--noredir				Do not discard STDERR
	-p|--progress				Show rudimentary progress indicator
	-q|--quiet				Do not show failures in-line
	-r|--reason <reason>		reason	Reason for running massfin. Required
	-R|--random				Randomize servers. Disable with --norandom
           --retry <# retries>                  Retry failed connection this many times
	-s|--skip				Skip prompt
	-S|--search <query>		list	Foreman query to add to server list
	-t|--tree <path>			Folder to send distributed output files to. One file per server within this folder
	-T|--timeout <seconds>			SSH connection timeout. Default $conf{ssh_opts}->{ssh_timeout}
	-u|--user <username>			Connect as this user
	-v|--verbose				Be verbose. Show servers prior to running
	-w|--windows|--win			Use windows mode for connecting to widnows with SSH
	-W|--nowrap				Do not pretty-wrap verbose output
END_HELP
    print( $help_message. "\n" );
    print("\n\tPossible massfin server types (type: list):\n");
    foreach my $fintype ( sort( keys %{ $conf{fintypes} } ) ) {
        printf( "\t\t%-16s = %s\n", $fintype, $conf{fintypes}->{$fintype}->{description} );
    }
    print("\n\tMassfin requires at least one each from types \"list\", \"cmd\", and \"reason\" in order to run.\n");
}

# The run loop
sub run_loop {
    my $massfin_error_file_temp = shift;
    my $retry_num               = shift;
    my @run_loop_list           = @_;
    my $pm                      = new Parallel::ForkManager($max_children);
    system("/usr/eig/bin/sshca.py");
    my $key_refresh_time = time();
    foreach my $server (@run_loop_list) {
        set_max_children($pm);
        $progress->update( $count, scalar(@run_loop_list) ) if ($progress);
        $count++                                            if ( !$retry_num );
        if ( $key_refresh_time < time() - 86100 ) {
            system("/usr/eig/bin/sshca.py --force --signtime 1440");
            $key_refresh_time = time();
        }
        my $pid = $pm->start and next;
        setpriority( 0, $$, $conf{massfin_opts}->{fork_niceness} ) if ( $pid == 0 );
        my $mininame = $server;
        $mininame =~ s/\.(hostgator|websitewelcome)\.com$//;
        local $0 = "[$mininame] $zeeroh";
        my $return = run($server);
        if ( $return != 1 ) {
            if ( !$quiet ) { print STDERR "[!] Failed on $server \n" }
            if ( sysopen( my $WAT, $massfin_error_file_temp, O_CREAT | O_WRONLY | O_APPEND ) ) {
                print $WAT "$server\n";
                close($WAT);
            }
        }
        usleep(3000);
        local $0 = $zeeroh;
        sleep($delay) if $delay;
        $pm->finish;
    }
    $pm->wait_all_children;
    $progress->stop() if ($progress);
}

# The actual push
sub run {
    my $input = shift;
    my ( $server, $port ) = split( ':', $input, 2 );
    $port = $conf{ssh_opts}->{$ostype}->{ssh_port} if ( !$port );

    # check to see if 22 is listening to avoid long timeouts.
    if ( !check_port( $server, $port ) ) {
        if ( !check_port( $server, $port ) ) {
            return 0;
        }
    }
    unshift( @commands, "source /etc/bashrc && source /etc/profile\n" ) if ( !$ostype eq 'windows' );
    if ($verbose) { print STDERR "[*] Pushing to $server.\n" }
    my ( @outputdump, $ssherror );
    my ( $_stdout, $_stderr, @_result ) = capture {
        my $ssh = Net::OpenSSH->new(
            $server,
            kill_ssh_on_timeout => 1,
            port                => $port,
            user                => $username,
            batch_mode          => 1,
            master_opts         => [
                -o => defined($debug) ? 'LogLevel=DEBUG3' : 'LogLevel=quiet',
                -o => 'StrictHostKeyChecking=no',
                -o => 'CertificateFile=~/.ssh/id_rsa-cert.pub',
		-o => 'ConnectionAttempts=2',
            ],
            timeout => $conf{ssh_opts}->{ssh_timeout},
        );
        return 0 if ( $ssh->error );
        $ssherror = $ssh->error;
        if ( $ostype eq 'windows' ) {
            @outputdump = $ssh->capture("@commands");    # Need this for windows
        }
        else {
            if ($scp) {
                $ssh->system("[[ -d /root/.massfin_tmp ]] || mkdir /root/.massfin_tmp") or do { $ssherror = $ssh->error; return (0); };
                $ssh->scp_put( abs_path($scp), '/root/.massfin_tmp' )                   or do { $ssherror = $ssh->error; return (0); };
            }
            @outputdump = $ssh->capture( { stdin_data => \@commands } );    # Linux
            if ($scp) {
                $ssh->system("[[ -d /root/.massfin_tmp ]] && rm -fr /root/.massfin_tmp/*") or return 0;
            }
        }
    };
    print Dumper ( $_stdout, $_stderr, @_result )                                                                                                         if ($debug);
    return (0)                                                                                                                                            if ( !defined($ssherror) || $ssherror != 0 );
    @outputdump = grep( !/^(Last failed login: .* from .* on|There (were|was) \d+ failed login attempts? since the last successful login)/, @outputdump ) if ( scalar(@outputdump) > 0 );
    if ( !defined($quiet) ) {
        open( my $WAT, '>>', "$log_path/$server.log" );
        print $WAT time2str( "%c", time ) . " - $reason\n@commands\nOUTPUT(if any): \n@outputdump\n\n";
        close($WAT);
    }
    if ( scalar @outputdump > 0 ) {
        local $| = 1;
        open( my $LOCK, '>', '/tmp/massfin-' . $massfin_pid . '.lock' ) or die( 'Could not open massfin lock file: ' . $! );
        flock( $LOCK, LOCK_EX )                                         or die( 'Could not get flock: ' . $! );
        chomp( $outputdump[$#outputdump] );
        if ( defined($tree_folder) && -d $tree_folder ) {
            open( my $output_file, ">", "$tree_folder/$server" ) or die("Could not open \"$tree_folder/$server\" for writing: $!");
            foreach (@outputdump) {
                print( $output_file $_ );
            }
            print( $output_file "\n" );
            close($output_file);
        }
        else {
            foreach (@outputdump) {
                print $_;
            }
            print("\n");
        }
        close($LOCK);
    }
    return 1;
}

# Read a file
sub slurp { local $/ = <> if local @ARGV = @_ }

# Remove lock, write out error report
sub finish {
    if ( -f $massfin_error_file ) {
        print STDERR "[!!!] The following servers did not accept the push: \n";
        open( my $DAT, '<', $massfin_error_file );
        while (<$DAT>) {
            $count_failures++;
            print STDERR $_;
        }
        close($DAT);
        unlink($massfin_error_file);
    }
    print STDERR "\n";
    print STDERR "[*] Executed on " . ( $count - $count_failures ) . "/$count servers\n" if ($count);
    print STDERR "[*] Total massfin runtime: " . strftime( "%Hh %Mm %Ss", gmtime( time() - $date ) ) . "\n";
    print STDERR "[*] Push execute time:     " . strftime( "%Hh %Mm %Ss", gmtime( time() - $pushdate ) ) . "\n";

    system( "rm -rf " . $conf{massfin_opts}->{option_path} . "/" . $massfin_pid )
      if ( exists( $conf{massfin_opts}->{option_path} )
        && length( $conf{massfin_opts}->{option_path} ) > 4
        && $conf{massfin_opts}->{option_path} . "/" . $massfin_pid ne "/"
        && -d $conf{massfin_opts}->{option_path} . "/" . $massfin_pid );
    if ( -e '/tmp/massfin-' . $massfin_pid . '.lock' ) {
        unlink( '/tmp/massfin-' . $massfin_pid . '.lock' );
    }
    print STDERR "\n";
    exit();
}

1;
