#!/usr/bin/perl
# massfin v3.0
# massively breaking shit across the server farm since '10
# THIS IS A SYMLINK. UPDATE THE REPO. /root/massfin.repo
#
# patrick p - jlavoy - dbarker
# hostgator llc - endurance international group
# "I wanna go fast" - ricky bobby
use strict;
use warnings;
use POSIX 'strftime';
use Parallel::ForkManager;
use Net::OpenSSH;
use Getopt::Long qw(:config gnu_getopt);
use Date::Format qw(!strftime);
use JSON::XS;
use YAML::XS;
use LWP::UserAgent;
use MIME::Base64;
use Time::HiRes qw(usleep);
use IO::Socket::INET;
use Fcntl qw(:DEFAULT :flock);
use sigtrap 'handler' => \&finish, 'INT';
require RPC::XML;
require RPC::XML::Client;
use MIME::Lite;
use URI;
use Capture::Tiny 'capture';
use Text::Wrap;
use Cwd 'abs_path';
use List::Util qw(shuffle);
use File::Copy qw(copy);
use lib '/opt/eig-perl';
use Progress;

# fintypes are now located in new configuration file: /root/massfin/massfin3.yaml

my $configuration_file             = '/etc/massfin3.yaml';
if ( -f $ENV{"HOME"} . '/.massfin/massfin3.yaml' ) {
  $configuration_file = $ENV{"HOME"} . '/.massfin/massfin3.yaml';
}
my ( $progress, $force_uppertier ) = (0,0);
my ( $count, $count_failures )     = (0,0);
our $massfin_pid                    = $$;
my $date                           = time();
my $pushdate                       = time();
my $massfin_error_file             = "/tmp/massfin.errors.$date";
my $massfin_error_file_retry       = $massfin_error_file . ".retry";
my $terminal_width                 = `tput cols`;
my $whoami                         = `logname`;
my $zeeroh                         = $0 . " " . ( join( " ", @ARGV ) );
my $logprefix                      = 'MASSFIN - ' . time2str( "%c", time ) . ' - ';
our $ostype                        = 'linux';
my ( @runlist, @runlist_retry, @serverlist, @commands, @manual_groups,
	$conf, $commandfile, $reason, $max_children, @customlist,
	$quiet, $verbose, $noredir, $skip, $nolog, $delay, $random,
	@_manual_groups, @environments, @_environments, $username,
	@_classes, @classes, @_fact_values, @fact_values,
	@runlist_tmp, $tree_folder, $search_query, @_runlist_tmp,
	@_customlist, $nowrap, $dryrun, $instruction,
        $help, $retry, $retry_sleep, $scp
);
my $globalexcludes                = 'sitestudio|supra|asp.websitewelcome|gatorasp|sitebuilder|whmcs|minidedi|staff'
				  . '|scripts|gator.hostgator|server.websitewelcome|suspended|toolbox|japan|basekit'
				  . '|laruetac|secure.hostgator.com|br01.hostgator.com.br|in1.hostgator.in'
				  . '|dev.davetha|pvamaster.hostgator|gator8001.hostgator.com|gatordev.hostgator.com'
				  . '|gator4999.hostgator.com';

my $henson_url;

chomp($whoami) if($whoami);

# Load configuration
our %conf = %{ YAML::XS::Load(slurp($configuration_file)) };
$random = $conf{massfin_opts}->{randomize_list} if($conf{massfin_opts}->{randomize_list});
my $log_path = $conf{massfin_opts}->{log_path};

# Read command line arguments
usage() if (!GetOptions(
		'c|command=s'     => \$commandfile,
		'r|reason=s'      => \$reason,
		'm|max=i'         => \$max_children,
		'l|list=s'        => \@_customlist,
		'q|quiet'         => \$quiet,
		'v|verbose'       => \$verbose,
		'n|noredir'       => \$noredir,
		's|skip'          => \$skip,
		'p|progress'      => \$progress,
		'b|nolog'         => \$nolog,
		'U|uppertier'     => \$force_uppertier,
		'd|delay=i'       => \$delay,
		'g|group=s'       => \@_manual_groups,
		'e|environment=s' => \@_environments,
		'C|class=s'       => \@_classes,
		'f|fact=s'        => \@_fact_values,
		't|tree=s'        => \$tree_folder,
		'S|search=s'      => \$search_query,
		'I|server=s'      => \@_runlist_tmp,
		'W|nowrap'        => \$nowrap,
		'D|dryrun'        => \$dryrun,
		'i|instruction=s' => \$instruction,
		'R|random!'       => \$random,
		'u|user=s'        => \$username,
		'w|windows|win'   => sub{$ostype='windows'},
		'T|timeout=i'     => \$conf{ssh_opts}->{ssh_timeout},
                'h|help'          => \$help,
		'retry=i'         => \$retry,
                'retrysleep=i'    => \$retry_sleep,
                'scp=s'    => \$scp,
	)
);

$username=$conf{ssh_opts}->{$ostype}->{user} if(!$username);
if(@_manual_groups){
	foreach my $_tmp (@_manual_groups){
		push(@manual_groups,split(/[,: ]+/,$_tmp));
	}
}
if(@_environments){
	foreach my $_tmp (@_environments){
		push(@environments,split(/[,: ]+/,$_tmp));
	}
}
if(@_classes){
	foreach my $_tmp (@_classes){
		push(@classes,split(/[, ]+/,$_tmp));
	}
}
if(@_fact_values){
	foreach my $_tmp (@_fact_values){
		push(@fact_values,split(/[, ]+/,$_tmp));
	}
}
if(@_runlist_tmp){
	foreach my $_tmp (@_runlist_tmp){
		push(@runlist_tmp,split(/[, ]+/,$_tmp));
	}
}
if(@_customlist){
	foreach my $_tmp (@_customlist){
		push(@customlist,split(/[ ]+/,$_tmp));
	}
}

my @fintype_list = @ARGV;
if(@fintype_list){
	foreach(@fintype_list){
		if(!exists($conf{fintypes}->{$_})) {
			print(STDERR "[-] Unknown fintype \"$_\"\n");
			usage();	
		}
	}
}

if(defined($tree_folder)){
	if(! -d $tree_folder){
		print(STDERR "[-] \"$tree_folder\" does not exist.\n");
		exit(1);
	}
}

$max_children = $conf{ssh_opts}->{max_children} if ( !defined($max_children) );
$retry = $conf{massfin_opts}->{retry} if(!$retry);
$retry_sleep = $conf{massfin_opts}->{retry_sleep} if(!$retry_sleep);

# Test for sane option combinations
usage() if( !(@fintype_list || @manual_groups || @environments || @classes || @fact_values || $search_query || @runlist_tmp || @customlist )
	|| ( !$dryrun && ( defined($commandfile) || (defined($instruction))) && !defined($reason) )
	|| ( !$dryrun && !defined($commandfile) && !defined($instruction) )
	|| ( $help )
);

open( STDERR, '>/dev/null' ) if ($noredir);

# Test for files existing and readable
foreach my $customlist_file (@customlist){
	if ( defined($customlist_file) ) {
		if ( !-f $customlist_file ) {
			senderror("[!] Unable to find server list file: $customlist_file");
		} else {
			$customlist_file = abs_path($customlist_file);
		}
	}
}
if($commandfile){
	if ( -f $commandfile ) {
		$commandfile = abs_path($commandfile);
		open( my $DAT, '<', $commandfile ) || senderror("[!] Unable to read in command file: $commandfile");
		@commands = <$DAT>;
		close($DAT);
	} else {
		senderror("[!] Unable to read in command file: $commandfile");
	}
} elsif($instruction){
	senderror("[!] Instruction too long. (".length($instruction)."/".$conf{massfin_opts}->{instruct_len}.") Use a command file: \"$instruction\"") if(length($instruction)>$conf{massfin_opts}->{instruct_len});
	$commands[0]=$instruction;
} elsif($dryrun){
	print(STDERR "[!] No commandfile or instruction found\n");
} else {
	senderror("[!] No commandfile or instruction found");
}

# Generate server list from Foreman.
print(STDERR show_info("*** DRY-RUN ***","")) if($dryrun);

my (@match, @match_tmp);

print(STDERR "[*] Generating server list..."); 
push(@match_tmp, @manual_groups);
foreach my $fintype (@fintype_list){
	push(@match_tmp, @{$conf{fintypes}->{$fintype}->{hostgroups}});
	$globalexcludes .= '|pvanode|basekit|dev.davetha' if ( $fintype eq 'hgvps' or 'latamvps' );
}

# Foreman auth stuff 
my $auth       = "Basic " . MIME::Base64::encode( $conf{henson}->{login} . ":" . $conf{henson}->{password} );
my $ua         = LWP::UserAgent->new;
$ua->ssl_opts(
    verify_hostname => 0,
    SSL_use_cert    => 0,
#    SSL_verify_mode => SSL_VERIFY_NONE,     # This breaks on wiz2, but makes work on wiz3
);
$ua->default_header( 'Authorization' => $auth );
$ua->default_header( 'Content-Type'  => 'application/json' );

# Get list of hostgroups from Foreman for class
foreach my $class (@classes){
	my $henson_url = URI->new($conf{henson}->{host}."/api/puppetclasses/$class");
	$henson_url->query_form(
		per_page => 10000,
	);
	my $henson_response = $ua->get($henson_url)->content();
	my $class_hostgroup_list     = decode_json($henson_response);
	if(!exists($class_hostgroup_list->{hostgroups}) || scalar(@{$class_hostgroup_list->{hostgroups}}) == 0){
		print(STDERR "\n[!] Unknown class \"$class\"\n");
		exit;
	}
	foreach my $class_hostgroup ( @{$class_hostgroup_list->{hostgroups}} ) {

		push( @match_tmp, $class_hostgroup->{name} );
	}
	print(STDERR ".");
}

# Get list of hostgroups from Foreman for environment
$henson_url = URI->new($conf{henson}->{host}.'/api/v1/hostgroups');
foreach my $environment (@environments){
	$henson_url->query_form(
		per_page => 10000,
		search   => "environment=$environment"
	);
	my $henson_response = $ua->get($henson_url)->content();
	my $env_hostgroup_list     = decode_json($henson_response);
	if(scalar(@{$env_hostgroup_list}) == 0){
		print(STDERR "\n[!] Unknown environment \"$environment\"\n");
		exit;
	}
	foreach my $env_hostgroup ( @{$env_hostgroup_list} ) {
		push( @match_tmp, $env_hostgroup->{hostgroup}->{name} );
	}
	print(STDERR ".");
}

# Deduplicate our match list (hostgroups specified multiple times or exist in massfin group
@match = do { my %seen; grep { !$seen{$_}++ } @match_tmp };

# Get list of servers from Foreman for hostroup
$henson_url = URI->new($conf{henson}->{host}.'/api/v1/hosts');
foreach my $host_group (@match) {
	$henson_url->query_form(
		per_page => 10000,
		search   => "hostgroup=$host_group"
	);
	my $henson_response = $ua->get($henson_url)->content();
	my $server_list     = decode_json($henson_response);
	### Commenting out because presently cannot tell difference between non-existent and empty hostgroup
	#if(scalar(@{$server_list}) == 0){
	#    print(STDERR "\n[!] Unknown hostgroup \"$host_group\"\n");
	#    exit;
	#}
	foreach my $server ( @{$server_list} ) {
		if ( $server->{host}->{name} !~ /$globalexcludes/ ) {
			if(grep {$_ eq $host_group} @{$conf{fintypes}->{hostgroups}->{'aso-legacy'}} or $host_group eq 'NAS800'){
				push( @runlist_tmp, $server->{host}->{name} . ':' . $conf{ssh_opts}->{$ostype}->{aso_ssh_port} );
			}
			else {
				push( @runlist_tmp, $server->{host}->{name} );
			}
		}
	}
	print(STDERR ".");
}

# Get list of servers from Foreman for fact match
$henson_url = URI->new($conf{henson}->{host}.'/api/v1/hosts');
while (scalar(@fact_values) > 0){
	my ($_fact_name, $_fact_op, $_fact_val) = (shift(@fact_values), shift(@fact_values),shift(@fact_values));
	$henson_url->query_form(
		per_page => 10000,
		search   => "facts.${_fact_name}${_fact_op}${_fact_val}"
	);
	my $henson_response = $ua->get($henson_url)->content();
	my $server_list     = decode_json($henson_response);
	foreach my $server ( @{$server_list} ) {
		if ( $server->{host}->{name} !~ /$globalexcludes/ ) {
			push( @runlist_tmp, $server->{host}->{name} );
		}
	}
	print(STDERR ".");
}

# Get list of servers from Foreman for search query
$henson_url = URI->new($conf{henson}->{host}.'/api/v1/hosts');
if($search_query){
	$henson_url->query_form(
		per_page => 10000,
		search   => "$search_query"
	);
	my $henson_response = $ua->get($henson_url)->content();
	my $server_list     = decode_json($henson_response);
	foreach my $server ( @{$server_list} ) {
		if ( $server->{host}->{name} !~ /$globalexcludes/ ) {
			push( @runlist_tmp, $server->{host}->{name} );
		}
	}
	print(STDERR ".");
}

# Direct lists from command line
foreach my $customlist_file (@customlist){
	my $linenum=0;
	open( my $DAT, '<', $customlist_file ) or die "[!] Unable to open custom list file \"$customlist_file\"!\n";
	while (<$DAT>) {
		chomp($_);
		if(/^\s*([a-zA-Z0-9-.]{1,63}(?::\d+)?)?\s*(\#.*)?$/){
			push(@runlist_tmp, $1) if($1);
		} else {
			die("[!] Unexpected data in custom list \"$customlist_file\", line $linenum: \"$_\"\n");
		}
	}
	close($DAT);
}
print(STDERR "Done!\n");   

# Deduplicate our runlist as fact checks + hostgroups could cause duplicate servers
@runlist = do { my %seen; grep { !$seen{$_}++ } @runlist_tmp };
@runlist = shuffle(@runlist) if ($random);

# Show some information before we run
print(STDERR show_info("reason", $reason)) if($reason);
print(STDERR show_info("random", $random)) if($random);
print(STDERR show_info("ostype", $ostype)) if($ostype);
print(STDERR show_info("command file", $commandfile)) if($commandfile);
print(STDERR show_info("instruction", $instruction)) if($instruction);
print(STDERR show_info("username", $username)) if($username);
print(STDERR show_info("massfin type", \@fintype_list)) if(@fintype_list);
print(STDERR show_info("lists", \@customlist)) if(@customlist);
print(STDERR show_info("environments", \@environments)) if(@environments);
print(STDERR show_info("classes", \@classes)) if(@classes);
print(STDERR show_info("facts", \@fact_values)) if(@fact_values);
print(STDERR show_info("foreman search", $search_query)) if($search_query);
print(STDERR show_info("hostgroups", \@match)) if(@match);
print(STDERR show_info("servers", \@runlist)) if($verbose && @runlist);
print(STDERR show_info("server count", scalar(@runlist)));
print(STDERR show_info("max children", $max_children));
print(STDERR show_info("output tree", $tree_folder)) if(defined($tree_folder));
print(STDERR show_info("process id", $massfin_pid));
print(STDERR show_info("retry", $retry)) if($retry);
print(STDERR show_info("retrysleep", $retry_sleep)) if($retry && $retry_sleep);
print(STDERR show_info("scp", $scp)) if($scp);

if(!-d "$log_path"){
	mkdir("$log_path",0751) or die("Cannot create logging directory \"$log_path\". Error: $?\n");
}

if(scalar(@runlist)<1){
	# Abort if no servers
	senderror("No servers found");
} elsif(!$skip){
	# Prompt to continue
	$| = 1;
	print(STDERR "[*] We're about to".($dryrun?' dry':' ')."run on " . scalar(@runlist) . " boxes".($dryrun?"\n":", press enter key to continue: "));
	my $read = <STDIN> if(!$dryrun);
}

if ($scp) {
if (! -f abs_path($scp) ) {
  print abs_path($scp);
  senderror("[!] $scp file not found");
}
}

# Affix username to reason for logging
$reason = "(".$whoami.") ".$reason if($whoami && $reason);

# Log to massfin.labs and send snitch emails only after we've commited to actually doing it
if (!$dryrun){
	if ( !$nolog ) {
		$ENV{'PERL_LWP_SSL_VERIFY_HOSTNAME'} = 0;
		my $cli = RPC::XML::Client->new(
			'https://192.185.178.7/rpc',
			ssl_opts => {
				verify_hostname => 0,
				SSL_use_cert    => 0,
				SSL_verify_mode => 'SSL_VERIFY_NONE'
			},
		);
		my $response = $cli->send_request(
			'postlog',
			{
				serverlist  => "@fintype_list",
				command     => ($commandfile?$commandfile:'(instruction)'),
				commandblob => join( "\n", @commands ),
				reason      => $reason,
			}
		);
	}
	else {
		my $message = MIME::Lite->new(
			From    => 'root@wizard2.hostgator.com',
			To      => 'level3s@hostgator.com',
			Subject => 'Massfin Logging Disabled Notification',
			Type    => 'text/plain',
			Data    => "Type: @fintype_list\nManual Groups: @manual_groups\nCommand File: ".($commandfile?$commandfile:'(instruction)')."\nReason: $reason\nCommands:\n @commands\n",
		);
		$message->send();
	}
	my $time    = localtime();
	my $message = MIME::Lite->new(
		From    => 'root@wizard2.hostgator.com',
		To      => 'directors@hostgator.com',
		Subject => 'Massfin Log ' . $time,
		Type    => 'text/plain',
		Data    => "Type: @fintype_list\nManual Groups: @manual_groups\nCommand File: ".($commandfile?$commandfile:'(instruction)')."\nReason: $reason\nCommands:\n @commands\n",
	);
	$message->send();
}

# Set-up progress stuff
if ($progress) {
	$progress = Progress->new( $count, scalar(@runlist) );
	$progress->start();
}

# Create lockfile
if ( !-e '/tmp/massfin-' . $massfin_pid . '.lock' ) {
	open( my $FH, '>', '/tmp/massfin-' . $massfin_pid . '.lock' ) or die('could not create lock file for I/O');
	close($FH);
}

$pushdate = time();

# Loop through servers
if(!$dryrun){
	run_loop($massfin_error_file, 0 , @runlist);
        copy($massfin_error_file, $massfin_error_file_retry);
        if (-f $massfin_error_file && $retry){
                for(my $i=1; $i <= $retry; $i++){
                        if(-f $massfin_error_file_retry){
                                open(my $retry_fh, '<', $massfin_error_file_retry);
                                chomp(my @runlist_retry = <$retry_fh>);
                                close($retry_fh);
                                unlink($massfin_error_file_retry);
                                if(scalar(@runlist_retry)>0){
                                        print(STDERR "\n[*] Retry $i/$retry with ".scalar(@runlist_retry)." failed servers in $retry_sleep seconds\n");
                                        sleep($retry_sleep);
                                        run_loop($massfin_error_file_retry, $i, @runlist_retry);
                                }
                        }
                }
                if(-f $massfin_error_file_retry){
                        unlink($massfin_error_file);
                        rename($massfin_error_file_retry,$massfin_error_file);
                }
        }
} else {
	print(STDERR show_info("*** DRY-RUN ***",""));
}
finish();



############################## ###  ##   #    Subs    #   ##  ### ##############################

# Adjust max_children on the fly
sub set_max_children {
        my $pm = shift;
        my $path = $conf{massfin_opts}->{option_path};
        mkdir($path) if(!-d $path);
        mkdir($path . "/$massfin_pid") if(!-e $path . "/$massfin_pid");
        if(-e $path . "/$massfin_pid/max_children"){
                chomp(my $procs = slurp($path . "/$massfin_pid/max_children"));
                $pm->set_max_procs($procs) if($procs =~ /^\d+$/);
        }
}

# Return informational output, like pre-run data for confirmation
sub show_info {
	my $decoration = "[*]";
	$Text::Wrap::columns = ($terminal_width - (length($decoration." ".$_[0].": ")+5));
	if(ref($_[1]) eq 'ARRAY'){
		return($decoration." ".$_[0].": ".($nowrap?"@{$_[1]}":wrap("",$decoration.(" "x(length($_[0])+3)),@{$_[1]}))."\n");
	} else {
		return($decoration." ".$_[0].": ".$_[1]."\n");
	}
}

# Write and log error message
sub senderror {
	my $errormsg = shift;
	print $logprefix . $errormsg . "\n";
	open( my $WAT, '>>', 'error_log' );
	print $WAT $logprefix . $errormsg . "\n";
	close($WAT);
	exit();
}

# Test port
sub check_port {
	if ( IO::Socket::INET->new( PeerAddr => $_[0], PeerPort => $_[1], Timeout => 3, Proto => 'tcp' ) ) {
		return 1;
	}
	else {
		return 0;
	}
}

# Show usage and die
sub usage {
        print("\n[!] Usage: $0 <types|-l <listfile>|-g <henson group>|-e <henson environment>|-C <henson class>|-f <fact comparison>|-S <search query>|-I <server>> < -c <command file>|-i <instruction> > -r <reason> [-m <integer>] [-q|--quiet] [-v|--verbose] [-p|--progress] [-n|--noredir] [-d|--delay <integer>] [-t|--tree <output path>] [-W|--nowrap] [-D|--dryrun] [-R|--(no)random] [-u|--user] [-w|--windows] [-T|--timeout] [-h|--help]\n\n");
        print("\tSee \"--help\" for further help\n") if(!$help);
        show_help() if($help);
        print("\n");
        exit(1);
}

# Show full help output
sub show_help {
    my $help_message = <<"END_HELP";
        OPTION				TYPE	DESCRIPTION
	-c|--command <filename>		cmd	Command file to run on target servers. Cannot be ran with --instruction
	-C|--class <classname>		list	Foreman class to add to server list. Can be specified more than once
	-d|--delay <seconds>			Pause between runs
	-D|--dryrun				Perform a dry-run. Show what it would do, but to not run on servers
	-e|--environment <environment>	list	Foreman environment to add to server list. Can be specified more than once
	-f|--fact <fact>		list	Puppet fact to add to server list. Can be specified more than once
	-g|--group <group>		list	Foreman group to add to server list. Can be specified more than once
	-h|--help				Show this help
	-i|--instruction <commands>	cmd	A short command to run on server. Cannot be ran with --command
	-I|--server <server>		list	Server to connect to. Can be specified more than once
	-l|--list <filename>		list	List of servers to connect to. Can be specified more than once
	-m|--max <num>				Maximum number of children to conenct to simultaneously. Default $max_children
	-n|--noredir				Do not discard STDERR
	-p|--progress				Show rudimentary progress indicator
	-q|--quiet				Do not show failures in-line
	-r|--reason <reason>		reason	Reason for running massfin. Required
	-R|--random				Randomize servers. Disable with --norandom
           --retry <# retries>                  Retry failed connection this many times
	-s|--skip				Skip prompt
	-S|--search <query>		list	Foreman query to add to server list
	-t|--tree <path>			Folder to send distributed output files to. One file per server within this folder
	-T|--timeout <seconds>			SSH connection timeout. Default $conf{ssh_opts}->{ssh_timeout}
	-u|--user <username>			Connect as this user
	-v|--verbose				Be verbose. Show servers prior to running
	-w|--windows|--win			Use windows mode for connecting to widnows with SSH
	-W|--nowrap				Do not pretty-wrap verbose output
END_HELP
    print($help_message."\n");
    print("\n\tPossible massfin server types (type: list):\n");
    foreach my $fintype (sort(keys %{$conf{fintypes}})){
            printf("\t\t%-16s = %s\n", $fintype, $conf{fintypes}->{$fintype}->{description});
    }
    print("\n\tMassfin requires at least one each from types \"list\", \"cmd\", and \"reason\" in order to run.\n");   
}

# The run loop
sub run_loop {
        my $massfin_error_file_temp = shift;
        my $retry_num               = shift;
        my @run_loop_list           = @_;
        my $pm = new Parallel::ForkManager($max_children);
        system("/usr/eig/bin/sshca.py");
	my $key_refresh_time = time();
        foreach my $server (@run_loop_list) {
                set_max_children($pm);
                $progress->update($count, scalar(@run_loop_list)) if($progress);
                $count++ if(!$retry_num);
		if ( $key_refresh_time < time() - 1800 ){
		  system("/usr/eig/bin/sshca.py");
		  $key_refresh_time = time();
		}
                my $pid = $pm->start and next;
                setpriority(0,$$,$conf{massfin_opts}->{fork_niceness}) if($pid == 0);
                my $mininame = $server;
                $mininame =~ s/\.(hostgator|websitewelcome)\.com$//;
                local $0 = "[$mininame] $zeeroh";
                my $return = run($server);
                if ( $return != 1 ) {
                        if ( !$quiet ) { print STDERR "[!] Failed on $server \n" }
                        if ( sysopen( my $WAT, $massfin_error_file_temp, O_CREAT | O_WRONLY | O_APPEND ) ) {
                                print $WAT "$server\n";
                                close($WAT);
                        }
                }
                usleep(3000);
                local $0 = $zeeroh;
                sleep($delay) if $delay;
                $pm->finish;
        }
        $pm->wait_all_children;
        $progress->stop() if($progress);
}

# The actual push
sub run {
	my $input = shift;
	my ( $server, $port ) = split ( ':' , $input, 2 );
	$port = $conf{ssh_opts}->{$ostype}->{ssh_port} if(!$port);
	# check to see if 22 is listening to avoid long timeouts.
	if ( !check_port( $server, $port ) ) {
		if ( !check_port( $server, $port ) ) {
			return 0;
		}
	}
	unshift(@commands, "source /etc/bashrc && source /etc/profile\n") if(!$ostype eq 'windows');
	if ($verbose) { print STDERR "[*] Pushing to $server.\n" }
	my (@outputdump, $ssherror);
	my ($_stdout, $_stderr, @_result) = capture {
		my $ssh = Net::OpenSSH->new(
			$server,
			kill_ssh_on_timeout	=>	1,
			port			=>	$port,
			user			=>	$username,
			batch_mode		=>	1,
			master_opts		=>	[
							-o => 'LogLevel=quiet',
							-o => 'StrictHostKeyChecking=no',
						],
			timeout			=>	$conf{ssh_opts}->{ssh_timeout},
		);
		return 0 if($ssh->error);
		$ssherror=$ssh->error;
		if($ostype eq 'windows'){
			@outputdump = $ssh->capture("@commands"); # Need this for windows
		}else{
                        if ($scp) {
                          $ssh->system("[[ -d /root/.massfin_tmp ]] || mkdir /root/.massfin_tmp") or do { $ssherror=$ssh->error; return(0); };
                          $ssh->scp_put(abs_path($scp), '/root/.massfin_tmp') or do { $ssherror=$ssh->error; return(0); };
                        }
			@outputdump = $ssh->capture({stdin_data => \@commands}); # Linux
                        if ($scp) {
                          $ssh->system("[[ -d /root/.massfin_tmp ]] && rm -fr /root/.massfin_tmp/*") or return 0;
                        }
		}
	};
	return(0) if(!defined($ssherror)||$ssherror!=0);
	@outputdump = grep(!/^(Last failed login: .* from .* on|There (were|was) \d+ failed login attempts? since the last successful login)/,@outputdump) if(scalar(@outputdump)>0);
	if ( !defined($quiet) ) {
		open( my $WAT, '>>', "$log_path/$server.log" );
		print $WAT time2str( "%c", time ) . " - $reason\n@commands\nOUTPUT(if any): \n@outputdump\n\n";
		close($WAT);
	}
	if ( scalar @outputdump > 0 ) {
		local $| = 1;
		open( my $LOCK, '>', '/tmp/massfin-' . $massfin_pid . '.lock' ) or die( 'Could not open massfin lock file: ' . $! );
		flock( $LOCK, LOCK_EX ) or die( 'Could not get flock: ' . $! );
		chomp($outputdump[$#outputdump]);
		if(defined($tree_folder) && -d $tree_folder){
			open(my $output_file, ">", "$tree_folder/$server") or die("Could not open \"$tree_folder/$server\" for writing: $!");
			foreach (@outputdump) {
				print($output_file $_);
			}
			print($output_file "\n");
			close($output_file);
		} else {
			foreach (@outputdump) {
				print $_;
			}
			print("\n");
		}
		close($LOCK);
	}
	return 1;
}

# Read a file
sub slurp {local$/=<>if local@ARGV=@_}

# Remove lock, write out error report
sub finish {
	if ( -f $massfin_error_file ) {
		print STDERR "[!!!] The following servers did not accept the push: \n";
		open( my $DAT, '<', $massfin_error_file );
		while (<$DAT>) {
			$count_failures++;
			print STDERR $_;
		}
		close($DAT);
		unlink($massfin_error_file);
	}
	print STDERR "\n";
	print STDERR "[*] Executed on ".($count - $count_failures)."/$count servers\n" if($count);
	print STDERR "[*] Total massfin runtime: ".strftime("%Hh %Mm %Ss", gmtime(time() - $date))."\n";
	print STDERR "[*] Push execute time:     ".strftime("%Hh %Mm %Ss", gmtime(time() - $pushdate))."\n"; 

        system("rm -rf ".$conf{massfin_opts}->{option_path}."/".$massfin_pid) if(exists($conf{massfin_opts}->{option_path}) && length($conf{massfin_opts}->{option_path}) > 4 && $conf{massfin_opts}->{option_path}."/".$massfin_pid ne "/" && -d $conf{massfin_opts}->{option_path}."/".$massfin_pid);
	if ( -e '/tmp/massfin-' . $massfin_pid . '.lock' ) {
		unlink( '/tmp/massfin-' . $massfin_pid . '.lock' );
	}
	print STDERR "\n";
	exit();
}

1;
