#!/usr/bin/perl

use strict;
use warnings;
use YAML::Syck;
use Proc::ProcessTable;
use Sys::Syslog qw(:standard :macros);
use sigtrap 'handler' => \&USR1handler, 'USR1';
use Getopt::Long;
use Proc::Daemon;
use File::Slurp;
use Storable;
use Data::Dumper;

$0 = "apache-monitor";

my $daemon = Proc::Daemon->new(
    work_dir => "/root/",
    pid_file => "/var/run/apache-monitor.pid"
);

my $pid = $daemon->Status();
my ( $start, $stop, $status, $procs_time, $debug );
my $stor_data = "/tmp/.$0";

GetOptions(
    "start"  => \&run,
    "status" => \&status,
    "stop"   => \&stop,
    "debug"  => \&debug
);

sub stop {
    if ($pid) {
        print "Stopping pid $pid...\n";
        if ( $daemon->Kill_Daemon() ) {
            print "Successfully stopped.\n";
        }
        else {
            print "Could not find $pid.  Was it running?\n";
        }
    }
    else {
        print "Not running, nothing to stop.\n";
    }
}

sub status {
    if ($pid) {
        print "Running with pid $pid.\n";
        exit 0;
    }
    else {
        print "Not running.\n";
        exit 1;
    }
}

sub run {
    if ( !$pid ) {
        print "Starting...\n";
        $daemon->Init;
        eval { $procs_time->{conf} = LoadFile('/opt/eig_linux/etc/apache-monitor.yaml') };
        if ($@) {
            openlog( $0, 'cons,pid', 'user' );
            syslog( 'info', '%s', "Failed loading /opt/eig_linux/etc/apache-monitor.yaml\n" . $@ );
            closelog();
            exit 1;
        }
        if ( -f $stor_data ) {
            $procs_time->{pid_status} = retrieve($stor_data);
        }
        while (1) {
            openlog( $0, 'cons,pid', 'user' );
            apache_check();
            closelog();
            if ( $procs_time->{pid_status} ) {
                store( $procs_time->{pid_status}, $stor_data );
                chmod 0600, $stor_data;
            }
            sleep(60);
        }
    }
    else {
        print "Already Running with pid $pid\n";
    }
}

sub apache_check {
    chomp( my @score_board = qx{timeout -s 9 15 lynx --dump localhost/whm-server-status?auto} );
    while ($?) {
        syslog( 'info', '%s', "localhost/whm-server-status?auto timed out" );
        sleep 5;
        chomp( @score_board = qx{timeout -s 9 15 lynx --dump localhost/whm-server-status?auto} );
    }
    foreach my $s_line (@score_board) {
        if ( $s_line =~ /^ServerMPM: (\w+)/ ) {
            $procs_time->{ServerMPM} = $1;
        }
        if ( $s_line =~ /^Stopping: (\d+)/ ) {
            $procs_time->{stopping_state_ct} = $1;
            last;
        }
    }

    if ( $procs_time->{ServerMPM} ne 'event' ) {
        syslog( 'err', '%s', "Only Event MPM is supported. Current MPM: $procs_time->{ServerMPM}" );
        die;
    }

    if ( $procs_time->{stopping_state_ct} ) {
        if ( $procs_time->{pid_status_tmp} ) {
            undef $procs_time->{pid_status_tmp};
        }
        chomp( my @score_board_full = qx{timeout -s 9 15 lynx --dump localhost/whm-server-status} );
        while ($?) {
            syslog( 'info', '%s', "localhost/whm-server-status timed out" );
            sleep 5;
            chomp( @score_board_full = qx{timeout -s 9 15 lynx --dump localhost/whm-server-status} );
        }
        my $match_count = 0;
        foreach my $s_line (@score_board_full) {
            if ( my ( $pid, $conn_total, $conn_busy, $conn_graceful, $conn_idle, $conn_writing, $conn_keepalive, $conn_closing ) =
                $s_line =~ /^\s+\d{1,2}\s+(\d{1,7})\s+yes\s+(?:\(old gen\)\s)?(\d+)\s+no\s+(\d+)\s+(\d+)\s+(\d+)\s+(?:\d+\s+)?(\d+)\s+(\d+)\s+(\d+)$/ )
            {
                $match_count++;
                if ( $procs_time->{pid_status}->{$pid} ) {
                    $procs_time->{pid_status}->{$pid}->{conn_total}     = $conn_total;
                    $procs_time->{pid_status}->{$pid}->{conn_busy}      = $conn_busy;
                    $procs_time->{pid_status}->{$pid}->{conn_graceful}  = $conn_graceful;
                    $procs_time->{pid_status}->{$pid}->{conn_idle}      = $conn_idle;
                    $procs_time->{pid_status}->{$pid}->{conn_writing}   = $conn_writing;
                    $procs_time->{pid_status}->{$pid}->{conn_keepalive} = $conn_keepalive;
                    $procs_time->{pid_status}->{$pid}->{conn_closing}   = $conn_closing;
                    $procs_time->{pid_status_tmp}->{$pid}               = $procs_time->{pid_status}->{$pid};
                }
                else {
                    $procs_time->{pid_status_tmp}->{$pid}->{conn_total}     = $conn_total;
                    $procs_time->{pid_status_tmp}->{$pid}->{conn_busy}      = $conn_busy;
                    $procs_time->{pid_status_tmp}->{$pid}->{conn_graceful}  = $conn_graceful;
                    $procs_time->{pid_status_tmp}->{$pid}->{conn_idle}      = $conn_idle;
                    $procs_time->{pid_status_tmp}->{$pid}->{conn_writing}   = $conn_writing;
                    $procs_time->{pid_status_tmp}->{$pid}->{conn_keepalive} = $conn_keepalive;
                    $procs_time->{pid_status_tmp}->{$pid}->{conn_closing}   = $conn_closing;
                    $procs_time->{pid_status_tmp}->{$pid}->{stop_time}      = time();
                }
            }
        }
        if ( $procs_time->{stopping_state_ct} > 0 && $match_count == 0 ) {
            syslog( 'err', '%s', "Apache reports $procs_time->{stopping_state_ct} stopping processes, but regex failed to match any in the full scoreboard. Format may have changed." );
        }
        if ( $procs_time->{pid_status_tmp} ) {
            $procs_time->{pid_status} = $procs_time->{pid_status_tmp};
        }
        if ( $procs_time->{stopping_state_ct} > $procs_time->{conf}->{max_procs} ) {
            syslog( 'info', '%s', "processes in stopping state: $procs_time->{stopping_state_ct}" );
            runtime_check();
        }
    }
}

sub runtime_check {
    my $over_limit = $procs_time->{stopping_state_ct} - $procs_time->{conf}->{max_procs};
    undef $procs_time->{pid_time};
    foreach my $pid ( keys %{ $procs_time->{pid_status} } ) {
        $procs_time->{pid_time}->{$pid} = $procs_time->{pid_status}->{$pid}->{stop_time};
        if ($debug) {
            syslog( 'info', '%s', "DEBUG: Proc in stopping state pid: $pid " . $procs_time->{pid_status}->{$pid}->{stop_time} );
        }
    }
    my $cnt = 0;
    foreach my $pid ( sort { $procs_time->{pid_time}->{$a} <=> $procs_time->{pid_time}->{$b} } keys %{ $procs_time->{pid_time} } ) {
        $cnt++;
        if (
            (
                   ( $procs_time->{pid_status}->{$pid}->{stop_time} < ( time() - $procs_time->{conf}->{proc_stop_ttl} ) )
                && ( $procs_time->{pid_status}->{$pid}->{conn_busy} == 0 )
                && ( $procs_time->{pid_status}->{$pid}->{conn_graceful} == 0 )
                && ( $procs_time->{pid_status}->{$pid}->{conn_idle} == 0 )
                && ( $procs_time->{pid_status}->{$pid}->{conn_writing} == 0 )
                && ( $procs_time->{pid_status}->{$pid}->{conn_keepalive} == 0 )
                && ( $procs_time->{pid_status}->{$pid}->{conn_closing} == 0 )
            )
            || ( $procs_time->{pid_status}->{$pid}->{stop_time} < ( time() - $procs_time->{conf}->{max_proc_stop_ttl} ) )
          )
        {
            kill_proc($pid);
        }
        if ( $cnt == $over_limit ) {
            last;
        }
    }
}

sub kill_proc {
    my $pid = shift;
    if ( ( -l "/proc/${pid}/exe" ) && ( readlink "/proc/${pid}/exe" eq '/usr/sbin/httpd' ) ) {
        my $message = 'pid:'
          . $pid
          . ' conn_total:'
          . $procs_time->{pid_status}->{$pid}->{conn_total}
          . ' conn_busy:'
          . $procs_time->{pid_status}->{$pid}->{conn_busy}
          . ' conn_graceful:'
          . $procs_time->{pid_status}->{$pid}->{conn_graceful}
          . ' conn_idle:'
          . $procs_time->{pid_status}->{$pid}->{conn_idle}
          . ' conn_writing:'
          . $procs_time->{pid_status}->{$pid}->{conn_writing}
          . ' conn_keepalive:'
          . $procs_time->{pid_status}->{$pid}->{conn_keepalive}
          . ' conn_closing:'
          . $procs_time->{pid_status}->{$pid}->{conn_closing}
          . ' stop_time:'
          . $procs_time->{pid_status}->{$pid}->{stop_time};

        my $rc = kill 'SIGABRT', $pid;
        if ( $rc == 1 ) {
            check_semaphores($pid);
        }
        syslog( 'info', '%s', "$message SIGABRT:$rc" );
    }
    else {
        syslog( 'info', '%s', "Failed safety check, not killing PID: $pid $procs_time->{pid_status}->{$pid}->{stop_time}" );
    }
}

sub check_semaphores {
    my $pid = shift;
    chomp( my @semaphore_list = qx{/usr/bin/lsipc -s -o ID,OWNER --noheadings} );
    foreach my $s_line (@semaphore_list) {
        my ( $id, $owner ) = split( " ", $s_line );
        if ( $owner eq 'nobody' ) {
            chomp( my @semaphore_info = qx{/usr/bin/lsipc -s -i $id} );
            foreach my $s_line (@semaphore_info) {
                if ( my ($sem_pid) = $s_line =~ /^\s+\d+\s+\d+\s+\d+\s+\d+\s+(\d+)\s+.*$/ ) {
                    if ($debug) {
                        syslog( 'info', '%s', "DEBUG: sem check pid:$pid sem_id:$id sem_pid:$sem_pid" );
                    }
                    if ( $pid == $sem_pid ) {

                        # leaving this commented out as there's no concrete evidence that we need to force clean them
                        # qx{/usr/bin/ipcrm -s $id};
                        if ($debug) {
                            syslog( 'info', '%s', "DEBUG: sem match sem_id:$id pid:$pid" );
                        }
                    }
                }
            }
        }
    }
}

sub USR1handler {
    write_file( '/tmp/apache-monitor.dump', { perms => 0600 }, Dumper($procs_time) );
}
