#!/usr/bin/perl

use strict;
use warnings;

use Search::Elasticsearch;
use HTTP::Tiny;
use JSON::MaybeXS qw(encode_json);
use MIME::Base64 qw(encode_base64);
use Getopt::Long qw(GetOptions);
use Sys::Hostname;

GetOptions(
    'nodes=s' => \my $nodes,
    'host=s'  => \my $host,
);

my $internal_metrics = {};
my $time_start = time();

my $e = Search::Elasticsearch->new(
    #nodes => '10.102.98.151:9200',
    nodes => $nodes // 'elastic.beta.pro1.techops.tools:9200',
);

## SETUP
#my $HOST    = 'bkafka1.t.techops.tools'; # '10.102.98.11'; # TODO should have defaults for beta and prod as well
    my $HOST    = $host // 'kafkarest.beta.pro1.techops.tools'; # '10.102.98.11'; # TODO should have defaults for beta and prod as well
    my $PORT    = 80;                        # 8082;
    my $PATH    = 'topics/user-graphite';


my $results = $e->search(
    body =>{
## Don't return the records, just the aggregate
        size => 0,
        query => {
            bool => {
                filter => {
                    range => {
                        timestamp => {
                            gte => 'now/h', ## last hour  now/d for one day
                            #gte => '2020-10-30',
                            #format => 'yyyy-MM-dd'

                        }
                    }
                }
            }
        },
        aggs => {
            group_by_domain => {
                terms => {
                    field => 'domain'
                },
                aggs => {
                    usernames => {
                        terms => {
                            field => 'acctname'
                        },
                    },
## how many unique IP
                    visitor_count => {
                        cardinality => {
                            field => 'IP'
                        }
                    },
## sum of bytes on the domain
                    bytes_count => {
                        sum => {
                            field => 'bytes'
                        }
                    },
## extended data on bandwidth usage
                    bandwidth => {
                        extended_stats => {
                            field => 'bytes'
                        }
                    },
                    group_by_source => {
                        terms => {
                            field => 'source'
                        },
                        aggs => {
                            host_bandwidth => {
                                sum => {
                                    field => 'bytes'
                                }
                            },
                            bandwidth_over_time => {
                                date_histogram => {
                                    field => 'timestamp',
                                    interval => '5m',
                                    #format => '-MM-dd',
                                },
                                aggs => {
                                    bandwidth_breakout => {
                                        extended_stats => {
                                            field => 'bytes'
                                        }
                                    },
                                }
                            },
                        }
                    },
## hits per IP

                }
            }
        }
    }
);

# capture some internal metrics off of the elastic-search results
$internal_metrics->{es_took} = $results->{took};
$internal_metrics->{es_shards_failed} = $results->{_shards}->{failed};
$internal_metrics->{es_shards_skipped} = $results->{_shards}->{skipped};
$internal_metrics->{es_shards_successful} = $results->{_shards}->{successful};
$internal_metrics->{es_shards_total} = $results->{_shards}->{total};
$internal_metrics->{es_hits_total} = $results->{hits}->{total} if defined($results->{hits}->{total});

my $url  = "http://$HOST:$PORT/$PATH";
if( $results->{'aggregations'} ) {
    my $prefix = 'user.cmh';
    my $ht = HTTP::Tiny->new;
## We got results
    foreach my $site_hits ( @{ $results->{'aggregations'}->{'group_by_domain'}->{'buckets'} } ){
        my $username = $site_hits->{'usernames'}->{'buckets'}->[0]->{'key'} // '-';
        my $domain = $site_hits->{'key'};
        $domain =~ s/[^a-z0-9-]/_/g;
        warn "Processing: ", $domain, "\n" if $ENV{'DEBUG_METRIC'};
        $internal_metrics->{total_site_count}++;
        my $name = "$domain.bandwidth";
##Top level info
# doc_count => record count
# key       => domain
# bytes_count => value => sum of bytes served (bandwidth) -- unnecessary if we do the larger stats dump
        warn "\tTotal Bandwidth: ", $site_hits->{'bytes_count'}->{'value'}, "\n" if $ENV{'DEBUG_METRIC'};
        $internal_metrics->{total_site_bandwidth} += $site_hits->{'bytes_count'}->{'value'};

## total for site
        my @mets = ();
        foreach my $server ( @{ $site_hits->{'group_by_source'}->{'buckets'} } ) {
            my $servername = $server->{'key'};
            warn "\t\tServer: ", $servername, "\t", $server->{'host_bandwidth'}->{'value'}, "\n" if $ENV{'DEBUG_METRIC'};
            $internal_metrics->{total_site_server_count}++;

            my ($short_host, $long_host) = split /\./, $servername, 2;
            $long_host =~ s/[^a-z0-9-]/_/g;
            $short_host =~ s/[^a-z0-9-]/_/g;
# host_bandwidth => value
            foreach my $five_min ( @{ $server->{'bandwidth_over_time'}->{'buckets'} } ) {
# bandwidth_over_time => buckets => [ {key / key_as_string / doc_count / bandwidth_breakout} ]
                # count / sum
                if($five_min->{'doc_count'} > 0 ) {
                    warn "\t\t\t", join(',', $five_min->{'key_as_string'}, $five_min->{'doc_count'}, $five_min->{'bandwidth_breakout'}->{'sum'} // 0), "\n" if $ENV{'DEBUG_METRIC'};
                    my $time = $five_min->{'key'} * .001; ## or do we want key_as_string ?
                    my $value = $five_min->{'bandwidth_breakout'}->{'sum'} // 0; ##error page hits that have 0 bandwidth value
                    my $metric = sprintf "%s.%s.%s.%s.%s %d %d\n", $prefix, $long_host, $short_host, $username, $name, $value, $time;
                    warn $metric if defined($ENV{'DEBUG_METRIC'}) and $ENV{'DEBUG_METRIC'} > 1;
                        # using domain instead of user, since we have it
                    $internal_metrics->{total_bucket_count}++;

                    push( @mets, { value => encode_base64($metric, '') } );
                }
            }
        }
## JSON encode
        my $json = encode_json({records => \@mets });
        $internal_metrics->{host_post_count}++;
        my $resp = $ht->request( POST => $url, {
            content => $json,
            headers => {
                'Content-Type' => 'application/vnd.kafka.binary.v2+json',
                Accept         => 'application/vnd.kafka.v2+json',
            },
        });
        if (! $resp->{'success'}) {
            warn "Failed sending data to $url (".($resp->{'code'} // 'undef').")" if $ENV{'DEBUG_METRIC'};
            print "ERROR\n";
            $internal_metrics->{host_post_error}++;
        }else{
            $internal_metrics->{host_post_success}++;
        }

    }
    print "SUCCESS\n";

}

my $time_end = time();
$internal_metrics->{runtime} = $time_end - $time_start;
my $local_hostname = hostname();
$local_hostname =~ s/\./_/g;
my @mets = ();
foreach my $metric_key (keys %$internal_metrics){
    my $metric = sprintf("%s.%s.%s %d %d\n","mwp_metrics_bandwidth",$local_hostname,$metric_key,$internal_metrics->{$metric_key},$time_start);
    warn $metric if $ENV{'DEBUG_METRIC'};
    push( @mets, { value => encode_base64($metric, '') } );
}
my $json = encode_json({records => \@mets });
my $ht = HTTP::Tiny->new;
my $resp = $ht->request( POST => $url, {
    content => $json,
    headers => {
        'Content-Type' => 'application/vnd.kafka.binary.v2+json',
        Accept         => 'application/vnd.kafka.v2+json',
    },
});
if (! $resp->{'success'}) {
     warn "Failed sending data to $url (".($resp->{'code'} // 'undef').")" if $ENV{'DEBUG_METRIC'};
     print "ERROR\n";
}

__END__
my $prefix = 'user.cmh';
    my ($short_host, $long_host) = split /\./, $hostname, 2;
    $long_host =~ s/[^a-z0-9-]/_/g;
    $name = "$domain.bandwidth";
    $name =~ s/[^a-z0-9-]/_/g;
    my $metric = sprintf "%s.%s.%s.%s.%s %d %d\n", $prefix, $long_host, $short_host, $user, $name, $value, $time;
