#!/usr/bin/perl

use v5.10;
use strict;
use warnings;

use Search::Elasticsearch qw();
use HTTP::Tiny    qw();
use JSON::MaybeXS qw(encode_json);
use MIME::Base64  qw(encode_base64);
use Getopt::Long  qw(GetOptions);
use Sys::Hostname qw(hostname);

GetOptions(
    'nodes=s'  => \my $nodes,
    'host=s'   => \my $host,
    'dry-run+' => \my $dry_run,
    'size=i'   => \my $size,
);
$size //= 500000;

## Kafka setup
my $HOST = $host // 'kafkarest.beta.pro1.techops.tools'; # '10.102.98.11'; # TODO should have defaults for beta and prod as well
my $PORT = 80; # 8082;
my $PATH = 'topics/user-graphite';
my $url  = "http://$HOST:$PORT/$PATH";
my $ht   = HTTP::Tiny->new;

my $time      = time; # don't have a better marker
my $snap_hour = int($time / 3600) * 3600 - 1;      # the last second of the previous hour
my $snap_day  = int(($snap_hour) / 86400) * 86400; # the first second of the day

## Elasticsearch setup
my $prefix = 'user.cmh';
$nodes //= 'elastic.beta.pro1.techops.tools:9200';
my $e = Search::Elasticsearch->new( nodes => $nodes );

my $results = $e->search(
    body => {
        size  => 0, # Don't return the records, just the aggregate
        query => {
            bool => {
                filter => {
                    range => {
                        timestamp => {
                            gte => format_time($snap_day),
                            lte => format_time($snap_hour),
                        }
                    }
                }
            }
        },
        aggs => {
            group_by_username => {
                terms => {
                    size  => $size, # TODO, find a way that won't bust things
                    field => 'acctname'
                },
                aggs => {
                    group_by_domain => {
                        terms => {
                            size  => 500, # Odds are a single username won't have more than this (we hope)
                            field => 'domain'
                        },
                        aggs => {
                            group_by_source => {
                                terms => {
                                    size  => 50, # We really shouldn't have very many "source" values (these are cluster nodes)
                                    field => 'source',
                                },
                                aggs => {
                                    visitor_count => {
                                        cardinality => {
                                            field => 'clientip' ## how many unique IP
                                            # Do not use the "IP" field, since it includes both source and destination IPs
                                        }
                                    },
                                },
                            }
                        },
                    },
                }
            }
        }
    }
);

# capture some internal metrics off of the elastic-search results
my $internal_metrics = {};
$internal_metrics->{es_took} = $results->{took};
$internal_metrics->{es_shards_failed} = $results->{_shards}->{failed};
$internal_metrics->{es_shards_skipped} = $results->{_shards}->{skipped};
$internal_metrics->{es_shards_successful} = $results->{_shards}->{successful};
$internal_metrics->{es_shards_total} = $results->{_shards}->{total};
$internal_metrics->{es_hits_total} = $results->{hits}->{total} if defined($results->{hits}->{total});
$internal_metrics->{host_post_error} = 0;
$internal_metrics->{host_post_success} = 0;
$internal_metrics->{host_post_dry_run} = 0;

my @mets = ();
if ( $results->{'aggregations'} ) {
    foreach my $user_hits ( @{ $results->{'aggregations'}->{'group_by_username'}->{'buckets'} } ) {
        my $username = $user_hits->{'key'};

        warn "Processing username: ", $username, "\n" if $ENV{'DEBUG_METRIC'};
      DOMAIN:
        foreach my $domain_hits ( @{ $user_hits->{'group_by_domain'}->{'buckets'} } ) {
            my $domain = $domain_hits->{'key'};
            $domain =~ s/[^a-z0-9-]/_/g;
            warn "Processing domain: ", $domain, "\n" if $ENV{'DEBUG_METRIC'};
            $internal_metrics->{'total_site_count'}++;

            my $name = "$domain.visitors";

            my $cluster;
            my $total_value = 0;
          CLUSTER_MEMBER:
            foreach my $user_hits ( @{ $domain_hits->{'group_by_source'}->{'buckets'} } ){
                my $value = $user_hits->{'visitor_count'}->{'value'} // 0;

                $internal_metrics->{'total_site_visitors'} += $value;
                $internal_metrics->{'total_doc_count'}     += $user_hits->{'doc_count'};
                next CLUSTER_MEMBER unless $value;

                $total_value += $value;
                $cluster //= do {
                    my $c = $user_hits->{'key'} // '-.-';
                    $c =~ s/^[^.]*?[.]//; # Take off the first segment of the hostname.
                    $c =~ s/[.]/_/g;
                    $c = 'mwp_techops_tools' if $c eq 'mwp11_cmh_pro1_eigdev_com';
                    $c;
                };
            }

            next DOMAIN unless $total_value;

            warn "\t$username total visitor: $total_value\n" if $ENV{'DEBUG_METRIC'} && $ENV{'DEBUG_METRIC'} < 2;
            my $metric = sprintf "%s.%s.%s.%s.%s %d %d\n", $prefix, $cluster, 'cp', $username, $name, $total_value, $snap_hour;
            warn $metric if $ENV{'DEBUG_METRIC'} > 1;
            push( @mets, { value => encode_base64($metric, '') } );
            send_to_kafka(\@mets) if scalar @mets >= 1000; # avoid size limits by sending in chunks
        }
    }
}
send_to_kafka(\@mets);

$internal_metrics->{'runtime'} = time() - $time;

my $local_hostname = hostname();
$local_hostname =~ s/[^a-z0-9-]/_/g;

while (my ($metric_key, $metric_value) = each %$internal_metrics) {
    my $metric = sprintf("%s.%s.%s %d %d\n", 'mwp_metrics_visitors', $local_hostname, $metric_key, $metric_value, $time);
    warn $metric if $ENV{'DEBUG_METRIC'};
    push( @mets, { value => encode_base64($metric, '') } );
}
send_to_kafka(\@mets);

exit 1 if $internal_metrics->{'host_post_error'};
exit;

sub format_time {
    my $t = shift;
    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = gmtime($t);
    sprintf("%04d-%02d-%02d %02d:%02d:%02d.000", $year+1900, $mon+1, $mday, $hour, $min, $sec);
}

sub send_to_kafka {
    my ($mets) = @_;

    return unless scalar @{$mets};
    my $json = encode_json({ records => $mets });
    warn "Sending ", scalar(@{$mets}), " records to kafka\n" if $ENV{'DEBUG_METRIC'};
    @{$mets} = ();

    if ($dry_run) {
        $internal_metrics->{'host_post_dry_run'}++;
        return;
    }

    my $resp = $ht->request( POST => $url, {
        content => $json,
        headers => {
            'Content-Type' => 'application/vnd.kafka.binary.v2+json',
            Accept         => 'application/vnd.kafka.v2+json',
        },
    });

    if ($resp->{'success'}) {
        $internal_metrics->{'host_post_success'}++;
        say 'SUCCESS';
    } else {
        warn "Failed sending data to $url (".($resp->{'code'} // 'undef').")" if $ENV{'DEBUG_METRIC'};
        say 'ERROR';
        $internal_metrics->{'host_post_error'}++;
    }
}


