#!/usr/bin/env python
import sys
import os

from subprocess import Popen, PIPE, STDOUT
from optparse import OptionParser
from decimal import Decimal
import operator
import time
import shutil
import glob
import math

import re

def parse_options():
    usage = "usage: %prog [options] --strace=\"/tmp/.strace.1316015714.25620.hgtest\""
    version = "\n"+"-"*80+"\nAnalizador v0.4 by Joshua Holmes\nMaintainers:\n\tKevin Schottie\n"+"-"*80+"\n"
    parser = OptionParser( usage=usage, version=version)
    parser.add_option("--strace",
                        action="store", dest="strace", default=False,
                        help="Strace to review, individual file")
    parser.add_option("--straceregex",
                        action="store", dest="straceregex", default=False,
                        help="Straces to review, filename regex, i.e. '/tmp/.strace.*.USER' ")
    (options, args) = parser.parse_args()
    if options.straceregex and options.strace:
        print "Please specify only one evaluation method."
        sys.exit(1)
    return options, args

class cls_strace_mysql:
    def __init__(self):
        self.data = list()
        self._queryStore = dict()
        self._queryTitleTime = 'time'
        self._queryTitleSize = 'size'
        self._queryTitleQuery = 'query'
        self.write_regobj = re.compile(r'(\s+)(\d+.\d+)(\swrite\(\d+,\s\"[A-Za-z0-9\\]+)(SELECT.*|UPDATE.*|INSERT.*)(\",\s)(\d+)(\)\s=\s)(\d+)(\s\<)(\d+.\d+)(\>)', re.I)
        self.read_regobj = re.compile(r'(\s+)(\d+.\d+)(\sread\(\d+.*)(,\s)(\d+)(\)\s=\s)(\d+)(\s<)(\d+.\d+)(>)')
        self._query_length = 130
    def handler(self, linenum, writeline, readline):
        self.linenum = linenum
        self.writeline = writeline
        self.readline = readline
        self.write_call()
        self.read_call()
        self.query_time = float(self.read_groups[1]) - float(self.write_groups[9])
        self.query_size = float(int(self.read_groups[6]) / 1024.0)
        self.query = self.write_groups[3][:self._query_length]
        self._queryStore = {
                        self._queryTitleTime: self.query_time,
                        self._queryTitleSize : self.query_size,
                        self._queryTitleQuery: self.query
                      }
        self.data.append( self._queryStore )
        #print "TIME: %4.6f \tSIZE: %10.4f kB \tQUERY: \" %s \" " % (self.query_time,self.query_size,self.query)
    def write_call(self):
        self.write_groups = self.write_regobj.match(self.writeline).groups()
    def read_call(self):
        try:
            self.read_groups = self.read_regobj.match(self.readline).groups()
        except:
            pass

class cls_top_signals:
    def __init__(self):
        self.data = list()
    def handler(self, fileContent):
        for linenum in range(len(fileContent)):
            try:
                self.data.append(( float(fileContent[linenum+1].split()[0]), fileContent[linenum]) )
            except:
                pass
    def report(self, length=None):
        if length is None: length = 20
        print "\n" + '='*80 + "\nTop %d Signals by Execution Time.\n" % length + '='*80
        print " Time \t\tSignal\n" + '-'*80
        for i in sorted(self.data, reverse=True)[:length]:
            print "%4.8f - %s" % ( i[0], ' '.join(i[1].split()[1:])[:100] )
        print "\n"

def median(values):
    length = len(values)
    if length%2:
        median_indeces = [length/2]
    else:
        median_indeces = [length/2-1, length/2]
    values = sorted(values)
    return sum([values[i] for i in median_indeces]) / len(median_indeces)

class MVSD(object):
    """ A class that calculates a running Mean / Variance / Standard Deviation"""
    def __init__(self):
        self.is_started = False
        self.ss = Decimal(0) # (running) sum of square deviations from mean
        self.m = Decimal(0) # (running) mean
        self.total_w = Decimal(0) # weight of items seen
    def add(self, x, w=1):
        """ add another datapoint to the Mean / Variance / Standard Deviation"""
        if not isinstance(x, Decimal):
            x = Decimal(str(x))
        if not self.is_started:
            self.m = x
            self.ss = Decimal(0)
            self.total_w = w
            self.is_started = True
        else:
            temp_w = self.total_w + w
            self.ss += (self.total_w * w * (x - self.m) * (x - self.m )) / temp_w
            self.m += (x - self.m) / temp_w
            self.total_w = temp_w
        # print "added %-2d mean=%0.2f var=%0.2f std=%0.2f" % (x, self.mean(), self.var(), self.sd())
    def var(self):
        return self.ss / self.total_w
    def sd(self):
        return math.sqrt(self.var())
    def mean(self):
        return self.m

def histogram(data):
    bucket_scale = 1
    #min_v = min(data)
    min_v = 0.001
    #max_v = max(data)
    max_v = max(data)
    buckets = 20
    diff = max_v - min_v
    step = diff / buckets
    bucket_counts = [0 for x in range(buckets)]
    boundaries = []
    for x in range(buckets):
        boundaries.append(min_v + (step * (x + 1)))
    skipped = 0
    samples = 0
    mvsd = MVSD()
    accepted_data = []
    for value in data:
        samples +=1
        mvsd.add(value)
        accepted_data.append(value)
        # find the bucket this goes in
        if value < min_v or value > max_v:
            skipped +=1
            continue
        for bucket_postion, boundary in enumerate(boundaries):
            if value <= boundary:
                bucket_counts[bucket_postion] +=1
                break
    # auto-pick the hash scale
    if max(bucket_counts) > 75:
        bucket_scale = int(max(bucket_counts) / 75)
    print "# Signals under %0.4f are automatically discarded from profile." % min_v
    print "# NumSamples = %d; Min = %0.4f; Max = %0.4f" % (samples, min_v, max_v)
    if skipped:
        print "# %d value%s outside of min/max" % (skipped, skipped > 1 and 's' or '')
    print "# Mean = %f; Variance = %f; StdDev = %f; Median %f" % (mvsd.mean(), mvsd.var(), mvsd.sd(), median(accepted_data))
    print "# each * represents a count of %d" % bucket_scale
    bucket_min = min_v
    bucket_max = min_v
    for bucket in range(buckets):
        bucket_min = bucket_max
        bucket_max = boundaries[bucket]
        bucket_count = bucket_counts[bucket]
        star_count = 0
        if bucket_count:
            star_count = bucket_count / bucket_scale
        print '%10.4f - %10.4f [%6d]: %s' % (bucket_min, bucket_max, bucket_count, '*' * star_count)

class cls_timer:
    def __init__(self):
        self.run_time = 0
        self.init_time = 0
        self.init_files = list()
        self.cwd = os.getcwd()
        self.stream = list()
        self.operdir = "open(\""+os.getcwd()
    def handler(self, fileContents):
        self.init_status = 1
        for line in fileContents:
            self.run_time += float(line.split()[0])
            self.stream.append(float(line.split()[0]))
            if self.init_status == 1:
                self.init_time += float(line.split()[0])
                if self.operdir in line:
                    self.init_status = 2
            elif self.init_status == 2:
                try:
                    if self.operdir in line:
                        try:
                            self.init_files.append( line.split('"')[1] )
                        except:
                            pass
                        self.init_status = 0
                except:
                    pass
    def get_initFiles(self):
        i = dict()
        for file in self.init_files:
            i[file] = i.has_key(file) and (i[file] + 1) or 1
        return sorted(i.iteritems(), key=operator.itemgetter(1), reverse=True)
    def report(self):
        print '='*80 + "\nSoftware Initialization Report\n" + '='*80
        print "List of initial file (PHP) called per HTTP request:"
        for k , v in self.get_initFiles():
            print "Count: %02d \tFile: %s" % ( v , k )
        print "\n" + """Sum of time spent initializing the processes: %2.6f seconds.
Percent of initialization to complete execution time: %2.2f%%
Total execution time for all sessions: %3.6f""" % (
                                self.init_time ,
                                self.init_time / self.run_time * 100 ,
                                self.run_time
                                )
        print "\n" + "="*80 + "\nExecution Profile\n" + "="*80
        histogram(self.stream)

class cls_curlCapture:
    def __init__(self):
        self.data = list()
        self.HTTP_methods = ['GET', 'POST', 'DELETE', 'HEAD', 'CONNECT']
    def handler(self, line):
        for method in self.HTTP_methods:
            if method in line:
                self.data.append(' '.join((line.strip("\n")).split()[1:]))
    def report(self):
        if self.data:
            print "\n" + '='*80 + "\nRemote Requests Report.\n" + '='*80
            for item in self.data: print item
            print "\n"

class cls_strace_process:
    def __init__(self, files):
        self.logfiles = list()
        self.logfileContent = list()
        self.mysqlStrace = cls_strace_mysql()
        self.topSignals = cls_top_signals()
        self.timer = cls_timer()
        self.curlCapture = cls_curlCapture()
        self.strace_func_list = list()
        self.strace_func_count = dict()
        self.strace_func_time = dict()
        self.logfiles = files
        self.openFiles = list()
        self.handler()
    def handler(self):
        #self.logfiles = files
        for self.filename in self.logfiles: self.process_log()
    def process_log(self):
        self.getlog()
        self.topSignals.handler(self.logfileContent)
        self.timer.handler(self.logfileContent)
        for linenum in range(len(self.logfileContent)):
            if options.strace:
                if " open" in self.logfileContent[linenum]:
                    if "public_html" in self.logfileContent[linenum]:
                        self.openFiles.append(self.logfileContent[linenum].split("\"")[1])
            if self.mysqlStrace.write_regobj.match(self.logfileContent[linenum]):
                self.mysqlStrace.handler(linenum, self.logfileContent[linenum], self.logfileContent[linenum+1])
            func = self.logfileContent[linenum].split()[1].split("(")[0]
            if 'send' == func:
                self.curlCapture.handler(self.logfileContent[linenum])
            if func not in self.strace_func_list:
                self.strace_func_list.append(func)
                self.strace_func_count[func] = 1
                try:
                    self.strace_func_time[func] = float(self.logfileContent[linenum+1].split()[0])
                except:
                    pass
            elif func in self.strace_func_list:
                self.strace_func_count[func] += 1
                try:
                    self.strace_func_time[func] += float(self.logfileContent[linenum+1].split()[0])
                except:
                    pass
    def getlog(self):
        file = open(self.filename)
        self.logfileContent = file.readlines()
        file.close()
    def report(self):
        self.strace_func_count_top = sorted(self.strace_func_count.iteritems(), key=operator.itemgetter(1), reverse=True)[:20]
        self.strace_func_time_top = sorted(self.strace_func_time.iteritems(), key=operator.itemgetter(1), reverse=True)[:20]
        if self.mysqlStrace.data:
            print "="*80 + "\n--== MySQL Activities ==--\n" + "="*80
            print " Time\t\tSize\t\tQuery"
            for i in sorted(self.mysqlStrace.data, key=lambda k: k['time'], reverse=True)[:20]: print " %4.6f \t %6.3f \t \" %s \" " % ( float(i['time']), float(i['size']), i['query'] )
        self.topSignals.report()
        self.timer.report()
        self.curlCapture.report()
        if options.strace:
            self.open_file_flow()
        print "\n\n" + "="*80 + "\nTop 20 System Calls/Singals by Counts & Time\n" + "="*80
        print "\n%20s%10s%20s%10s" % ( "Function", "Count", "Function", "Time" )
        for i in range(20): print "%20s%10s%20s%10.4f" % ( self.strace_func_count_top[i][0], self.strace_func_count_top[i][1], self.strace_func_time_top[i][0], self.strace_func_time_top[i][1] )
        print "\n\n"
    def open_file_flow(self):
        olddir = None
        print "\n" + "="*80 + "\nSoftware File Flow Report\n" + "="*80
        for name in uniqList(self.openFiles):
            dirname, fname = os.path.split(name)
            if olddir != dirname:
                prefix = ' ' * (len(dirname) +1) + "-/"
                olddir = dirname
                print(name)
            else:
                print(prefix + fname)

def uniqList(data):
    # order preserving
    checked = []
    for e in data:
        if e not in checked:
            checked.append(e)
    return checked

def inject_monitor(user):
    monitor_lines = """
######################################
#####  HostGator '.htaccess' Mod  ####
######################################
######################################
Action application/x-hg-php-debug /cgi-sys/php-debug
AddType application/x-hg-php-debug .php
######################################
######################################
"""
    file = "/home/"+user+"/.htaccess"
    original_content = list()
    try:
        readFile = open(file)
        original_content = readFile.readlines()
        readFile.close()
    except:
        pass
    w = open(file,'w')
    w.writelines(monitor_lines)
    w.writelines([item for item in original_content])
    w.close()
    return original_content

def eject_monitor(user, original_content):
    file = "/home/"+user+"/.htaccess"
    w = open(file,'w')
    w.writelines([item for item in original_content])
    w.close()

def get_docroots(user):
    """Returns the document roots for the given user in a list"""
    return [line[17:].strip('\r\n\t ') for line in open('/usr/local/apache/conf/httpd.conf', 'r').readlines() if line.find('DocumentRoot') != -1 and re.search(r'/home\d*/'+user+'/', line) != None]

def getstracefiles():
    return glob.glob(".strace.*")

def moveLogFiles(logfiles, user):
    print "\nProcess log files stored in:"
    for file in logfiles:
        dest = "/tmp/"+file+"."+user
        print dest ,
        shutil.move(file, dest)
	os.chmod(dest, 0400)

def session_monitor():
    filelist = list()
    print """
    HTTP Session monitoring has begun.
    Please browse the site of interest at this time.
    Once log files have been generated press Ctrl+C to proceed with analysis.
    """
    while True:
        try:
            logfiles = getstracefiles()
            for file in logfiles:
                if file not in filelist:
                    filelist.append(file)
                    sys.stdout.write("\rSessions Captured: %d" % len(filelist) )
                    sys.stdout.flush()
            time.sleep(0.5)
        except KeyboardInterrupt:
            print '\nYou pressed Ctrl+C! Attempting to process session logs now.\n'
            eject_monitor(user, htaccess_original_content)
            if not filelist:
                print "\nNo Sessions Captured.\n"
                sys.exit()
            time.sleep(3)
            print "="*80 + "\n\n"
            break

if __name__ == '__main__':
    options, parser = parse_options()
    if options.strace:
        logfiles = [ options.strace ]
    elif options.straceregex:
        logfiles = [ x for x in glob.glob( options.straceregex )]
    else:
        try:
            user = os.getcwd().split("/")[2]
        except:
            print "Error retrieving cwd. Are you in a user's home directory?"
            sys.exit()
        if not os.getcwd() in get_docroots(user):
            print "!! Error retrieving docRoot. Are you in a user's docRoot directory?"
            sys.exit()
        htaccess_original_content = inject_monitor(user)
        session_monitor()
        logfiles = getstracefiles()
    processObject = cls_strace_process(logfiles)
    processObject.report()
    if not options.strace or options.straceregex:
        moveLogFiles(logfiles, user)


