#!/usr/bin/env python
"""
SuperQ, process queue throttler.

    This program checks a set of definitions against running processes
to see if there are too many instances of a particular process. If there are,
it freezes the remaining ones and queues them. Once the number of seconds
defined in INTERVAL has passed, it scans all of the processes, freezing all of
the new matching processes and placing them into purgatory, and then activates
any ones that have been waiting up until the quota set by a definition's
'instances' limit has been filled.

Definitions are stored in a json file in the same directory as superq.

    If the number of instances in purgatory is greater than
'max_instances' for a definition, the process will be killed. To disable this for
a process type, set its 'max_instances' to a false value, like null.

    The definitions are made by having a name for the type of process watched
for as the key (this can be whatever you like), and a dictionary containing a regex
pattern that the process name matches, the number of permitted running instances,
and the maximum number of instances that may be queued.

    One should be careful: if a pattern matches the child process of another
pattern, there could be a conflict. Also, regexes are error-prone, so test
first.

    SuperQ's logging level can also be set in the configuration file.
Acceptable levels are INFO, WARNING, and DEBUG. ERROR, or CRITICAL.

    SuperQ's configuration can be reloaded if sent the SIGUSR signal. The
init.d script that comes packed with SuperQ sends the signal upon reload.

    Purgatory is a dictionary. Keys are created from the names defined in the
definitions dictionary, and the values are lists containing the psutil process
objects that were harvested by harvest_jobs for that pattern.

    Paradise is also a dictionary. Keys are created from the names
defined in the definitions dictionary, and the values are lists containing the
psutil process objects that have been given the green light to execute.

    This is not an ultimate solution for process problems. It is a
sledgehammer solution for processes that spawn unpredictably and which cannot
be fixed in-solution, or which fixing 'in-solution' creates additional
complexity that this script would be preferable to.
"""

import errno
import json
import logging
import os
import psutil
import re
import signal
import sys
import time
import traceback

VERSION = 2.0
INTERVAL = 60

purgatory = {}
paradise = {}
base_dir = os.path.dirname(__file__)
pid_dir = '/var/run'
definitions_filename = base_dir + '/superq.json'
log_filename = base_dir + '/logs/superq.log'
definitions = {}
base_dir = os.path.dirname(__file__)
format = '[%(asctime)s %(levelname)s] %(message)s'

logger = logging.getLogger(__name__)

# Common exception class for psutils
NoSuchProcess = psutil.error.NoSuchProcess


def _handle_lock(set_lock):
    """
    Set or remove the execution lock for superq. True to set, False to clear.
    """
    lock_source = pid_dir + '/superq.pid'
    lock = pid_dir + '/superq_lock'
    if set_lock:
        try:
            if os.path.exists(lock_source):
                f = open(lock_source,'r')
                oldpid = f.read(4096)
                f.close()
                if len(oldpid) == 0:
                    px = psutil.process_iter()
                    for p in px:
                        if ' '.join(p.cmdline).find('superq') >= 0 and p.name.find('python') >= 0 and p.pid != os.getpid():
                            sys.stderr.write("Could not acquire lock at %s (legacy).  Halting execution.")
                            os._exit(1)
                elif os.path.exists('/proc/%s' % oldpid):
                    f = open('/proc/%s/cmdline' % oldpid)
                    cmd = f.read(4096)
                    f.close()
                    if cmd.find('superq') >= 0:
                        sys.stderr.write("Could not acquire lock at '%s' (%s). Halting execution.\n" % (lock,oldpid) )
                        os._exit(1)
            f = open(lock_source, mode='w+')
            f.write(str(os.getpid()))
            f.close()
            # Symlink creation is atomic, file creation/checking is not. ... not needed anymore
            #os.symlink(lock_source, lock)
        except OSError:
            sys.stderr.write("Could not acquire lock at '%s'. Halting execution.\n" % lock )
            os._exit(1)
    else:
        try:
            os.remove(lock_source)
            os.remove(lock)
        except OSError:
            logger.warning("Failed to remove lock at %s" % lock)


def _initial_setup():
    global logger
    # Let's set up the environment if it's not already set up.
    _handle_lock(True)
    try:
        os.makedirs(os.path.dirname(log_filename))
    except OSError as exception:
        if exception.errno != errno.EEXIST:
            raise
    log_file = logging.FileHandler(base_dir + '/logs/superq.log', mode='a+')
    format = logging.Formatter('[%(asctime)s %(levelname)s] %(message)s')
    log_file.setFormatter(format)
    logger.addHandler(log_file)


def load_definitions():
    """
    Load the definitions from the definitions file.
    """
    global definitions
    global INTERVAL
    logger.info("Loading definitions file...")
    
    try:
        file = open(definitions_filename)
    except IOError as e:
        logger.warning("Could not open definitions file: %s" % e)
        return
    try:
        defs_file = json.load(file)
        definitions = defs_file['definitions']
        INTERVAL  = defs_file['interval']
        log_level = getattr(logging, defs_file['log_level'])
        logger.setLevel(log_level)

    except (ValueError, KeyError) as e:
        logger.warning("Invalid JSON in definitions file: %s" % e)
        return
    logger.info("Definition files loaded.")
    return True

def get_tree_list(pid):
    """
    Get a PID and all of its children, recursively.
    """
    return_list = [pid]
    try:
        pid_list = pid.get_children()
        return_list += pid_list
    except NoSuchProcess:
        return []
    for child in pid_list:
        return_list += get_tree_list(child)
    return return_list

def toggle_job(pid, thaw):
    """
    Take a process and freeze or thaw its tree.
    """
    to_toggle = get_tree_list(pid)
    logger.debug(to_toggle)
    for child in to_toggle:
        try:
            if thaw:
                logger.debug("SIGCONT to " + repr(child))
                child.resume()
            else:
                logger.debug("SIGSTOP to " + repr(child))
                child.suspend()
        except NoSuchProcess:
            continue

def init_queues():
    """
    Set up purgatory with empty lists for each definition.
    """
    for key in definitions.keys():
        purgatory[key] = []
        paradise[key] = []

def harvest_jobs():
    """
        Find all jobs that match a pattern, freeze them and add them to
    purgatory
    """
    to_freeze = [ ]
    pids = psutil.process_iter()
    already_processed = [ ]
    # Get all jobs currently frozen to avoid dual freezing.
    for jobs in purgatory.values() + paradise.values():
        for job in jobs:
            try:
                already_processed.append(job.pid)
            except NoSuchProcess:
                continue

    for key, value in definitions.items():
        pattern = value['pattern']
        for pid in pids:
            try:
                if re.search(pattern, pid.name) and pid.pid not in already_processed:
                    if pid.uids.real == 0 and not re.search(pattern, psutil.Process(pid.ppid).name):
                        to_freeze.append((key, pid))
            except NoSuchProcess:
                continue
    for key, pid in to_freeze:
        try:
            logger.info('Freezing "%s" task pid %i and its children. Name of process is "%s"' % (key, pid.pid, pid.name))
            toggle_job(pid, False)
            # Not using append here, because we want a FIFO queue.
            purgatory[key] = [pid] + purgatory[key]
        except NoSuchProcess:
            continue

def clear_jobs(job_list):
    """
        Iterate through a list of psutil process objects and eliminate the ones
    that are no longer running. The modified list is returned.
    """
    new_job_list = [ ]
    for job in job_list:
        try:
            if job.status != psutil.STATUS_ZOMBIE:
                new_job_list.append(job)
        except NoSuchProcess:
            continue
    return new_job_list

def prune_jobs(to_prune, max_size, key):
    """
        If a true valued index marker is specified, returns a list slice from
    the beginning to the index marker specified. Otherwise, returns the list
    unmodified.

        The jobs which are pruned are killed.
    """
    if max_size:
        to_kill = to_prune[max_size:]
        current_size = len(to_prune)
        for process in to_kill:
            try:
                logger.info("Too many instances of %s! Maximium instances permitted is %i; currently have %i. Killing pid %i. Name: %s"
                    % (key, max_size, current_size, process.pid, process.name)
                )
                process.kill()
            except NoSuchProcess:
                continue
            current_size += -1
        return to_prune[:max_size]
    return to_prune

def cycle_jobs():
    """
        Clear out the queues for each job type, then pluck a number of
    processes for each queue and unfreeze them until we have the maximum number
    or no more left.
    """
    for key, value in definitions.items():
        paradise[key] = clear_jobs(paradise[key])
        logger.debug("Cycle Jobs Key: %s" % key)
        logger.debug("Cycle Jobs Value: %s" % (value,))
        logger.debug("Instances allowed: %i" % value['instances'])
        logger.debug("Paradise queue: %s" % paradise[key])
        to_queue = value['instances'] - len(paradise[key])
        logger.debug("to_queue: %s" % to_queue)
        logger.debug("Purgatory queue: %s" % purgatory[key])
        for i in range(to_queue):
           try:
               job = purgatory[key].pop()
               logger.info('Thawing "%s" task pid %i and its children. Name of process is "%s"' % (key, job.pid, job.name))
               toggle_job(job, True)
               paradise[key].append(job)
           except IndexError:
               # No jobs left.
               break
           except NoSuchProcess:
               continue
        paradise[key] = prune_jobs(paradise[key], value['max_instances'], key)

def main_loop():
    """
    The primary loop of the program. Runs until killed.
    """
    init_queues()
    while True:
        harvest_jobs()
        cycle_jobs()
        time.sleep(INTERVAL)

def _daemonize():
    """
    Turns the script into a proper *NIX daemon.
    """
    if (hasattr(os, "devnull")):
        REDIRECT_TO = os.devnull
    else:
       REDIRECT_TO = "/dev/null"

    pid = os.fork()

    # Create a new process session. May be overkill, but it's clean.
    if (pid == 0):
        os.setsid()
        pid = os.fork()
        if (pid != 0):
            os._exit(0)
    else:
        os._exit(0)

    os.open(REDIRECT_TO, os.O_RDWR)
    # Duplicate standard input to standard output and standard error.
    os.dup2(0, 1)                       # standard output (1)
    os.dup2(0, 2)                       # standard error (2)
    flk = open(pid_dir + '/superq.pid',"w")
    flk.write(str(os.getpid()))
    flk.close()

def thaw_all():
    """
    Thaws everything in purgatory.
    """
    for key, value in definitions.items():
        while purgatory[key]:
            toggle_job(purgatory[key].pop(), True)

def _reload_handler(signal, frame):
    global definitions
    logger.info("Received reload signal.")
    if not load_definitions():
        logger.info("Reload failed. Continuing with previous definitions and state.")
    else:
        logger.info("Reload succeeded. Thawing all jobs, clearing the queues, and waiting for next cycle...")
        thaw_all()
        purgatory = []
        paradise = []
        init_queues()

def _termination_handler(signal, frame):
    """
    Exit cleanly.
    """
    logger.info('Terminating. Thawing all processes before halting execution.')
    try:
        thaw_all()
    except Exception as e:
        logger.error(repr(e))
        logger.error("Some processes may have been left unterminated. Raw dictionary:")
        logger.error(repr(purgatory))
    _handle_lock(False)
    logger.info("Execution halted.")
    os._exit(0)

if __name__ == '__main__':
    """
    Only execute if we weren't imported. Let's become a daemon.
    """
    _initial_setup()
    logger.info("SuperQ version %0.2f starting." % VERSION)
    if not load_definitions():
        logger.critical("Definition load failure! Execution halted.")
        _handle_lock(False)
        os._exit(1)
    _daemonize()
    signal.signal(signal.SIGINT, _termination_handler)
    signal.signal(signal.SIGTERM, _termination_handler)
    signal.signal(signal.SIGUSR1, _reload_handler)
    logger.info("SuperQ version %0.2f starting." % VERSION)
    try:
        main_loop()
    except:
        logger.critical('\n' + traceback.format_exc() + '\nExecution halted.')
        _termination_handler(None, None)
