meeseeks_launcher.py 1.63 KB
Newer Older
1 2 3 4 5 6 7 8
"""
Module that launches the status daemon in a Mr Meeseeks way
"""
import os
import shlex
import subprocess

import app.app_logging as app_logging
9 10
from app.models import delayed_job_models
from app import utils
11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27


def summon_meeseeks_for_job(job_id):
    """
    starts the job checking process. It will disappear once the job is finished or errored.
    :param job_id: id of the job to monitor
    """

    app_logging.debug('Going to launch Mr Meeseeks to monitor the job status')
    meeseeks_command = f'/app/run_daemon.sh {job_id}'
    app_logging.debug(f'Mr Meeseeks command is: {meeseeks_command}')
    checker_env = {
        'CONFIG_FILE_PATH': os.getenv('CONFIG_FILE_PATH')
    }
    app_logging.debug(f'Mr Meeseeks env is: {checker_env}')
    meeseeks_process = subprocess.Popen(shlex.split(meeseeks_command), env=checker_env)
    app_logging.debug(f'Mr Meeseeks summoned! PID is {meeseeks_process.pid}')
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43


def summon_new_meeseeks_if_previous_died_prematurely(job):
    """
    If if is detected that the previous job checker died prematurely, a new one is summoned
    :param job:   job to monitor
    """
    if job.needs_to_be_checked_in_lsf() and job.job_checker_seems_to_have_died():
        msg = f'The job {job.id} needs to be checked in LSF and the previous checker seems to have died '
        f'previously. I will summon another Meeseeks.'
        app_logging.debug(msg)
        delayed_job_models.append_to_lsf_check_log(job.id, msg)
        summon_meeseeks_for_job(job.id)
        # I update the last checked at to avoid many summonings at the same time.
        job.last_lsf_checked_at = utils.get_utc_now()
        delayed_job_models.save_job(job)