Commit a9a7bcb8 authored by David Mendez's avatar David Mendez
Browse files

Status daemon: set last checked at and append output to log after checing lsf

parent 6e5e5afd
......@@ -138,23 +138,32 @@ def prepare_job_status_check_script(job_id, lsf_job_id):
# ----------------------------------------------------------------------------------------------------------------------
# Parsing status script output
# ----------------------------------------------------------------------------------------------------------------------
def get_status_script_output(script_path):
def get_status_script_output(job_id, checker_name, script_path):
"""
Runs the status script and returns a text with the output obtained, if there is an error raises an exception
:param job_id: id of the job, used to save the outputs to the check log
:param checker_name: name of the process that checks the job in lsf
:param script_path: path of the script
:return: the text output of stdout
"""
lsf_config = RUN_CONFIG.get('lsf_submission')
id_rsa_path = lsf_config['id_rsa_file']
run_command = f'{script_path} {id_rsa_path}'
print(f'Going to run job status script, command: {run_command}')
check_msg = f'Going to run job status script, command: {run_command}'
delayed_job_models.append_to_lsf_check_log(job_id, checker_name, check_msg)
status_check_process = subprocess.run(run_command.split(' '), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
print(f'Output: \n {status_check_process.stdout}')
print(f'Error: \n {status_check_process.stderr}')
check_msg = f'Output: \n {status_check_process.stdout}'
delayed_job_models.append_to_lsf_check_log(job_id, checker_name, check_msg)
check_msg = f'Error: \n {status_check_process.stderr}'
delayed_job_models.append_to_lsf_check_log(job_id, checker_name, check_msg)
return_code = status_check_process.returncode
print(f'script return code was: {return_code}')
check_msg = f'script return code was: {return_code}'
delayed_job_models.append_to_lsf_check_log(job_id, checker_name, check_msg)
delayed_job_models.set_last_lsf_check_status(job_id, return_code)
delayed_job_models.set_last_lsf_checked_at(job_id, datetime.datetime.utcnow())
if return_code not in [0, 255]:
......@@ -167,6 +176,9 @@ def get_status_script_output(script_path):
with open(status_error_path, 'wb') as status_err_file:
status_err_file.write(status_check_process.stderr)
check_msg = f'There was an error when running the job status script! Check the directory {script_path} for logs!'
delayed_job_models.append_to_lsf_check_log(job_id, checker_name, check_msg)
raise JobStatusDaemonError('There was an error when running the job status script! Please check the logs')
else:
return status_check_process.stdout.decode()
......
......@@ -491,6 +491,28 @@ def append_to_lsf_check_log(job_id, checker_name, check_log):
save_job(job)
def set_last_lsf_check_status(job_id, script_status):
"""
Sets the last status of the checking script
:param job_id: id of the job to modify
:param script_status: status code of the script
"""
job = get_job_by_id(job_id)
job.last_lsf_check_status = script_status
save_job(job)
def set_last_lsf_checked_at(job_id, last_lsf_checked_at):
"""
Sets the last time the job was checked in lsf
:param job_id: id of the job to modify
:param last_lsf_checked_at: last time when the job was checked
"""
job = get_job_by_id(job_id)
job.last_lsf_checked_at = last_lsf_checked_at
save_job(job)
def add_input_file_to_job(job, input_file):
"""
Adds an input file to a job and saves the job
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment