Commit 71fba523 authored by David Mendez's avatar David Mendez
Browse files

Implement determining when a job needs to be checked

parent a9a7bcb8
......@@ -9,14 +9,13 @@ from os import path
import shutil
import os
from sqlalchemy import and_
from app import create_app
from app.models import delayed_job_models
from app.config import RUN_CONFIG
from app.job_status_daemon import daemon
from app.blueprints.job_submission.services import job_submission_service
from app.job_status_daemon import locks
from app import utils
class TestJobStatusDaemon(unittest.TestCase):
......
......@@ -204,6 +204,13 @@ class DelayedJob(DB.Model):
"""
return len(self.executions)
def needs_to_be_checked_in_lsf(self):
"""
:return: True if the job needs to be checked in lsf, false otherwise
"""
needs_to_be_checked_by_status = self.status in [JobStatuses.QUEUED, JobStatuses.RUNNING, JobStatuses.UNKNOWN]
return needs_to_be_checked_by_status
# ----------------------------------------------------------------------------------------------------------------------
# Helper functions
......
"""
Tests related to the status of the job in LSF
"""
import unittest
import datetime
from app import create_app
from app.models import delayed_job_models
class TestJobLSFStatus(unittest.TestCase):
"""
Class to test the columns related to the job lsf status checking
"""
def setUp(self):
self.flask_app = create_app()
self.client = self.flask_app.test_client()
def tearDown(self):
with self.flask_app.app_context():
delayed_job_models.delete_all_jobs()
def test_determines_when_a_job_needs_to_be_checked(self):
"""
Test that determines when a job needs to be checked in LSF because of its status
and when it does noe need to be checked
"""
with self.flask_app.app_context():
job_type = 'SIMILARITY'
params = {
'search_type': 'SIMILARITY',
'structure': '[H]C1(CCCN1C(=N)N)CC1=NC(=NO1)C1C=CC(=CC=1)NC1=NC(=CS1)C1C=CC(Br)=CC=1',
'threshold': '70'
}
docker_image_url_must_be = 'some_url'
job = delayed_job_models.get_or_create(job_type, params, docker_image_url_must_be)
for status in [delayed_job_models.JobStatuses.QUEUED, delayed_job_models.JobStatuses.RUNNING,
delayed_job_models.JobStatuses.UNKNOWN]:
job.status = status
needs_to_be_checked_got = job.needs_to_be_checked_in_lsf()
self.assertTrue(needs_to_be_checked_got, msg=f'A job with status {status} need to be checked in LSF!')
for status in [delayed_job_models.JobStatuses.CREATED, delayed_job_models.JobStatuses.ERROR,
delayed_job_models.JobStatuses.FINISHED]:
job.status = status
needs_to_be_checked_got = job.needs_to_be_checked_in_lsf()
self.assertFalse(needs_to_be_checked_got,
msg=f'A job with status {status} does not need to be checked in LSF!')
def test_determines_when_the_job_checker_seems_to_have_died(self):
"""
Test that determines when a job needs to be checked in LSF because the previous script failed
"""
with self.flask_app.app_context():
job_type = 'SIMILARITY'
params = {
'search_type': 'SIMILARITY',
'structure': '[H]C1(CCCN1C(=N)N)CC1=NC(=NO1)C1C=CC(=CC=1)NC1=NC(=CS1)C1C=CC(Br)=CC=1',
'threshold': '70'
}
docker_image_url_must_be = 'some_url'
job = delayed_job_models.get_or_create(job_type, params, docker_image_url_must_be)
last_lsf_checked_at = datetime.datetime.utcnow() + datetime.timedelta(seconds=-3)
print('last_lsf_checked_at: ', last_lsf_checked_at)
# job.last_lsf_checked_at = 0
# the last checked at time must be older than the seconds at which it is considered dead
# for status in [delayed_job_models.JobStatuses.QUEUED, delayed_job_models.JobStatuses.RUNNING,
# delayed_job_models.JobStatuses.UNKNOWN]:
# job.status = status
# needs_to_be_checked_got = job.needs_to_be_checked_in_lsf()
# self.assertTrue(needs_to_be_checked_got, msg=f'A job with status {status} need to be checked in LSF!')
......@@ -3,10 +3,12 @@
"""
import shutil
import subprocess
import datetime
import app.app_logging as app_logging
from app.errors import DelayedJobsError
def delete_directory_robustly(dir_path):
"""
Deletes a directory in a more robust way, if it catches and exception tries other methods
......@@ -22,7 +24,7 @@ def delete_directory_robustly(dir_path):
delete_command = f'rm -rf {dir_path}'
deletion_process = subprocess.run(delete_command.split(' '), stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stderr=subprocess.PIPE)
return_code = deletion_process.returncode
app_logging.debug(f'deletion return code was: {return_code}')
......@@ -32,3 +34,10 @@ def delete_directory_robustly(dir_path):
app_logging.debug(f'Deletion stdout: \n {deletion_process.stdout}')
app_logging.debug(f'Deletion stderr: \n {deletion_process.stderr}')
def get_utc_now():
"""
:return: the UTC current time with the timezone info
"""
return datetime.utcnow().replace(tzinfo=datetime.timezone.utc)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment