Create singularity docker dir in LSF if it does not exist

.gitlab-ci.yml

@@ -173,9 +173,9 @@ deploy_to_staging:
   - kubectl config use-context ${CHEMBL_NS_STAGING}
   - echo ${CHEMBL_NS_STAGING} ${KUBE_URL_HX} ${KUBE_CLUS_NAME_HX} ${KUBE_USER_HX}
   - kubectl get pods -n ${CHEMBL_NS_STAGING}
-  - kubectl create secret generic ${ID_RSA_SECRET_NAME_STAGING} --from-file=ID_RSA_LSF=${ID_RSA_LSF_HX} -o yaml -n ${CHEMBL_NS_STAGING} --dry-run | kubectl apply -f -
-  - kubectl create secret generic ${RUN_CONFIG_SECRET_NAME_STAGING} --from-file=RUN_CONFIG.yml=${CONFIG_STAGING_FILE_PATH} -o yaml -n ${CHEMBL_NS_STAGING} --dry-run | kubectl apply -f -
-  - kubectl create secret generic ${GUNICORN_CONFIG_SECRET_NAME_STAGING} --from-file=GUNICORN_CONFIG.py=${CONFIG_GUNICORN_STAGING_FILE_PATH} -o yaml -n ${CHEMBL_NS_STAGING} --dry-run | kubectl apply -f -
+  - kubectl create secret generic ${ID_RSA_SECRET_NAME_STAGING} --from-file=ID_RSA_LSF=${ID_RSA_LSF_HX} -o yaml -n ${CHEMBL_NS_STAGING} --dry-run | kubectl replace -f -
+  - kubectl create secret generic ${RUN_CONFIG_SECRET_NAME_STAGING} --from-file=RUN_CONFIG.yml=${CONFIG_STAGING_FILE_PATH} -o yaml -n ${CHEMBL_NS_STAGING} --dry-run | kubectl replace -f -
+  - kubectl create secret generic ${GUNICORN_CONFIG_SECRET_NAME_STAGING} --from-file=GUNICORN_CONFIG.py=${CONFIG_GUNICORN_STAGING_FILE_PATH} -o yaml -n ${CHEMBL_NS_STAGING} --dry-run | kubectl replace -f -
   - cat ${CONFIG_STAGING_FILE_PATH}
   - cat ${CONFIG_GUNICORN_STAGING_FILE_PATH}
   - sed -i "s~<NAMESPACE>~${CHEMBL_NS_STAGING}~" k8s-deployment.yaml
@@ -264,9 +264,9 @@ deploy_to_prod_hx:
     - kubectl config use-context ${CHEMBL_NS_PROD}
     - echo ${CHEMBL_NS_PROD} ${KUBE_URL_HX} ${KUBE_CLUS_NAME_HX} ${KUBE_USER_HX}
     - kubectl get pods -n ${CHEMBL_NS_PROD}
-    - kubectl create secret generic ${ID_RSA_SECRET_NAME_PROD_HX} --from-file=ID_RSA_LSF=${ID_RSA_LSF_HX} -o yaml -n ${CHEMBL_NS_PROD} --dry-run | kubectl replace -f -
-    - kubectl create secret generic ${RUN_CONFIG_SECRET_NAME_PROD_HX} --from-file=RUN_CONFIG.yml=${CONFIG_PROD_HX_FILE_PATH} -o yaml -n ${CHEMBL_NS_PROD} --dry-run | kubectl replace -f -
-    - kubectl create secret generic ${GUNICORN_CONFIG_SECRET_NAME_PROD_HX} --from-file=GUNICORN_CONFIG.py=${CONFIG_GUNICORN_PROD_HX_FILE_PATH} -o yaml -n ${CHEMBL_NS_PROD} --dry-run | kubectl replace -f -
+    - kubectl create secret generic ${ID_RSA_SECRET_NAME_PROD_HX} --from-file=ID_RSA_LSF=${ID_RSA_LSF_HX} -o yaml -n ${CHEMBL_NS_PROD} --dry-run | kubectl apply -f -
+    - kubectl create secret generic ${RUN_CONFIG_SECRET_NAME_PROD_HX} --from-file=RUN_CONFIG.yml=${CONFIG_PROD_HX_FILE_PATH} -o yaml -n ${CHEMBL_NS_PROD} --dry-run | kubectl apply -f -
+    - kubectl create secret generic ${GUNICORN_CONFIG_SECRET_NAME_PROD_HX} --from-file=GUNICORN_CONFIG.py=${CONFIG_GUNICORN_PROD_HX_FILE_PATH} -o yaml -n ${CHEMBL_NS_PROD} --dry-run | kubectl apply -f -
     - cat ${CONFIG_PROD_HX_FILE_PATH}
     - cat ${CONFIG_GUNICORN_PROD_HX_FILE_PATH}
     - sed -i "s~<NAMESPACE>~${CHEMBL_NS_PROD}~" k8s-deployment.yaml

app/blueprints/job_submission/services/job_submission_service.py

@@ -485,6 +485,7 @@ def prepare_job_submission_script(job):
             set_docker_registry_credentials = ''
 
         resources_params = get_job_resources_params(job)
+        docker_singularity_dir = RUN_CONFIG.get('docker_singularity_dir', 'some_dir')
 
         job_submission_script = submit_job_template.format(
             JOB_ID=job.id,
@@ -494,7 +495,8 @@ def prepare_job_submission_script(job):
             DOCKER_IMAGE_URL=job.docker_image_url,
             SET_DOCKER_REGISTRY_CREDENTIALS=set_docker_registry_credentials,
             RUN_DIR=get_job_run_dir(job),
-            RESOURCES_PARAMS=resources_params
+            RESOURCES_PARAMS=resources_params,
+            DOCKER_SINGULARITY_DIR=docker_singularity_dir
 
         )
         job.requirements_parameters_string = resources_params

templates/submit_job.sh

@@ -9,5 +9,5 @@ echo "I am going to submit the job {JOB_ID}"
 ssh {LSF_USER}@{LSF_HOST} -i $IDENTITY_FILE -oStrictHostKeyChecking=no <<ENDSSH
 {SET_DOCKER_REGISTRY_CREDENTIALS}
 export LSB_JOB_REPORT_MAIL=N
-bsub {RESOURCES_PARAMS} -J {JOB_ID} -o {RUN_DIR}/job_run.out -e {RUN_DIR}/job_run.err "singularity exec {DOCKER_IMAGE_URL} /app/run_job.sh {RUN_PARAMS_FILE}"
+bsub {RESOURCES_PARAMS} -J {JOB_ID} -o {RUN_DIR}/job_run.out -e {RUN_DIR}/job_run.err "mkdir -p {DOCKER_SINGULARITY_DIR};singularity exec {DOCKER_IMAGE_URL} /app/run_job.sh {RUN_PARAMS_FILE}"
 ENDSSH