diff --git a/slurm_jobs/chebi_release_process.sh b/slurm_jobs/chebi_release_process.sh index 1b5e23b92f72d42b13059a1e4e26e5f21bd5d6de..2577f0bdf3564869c5c70a6aafbdecbd4cecbc22 100644 --- a/slurm_jobs/chebi_release_process.sh +++ b/slurm_jobs/chebi_release_process.sh @@ -16,6 +16,7 @@ module purge # General variables RELEASE_VERSION=<RELEASE_VERSION> PREVIOUS_RELEASE="$((${RELEASE_VERSION} - 1))" +FTP=<FTP> # Variables used by chebi-ontology-generator container. TMP_ONTOLOGY_FOLDER=$(pwd)/ontology_files # This is the folder where the chebi-ontology-generator image will save the generated ontology files, it will be used as a volume by Singularity. @@ -30,15 +31,15 @@ TMP_SQL_DUMPS_FOLDER=$(pwd)/generic_dumps TMP_SDF_DUMPS_FOLDER=$(pwd)/SDF #FTP related variables: Where the different files will be sent once they have been created in the above temporal folders -FTP_ONTOLOGY_FOLDER=<FTP>/ontology #This is the folder where the last version of the ontology is saved in the FTP -FTP_SQL_DUMPS_FOLDER=<FTP>/generic_dumps # This is the folder where the last version of the SQL dumps are saved in the FTP -FTP_SDF_DUMPS_FOLDER=<FTP>/SDF -FTP_ARCHIVE_FOLDER=<FTP>/archive # This folder is where all the past releases are saved. +FTP_ONTOLOGY_FOLDER=${FTP}/ontology #This is the folder where the last version of the ontology is saved in the FTP +FTP_SQL_DUMPS_FOLDER=${FTP}/generic_dumps # This is the folder where the last version of the SQL dumps are saved in the FTP +FTP_SDF_DUMPS_FOLDER=${FTP}/SDF +FTP_ARCHIVE_FOLDER=${FTP}/archive # This folder is where all the past releases are saved. OWNER='<REMOTE_USER>' GROUP='chebi' -# 1. Archive the current version of the release. It means, save in: <FTP>/archive all the data published the past month. +# 1. Archive the current version of the release. It means, save in: ${FTP}/archive all the data published the past month. function archive_previous_version() { # Call this function with the root/to/folder/ you want to archive as an argument. # This is useful for archiving folders already published on the public FTP. @@ -67,11 +68,11 @@ function execute_chebi_ontology_generator() { --env ROBOT_JAVA_ARGS=-Xmx4G chebi_ontology_generator.sif --release-version=${RELEASE_VERSION} } -# 3. Apart from the ontology, we need to generate Dump files which are: +# 3. Apart from the ontology, we need to generate Dump files which are: SQL Dumps, SDF files and TSV flat files. # We're utilizing an ELT framework called dbt (https://docs.getdbt.com), which dynamically creates configuration folders during execution. # However, since files within a singularity container are read-only, new folders/files can't be created without assigning a volume. # To address this, we need to create three "dbt" folders to serve as "volume bind mounts" in the singularity container. -# Additionally, we require folders to store Luigi checkpoint files and the SQL dumps themselves. +# Additionally, we require folders to store Luigi checkpoint files and the dumps themselves. function execute_chebi_dump_generator() { mkdir -p ${TMP_DBT_LOGS_FOLDER} ${TMP_DBT_TARGET_FOLDER} ${TMP_DBT_PACKAGES_FOLDER} ${TMP_LUIGI_CHECKPOINTS} ${TMP_SQL_DUMPS_FOLDER} ${TMP_SDF_DUMPS_FOLDER} @@ -102,7 +103,7 @@ function publish_files() { --exclude="README.md" ${local_folder_to_publish}/ ${ftp_destination_folder} } - +# 5. With all the functions defined above, we create two workflows functions: First one to the ontology, second one to the dumps files. function ontology_publication_workflow() { # Ontology release steps: # 1. Execute the ontology generator to get the .owl, .obo and .json files in the temporally folder. @@ -137,7 +138,7 @@ function run_monthly_release() { function run_daily_update() { # Ontology is updated daily in a different path. This folder is to update the ontology nightly. - FTP_ONTOLOGY_FOLDER=<FTP>/ontology/nightly + FTP_ONTOLOGY_FOLDER=${FTP}/ontology/nightly # In this case we don't archive anything, for the above we can't call ontology_publication_workflow function here. execute_chebi_ontology_generator && publish_files ${TMP_ONTOLOGY_FOLDER} ${FTP_ONTOLOGY_FOLDER}