Skip to content
Snippets Groups Projects
Commit 57c64720 authored by Benjamin Wingfield's avatar Benjamin Wingfield
Browse files

improve rendered template

parent 329fd577
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Cargo.lock
+ 89
− 3
View file @ 57c64720
......@@ -30,6 +30,21 @@ version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56fc6cf8dc8c4158eed8649f9b8b0ea1518eb62b544fe9490d66fa0b349eafe9"
[[package]]
name = "android-tzdata"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
[[package]]
name = "android_system_properties"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
dependencies = [
"libc",
]
[[package]]
name = "anstream"
version = "0.3.2"
......@@ -154,6 +169,21 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
version = "0.4.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5"
dependencies = [
"android-tzdata",
"iana-time-zone",
"js-sys",
"num-traits",
"time 0.1.45",
"wasm-bindgen",
"winapi",
]
[[package]]
name = "clap"
version = "4.3.4"
......@@ -202,6 +232,12 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
[[package]]
name = "core-foundation-sys"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
[[package]]
name = "encoding_rs"
version = "0.8.32"
......@@ -349,7 +385,7 @@ dependencies = [
"cfg-if",
"js-sys",
"libc",
"wasi",
"wasi 0.11.0+wasi-snapshot-preview1",
"wasm-bindgen",
]
......@@ -402,6 +438,7 @@ name = "hattivatti"
version = "0.1.0"
dependencies = [
"anyhow",
"chrono",
"clap",
"env_logger",
"jsonschema",
......@@ -498,6 +535,29 @@ dependencies = [
"want",
]
[[package]]
name = "iana-time-zone"
version = "0.1.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
"wasm-bindgen",
"windows",
]
[[package]]
name = "iana-time-zone-haiku"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
dependencies = [
"cc",
]
[[package]]
name = "idna"
version = "0.4.0"
......@@ -596,7 +656,7 @@ dependencies = [
"reqwest",
"serde",
"serde_json",
"time",
"time 0.3.22",
"url",
"uuid",
]
......@@ -671,7 +731,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2"
dependencies = [
"libc",
"wasi",
"wasi 0.11.0+wasi-snapshot-preview1",
"windows-sys",
]
......@@ -1042,6 +1102,17 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "time"
version = "0.1.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a"
dependencies = [
"libc",
"wasi 0.10.0+wasi-snapshot-preview1",
"winapi",
]
[[package]]
name = "time"
version = "0.3.22"
......@@ -1220,6 +1291,12 @@ dependencies = [
"try-lock",
]
[[package]]
name = "wasi"
version = "0.10.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
......@@ -1333,6 +1410,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-sys"
version = "0.48.0"
......
Cargo.toml
+ 1
− 0
View file @ 57c64720
......@@ -16,3 +16,4 @@ anyhow = "1.0.71"
clap = { version = "4.3.4", features = ["derive"] }
rusqlite = { version = "0.29.0", features = ["bundled"] }
tinytemplate = "1.2.1"
chrono = "0.4.26"
data/templates/env_vars.txt
+ 2
− 0
View file @ 57c64720
# these variables shouldn't be sensitive
export NXF_OPTS="-Xms500M -Xmx4G"
export NXF_ANSI_TERM=false
export NXF_EXECUTOR=local
......@@ -5,3 +6,4 @@ export NXF_WORK=$TMPDIR/work
export GLOBUS_BASE_URL={globus_base_url}
export GLOBUS_GUEST_COLLECTION_ID={guest_collection_id}
export JOB_MESSAGE="{message}"
data/templates/header.txt
+ 7
− 2
View file @ 57c64720
#!/usr/bin/env bash
# this script processes sensitive data (human genomes) to calculate polygenic
# risk scores with the Polygenic Score (PGS) Catalog Calculator (pgsc_calc). the
# script was automatically generated by hattivatti for the INTERVENE platform.
# created at {time_now}
#SBATCH --job-name={name}
#SBATCH --account=project_2004504
#SBATCH --partition=small
#SBATCH --time={time}
#SBATCH --time={job_time}
#SBATCH --gres=nvme:256
#SBATCH --exclusive
#SBATCH --mem=32G
# pass through secrets from hattivatti deployment
# pass through secrets from deployment environment
#SBATCH --export=AWS_ACCESS_KEY_ID,AWS_SECRET_ACCESS_KEY,GLOBUS_SECRET_TOKEN
data/templates/nxf.sh → data/templates/nxf.txt
+ 16
− 0
View file @ 57c64720
# step 0. load required modules
module load nextflow/22.10.1
# step 1: transfer data using HTTPS + curl
# (assume transfer.sh is on PATH)
transfer.sh {work_dir}
# step 2. run pgsc_calc!
nextflow run pgscatalog/pgsc_calc -profile test,singularity \
-c ${HATTIVATTI_WORK_DIR}/{name}/config \
-c ${HATTIVATTI_WORK_DIR}/allas.config \
--input ${HATTIVATTI_WORK_DIR}/{name}/input.json \
-c {work_dir}/{name}/config \
-c {work_dir}/{name}/allas.config \
--input {work_dir}/{name}/input.json \
--min_overlap 0.01 \
--max_cpus 40 \
--max_memory "32.GB" \
......
src/main.rs
+ 5
− 2
View file @ 57c64720
......@@ -23,6 +23,7 @@ mod slurm;
"This program reads job request messages from the INTERVENE backend and submits a sensitive data
processing task to the SLURM scheduler. The program also monitors the state of submitted jobs,
and notifies the INTERVENE backend when a requested job has succeeded or failed.")]
struct Args {
#[arg(short, long)]
schema_dir: PathBuf,
......@@ -34,12 +35,14 @@ struct Args {
work_dir: PathBuf
}
pub struct WorkingDirectory { path: PathBuf }
fn main() {
env_logger::init();
info!("terve! starting up :)");
let args = Args::parse();
let wd = WorkingDirectory { path: args.work_dir };
let conn: Connection = db::open::open_db(Path::new(&args.db_path))
.expect("Database connection");
......@@ -56,7 +59,7 @@ fn main() {
info!("{:?}", jobs);
for job in jobs {
let _ = create_job(job);
let _ = create_job(job, &wd);
}
......
src/slurm/job.rs
+ 43
− 15
View file @ 57c64720
......@@ -2,25 +2,33 @@ use std::fs::OpenOptions;
use std::io;
use std::io::Write;
use std::path::Path;
use log::info;
use tinytemplate::TinyTemplate;
use crate::slurm::job_request::{JobRequest, NxfParamsFile, PipelineParam};
use serde::Serialize;
use serde_json::Value;
use tinytemplate::error::Error;
use tinytemplate::TinyTemplate;
pub fn create_job(request: JobRequest) {
use chrono::Utc;
use crate::slurm::job_request::{JobRequest, NxfParamsFile, PipelineParam};
use crate::WorkingDirectory;
pub fn create_job(request: JobRequest, wd: &WorkingDirectory) {
info!("Creating job {}", &request.pipeline_param.id);
let header: Header = render_header(&request.pipeline_param);
let vars: EnvVars = get_environment_variables(&request);
let job = JobTemplate { header, vars };
let vars: EnvVars = render_environment_variables(&request);
let workflow: Workflow = render_nxf(&request.pipeline_param, &wd.path);
let job = JobTemplate { header, vars, workflow };
job.write(Path::new("/Users/bwingfield/Downloads/test.txt")).expect("out");
let _ = make_input_file(&request.pipeline_param.nxf_params_file);
}
struct JobTemplate {
header: Header,
vars: EnvVars,
workflow: Workflow,
}
impl JobTemplate {
......@@ -30,7 +38,7 @@ impl JobTemplate {
.append(true)
.open(out_path)?;
[self.header.content, self.vars.content].map(
[self.header.content, self.vars.content, self.workflow.content].map(
|str| {
file.write_all(str.as_bytes());
}
......@@ -41,26 +49,38 @@ impl JobTemplate {
}
struct Header {
content: String
content: String,
}
struct EnvVars {
content: String
content: String,
}
struct Workflow {
content: String,
}
#[derive(Serialize)]
struct HeaderContext {
name: String,
time: String
job_time: String,
time_now: String
}
#[derive(Serialize)]
struct EnvVarContext {
globus_base_url: String,
guest_collection_id: String,
message: String
message: String,
}
#[derive(Serialize)]
struct NextflowContext {
name: String,
work_dir: String,
}
fn render_header(param: &PipelineParam) -> Header {
static HEADER: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/data/templates/header.txt"));
let mut tt = TinyTemplate::new();
......@@ -69,16 +89,17 @@ fn render_header(param: &PipelineParam) -> Header {
let context = HeaderContext {
name: param.id.to_string(),
// (todo: run job for 1 hour)
time: "01:00:00".to_string()
job_time: "01:00:00".to_string(),
time_now: Utc::now().to_string()
};
Header { content: tt.render("header", &context).expect("Rendered document") }
}
fn get_environment_variables(request: &JobRequest) -> EnvVars {
fn render_environment_variables(request: &JobRequest) -> EnvVars {
static ENV_VARS: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/data/templates/env_vars.txt"));
let mut tt = TinyTemplate::new();
// html escape breaks JSON
// html escape breaks JSON
tt.set_default_formatter(&tinytemplate::format_unescaped);
tt.add_template("env_var", ENV_VARS).expect("Template");
......@@ -90,12 +111,19 @@ fn get_environment_variables(request: &JobRequest) -> EnvVars {
EnvVars { content: tt.render("env_var", &context).expect("Rendered document") }
}
fn render_nxf(param: &PipelineParam) {
fn render_nxf(param: &PipelineParam, work_dir: &Path) -> Workflow {
static NXF: &str = include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/data/templates/nxf.txt"));
let mut tt = TinyTemplate::new();
tt.add_template("nxf", NXF).expect("Template");
let name: &String = &param.id;
let wd = work_dir.to_str().expect("path").to_string();
let context = NextflowContext { name: name.clone(), work_dir: wd };
Workflow { content: tt.render("nxf", &context).expect("Rendered document") }
}
struct AllasConfig {
content: String
content: String,
}
fn allas_config() -> AllasConfig {
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment