@@ -134,5 +134,12 @@ sub print_active_role_counts {
}
sub fetch_all_finished_roles_with_unfinished_jobs{
my$self=shift;
return$self->fetch_all("JOIN job USING(role_id) WHERE when_finished IS NOT NULL AND status NOT IN ('DONE', 'READY', 'FAILED', 'PASSED_ON') GROUP BY role_id");
return$self->fetch_all("JOIN job j USING(worker_id) WHERE worker.status='DEAD' AND j.status NOT IN ('DONE', 'READY', 'FAILED', 'PASSED_ON') GROUP BY worker_id");
-- First remove the ForeignKeys from job.worker_id and job_file.worker_id:
ALTER TABLE job DROP FOREIGN KEY job_ibfk_4;
ALTER TABLE job_file DROP FOREIGN KEY job_file_ibfk_2;
-- Also remove Indices from the old columns:
ALTER TABLE job DROP INDEX worker_id;
ALTER TABLE job_file DROP INDEX worker_id;
-- Use the existing worker_id columns as reference to add role_id columns:
ALTER TABLE job ADD COLUMN role_id INTEGER DEFAULT NULL AFTER worker_id;
ALTER TABLE job_file ADD COLUMN role_id INTEGER DEFAULT NULL AFTER worker_id;
-- Pretend we had role entries from the very beginning (the data is very approximately correct!):
UPDATE job j, role r SET j.role_id=r.role_id WHERE r.worker_id=j.worker_id AND CASE WHEN completed IS NOT NULL THEN when_started<=completed AND (when_finished IS NULL OR completed<=when_finished) ELSE when_finished IS NULL END;
UPDATE job_file jf, job j SET jf.role_id=j.role_id WHERE jf.job_id=j.job_id;
-- Now we can drop the columns themselves:
ALTER TABLE job DROP COLUMN worker_id;
ALTER TABLE job_file DROP COLUMN worker_id;
-- Add new Indices:
ALTER TABLE job ADD INDEX role_status (role_id, status);
ALTER TABLE job_file ADD INDEX role (role_id);
-- Add ForeignKeys on the new columns:
ALTER TABLE job ADD FOREIGN KEY (role_id) REFERENCES role(role_id) ON DELETE CASCADE;
ALTER TABLE job_file ADD FOREIGN KEY (role_id) REFERENCES role(role_id) ON DELETE CASCADE;
-- UPDATE hive_sql_schema_version
UPDATE hive_meta SET meta_value=62 WHERE meta_key='hive_sql_schema_version' AND meta_value='61';
-- First remove the ForeignKeys from job.worker_id and job_file.worker_id:
ALTER TABLE job DROP CONSTRAINT job_worker_id_fkey;
ALTER TABLE job_file DROP CONSTRAINT job_file_worker_id_fkey;
-- Also remove Indices from the old columns:
DROP INDEX job_worker_id_status_idx;
DROP INDEX job_file_worker_id_idx;
-- Add role_id columns:
ALTER TABLE job ADD COLUMN role_id INTEGER DEFAULT NULL;
ALTER TABLE job_file ADD COLUMN role_id INTEGER DEFAULT NULL;
-- Pretend we had role entries from the very beginning (the data is very approximately correct!):
UPDATE job j set role_id = (SELECT r.role_id FROM role r WHERE r.worker_id=j.worker_id AND CASE WHEN completed IS NOT NULL THEN when_started<=completed AND (when_finished IS NULL OR completed<=when_finished) ELSE when_finished IS NULL END);
UPDATE job_file jf set role_id = (SELECT role_id FROM job j WHERE j.job_id=jf.job_id);
-- Now we can drop the columns themselves:
ALTER TABLE job DROP COLUMN worker_id;
ALTER TABLE job_file DROP COLUMN worker_id;
-- Add new Indices:
CREATE INDEX ON job (role_id, status);
CREATE INDEX ON job_file (role_id);
-- Add ForeignKeys on the new columns:
ALTER TABLE job ADD FOREIGN KEY (role_id) REFERENCES role(role_id) ON DELETE CASCADE;
ALTER TABLE job_file ADD FOREIGN KEY (role_id) REFERENCES role(role_id) ON DELETE CASCADE;
-- UPDATE hive_sql_schema_version
UPDATE hive_meta SET meta_value=62 WHERE meta_key='hive_sql_schema_version' AND meta_value='61';
@column input_id input data passed into Analysis:RunnableDB to control the work
@column param_id_stack a CSV of job_ids whose input_ids contribute to the stack of local variables for the job
@column accu_id_stack a CSV of job_ids whose accu's contribute to the stack of local variables for the job
@column worker_id link to worker table to define which worker claimed this job
@column role_id links to the Role that claimed this job (NULL means it has never been claimed)
@column status state the job is in
@column retry_count number times job had to be reset when worker failed to run it
@column completed when the job was completed
...
...
@@ -329,7 +329,7 @@ CREATE TABLE job (
input_id CHAR(255) NOT NULL,
param_id_stack CHAR(64) NOT NULL DEFAULT '',
accu_id_stack CHAR(64) NOT NULL DEFAULT '',
worker_id INTEGER DEFAULT NULL,
role_id INTEGER DEFAULT NULL,
status ENUM('SEMAPHORED','READY','CLAIMED','COMPILATION','PRE_CLEANUP','FETCH_INPUT','RUN','WRITE_OUTPUT','POST_CLEANUP','DONE','FAILED','PASSED_ON') DEFAULT 'READY' NOT NULL,
retry_count INTEGER NOT NULL DEFAULT 0,
completed TIMESTAMP NULL, -- mysql's special for "TIMESTAMP DEFAULT NULL"
@column input_id input data passed into Analysis:RunnableDB to control the work
@column param_id_stack a CSV of job_ids whose input_ids contribute to the stack of local variables for the job
@column accu_id_stack a CSV of job_ids whose accu's contribute to the stack of local variables for the job
@column worker_id link to worker table to define which worker claimed this job
@column role_id links to the Role that claimed this job (NULL means it has never been claimed)
@column status state the job is in
@column retry_count number times job had to be reset when worker failed to run it
@column completed when the job was completed
...
...
@@ -327,7 +327,7 @@ CREATE TABLE job (
input_id CHAR(255) NOT NULL,
param_id_stack CHAR(64) NOT NULL DEFAULT '',
accu_id_stack CHAR(64) NOT NULL DEFAULT '',
worker_id INTEGER DEFAULT NULL,
role_id INTEGER DEFAULT NULL,
status TEXT NOT NULL DEFAULT 'READY', /* enum('SEMAPHORED','READY','CLAIMED','COMPILATION','FETCH_INPUT','RUN','WRITE_OUTPUT','DONE','FAILED','PASSED_ON') DEFAULT 'READY' NOT NULL, */
retry_count INTEGER NOT NULL DEFAULT 0,
completed TIMESTAMP DEFAULT NULL,
...
...
@@ -339,7 +339,7 @@ CREATE TABLE job (
);
CREATE UNIQUE INDEX job_input_id_stacks_analysis_idx ON job (input_id, param_id_stack, accu_id_stack, analysis_id);
CREATE INDEX job_analysis_status_retry_idx ON job (analysis_id, status, retry_count);
CREATE INDEX job_worker_idx ON job (worker_id);
CREATE INDEX job_role_id_status_id_idx ON job (role_id);
/**
...
...
@@ -353,8 +353,8 @@ CREATE INDEX job_worker_idx ON job (worker_id);
There is max one entry per job_id and retry.
@column job_id foreign key
@column worker_id link to worker table to define which worker claimed this job
@column retry copy of retry_count of job as it was run
@column role_id links to the Role that claimed this job
@column stdout_file path to the job's STDOUT log
@column stderr_file path to the job's STDERR log
*/
...
...
@@ -362,13 +362,13 @@ CREATE INDEX job_worker_idx ON job (worker_id);
CREATE TABLE job_file (
job_id INTEGER NOT NULL,
retry INTEGER NOT NULL,
worker_id INTEGER NOT NULL,
role_id INTEGER NOT NULL,
stdout_file VARCHAR(255),
stderr_file VARCHAR(255),
PRIMARY KEY (job_id, retry)
);
CREATE INDEX job_file_worker_idx ON job_file (worker_id);
CREATE INDEX job_file_role_id_idx ON job_file (role_id);
