HivePipeline.pm 40.2 KB
Newer Older
1 2 3
=head1 LICENSE

Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
nwillhoft's avatar
nwillhoft committed
4
Copyright [2016-2021] EMBL-European Bioinformatics Institute
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

=cut

20 21 22 23 24
package Bio::EnsEMBL::Hive::HivePipeline;

use strict;
use warnings;

25
use Bio::EnsEMBL::Hive::TheApiary;
26
use Bio::EnsEMBL::Hive::DBSQL::DBAdaptor;
27
use Bio::EnsEMBL::Hive::Utils ('stringify', 'destringify', 'throw');
28
use Bio::EnsEMBL::Hive::Utils::Collection;
29
use Bio::EnsEMBL::Hive::Utils::PCL;
30
use Bio::EnsEMBL::Hive::Utils::URL;
31

32 33 34 35
    # needed for offline graph generation:
use Bio::EnsEMBL::Hive::Accumulator;
use Bio::EnsEMBL::Hive::NakedTable;

36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55
use constant {
    TWEAK_ERROR_MSG => {
        PARSE_ERROR  => 'Tweak cannot be parsed',
        ACTION_ERROR => 'Action is not supported',
        FIELD_ERROR  => 'Field not recognized',
        VALUE_ERROR  => 'Invalid value',
    },
    TWEAK_ACTION => {
        '=' => 'SET',
        '+' => 'SET',
        '?' => 'SHOW',
        '#' => 'DELETE',
    },
    TWEAK_OBJECT_TYPE => {
        PIPELINE       => 'Pipeline',
        ANALYSIS       => 'Analysis',
        RESOURCE_CLASS => 'Resource class',
    },
};

56 57 58 59 60 61

sub hive_dba {      # The adaptor for HivePipeline objects
    my $self = shift @_;

    if(@_) {
        $self->{'_hive_dba'} = shift @_;
62
        $self->{'_hive_dba'}->hive_pipeline($self) if $self->{'_hive_dba'};
63 64 65 66 67
    }
    return $self->{'_hive_dba'};
}


68 69 70 71
sub display_name {
    my $self = shift @_;

    if(my $dbc = $self->hive_dba && $self->hive_dba->dbc) {
72
        return $dbc->dbname . '@' .($dbc->host||'');
73 74 75 76 77 78
    } else {
        return '(unstored '.$self->hive_pipeline_name.')';
    }
}


79 80 81 82 83 84 85 86 87 88 89
sub unambig_key {   # based on DBC's URL if present, otherwise on pipeline_name
    my $self = shift @_;

    if(my $dbc = $self->hive_dba && $self->hive_dba->dbc) {
        return Bio::EnsEMBL::Hive::Utils::URL::hash_to_unambig_url( $dbc->to_url_hash );
    } else {
        return 'unstored:'.$self->hive_pipeline_name;
    }
}


90 91 92 93
sub collection_of {
    my $self = shift @_;
    my $type = shift @_;

94 95 96
    if (@_) {
        $self->{'_cache_by_class'}->{$type} = shift @_;
    } elsif (not $self->{'_cache_by_class'}->{$type}) {
97

98
        if( (my $hive_dba = $self->hive_dba) and ($type ne 'NakedTable') and ($type ne 'Accumulator') and ($type ne 'Job') and ($type ne 'AnalysisJob')) {
99 100 101 102 103 104
            my $adaptor = $hive_dba->get_adaptor( $type );
            my $all_objects = $adaptor->fetch_all();
            if(@$all_objects and UNIVERSAL::can($all_objects->[0], 'hive_pipeline') ) {
                $_->hive_pipeline($self) for @$all_objects;
            }
            $self->{'_cache_by_class'}->{$type} = Bio::EnsEMBL::Hive::Utils::Collection->new( $all_objects );
105
#            warn "initialized collection_of($type) by loading all ".scalar(@$all_objects)."\n";
106 107
        } else {
            $self->{'_cache_by_class'}->{$type} = Bio::EnsEMBL::Hive::Utils::Collection->new();
108
#            warn "initialized collection_of($type) as an empty one\n";
109
        }
110
    }
111

112
    return $self->{'_cache_by_class'}->{$type};
113 114 115
}


116 117 118
sub find_by_query {
    my $self            = shift @_;
    my $query_params    = shift @_;
119
    my $no_die          = shift @_;
120

121 122
    if(my $object_type = delete $query_params->{'object_type'}) {
        my $object;
123

124
        if($object_type eq 'Accumulator' or $object_type eq 'NakedTable') {
125

126
            unless($object = $self->collection_of($object_type)->find_one_by( %$query_params )) {
127

128
                my @specific_adaptor_params = ($object_type eq 'NakedTable')
129 130 131 132 133
                    ? ('table_name' => $query_params->{'table_name'},
                        $query_params->{'insertion_method'}
                            ? ('insertion_method' => $query_params->{'insertion_method'})
                            : ()
                      )
134
                    : ();
135 136 137 138 139
                ($object) = $self->add_new_or_update( $object_type, # NB: add_new_or_update returns a list
                    %$query_params,
                    $self->hive_dba ? ('adaptor' => $self->hive_dba->get_adaptor($object_type, @specific_adaptor_params)) : (),
                );
            }
140 141 142
        } elsif($object_type eq 'AnalysisJob' or $object_type eq 'Semaphore') {
            my $id_name = { 'AnalysisJob' => 'job_id', 'Semaphore' => 'semaphore_id' }->{$object_type};
            my $dbID    = $query_params->{$id_name};
143 144 145 146 147 148 149 150
            my $coll    = $self->collection_of($object_type);
            unless($object = $coll->find_one_by( 'dbID' => $dbID )) {

                my $adaptor = $self->hive_dba->get_adaptor( $object_type );
                if( $object = $adaptor->fetch_by_dbID( $dbID ) ) {
                    $coll->add( $object );
                }
            }
151 152
        } else {
            $object = $self->collection_of($object_type)->find_one_by( %$query_params );
153 154
        }

155 156
        return $object if $object || $no_die;
        throw("Could not find an '$object_type' object from query ".stringify($query_params)." in ".$self->display_name);
157 158

    } else {
159
        throw("Could not find or guess the object_type from the query ".stringify($query_params)." , so could not find the object");
160 161 162
    }
}

163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190
sub test_connections {
    my $self = shift;

    my @warnings;

    foreach my $dft ($self->collection_of('DataflowTarget')->list) {
        my $analysis_url = $dft->to_analysis_url;
        if ($analysis_url =~ m{^\w+$}) {
            my $heir_analysis = $self->collection_of('Analysis')->find_one_by('logic_name', $analysis_url)
                or push @warnings, "Could not find a local analysis named '$analysis_url' (dataflow from analysis '".($dft->source_dataflow_rule->from_analysis->logic_name)."')";
        }
    }

    foreach my $cf ($self->collection_of('AnalysisCtrlRule')->list) {
        my $analysis_url = $cf->condition_analysis_url;
        if ($analysis_url =~ m{^\w+$}) {
            my $heir_analysis = $self->collection_of('Analysis')->find_one_by('logic_name', $analysis_url)
                or push @warnings, "Could not find a local analysis named '$analysis_url' (control-flow for analysis '".($cf->ctrled_analysis->logic_name)."')";
        }

    }

    if (@warnings) {
        push @warnings, '', 'Please fix these before running the pipeline';
        warn join("\n", '', '# ' . '-' x 26 . '[WARNINGS]' . '-' x 26, '', @warnings), "\n";
    }
}

191

192
sub new {       # construct an attached or a detached Pipeline object
193
    my $class           = shift @_;
194 195 196

    my $self = bless {}, $class;

197
    my %dba_flags           = @_;
198
    my $existing_dba        = delete $dba_flags{'-dba'};
199 200 201 202

    if(%dba_flags) {
        my $hive_dba    = Bio::EnsEMBL::Hive::DBSQL::DBAdaptor->new( %dba_flags );
        $self->hive_dba( $hive_dba );
203 204
    } elsif ($existing_dba) {
        $self->hive_dba( $existing_dba );
205
    } else {
206
#       warn "Created a standalone pipeline";
207 208
    }

209 210
    Bio::EnsEMBL::Hive::TheApiary->pipelines_collection->add( $self );

211 212 213 214
    return $self;
}


215 216
    # If there is a DBAdaptor, collection_of() will fetch a collection on demand:
sub invalidate_collections {
217 218
    my $self = shift @_;

219 220
    delete $self->{'_cache_by_class'};
    return;
221 222 223 224 225 226 227 228
}


sub save_collections {
    my $self = shift @_;

    my $hive_dba = $self->hive_dba();

229 230 231
    my @adaptor_types = ('MetaParameters', 'PipelineWideParameters', 'ResourceClass', 'ResourceDescription', 'Analysis', 'AnalysisStats', 'AnalysisCtrlRule', 'DataflowRule', 'DataflowTarget');

    foreach my $AdaptorType (reverse @adaptor_types) {
232
        my $adaptor = $hive_dba->get_adaptor( $AdaptorType );
233 234 235 236 237 238 239 240
        my $coll    = $self->collection_of( $AdaptorType );
        if( my $dark_collection = $coll->dark_collection) {
            foreach my $obj_to_be_deleted ( $coll->dark_collection->list ) {
                $adaptor->remove( $obj_to_be_deleted );
#                warn "Deleted ".(UNIVERSAL::can($obj_to_be_deleted, 'toString') ? $obj_to_be_deleted->toString : stringify($obj_to_be_deleted))."\n";
            }
            $coll->dark_collection( undef );
        }
241 242
    }

243 244 245 246 247 248 249 250 251 252
    foreach my $AdaptorType (@adaptor_types) {
        my $adaptor = $hive_dba->get_adaptor( $AdaptorType );
        my $class   = 'Bio::EnsEMBL::Hive::'.$AdaptorType;
        my $coll    = $self->collection_of( $AdaptorType );
        foreach my $storable_object ( $coll->list ) {
            $adaptor->store_or_update_one( $storable_object, $class->unikey() );
#            warn "Stored/updated ".$storable_object->toString()."\n";
        }
    }

253
    my $job_adaptor = $hive_dba->get_AnalysisJobAdaptor;
254
    foreach my $analysis ( $self->collection_of( 'Analysis' )->list ) {
255 256
        if(my $our_jobs = $analysis->jobs_collection ) {
            $job_adaptor->store( $our_jobs );
257
#            foreach my $job (@$our_jobs) {
258
#                warn "Stored ".$job->toString()."\n";
259
#            }
260 261 262 263 264 265 266 267 268
        }
    }
}


sub add_new_or_update {
    my $self = shift @_;
    my $type = shift @_;

269 270 271
    # $verbose is an extra optional argument that sits between the type and the object hash
    my $verbose = scalar(@_) % 2 ? shift : 0;

272 273
    my $class   = 'Bio::EnsEMBL::Hive::'.$type;
    my $coll    = $self->collection_of( $type );
274 275

    my $object;
276
    my $newly_made = 0;
277 278 279 280 281 282

    if( my $unikey_keys = $class->unikey() ) {
        my %other_pairs = @_;
        my %unikey_pairs;
        @unikey_pairs{ @$unikey_keys} = delete @other_pairs{ @$unikey_keys };

283
        if( $object = $coll->find_one_by( %unikey_pairs ) ) {
284
            my $found_display = $verbose && (UNIVERSAL::can($object, 'toString') ? $object->toString : stringify($object));
285
            if(keys %other_pairs) {
286
                print "Updating $found_display with (".stringify(\%other_pairs).")\n" if $verbose;
287 288 289 290 291 292 293 294
                if( ref($object) eq 'HASH' ) {
                    @$object{ keys %other_pairs } = values %other_pairs;
                } else {
                    while( my ($key, $value) = each %other_pairs ) {
                        $object->$key($value);
                    }
                }
            } else {
295
                print "Found a matching $found_display\n" if $verbose;
296
            }
297 298 299
        } elsif( my $dark_coll = $coll->dark_collection) {
            if( my $shadow_object = $dark_coll->find_one_by( %unikey_pairs ) ) {
                $dark_coll->forget( $shadow_object );
300
                my $found_display = $verbose && (UNIVERSAL::can($shadow_object, 'toString') ? $shadow_object->toString : stringify($shadow_object));
301
                print "Undeleting $found_display\n" if $verbose;
302
            }
303 304 305 306 307 308 309
        }
    } else {
        warn "$class doesn't redefine unikey(), so unique objects cannot be identified";
    }

    unless( $object ) {
        $object = $class->can('new') ? $class->new( @_ ) : { @_ };
310
        $newly_made = 1;
311

312
        $coll->add( $object );
313

314
        $object->hive_pipeline($self) if UNIVERSAL::can($object, 'hive_pipeline');
315

316
        my $found_display = $verbose && (UNIVERSAL::can($object, 'toString') ? $object->toString : 'naked entry '.stringify($object));
317
        print "Created a new $found_display\n" if $verbose;
318 319
    }

320
    return ($object, $newly_made);
321 322 323
}


324 325 326 327 328 329 330 331 332
=head2 get_source_analyses

    Description: returns a listref of analyses that do not have local inflow ("source analyses")

=cut

sub get_source_analyses {
    my $self = shift @_;

333
    my %analyses_to_discard = map {scalar($_->to_analysis) => 1} $self->collection_of( 'DataflowTarget' )->list;
334

335
    return [grep {!$analyses_to_discard{"$_"}} $self->collection_of( 'Analysis' )->list];
336 337 338
}


339
=head2 _meta_value_by_key
340

341
    Description: getter/setter for a particular meta_value from 'MetaParameters' collection given meta_key
342 343 344

=cut

345
sub _meta_value_by_key {
346 347
    my $self    = shift @_;
    my $meta_key= shift @_;
348

349
    my $hash = $self->collection_of( 'MetaParameters' )->find_one_by( 'meta_key', $meta_key );
350 351 352 353 354 355 356 357 358 359 360 361 362 363

    if(@_) {
        my $new_value = shift @_;

        if($hash) {
            $hash->{'meta_value'} = $new_value;
        } else {
            ($hash) = $self->add_new_or_update( 'MetaParameters',
                'meta_key'      => $meta_key,
                'meta_value'    => $new_value,
            );
        }
    }

364
    return $hash && $hash->{'meta_value'};
365 366 367
}


368 369
=head2 hive_use_param_stack

370
    Description: getter/setter via MetaParameters. Defines which one of two modes of parameter propagation is used in this pipeline
371 372 373 374 375 376

=cut

sub hive_use_param_stack {
    my $self = shift @_;

377
    return $self->_meta_value_by_key('hive_use_param_stack', @_) // 0;
378 379 380
}


381 382
=head2 hive_pipeline_name

383
    Description: getter/setter via MetaParameters. Defines the symbolic name of the pipeline.
384 385 386 387 388 389

=cut

sub hive_pipeline_name {
    my $self = shift @_;

390
    return $self->_meta_value_by_key('hive_pipeline_name', @_) // '';
391 392 393
}


394 395 396 397 398 399 400 401 402
=head2 hive_auto_rebalance_semaphores

    Description: getter/setter via MetaParameters. Defines whether beekeeper should attempt to rebalance semaphores on each iteration.

=cut

sub hive_auto_rebalance_semaphores {
    my $self = shift @_;

403
    return $self->_meta_value_by_key('hive_auto_rebalance_semaphores', @_) // '0';
404 405 406
}


407 408 409 410 411 412 413 414 415 416 417 418 419
=head2 hive_use_triggers

    Description: getter via MetaParameters. Defines whether SQL triggers are used to automatically update AnalysisStats counters

=cut

sub hive_use_triggers {
    my $self = shift @_;

    if(@_) {
        throw('HivePipeline::hive_use_triggers is not settable, it is only a getter');
    }

420
    return $self->_meta_value_by_key('hive_use_triggers') // '0';
421 422
}

423 424 425 426 427 428 429 430 431 432 433 434
=head2 hive_default_max_retry_count

    Description: getter/setter via MetaParameters. Defines the default value for analysis_base.max_retry_count

=cut

sub hive_default_max_retry_count {
    my $self = shift @_;

    return $self->_meta_value_by_key('hive_default_max_retry_count', @_) // 0;
}

435

436 437 438 439 440 441 442 443 444 445 446 447 448
=head2 list_all_hive_tables

    Description: getter via MetaParameters. Lists the (MySQL) table names used by the HivePipeline

=cut

sub list_all_hive_tables {
    my $self = shift @_;

    if(@_) {
        throw('HivePipeline::list_all_hive_tables is not settable, it is only a getter');
    }

449
    return [ split /,/, ($self->_meta_value_by_key('hive_all_base_tables') // '') ];
450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465
}


=head2 list_all_hive_views

    Description: getter via MetaParameters. Lists the (MySQL) view names used by the HivePipeline

=cut

sub list_all_hive_views {
    my $self = shift @_;

    if(@_) {
        throw('HivePipeline::list_all_hive_views is not settable, it is only a getter');
    }

466
    return [ split /,/, ($self->_meta_value_by_key('hive_all_views') // '') ];
467 468 469
}


470 471 472 473 474 475 476 477 478 479 480 481 482
=head2 hive_sql_schema_version

    Description: getter via MetaParameters. Defines the Hive SQL schema version of the database if it has been stored

=cut

sub hive_sql_schema_version {
    my $self = shift @_;

    if(@_) {
        throw('HivePipeline::hive_sql_schema_version is not settable, it is only a getter');
    }

483
    return $self->_meta_value_by_key('hive_sql_schema_version') // 'N/A';
484 485 486
}


487 488 489 490 491 492 493 494 495 496 497 498 499 500
=head2 params_as_hash

    Description: returns the destringified contents of the 'PipelineWideParameters' collection as a hash

=cut

sub params_as_hash {
    my $self = shift @_;

    my $collection = $self->collection_of( 'PipelineWideParameters' );
    return { map { $_->{'param_name'} => destringify($_->{'param_value'}) } $collection->list() };
}


501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533
=head2 get_cached_hive_current_load

    Description: Proxy for RoleAdaptor::get_hive_current_load() that caches the last value.

=cut

sub get_cached_hive_current_load {
    my $self = shift @_;

    if (not exists $self->{'_cached_hive_load'}) {
        if ($self->hive_dba) {
            $self->{'_cached_hive_load'} = $self->hive_dba->get_RoleAdaptor->get_hive_current_load();
        } else {
            $self->{'_cached_hive_load'} = 0;
        }
    }
    return $self->{'_cached_hive_load'};
}


=head2 invalidate_hive_current_load

    Description: Method that forces the next get_cached_hive_current_load() call to fetch a fresh value from the database

=cut

sub invalidate_hive_current_load {
    my $self = shift @_;

    delete $self->{'_cached_hive_load'};
}


534 535 536 537 538 539
=head2 print_diagram

    Description: prints a "Unicode art" textual representation of the pipeline's flow diagram

=cut

540 541 542 543 544
sub print_diagram {
    my $self = shift @_;

    print ''.(''x20).'[ '.$self->display_name.' ]'.(''x20)."\n";

545
    my %seen = ();
546 547
    foreach my $source_analysis ( @{ $self->get_source_analyses } ) {
        print "\n";
548
        $source_analysis->print_diagram_node($self, '', \%seen);
549
    }
550 551 552 553 554
    foreach my $cyclic_analysis ( $self->collection_of( 'Analysis' )->list ) {
        next if $seen{$cyclic_analysis};
        print "\n";
        $cyclic_analysis->print_diagram_node($self, '', \%seen);
    }
555 556 557
}


558 559
=head2 apply_tweaks

560
    Description: changes attributes of Analyses|ResourceClasses|ResourceDescriptions or values of pipeline/analysis parameters
561 562 563

=cut

564 565 566
sub apply_tweaks {
    my $self    = shift @_;
    my $tweaks  = shift @_;
Mira's avatar
Mira committed
567 568
    my @response;
    my $responseStructure;
569
    my $need_write = 0;
Mira's avatar
Mira committed
570
    $responseStructure->{Tweaks} = [];
571
    foreach my $tweak (@$tweaks) {
Mira's avatar
Mira committed
572
        push @response, "\nTweak.Request\t$tweak\n";
573

574
        if($tweak=~/^pipeline\.param\[(\w+)\](\?|#|=(.+))$/) {
575
            my ($param_name, $operator, $new_value_str) = ($1, $2, $3);
576 577
            my $pwp_collection  = $self->collection_of( 'PipelineWideParameters' );
            my $hash_pair       = $pwp_collection->find_one_by('param_name', $param_name);
Mira's avatar
Mira committed
578 579
            my $value           = $hash_pair ? $hash_pair->{'param_value'} : undef;
            my $tweakStructure;
580 581
            $tweakStructure->{Action} = TWEAK_ACTION->{substr($operator, 0, 1)};
            $tweakStructure->{Object}->{Type} =  TWEAK_OBJECT_TYPE->{PIPELINE};
Mira's avatar
Mira committed
582 583 584 585
            $tweakStructure->{Object}->{Id} = undef;
            $tweakStructure->{Object}->{Name} = undef;
            $tweakStructure->{Return}->{Field} = $param_name;
            $tweakStructure->{Return}->{OldValue} = $value;
586

587
            if($operator eq '?') {
Mira's avatar
Mira committed
588 589
                $tweakStructure->{Return}->{NewValue} = $value;
                push @response, "Tweak.Show    \tpipeline.param[$param_name] ::\t"
590
                     . ($hash_pair ? $hash_pair->{'param_value'} : '(missing_value)') . "\n";
591
            } elsif($operator eq '#') {
Mira's avatar
Mira committed
592
                $tweakStructure->{Return}->{NewValue} = undef;
593 594 595
                if ($hash_pair) {
                    $need_write = 1;
                    $pwp_collection->forget_and_mark_for_deletion( $hash_pair );
Mira's avatar
Mira committed
596
                    push @response, "Tweak.Deleting\tpipeline.param[$param_name] ::\t".stringify($hash_pair->{'param_value'})." --> (missing value)\n";
597
                } else {
Mira's avatar
Mira committed
598
                    push @response, "Tweak.Deleting\tpipeline.param[$param_name] skipped (does not exist)\n";
599
                }
600
            } else {
601
                $need_write = 1;
602
                my $new_value = destringify( $new_value_str );
603
                $new_value_str = stringify($new_value);
Mira's avatar
Mira committed
604
                $tweakStructure->{Return}->{NewValue} = $new_value_str;
605
                if($hash_pair) {
Mira's avatar
Mira committed
606
                    push @response, "Tweak.Changing\tpipeline.param[$param_name] ::\t$hash_pair->{'param_value'} --> $new_value_str\n";
607

608
                    $hash_pair->{'param_value'} = $new_value_str;
609
                } else {
Mira's avatar
Mira committed
610
                    push @response, "Tweak.Adding  \tpipeline.param[$param_name] ::\t(missing value) --> $new_value_str\n";
611 612
                    $self->add_new_or_update( 'PipelineWideParameters',
                        'param_name'    => $param_name,
613
                        'param_value'   => $new_value_str,
614
                    );
615
                }
616
            }
Mira's avatar
Mira committed
617
            push @{$responseStructure->{Tweaks}}, $tweakStructure;
618

619
        } elsif($tweak=~/^pipeline\.(\w+)(\?|=(.+))$/) {
620
            my ($attrib_name, $operator, $new_value_str) = ($1, $2, $3);
Mira's avatar
Mira committed
621
            my $tweakStructure;
622
            $tweakStructure->{Object}->{Type} = TWEAK_OBJECT_TYPE->{PIPELINE};
Mira's avatar
Mira committed
623 624 625
            $tweakStructure->{Object}->{Id} = undef;
            $tweakStructure->{Object}->{Name} = undef;
            $tweakStructure->{Return}->{Field} = $attrib_name;
626
            $tweakStructure->{Action} = TWEAK_ACTION->{substr($operator, 0, 1)};
627

628 629
            if($self->can($attrib_name)) {
                my $old_value = stringify( $self->$attrib_name() );
Mira's avatar
Mira committed
630
                $tweakStructure->{Return}->{OldValue} = $old_value;
631
                if($operator eq '?') {
Mira's avatar
Mira committed
632 633
                    $tweakStructure->{Return}->{NewValue} = $old_value;
                    push @response, "Tweak.Show    \tpipeline.$attrib_name ::\t$old_value\n";
634
                } else {
Mira's avatar
Mira committed
635 636
                    $tweakStructure->{Return}->{NewValue} = $new_value_str;
                    push @response, "Tweak.Changing\tpipeline.$attrib_name ::\t$old_value --> $new_value_str\n";
637 638

                    $self->$attrib_name( $new_value_str );
639
                    $need_write = 1;
640
                }
641 642

            } else {
643
                $tweakStructure->{Error} = TWEAK_ERROR_MSG->{FIELD_ERROR};
Mira's avatar
Mira committed
644
                push @response, "Tweak.Error   \tCould not find the pipeline-wide '$attrib_name' method\n";
645
            }
Mira's avatar
Mira committed
646
            push @{$responseStructure->{Tweaks}}, $tweakStructure;
647 648
        } elsif($tweak=~/^analysis\[([^\]]+)\]\.param\[(\w+)\](\?|#|=(.+))$/) {
            my ($analyses_pattern, $param_name, $operator, $new_value_str) = ($1, $2, $3, $4);
649
            my $analyses = $self->collection_of( 'Analysis' )->find_all_by_pattern( $analyses_pattern );
Mira's avatar
Mira committed
650
            push @response, "Tweak.Found   \t".scalar(@$analyses)." analyses matching the pattern '$analyses_pattern'\n";
651 652

            my $new_value = destringify( $new_value_str );
653
            $new_value_str = stringify( $new_value );
654 655 656 657 658

            foreach my $analysis (@$analyses) {
                my $analysis_name = $analysis->logic_name;
                my $old_value = $analysis->parameters;
                my $param_hash  = destringify( $old_value );
Mira's avatar
Mira committed
659
                my $tweakStructure;
660 661
                $tweakStructure->{Object}->{Type} = TWEAK_OBJECT_TYPE->{ANALYSIS};
                $tweakStructure->{Action} = TWEAK_ACTION->{substr($operator, 0, 1)};
662
                $tweakStructure->{Object}->{Id} = defined $analysis->dbID ? $analysis->dbID + 0 : undef;
Mira's avatar
Mira committed
663 664 665
                $tweakStructure->{Object}->{Name} = $analysis_name;
                $tweakStructure->{Return}->{Field} = $param_name;
                $tweakStructure->{Return}->{OldValue} =  exists($param_hash->{ $param_name }) ? stringify($param_hash->{ $param_name }) : undef;
666 667

                if($operator eq '?') {
Mira's avatar
Mira committed
668 669
                    $tweakStructure->{Return}->{NewValue} = $tweakStructure->{Return}->{OldValue};
                    push @response, "Tweak.Show    \tanalysis[$analysis_name].param[$param_name] ::\t"
670 671 672
                        . (exists($param_hash->{ $param_name }) ? stringify($param_hash->{ $param_name }) : '(missing value)')
                        ."\n";
                } elsif($operator eq '#') {
Mira's avatar
Mira committed
673 674
                    $tweakStructure->{Return}->{NewValue} = undef;
                    push @response, "Tweak.Deleting\tanalysis[$analysis_name].param[$param_name] ::\t".stringify($param_hash->{ $param_name })." --> (missing value)\n";
675 676 677

                    delete $param_hash->{ $param_name };
                    $analysis->parameters( stringify($param_hash) );
678
                    $need_write = 1;
679
                } else {
Mira's avatar
Mira committed
680
                    $tweakStructure->{Return}->{NewValue} = $new_value_str;
681
                    if(exists($param_hash->{ $param_name })) {
Mira's avatar
Mira committed
682
                        push @response, "Tweak.Changing\tanalysis[$analysis_name].param[$param_name] ::\t".stringify($param_hash->{ $param_name })." --> $new_value_str\n";
683
                    } else {
Mira's avatar
Mira committed
684
                        push @response, "Tweak.Adding  \tanalysis[$analysis_name].param[$param_name] ::\t(missing value) --> $new_value_str\n";
685
                    }
686

687 688
                    $param_hash->{ $param_name } = $new_value;
                    $analysis->parameters( stringify($param_hash) );
689
                    $need_write = 1;
690
                }
Mira's avatar
Mira committed
691
                push @{$responseStructure->{Tweaks}}, $tweakStructure;
692 693
            }

Mira's avatar
Mira committed
694

695
        } elsif($tweak=~/^analysis\[([^\]]+)\]\.(wait_for|flow_into)(\?|#|\+?=(.+))$/) {
Mira's avatar
Mira committed
696

697 698 699
            my ($analyses_pattern, $attrib_name, $operation, $new_value_str) = ($1, $2, $3, $4);
            $operation=~/^(\?|#|\+?=)/;
            my $operator = $1;
700

701
            my $analyses = $self->collection_of( 'Analysis' )->find_all_by_pattern( $analyses_pattern );
Mira's avatar
Mira committed
702
            push @response, "Tweak.Found   \t".scalar(@$analyses)." analyses matching the pattern '$analyses_pattern'\n";
703 704 705

            my $new_value = destringify( $new_value_str );

706
            foreach my $analysis (@$analyses) {
707
                my $analysis_name = $analysis->logic_name;
Mira's avatar
Mira committed
708
                my $tweakStructure;
709 710
                $tweakStructure->{Object}->{Type} = TWEAK_OBJECT_TYPE->{ANALYSIS};
                $tweakStructure->{Action} = TWEAK_ACTION->{substr($operator, 0, 1)};
711
                $tweakStructure->{Object}->{Id} = defined $analysis->dbID ? $analysis->dbID + 0 : undef;
Mira's avatar
Mira committed
712 713
                $tweakStructure->{Object}->{Name} = $analysis_name;
                $tweakStructure->{Return}->{Field} = $attrib_name;
714 715 716
                if( $attrib_name eq 'wait_for' ) {
                    my $cr_collection   = $self->collection_of( 'AnalysisCtrlRule' );
                    my $acr_collection  = $analysis->control_rules_collection;
Mira's avatar
Mira committed
717
                    $tweakStructure->{Return}->{OldValue} = [map { $_->condition_analysis_url } @$acr_collection];
718
                    if($operator eq '?') {
Mira's avatar
Mira committed
719 720
                        $tweakStructure->{Return}->{NewValue} = $tweakStructure->{Return}->{OldValue};
                        push @response, "Tweak.Show    \tanalysis[$analysis_name].wait_for ::\t[".join(', ', map { $_->condition_analysis_url } @$acr_collection )."]\n";
721 722 723
                    }

                    if($operator eq '#' or $operator eq '=') {     # delete the existing rules
Mira's avatar
Mira committed
724
                        $tweakStructure->{Return}->{NewValue} = undef;
725 726
                        foreach my $c_rule ( @$acr_collection ) {
                            $cr_collection->forget_and_mark_for_deletion( $c_rule );
727
                            $need_write = 1;
728

Mira's avatar
Mira committed
729
                            push @response, "Tweak.Deleting\t".$c_rule->toString." --> (missing value)\n";
730
                        }
731 732 733
                    }

                    if($operator eq '=' or $operator eq '+=') {     # create new rules
734
                        Bio::EnsEMBL::Hive::Utils::PCL::parse_wait_for($self, $analysis, $new_value);
Mira's avatar
Mira committed
735 736 737
                        my $acr_collection  = $analysis->control_rules_collection;
                        foreach my $c_rule ( @$acr_collection ) {
                            push @response, "Tweak.Adding\t".$c_rule->toString."\n";
738
                        }
Mira's avatar
Mira committed
739 740
                        $tweakStructure->{Return}->{NewValue} = [map { $_->condition_analysis_url } @$acr_collection];

741
                        $need_write = 1;
742 743 744
                    }

                } elsif( $attrib_name eq 'flow_into' ) {
Mira's avatar
Mira committed
745
                    $tweakStructure->{Warning} = "Value can't be displayed";
746
                    if($operator eq '?') {
747
                        # FIXME: should not recurse
Mira's avatar
Mira committed
748
                        #$analysis->print_diagram_node($self, '', {}); TODO: refactor with formatter.pm
749 750 751
                    }

                    if($operator eq '#' or $operator eq '=') {     # delete the existing rules
752 753 754 755 756 757 758 759 760 761 762
                        my $dfr_collection = $self->collection_of( 'DataflowRule' );
                        my $dft_collection = $self->collection_of( 'DataflowTarget' );

                        foreach my $group ( @{$analysis->get_grouped_dataflow_rules} ) {
                            my ($funnel_dfr, $fan_dfrs, $funnel_df_targets) = @$group;

                            foreach my $df_rule (@$fan_dfrs, $funnel_dfr) {

                                foreach my $df_target ( @{$df_rule->get_my_targets} ) {
                                    $dft_collection->forget_and_mark_for_deletion( $df_target );

Mira's avatar
Mira committed
763
                                    push @response, "Tweak.Deleting\t".$df_target->toString." --> (missing value)\n";
764 765
                                }
                                $dfr_collection->forget_and_mark_for_deletion( $df_rule );
766
                                $need_write = 1;
767

Mira's avatar
Mira committed
768
                                push @response, "Tweak.Deleting\t".$df_rule->toString." --> (missing value)\n";
769 770
                            }
                        }
771 772 773
                    }

                    if($operator eq '=' or $operator eq '+=') {     # create new rules
774
                        $need_write = 1;
775 776
                        Bio::EnsEMBL::Hive::Utils::PCL::parse_flow_into($self, $analysis, $new_value );
                    }
777
                }
Mira's avatar
Mira committed
778
                push @{$responseStructure->{Tweaks}}, $tweakStructure;
779 780 781
            }

        } elsif($tweak=~/^analysis\[([^\]]+)\]\.(\w+)(\?|#|=(.+))$/) {
Mira's avatar
Mira committed
782

783 784 785
            my ($analyses_pattern, $attrib_name, $operator, $new_value_str) = ($1, $2, $3, $4);

            my $analyses = $self->collection_of( 'Analysis' )->find_all_by_pattern( $analyses_pattern );
Mira's avatar
Mira committed
786
            push @response, "Tweak.Found   \t".scalar(@$analyses)." analyses matching the pattern '$analyses_pattern'\n";
787 788 789 790 791 792

            my $new_value = destringify( $new_value_str );

            foreach my $analysis (@$analyses) {

                my $analysis_name = $analysis->logic_name;
Mira's avatar
Mira committed
793
                my $tweakStructure;
794
                $tweakStructure->{Object}->{Type} = TWEAK_OBJECT_TYPE->{ANALYSIS};
795
                $tweakStructure->{Object}->{Id} = defined $analysis->dbID ? $analysis->dbID + 0 : undef;
Mira's avatar
Mira committed
796
                $tweakStructure->{Object}->{Name} = $analysis_name;
797
                $tweakStructure->{Action} = TWEAK_ACTION->{substr($operator, 0, 1)};
Mira's avatar
Mira committed
798
                $tweakStructure->{Return}->{Field} = $attrib_name;
799
                if( $attrib_name eq 'resource_class' ) {
800
                    $tweakStructure->{Return}->{OldValue} = $analysis->resource_class->name;
801

802
                    if($operator eq '?') {
Mira's avatar
Mira committed
803
                        $tweakStructure->{Return}->{NewValue} = $tweakStructure->{Return}->{OldValue};
804 805
                        my $old_value = $analysis->resource_class;
                        push @response, "Tweak.Show    \tanalysis[$analysis_name].resource_class ::\t".$old_value->name."\n";
806
                    } elsif($operator eq '#') {
807
                        $tweakStructure->{Error} = TWEAK_ERROR_MSG->{ACTION_ERROR};
808
                        push @response, "Tweak.Error   \tDeleting of an Analysis' resource-class is not supported\n";
809
                    } else {
Mira's avatar
Mira committed
810
                        $tweakStructure->{Return}->{NewValue} = $new_value_str;
811 812
                        my $old_value = $analysis->resource_class;
                        push @response, "Tweak.Changing\tanalysis[$analysis_name].resource_class ::\t".$old_value->name." --> $new_value_str\n";
813

814 815
                        my $resource_class;
                        if($resource_class = $self->collection_of( 'ResourceClass' )->find_one_by( 'name', $new_value )) {
Mira's avatar
Mira committed
816
                            push @response, "Tweak.Found   \tresource_class[$new_value_str]\n";
817 818
                            $analysis->resource_class( $resource_class );
                            $need_write = 1;
819
                        } else {
820 821
                            $tweakStructure->{Error} = TWEAK_ERROR_MSG->{VALUE_ERROR};
                            push @response, "Tweak.Error    \t'$new_value_str' is not a known resource-class\n";
822
                        }
823
                    }
824

825 826
                } elsif( $attrib_name eq 'is_excluded' ) {
                    my $analysis_stats = $analysis->stats();
Mira's avatar
Mira committed
827
                    $tweakStructure->{Return}->{OldValue} = $analysis_stats->is_excluded();
828
                    if($operator eq '?') {
Mira's avatar
Mira committed
829 830
                        $tweakStructure->{Return}->{NewValue} = $tweakStructure->{Return}->{OldValue};
                        push @response, "Tweak.Show    \tanalysis[$analysis_name].is_excluded ::\t".$analysis_stats->is_excluded()."\n";
831
                    } elsif($operator eq '#') {
832
                        $tweakStructure->{Error} = TWEAK_ERROR_MSG->{ACTION_ERROR};
Mira's avatar
Mira committed
833
                        push @response, "Tweak.Error   \tDeleting of excluded status is not supported\n";
834
                    } else {
Mira's avatar
Mira committed
835
                        $tweakStructure->{Return}->{NewValue} = $new_value_str;
836
                        if(!($new_value =~ /^[01]$/)) {
837
                            $tweakStructure->{Error} = TWEAK_ERROR_MSG->{VALUE_ERROR};
Mira's avatar
Mira committed
838
                            push @response, "Tweak.Error    \tis_excluded can only be 0 (no) or 1 (yes)\n";
839
                        } elsif ($new_value == $analysis_stats->is_excluded()) {
Mira's avatar
Mira committed