Merge branch 'version/2.1' into version/2.2

* version/2.1:
  Should ignore psql from the coveralls statistics
  [test-suite] collection.t should also be executable
  [test-suite] added back LongMultWf_conf since it seems to work well
  [test-suite] The filename format for output and error logging has changed
  [test-suite] We can now rely on runWorker's return code
  [test-suite] Backported the test of the Collection object
  [test-suite] LSF gained get_report_entries_*() methods
  [test-suite] Parameters of the constructor now need to be method names (without an hyphen)
  [test-suite] Removed scripts that are now tests
  [test-suite] Don't overwrite en existing $EHIVE_ROOT_DIR
  [test-suite] More tests for LSF (copied from master)
  [test-suite] More tests for the AnalysisData adaptor
  [test-suite] don't change the current working-directory
  made most of the tests in t/ runnable as standalone scripts
  [test-suite] NakedTable adaptors are cached by table_name, so we can now check their unicity
  [test-suite] Ensembl Core API is not needed any more
  [test-suite] Updated the version numbers in the test
  Copied and adapted the test-suite from 2.3
  PostgreSQL doesn't know GROUP_CONCAT()

.travis.yml

+language: "perl"
+
+perl:
+  - "5.10"
+  - "5.14"
+
+env:
+  - COVERALLS=true
+  - COVERALLS=false
+
+sudo: false
+
+before_install:
+    - echo -e '#!/bin/bash\n\nperl '`which psql`' "$@"' > psql
+    - chmod +x psql
+    - export PATH=$PWD:$PATH
+    - ln -s /usr/share/perl5/PgCommon.pm modules/
+
+
+install:
+    - cpanm -v --installdeps --notest .
+    - cpanm -n Devel::Cover::Report::Coveralls
+
+script: "./scripts/travis_run_tests.sh"
+
+notifications:
+  email:
+    on_success: always
+    on_failure: always
+  slack:
+    secure: "fbxQ+yuUAKKI10MUmNeCUbLqmVoOHUhGdM2KetqT5DmSn2rHxb099040Ira2tBYmUEPb5QvIMfPZ1xgS27he4xsZIpvOLsd1Or9OOL8XqyeTLcZ/IG4rpky0ehltVXs3s4ClM+YXwCHIoR2FwVghLVn6znmRkYm3TreSDtEUC2o="
+
+# Get the matrix to only build coveralls support when on 5.10
+matrix:
+  exclude:
+    - perl: "5.10"
+      env: COVERALLS=false
+    - perl: "5.14"
+      env: COVERALLS=true
+

cpanfile

+requires 'DBI';
+requires 'DBD::mysql';
+requires 'DBD::SQLite';
+requires 'DBD::Pg';
+
+requires 'JSON';
+
+on 'test' => sub {
+	requires 'Capture::Tiny';
+	requires 'Test::Exception';
+	requires 'Test::More';
+}
+

modules/Bio/EnsEMBL/Hive/Utils/Test.pm

+=head1 LICENSE
+
+Copyright [1999-2016] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+=cut
+
+
+package Bio::EnsEMBL::Hive::Utils::Test;
+
+use strict;
+use warnings;
+no warnings qw( redefine );
+
+use Exporter;
+use Carp qw{croak};
+
+use Test::More;
+use Test::Exception;
+
+use Bio::EnsEMBL::Hive::Process;
+use Bio::EnsEMBL::Hive::AnalysisJob;
+use Bio::EnsEMBL::Hive::DBSQL::DBAdaptor;
+use Bio::EnsEMBL::Hive::Utils ('load_file_or_module', 'stringify', 'destringify');
+
+
+our @ISA         = qw(Exporter);
+our @EXPORT      = ();
+our %EXPORT_TAGS = ();
+our @EXPORT_OK   = qw( init_pipeline runWorker );
+
+our $VERSION = '0.00';
+
+
+sub init_pipeline {
+    my ($file_or_module, $pipeline_url, $options) = @_;
+
+    $options ||= [];
+
+    my $hive_dba;
+    local @ARGV = @$options;
+    local %Bio::EnsEMBL::Hive::Cacheable::cache_by_class;
+
+    lives_ok(sub {
+        my @cmd = ($ENV{'EHIVE_ROOT_DIR'}.'/scripts/init_pipeline.pl', $file_or_module, '-pipeline_url', $pipeline_url, @$options);
+        my $rc = system(@cmd);
+        ok(!$rc, 'pipeline initialized');
+        $hive_dba = Bio::EnsEMBL::Hive::DBSQL::DBAdaptor->new(-url => $pipeline_url);
+        ok($hive_dba, 'DBAdaptor created');
+    }, sprintf('init_pipeline("%s on %s", %s)', $file_or_module, $pipeline_url, stringify($options)));
+
+    return $hive_dba;
+}
+
+
+sub runWorker {
+    my ($hive_dba, $specialization_options, $life_options, $execution_options) = @_;
+
+    $specialization_options ||= {};
+    $life_options ||= {};
+    $execution_options ||= {};
+
+    lives_ok(sub {
+        my $pipeline_url = $hive_dba->dbc->url;
+        my @cmd = ($ENV{'EHIVE_ROOT_DIR'}.'/scripts/runWorker.pl', '-url', $pipeline_url);
+        my %super_hash = (%$specialization_options, %$life_options, %$execution_options);
+        foreach my $option (keys %super_hash) {
+            push @cmd, ('-'.$option, $super_hash{$option});
+        }
+        my $rc = system(@cmd);
+        ok(!$rc, 'worker successful');
+    }, sprintf('runWorker()'));
+}
+
+
+1;

scripts/travis_run_tests.sh

+#!/bin/bash
+# Copyright [1999-2016] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#      http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+export PERL5LIB=$PWD/modules:$PERL5LIB
+
+    # for the t/10.pipeconfig/longmult.t test
+export EHIVE_TEST_PIPELINE_URLS='mysql://travis@127.0.0.1/ehive_test_pipeline_db pgsql://postgres@127.0.0.1/ehive_test_pipeline_db sqlite:///ehive_test_pipeline_db'
+
+echo "Running test suite"
+if [ "$COVERALLS" = 'true' ]; then
+  PERL5OPT="-MDevel::Cover=+ignore,/usr/bin/psql,-db,$PWD/cover_db/" prove -rv t
+else
+  prove -r t
+fi
+
+rt=$?
+if [ $rt -eq 0 ]; then
+  if [ "$COVERALLS" = 'true' ]; then
+    echo "Running Devel::Cover coveralls report"
+    cover --nosummary -report coveralls
+  fi
+  exit $?
+else
+  exit $rt
+fi

t/01.utils/collection.t

+#!/usr/bin/env perl
+
+# Copyright [1999-2016] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#      http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+use strict;
+use warnings;
+
+use Test::More;
+use Data::Dumper;
+
+BEGIN {
+    ## at least it compiles
+    use_ok( 'Bio::EnsEMBL::Hive::Utils::Collection' );
+}
+#########################
+
+my $collection = Bio::EnsEMBL::Hive::Utils::Collection->new(['the']);
+ok($collection, 'and creates objects');
+
+my $ref = $collection->listref;
+is(@$ref, 1, 'correct size');
+
+my @ref = $collection->list;
+is(@ref, 1, 'still correct size');
+
+my $i = 1;
+foreach my $member(qw{quick brown fox}) { 
+    $collection->add($member);
+    is(@$ref, ++$i, 'addition - same object');
+    is(@ref, 1, 'no addition as it was a copy');
+}
+
+$collection = Bio::EnsEMBL::Hive::Utils::Collection->new([{ foo => undef}]);
+$collection->add({ foo => undef });
+$collection->add({ bar => 'foobar' });
+$collection->add({ foo => undef });
+
+my $listref = $collection->listref;
+is(@$listref, 4, '4 elements');
+
+my $result = $collection->find_one_by(bar => 'foobar');
+ok($result, 'found something');
+ok(exists($result->{'bar'}), 'we think it is correct');
+is($result->{'bar'}, 'foobar');
+
+done_testing();

t/01.utils/command_line.t

+#!/usr/bin/env perl
+
+# Copyright [1999-2016] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#      http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+use strict;
+use warnings;
+
+use Test::More;
+use Data::Dumper;
+
+BEGIN {
+    use_ok( 'Bio::EnsEMBL::Hive::Utils', 'parse_cmdline_options' );
+}
+#########################
+
+## block 1 - simple single and double -- options
+{
+    ## note we do not require the double minus version
+    local @ARGV = qw{-foo bar --alpha beta -count 3};
+    my ($pairs, $list) = parse_cmdline_options();
+    ok($pairs, 'command line parsed to a hash');
+    isa_ok($pairs, 'HASH'); ## really a hash
+    ok($list, 'list returned');
+    isa_ok($list, 'ARRAY'); ## really a list ref
+    is(@$list, 0, 'empty list');
+    is($pairs->{'foo'}, 'bar', 'foo option set');
+    is($pairs->{'alpha'}, 'beta', 'alpha option set');
+    is($pairs->{'count'}, 3, 'count option set');
+}
+
+## what happens with one too many - ?
+{
+    local @ARGV = qw{---wrong option};
+    my ($pairs, $list) = parse_cmdline_options();
+    is(scalar(keys %$pairs), 0, 'ignore the --- option and assign to list instead');
+    is(@$list, 2, 'list is populated');
+    is($list->[0], '---wrong', 'no pruning of -');
+    is($list->[1], 'option', 'fidelity');
+}
+
+## mix of options and extra
+{
+    local @ARGV = qw{-foo bar alpha beta --lorem ipsum};
+    my ($pairs, $list) = parse_cmdline_options();
+    ok($pairs, 'good');
+    ok($list, 'still good');
+    is($pairs->{'foo'}, 'bar', 'option foo set to bar');
+    is($list->[0], 'alpha', 'centuri');
+    is($list->[1], 'beta', 'globulin');
+    is($pairs->{'lorem'}, 'ipsum', 'still latin');
+}
+
+## complex option array
+{
+    local @ARGV = (q!--foo=["bar","foobar"]!, 'alpha');
+    my ($pairs, $list) = parse_cmdline_options();
+    ok($pairs, 'pairs');
+    isa_ok($pairs->{'foo'}, 'ARRAY'); ## array value
+    is(@$list, 1, 'still set');
+
+}
+
+## complex option hash
+{
+    local @ARGV = (q!--foo={"bar" => "foobar"}!, 'alpha');
+    my ($pairs, $list) = parse_cmdline_options();
+    ok($pairs, 'pairs');
+    isa_ok($pairs->{'foo'}, 'HASH'); ## hash value
+    is(@$list, 1, 'still set');
+}
+
+## trailing key
+{
+    local @ARGV = qw{-foo bar alpha -beta};
+    my ($pairs, $list) = parse_cmdline_options();
+    ok($pairs, 'pairs');
+    ok($list, 'list');
+    is(@$list, 1, 'alpha only');
+    is($list->[0], 'alpha', 'no beta here');
+    is($pairs->{'foo'}, 'bar', 'bar still');
+}
+
+done_testing();

t/01.utils/config.t

+#!/usr/bin/env perl
+
+# Copyright [1999-2016] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#      http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+use strict;
+use warnings;
+
+use Cwd;
+use File::Basename;
+use Test::More;
+use Data::Dumper;
+
+BEGIN {
+    use_ok( 'Bio::EnsEMBL::Hive::Utils::Config' );
+    
+}
+#########################
+
+# Need EHIVE_ROOT_DIR to access the default config file
+$ENV{'EHIVE_ROOT_DIR'} ||= File::Basename::dirname( File::Basename::dirname( File::Basename::dirname( Cwd::realpath($0) ) ) );
+
+my @config_files = Bio::EnsEMBL::Hive::Utils::Config->default_config_files();
+# diag "@config_files"; 
+
+cmp_ok(@config_files, '>', 0, 'at least one got returned');
+cmp_ok(@config_files, '<', 3, '1 or 2');
+
+my $config = bless {_config_hash => {}}, 'Bio::EnsEMBL::Hive::Utils::Config';
+
+my $json = $ENV{EHIVE_ROOT_DIR}.'/t/test_config.json';
+
+my $simple = $config->load_from_json($json);
+isa_ok($simple, 'HASH');
+ok(exists($simple->{'Meadow'}), 'exists');
+
+my $simpler = $config->load_from_json($json . '.notexist');
+is($simpler, undef, 'undef, but no death');
+
+$config->merge($simple);
+my $merged = $config->config_hash;
+ok($merged, 'merged hash');
+
+
+$config = Bio::EnsEMBL::Hive::Utils::Config->new();
+ok($config, 'new instance');
+
+
+my $content = $config->config_hash;
+
+isa_ok($content, 'HASH');
+ok(exists($content->{'Meadow'}), 'Ok you are in a forest. Forest? With Heather...');
+
+isa_ok($content->{'Meadow'}, 'HASH');
+
+ok(exists($content->{'Valley'}), 'alpine valley?');
+
+isa_ok($content->{'Valley'}, 'HASH');
+
+ok(exists($content->{'Graph'}), 'lets plot that');
+
+isa_ok($content->{'Graph'}, 'HASH');
+
+ok(exists($content->{'VERSION'}), 'party on');
+
+$config->merge($simple);
+my $merged_hash = $config->config_hash;
+isa_ok($merged_hash, 'HASH');
+ok(exists($merged_hash->{'Meadow'}{'OPENLAVA'}),                               'first level merge');
+ok(exists($merged_hash->{'Meadow'}{'OPENLAVA'}{'omics'}),                      'second level merge');
+ok(exists($merged_hash->{'Meadow'}{'OPENLAVA'}{'omics'}{'SubmissionOptions'}), 'third level merge');
+is($merged_hash->{'Meadow'}{'OPENLAVA'}{'TotalRunningWorkersMax'}, 1000,       'data check');
+
+my $ol_submission_opts = $config->get(qw{Meadow OPENLAVA omics SubmissionOptions});
+is($ol_submission_opts, '-q special-branch', 'get() functions');
+
+$config->set(qw{Meadow OPENLAVA TotalRunningWorkersMax}, 20);
+my $ol_total = $config->get(qw{Meadow OPENLAVA TotalRunningWorkersMax});
+is($ol_total, 20, 'roundtrip set() and get() function');
+
+done_testing();

t/01.utils/directory_hash.t

+#!/usr/bin/env perl
+
+# Copyright [1999-2016] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#      http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+use strict;
+use warnings;
+
+use Test::More;
+use Data::Dumper;
+
+BEGIN {
+    use_ok( 'Bio::EnsEMBL::Hive::Utils', 'dir_revhash' );
+}
+#########################
+
+my $directory_structure = dir_revhash 123456789;
+ok( $directory_structure, 'simply returns a string' );
+is( $directory_structure, '9/8/7/6/5/4/3/2', 'string is expected');
+
+done_testing();

t/01.utils/modules.t

+#!/usr/bin/env perl
+
+# Copyright [1999-2016] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#      http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+use strict;
+use warnings;
+
+use Test::More;
+use Data::Dumper;
+
+BEGIN {
+    use_ok( 'Bio::EnsEMBL::Hive::Utils', 'load_file_or_module', 'find_submodules' );
+}
+#########################
+
+my $hive = find_submodules 'Bio::EnsEMBL::Hive';
+isa_ok($hive, 'ARRAY');
+cmp_ok(scalar(@$hive), '>', 0, 'modules loaded');
+
+my $pipe_configs = find_submodules 'Bio::EnsEMBL::Hive::PipeConfig';
+ok($pipe_configs, 'configs found');
+isa_ok($pipe_configs, 'ARRAY'); ## still an array
+my @sorted_configs = sort { $a cmp $b } @$pipe_configs;
+my ($generic) = grep { /HiveGeneric_conf/ } @sorted_configs;
+ok($generic, 'found generic base config');
+
+## loading the module
+
+my $module = load_file_or_module( $generic );
+is( $module, $generic, 'all good' );
+
+## and by file name ...
+
+
+done_testing();

t/01.utils/redirect_stack.t

+#!/usr/bin/env perl
+
+# Copyright [1999-2016] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#      http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+use strict;
+use warnings;
+
+use Test::More tests => 8;
+
+use Capture::Tiny ':all';
+use File::Temp qw{tempdir};
+
+BEGIN {
+    use_ok( 'Bio::EnsEMBL::Hive::Utils::RedirectStack' );
+}
+#########################
+
+my $dir = tempdir CLEANUP => 1;
+my $original = chdir $dir;
+
+my $rs_stdout = Bio::EnsEMBL::Hive::Utils::RedirectStack->new(\*STDOUT);
+my $stdout = capture_stdout {
+    print "Message 1\n";            # gets displayed on the screen
+    $rs_stdout->push('foo');
+        print "Message 2\n";            # goes to 'foo'
+        $rs_stdout->push('bar');
+            print "Message 3\n";            # goes to 'bar'
+            system('echo subprocess A');    # it works for subprocesses too
+            $rs_stdout->pop;
+        print "Message 4\n";            # goes to 'foo'
+        system('echo subprocess B');    # again, works for subprocesses as well
+        $rs_stdout->push('baz');
+            print "Message 5\n";            # goest to 'baz'
+            $rs_stdout->pop;
+        print "Message 6\n";            # goes to 'foo'
+        $rs_stdout->pop;
+    print "Message 7\n";            # gets displayed on the screen
+};
+
+is($stdout, qq{Message 1\nMessage 7\n}, 'stdout output');
+ok(-e 'foo', '"foo" exists');
+is(`cat foo`, qq{Message 2\nMessage 4\nsubprocess B\nMessage 6\n}, 'foo output');
+ok(-e 'bar', '"bar" exists');
+is(`cat bar`, qq{Message 3\nsubprocess A\n}, 'bar output');
+ok(-e 'baz', '"bar" exists');
+is(`cat baz`, qq{Message 5\n}, 'baz output');
+
+done_testing();
+
+chdir $original;
+

t/01.utils/stopwatch.t

+#!/usr/bin/env perl
+
+# Copyright [1999-2016] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#      http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+use strict;
+use warnings;
+
+use Test::More;
+use Data::Dumper;
+
+BEGIN {
+    ## at least it compiles
+    use_ok( 'Bio::EnsEMBL::Hive::Utils::Stopwatch' );
+}
+#########################
+