Merge pull request #89 from Ensembl/merge_conflict

Merge conflict

.travis.yml

@@ -3,7 +3,7 @@ os:
 services: docker
 
 before_install:
-    - docker pull smmebi/otter_test_update
+    - docker pull smmebi/otter_test_nomerge
 install:
     - cd ../
     - export ROOT_PATH=$PWD
@@ -13,11 +13,19 @@ install:
     - git clone --branch master --depth 1 https://github.com/Ensembl/team_tools.git
     - git clone --branch master --depth 1 https://github.com/Ensembl/webvm.git
     - git clone --branch master --depth 1 https://github.com/Ensembl/webvm-deps.git
+    - cd webvm-deps/
+    - mkdir ensembl-branch-99/
+    - cd ensembl-branch-99/
+    - git clone --branch release/99 https://github.com/Ensembl/ensembl.git
+    - git clone --branch release/99 https://github.com/Ensembl/ensembl-variation.git
+    - git clone --branch release/99 https://github.com/Ensembl/ensembl-funcgen.git
+    - git clone --branch release/99 https://github.com/Ensembl/ensembl-compara.git
+    - cd ../../
     - git clone --branch master --depth 1 https://github.com/Ensembl/zircon.git
     - git clone --branch master --depth 1 https://github.com/Ensembl/PerlModules.git
 
 
-script: "docker run -i --name devtest -v $ROOT_PATH/app:/loutre smmebi/otter_test_update:latest"
+script: "docker run -i --name devtest -v $ROOT_PATH/app:/loutre smmebi/otter_test_nomerge:latest"
 
 notifications:
   slack:

data/assembly.sql

 -- Copyright [2018-2019] EMBL-European Bioinformatics Institute
--- 
+--
 -- Licensed under the Apache License, Version 2.0 (the "License");
 -- you may not use this file except in compliance with the License.
 -- You may obtain a copy of the License at
--- 
+--
 --      http://www.apache.org/licenses/LICENSE-2.0
--- 
+--
 -- Unless required by applicable law or agreed to in writing, software
 -- distributed under the License is distributed on an "AS IS" BASIS,
 -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

data/tinyassembly.sql

 -- Copyright [2018-2019] EMBL-European Bioinformatics Institute
--- 
+--
 -- Licensed under the Apache License, Version 2.0 (the "License");
 -- you may not use this file except in compliance with the License.
 -- You may obtain a copy of the License at
--- 
+--
 --      http://www.apache.org/licenses/LICENSE-2.0
--- 
+--
 -- Unless required by applicable law or agreed to in writing, software
 -- distributed under the License is distributed on an "AS IS" BASIS,
 -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

dev/otter_live_bash

 #! /bin/sh
 # Copyright [2018-2019] EMBL-European Bioinformatics Institute
-# 
+#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 #      http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

dev/otter_this

 #!/bin/bash
 # Copyright [2018-2019] EMBL-European Bioinformatics Institute
-# 
+#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 #      http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

dist/conf/client_ensembl_version

-83
+99
 
 # This is the version of EnsEMBL that the Otter client uses.  The
 # install script uses this to check out the correct branch from the

dist/conf/server_ensembl_version

-83
+99
 
 # This is the version of EnsEMBL that the Otter server uses.

dist/conf/version_major

-109
+110
 
 # This is the Otter major version.  Together with the
 # version_minor configuration parameter it determines the installation

docs/xml_description_tables

 #!/usr/bin/env perl
 # Copyright [2018-2019] EMBL-European Bioinformatics Institute
-# 
+#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 #      http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

modules/Bio/Otter/Lace/AceDatabase.pm

@@ -349,7 +349,7 @@ sub recover_slice_from_region_xml {
 
     my $parser = Bio::Vega::Transform::XMLToRegion->new;
     $parser->analysis_from_transcript_class(1);
-    $parser->coord_system_factory(Bio::Vega::CoordSystemFactory->new); # Should we get this from somewhere else?
+    $parser->coord_system_factory(Bio::Vega::CoordSystemFactory->new( dba => $self->DB->vega_dba )); # Should we get this from somewhere else?
     my $region = $parser->parse($xml);
 
     my $slice = Bio::Otter::Lace::Slice->new_from_region($client, $region);
@@ -835,6 +835,7 @@ sub generate_XML_from_sqlite {
 
     $self->DB->vega_dba->clear_caches;
     my $region = Bio::Vega::Region->new_from_otter_db( slice => $self->DB->session_slice );
+    $region->{species} = $self->DB->species;
     $region->check_transcript_stable_ids;
     my $formatter = Bio::Vega::Transform::RegionToXML->new;
     $formatter->region($region);

modules/Bio/Otter/Lace/Client.pm

@@ -895,7 +895,6 @@ sub _do_http_request {
         $request->content($paramstring);
 
         $self->_client_logger->debug("POST  $url");
-        # $self->_client_logger->debug("paramstring: $paramstring");
     }
     else {
         $self->logger->logconfess("method '$method' is not supported");
@@ -950,8 +949,7 @@ sub status_refresh_for_DataSet_SequenceSet{
 }
 
 sub find_clones {
-    my ($self, $dsname, $qnames_list) = @_;
-
+    my ($self, $dsname, $qnames_list, $ss) = @_;
     my $qnames_string = join(',', @$qnames_list);
 
     my $response = $self->http_response_content(
@@ -960,7 +958,9 @@ sub find_clones {
         {
             'dataset'  => $dsname,
             'qnames'   => $qnames_string,
-            'author'   => $self->author
+            'author'   => $self->author,
+            'coord_system_name' => $ss->coord_system_name,
+            'coord_system_version' => $ss->coord_system_version,
         },
     );
 
@@ -992,8 +992,9 @@ sub get_meta {
 }
 
 sub get_db_info {
-    my ($self, $dsname) = @_;
-    my $hashref = $self->otter_response_content(GET => 'get_db_info', { dataset => $dsname, 'author' => $self->author });
+    my ($self, $dsname, $coord_system_name, $coord_system_version) = @_;
+    my $hashref = $self->otter_response_content(GET => 'get_db_info', { dataset => $dsname, 'coord_system_name' => $coord_system_name,
+                                                                         'coord_system_version' => $coord_system_version, 'author' => $self->author });
     return $hashref;
 }
 
@@ -1001,8 +1002,12 @@ sub lock_refresh_for_DataSet_SequenceSet {
     my ($self, $ds, $ss) = @_;
     my $response =
        $self->_DataSet_SequenceSet_response_content(
-           $ds, $ss, 'GET', 'get_locks', {'author' => $self->author});
-
+           $ds, $ss, 'GET', 'get_locks',
+        {
+            'coord_system_name' => $ss->coord_system_name,
+            'coord_system_version' => $ss->coord_system_version,
+            'author' => $self->author
+        });
     my @slice_lock = map { Bio::Vega::SliceLock->new_from_json($_) }
       @{ $response->{SliceLock} || [] };
 
@@ -1025,7 +1030,7 @@ sub fetch_all_SequenceNotes_for_DataSet_SequenceSet {
 
     my $response =
         $self->_DataSet_SequenceSet_response_content(
-            $ds, $ss, 'GET', 'get_sequence_notes', {'author' => $self->author});
+            $ds, $ss, 'GET', 'get_sequence_notes',{'author' => $self->author});
 
     my %ctgname2notes = ();
 
@@ -1097,7 +1102,7 @@ sub _sequence_note_action {
             'contig'    => $contig_name,
             'timestamp' => $seq_note->timestamp(),
             'text'      => $seq_note->text(),
-            'author'    => $self->author,
+            'author'    => $self->author
         },
     );
 
@@ -1131,6 +1136,15 @@ sub _get_DataSets_hash {
     my $datasets_hash = $self->otter_response_content
         ('GET', 'get_datasets', {'author' => $self->author});
 
+
+    return $datasets_hash;
+}
+
+sub fetch_fasta_seqence {
+    my ($self, $acc) = @_;
+    my $datasets_hash = $self->otter_response_content
+        ('GET', 'get_fasta_sequence', {'id'=>$acc, 'author' => $self->author});
+
     return $datasets_hash;
 }
 
@@ -1238,7 +1252,9 @@ sub get_server_ensembl_version {
 # same as Bio::Otter::Server::Config->designations (fresh every time)
 sub _get_designations {
     my ($self) = @_;
-    my $hashref = $self->otter_response_content(GET => 'get_config', { key => 'designations', 'author' => $self->author });
+
+    my $hashref = $self->otter_response_content(GET => 'get_config', { key => 'designations','author' => $self->author });
+
     return $hashref;
 }
 
@@ -1334,15 +1350,16 @@ sub get_slice_DE {
 sub slice_query {
     my ($self, $slice) = @_;
     die unless wantarray;
-    return (
-        'dataset' => $slice->dsname(),
-        'chr'     => $slice->ssname(),
-        'cs'      => $slice->csname(),
-        'csver'   => $slice->csver(),
-        'name'    => $slice->seqname(),
-        'start'   => $slice->start(),
-        'end'     => $slice->end(),
-    );
+
+    return ('dataset' => $slice->dsname(),
+            'chr'     => $slice->ssname(),
+            'cs'      => $slice->csname(),
+            'csver'   => $slice->csver(),
+            'name'    => $slice->seqname(),
+            'start'   => $slice->start(),
+            'end'     => $slice->end(),
+           );
+
 }
 
 
@@ -1365,9 +1382,7 @@ sub get_all_SequenceSets_for_DataSet {
 
   my $sequencesets_xml =
       $self->http_response_content(
-          'GET', 'get_sequencesets', {
-              'dataset' => $dataset_name, 'author' => $self->author,
-          });
+          'GET', 'get_sequencesets', {'dataset' => $dataset_name, 'author' => $self->author});
 
   local $XML::Simple::PREFERRED_PARSER = 'XML::Parser';
   # configure expat for speed, also used in Bio::Vega::XML::Parser
@@ -1416,6 +1431,9 @@ sub _make_SequenceSet {
                     $sr_name, [split(/,/, $sr_params->{content})]);
             }
         }
+        elsif ($key eq 'coord_system_name' or $key eq 'coord_system_version') {
+            $sequenceset->$key($value);
+        }
         elsif ($sequenceset->can($key)) {
             die "Bad key $key" unless $key =~ /^[_A-Za-z]{1,16}$/;
             $sequenceset->$key($value);
@@ -1433,6 +1451,7 @@ sub get_all_CloneSequences_for_DataSet_SequenceSet { # without any lock info
 
   my $dataset_name     = $ds->name;
   my $sequenceset_name = $ss->name;
+  $ds->selected_SequenceSet($ss);
 
   my $clonesequences_xml = $self->http_response_content(
         'GET',
@@ -1440,7 +1459,10 @@ sub get_all_CloneSequences_for_DataSet_SequenceSet { # without any lock info
         {
             'dataset'     => $dataset_name,
             'sequenceset' => $sequenceset_name,
-            'author'      => $self->author
+            'coord_system_name' => $ss->coord_system_name,
+            'coord_system_version' => $ss->coord_system_version,
+            'author' => $self->author
+
         }
     );
 
@@ -1466,8 +1488,7 @@ sub get_all_CloneSequences_for_DataSet_SequenceSet { # without any lock info
               $dataset_name, $sequenceset_name, $_);
       } @{$clonesequences_array} ];
   $ss->CloneSequence_list($clonesequences);
-
-  return $clonesequences;
+ return $clonesequences;
 }
 
 sub _make_CloneSequence {
@@ -1479,6 +1500,9 @@ sub _make_CloneSequence {
         if ($key eq 'chr') {
             $clonesequence->chromosome($value->{name});
         }
+        elsif ($key eq 'coord_system_name' or $key eq 'coord_system_version') {
+            $clonesequence->$key($value);
+        }
         elsif ($clonesequence->can($key)) {
             die "Bad key $key" unless $key =~ /^[_A-Za-z]{1,16}$/;
             $clonesequence->$key($value);
@@ -1553,7 +1577,7 @@ sub save_otter_xml {
 # lock_region, unlock_region : see Bio::Otter::Lace::AceDatabase
 
 sub _DataSet_SequenceSet_response_content {
-    my ($self, $ds, $ss, $method, $script) = @_;
+    my ($self, $ds, $ss, $method, $script, $extra) = @_;
 
     my $query = {
         'dataset'  => $ds->name,
@@ -1561,6 +1585,9 @@ sub _DataSet_SequenceSet_response_content {
         'author'   => $self->author
     };
 
+    if ($extra and ref($extra) eq 'HASH') {
+      %$query = (%$query, %$extra);
+    }
     my $content =
         $self->otter_response_content($method, $script, $query);
 
@@ -1795,3 +1822,4 @@ An ordinary constructor, making instances as requested.
 =head1 AUTHOR
 
 Ana Code B<email> anacode@sanger.ac.uk
+

modules/Bio/Otter/Lace/CloneSequence.pm

@@ -31,6 +31,24 @@ sub new {
     return bless {}, $pkg;
 }
 
+sub coord_system_name {
+    my ($self, $coord_system_name) = @_;
+
+    if ($coord_system_name) {
+        $self->{'_coord_system_name'} = $coord_system_name;
+    }
+    return $self->{'_coord_system_name'};
+}
+
+sub coord_system_version {
+    my ($self, $coord_system_version) = @_;
+
+    if ($coord_system_version) {
+        $self->{'_coord_system_version'} = $coord_system_version;
+    }
+    return $self->{'_coord_system_version'};
+}
+
 sub accession {
     my ($self, $accession) = @_;
 

modules/Bio/Otter/Lace/DB.pm

@@ -298,6 +298,7 @@ sub load_dataset_info {
 
     my $dbh = $dbh{$self};
 
+    my $select_sth = $dbh->prepare(q{ SELECT species_id, meta_key, meta_value FROM meta WHERE species_id = ? AND meta_key = ? AND meta_value = ? });
     my $meta_sth = $dbh->prepare(q{ INSERT INTO meta (species_id, meta_key, meta_value) VALUES (?, ?, ?) });
     my $meta_hash = $dataset->meta_hash;
 
@@ -307,15 +308,6 @@ sub load_dataset_info {
 
     # I'm not really sure we need to do this - we could just use a local version
     #
-    my $cs_chr  = $dataset->get_db_info_item('coord_system.chromosome');
-    my $local_cs_spec = {
-        $cs_chr->{name} => {
-            '-version'        => $cs_chr->{version},
-            '-rank'           => $cs_chr->{rank},
-            '-default'        => $cs_chr =~ m/default_version/,
-            '-sequence_level' => $cs_chr =~ m/sequence_level/,
-        },
-    };
 
     my @at_cols = qw(                                       attrib_type_id  code  name  description );
     my $at_sth  = $dbh->prepare(q{ INSERT INTO attrib_type (attrib_type_id, code, name, description)
@@ -323,30 +315,44 @@ sub load_dataset_info {
     my $at_list = $dataset->get_db_info_item('attrib_type');
 
     my $_dba = $self->_dba('_coords');     # we throw this one away
+    my $override_specs = $dataset->get_db_info_item('coord_systems');
+    my $dna_cs_rank;
+    foreach my $value (values %$override_specs) {
+      if ($value->{'-sequence_level'}) {
+        $value->{'-sequence_level'} = 0;
+        $dna_cs_rank = $value->{'-rank'}+1;
+      }
+    }
 
     my $cs_factory = Bio::Vega::CoordSystemFactory->new(
         dba           => $_dba,
         create_in_db  => 1,
-        override_spec => $local_cs_spec,
+        override_spec => $override_specs,
         );
+    my $toplevel_cs = $dataset->get_db_info_item('coord_system.chromosome');
 
-    my %local_meta = (
-        'assembly.mapping' => {
-            species_id => 1,
-            values     => [
-                $cs_factory->assembly_mappings,
-            ],
-        },
-        );
+    $override_specs->{dna_contig} = {
+      '-rank' => $dna_cs_rank,
+      '-sequence_level' => 1,
+      '-default' => 1,
+      'version'  => $toplevel_cs->{version},
+    };
+    $meta_hash->{'assembly.mapping'}->{species_id} = $meta_hash->{'species.classification'}->{species_id};
+    push(@{$meta_hash->{'assembly.mapping'}->{values}}, $toplevel_cs->{name}.':'.$toplevel_cs->{version}.'|dna_contig:'.$toplevel_cs->{version});
 
     $dbh->begin_work;
 
     # Meta first, so that CoordSystemAdaptor doesn't complain about missing schema_version
     #
     while (my ($key, $details) = each %$meta_hash) {
-        next if $key eq 'assembly.mapping'; # we do our own mapping, below...
         foreach my $value (@{$details->{values}}) {
-            $meta_sth->execute($details->{species_id}, $key, $value);
+            $select_sth->execute($details->{species_id}, $key, $value);
+            if($select_sth->fetchrow_array) {
+                next;
+            }
+            else {
+                $meta_sth->execute($details->{species_id}, $key, $value);
+            }
         }
     }
 
@@ -358,22 +364,14 @@ sub load_dataset_info {
 
     $dbh->commit;
 
-    $dbh->begin_work;
 
     # Coord systems via factory
     #
+    $cs_factory->known;
     $cs_factory->instantiate_all;
 
     # Mappings
     #
-    my $mca = $_dba->get_MetaContainer;
-    while (my ($key, $details) = each %local_meta) {
-        foreach my $value (@{$details->{values}}) {
-            $mca->store_key_value($key, $value);
-        }
-    }
-
-    $dbh->commit;
 
     $self->_is_loaded('dataset_info', 1);
 

modules/Bio/Otter/Lace/DataSet.pm

@@ -481,8 +481,14 @@ sub get_meta_value {
 sub db_info_hash {
     my ($self) = @_;
     # Get all db_info in one call
+    my $cs_name = 'chromosome';
+    my $cs_version = 'Otter';
+    if ($self->selected_SequenceSet) {
+      $cs_name = $self->selected_SequenceSet->coord_system_name;
+      $cs_version = $self->selected_SequenceSet->coord_system_version;
+    }
     return $self->{'_db_info_hash'} ||=
-        $self->Client->get_db_info($self->name);
+        $self->Client->get_db_info($self->name, $cs_name, $cs_version);
 }
 
 sub get_db_info_item {
@@ -682,7 +688,7 @@ sub get_cached_DBAdaptor {
         $self->_attach_DNA_DBAdaptor($tmp) if $self->DNA_DBNAME;
         $self->{'_dba_cache'} = $tmp;
     }
-    #warn "OTTER DBADAPTOR = '$dba'";
+
     return $self->{'_dba_cache'};
 }
 

modules/Bio/Otter/Lace/Defaults.pm

@@ -685,8 +685,9 @@ __DATA__
 # e.g. short_window_title_prefix and password_timeout accept -1 to
 # disable.
 
+
 [client]
-url=http://193.62.52.185:82/cgi-bin/otter
+url=http://otter-server.ebi.ac.uk:8004/cgi-bin/otter
 write_access=1
 gff_version=3
 acedb_version=4.9.61

modules/Bio/Otter/Lace/OnTheFly/QueryValidator.pm

@@ -16,6 +16,8 @@ limitations under the License.
 
 =cut
 
+### Bio::Otter::Lace::OnTheFly::QueryValidator
+
 package Bio::Otter::Lace::OnTheFly::QueryValidator;
 
 use namespace::autoclean;
@@ -236,41 +238,68 @@ sub Client {
 # Adds sequences to $self->seqs
 #
 sub _fetch_sequences {
-  my ($self, @to_fetch) = @_;
-
-  my $cache = $self->accession_type_cache;
-
-  @to_fetch = uniq @to_fetch;
-  $self->logger->debug('Need seq for: ', join(',', @to_fetch) || '<none>');
-  my $client = Bio::Otter::Lace::Defaults::make_Client();
-  my $seqs = $client->fetch_fasta_sequence(@to_fetch);
-  my @seq_array = split('>', $seqs);
-
-
-  foreach my $sequence (@seq_array) {
-      if (length($sequence) < 2) {
-          next;
-      };
-
-      my $seq = $self->parse_fasta_sequence('>' . $sequence);
-      my $iteration = 0;
-      my @to_fetch_cut = @to_fetch;
-      @to_fetch_cut = map { m/\-/ && s/\.\d+$//; $_ } @to_fetch_cut;
-      while ((index $seq->name, $to_fetch_cut[$iteration]) == -1 && $iteration < scalar(@to_fetch)) {
-            $iteration++;
-      }
+    my ($self, @to_fetch) = @_;
+
+    my $cache = $self->accession_type_cache;
+
+    @to_fetch = uniq @to_fetch;
+    $self->logger->debug('Need seq for: ', join(',', @to_fetch) || '<none>');
+    my $client = Bio::Otter::Lace::Defaults::make_Client();
+    foreach my $acc (@to_fetch) {
+
+        my $seq = $client->fetch_fasta_seqence($acc);
+        if (substr($seq, 0, 1) eq ">") {
+          $seq = $self->parse_fasta_sequence($seq);
+          push(@{$self->seqs}, $seq);
+          my ($type, $full) = @{$self->_acc_type_full($acc)};
+          unless ($type) {
+              $self->_add_missing_warning($acc => 'illegal evidence type');
+              next;
+          }
+          $seq->type($type);
+          $seq->name($acc);
+        } else {
+          $self->_add_missing_warning($acc, "unknown accession or illegal evidence type");
+        }
 
-      my ($type, $full) = @{$self->_acc_type_full($to_fetch[$iteration])};
-      unless ($type) {
-          $self->_add_missing_warning($to_fetch[$iteration] => 'illegal evidence type');
-          next;
-      }
-      $seq->type($type);
-      $seq->name($full);
-      push(@{$self->seqs}, $seq);
-  }
+#        my ($type, $full) = @{$self->_acc_type_full($acc)};
+#        unless ($type) {
+#            $self->_add_missing_warning($acc => 'illegal evidence type');
+#            next;
+#        }
+
+#        my $info = $cache->feature_accession_info($acc);
+#        unless ($info) {
+#            $self->logger->error("No info for '$acc' - this should not happen");
+#            $self->_add_missing_warning($acc => 'internal error');
+#            next;