Unverified Commit f366fa84 authored by s-mm's avatar s-mm Committed by GitHub
Browse files

Merge pull request #89 from Ensembl/merge_conflict

Merge conflict
parents 8cfbcfb6 6c876b1f
......@@ -3,7 +3,7 @@ os:
services: docker
before_install:
- docker pull smmebi/otter_test_update
- docker pull smmebi/otter_test_nomerge
install:
- cd ../
- export ROOT_PATH=$PWD
......@@ -13,11 +13,19 @@ install:
- git clone --branch master --depth 1 https://github.com/Ensembl/team_tools.git
- git clone --branch master --depth 1 https://github.com/Ensembl/webvm.git
- git clone --branch master --depth 1 https://github.com/Ensembl/webvm-deps.git
- cd webvm-deps/
- mkdir ensembl-branch-99/
- cd ensembl-branch-99/
- git clone --branch release/99 https://github.com/Ensembl/ensembl.git
- git clone --branch release/99 https://github.com/Ensembl/ensembl-variation.git
- git clone --branch release/99 https://github.com/Ensembl/ensembl-funcgen.git
- git clone --branch release/99 https://github.com/Ensembl/ensembl-compara.git
- cd ../../
- git clone --branch master --depth 1 https://github.com/Ensembl/zircon.git
- git clone --branch master --depth 1 https://github.com/Ensembl/PerlModules.git
script: "docker run -i --name devtest -v $ROOT_PATH/app:/loutre smmebi/otter_test_update:latest"
script: "docker run -i --name devtest -v $ROOT_PATH/app:/loutre smmebi/otter_test_nomerge:latest"
notifications:
slack:
......
-- Copyright [2018-2019] EMBL-European Bioinformatics Institute
--
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
......
-- Copyright [2018-2019] EMBL-European Bioinformatics Institute
--
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
......
#! /bin/sh
# Copyright [2018-2019] EMBL-European Bioinformatics Institute
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#
# http://www.apache.org/licenses/LICENSE-2.0
#
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
......
#!/bin/bash
# Copyright [2018-2019] EMBL-European Bioinformatics Institute
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#
# http://www.apache.org/licenses/LICENSE-2.0
#
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
......
83
99
# This is the version of EnsEMBL that the Otter client uses. The
# install script uses this to check out the correct branch from the
......
83
99
# This is the version of EnsEMBL that the Otter server uses.
109
110
# This is the Otter major version. Together with the
# version_minor configuration parameter it determines the installation
......
#!/usr/bin/env perl
# Copyright [2018-2019] EMBL-European Bioinformatics Institute
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#
# http://www.apache.org/licenses/LICENSE-2.0
#
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
......
......@@ -349,7 +349,7 @@ sub recover_slice_from_region_xml {
my $parser = Bio::Vega::Transform::XMLToRegion->new;
$parser->analysis_from_transcript_class(1);
$parser->coord_system_factory(Bio::Vega::CoordSystemFactory->new); # Should we get this from somewhere else?
$parser->coord_system_factory(Bio::Vega::CoordSystemFactory->new( dba => $self->DB->vega_dba )); # Should we get this from somewhere else?
my $region = $parser->parse($xml);
my $slice = Bio::Otter::Lace::Slice->new_from_region($client, $region);
......@@ -835,6 +835,7 @@ sub generate_XML_from_sqlite {
$self->DB->vega_dba->clear_caches;
my $region = Bio::Vega::Region->new_from_otter_db( slice => $self->DB->session_slice );
$region->{species} = $self->DB->species;
$region->check_transcript_stable_ids;
my $formatter = Bio::Vega::Transform::RegionToXML->new;
$formatter->region($region);
......
......@@ -895,7 +895,6 @@ sub _do_http_request {
$request->content($paramstring);
$self->_client_logger->debug("POST $url");
# $self->_client_logger->debug("paramstring: $paramstring");
}
else {
$self->logger->logconfess("method '$method' is not supported");
......@@ -950,8 +949,7 @@ sub status_refresh_for_DataSet_SequenceSet{
}
sub find_clones {
my ($self, $dsname, $qnames_list) = @_;
my ($self, $dsname, $qnames_list, $ss) = @_;
my $qnames_string = join(',', @$qnames_list);
my $response = $self->http_response_content(
......@@ -960,7 +958,9 @@ sub find_clones {
{
'dataset' => $dsname,
'qnames' => $qnames_string,
'author' => $self->author
'author' => $self->author,
'coord_system_name' => $ss->coord_system_name,
'coord_system_version' => $ss->coord_system_version,
},
);
......@@ -992,8 +992,9 @@ sub get_meta {
}
sub get_db_info {
my ($self, $dsname) = @_;
my $hashref = $self->otter_response_content(GET => 'get_db_info', { dataset => $dsname, 'author' => $self->author });
my ($self, $dsname, $coord_system_name, $coord_system_version) = @_;
my $hashref = $self->otter_response_content(GET => 'get_db_info', { dataset => $dsname, 'coord_system_name' => $coord_system_name,
'coord_system_version' => $coord_system_version, 'author' => $self->author });
return $hashref;
}
......@@ -1001,8 +1002,12 @@ sub lock_refresh_for_DataSet_SequenceSet {
my ($self, $ds, $ss) = @_;
my $response =
$self->_DataSet_SequenceSet_response_content(
$ds, $ss, 'GET', 'get_locks', {'author' => $self->author});
$ds, $ss, 'GET', 'get_locks',
{
'coord_system_name' => $ss->coord_system_name,
'coord_system_version' => $ss->coord_system_version,
'author' => $self->author
});
my @slice_lock = map { Bio::Vega::SliceLock->new_from_json($_) }
@{ $response->{SliceLock} || [] };
......@@ -1025,7 +1030,7 @@ sub fetch_all_SequenceNotes_for_DataSet_SequenceSet {
my $response =
$self->_DataSet_SequenceSet_response_content(
$ds, $ss, 'GET', 'get_sequence_notes', {'author' => $self->author});
$ds, $ss, 'GET', 'get_sequence_notes',{'author' => $self->author});
my %ctgname2notes = ();
......@@ -1097,7 +1102,7 @@ sub _sequence_note_action {
'contig' => $contig_name,
'timestamp' => $seq_note->timestamp(),
'text' => $seq_note->text(),
'author' => $self->author,
'author' => $self->author
},
);
......@@ -1131,6 +1136,15 @@ sub _get_DataSets_hash {
my $datasets_hash = $self->otter_response_content
('GET', 'get_datasets', {'author' => $self->author});
return $datasets_hash;
}
sub fetch_fasta_seqence {
my ($self, $acc) = @_;
my $datasets_hash = $self->otter_response_content
('GET', 'get_fasta_sequence', {'id'=>$acc, 'author' => $self->author});
return $datasets_hash;
}
......@@ -1238,7 +1252,9 @@ sub get_server_ensembl_version {
# same as Bio::Otter::Server::Config->designations (fresh every time)
sub _get_designations {
my ($self) = @_;
my $hashref = $self->otter_response_content(GET => 'get_config', { key => 'designations', 'author' => $self->author });
my $hashref = $self->otter_response_content(GET => 'get_config', { key => 'designations','author' => $self->author });
return $hashref;
}
......@@ -1334,15 +1350,16 @@ sub get_slice_DE {
sub slice_query {
my ($self, $slice) = @_;
die unless wantarray;
return (
'dataset' => $slice->dsname(),
'chr' => $slice->ssname(),
'cs' => $slice->csname(),
'csver' => $slice->csver(),
'name' => $slice->seqname(),
'start' => $slice->start(),
'end' => $slice->end(),
);
return ('dataset' => $slice->dsname(),
'chr' => $slice->ssname(),
'cs' => $slice->csname(),
'csver' => $slice->csver(),
'name' => $slice->seqname(),
'start' => $slice->start(),
'end' => $slice->end(),
);
}
......@@ -1365,9 +1382,7 @@ sub get_all_SequenceSets_for_DataSet {
my $sequencesets_xml =
$self->http_response_content(
'GET', 'get_sequencesets', {
'dataset' => $dataset_name, 'author' => $self->author,
});
'GET', 'get_sequencesets', {'dataset' => $dataset_name, 'author' => $self->author});
local $XML::Simple::PREFERRED_PARSER = 'XML::Parser';
# configure expat for speed, also used in Bio::Vega::XML::Parser
......@@ -1416,6 +1431,9 @@ sub _make_SequenceSet {
$sr_name, [split(/,/, $sr_params->{content})]);
}
}
elsif ($key eq 'coord_system_name' or $key eq 'coord_system_version') {
$sequenceset->$key($value);
}
elsif ($sequenceset->can($key)) {
die "Bad key $key" unless $key =~ /^[_A-Za-z]{1,16}$/;
$sequenceset->$key($value);
......@@ -1433,6 +1451,7 @@ sub get_all_CloneSequences_for_DataSet_SequenceSet { # without any lock info
my $dataset_name = $ds->name;
my $sequenceset_name = $ss->name;
$ds->selected_SequenceSet($ss);
my $clonesequences_xml = $self->http_response_content(
'GET',
......@@ -1440,7 +1459,10 @@ sub get_all_CloneSequences_for_DataSet_SequenceSet { # without any lock info
{
'dataset' => $dataset_name,
'sequenceset' => $sequenceset_name,
'author' => $self->author
'coord_system_name' => $ss->coord_system_name,
'coord_system_version' => $ss->coord_system_version,
'author' => $self->author
}
);
......@@ -1466,8 +1488,7 @@ sub get_all_CloneSequences_for_DataSet_SequenceSet { # without any lock info
$dataset_name, $sequenceset_name, $_);
} @{$clonesequences_array} ];
$ss->CloneSequence_list($clonesequences);
return $clonesequences;
return $clonesequences;
}
sub _make_CloneSequence {
......@@ -1479,6 +1500,9 @@ sub _make_CloneSequence {
if ($key eq 'chr') {
$clonesequence->chromosome($value->{name});
}
elsif ($key eq 'coord_system_name' or $key eq 'coord_system_version') {
$clonesequence->$key($value);
}
elsif ($clonesequence->can($key)) {
die "Bad key $key" unless $key =~ /^[_A-Za-z]{1,16}$/;
$clonesequence->$key($value);
......@@ -1553,7 +1577,7 @@ sub save_otter_xml {
# lock_region, unlock_region : see Bio::Otter::Lace::AceDatabase
sub _DataSet_SequenceSet_response_content {
my ($self, $ds, $ss, $method, $script) = @_;
my ($self, $ds, $ss, $method, $script, $extra) = @_;
my $query = {
'dataset' => $ds->name,
......@@ -1561,6 +1585,9 @@ sub _DataSet_SequenceSet_response_content {
'author' => $self->author
};
if ($extra and ref($extra) eq 'HASH') {
%$query = (%$query, %$extra);
}
my $content =
$self->otter_response_content($method, $script, $query);
......@@ -1795,3 +1822,4 @@ An ordinary constructor, making instances as requested.
=head1 AUTHOR
Ana Code B<email> anacode@sanger.ac.uk
......@@ -31,6 +31,24 @@ sub new {
return bless {}, $pkg;
}
sub coord_system_name {
my ($self, $coord_system_name) = @_;
if ($coord_system_name) {
$self->{'_coord_system_name'} = $coord_system_name;
}
return $self->{'_coord_system_name'};
}
sub coord_system_version {
my ($self, $coord_system_version) = @_;
if ($coord_system_version) {
$self->{'_coord_system_version'} = $coord_system_version;
}
return $self->{'_coord_system_version'};
}
sub accession {
my ($self, $accession) = @_;
......
......@@ -298,6 +298,7 @@ sub load_dataset_info {
my $dbh = $dbh{$self};
my $select_sth = $dbh->prepare(q{ SELECT species_id, meta_key, meta_value FROM meta WHERE species_id = ? AND meta_key = ? AND meta_value = ? });
my $meta_sth = $dbh->prepare(q{ INSERT INTO meta (species_id, meta_key, meta_value) VALUES (?, ?, ?) });
my $meta_hash = $dataset->meta_hash;
......@@ -307,15 +308,6 @@ sub load_dataset_info {
# I'm not really sure we need to do this - we could just use a local version
#
my $cs_chr = $dataset->get_db_info_item('coord_system.chromosome');
my $local_cs_spec = {
$cs_chr->{name} => {
'-version' => $cs_chr->{version},
'-rank' => $cs_chr->{rank},
'-default' => $cs_chr =~ m/default_version/,
'-sequence_level' => $cs_chr =~ m/sequence_level/,
},
};
my @at_cols = qw( attrib_type_id code name description );
my $at_sth = $dbh->prepare(q{ INSERT INTO attrib_type (attrib_type_id, code, name, description)
......@@ -323,30 +315,44 @@ sub load_dataset_info {
my $at_list = $dataset->get_db_info_item('attrib_type');
my $_dba = $self->_dba('_coords'); # we throw this one away
my $override_specs = $dataset->get_db_info_item('coord_systems');
my $dna_cs_rank;
foreach my $value (values %$override_specs) {
if ($value->{'-sequence_level'}) {
$value->{'-sequence_level'} = 0;
$dna_cs_rank = $value->{'-rank'}+1;
}
}
my $cs_factory = Bio::Vega::CoordSystemFactory->new(
dba => $_dba,
create_in_db => 1,
override_spec => $local_cs_spec,
override_spec => $override_specs,
);
my $toplevel_cs = $dataset->get_db_info_item('coord_system.chromosome');
my %local_meta = (
'assembly.mapping' => {
species_id => 1,
values => [
$cs_factory->assembly_mappings,
],
},
);
$override_specs->{dna_contig} = {
'-rank' => $dna_cs_rank,
'-sequence_level' => 1,
'-default' => 1,
'version' => $toplevel_cs->{version},
};
$meta_hash->{'assembly.mapping'}->{species_id} = $meta_hash->{'species.classification'}->{species_id};
push(@{$meta_hash->{'assembly.mapping'}->{values}}, $toplevel_cs->{name}.':'.$toplevel_cs->{version}.'|dna_contig:'.$toplevel_cs->{version});
$dbh->begin_work;
# Meta first, so that CoordSystemAdaptor doesn't complain about missing schema_version
#
while (my ($key, $details) = each %$meta_hash) {
next if $key eq 'assembly.mapping'; # we do our own mapping, below...
foreach my $value (@{$details->{values}}) {
$meta_sth->execute($details->{species_id}, $key, $value);
$select_sth->execute($details->{species_id}, $key, $value);
if($select_sth->fetchrow_array) {
next;
}
else {
$meta_sth->execute($details->{species_id}, $key, $value);
}
}
}
......@@ -358,22 +364,14 @@ sub load_dataset_info {
$dbh->commit;
$dbh->begin_work;
# Coord systems via factory
#
$cs_factory->known;
$cs_factory->instantiate_all;
# Mappings
#
my $mca = $_dba->get_MetaContainer;
while (my ($key, $details) = each %local_meta) {
foreach my $value (@{$details->{values}}) {
$mca->store_key_value($key, $value);
}
}
$dbh->commit;
$self->_is_loaded('dataset_info', 1);
......
......@@ -481,8 +481,14 @@ sub get_meta_value {
sub db_info_hash {
my ($self) = @_;
# Get all db_info in one call
my $cs_name = 'chromosome';
my $cs_version = 'Otter';
if ($self->selected_SequenceSet) {
$cs_name = $self->selected_SequenceSet->coord_system_name;
$cs_version = $self->selected_SequenceSet->coord_system_version;
}
return $self->{'_db_info_hash'} ||=
$self->Client->get_db_info($self->name);
$self->Client->get_db_info($self->name, $cs_name, $cs_version);
}
sub get_db_info_item {
......@@ -682,7 +688,7 @@ sub get_cached_DBAdaptor {
$self->_attach_DNA_DBAdaptor($tmp) if $self->DNA_DBNAME;
$self->{'_dba_cache'} = $tmp;
}
#warn "OTTER DBADAPTOR = '$dba'";
return $self->{'_dba_cache'};
}
......
......@@ -685,8 +685,9 @@ __DATA__
# e.g. short_window_title_prefix and password_timeout accept -1 to
# disable.
[client]
url=http://193.62.52.185:82/cgi-bin/otter
url=http://otter-server.ebi.ac.uk:8004/cgi-bin/otter
write_access=1
gff_version=3
acedb_version=4.9.61
......
......@@ -16,6 +16,8 @@ limitations under the License.
=cut
### Bio::Otter::Lace::OnTheFly::QueryValidator
package Bio::Otter::Lace::OnTheFly::QueryValidator;
use namespace::autoclean;
......@@ -236,41 +238,68 @@ sub Client {
# Adds sequences to $self->seqs
#
sub _fetch_sequences {
my ($self, @to_fetch) = @_;
my $cache = $self->accession_type_cache;
@to_fetch = uniq @to_fetch;
$self->logger->debug('Need seq for: ', join(',', @to_fetch) || '<none>');
my $client = Bio::Otter::Lace::Defaults::make_Client();
my $seqs = $client->fetch_fasta_sequence(@to_fetch);
my @seq_array = split('>', $seqs);
foreach my $sequence (@seq_array) {
if (length($sequence) < 2) {
next;
};
my $seq = $self->parse_fasta_sequence('>' . $sequence);
my $iteration = 0;
my @to_fetch_cut = @to_fetch;
@to_fetch_cut = map { m/\-/ && s/\.\d+$//; $_ } @to_fetch_cut;
while ((index $seq->name, $to_fetch_cut[$iteration]) == -1 && $iteration < scalar(@to_fetch)) {
$iteration++;
}
my ($self, @to_fetch) = @_;
my $cache = $self->accession_type_cache;
@to_fetch = uniq @to_fetch;
$self->logger->debug('Need seq for: ', join(',', @to_fetch) || '<none>');
my $client = Bio::Otter::Lace::Defaults::make_Client();
foreach my $acc (@to_fetch) {
my $seq = $client->fetch_fasta_seqence($acc);
if (substr($seq, 0, 1) eq ">") {
$seq = $self->parse_fasta_sequence($seq);
push(@{$self->seqs}, $seq);
my ($type, $full) = @{$self->_acc_type_full($acc)};
unless ($type) {
$self->_add_missing_warning($acc => 'illegal evidence type');
next;
}
$seq->type($type);
$seq->name($acc);
} else {
$self->_add_missing_warning($acc, "unknown accession or illegal evidence type");
}
my ($type, $full) = @{$self->_acc_type_full($to_fetch[$iteration])};
unless ($type) {
$self->_add_missing_warning($to_fetch[$iteration] => 'illegal evidence type');
next;
}
$seq->type($type);
$seq->name($full);
push(@{$self->seqs}, $seq);
}
# my ($type, $full) = @{$self->_acc_type_full($acc)};
# unless ($type) {
# $self->_add_missing_warning($acc => 'illegal evidence type');
# next;
# }
# my $info = $cache->feature_accession_info($acc);
# unless ($info) {
# $self->logger->error("No info for '$acc' - this should not happen");
# $self->_add_missing_warning($acc => 'internal error');
# next;