CollectionAdaptor.pm 7.93 KB
Newer Older
1 2
=head1 LICENSE

3
Copyright [1999-2013] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
4

5 6 7 8 9 10 11 12 13 14 15 16 17
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

=cut
18 19 20 21 22


=head1 CONTACT

  Please email comments or questions to the public Ensembl
Magali Ruffier's avatar
Magali Ruffier committed
23
  developers list at <http://lists.ensembl.org/mailman/listinfo/dev>.
24 25

  Questions may also be sent to the Ensembl help desk at
Magali Ruffier's avatar
Magali Ruffier committed
26
  <http://www.ensembl.org/Help/Contact>.
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73

=cut

=head1 NAME

Bio::EnsEMBL::DBFile::CollectionAdaptor

=head1 SYNOPSIS

For use with a Bio::EnsEMBL::Collector e.g.

    package Bio::EnsEMBL::Funcgen::DBSQL::ResultFeatureAdaptor;

    @ISA = qw(Bio::EnsEMBL::Funcgen::DBSQL::BaseFeatureAdaptor 
              Bio::EnsEMBL::Funcgen::Collector::ResultFeature 
              Bio::EnsEMBL::DBFile::CollectionAdaptor);
    #DBSQL and DBFile inheritance here due to dynamic nature of ResultFeatureAdaptor


Fetch wrapper methods access file based data via read_collection_blob:

    sub _fetch_from_file_by_Slice_ResultSet{

	    #define filepath/config

        my $packed_scores =  $self->read_collection_blob(
		    										   $filepath,
			    									   $efg_sr_id,
				    								   $conf->{$window_size}{'byte_offset'},
					    							   $conf->{$window_size}{'byte_length'},
						    						  );

        #Do unpacking and object creation here

    }

=head1 DESCRIPTION

Adaptor for direct collection(.col) file access, which are binary compressed fixed 
width format files providing window based values across the genome. Collection files
integrate an index block which contains seq_region byte off set values.

NOTE: By default all collection files are generated and packed using little endian encoding. 
Due to the lack of standards of float encoding(wrt to endianess) perl packs using the 
implicit endianess of the underlying architecture. This means that accessing float
collection files located on a big endian architecture will produce unexpected results.

74
# endian issues will disappear with knetfile xsubs
75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169

=head1 SEE ALSO

Bio::EnsEMBL::DBFile::FileAdaptor

=cut



package Bio::EnsEMBL::DBFile::CollectionAdaptor;

use strict;
use warnings;

use Bio::EnsEMBL::DBFile::FileAdaptor;
use Bio::EnsEMBL::Utils::Exception qw(throw warning deprecate);
use vars qw(@ISA);
@ISA = qw(Bio::EnsEMBL::DBFile::FileAdaptor);


=head2 initialise_filehandle

  Arg[1]     : string  - filepath
  Example    : $self->initialise_filehandle($filepath);
  Description: Initialises the filehandle for use, in this case reads 
               the index (seq_region offsets)
  Returntype : None
  Exceptions : warns if read fails
  Caller     : Bio::EnsEMBL::DBFile::FileAdaptor::get_filehandle
  Status     : at risk

=cut

sub initialise_filehandle{
  my ($self, $filepath) = @_;
  my $fh = $self->{file_cache}{$filepath}{filehandle};
  
  #offsets include the length of the complete index block
  my ($index_size, $read_bytes, $index, $num_keys, %offset_index);
  
  ### INDEX FORMAT ###
  #First block of the index the index size in bytes(not inc size block).
  #
  #Rest of index is a hash of sr_id(v 2 bytes) key offset(V 4 bytes) value pairs
  #V (long) is 4 bytes(via sys/read), which is actually an Config{intsize} i.e. i? 
  #long is 8 bytes according to Config{longsize}!

  #read uses logical characters not necessarily in bytes
  #altho this does seem to read bytes, maybe due to binmode?
  #seek is in bytes
  #Changed to sysread/read which both use bytes explicitly
  #Can't mix sysread/seek due to I/O buffering differences

  
  #Read index_size first encoded as v(2 bytes)
  $read_bytes = sysread($fh, $index_size, 2);
    
  if(! ((defined $read_bytes) && ($read_bytes == 2))){
	#! defined is error 0 is end of file
	warn "Failed to read index size from $filepath\n$!";

	#Delete fh as it is useless/unsafe to retry
	undef $self->{file_cache}{$filepath}{filehandle};
  }
  else{	#Read index
	($index_size) = unpack('v', $index_size);
	$read_bytes = sysread($fh, $index, $index_size);  #Now read index proper
	
	if(! ((defined $read_bytes) && ($read_bytes == $index_size))){
	  #! defined is error 0 is end of file
	  warn "Failed to read index from $filepath\n$!";

	  #Delete fh as it is useless/unsafe to retry
	  undef $self->{file_cache}{$filepath}{filehandle};
	}
	else{
	  #Number of key-value pairs => $index_size /(size of key(v 2bytes) + size of offset(V 4bytes))
	  $num_keys        = $index_size/6;
	  my $unpack_template = '(vV)'.$num_keys,;
	  
	  %offset_index = unpack($unpack_template, $index);
	  $self->{file_cache}{$filepath}{off_sets} = \%offset_index;
	}
  }

  return $self->{file_cache}{$filepath}{off_sets};
}


=head2 read_collection_blob

  Arg[1]     : string - filepath
  Arg[2]     : int    - seq_region_id
  Arg[3]     : int    - seq_region offset. The byte offset required to
                        locate the required start position
Kieron Taylor's avatar
Kieron Taylor committed
170
  Arg[4]     : int    - byte length to read
171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252
  Example    : my $blob_substr = $self->read_collection_blob($filepath,
                                                             $sr_key,
                                                             $sr_offset,
                                                             $byte_length);
  Description: Reads bytes from file given a seq_region_key, byte offset and byte length.
               Sets filehandle to undef if read fails.
  Returntype : string - packed binary data
  Exceptions : warns if seek or read errors
  Caller     : general e.g. fetch_from_file_by_Slice_ResultSet
  Status     : at risk

=cut

# We could change this to take a Slice, hence we could check 
# whether an EOF error is because the slice is out of range 
# and undef only if it is in range i.e. the index/file is corrupt
# overkill?
# This is something the Slice API should warn about
# but will still cause undef'd filehandle here
# Index should also contain ends, so we can validate whether the slice is out of range???


sub read_collection_blob{
  my($self, $filepath, $sr_key, $sr_offset, $byte_length) = @_;
	
  my $blob_substr;
  my $fh = $self->get_filehandle($filepath, {-binmode => 1});

  if(defined $fh){
	#Return from query cache here?
	#cache key = "$filepath:$key:$sr_offset:$byte_length"

	#define total offset

	#if(! exists $self->{file_cache}{$filepath}{off_sets}{$sr_key}){
	#  #warn "sr_key($sr_key) is not part of index for $filepath\n";
	#}
	#else{

	if(exists $self->{file_cache}{$filepath}{off_sets}{$sr_key}){

 	  my $total_offset = $self->{file_cache}{$filepath}{off_sets}{$sr_key} + $sr_offset;
	  my $seeked = sysseek($fh, $total_offset, 0);#0(whence) is SEEK_SET.

	  if(! $seeked){
		warn("Failed to seek to byte $total_offset in $filepath");
		#Don't undef fh here as this valid Slice maybe out of range
		#and we don't want to kill a valid fh
		#i.e. Slice start/end is past end of seq_region
	  }
	  else{
		my $read_bytes = sysread($fh, $blob_substr, $byte_length);
		
		if(! ((defined $read_bytes) && ($read_bytes == $byte_length))){
		  #! defined is error 0 is end of file
		  warn "Failed to read from $filepath\n$!";

		  if($read_bytes == 0){
			#This maybe because the slice is out of range!
			#The API gives no warning about this
						
			warn "End Of File encountered\n";
			warn "Total offset:\t".$self->{file_cache}{$filepath}{off_sets}{$sr_key}.
			  "  key($sr_key)  + $sr_offset = $total_offset\n";

			#add some checks against the theoretical/true length of the file?
		  }
		  else{  #Delete fh as it is useless/unsafe to retry
			undef $self->{file_cache}{$filepath}{filehandle};
			#$blob_substr is now set to empty string by read
			undef $blob_substr;
		  }
		}		
	  }
	}	
  }

  return $blob_substr;
}


1;