moving from ensembl-personal

misc-scripts/cloud/EnsCloud/Cmd.pm

+package EnsCloud::Cmd;
+use Moose;
+use MooseX::StrictConstructor;
+extends qw(MooseX::App::Cmd);
+__PACKAGE__->meta->make_immutable;
+
+1;
+

misc-scripts/cloud/EnsCloud/Cmd/Command/SpeciesToVolumes.pm

+package EnsCloud::Cmd::Command::SpeciesToVolumes;
+use Moose;
+use autodie qw(:all);
+use Data::Dump qw(dump);
+use Data::Dumper;
+use File::Spec;
+use Moose::Util::TypeConstraints;
+use EnsCloud::Image;
+use EnsCloud::Image::VolumeBundle;
+use EnsCloud::Image::VolumeBundle::Volume;
+use EnsCloud::Image::VolumeBundle::Volume::DatabaseDetails;
+use namespace::autoclean;
+use IPC::Cmd qw[can_run run];
+
+# use List::Utils qw(first);
+use Log::Log4perl qw(:easy);
+with 'MooseX::Log::Log4perl';
+extends qw(MooseX::App::Cmd::Command);
+
+BEGIN {
+    Log::Log4perl->easy_init();
+}
+
+# ABSTRACT: list the application's commands
+sub abstract {
+
+	return "build ensembl MySQL instances by species";
+
+    }
+
+
+# Instructions
+#
+# * This expects a volume containing the Ensembl MYDs attached to the instance (Public Snapshot of 65 dbs = snap-56c9ab32)
+# - default path is /vols/ensembl_mysql_data
+# Run like this: ( N.B --base_snapshot is the base AMI with the OS)
+#
+# ecloud speciestovolumes   --base_snapshot snap-e36fde86  --species saccharomyces_cerevisiae (omit --species to do all)
+# or by dbtype
+# ecloud speciestovolumes   --base_snapshot snap-e36fde86  --species homo_sapiens --dbtype variation
+# 
+# * Let it run until completion.  then wait until all PENDING snapshots are complete.
+# Once all PENDING snapshots are complete you will have one snapshot per species
+# (Do not start process again while there are are still PENDING snapshots) 
+# Now run it again with the same parameters - this will loop the completed snapshots to combine each with the base_image OS 
+
+# has 'type' => (
+#     is     => 'rw',
+#     isa    => 'Str',
+#     traits => ['Getopt'],
+#
+#     #     cmd_aliases   => "h",
+#     documentation => "instances volumes or images",
+#     required      => 1,
+#
+#     #     default       => sub { die "bucket name required" },
+# );
+
+has 'region_alias' => (
+    is     => 'rw',
+    isa    => 'Str',
+    traits => ['Getopt'],
+
+    #     cmd_aliases   => "h",
+    documentation => "asia, useast, uswest or eu",
+    required      => 1,
+    default       => 'useast',
+);
+
+has 'volume_path' => (
+
+    is     => 'ro',
+    isa    => 'Str',
+    traits => ['Getopt'],
+
+    #     cmd_aliases   => "h",
+    documentation => "path to the species DBs",
+    required      => 1,
+    default       => '/vols/ensembl_mysql_data',
+
+);
+
+enum 'CompositeGroup' => qw(core coreplusvariation corepluscompara variation compara all);
+has 'dbtypes'         => (
+
+    is     => 'ro',
+    isa    => 'CompositeGroup',
+    traits => ['Getopt'],
+
+    #     cmd_aliases   => "h",
+    documentation => "database types to copy: default=core",
+    required      => 1,
+    default       => 'all',
+
+);
+
+has db_type_lookup => (
+    is       => 'ro',
+    isa      => 'HashRef',
+    required => 1,
+    default  => sub {
+        {
+            core => [qw(core coreexpressionatlas coreexpressionest funcgen otherfeatures rnaseq vega )],
+            coreplusvariation =>
+              [qw(core coreexpressionatlas coreexpressionest funcgen otherfeatures rnaseq vega variation)],
+            corepluscompara =>
+              [qw(core coreexpressionatlas coreexpressionest funcgen otherfeatures rnaseq vega compara )],
+            all =>
+              [qw(core coreexpressionatlas coreexpressionest funcgen otherfeatures rnaseq vega compara variation )],
+            variation => [qw(variation)],
+            compara   => [qw(compara)],
+        };
+    },
+
+);
+
+has 'species' => (
+    is     => 'ro',
+    isa    => 'Str',
+    traits => ['Getopt'],
+
+    #     cmd_aliases   => "h",
+    documentation => "the species to copy. default=all",
+    required      => 1,
+    default       => 'all',
+
+);
+
+has 'start_from' => (
+    is     => 'ro',
+    isa    => 'Str',
+    traits => ['Getopt'],
+
+    #     cmd_aliases   => "h",
+    documentation => "the species to start dumping from",
+    default       => '.*'
+);
+
+has 'end_at' => (
+    is     => 'ro',
+    isa    => 'Str',
+    traits => ['Getopt'],
+
+    #     cmd_aliases   => "h",
+    documentation => "the species to stop dumping at",
+    default       => '^$'
+);
+has 'volume_dump_queue' => (
+    traits  => ['Array'],
+    handles => {
+        all_queued_volumes      => 'elements',
+        volume_add_to_queue     => 'push',
+        next_volume_from_queue  => 'shift',
+        sort_queue_by_size_desc => [ sort_in_place => ( sub { $_[1]->total_size <=> $_[0]->total_size } ) ],
+        find_tag                => 'first',
+        queue_length            => 'count',
+        filter_queue            => 'grep'
+    },
+    isa => 'ArrayRef[EnsCloud::Image::VolumeBundle::Volume]',
+);
+
+has db_species_details => (
+    is => 'rw',
+
+    #isa => 'HashRef',
+    isa     => 'HashRef[EnsCloud:::DatabaseDetails]',
+    default => sub { {} }
+);
+
+has this_instance_id => (
+    is       => 'ro',
+    isa      => 'Str',
+    default  => sub { return `curl -s 169.254.169.254/latest/meta-data/instance-id` },
+    required => 1
+
+);
+
+has myd_destination_folder => (
+    is       => 'ro',
+    isa      => 'Str',
+    default  => '/vols/MYDCOPY',
+    required => 1
+
+);
+has device => (
+    is       => 'ro',
+    isa      => 'Str',
+    default  => '/dev/sdi',
+    required => 1
+
+);
+has base_snapshot => (
+    is       => 'ro',
+    isa      => 'Str',
+    required => 1
+
+);
+has this_zone => (
+    is       => 'ro',
+    isa      => 'Str',
+    default  => sub { return `curl -s 169.254.169.254/latest/meta-data/placement/availability-zone/` },
+    required => 1
+
+);
+with 'EnsCloud::Describer';
+
+sub execute {
+    my ($self) = @_;
+    $self->build_queue();
+
+    $self->log->fatal("No databases in the queue");
+    my @image_list = ();
+
+    $self->sort_queue_by_size_desc;
+
+    foreach my $volume ( $self->all_queued_volumes ) {
+        $self->log->info( "Doing: " . $volume->species );
+        $self->update_queue;
+        if ( $volume->snapshot_id && $volume->status eq 'completed' ) {
+
+            #  $self->log->info($volume->snapshot_id): $self->log->warn("No snapshot for: " . $volume->tag);
+            my $image = $self->make_image($volume);
+            push @image_list, $image;
+        }
+        else {
+            $self->make_snapshot($volume);
+        }
+
+    }
+    $self->update_queue;
+    my $image_count = @image_list > 0 ? scalar @image_list : 0;
+    $self->log->info( "Finished, made ", $self->queue_length, " Volumes" );
+    $self->log->info("Finished, made $image_count Images");
+    my @no_snapshots = $self->filter_queue( sub { $_->status =~ /no snapshot/ } );
+    $self->log->info( scalar @no_snapshots . " Snapshots still waiting" );
+}
+
+sub update_queue {
+    my ($self) = @_;
+
+    my $ec2_snapshots = $self->ec2->describe_snapshots( Owner => 'self' );
+    foreach my $ec2_snap (@$ec2_snapshots) {
+        my $description = $ec2_snap->{description} || next;
+        $description =~ s/\s+$//;
+        if ( my $has_snapshot = $self->find_tag( sub { $_->tag eq $description } ) ) {
+            $has_snapshot->snapshot_id( $ec2_snap->{snapshot_id} );
+            $has_snapshot->status( $ec2_snap->{status} );
+        }
+
+    }
+
+}
+
+sub make_image {
+    my ( $self, $snapshot ) = @_;
+
+    $self->log->debug("MAKE IMAGE");
+
+    my $tag =
+        "Ensembl"
+      . $snapshot->ensembl_release . " "
+      . $snapshot->species . " ["
+      . ( join " ", map { $_->type } $snapshot->all_dbs ) . "]";
+    my $name = "Ensembl" . $snapshot->ensembl_release . " " . $snapshot->species . " MySQL AMI";
+
+    # has it already been created
+    my $existing_images = $self->ec2->describe_images( Owner => 'self' );
+    foreach my $i (@$existing_images) {
+        if ( $i->{description} eq $tag ) {
+            $self->log->warn( "Image " . $i->{image_id} . " already exists for " . $tag );
+            return undef;
+        }
+    }
+    my $create_image = $self->ec2->register_image(
+        Name               => $name,
+        Description        => $tag,
+        Architecture       => 'x86_64',
+        KernelId           => 'aki-427d952b',
+        RootDeviceName     => '/dev/sda1',
+        BlockDeviceMapping => [
+            {
+                deviceName => '/dev/sda1',
+                ebs        => {
+                    snapshotId          => $self->base_snapshot,
+                    deleteOnTermination => 'true'
+                }
+            },
+            {
+                deviceName => '/dev/sdh',
+                ebs        => { snapshotId => $snapshot->{snapshot_id}, deleteOnTermination => 'true' }
+            },
+        ]
+    );
+    if ( $create_image->can('errors') ) {
+        $self->log->error( "[Error Creating Image] " . $self->pp_ec2_errors( $create_image->errors ) );
+        return;
+    }
+    $self->log->info("Created Image $create_image for $tag");
+    return $create_image;
+}
+
+sub make_snapshot {
+    my ( $self, $bag_of_dbs ) = @_;
+    my $size_as_float = $bag_of_dbs->total_size;
+    $self->log->debug( "Making SNAPSHOT " . $bag_of_dbs->tag . " Current status=" . $bag_of_dbs->status );
+
+    # round up volumesize to nearest Gb +1
+    # rounding to nearest Gig doesn't seem to enough
+    my $round_up_size = int( $size_as_float + 2 );
+
+    #$self->clear_ec2;
+    my $volume = $self->ec2->create_volume( Size => $round_up_size, AvailabilityZone => $self->this_zone );
+
+    if ( $volume->can('errors') ) {
+        $self->log->error( "[Error Creating Volume] " . $self->pp_ec2_errors( $volume->errors ) );
+        return;
+    }
+    else {
+        $self->log->info( "Created ", $volume->volume_id, " ", $volume->size, " Gb" );
+    }
+    my $do_attach = $self->ec2->attach_volume(
+        VolumeId   => $volume->volume_id,
+        InstanceId => $self->this_instance_id,
+        Device     => $self->device
+    );
+    $self->log->info( "Attaching " . $volume->volume_id . " Device " . $self->device );
+    if ( $do_attach->can('errors') ) {
+        $self->log->error( "[Error attaching Volume] " . $self->pp_ec2_errors( $do_attach->errors ) );
+        $self->log->info( "Deleting " . $volume->volume_id );
+        $self->ec2->delete_volume( VolumeId => $volume->volume_id );
+        return;
+    }
+    else {
+        my $wait_time     = 0;
+        my $attach_status = '';
+        while ( $attach_status ne 'attached' && $wait_time < 30 ) {
+
+            $self->log->info("Waiting 10 seconds for volume to become available");
+            sleep 10;
+            $wait_time += 10;
+            $attach_status =
+              $self->ec2->describe_volumes( VolumeId => $volume->volume_id )->[0]->attachments->[0]->{status};
+            $self->log->info("Volume $attach_status");
+        }
+        unless ( $attach_status eq 'attached' ) {
+            $self->log->error("ERROR ATTACHING VOLUME AFTER 30 seconds... DETACHING AND DELETING");
+            $self->clean_up_by_volume($volume);
+
+            # todo, check for errors in the delete call
+            $self->log->fatal("COULD NOT ATTACH VOLUME. THIS IS BAD. EXITING") && die;
+        }
+
+    }
+
+    # Make filesystem on the new volume
+    $self->log->info( "Making filesystem on " . $self->device );
+    my $mkfs_path = can_run('mkfs.xfs') or $self->log->warn('mkfs.xfs not installed!');
+    my $mkfs_cmd = [ 'sudo', $mkfs_path, $self->device ];
+    unless ( $self->run_command($mkfs_cmd) ) {
+        $self->log->info("Cleaning up after failed mkfs");
+        $self->clean_up_by_volume($volume);
+        return;
+
+    }
+
+    # Make the directory
+    my $mkdir_cmd = [ 'sudo', 'mkdir', '-p', $self->myd_destination_folder ];
+    unless ( $self->run_command($mkdir_cmd) ) {
+        $self->log->info("Cleaning up after failed mkdir");
+        $self->clean_up_by_volume($volume);
+        return;
+
+    }
+
+    # Mount new volume to it
+    $self->log->info( "Mounting " . $self->device );
+    my $mount_path = can_run('mount') or $self->log->warn('mount not installed!');
+    my $mount_cmd = [ 'sudo', $mount_path, $self->device, $self->myd_destination_folder ];
+    unless ( $self->run_command($mount_cmd) ) {
+        $self->log->info("Cleaning up after failed mount");
+        $self->clean_up_by_volume($volume);
+        return;
+    }
+
+    # Copy the each MYD dir
+    foreach my $myd_dir ( $bag_of_dbs->all_dbs ) {
+        my $copy_cmd = [ 'sudo', 'cp', '-r', $myd_dir->myd_path, $self->myd_destination_folder ];
+        $self->log->debug( join " ", @$copy_cmd );
+        unless ( $self->run_command($copy_cmd) ) {
+
+            $self->log->info("Cleaning up after failed copy");
+            my $umount_path = can_run('umount') or $self->log->warn('umount not installed!');
+            my $umount_cmd = [ 'sudo', $umount_path, $self->device ];
+            unless ( $self->run_command($umount_cmd) ) {
+                $self->log->fatal( "CANNOT UMOUNT " . $self->device . "EXITING" ) && die;
+            }
+            $self->clean_up_by_volume($volume);
+            return;
+
+        }
+        $myd_dir->is_copied(1);
+
+        #         $snapshot_description .= $myd_dir->name . " ";
+    }
+
+    $self->log->info( "umounting " . $self->device );
+    my $umount_path = can_run('umount') or $self->log->warn('umount not installed!');
+    my $umount_cmd = [ 'sudo', $umount_path, $self->device ];
+    return unless $self->run_command($umount_cmd);
+
+    my $wait_time = 0;
+    $self->ec2->detach_volume( VolumeId => $volume->volume_id );
+    while ( defined eval { $self->ec2->describe_volumes( VolumeId => $volume->volume_id )->[0]->attachments }
+        && $wait_time < 60 )
+    {
+        $self->log->info("Waiting 10 seconds for volume to detach");
+        sleep 10;
+        $wait_time += 10;
+    }
+
+    my $snapshot_description = $bag_of_dbs->{tag};
+    $self->log->info("Creating Snapshot");
+    my $snapshot = $self->ec2->create_snapshot( VolumeId => $volume->volume_id, Description => $snapshot_description );
+
+    if ( $snapshot->can('errors') ) {
+        $self->log->error( "[Snapshot Creation Error] " . $self->pp_ec2_errors( $snapshot->errors ) );
+        return;
+    }
+    else {
+        $self->log->info( "Created Snapshot ", $snapshot->snapshot_id, " ", $volume->size, " Gb" );
+        $self->log->info( "Deleting " . $volume->volume_id );
+        $self->ec2->delete_volume( VolumeId => $volume->volume_id );
+        $self->log->info("Tagging the Snapshot");
+        my $tag_path = can_run('ec2-create-tags') or $self->log->warn('ec2-describe-tags not found');
+        my $tag_cmd = [ $tag_path, $snapshot->snapshot_id, '-t', "Name=$snapshot_description" ];
+        return unless $self->run_command($tag_cmd);
+
+        # $self->log->info("Detaching Temp Volume");
+        # $self->ec2->detach_volume(VolumeId => $volume->volume_id);
+        # $self->log->info("Deleting Temp Volume");
+        # $self->ec2->delete_volume(VolumeId => $volume->volume_id);
+
+    }
+
+}
+
+sub clean_up_by_volume {
+    my ( $self, $volume ) = @_;
+
+    #detach volume
+    $self->log->info( "CLEANUP: Detaching " . $volume->volume_id );
+    my $wait_time = 0;
+    my $do_detach = $self->ec2->detach_volume( VolumeId => $volume->volume_id );
+
+    if ( $do_detach->can('errors') ) {
+        $self->log->error( "[Error detaching Volume] " . $self->pp_ec2_errors( $do_detach->errors ) );
+
+        $self->log->fatal( "CANNOT DETACH VOLUME DURING CLEANUP. THIS IS BAD. EXITING :" . $volume->volume_id )
+          && die;
+    }
+
+    while (
+        defined eval { $self->ec2->describe_volumes( VolumeId => $volume->volume_id )->[0]->attachments }
+
+        && $wait_time < 60
+      )
+    {
+
+        #                 $self->log->info("Volume $attach_status");
+
+        $self->log->info("Waiting 10 seconds for volume to detach");
+        sleep 10;
+        $wait_time += 10;
+    }
+
+    # delete it
+    $self->log->info( "CLEANUP: Deleting " . $volume->volume_id );
+    $self->ec2->delete_volume( VolumeId => $volume->volume_id );
+    return;
+
+}
+
+sub pp_ec2_errors {
+    my ( $self, $error_obj ) = @_;
+    my $full_message;
+    foreach my $error (@$error_obj) {
+        $full_message .= $error->message . "\n";
+
+    }
+    return $full_message;
+}
+
+sub run_command {
+    my ( $self, $cmd ) = @_;
+
+    ### in list context ###
+    my ( $success, $error_message, $full_buf, $stdout_buf, $stderr_buf ) = run( command => $cmd, verbose => 0 );
+
+    if ($success) {
+
+        $self->log->info( join " ", @$cmd, " Successful" );
+
+        #        return $stdout_buf;
+        #         print " is what the command printed:\n";
+        my $stdout_str = join "", @$stdout_buf;
+        return length $stdout_str > 0 ? $stdout_str : $success;
+    }
+    else {
+
+        $self->log->error( "Failed!!: $error_message\n" . ( join "", @$stderr_buf ) );
+        return 0;
+    }
+
+}
+
+sub build_queue {
+    my ($self) = @_;
+
+    my $db_list;
+    my @du = ( "du", $self->volume_path, );
+
+    # Get a list of files
+    $self->log->info("Getting Directory Sizes");
+    my $du_path = can_run('du') or $self->log->warn('du not installed!');
+    my $du_command = [ 'du', $self->volume_path, '|', 'sort', '-n', '-k', '1' ];
+    my $du_output = $self->run_command($du_command);
+
+    #     my @dir_size = qx{@du};
+    my $compara_db;
+    my $species_filter = $self->species eq 'all' ? '.*' : $self->species;
+    my $start_from_filter = $self->start_from ? $self->start_from : '.*';
+    my $dbgroup_to_dbnames = {
+        core => [qw(core coreexpressionatlas coreexpressionest funcgen otherfeatures rnaseq vega )],
+        coreplusvariation =>
+          [qw(core coreexpressionatlas coreexpressionest funcgen otherfeatures rnaseq vega variation)],
+        corepluscompara => [qw(core coreexpressionatlas coreexpressionest funcgen otherfeatures rnaseq vega compara )],
+        all => [qw(core coreexpressionatlas coreexpressionest funcgen otherfeatures rnaseq vega compara variation )],
+        variation => [qw(variation)],
+        compara   => [qw(compara)],
+
+    };
+    my $wanted_dbs = $dbgroup_to_dbnames->{ $self->dbtypes };
+
+    my @lines;
+    my ( $start, $end ) = ( $self->start_from, $self->end_at );
+    map { push @lines, $_ if ( /$start/ .. /$end/ ) } split "\n", $du_output;
+    $self->log->fatal( "Nothing found at " . $self->volume_path ) && die if @lines == 0;
+
+    my $db_hash;
+
+    foreach my $du_line (@lines) {
+        my ( $db_size, $db_path ) = split /\s+/, $du_line;
+        my ( $volume, $directories, $db_name ) = File::Spec->splitpath($db_path);
+        my ( $db_species, $db_release, $db_type );
+        if ( ( $db_species, $db_type, $db_release ) = $db_name =~ /^([a-z]+_[a-z]+)_([a-z]+)_(\d+)_\w+$/ ) {
+            next unless $db_species =~ /$species_filter/;
+            next unless $db_type ~~ @$wanted_dbs;
+            push @{ $db_hash->{$db_species} }, [ $db_size, $db_path, $db_type, $db_release, $db_name ];
+        }
+        elsif ( $db_name =~ /ensembl_compara_(\d+)/ && 'compara' ~~ @$wanted_dbs ) {
+            push @{ $db_hash->{$db_name} }, [ $db_size, $db_path, 'compara', $1, $db_name ];
+        }
+    }
+    foreach my $species ( keys %$db_hash ) {
+
+        my $dbs = $db_hash->{$species};
+        my $volume = EnsCloud::Image::VolumeBundle::Volume->new( species => $species );
+        foreach my $db (@$dbs) {
+            my $db_detail = EnsCloud::Image::VolumeBundle::Volume::DatabaseDetails->new(
+                myd_path => $db->[1],
+                name     => $db->[4],
+                type     => $db->[2],
+                size     => $db->[0] / 1024 / 1024
+
+            );
+            $volume->add_db($db_detail);
+            $volume->add_size( $db_detail->size );
+            my $tag = "e$db->[3] MYD $species [" . ( join " ", map { $_->type } $volume->all_dbs ) . "]";
+
+            #             my $tag = join " ", map { $_->name } $volume->all_dbs;
+            $volume->tag($tag);
+            $volume->ensembl_release( $db->[3] );
+        }
+        $volume->sort_in_place_curried;
+        $self->volume_add_to_queue($volume);
+    }
+    return;
+}
+
+__PACKAGE__->meta->make_immutable;
+
+1;
+__END__
+
+
+

misc-scripts/cloud/EnsCloud/Cmd/Command/list.pm

+package EnsCloud::Cmd::Command::list;
+use Moose;
+use MooseX::StrictConstructor;
+
+extends qw(MooseX::App::Cmd::Command);
+use CHI;
+
+# ABSTRACT: list the application's commands
+
+
+has 'region_alias' => (
+    is     => 'rw',
+    isa    => 'Str',
+    traits => ['Getopt'],
+    #     cmd_aliases   => "h",
+    documentation => "asia, useast, uswest or eu",
+    required      => 1,
+    default => 'useast',
+);
+
+has 'refresh' => (
+    is => 'ro',
+    isa => 'Str',
+);
+
+
+with 'EnsCloud::Describer';
+sub execute {
+    my ( $self, $opt, $arg ) = @_;
+    
+#     my $edescriber = EnsCloud::Describer->new( region_alias => $self->region_alias );
+    my $cache = CHI->new(
+        driver   => 'File',
+        root_dir => $ENV{HOME} . '/' . '/ec2cache/' . $self->region_alias,
+    );
+    my $name           = 'useast_instances';
+    my $instance_table = $cache->get($name);
+    $DB::single = 1;
+    if ( !defined $instance_table || $self->refresh ) {
+#}         $instance_table = $edescriber->list_instances;
+         $instance_table = $self->list_instances;
+
+        $cache->set( $name, $instance_table, "10 minutes" );
+    }
+    print $instance_table;
+}
+
+sub abstract {
+
+    return 'list cloud things';
+
+}
+__PACKAGE__->meta->make_immutable;
+
+1;
+__END__
+

misc-scripts/cloud/EnsCloud/Describer.pm

+package EnsCloud::Describer;
+
+use Moose::Role;
+requires 'region_alias';
+
+use Net::Amazon::EC2;
+use Text::SimpleTable;
+our $col_widths_headings = {
+    tags          => [ 22, 'Tag' ],
+    instance_id   => [ 10, 'Instance' ],
+    ip_address    => [ 14, 'Ip Address' ],
+    dns_name      => [ 57, 'DNS Name' ],
+    instance_type => [ 9,  'Type' ],
+    launch_time   => [ 16, 'Launch' ],
+    image_id      => [ 12, 'Ami id' ],
+    name          => [ 45, 'Name' ],
+    description   => [ 45, 'Description' ],
+    image_state   => [ 7,  'State' ],
+
+    # snap_creation =>    [ 23, 'Snap Creation' ],
+    #     [ 13, 'Vol Creation' ]
+    size        => [ 5,  'Size' ],
+    create_time => [ 20, 'Create Time' ],
+
+    #     [ 20, 'Attach Time' ],
+    #     [ 13, 'Volume Id' ],
+    #     [ 10, 'Device' ],
+    #     [ 10, 'Vol. status' ],
+    #     [ 8,  'Attach status' ],
+    # zone =>    [ 10, 'Zone' ]
+    #     [ 13, 'Snapshot Id' ],
+    #     [ 19, 'Start Time' ],
+    #     [ 8,  'Progress' ],
+    # status =>    [ 10, 'Status' ],
+    #     [ 12, 'owner' ],
+    #     [ 11, 'owner alias' ],
+};
+
+has ec2_wanted_version => (
+    is      => 'rw',
+    default => '2011-02-28'
+
+);
+
+#has 'ec2' => (
+#    is         => 'ro',
+#    isa        => 'Net::Amazon::EC2',
+#lazy_
+#);
+
+# has 'region_alias' => (
+#     is  => 'rw',
+#     isa => 'Str',
+#     required =>1
+# );
+
+sub ec2 {
+    my $self          = shift;
+    my $region_lookup = {
+        'eu'     => 'eu-west-1',
+        'useast' => 'us-east-1',
+        'uswest' => 'us-west-1',
+        'asia'   => 'ap-southeast-1',
+    };
+    return Net::Amazon::EC2->new(
+        AWSAccessKeyId  => $ENV{AWS_ACCESS_KEY_ID},
+        SecretAccessKey => $ENV{AWS_ACCESS_KEY_SECRET},
+        debug           => 0,
+        region          => $region_lookup->{ $self->region_alias },
+        version         => $self->ec2_wanted_version
+    );
+
+}
+
+sub list_volumes {
+
+    my $self           = shift;
+    my $volumes        = $self->ec2->describe_volumes;
+    my @sorted_volumes = sort { $a->{size} <=> $b->{size} } @$volumes;
+
+    my $instances = $self->ec2->describe_instances;
+
+    my $instance2ip;
+
+    foreach my $instance (@$instances) {
+
+        #         print dump $instance->instances_set;
+
+        my $instance_set = $instance->instances_set->[0];
+        $instance2ip->{ $instance_set->{instance_id} } =
+          $instance_set->{ip_address};
+
+    }
+
+    #     die dump $volumes;
+    my $table = Text::SimpleTable->new(
+        [ 5,  'Size' ],
+        [ 20, 'Create Time' ],
+        [ 20, 'Attach Time' ],
+        [ 13, 'Volume Id' ],
+        [ 13, 'Instance Id' ],
+        [ 15, 'Ip Address' ],
+        [ 10, 'Device' ],
+        [ 10, 'Vol. status' ],
+        [ 8,  'Attach status' ],
+        [ 13, 'Snapshot Id' ],
+        [ 10, 'Zone' ]
+    );
+
+    foreach my $volume (@sorted_volumes) {
+
+        #         print dump $instance->instances_set;
+
+        my $attachments = $volume->attachments;
+
+        print
+"Volume [$volume->{volume_id}] is attached to multiple instances: this not handled by this script\n"
+          and exit
+          if @$attachments > 1;
+        $volume->{create_time} =~ s/.000Z$//;
+        $attachments->[0]->{attach_time} =~ s/.000Z$//;
+
+        #         die dump $instance_set;
+        $table->row(
+            $volume->{size},
+            $volume->{create_time},
+            $attachments->[0]->{attach_time} || '',
+            $volume->{volume_id},
+            $attachments->[0]->{instance_id}                   || '',
+            $instance2ip->{ $attachments->[0]->{instance_id} } || '',
+            $attachments->[0]->{device}                        || '',
+            $volume->{status},
+            $attachments->[0]->{status} || '',
+
+            $volume->{snapshot_id} || 'no snapshot',
+            $volume->{zone}
+        );
+    }
+
+    print $table->draw;
+
+}
+
+sub list_instances {
+    my ($self) = @_;
+
+    my $instances = $self->ec2->describe_instances;
+    my $table     = Text::SimpleTable->new(
+        $col_widths_headings->{tags},
+        $col_widths_headings->{instance_id},
+        $col_widths_headings->{ip_address},
+        $col_widths_headings->{dns_name},
+        $col_widths_headings->{instance_type},
+        $col_widths_headings->{launch_time},
+        $col_widths_headings->{image_state},
+        $col_widths_headings->{image_id}
+    );
+    foreach my $instance (@$instances) {
+
+        #    print dump $instance->instances_set;
+
+        my $instance_set = $instance->instances_set->[0];
+        $instance_set->{launch_time} =~ s/:\d+\.000Z$//;
+
+        #die dump $instance_set;
+        $table->row(
+            $instance_set->{tags} || '',
+            $instance_set->{instance_id},
+            $instance_set->{ip_address} || '',
+            $instance_set->{dns_name}   || '',
+            $instance_set->{instance_type},
+            $instance_set->{launch_time},
+            $instance_set->{instance_state}->name,
+            $instance_set->{image_id},
+        );
+    }
+    return $table->draw;
+}
+
+# __PACKAGE__->meta->make_immutable;
+
+1;
+
+__END__

misc-scripts/cloud/EnsCloud/Image.pm

+package EnsCloud::Image;
+use Moose;
+
+
+has volume_bundle => (isa => 'EnsCloud::Image::VolumeBundle', is=>'rw', required => 1);
+
+has 'root_device_snapshot' => (isa => 'Str', is => 'rw') ;
+has 'root_device' => (isa => 'Str', is => 'rw') ;
+
+has 'kernel' => (isa => 'Str', is => 'rw') ;
+has 'species' => (isa => 'Str', is => 'rw', required => 1) ;
+has 'tag' => (isa => 'Str', is => 'rw', required => 1) ;
+
+1;
+
+__END__

misc-scripts/cloud/EnsCloud/Image/VolumeBundle.pm

+package EnsCloud::Image::VolumeBundle;
+
+use Moose;
+
+has 'volumes' => (
+    traits  => ['Array'],
+    handles => {
+        all_volumes => 'elements',
+        add_volumes => 'push',
+        next_volume => 'shift',
+    },
+    isa => 'ArrayRef[EnsCloud::Image::VolumeBundle::Volume]',
+
+);
+
+1;
+
+__END__

misc-scripts/cloud/EnsCloud/Image/VolumeBundle/Volume.pm

+package EnsCloud::Image::VolumeBundle::Volume;
+use Moose;
+use Data::Dumper;
+has 'tag' => ( isa => 'Str', is => 'rw' );
+has 'species' => ( isa => 'Str', is => 'rw', required => 1 );
+
+
+has 'total_size' => (
+    traits => ['Number'],
+    is     => 'rw',
+
+    #      isa     => 'Int',
+    default => 0,
+    handles => {
+        add_size => 'add',
+    }
+);
+
+has 'volume_id'   => ( isa => 'Str', is => 'rw' );
+has 'ensembl_release'   => ( isa => 'Int', is => 'rw' );
+has 'snapshot_id' => ( isa => 'Str', is => 'rw' );
+has 'status' => ( isa => 'Str', is => 'rw', required =>1 , default => 'no snapshot' );
+has 'device'      => ( isa => 'Str', is => 'rw' );
+has 'dbs'         => (
+    traits  => ['Array'],
+    handles => {
+        all_dbs => 'elements',
+        add_db  => 'push',
+        next_db => 'shift',
+	sort_in_place_curried => [sort_in_place => sub {$_[0]->type cmp $_[1]->type}]
+    },
+    isa => 'ArrayRef[EnsCloud::Image::VolumeBundle::Volume::DatabaseDetails]',
+);
+
+1;
+
+__END__

misc-scripts/cloud/EnsCloud/Image/VolumeBundle/Volume/DatabaseDetails.pm

+package EnsCloud::Image::VolumeBundle::Volume::DatabaseDetails;
+use Moose;
+use CHI;
+
+
+has 'myd_path' => (isa => 'Str' , is => 'rw', required => 1);
+has 'name' => (isa => 'Str', is => 'rw', required => 1 ) ;
+has 'size' => (isa => 'Str', is => 'rw', required => 1);
+has 'type' => (isa => 'Str', is => 'rw', required => 1);
+has 'copy_path' => (isa => 'Str', is => 'rw', );
+has 'is_copied' => (isa => 'Str', is => 'rw', required => 1, default => 0);
+
+
+
+
+
+
+
+
+1;
+
+__END__

misc-scripts/cloud/ecloud

+#!/usr/bin/env perl
+
+
+use EnsCloud::Cmd;
+
+EnsCloud::Cmd->run;
+
+
+