Simple S3 utils - copy bucket to bucket

wiseleyb (ben) — Sun, 30 Dec 2007 20:13:56 GMT

Some simple S3 utils in Ruby

Specifically written to copy one bucket to another (for testing on production on staging)

PLEASE BE CAREFUL WITH THIS - THERE IS CODE THAT DELETES ALL CONTENTS OF A BUCKET

Example:



a = AmazoneS3Asset.new

a.copy_over_bucket("myapp_production", "myapp_production")



require 'aws/s3'

require 'mechanize'



class AmazonS3Asset

  

  include AWS::S3

  S3ID = "your s3 id"

  S3KEY = "your s3 key"

  

  def initialize

    puts "connecting..."

    AWS::S3::Base.establish_connection!(

      :access_key_id     => S3ID,

      :secret_access_key => S3KEY

    )

  end



  def delete_key(bucket, key)

    if exists?(bucket, key) 

      S3Object.delete key, bucket

    end

  end

  

  def empty_bucket(bucket)

    bucket_keys(bucket).each do |k|

      puts "deleting #{k}"

      delete_key(bucket,k)

    end

  end

  

  def bucket_keys(bucket)

    b = Bucket.find(bucket)

    b.objects.collect {|o| o.key}

  end



  def copy_over_bucket(from_bucket, to_bucket)

    puts "Replacing #{to_bucket} with contents of #{from_bucket}"

    #delete to_bucket

    empty_bucket(to_bucket)

    bucket_keys(from_bucket).each do |k|

      copy_between_buckets(from_bucket, to_bucket, k)

    end

  end

  

  def copy_between_buckets(from_bucket, to_bucket, from_key, to_key = nil)

    if exists?(from_bucket, from_key)

      to_key = from_key if to_key.nil?

      puts "Copying #{from_bucket}.#{from_key} to #{to_bucket}.#{to_key}"

      url = "http://s3.amazonaws.com/#{from_bucket}/#{from_key}"

      filename = download(url)

      store_file(to_bucket,to_key,filename)

      File.delete(filename)

      return "http://s3.amazonaws.com/#{to_bucket}/#{to_key}"

    else

      puts "#{from_bucket}.#{from_key} didn't exist"

      return nil

    end

  end



  def store_file(bucket, key, filename)

     puts "Storing #{filename} in #{bucket}.#{key}"

     S3Object.store(

      key,

      File.open(filename),

      bucket,

      :access => :public_read

      )

  end



  def download(url, save_as = nil)

    if save_as.nil?

      Dir.mkdir("amazon_s3_temp") if !File.exists?("amazon_s3_temp")

      save_as = File.join("amazon_s3_temp",File.basename(url))

    end

    begin

      puts "Saving #{url} to #{save_as}"

      agent = WWW::Mechanize.new {|a| a.log = Logger.new(STDERR) }

      img = agent.get(url)

      img.save_as(save_as)

      return save_as

    rescue

      raise "Failed on " + url + "  " + save_as

    end

  end



  def exists?(bucket,key)

    begin

      res = S3Object.find key, bucket

    rescue 

      res = nil

    end

    return !res.nil?

  end

      

end

DZone Snippets: util code

Simple S3 utils - copy bucket to bucket