DZone Snippets: rsync code

Use rsync to mirror directories between Virtual Hosts.

jrobertson (James Robertson) — Thu, 29 May 2008 23:16:58 GMT

The following shell command will rsync the mysite.com directory 'feed' to mysite2.com



rsync -a /var/www/mysite.com/feed /var/www/mysite2.com

Active Record YAML Backup Solution - Drop in rake task and config file to backup db and directorys of your choice (user uploaded files)

rawdod (David Richards) — Fri, 16 Nov 2007 04:38:47 GMT

EZ drop in backup rake task for your rails projects - works well - 5 - 10 min set up, one rake file and one config file
backs up db and any directory's to any number of servers with rsync

BSD License or whatever, but it would be cool if you told me your using it
2007 ISS http://industrialstrengthinc.com

this is a sample config file and a rake task you can drop into any rails project to do backups, easy to automate.
Backs up your specified environment db to activerecord yml files (one per table) and zips them up in a human readable timestamped file
syncs that and any other set of arbitrary directory's to any number of arbitrary servers with rsync
be sure to set ssh key based logins
to run: rake backup
also: rake backup:db, rake backup:restoredb (promts for a file created by backup:db), rake backup:push (to push out what u got)
note: this uses something like 30 lines of code from some blog site I got it from that I cant recall, yay for that guy, thanks



## example crontab entry:



3 * * * * cd /rails_deployment_dir/current && nice rake backup RAILS_ENV=production >> /rails_deployment_dir/production_backup_system.log



## config file - goes in config/backup.yml



production: 

  dirs: 

   - db/backups

   - public/uploaded_images

  servers: 

    -  name: backup server number 1

       host: gridserver.com

       port: 22

       user: blah@whatever.com

       dir: /home/blah/backups

    -  name: backup server two

       host: kradradio.com

       port: 22

       user: kraduser

       dir: /home/kraduser/backups_from_my_rails_proj





development: 

  dirs: 

   - db/backups

   - public/uploaded_images

  servers: 

    -  name: local self

       host: localhost

       port: 5222

       user: oneman

       dir: /home/oneman/Documents/development_backup





## rake file lib/tasks/backup.rake



desc "Backup Everything Specified in config/backup.yml"

task :backup => [ "backup:db",  "backup:push"]



namespace :backup do

 

    RAILS_APPDIR = RAILS_ROOT.sub("/config/..","")

    

   def interesting_tables

     ActiveRecord::Base.connection.tables.sort.reject! do |tbl|

       ['schema_info', 'sessions', 'public_exceptions'].include?(tbl)

     end

   end

  

   desc "Push backup to remote server"

   task :push  => [:environment] do 

      FileUtils.chdir(RAILS_APPDIR)

      backup_config = YAML::load( File.open( 'config/backup.yml' ) )[RAILS_ENV]

      for server in backup_config["servers"]

       puts "Backing up #{RAILS_ENV} directorys #{backup_config['dirs'].join(', ')} to #{server['name']}"

       puts "Time is " + Time.now.rfc2822 + "\n\n"

         for dir in backup_config["dirs"]

          local_dir = RAILS_APPDIR + "/" + dir + "/"

          remote_dir = server['dir'] + "/" + dir.split("/").last + "/"

          puts "Syncing #{local_dir} to #{server['host']}#{remote_dir}"

          sh "/usr/bin/rsync -avz -e 'ssh -p#{server['port']} ' #{local_dir} #{server['user']}@#{server['host']}:#{remote_dir}"

         end

       puts "Completed backup to #{server['name']}\n\n"

      end

   end



    task :storedb => :environment do 



      backupdir = RAILS_APPDIR + '/db/backup'

      FileUtils.mkdir_p(backupdir)

      FileUtils.chdir(backupdir)

      puts "Dumping database to activerecord yaml files in #{backupdir}"

      interesting_tables.each do |tbl|



        klass = tbl.classify.constantize

        puts "Writing #{tbl}..."

        File.open("#{tbl}.yml", 'w+') { |f| YAML.dump klass.find(:all).collect(&:attributes), f }      

      end

      puts "Database Dumped.\n\n"

    end



    desc "Dump Current Environment Db to file"    

    task :db => [:environment, :storedb ] do

      backupdir = RAILS_APPDIR + '/db/backup'

      archivedir = RAILS_APPDIR + '/db/backups'

      backup_filename = "#{RAILS_ENV}_db_backup_#{Time.now.strftime("%B.%d.%Y_at_%I.%M.%S%p_%Z")}.tar.bz2"

      FileUtils.mkdir_p(archivedir)

      puts "Archiving #{backupdir} yaml files to #{backup_filename}\n\n"

      `tar -C #{backupdir} -cjf #{backup_filename} *`

      `mv #{backup_filename} #{archivedir}`

    end



    desc "Restore Current Environment Db from a file"    

    task :restoredb => [:environment] do 

        backupdir = RAILS_APPDIR + '/db/backup'

        archivedir = RAILS_APPDIR + '/db/backups'

        print "Input a file to load into the db: #{archivedir}/"

        backup_filename = STDIN.gets.chomp

        puts "Loading backup file: #{backup_filename}"

        FileUtils.chdir(archivedir)

        `tar -xjf #{backup_filename}`

        `mv *.yml #{backupdir}`

        FileUtils.mkdir_p(backupdir)

        FileUtils.chdir(backupdir)

    

        interesting_tables.each do |tbl|

        puts "Clearing #{tbl} table.."

        ActiveRecord::Base.connection.execute "TRUNCATE #{tbl}"

        puts "Loading #{tbl} backup file..."

        table = YAML.load_file("#{tbl}.yml")        



        if table.length > 0 && table.first.key?("id")

            highestid = 0

            table.each do |fixture|

             if fixture["id"] > highestid

                highestid = fixture["id"]

             end

            end



            ActiveRecord::Base.connection.execute "SELECT setval('#{tbl}_id_seq',#{highestid})"

            puts "Setting #{tbl}_id sequence to #{highestid}"

        end

         

        #klass = tbl.classify.constantize

        ActiveRecord::Base.transaction do 

        

          puts "Inserting #{table.length} values into #{tbl}"

          table.each do |fixture|

            ActiveRecord::Base.connection.execute "INSERT INTO #{tbl} (#{fixture.keys.join(",")}) VALUES (#{fixture.values.collect { |value| ActiveRecord::Base.connection.quote(value) }.join(",")})", 'Fixture Insert'

          end        

          puts "#{tbl} table restored.\n\n"

        end

       end

    end



 

end

copy files using rsync and ssh

jrobertson (James Robertson) — Tue, 30 Oct 2007 13:03:36 GMT

source: http://www.mikerubel.org/computers/rsync_snapshots/#Abstract ; switches: -a = archive mode -e specifies the remote shell to use



rsync -a -e ssh source/ username@remotemachine.com:/path/to/destination/

rsync backup of a Mac

willcodeforfoo () — Sun, 15 Oct 2006 08:37:42 GMT

sudo /usr/local/bin/rsync -aREx --delete --exclude='.Spotlight-*' --exclude '/private/var/vm/*' [IP-address of Mac mini]::PowerBookBackup

Cheapest rsync replacement (with Ruby)

413x () — Fri, 31 Mar 2006 15:06:47 GMT

I often use rsync to keep a local copy of some HTTPD logs (around ~200MB atm.). Since they are append-only, having rsync compute and compare the checksums for the parts I already have seems wasteful: both my box and the one I'm copying from would be happier if they didn't have to process a couple hundred MBs for nothing. (...)



#!/usr/bin/env ruby



REMOTE_RUBY = "ruby"

# TODO: allow REMOTE_RUBY to be specified via a cmdline opt



if ARGV.size != 2 || ARGV[0][/:/].nil? || !File.exist?(ARGV[1])

  puts <  ruby logfetcher.rb host:path/to/src dst

EOF

  exit

end



FILE = ARGV[1]

REMOTE_HOST, REMOTE_FILE = ARGV[0].split(/:/)

BLOCK_SIZE = 8192



osize = File.size(FILE)

#FIXME: cheap escaping

command = "File.open(#{REMOTE_FILE.inspect}){|f| " + 

          "f.pos = #{osize}; print f.read(#{BLOCK_SIZE}) until f.eof? }"



command.gsub!(/"/){'\\"'}

fetched = 0

t = nil

$stdout.sync = true

print "Establishing connection\r"

File.open(FILE, "a") do |os|

  IO.popen(%{ssh #{REMOTE_HOST} ruby -e '"#{command}"'}) do |is|

    until is.eof?

      data = is.read(BLOCK_SIZE)

      t ||= Time.new # ignore the time it takes to establish the SSH connection

      fetched += data.size

      print "Read #{fetched}                          \r"

      os.write(data)

    end

  end

end

print(" " * 50  + "\r")



dt = Time.new - t

puts "Fetched #{fetched} bytes."

puts "Total size #{osize + fetched}."

puts "Needed %4.1f seconds." % dt

puts "Average speed %d bytes/sec." % (fetched / dt)

Source: Cheapest rsync replacement

Resuming an aborted scp transfer

plindberg (Peter Lindberg) — Thu, 21 Apr 2005 20:58:11 GMT

Assume you have copied a file across the network using something like this:

scp host:/path/to/file .

Then assume something goes wrong, so the transfer is interrupted. To continue the transfer, do this:

rsync --partial host:/path/to/file .

(Or, rather, always use rsync as above when transferring very large files.)

(Also note: I don't know how frequently rsync flushes its buffer, but don't assume something is wrong just because the file you're downloading doesn't increase in size in the file system. When you interrupt rsync using Ctrl-C, it will flush its buffers.)