manheim/backupsss

View on GitHub
lib/backupsss/backup.rb

Summary

Maintainability
A
0 mins
Test Coverage
require 'parallel'

module Backupsss
  # A class for delivering a tar to S3
  class Backup
    MAX_FILE_SIZE = 1024 * 1024 * 100 # 100MB

    attr_reader :config, :client, :filename

    def initialize(config, client)
      @config   = config
      @client   = client
      @filename = config[:filename]
    end

    def put_file(file)
      large_file(file) ? multi_upload(file) : single_upload(file)
    end

    private

    def create_multipart_upload
      $stdout.puts 'Creating a multipart upload'

      client.create_multipart_upload(bucket_opts)
    end

    def large_file(file)
      $stdout.puts 'Checking backup size ...'
      is_lg  = file.size > MAX_FILE_SIZE
      status = is_lg ? 'greater than' : 'less than or equal to'
      $stdout.puts "Size of backup is #{status} 100MB"

      is_lg
    end

    def complete_multipart_upload_request(upload_id, parts)
      bucket_opts.merge(
        upload_id: upload_id, multipart_upload: { parts: parts }
      )
    end

    def timed_multipart_upload
      s = Time.now
      $stdout.puts "Starting multipart upload at #{s}"
      yield
      e = Time.now
      duration = ((e - s) / 60).round(2)
      output   = ["Completed multipart upload at #{e}"]
      output << "Completed in #{duration} minutes."

      $stdout.puts output.join("\n")
    end

    def abort_multipart_message(error, upload_id)
      "#{error}\nAborting multipart upload : #{upload_id}"
    end

    def multi_upload(file)
      upload_id = create_multipart_upload.upload_id

      bail_multipart_on_fail(upload_id) do
        timed_multipart_upload do
          parts = upload_parts(file, upload_id).sort do |a, b|
            a[:part_number] <=> b[:part_number]
          end

          req = complete_multipart_upload_request(upload_id, parts)
          client.complete_multipart_upload(req)
        end
      end
    end

    def upload_parts(file, upload_id)
      Parallel.map(1..part_count(file), in_threads: 10) do |part|
        bail_upload_part_on_fail(part, upload_id) do
          $stdout.puts "Uploading part number #{part} : #{upload_id}\n"
          r = client.upload_part(upload_part_params(file, part, upload_id))
          success_msg = "Completed uploading part number #{part} : #{upload_id}"
          r.on_success { $stdout.puts success_msg }

          { etag: r.etag, part_number: part }
        end
      end
    end

    def bail_multipart_on_fail(upload_id)
      yield
    rescue StandardError => e
      $stdout.puts abort_multipart_message(e, upload_id)
      client.abort_multipart_upload(bucket_opts.merge(upload_id: upload_id))
    end

    def bail_upload_part_on_fail(part, upload_id)
      yield
    rescue StandardError => e
      output = ["Failed to upload part number #{part} : #{upload_id}"]
      output << "because of #{e.message}"
      output << "Aborting remaining parts : #{upload_id}"
      raise output.join("\n")
    end

    def upload_part_params(file, part, upload_id)
      start = (part - 1) * MAX_FILE_SIZE
      body  = IO.read(file.path, MAX_FILE_SIZE, start)

      bucket_opts.merge(part_number: part, body: body, upload_id: upload_id)
    end

    def part_count(file)
      c = (file.size.to_f / MAX_FILE_SIZE.to_f).ceil
      $stdout.puts "Uploading backup as #{c} parts"

      c
    end

    def single_upload(file)
      client.put_object(bucket_opts.merge(body: file.read))
    end

    def bucket_opts
      { bucket: config[:s3_bucket],
        key:    "#{config[:s3_bucket_prefix]}/#{filename}" }
    end
  end
end