Bump to v0.3.0. Enable multiple instances of script. Enable encryption. Add time…

…stam to stored backups.

@@ -8,16 +8,16 @@ driver:
   subnet_id: subnet-d530d8b1
   instance_type: t2.micro
   associate_public_ip: true
-  require_chef_omnibus: true
+  require_chef_omnibus: 12.12.15
   shared_credentials_profile: earth
 
 provisioner:
-  name: chef_solo
+  name: chef_zero
 
 platforms:
   - name: ubuntu-14.04
     driver:
-      image_id: ami-50946030
+      image_id: ami-548e2d34
     transport:
       username: ubuntu
       ssh_key: ~/.ssh/cfe_stg_20160222.pem

-# 0.2.1
+## 0.3.0 - 2016-11-25
+### Changed
+- 'dirs' attribute has been replaced with 'backups' which gives the option of creating multiple instances of the script, each having a different filename.
+- Maximum number of backup copies can now be specified for each file/directory to be backed up.
+- Attributes for cron entries have been simplified.
+- Definition for 'aws_tar_extract' now includes options for decrypting encrypted tarballs. The contents of the private key file must be provided in this case.
 
-Include tar cookbook in default recipe.
+### Added
+- An option whether or not to use encryption for each file/directory backup is provided. If encryption is used, proper encryption keys must be provided beforehand as node attributes.
+- Backup copies stored in S3 are now stored in their own subfolders (prefixes) along with a text file indicating the timestamp for when that backup was originally uploaded.
 
-# 0.2.0
+## 0.2.1
+- Include tar cookbook in default recipe.
 
-Add definition aws_tar_extract to abstract reloading of backups.
+## 0.2.0
+- Add definition aws_tar_extract to abstract reloading of backups.
 
-# 0.1.0
-
-Initial release of backup-file2s3
+## 0.1.0
+- Initial release of backup-file2s3

 # backup-file2s3-cookbook
 
-Installs a script that backs up one or more directories into an S3 bucket. Also sets up the cronjob to regularly run said script.
+Installs script/s that backs up one or more files/directories into an S3 bucket. Also sets up the cronjob to regularly run said script/s.
 
-There is also a definition `aws_tar_extract` to recover those backup files and unpack them.
+There is an option to encrypt the backups with OpenSSL. If encryption is used, the contents of the public PEM key must be provided.
+
+There is also a definition `aws_tar_extract` to recover those backup files and unpack them. This definition can also decrypt the tarballs using private key contents in a node attribute (default), or explicitly passed to the definition.
 
 ## Supported Platforms
 
@@ -30,21 +32,27 @@ Ubuntu 14.04
     <td><tt>'us-east-1'</tt></td>
   </tr>
   <tr>
-    <td><tt>['backup-file2s3']['max_backups']</tt></td>
-    <td>Integer</td>
-    <td>Number of old backup tarballs to retain in S3.</td>
-    <td><tt>30</tt></td>
-  </tr>
-  <tr>
-    <td><tt>['backup-file2s3']['dirs']</tt></td>
+    <td><tt>['backup-file2s3']['backups']</tt></td>
     <td>Array</td>
-    <td>An array of directories to be backed up.</td>
+    <td>An array of hashes that details the script names and directories (or files) that are to be backed up. This is also where you specify whether or not you want to encrypt the backup. Please see the default attribs file for more details.</td>
     <td><tt>[]</tt></td>
   </tr>
   <tr>
-    <td><tt>['backup-file2s3']['cron']['min']</tt></td>
+    <td><tt>['backup-file2s3']['cron']['sched']</tt></td>
+    <td>String</td>
+    <td>Crontab syntax for scheduling how often the backup script will run.</td>
+    <td><tt>'0 0 * * *'</tt></td>
+  </tr>
+  <tr>
+    <td><tt>['backup-file2s3']['encrypt']['pub_key']</tt></td>
+    <td>String</td>
+    <td>If encryption is used by any of the paths under 'backups', then this attribute should contain the public key used to encrypt.</td>
+    <td><tt>nil</tt></td>
+  </tr>
+  <tr>
+    <td><tt>['backup-file2s3']['encrypt']['priv_key']</tt></td>
     <td>String</td>
-    <td>Related cron attributes are: `hour`, `day`, `mon`, `wday`, each specifying a corresponding crontab value. This cron job will determine how often the backup script is run.</td>
+    <td>If an encrypted tarball is fetched using 'aws_tar_extract', then this attribute should contain the default private key to decrypt. Although the private key can also be explicitly passed to the 'aws_tar_extract' definition.</td>
     <td><tt>nil</tt></td>
   </tr>
 </table>

@@ -18,28 +18,36 @@
 # limitations under the License.
 #
 
-default['backup-file2s3']['bucket']      = 'bucketname'
-default['backup-file2s3']['region']      = 'us-east-1'
-default['backup-file2s3']['max_backups'] = 30
+default['backup-file2s3']['bucket'] = 'bucketname'
+default['backup-file2s3']['region'] = 'us-east-1'
 
-# The array of file directories to be backed up to S3
-default['backup-file2s3']['dirs'] = []
+# Contains hashes with details of file/directories to be backed up to S3.
+default['backup-file2s3']['backups'] = [
+#  {
+#    :script_name => 'default', # default: 'default', but only
+#                               #          one default can exist at a time
+#    :enable      => true # default: true
+#    :paths => [
+#      {
+#        :path          => '/my/dir',
+#        :bak_filename  => 'dir', # default: basename of path
+#        :bak_encrypted => false, # default: false
+#        :bak_maxcopies => 30     # default: 30
+#      }
+#    ]
+#  }
+]
 
-default['backup-file2s3']['script_dir'] = '/etc/backup_file_to_s3'
-default['backup-file2s3']['log_dir'] = '/var/log/backup_file_to_s3'
-default['backup-file2s3']['tmp_dir'] = '/tmp/backup_files'
+# Encryption keys (string contents of the actual keys, if used)
+default['backup-file2s3']['encrypt']['pub_key']  = nil
+default['backup-file2s3']['encrypt']['priv_key'] = nil
 
-default['backup-file2s3']['aws_bin'] = value_for_platform(
-  'ubuntu'  => { 'default' => '/usr/local/bin/aws' },
-  'default' => '/usr/local/bin/aws'   # haven't tested on other platforms yet
-)
+default['backup-file2s3']['script_dir'] = '/etc/backup_file_to_s3'
+default['backup-file2s3']['log_dir']    = '/var/log/backup_file_to_s3'
+default['backup-file2s3']['tmp_dir']    = '/tmp/backup_files'
 
 # Basic options for cron
-default['backup-file2s3']['cron']['min']    = '0'
-default['backup-file2s3']['cron']['hour']   = '0'
-default['backup-file2s3']['cron']['day']    = '*'
-default['backup-file2s3']['cron']['mon']    = '*'
-default['backup-file2s3']['cron']['wday']   = '*'
+default['backup-file2s3']['cron']['sched']  = '0 0 * * *'
 default['backup-file2s3']['cron']['mailto'] = "''"
 
 # Basic options for logrotate
@@ -51,3 +59,10 @@ default['backup-file2s3']['logrotate']['options']  = %w{
   compress
   notifempty
 }
+
+# Constants
+default['backup-file2s3']['tar_bin'] = '/bin/tar'
+default['backup-file2s3']['aws_bin'] = value_for_platform(
+  'ubuntu'  => { 'default' => '/usr/local/bin/aws' },
+  'default' => '/usr/local/bin/aws' # haven't tested on other platforms yet
+)

@@ -19,45 +19,79 @@
 #
 
 # Gets a tarball from AWS S3, then unpack it into a directory.
-# Parameters (all mandatory):
+# Parameters:
 #   :name | :file => The name of the backup tarball, without the extension
 #   :region       => AWS region
 #   :bucket       => AWS bucket
 #   :target_dir   => Where the tarball is to be unpacked. If not
 #                    exists, it will be created
 #   :creates      => A file path used for idempotency
+#   :encrypted    => Boolean. Whether these backup files are encrypted.
+#   :priv_key     => String. Contents of private key, if used.
 
 define :aws_tar_extract,
        :file       => nil, # default is params[:name]
-       :region     => 'us-east-1',
+       :region     => nil,
        :bucket     => nil,
        :target_dir => nil,
-       :creates    => nil do
-  file = params[:file] || params[:name]
+       :creates    => nil,
+       :encrypted  => false,
+       :priv_key   => nil do
 
-  tmp_dir = ::File.join(Chef::Config[:file_cache_path], 'backups')
-  filetgz = "#{tmp_dir}/#{file}.tar.gz"
+  fname    = params[:file] || params[:name]
+  region   = params[:region] || node['backup-file2s3']['region']
+  bucket   = params[:bucket] || node['backup-file2s3']['bucket']
+  priv_key = params[:priv_key] || node['backup-file2s3']['encrypt']['priv_key']
+  tmp_dir  = ::File.join(Chef::Config[:file_cache_path], 'f2s3_backups')
 
   include_recipe 'awscli'
   include_recipe 'tar'
 
-  [ tmp_dir, params[:target_dir] ].each do |ndir|
-    directory ndir do
-      recursive true
+  unless ::File.exist?(params[:creates])
+
+    directory(tmp_dir) { recursive true }
+    directory(params[:target_dir]) { recursive true }
+
+    file_priv_key = "#{tmp_dir}/priv.key"
+    fname_tgz     = "#{tmp_dir}/#{fname}.tar.gz"
+    fname_path    = "#{tmp_dir}/#{fname}.tar.gz"
+
+    if params[:encrypted]
+      fname_path << '.enc'
+
+      file file_priv_key do
+        content   priv_key
+        mode      0600
+        sensitive true
+      end
+
+      execute "decrypt_#{fname}" do
+        command  "openssl smime -decrypt -binary -inkey #{file_priv_key} "\
+                 "-in #{fname_path} -out #{fname_tgz} -inform DEM"
+        notifies :delete, "file[#{fname_path}]"
+        notifies :delete, "file[#{file_priv_key}]"
+        action   :nothing
+      end
+
+      file(fname_path) { action :nothing }
     end
-  end
 
-  unless ::File.exist?(params[:creates])
-    awscli_s3_file filetgz do
-      region  params[:region]
-      bucket  params[:bucket]
-      key     "#{file}.tar.gz"
+    awscli_s3_file fname_path do
+      region region
+      bucket bucket
+      key    "#{fname}/#{::File.basename(fname_path)}"
+      if params[:encrypted]
+        notifies :run, "execute[decrypt_#{fname}]", :immediately
+      end
     end
 
-    tar_extract filetgz do
+    tar_extract fname_tgz do
       action     :extract_local
       target_dir params[:target_dir]
       creates    params[:creates]
+      notifies   :delete, "file[#{fname_tgz}]"
     end
+
+    file(fname_tgz) { action :nothing }
   end
 end

@@ -4,7 +4,7 @@ maintainer_email 'sysadmin@chromedia.com'
 license          'Apache License'
 description      'Creates a script to backup directories into an S3 bucket.'
 long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
-version          '0.2.1'
+version          '0.3.0'
 
 depends 'awscli', '~> 1.0.1'
 depends 'cron', '~> 1.7.4'

@@ -22,40 +22,70 @@ include_recipe 'awscli'
 include_recipe 'tar'
 
 attribs = node['backup-file2s3']
-scr_dir = attribs['script_dir']
-sname   = 'backup_file_to_s3'
+pub_key_file = "#{attribs['script_dir']}/pub.key"
 
-template "#{scr_dir}/#{sname}" do
-  mode    0644
-  only_if "test -d #{scr_dir} || mkdir -p #{scr_dir}"
-  variables(
-    :aws_bin     => attribs['aws_bin'],
-    :log_dir     => attribs['log_dir'],
-    :tmp_dir     => attribs['tmp_dir'],
-    :bucket      => attribs['bucket'],
-    :region      => attribs['region'] || 'us-east-1',
-    :max_backups => attribs['max_backups'] || 30,
-    :dirs        => attribs['dirs']
-  )
+directory attribs['script_dir'] { recursive true }
+directory attribs['log_dir'] { recursive true }
+
+is_any_enc = attribs['backups'].any? do |back|
+  back[:paths].any? do |path|
+    path[:bak_encrypted]
+  end
 end
+if !attribs['encrypt']['pub_key'] && is_any_enc
+  Chef::Application.fatal!('No encryption public key contents supplied')
+end
+
+file pub_key_file do
+  content attribs['encrypt']['pub_key']
+  mode      0600
+  owner     'root'
+  group     'root'
+  sensitive true
+  only_if   { attribs['encrypt']['pub_key'] }
+end
+
+attribs['backups'].each do |back|
+  snam   = back[:script_name] || 'default'
+  sname  = "#{snam.gsub(' ', '-')}_backup2s3"
+  enable = back.has_key?(:enable) ? back[:enable] : true
+
+  template "#{attribs['script_dir']}/#{sname}" do
+    mode   0644
+    source 'backup_file_to_s3.erb'
+    variables(
+      :aws_bin      => attribs['aws_bin'],
+      :tar_bin      => attribs['tar_bin'],
+      :tmp_dir      => attribs['tmp_dir'],
+      :bucket       => attribs['bucket'],
+      :region       => attribs['region'],
+      :pub_key_file => pub_key_file,
+      :paths        => back[:paths]
+    )
+    action( enable ? :create : :delete )
+  end
+
+  sched = attribs['cron']['sched'].split(' ')
+  cron_d sname do
+    command "bash #{attribs['script_dir']}/#{sname} >> "\
+            "#{attribs['log_dir']}/#{sname}.log 2>&1"
+    minute  sched[0]
+    hour    sched[1]
+    day     sched[2]
+    month   sched[3]
+    weekday sched[4]
+    mailto  attribs['cron']['mailto']
+    path    '/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin'
 
-cra = attribs['cron']
-cron_d sname do
-  command "bash #{scr_dir}/#{sname}"
-  minute  cra['min']
-  hour    cra['hour']
-  day     cra['day']
-  month   cra['mon']
-  weekday cra['wday']
-  mailto  cra['mailto']
-  path    '/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin'
+    action( enable ? :create : :delete )
+  end
 end
 
 package 'logrotate'
 
 loa = attribs['logrotate']
-template "#{loa['conf_dir']}/#{sname}" do
-  source  "#{sname}_logrotate.erb"
+template "#{loa['conf_dir']}/backup_file_to_s3" do
+  source  'backup_file_to_s3_logrotate.erb'
   only_if "test -d #{loa['conf_dir']} || mkdir -p #{loa['conf_dir']}"
   variables(
     :log_dir => attribs['log_dir'],

@@ -2,59 +2,66 @@
 #
 # Generated by Chef
 #
-# Back up directories into an S3 bucket
+# Back up directories/files into an S3 bucket
 
 set -e
 
 suffix=.backup_file_to_s3
-[ -f /tmp/*"$suffix" ] && exit 200
-tmp_file=$( mktemp --suffix "$suffix" )
-
-log_dir=<%= @log_dir %>
-if [[ ! -d "$log_dir" ]] ; then
-    mkdir -p "$log_dir"
+if [ -f /tmp/*"$suffix" ] ; then
+    ( >&2 echo "[ERROR] Another operation might still be in progress" )
+    exit 200
 fi
-
-aws_cmd=<%= @aws_bin %>
-
-exec 3>&1 4>&2
-trap 'exec 2>&4 1>&3' 0 1 2 3
-exec 1>>"${log_dir}/backup_file_to_s3.log" 2>&1
+tmp_file=$( mktemp --suffix "$suffix" )
 
 bucket=<%= @bucket %>
 region=<%= @region %>
-max_backups=<%= @max_backups %>
 bak_dir=<%= @tmp_dir %>
 
-# Array of directories to be backed up.
-# 
-# Example:
-# declare -a tar_dirs=(
-#   "/path/to/dira"
-#   "/another/path/to/dirb"
-# )
-#
-# Tarball names will be the basename of each path given.
-declare -a tar_dirs=(
-<% @dirs.each do |dirx| -%>
-    "<%= dirx %>"
-<% end -%>
-)
+aws_bin=<%= @aws_bin %>
+tar_bin=<%= @tar_bin || '/bin/tar' %>
 
-if [[ ! -d "$bak_dir" ]] ; then
-    echo "$(date) : Missing backup directory. Creating."
-    mkdir -p "$bak_dir"
-fi
+pub_key_file=<%= @pub_key_file %>
+
+# Create the tarball.
+# Args:
+#   $1 = full path to be backed up, e.g. '/var/src/mydir'
+#   $2 = desired filename, sans '.tar.gz' extension, e.g. 'myfilename'
+tar_dir() {
+    bname=$( basename "$1" )
+    dname=$( dirname "$1" )
+
+    echo "$(date) : Creating tarball from ${1}"
+    "$tar_bin" -C "${dname}" -czf "${bak_dir}/${2}.tar.gz" "$bname"
+}
+
+# Encrypt a tarball using OpenSSL and a given public key.
+# The original file will be replaced by a new one with the same filename,
+# suffixed with '.enc'.
+#
+# Given file should be located in $bak_dir.
+# Args:
+#   $1 = filename, sans '.tar.gz' extension, e.g. 'myfilename'
+encrypt_tarball() {
+    echo "$(date) : Encrypt file ${1}.tar.gz"
+    openssl smime -encrypt -binary -text -aes256 -in "${bak_dir}/${1}.tar.gz" \
+            -out "${bak_dir}/${1}.tar.gz.enc" -outform DER "${pub_key_file}"
+    rm "${bak_dir}/${1}.tar.gz"
+}
 
-# Rotate the current backups in S3
-# $1 = directory to be tarred
+# Rotate the number suffixes of the current backups in S3
+# Args:
+#   $1 = filename, sans '.tar.gz' extension, e.g. 'myfilename'
+#   $2 = max number of backup copies to store at a time
 increment_backup_names() {
-    fname=$( basename "$1" )
-    bak_keyname="${fname}.tar.gz"
+    bak_keyname=$1
+    max_backups=$2
 
-    baks=$( "$aws_cmd" --output text --region "$region" \
-                s3api list-objects --bucket "$bucket" \
-            | grep '^CONTENTS' | cut -f3 | grep "^${bak_keyname}" || echo "" )
+    # Backups will stored inside subdirectories (prefixes)
+    # in S3, so only look at 'PRE' objects.
+    baks=$( "$aws_bin" --output text --region "$region" \
+            s3 ls "s3://${bucket}/" | grep '^\s*PRE' | \
+            sed -e 's/^ *PRE //' -e 's/\/$//' | \
+            grep "^${bak_keyname}" || echo "" )
 
     echo "$(date) : Backup rotation for ${bak_keyname}"
     start=$((max_backups - 1))
@@ -63,48 +70,70 @@ increment_backup_names() {
         if echo "$baks" | grep "^${bak_keyname}\\.${x}\$" ; then
             newx=$((x + 1))
             if [[ $newx -lt $max_backups ]] ; then
-                "$aws_cmd" --region "$region" \
-                    s3 cp "s3://${bucket}/${bak_keyname}.${x}" \
+                "$aws_bin" --region "$region" \
+                    s3 mv --recursive "s3://${bucket}/${bak_keyname}.${x}" \
                           "s3://${bucket}/${bak_keyname}.${newx}"
+            else
+                "$aws_bin" --region "$region" \
+                    s3 rm --recursive "s3://${bucket}/${bak_keyname}.${x}"
             fi
         fi
     done
 
     if echo "$baks" | grep "^${bak_keyname}\$" ; then
-        "$aws_cmd" --region "$region" \
-            s3 cp "s3://${bucket}/${bak_keyname}" \
+        "$aws_bin" --region "$region" \
+            s3 mv --recursive "s3://${bucket}/${bak_keyname}" \
                   "s3://${bucket}/${bak_keyname}.1"
     fi
 }
 
-# Tar up the directory
-# $1 = directory to be tarred
-tar_dir() {
-    fname=$( basename "$1" )
-    parent=$( dirname "$1" )
-    echo "$(date) : Tar up ${1}"
+# Upload the tarball to the S3 bucket. It will be uploaded
+# to this location: ${bucket}/basename/myfile.tar.gz.enc
+#
+# A timestamp file will also be uploaded to this location:
+# ${bucket}/basename/YYYY-MM-DDThh:mm:ss.txt
+# Args:
+#   $1 = filename, sans '.tar.gz' extension, e.g. 'myfilename'
+#   $2 = if file is encrypted or not (boolean)
+upload_to_s3() {
+    bak_keyname=$1
+    if [ "$2" = true ] ; then
+        fname="${1}.tar.gz.enc"
+    else
+        fname="${1}.tar.gz"
+    fi
 
-    tar -C "$parent" -czf "${bak_dir}/${fname}.tar.gz" "$fname"
-}
+    echo "$(date) : Upload ${fname} to S3 bucket ${bucket}"
 
-# $1 = directory to be tarred
-upload_to_s3() {
-    fname=$( basename "$1" )
-    echo "$(date) : Upload ${fname}.tar.gz to S3 bucket ${bucket}"
+    stamp=$( date +"%FT%T )
+    echo "Uploaded: ${stamp}" > "${bak_dir}/${stamp}.txt"
 
-    "$aws_cmd" --region "$region" \
-        s3 mv "${bak_dir}/${fname}.tar.gz" "s3://${bucket}/${fname}.tar.gz"
+    "$aws_bin" --region "$region" \
+        s3 mv "${bak_dir}/${fname}" \
+        "s3://${bucket}/${bak_keyname}/${fname}"
+    "$aws_bin" --region "$region" \
+        s3 mv "${bak_dir}/${stamp}.txt" \
+        "s3://${bucket}/${bak_keyname}/${stamp}.txt"
 }
 
-for dirx in "${tar_dirs[@]}" ; do
-    if [[ -d "$dirx" ]] ; then
-        increment_backup_names "$dirx"
-        tar_dir "$dirx"
-        upload_to_s3 "$dirx"
-    else
-        echo "$(date) : WARNING : Directory ${dirx} does not exist. Skipping."
-    fi
-done
+if [[ ! -d "$bak_dir" ]] ; then
+    mkdir -p "$bak_dir"
+fi
+
+<% @paths.each do |path| -%>
+<%   bname  = path[:bak_filename] || ::File.basename(path[:path]) -%>
+<%   is_enc = path.has_key?(:bak_encrypted) ? path[:bak_encrypted] : false -%>
+if [[ -d <%= path[:path] %> || -f <%= path[:path] %> ]] ; then
+    increment_backup_names <%= bname %> <%= path[:bak_maxcopies] || 30 %>
+    tar_dir <%= path[:path] %> <%= bname %>
+<%   if is_enc -%>
+    encrypt_tarball <%= bname %>
+<%   end -%>
+    upload_to_s3 <%= bname %> <%= is_enc %>
+else
+    >&2 echo "$(date) [WARNING] Path <%= path[:path] %> does not exist"
+fi
 
+<% end -%>
 rm "$tmp_file"
 echo "$(date) : Done"