Bump to v0.4.0.

- Install mariadb::client before mariadb::default. Backup script will now compress first before encrypting.
- Modify the way the backup script is called. Optimize cookbook a bit. Modify reload_from_s3 recipe to account for new compression-then-encryption order.

@@ -8,16 +8,16 @@ driver:
   subnet_id: subnet-d530d8b1
   subnet_id: subnet-d530d8b1
   instance_type: t2.micro
   instance_type: t2.micro
   associate_public_ip: true
   associate_public_ip: true
-  require_chef_omnibus: true
+  require_chef_omnibus: 12.12.15
   shared_credentials_profile: earth
   shared_credentials_profile: earth
 
 
 provisioner:
 provisioner:
-  name: chef_solo
+  name: chef_zero
 
 
 platforms:
 platforms:
   - name: ubuntu-14.04
   - name: ubuntu-14.04
     driver:
     driver:
-      image_id: ami-50946030
+      image_id: ami-d732f0b7
     transport:
     transport:
       username: ubuntu
       username: ubuntu
       ssh_key: ~/.ssh/cfe_stg_20160222.pem
       ssh_key: ~/.ssh/cfe_stg_20160222.pem

+## 0.4.0
+### Fixed
+- Install `mariadb::client` first, before doing `mariadb::default`. Otherswise, it might error out.
+- Fixed recipes to account for the above change, including `reload_from_s3`.
+- Backup script now properly compresses the file before encrypting it.
+
+### Changed
+- Optimized certain variables and recipt for better understandability. Some variables are now incompatible with previous versions.
+
 # 0.3.1
 # 0.3.1
 
 
 Use encoding utf8 and collation utf8_general_ci as defaults when creating databases
 Use encoding utf8 and collation utf8_general_ci as defaults when creating databases

@@ -8,6 +8,9 @@ The server is assumed to be using an IAM role with S3 bucket read/write access,
 
 
 When encryption is enabled for DB backups, the private and public keys are shared across all databases in the `db_map` attribute. Encryption is enabled separately for each individual database (see Usage below).
 When encryption is enabled for DB backups, the private and public keys are shared across all databases in the `db_map` attribute. Encryption is enabled separately for each individual database (see Usage below).
 
 
+
+Note that enabling encryption can cause the backup process to take a very long time depending on how large the database is. That said, it is still very much recommended to do so.
+
 ## Supported Platforms
 ## Supported Platforms
 
 
 Ubuntu 14.04
 Ubuntu 14.04

@@ -47,17 +47,15 @@ else
   default['cfe-mariadb']['backup']['aws_bin']       = '/usr/local/bin/aws'
   default['cfe-mariadb']['backup']['aws_bin']       = '/usr/local/bin/aws'
   default['cfe-mariadb']['backup']['mysqldump_bin'] = '/usr/bin/mysqldump'
   default['cfe-mariadb']['backup']['mysqldump_bin'] = '/usr/bin/mysqldump'
 end
 end
+
 # Path to directory where the backup script should be placed.
 # Path to directory where the backup script should be placed.
 # Uncomment to set custom locations.
 # Uncomment to set custom locations.
 #default['cfe-mariadb']['backup']['script_dir'] = ::File.join(
 #default['cfe-mariadb']['backup']['script_dir'] = ::File.join(
 #  node['mariadb']['configuration']['path'], 'scripts'
 #  node['mariadb']['configuration']['path'], 'scripts'
 #)
 #)
+default['cfe-mariadb']['backup']['log_dir'] = '/var/log/backup_db_to_s3'
 
 
-default['cfe-mariadb']['backup']['cron']['min']    = '0'
-default['cfe-mariadb']['backup']['cron']['hour']   = '0'
-default['cfe-mariadb']['backup']['cron']['day']    = '*'
-default['cfe-mariadb']['backup']['cron']['mon']    = '*'
-default['cfe-mariadb']['backup']['cron']['wday']   = '*'
+default['cfe-mariadb']['backup']['cron']['sched']  = '0 0 * * *'
 default['cfe-mariadb']['backup']['cron']['mailto'] = "''" # Empty single quotes
 default['cfe-mariadb']['backup']['cron']['mailto'] = "''" # Empty single quotes
 
 
 # Basic options for logrotate
 # Basic options for logrotate
@@ -91,11 +89,6 @@ default['cfe-mariadb']['backup']['logrotate']['options']  = %w{
 #
 #
 #              The key file will be stored in the same directory
 #              The key file will be stored in the same directory
 #              as the script as 'pub.key'.
 #              as the script as 'pub.key'.
-# NOTE:
-#   Enabling encryption will result in HUGE file sizes and,
-#   depending on how large a database is, can take a LOT of time
-#   during the backup process. That said, it is still recommended to
-#   enforce encryption on DB backups.
 default['cfe-mariadb']['encrypt']['priv_key'] = nil
 default['cfe-mariadb']['encrypt']['priv_key'] = nil
 default['cfe-mariadb']['encrypt']['pub_key']  = nil
 default['cfe-mariadb']['encrypt']['pub_key']  = nil
 
 

@@ -4,7 +4,7 @@ maintainer_email 'sysadmin @ chromedia.com'
 license          'Apache License'
 license          'Apache License'
 description      'Simplifies setup of MariaDB in Chromedia.'
 description      'Simplifies setup of MariaDB in Chromedia.'
 long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
 long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
-version          '0.3.1'
+version          '0.4.0'
 
 
 {
 {
   'mariadb'         => '0.3.1',
   'mariadb'         => '0.3.1',

@@ -50,6 +50,10 @@ file pub_key_file do
   only_if { md['encrypt']['pub_key'] }
   only_if { md['encrypt']['pub_key'] }
 end
 end
 
 
+directory mdb['log_dir'] do
+  recursive true
+end
+
 template "#{mdb['script_dir']}/backup_db_to_s3" do
 template "#{mdb['script_dir']}/backup_db_to_s3" do
   variables(
   variables(
     :db_map        => md['db_map'],
     :db_map        => md['db_map'],
@@ -66,13 +70,15 @@ end
 # Chef built-in resource 'cron' seems to have a problem with
 # Chef built-in resource 'cron' seems to have a problem with
 # the syntax here. But they suggest using 'cron_d', instead, from
 # the syntax here. But they suggest using 'cron_d', instead, from
 # the cron cookbook (https://docs.chef.io/resource_cron.html).
 # the cron cookbook (https://docs.chef.io/resource_cron.html).
+sched = mdbc['sched'].split(' ')
 cron_d 'backup_db_to_s3' do
 cron_d 'backup_db_to_s3' do
-  command "bash #{mdb['script_dir']}/backup_db_to_s3"
-  minute  mdbc['min']
-  hour    mdbc['hour']
-  day     mdbc['day']
-  month   mdbc['mon']
-  weekday mdbc['wday']
+  command "bash #{mdb['script_dir']}/backup_db_to_s3 "\
+          ">> #{mdb['log_dir']}/backup_db_to_s3.log 2>&1"
+  minute  sched[0]
+  hour    sched[1]
+  day     sched[2]
+  month   sched[3]
+  weekday sched[4]
   mailto  mdbc['mailto']
   mailto  mdbc['mailto']
   path    '/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin'
   path    '/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin'
 end
 end

@@ -30,6 +30,8 @@ chef_gem 'chef-rewind' do
 end
 end
 require 'chef/rewind'
 require 'chef/rewind'
 
 
+# mysql2_chef_gem seems to want the client to be installed first.
+include_recipe 'mariadb::client'
 include_recipe 'mariadb'
 include_recipe 'mariadb'
 
 
 # It looks like the service is sometimes not being restarted
 # It looks like the service is sometimes not being restarted

@@ -49,11 +49,10 @@ node['cfe-mariadb']['db_map'].each do |dbx|
     dbx_name = dbx[:db_name]
     dbx_name = dbx[:db_name]
   end
   end
 
 
-  keyname = "#{dbx[:bak_filename]}#{dbx[:bak_encrypted] ? '.enc.gz' : '.gz'}"
-  filegz  = "#{tmp_dir}/#{keyname}"
-  filesql = "#{tmp_dir}/#{dbx[:bak_filename]}"
+  keyname  = "#{dbx[:bak_filename]}.gz#{dbx[:bak_encrypted] ? '.enc' : ''}"
+  filepath = "#{tmp_dir}/#{dbx[:bak_filename]}"
 
 
-  awscli_s3_file filegz do
+  awscli_s3_file "#{tmp_dir}/#{keyname}" do
     region   node['cfe-mariadb']['s3_region']
     region   node['cfe-mariadb']['s3_region']
     bucket   node['cfe-mariadb']['s3_bucket']
     bucket   node['cfe-mariadb']['s3_bucket']
     key      keyname
     key      keyname
@@ -64,30 +63,28 @@ node['cfe-mariadb']['db_map'].each do |dbx|
     end
     end
   end
   end
 
 
-  execute "unpack_#{filegz}" do
-    command  "gzip -d #{filegz}"
-  end
-
-  execute "decrypt_#{filesql}.enc" do
+  execute "decrypt_#{filepath}.gz.enc" do
     command  "openssl smime -decrypt -binary -inkey #{priv_key_file} "\
     command  "openssl smime -decrypt -binary -inkey #{priv_key_file} "\
-             "-in #{filesql}.enc -out #{filesql} -inform DEM"
-    only_if  { dbx[:bak_encrypted] }
-    notifies :delete, "file[#{filesql}.enc]"
+             "-in #{filepath}.gz.enc -out #{filepath}.gz -inform DEM"
+    only_if  { ::File.exist?("#{filepath}.gz.enc") }
+    notifies :delete, "file[#{filepath}.gz.enc]"
   end
   end
 
 
-  execute "reload_#{filesql}" do
+  execute "gzip -d #{filepath}.gz"
+
+  execute "reload_#{filepath}" do
     command   "mysql -h #{node['mariadb']['mysqld']['bind_address']} "\
     command   "mysql -h #{node['mariadb']['mysqld']['bind_address']} "\
               "-P #{node['mariadb']['mysqld']['port']} -u #{dbx[:db_user]} "\
               "-P #{node['mariadb']['mysqld']['port']} -u #{dbx[:db_user]} "\
-              "-p'#{dbx[:db_pass]}' -D #{dbx_name} < #{filesql}"
-    notifies  :delete, "file[#{filesql}]"
+              "-p'#{dbx[:db_pass]}' -D #{dbx_name} < #{filepath}"
+    notifies  :delete, "file[#{filepath}]"
     sensitive true
     sensitive true
   end
   end
 
 
-  file "#{filesql}.enc" do
+  file "#{filepath}.gz.enc" do
     action :nothing
     action :nothing
   end
   end
 
 
-  file filesql do
+  file filepath do
     action :nothing
     action :nothing
   end
   end
 end
 end

@@ -10,7 +10,10 @@
 set -e
 set -e
 
 
 suffix=.backup_db_to_s3
 suffix=.backup_db_to_s3
-[ -f /tmp/*"$suffix" ] && exit 200
+if [ -f /tmp/*"$suffix" ] ; then
+    ( >&2 echo "[ERROR] Another operation might still be in progress" )
+    exit 200
+fi
 tmp_file=$( mktemp --suffix "$suffix" )
 tmp_file=$( mktemp --suffix "$suffix" )
 
 
 <% bak_dir = "#{Chef::Config[:file_cache_path]}/backup_db_to_s3" -%>
 <% bak_dir = "#{Chef::Config[:file_cache_path]}/backup_db_to_s3" -%>
@@ -24,15 +27,6 @@ aws_bin=<%= @aws_bin %>
 mysqldump_bin=<%= @mysqldump_bin %>
 mysqldump_bin=<%= @mysqldump_bin %>
 pub_key_file=<%= @pub_key_file %>
 pub_key_file=<%= @pub_key_file %>
 
 
-log_dir=/var/log/backup_db_to_s3
-if [[ ! -d "$log_dir" ]] ; then
-    mkdir -p "$log_dir"
-fi
-
-exec 3>&1 4>&2
-trap 'exec 2>&4 1>&3' 0 1 2 3
-exec 1>>"${log_dir}/backup_db_to_s3.log" 2>&1
-
 if [[ ! -d "$bak_dir" ]] ; then
 if [[ ! -d "$bak_dir" ]] ; then
     echo "$(date) : Create backup directory."
     echo "$(date) : Create backup directory."
     mkdir -p "$bak_dir"
     mkdir -p "$bak_dir"
@@ -55,7 +49,7 @@ export_db() {
 # Encrypt a file using OpenSSL and a given public key.
 # Encrypt a file using OpenSSL and a given public key.
 # The original file will be replaced by a new file, suffixed with '.enc'.
 # The original file will be replaced by a new file, suffixed with '.enc'.
 # Args:
 # Args:
-#   $1 = dump file filename, e.g. 'mydb.sql'
+#   $1 = compressed dump file filename, e.g. 'mydb.sql.gz'
 encrypt_file() {
 encrypt_file() {
     echo "$(date) : Encrypt file ${1}."
     echo "$(date) : Encrypt file ${1}."
     openssl smime -encrypt -binary -text -aes256 -in "${bak_dir}/${1}" \
     openssl smime -encrypt -binary -text -aes256 -in "${bak_dir}/${1}" \
@@ -65,7 +59,7 @@ encrypt_file() {
 
 
 # Compress the backup file with gzip.
 # Compress the backup file with gzip.
 # Args:
 # Args:
-#   $1 = dump file filename, e.g. 'mydb.sql', 'mydb.sql.enc'
+#   $1 = dump file filename, e.g. 'mydb.sql'
 compress_backup_file() {
 compress_backup_file() {
     echo "$(date) : Gzip file ${1}."
     echo "$(date) : Gzip file ${1}."
     gzip "${bak_dir}/${1}"
     gzip "${bak_dir}/${1}"
@@ -73,15 +67,14 @@ compress_backup_file() {
 
 
 # Rotate the current backups in S3.
 # Rotate the current backups in S3.
 # Args:
 # Args:
-#   $1 = dump file filename, e.g. 'mydb.sql', 'mydb.sql.enc'
+#   $1 = resulting dump filename, e.g. 'mydb.sql.gz', 'mydb.sql.gz.enc'
 #   $2 = max number of backup files to store at a time
 #   $2 = max number of backup files to store at a time
 increment_backup_names() {
 increment_backup_names() {
-    bak_keyname="${1}.gz"
+    bak_keyname=$1
     max_backups=$2
     max_backups=$2
 
 
     baks=$( "$aws_bin" --output text --region "$region" \
     baks=$( "$aws_bin" --output text --region "$region" \
-                       s3api list-objects --bucket "$bucket" \
-            | grep '^CONTENTS' | cut -f3 | grep "^${bak_keyname}" || echo "" )
+            s3 ls "s3://${bucket}/" | awk '{ printf("%s\n", $4); }' || echo "" )
 
 
     echo "$(date) : Backup rotation for ${bak_keyname}."
     echo "$(date) : Backup rotation for ${bak_keyname}."
     start=$((max_backups - 1))
     start=$((max_backups - 1))
@@ -106,15 +99,16 @@ increment_backup_names() {
 
 
 # Upload the compressed db backup file.
 # Upload the compressed db backup file.
 # Args:
 # Args:
-#   $1 = dump file filename, e.g. 'mydb.sql', 'mydb.sql.enc'
+#   $1 = resulting dump filename, e.g. 'mydb.sql.gz', 'mydb.sql.gz.enc'
 upload_to_s3() {
 upload_to_s3() {
-    echo "$(date) : Upload ${1}.gz to S3 bucket ${bucket}."
+    echo "$(date) : Upload ${1} to S3 bucket ${bucket}."
     "$aws_bin" --region "$region" \
     "$aws_bin" --region "$region" \
-               s3 mv "${bak_dir}/${1}.gz" "s3://${bucket}/${1}.gz"
+               s3 mv "${bak_dir}/${1}" "s3://${bucket}/${1}"
 }
 }
 
 
 # First, perform mysqldump on each database (and encrypt if desired):
 # First, perform mysqldump on each database (and encrypt if desired):
 
 
+<% bfname = '' -%>
 <% @db_map.each do |db| -%>
 <% @db_map.each do |db| -%>
 <%   if db.is_a?(Array) -%>
 <%   if db.is_a?(Array) -%>
 <%     db_name = db[0] -%>
 <%     db_name = db[0] -%>
@@ -123,17 +117,19 @@ upload_to_s3() {
 <%     db_name = db[:db_name] -%>
 <%     db_name = db[:db_name] -%>
 <%   end -%>
 <%   end -%>
 export_db <%= db_name %> <%= db[:db_user] %> '<%= db[:db_pass] %>' <%= db[:bak_filename] %>
 export_db <%= db_name %> <%= db[:db_user] %> '<%= db[:db_pass] %>' <%= db[:bak_filename] %>
+compress_backup_file <%= db[:bak_filename] %>
 <%   if db[:bak_encrypted] -%>
 <%   if db[:bak_encrypted] -%>
-encrypt_file <%= db[:bak_filename] %>
+<%     bfname = "#{db[:bak_filename]}.gz.enc" -%>
+encrypt_file <%= db[:bak_filename] %>.gz
+<%   else -%>
+<%     bfname = "#{db[:bak_filename]}.gz" -%>
 <%   end -%>
 <%   end -%>
 <% end -%>
 <% end -%>
 
 
-# Then compress and upload the backup files one by one:
+# Then upload the backup files one by one:
 
 
 <% @db_map.each do |db| -%>
 <% @db_map.each do |db| -%>
 <%   if db.is_a?(Array) then db = db[1] end -%>
 <%   if db.is_a?(Array) then db = db[1] end -%>
-<%     bfname = db[:bak_encrypted] ? "#{db[:bak_filename]}.enc" : db[:bak_filename] -%>
-compress_backup_file <%= bfname %>
 increment_backup_names <%= bfname %> <%= db[:bak_maxcopies] %>
 increment_backup_names <%= bfname %> <%= db[:bak_maxcopies] %>
 upload_to_s3 <%= bfname %>
 upload_to_s3 <%= bfname %>