Commit 9f0ea9b0837d8fe0cd6bddcfacd8979091b673cf

Authored by nollieheel
Committed by Earth Ugat
1 parent 986ae38c

Version 0.1.0 release

@@ -13,6 +13,14 @@ driver: @@ -13,6 +13,14 @@ driver:
13 provisioner: 13 provisioner:
14 name: chef_zero 14 name: chef_zero
15 15
  16 +platforms:
  17 + - name: ubuntu-14.04
  18 + driver:
  19 + image_id: ami-xxx
  20 + transport:
  21 + username: ubuntu
  22 + ssh_key: ~/.ssh/xxx.pem
  23 +
16 suites: 24 suites:
17 - name: default 25 - name: default
18 run_list: 26 run_list:
@@ -4,7 +4,7 @@ Just a wrapper around [`mongodb3`](https://supermarket.chef.io/cookbooks/mongodb @@ -4,7 +4,7 @@ Just a wrapper around [`mongodb3`](https://supermarket.chef.io/cookbooks/mongodb
4 4
5 ## Supported Platforms 5 ## Supported Platforms
6 6
7 -The cookbook `mongodb3` supports the most common Linux distros. 7 +The cookbook `mongodb3` supports the most common Linux distros, but this cookbook has some constants that might be Ubuntu-specific only.
8 8
9 ## Attributes 9 ## Attributes
10 10
@@ -16,10 +16,76 @@ The cookbook `mongodb3` supports the most common Linux distros. @@ -16,10 +16,76 @@ The cookbook `mongodb3` supports the most common Linux distros.
16 <th>Default</th> 16 <th>Default</th>
17 </tr> 17 </tr>
18 <tr> 18 <tr>
19 - <td><tt>['cfe-mongodb']['']</tt></td>  
20 - <td>Boolean</td>  
21 - <td>Desctd>  
22 - <td><tt>True</tt></td> 19 + <td><tt>['cfe-mongodb']['local_ipv4']</tt></td>
  20 + <td>String</td>
  21 + <td>The local IP where mongod should be listening. Should be at least a member of the mongod bind IPs. (The node's IP in a replication set.)<td>
  22 + <td><tt>'127.0.0.1'</tt></td>
  23 + </tr>
  24 + <tr>
  25 + <td><tt>['cfe-mongodb']['s3_region']</tt></td>
  26 + <td>String</td>
  27 + <td>S3 region if using the backup script.<td>
  28 + <td><tt>'us-east-1'</tt></td>
  29 + </tr>
  30 + <tr>
  31 + <td><tt>['cfe-mongodb']['s3_bucket']</tt></td>
  32 + <td>String</td>
  33 + <td>S3 bucket if using the backup script<td>
  34 + <td><tt>'test-bucket'</tt></td>
  35 + </tr>
  36 + <tr>
  37 + <td><tt>['cfe-mongodb']['db']['map']</tt></td>
  38 + <td>Hash/Array</td>
  39 + <td>Details of every database to set up, including users and passwords, etc. Please refer to the default attributes file.<td>
  40 + <td><tt>{}</tt></td>
  41 + </tr>
  42 + <tr>
  43 + <td><tt>['cfe-mongodb']['db']['pass_root']</tt></td>
  44 + <td>String</td>
  45 + <td>The mongodb password for user 'root'<td>
  46 + <td><tt>'secret'</tt></td>
  47 + </tr>
  48 + <tr>
  49 + <td><tt>['cfe-mongodb']['db']['pass_backup']</tt></td>
  50 + <td>String</td>
  51 + <td>The mongodb password for user 'backup', used for performing mongodumps during the backup script operation.<td>
  52 + <td><tt>'secret'</tt></td>
  53 + </tr>
  54 + <tr>
  55 + <td><tt>['cfe-mongodb']['rs']['key']</tt></td>
  56 + <td>String</td>
  57 + <td>Contents of replication key.<td>
  58 + <td><tt>'xxxx'</tt></td>
  59 + </tr>
  60 + <tr>
  61 + <td><tt>['cfe-mongodb']['rs']['nodes']['primary']</tt></td>
  62 + <td>String</td>
  63 + <td>The primary mongodb node during initial Chef run, e.g. '10.0.0.1:27017'.<td>
  64 + <td><tt>''</tt></td>
  65 + </tr>
  66 + <tr>
  67 + <td><tt>['cfe-mongodb']['rs']['nodes']['secondary']</tt></td>
  68 + <td>Array</td>
  69 + <td>Array of secondary mongodb nodes during initial Chef run, e.g. ['10.0.0.2:27017', '10.0.0.3:27017'].<td>
  70 + <td><tt>[]</tt></td>
  71 + </tr>
  72 + <tr>
  73 + <td><tt>['cfe-mongodb']['rs']['nodes']['arbiter']</tt></td>
  74 + <td>Array</td>
  75 + <td>Array of arbiter mongodb nodes during initial Chef run, e.g. ['10.0.0.4:27017'].<td>
  76 + <td><tt>[]</tt></td>
  77 + </tr>
  78 + <tr>
  79 + <td><tt>['cfe-mongodb']['install']['bak_sched']</tt></td>
  80 + <td>String</td>
  81 + <td>Crontab schedule for running the backup script.<td>
  82 + <td><tt>'0 7 * * *'</tt></td>
  83 + </tr>
  84 + <tr>
  85 + <td><tt>['cfe-mongodb']['encrypt']['pub_key']</tt></td>
  86 + <td>String</td>
  87 + <td>Encryption public key if at least one of the databases specified in the DB map is to be encrypted when it is automatically backed up.<td>
  88 + <td><tt>nil</tt></td>
23 </tr> 89 </tr>
24 </table> 90 </table>
25 91
@@ -37,6 +103,19 @@ After setting proper node attributes, include `cfe-mongodb` in your node's `run_ @@ -37,6 +103,19 @@ After setting proper node attributes, include `cfe-mongodb` in your node's `run_
37 } 103 }
38 ``` 104 ```
39 105
  106 +### cfe-mongodb::backup2s3
  107 +
  108 +Sets up an automated backup script that uploads DB backups into an S3 bucket:
  109 +
  110 +```json
  111 +{
  112 + "run_list": [
  113 + "recipe[cfe-mongodb::default]",
  114 + "recipe[cfe-mongodb::backup2s3]"
  115 + ]
  116 +}
  117 +```
  118 +
40 ## License and Authors 119 ## License and Authors
41 120
42 Author:: Earth U. (<sysadmin @ chromedia.com>) 121 Author:: Earth U. (<sysadmin @ chromedia.com>)
@@ -17,3 +17,76 @@ @@ -17,3 +17,76 @@
17 # See the License for the specific language governing permissions and 17 # See the License for the specific language governing permissions and
18 # limitations under the License. 18 # limitations under the License.
19 # 19 #
  20 +
  21 +default['cfe-mongodb']['local_ipv4'] = '127.0.0.1'
  22 +
  23 +default['cfe-mongodb']['s3_region'] = 'us-east-1'
  24 +default['cfe-mongodb']['s3_bucket'] = 'test-bucket'
  25 +
  26 +default['cfe-mongodb']['db']['map'] = {
  27 + # The db_map format is:
  28 + # 'example_db_name' => {
  29 + # :db_user => '<name_of_custom_user>',
  30 + # :db_pass => '<name_of_custom_pass>',
  31 + #
  32 + # Optional:
  33 + # :db_auth => '<name_of_authentication_db>' # default is db_name
  34 + # :backup => # default: true
  35 + # :bak_encrypted => # default: false
  36 + # :bak_filename => # default is db_name
  37 + # :bak_maxcopies => # default: 30
  38 + # }
  39 +}
  40 +default['cfe-mongodb']['db']['pass_root'] = 'secret'
  41 +default['cfe-mongodb']['db']['pass_backup'] = 'secret'
  42 +
  43 +# Create custom key contents with:
  44 +# $ openssl rand -base64 756
  45 +default['cfe-mongodb']['rs']['key'] = 'supersecretkeyxxx'
  46 +default['cfe-mongodb']['rs']['delay'] = 2
  47 +# Note: the following attributes for primary/secondary/arbiter
  48 +# should all include the port number (e.g. '1.2.3.4:27017')
  49 +default['cfe-mongodb']['rs']['nodes']['primary'] = ''
  50 +default['cfe-mongodb']['rs']['nodes']['secondary'] = []
  51 +default['cfe-mongodb']['rs']['nodes']['arbiter'] = []
  52 +
  53 +default['cfe-mongodb']['install']['priv_dir'] = '/opt/mongodb/priv'
  54 +default['cfe-mongodb']['install']['bak_log_dir'] = '/var/log/mongodb_backup2s3'
  55 +default['cfe-mongodb']['install']['bak_sched'] = '0 7 * * *'
  56 +
  57 +default['cfe-mongodb']['encrypt']['priv_key'] = nil
  58 +default['cfe-mongodb']['encrypt']['pub_key'] = nil
  59 +
  60 +## Constant logrotate options if automated backups are used
  61 +
  62 +default['cfe-mongodb']['logrotate']['conf_dir'] = '/etc/logrotate.d'
  63 +default['cfe-mongodb']['logrotate']['options'] = %w{
  64 + weekly
  65 + rotate\ 12
  66 + missingok
  67 + compress
  68 + notifempty
  69 +}
  70 +
  71 +## Constant location of binaries (at least for Ubuntu 14.04)
  72 +
  73 +default['cfe-mongodb']['bin']['aws'] = '/usr/local/bin/aws'
  74 +default['cfe-mongodb']['bin']['mongo'] = '/usr/bin/mongo'
  75 +default['cfe-mongodb']['bin']['mongodump'] = '/usr/bin/mongodump'
  76 +default['cfe-mongodb']['bin']['openssl'] = '/usr/bin/openssl'
  77 +
  78 +## mongodb3 attributes
  79 +
  80 +default['mongodb3']['version'] = '3.2.8'
  81 +
  82 +default['mongodb3']['mongod']['disable-transparent-hugepages'] = true
  83 +
  84 +default['mongodb3']['config']['mongod']['net']['port'] = 27017
  85 +default['mongodb3']['config']['mongod']['net']['bindIp'] = '127.0.0.1'
  86 +
  87 +default['mongodb3']['config']['mongod']['security']['authorization'] = 'enabled'
  88 +default['mongodb3']['config']['mongod']['security']['keyFile'] =
  89 + "#{node['cfe-mongodb']['install']['priv_dir']}/mongod.rs.key"
  90 +
  91 +default['mongodb3']['config']['mongod']['replication']['replSetName'] = 'test'
  92 +default['mongodb3']['config']['mongod']['replication']['oplogSizeMB'] = '1024'
@@ -5,3 +5,10 @@ license 'Apache License' @@ -5,3 +5,10 @@ license 'Apache License'
5 description 'Simplifies setup of staging MongoDB servers' 5 description 'Simplifies setup of staging MongoDB servers'
6 long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) 6 long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
7 version '0.1.0' 7 version '0.1.0'
  8 +
  9 +depends 'mongodb3', '~> 5.3.0'
  10 +depends 'awscli', '~> 1.0.1'
  11 +depends 'openssl', '~> 4.4.0'
  12 +depends 'cron', '~> 1.7.4'
  13 +
  14 +supports 'ubuntu', '14.04'
  1 +#
  2 +# Author:: Earth U (<sysadmin @ chromedia.com>)
  3 +# Cookbook Name:: cfe-mongodb
  4 +# Recipe:: backup2s3
  5 +#
  6 +# Copyright (C) 2016, Chromedia Far East, Inc.
  7 +#
  8 +# Licensed under the Apache License, Version 2.0 (the "License");
  9 +# you may not use this file except in compliance with the License.
  10 +# You may obtain a copy of the License at
  11 +#
  12 +# http://www.apache.org/licenses/LICENSE-2.0
  13 +#
  14 +# Unless required by applicable law or agreed to in writing, software
  15 +# distributed under the License is distributed on an "AS IS" BASIS,
  16 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 +# See the License for the specific language governing permissions and
  18 +# limitations under the License.
  19 +#
  20 +
  21 +package 'gzip'
  22 +package 'logrotate'
  23 +include_recipe 'openssl::upgrade'
  24 +include_recipe 'awscli'
  25 +
  26 +pub_key = "#{node[cookbook_name]['install']['priv_dir']}/pub.key"
  27 +bscript = "#{node[cookbook_name]['install']['priv_dir']}/mongodb_backup2s3"
  28 +
  29 +ip = (node['mongodb3']['config']['mongod']['net']['bindIp'].split(','))[-1]
  30 +
  31 +directory(node[cookbook_name]['install']['bak_log_dir']) { recursive true }
  32 +
  33 +is_any_enc = node[cookbook_name]['db']['map'].any? do |x|
  34 + if x.is_a?(Array)
  35 + x = x[1]
  36 + end
  37 + do_backup = x.has_key?(:backup) ? x[:backup] : true
  38 + do_backup ? x[:bak_encrypted] : false
  39 +end
  40 +if !node[cookbook_name]['encrypt']['pub_key'] && is_any_enc
  41 + Chef::Application.fatal!('No encryption public key contents supplied')
  42 +end
  43 +
  44 +file pub_key do
  45 + content node[cookbook_name]['encrypt']['pub_key']
  46 + mode 0600
  47 + owner 'root'
  48 + group 'root'
  49 + sensitive true
  50 + only_if { is_any_enc }
  51 +end
  52 +
  53 +template bscript do
  54 + mode 0700
  55 + owner 'root'
  56 + group 'root'
  57 + sensitive true
  58 + variables(
  59 + :bin_aws => node[cookbook_name]['bin']['aws'],
  60 + :bin_mongo => node[cookbook_name]['bin']['mongo'],
  61 + :bin_mongodump => node[cookbook_name]['bin']['mongodump'],
  62 + :bin_openssl => node[cookbook_name]['bin']['openssl'],
  63 +
  64 + :db_host => ip,
  65 + :db_port => node['mongodb3']['config']['mongod']['net']['port'],
  66 + :db_map => node[cookbook_name]['db']['map'],
  67 +
  68 + :backup_user => 'backup',
  69 + :backup_pass => node[cookbook_name]['db']['pass_backup'],
  70 + :backup_auth => 'admin',
  71 +
  72 + :s3_region => node[cookbook_name]['s3_region'],
  73 + :s3_bucket => node[cookbook_name]['s3_bucket'],
  74 +
  75 + :pub_key => pub_key
  76 + )
  77 +end
  78 +
  79 +sched = node[cookbook_name]['install']['bak_sched'].split(' ')
  80 +cron_d 'mongodb_backup2s3' do
  81 + command "bash #{bscript} >> #{node[cookbook_name]['install']['bak_log_dir']}"\
  82 + '/mongodb_backup2s3.log 2>&1'
  83 + minute sched[0]
  84 + hour sched[1]
  85 + day sched[2]
  86 + month sched[3]
  87 + weekday sched[4]
  88 + mailto "''"
  89 + path '/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin'
  90 +end
  91 +
  92 +template "#{node[cookbook_name]['logrotate']['conf_dir']}/mongodb_backup2s3" do
  93 + source 'logrotate.erb'
  94 + variables(
  95 + :log_dir => node[cookbook_name]['install']['bak_log_dir'],
  96 + :opts => node[cookbook_name]['logrotate']['options']
  97 + )
  98 +end
@@ -17,3 +17,6 @@ @@ -17,3 +17,6 @@
17 # See the License for the specific language governing permissions and 17 # See the License for the specific language governing permissions and
18 # limitations under the License. 18 # limitations under the License.
19 # 19 #
  20 +
  21 +include_recipe 'mongodb3'
  22 +include_recipe "#{cookbook_name}::replicate"
  1 +#
  2 +# Author:: Earth U (<sysadmin @ chromedia.com>)
  3 +# Cookbook Name:: cfe-mongodb
  4 +# Recipe:: replicate
  5 +#
  6 +# Copyright (C) 2016, Chromedia Far East, Inc.
  7 +#
  8 +# Licensed under the Apache License, Version 2.0 (the "License");
  9 +# you may not use this file except in compliance with the License.
  10 +# You may obtain a copy of the License at
  11 +#
  12 +# http://www.apache.org/licenses/LICENSE-2.0
  13 +#
  14 +# Unless required by applicable law or agreed to in writing, software
  15 +# distributed under the License is distributed on an "AS IS" BASIS,
  16 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 +# See the License for the specific language governing permissions and
  18 +# limitations under the License.
  19 +#
  20 +
  21 +directory(node[cookbook_name]['install']['priv_dir']) { recursive true }
  22 +
  23 +file node['mongodb3']['config']['mongod']['security']['keyFile'] do
  24 + content node[cookbook_name]['rs']['key']
  25 + owner node['mongodb3']['user']
  26 + group node['mongodb3']['group']
  27 + mode 0400
  28 + sensitive true
  29 + notifies :restart, 'service[mongod]', :immediately
  30 + notifies :reboot_now, 'reboot[reboot_before_replication]', :immediately
  31 + only_if { node[cookbook_name]['rs']['key'] }
  32 +end
  33 +
  34 +reboot 'reboot_before_replication' do
  35 + action :nothing
  36 + reason 'Node needs to restart before initiating MongoDB replication'
  37 +end
  38 +
  39 +server = "#{node[cookbook_name]['local_ipv4']}"\
  40 + ":#{node['mongodb3']['config']['mongod']['net']['port']}"
  41 +if node['cfe-mongodb']['rs']['nodes']['primary'] == server
  42 + include_recipe "#{cookbook_name}::replicate_pri_init"
  43 + include_recipe "#{cookbook_name}::replicate_pri_addnodes"
  44 +end
  1 +#
  2 +# Author:: Earth U (<sysadmin @ chromedia.com>)
  3 +# Cookbook Name:: cfe-mongodb
  4 +# Recipe:: replicate_pri_addnodes
  5 +#
  6 +# Copyright (C) 2016, Chromedia Far East, Inc.
  7 +#
  8 +# Licensed under the Apache License, Version 2.0 (the "License");
  9 +# you may not use this file except in compliance with the License.
  10 +# You may obtain a copy of the License at
  11 +#
  12 +# http://www.apache.org/licenses/LICENSE-2.0
  13 +#
  14 +# Unless required by applicable law or agreed to in writing, software
  15 +# distributed under the License is distributed on an "AS IS" BASIS,
  16 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 +# See the License for the specific language governing permissions and
  18 +# limitations under the License.
  19 +#
  20 +
  21 +mongod = node['mongodb3']['config']['mongod']
  22 +ip = (mongod['net']['bindIp'].split(','))[-1]
  23 +port = mongod['net']['port']
  24 +
  25 +com = "sleep #{node[cookbook_name]['rs']['delay']} "\
  26 + "&& mongo --host #{ip} --port #{port} "\
  27 + "-u root -p #{node[cookbook_name]['db']['pass_root']} "\
  28 + "--authenticationDatabase admin --eval 'rs.add<<1>>(\"<<2>>\")' "\
  29 + ">> #{::File.dirname(mongod['systemLog']['path'])}/primary_addnodes"
  30 +
  31 +node[cookbook_name]['rs']['nodes']['secondary'].each do |sec|
  32 + execute com.sub('<<1>>', '').sub('<<2>>', sec)
  33 +end
  34 +
  35 +node[cookbook_name]['rs']['nodes']['arbiter'].each do |arb|
  36 + execute com.sub('<<1>>', 'Arb').sub('<<2>>', arb)
  37 +end
  1 +#
  2 +# Author:: Earth U (<sysadmin @ chromedia.com>)
  3 +# Cookbook Name:: cfe-mongodb
  4 +# Recipe:: replicate_pri_init
  5 +#
  6 +# Copyright (C) 2016, Chromedia Far East, Inc.
  7 +#
  8 +# Licensed under the Apache License, Version 2.0 (the "License");
  9 +# you may not use this file except in compliance with the License.
  10 +# You may obtain a copy of the License at
  11 +#
  12 +# http://www.apache.org/licenses/LICENSE-2.0
  13 +#
  14 +# Unless required by applicable law or agreed to in writing, software
  15 +# distributed under the License is distributed on an "AS IS" BASIS,
  16 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  17 +# See the License for the specific language governing permissions and
  18 +# limitations under the License.
  19 +#
  20 +
  21 +# NOTE:
  22 +# The recipe cfe-mongodb::replicate must be run before this
  23 +# in order to restart the node beforehand.
  24 +# Otherwise, manually restart the node first because that seems to
  25 +# be the only way for an automated rs.initiate() to proceed.
  26 +# Doing it immediately after mongodb install does not work,
  27 +# nor does implementing delay, or even restarting the mongod service.
  28 +
  29 +# NOTE:
  30 +# It is recommended that this cookbook be run on secondary and
  31 +# arbiter nodes first, before finally running it on the primary node.
  32 +
  33 +log = node['mongodb3']['config']['mongod']['systemLog']
  34 +port = node['mongodb3']['config']['mongod']['net']['port']
  35 +
  36 +boot_mark = "#{::File.dirname(log['path'])}/primary_init_mark"
  37 +boot_log = "#{::File.dirname(log['path'])}/primary_init"
  38 +is_logrename = log['logRotate'] == 'rename'
  39 +
  40 +boot_script = "#{Chef::Config[:file_cache_path]}/mongodb_pri_bootstrap.js"
  41 +
  42 +template boot_script do
  43 + sensitive true
  44 + variables(
  45 + :db_map => node[cookbook_name]['db']['map'],
  46 + :db_pass_root => node[cookbook_name]['db']['pass_root'],
  47 + :db_pass_backup => node[cookbook_name]['db']['pass_backup'],
  48 + :is_logrename => is_logrename
  49 + )
  50 +end
  51 +
  52 +script 'mongodb_pri_bootstrap' do
  53 + interpreter 'bash'
  54 + not_if { ::File.exist?(boot_mark) }
  55 + code <<-CODE
  56 + set -e
  57 + mongo --host 127.0.0.1 --port #{port} --eval "rs.initiate()" >> #{boot_log}
  58 + sleep #{node[cookbook_name]['rs']['delay']}
  59 + mongo #{boot_script} --host 127.0.0.1 --port #{port} >> #{boot_log}
  60 + date > #{boot_mark}
  61 + CODE
  62 +end
  63 +
  64 +file boot_script do
  65 + action :delete
  66 +end
  1 +<%= @log_dir %>/*.log {
  2 +<% @opts.each do |opt| -%>
  3 + <%= opt %>
  4 +<% end -%>
  5 +}
  1 +#!/bin/bash
  2 +#
  3 +# This file was generated by CHEF. Changes will be overwritten!
  4 +
  5 +# Backup script for MongoDB in AWS EC2 instances.
  6 +#
  7 +# If this mongod instance is primary, then backup the
  8 +# designated database/s and upload it to S3. Encrypt first if needed.
  9 +#
  10 +# Depends on:
  11 +# MongoDB
  12 +# AWSCLI
  13 +# OpenSSL
  14 +
  15 +set -e
  16 +
  17 +suffix=.mongodb_backup2s3
  18 +if [ -f /tmp/*"$suffix" ] ; then
  19 + ( >&2 echo "[ERROR] Another operation might still be in progress" )
  20 + exit 200
  21 +fi
  22 +tmp_file=$( mktemp --suffix "$suffix" )
  23 +
  24 +bin_aws=<%= @bin_aws %>
  25 +bin_mongo=<%= @bin_mongo %>
  26 +bin_mongodump=<%= @bin_mongodump %>
  27 +bin_openssl=<%= @bin_openssl %>
  28 +
  29 +db_host=<%= @db_host %>
  30 +db_port=<%= @db_port %>
  31 +db_user='<%= @backup_user %>'
  32 +db_pass='<%= @backup_pass %>'
  33 +db_auth=<%= @backup_auth %>
  34 +
  35 +s3_bucket=<%= @s3_bucket %>
  36 +s3_region=<%= @s3_region %>
  37 +
  38 +pub_key=<%= @pub_key %>
  39 +<% bak_dir = "#{Chef::Config[:file_cache_path]}/mongodb_backup2s3" -%>
  40 +bak_dir=<%= bak_dir %>
  41 +
  42 +# Perform the actual mongodump
  43 +# Args:
  44 +# $1 = db name
  45 +# $2 = backup dump filename, e.g. 'mydb'
  46 +export_db() {
  47 + echo "$(date) : Export database ${1} to ${bak_dir}."
  48 + rm -f "${bak_dir}/${2}.gz"
  49 +
  50 + "$bin_mongodump" --host="${db_host}:${db_port}" \
  51 + -u "$db_user" -p "$db_pass" --authenticationDatabase="${db_auth}" \
  52 + --dumpDbUsersAndRoles --gzip -d "${1}" --archive="${bak_dir}/${1}.gz"
  53 +}
  54 +
  55 +# Encrypt the backup file with OpenSSL
  56 +# Args:
  57 +# $1 = compressed dump filename, e.g. 'mydb.gz'
  58 +encrypt_backup() {
  59 + echo "$(date) : Encrypt file ${1}."
  60 + rm -f "${bak_dir}/${1}.enc"
  61 +
  62 + "$bin_openssl" smime -encrypt -binary -text -aes256 \
  63 + -in "${bak_dir}/${1}" -out "${bak_dir}/${1}.enc" \
  64 + -outform DER "$pub_key"
  65 + rm "${bak_dir}/${1}"
  66 +}
  67 +
  68 +# Rotate the current backups in S3
  69 +# Args:
  70 +# $1 = backup dump filename, e.g. 'mydb'
  71 +# $2 = max number of backup files to store at a time
  72 +rotate_backup() {
  73 + folder=$1
  74 + max=$2
  75 +
  76 + # Backups will stored inside subfolders (prefixes)
  77 + # in S3, so only look at 'PRE' objects.
  78 + baks=$( "$bin_aws" --output text --region "$s3_region" \
  79 + s3 ls "s3://${s3_bucket}/" | grep '^\s*PRE' | \
  80 + sed -e 's/^ *PRE //' -e 's/\/$//' | \
  81 + grep "^${folder}" || echo "" )
  82 +
  83 + echo "$(date) : Backup rotation for ${folder}."
  84 + start=$((max - 1))
  85 +
  86 + for (( x=start ; x > 0 ; x-- )) ; do
  87 + if echo "$baks" | grep "^${folder}\\.${x}\$" ; then
  88 + newx=$((x + 1))
  89 + if [[ $newx -lt $max ]] ; then
  90 + "$bin_aws" --region "$s3_region" \
  91 + s3 mv --recursive "s3://${s3_bucket}/${folder}.${x}" \
  92 + "s3://${s3_bucket}/${folder}.${newx}"
  93 + else
  94 + "$bin_aws" --region "$s3_region" \
  95 + s3 rm --recursive "s3://${s3_bucket}/${folder}.${x}"
  96 + fi
  97 + fi
  98 + done
  99 +
  100 + if echo "$baks" | grep "^${folder}\$" ; then
  101 + if [[ $max -gt 1 ]] ; then
  102 + "$bin_aws" --region "$s3_region" \
  103 + s3 mv --recursive "s3://${s3_bucket}/${folder}" \
  104 + "s3://${s3_bucket}/${folder}.1"
  105 + else
  106 + "$bin_aws" --region "$s3_region" \
  107 + s3 rm --recursive "s3://${s3_bucket}/${folder}"
  108 + fi
  109 + fi
  110 +}
  111 +
  112 +# Upload the compressed db backup file. It will be uploaded
  113 +# to this example location: ${s3_bucket}/dump_filename/dump_filename.gz.enc
  114 +#
  115 +# A timestamp file will also be uploaded to this location:
  116 +# ${s3_bucket}/dump_filename/YYYY-MM-DDThh:mm:ss.txt
  117 +# Args:
  118 +# $1 = backup dump filename, e.g. 'mydb'
  119 +# $2 = if file is encrypted or not (boolean)
  120 +upload_to_s3() {
  121 + keyname=$1
  122 + if [ "$2" = true ] ; then
  123 + fname="${keyname}.gz.enc"
  124 + else
  125 + fname="${keyname}.gz"
  126 + fi
  127 +
  128 + echo "$(date) : Upload ${fname} to S3 bucket ${s3_bucket}."
  129 +
  130 + stamp=$( date +"%FT%T" )
  131 + echo "Uploaded: ${stamp}" > "${bak_dir}/${stamp}.txt"
  132 +
  133 + "$bin_aws" --region "$s3_region" \
  134 + s3 mv "${bak_dir}/${fname}" \
  135 + "s3://${s3_bucket}/${keyname}/${fname}"
  136 + "$bin_aws" --region "$s3_region" \
  137 + s3 mv "${bak_dir}/${stamp}.txt" \
  138 + "s3://${s3_bucket}/${keyname}/${stamp}.txt"
  139 +}
  140 +
  141 +## Do the backup only if this node is the primary:
  142 +if "$bin_mongo" --host="${db_host}:${db_port}" -u "$db_user" \
  143 + -p "$db_pass" --authenticationDatabase="${db_auth}" \
  144 + --eval 'db.isMaster()["ismaster"]' | grep -q true ; then
  145 +
  146 + if [[ ! -d "$bak_dir" ]] ; then
  147 + mkdir -p "$bak_dir"
  148 + fi
  149 +
  150 +<% @db_map.each do |x| -%>
  151 +<%
  152 + if x.is_a?(Array)
  153 + db_name = x[0]
  154 + x = x[1]
  155 + else
  156 + db_name = x[:db_name]
  157 + end
  158 +
  159 + do_backup = x.has_key?(:backup) ? x[:backup] : true
  160 + is_enc = x.has_key?(:bak_encrypted) ? x[:bak_encrypted] : false
  161 + bak_filename = x[:bak_filename] || db_name
  162 + bak_maxcopies = x[:bak_maxcopies] || 30
  163 +-%>
  164 +<% if do_backup -%>
  165 + # Database: <%= db_name %>
  166 + export_db <%= db_name %> <%= bak_filename %>
  167 +<% if is_enc -%>
  168 + encrypt_backup <%= bak_filename %>.gz
  169 +<% end -%>
  170 + rotate_backup <%= bak_filename %> <%= bak_maxcopies %>
  171 + upload_to_s3 <%= bak_filename %> <%= is_enc %>
  172 +
  173 +<% end -%>
  174 +<% end -%>
  175 + echo "$(date) : Done."
  176 +fi
  177 +
  178 +rm "$tmp_file"
  1 +var db_pass_root = "<%= @db_pass_root %>";
  2 +var db_pass_backup = "<%= @db_pass_backup %>";
  3 +
  4 +db = db.getSiblingDB("admin");
  5 +db.createUser( { user: "root", pwd: db_pass_root, roles: [ "root", "__system" ] } );
  6 +db.auth("root", db_pass_root);
  7 +
  8 +<% if @is_logrename -%>
  9 +db.runCommand( { logRotate : 1 } );
  10 +
  11 +<% end -%>
  12 +db.createUser( { user: "backup", pwd: db_pass_backup, roles: [ "backup" ] } );
  13 +
  14 +<% @db_map.each do |x| -%>
  15 +<%
  16 + if x.is_a?(Array)
  17 + db_name = x[0]
  18 + x = x[1]
  19 + else
  20 + db_name = x[:db_name]
  21 + end
  22 +
  23 + db_auth = x.has_key?(:db_auth) ? x[:db_auth] : db_name
  24 +-%>
  25 +db = db.getSiblingDB("<%= db_auth %>");
  26 +db.createUser( { user: "<%= x[:db_user] %>", pwd: "<%= x[:db_pass] %>", roles: [ { role: "readWrite", db: "<%= db_name %>" }, { role: "dbAdmin", db: "<%= db_name %>" } ] } );
  27 +
  28 +<% end -%>