Commit 9f0ea9b0837d8fe0cd6bddcfacd8979091b673cf
Committed by
Earth Ugat

1 parent
986ae38c
Version 0.1.0 release
Showing
13 changed files
with
631 additions
and
5 deletions
... | ... | @@ -4,7 +4,7 @@ Just a wrapper around [`mongodb3`](https://supermarket.chef.io/cookbooks/mongodb |
4 | 4 | |
5 | 5 | ## Supported Platforms |
6 | 6 | |
7 | -The cookbook `mongodb3` supports the most common Linux distros. | |
7 | +The cookbook `mongodb3` supports the most common Linux distros, but this cookbook has some constants that might be Ubuntu-specific only. | |
8 | 8 | |
9 | 9 | ## Attributes |
10 | 10 | |
... | ... | @@ -16,10 +16,76 @@ The cookbook `mongodb3` supports the most common Linux distros. |
16 | 16 | <th>Default</th> |
17 | 17 | </tr> |
18 | 18 | <tr> |
19 | - <td><tt>['cfe-mongodb']['']</tt></td> | |
20 | - <td>Boolean</td> | |
21 | - <td>Desctd> | |
22 | - <td><tt>True</tt></td> | |
19 | + <td><tt>['cfe-mongodb']['local_ipv4']</tt></td> | |
20 | + <td>String</td> | |
21 | + <td>The local IP where mongod should be listening. Should be at least a member of the mongod bind IPs. (The node's IP in a replication set.)<td> | |
22 | + <td><tt>'127.0.0.1'</tt></td> | |
23 | + </tr> | |
24 | + <tr> | |
25 | + <td><tt>['cfe-mongodb']['s3_region']</tt></td> | |
26 | + <td>String</td> | |
27 | + <td>S3 region if using the backup script.<td> | |
28 | + <td><tt>'us-east-1'</tt></td> | |
29 | + </tr> | |
30 | + <tr> | |
31 | + <td><tt>['cfe-mongodb']['s3_bucket']</tt></td> | |
32 | + <td>String</td> | |
33 | + <td>S3 bucket if using the backup script<td> | |
34 | + <td><tt>'test-bucket'</tt></td> | |
35 | + </tr> | |
36 | + <tr> | |
37 | + <td><tt>['cfe-mongodb']['db']['map']</tt></td> | |
38 | + <td>Hash/Array</td> | |
39 | + <td>Details of every database to set up, including users and passwords, etc. Please refer to the default attributes file.<td> | |
40 | + <td><tt>{}</tt></td> | |
41 | + </tr> | |
42 | + <tr> | |
43 | + <td><tt>['cfe-mongodb']['db']['pass_root']</tt></td> | |
44 | + <td>String</td> | |
45 | + <td>The mongodb password for user 'root'<td> | |
46 | + <td><tt>'secret'</tt></td> | |
47 | + </tr> | |
48 | + <tr> | |
49 | + <td><tt>['cfe-mongodb']['db']['pass_backup']</tt></td> | |
50 | + <td>String</td> | |
51 | + <td>The mongodb password for user 'backup', used for performing mongodumps during the backup script operation.<td> | |
52 | + <td><tt>'secret'</tt></td> | |
53 | + </tr> | |
54 | + <tr> | |
55 | + <td><tt>['cfe-mongodb']['rs']['key']</tt></td> | |
56 | + <td>String</td> | |
57 | + <td>Contents of replication key.<td> | |
58 | + <td><tt>'xxxx'</tt></td> | |
59 | + </tr> | |
60 | + <tr> | |
61 | + <td><tt>['cfe-mongodb']['rs']['nodes']['primary']</tt></td> | |
62 | + <td>String</td> | |
63 | + <td>The primary mongodb node during initial Chef run, e.g. '10.0.0.1:27017'.<td> | |
64 | + <td><tt>''</tt></td> | |
65 | + </tr> | |
66 | + <tr> | |
67 | + <td><tt>['cfe-mongodb']['rs']['nodes']['secondary']</tt></td> | |
68 | + <td>Array</td> | |
69 | + <td>Array of secondary mongodb nodes during initial Chef run, e.g. ['10.0.0.2:27017', '10.0.0.3:27017'].<td> | |
70 | + <td><tt>[]</tt></td> | |
71 | + </tr> | |
72 | + <tr> | |
73 | + <td><tt>['cfe-mongodb']['rs']['nodes']['arbiter']</tt></td> | |
74 | + <td>Array</td> | |
75 | + <td>Array of arbiter mongodb nodes during initial Chef run, e.g. ['10.0.0.4:27017'].<td> | |
76 | + <td><tt>[]</tt></td> | |
77 | + </tr> | |
78 | + <tr> | |
79 | + <td><tt>['cfe-mongodb']['install']['bak_sched']</tt></td> | |
80 | + <td>String</td> | |
81 | + <td>Crontab schedule for running the backup script.<td> | |
82 | + <td><tt>'0 7 * * *'</tt></td> | |
83 | + </tr> | |
84 | + <tr> | |
85 | + <td><tt>['cfe-mongodb']['encrypt']['pub_key']</tt></td> | |
86 | + <td>String</td> | |
87 | + <td>Encryption public key if at least one of the databases specified in the DB map is to be encrypted when it is automatically backed up.<td> | |
88 | + <td><tt>nil</tt></td> | |
23 | 89 | </tr> |
24 | 90 | </table> |
25 | 91 | |
... | ... | @@ -37,6 +103,19 @@ After setting proper node attributes, include `cfe-mongodb` in your node's `run_ |
37 | 103 | } |
38 | 104 | ``` |
39 | 105 | |
106 | +### cfe-mongodb::backup2s3 | |
107 | + | |
108 | +Sets up an automated backup script that uploads DB backups into an S3 bucket: | |
109 | + | |
110 | +```json | |
111 | +{ | |
112 | + "run_list": [ | |
113 | + "recipe[cfe-mongodb::default]", | |
114 | + "recipe[cfe-mongodb::backup2s3]" | |
115 | + ] | |
116 | +} | |
117 | +``` | |
118 | + | |
40 | 119 | ## License and Authors |
41 | 120 | |
42 | 121 | Author:: Earth U. (<sysadmin @ chromedia.com>) | ... | ... |
... | ... | @@ -17,3 +17,76 @@ |
17 | 17 | # See the License for the specific language governing permissions and |
18 | 18 | # limitations under the License. |
19 | 19 | # |
20 | + | |
21 | +default['cfe-mongodb']['local_ipv4'] = '127.0.0.1' | |
22 | + | |
23 | +default['cfe-mongodb']['s3_region'] = 'us-east-1' | |
24 | +default['cfe-mongodb']['s3_bucket'] = 'test-bucket' | |
25 | + | |
26 | +default['cfe-mongodb']['db']['map'] = { | |
27 | + # The db_map format is: | |
28 | + # 'example_db_name' => { | |
29 | + # :db_user => '<name_of_custom_user>', | |
30 | + # :db_pass => '<name_of_custom_pass>', | |
31 | + # | |
32 | + # Optional: | |
33 | + # :db_auth => '<name_of_authentication_db>' # default is db_name | |
34 | + # :backup => # default: true | |
35 | + # :bak_encrypted => # default: false | |
36 | + # :bak_filename => # default is db_name | |
37 | + # :bak_maxcopies => # default: 30 | |
38 | + # } | |
39 | +} | |
40 | +default['cfe-mongodb']['db']['pass_root'] = 'secret' | |
41 | +default['cfe-mongodb']['db']['pass_backup'] = 'secret' | |
42 | + | |
43 | +# Create custom key contents with: | |
44 | +# $ openssl rand -base64 756 | |
45 | +default['cfe-mongodb']['rs']['key'] = 'supersecretkeyxxx' | |
46 | +default['cfe-mongodb']['rs']['delay'] = 2 | |
47 | +# Note: the following attributes for primary/secondary/arbiter | |
48 | +# should all include the port number (e.g. '1.2.3.4:27017') | |
49 | +default['cfe-mongodb']['rs']['nodes']['primary'] = '' | |
50 | +default['cfe-mongodb']['rs']['nodes']['secondary'] = [] | |
51 | +default['cfe-mongodb']['rs']['nodes']['arbiter'] = [] | |
52 | + | |
53 | +default['cfe-mongodb']['install']['priv_dir'] = '/opt/mongodb/priv' | |
54 | +default['cfe-mongodb']['install']['bak_log_dir'] = '/var/log/mongodb_backup2s3' | |
55 | +default['cfe-mongodb']['install']['bak_sched'] = '0 7 * * *' | |
56 | + | |
57 | +default['cfe-mongodb']['encrypt']['priv_key'] = nil | |
58 | +default['cfe-mongodb']['encrypt']['pub_key'] = nil | |
59 | + | |
60 | +## Constant logrotate options if automated backups are used | |
61 | + | |
62 | +default['cfe-mongodb']['logrotate']['conf_dir'] = '/etc/logrotate.d' | |
63 | +default['cfe-mongodb']['logrotate']['options'] = %w{ | |
64 | + weekly | |
65 | + rotate\ 12 | |
66 | + missingok | |
67 | + compress | |
68 | + notifempty | |
69 | +} | |
70 | + | |
71 | +## Constant location of binaries (at least for Ubuntu 14.04) | |
72 | + | |
73 | +default['cfe-mongodb']['bin']['aws'] = '/usr/local/bin/aws' | |
74 | +default['cfe-mongodb']['bin']['mongo'] = '/usr/bin/mongo' | |
75 | +default['cfe-mongodb']['bin']['mongodump'] = '/usr/bin/mongodump' | |
76 | +default['cfe-mongodb']['bin']['openssl'] = '/usr/bin/openssl' | |
77 | + | |
78 | +## mongodb3 attributes | |
79 | + | |
80 | +default['mongodb3']['version'] = '3.2.8' | |
81 | + | |
82 | +default['mongodb3']['mongod']['disable-transparent-hugepages'] = true | |
83 | + | |
84 | +default['mongodb3']['config']['mongod']['net']['port'] = 27017 | |
85 | +default['mongodb3']['config']['mongod']['net']['bindIp'] = '127.0.0.1' | |
86 | + | |
87 | +default['mongodb3']['config']['mongod']['security']['authorization'] = 'enabled' | |
88 | +default['mongodb3']['config']['mongod']['security']['keyFile'] = | |
89 | + "#{node['cfe-mongodb']['install']['priv_dir']}/mongod.rs.key" | |
90 | + | |
91 | +default['mongodb3']['config']['mongod']['replication']['replSetName'] = 'test' | |
92 | +default['mongodb3']['config']['mongod']['replication']['oplogSizeMB'] = '1024' | ... | ... |
... | ... | @@ -5,3 +5,10 @@ license 'Apache License' |
5 | 5 | description 'Simplifies setup of staging MongoDB servers' |
6 | 6 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) |
7 | 7 | version '0.1.0' |
8 | + | |
9 | +depends 'mongodb3', '~> 5.3.0' | |
10 | +depends 'awscli', '~> 1.0.1' | |
11 | +depends 'openssl', '~> 4.4.0' | |
12 | +depends 'cron', '~> 1.7.4' | |
13 | + | |
14 | +supports 'ubuntu', '14.04' | ... | ... |
recipes/backup2s3.rb
0 → 100644
1 | +# | |
2 | +# Author:: Earth U (<sysadmin @ chromedia.com>) | |
3 | +# Cookbook Name:: cfe-mongodb | |
4 | +# Recipe:: backup2s3 | |
5 | +# | |
6 | +# Copyright (C) 2016, Chromedia Far East, Inc. | |
7 | +# | |
8 | +# Licensed under the Apache License, Version 2.0 (the "License"); | |
9 | +# you may not use this file except in compliance with the License. | |
10 | +# You may obtain a copy of the License at | |
11 | +# | |
12 | +# http://www.apache.org/licenses/LICENSE-2.0 | |
13 | +# | |
14 | +# Unless required by applicable law or agreed to in writing, software | |
15 | +# distributed under the License is distributed on an "AS IS" BASIS, | |
16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
17 | +# See the License for the specific language governing permissions and | |
18 | +# limitations under the License. | |
19 | +# | |
20 | + | |
21 | +package 'gzip' | |
22 | +package 'logrotate' | |
23 | +include_recipe 'openssl::upgrade' | |
24 | +include_recipe 'awscli' | |
25 | + | |
26 | +pub_key = "#{node[cookbook_name]['install']['priv_dir']}/pub.key" | |
27 | +bscript = "#{node[cookbook_name]['install']['priv_dir']}/mongodb_backup2s3" | |
28 | + | |
29 | +ip = (node['mongodb3']['config']['mongod']['net']['bindIp'].split(','))[-1] | |
30 | + | |
31 | +directory(node[cookbook_name]['install']['bak_log_dir']) { recursive true } | |
32 | + | |
33 | +is_any_enc = node[cookbook_name]['db']['map'].any? do |x| | |
34 | + if x.is_a?(Array) | |
35 | + x = x[1] | |
36 | + end | |
37 | + do_backup = x.has_key?(:backup) ? x[:backup] : true | |
38 | + do_backup ? x[:bak_encrypted] : false | |
39 | +end | |
40 | +if !node[cookbook_name]['encrypt']['pub_key'] && is_any_enc | |
41 | + Chef::Application.fatal!('No encryption public key contents supplied') | |
42 | +end | |
43 | + | |
44 | +file pub_key do | |
45 | + content node[cookbook_name]['encrypt']['pub_key'] | |
46 | + mode 0600 | |
47 | + owner 'root' | |
48 | + group 'root' | |
49 | + sensitive true | |
50 | + only_if { is_any_enc } | |
51 | +end | |
52 | + | |
53 | +template bscript do | |
54 | + mode 0700 | |
55 | + owner 'root' | |
56 | + group 'root' | |
57 | + sensitive true | |
58 | + variables( | |
59 | + :bin_aws => node[cookbook_name]['bin']['aws'], | |
60 | + :bin_mongo => node[cookbook_name]['bin']['mongo'], | |
61 | + :bin_mongodump => node[cookbook_name]['bin']['mongodump'], | |
62 | + :bin_openssl => node[cookbook_name]['bin']['openssl'], | |
63 | + | |
64 | + :db_host => ip, | |
65 | + :db_port => node['mongodb3']['config']['mongod']['net']['port'], | |
66 | + :db_map => node[cookbook_name]['db']['map'], | |
67 | + | |
68 | + :backup_user => 'backup', | |
69 | + :backup_pass => node[cookbook_name]['db']['pass_backup'], | |
70 | + :backup_auth => 'admin', | |
71 | + | |
72 | + :s3_region => node[cookbook_name]['s3_region'], | |
73 | + :s3_bucket => node[cookbook_name]['s3_bucket'], | |
74 | + | |
75 | + :pub_key => pub_key | |
76 | + ) | |
77 | +end | |
78 | + | |
79 | +sched = node[cookbook_name]['install']['bak_sched'].split(' ') | |
80 | +cron_d 'mongodb_backup2s3' do | |
81 | + command "bash #{bscript} >> #{node[cookbook_name]['install']['bak_log_dir']}"\ | |
82 | + '/mongodb_backup2s3.log 2>&1' | |
83 | + minute sched[0] | |
84 | + hour sched[1] | |
85 | + day sched[2] | |
86 | + month sched[3] | |
87 | + weekday sched[4] | |
88 | + mailto "''" | |
89 | + path '/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin' | |
90 | +end | |
91 | + | |
92 | +template "#{node[cookbook_name]['logrotate']['conf_dir']}/mongodb_backup2s3" do | |
93 | + source 'logrotate.erb' | |
94 | + variables( | |
95 | + :log_dir => node[cookbook_name]['install']['bak_log_dir'], | |
96 | + :opts => node[cookbook_name]['logrotate']['options'] | |
97 | + ) | |
98 | +end | ... | ... |
recipes/replicate.rb
0 → 100644
1 | +# | |
2 | +# Author:: Earth U (<sysadmin @ chromedia.com>) | |
3 | +# Cookbook Name:: cfe-mongodb | |
4 | +# Recipe:: replicate | |
5 | +# | |
6 | +# Copyright (C) 2016, Chromedia Far East, Inc. | |
7 | +# | |
8 | +# Licensed under the Apache License, Version 2.0 (the "License"); | |
9 | +# you may not use this file except in compliance with the License. | |
10 | +# You may obtain a copy of the License at | |
11 | +# | |
12 | +# http://www.apache.org/licenses/LICENSE-2.0 | |
13 | +# | |
14 | +# Unless required by applicable law or agreed to in writing, software | |
15 | +# distributed under the License is distributed on an "AS IS" BASIS, | |
16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
17 | +# See the License for the specific language governing permissions and | |
18 | +# limitations under the License. | |
19 | +# | |
20 | + | |
21 | +directory(node[cookbook_name]['install']['priv_dir']) { recursive true } | |
22 | + | |
23 | +file node['mongodb3']['config']['mongod']['security']['keyFile'] do | |
24 | + content node[cookbook_name]['rs']['key'] | |
25 | + owner node['mongodb3']['user'] | |
26 | + group node['mongodb3']['group'] | |
27 | + mode 0400 | |
28 | + sensitive true | |
29 | + notifies :restart, 'service[mongod]', :immediately | |
30 | + notifies :reboot_now, 'reboot[reboot_before_replication]', :immediately | |
31 | + only_if { node[cookbook_name]['rs']['key'] } | |
32 | +end | |
33 | + | |
34 | +reboot 'reboot_before_replication' do | |
35 | + action :nothing | |
36 | + reason 'Node needs to restart before initiating MongoDB replication' | |
37 | +end | |
38 | + | |
39 | +server = "#{node[cookbook_name]['local_ipv4']}"\ | |
40 | + ":#{node['mongodb3']['config']['mongod']['net']['port']}" | |
41 | +if node['cfe-mongodb']['rs']['nodes']['primary'] == server | |
42 | + include_recipe "#{cookbook_name}::replicate_pri_init" | |
43 | + include_recipe "#{cookbook_name}::replicate_pri_addnodes" | |
44 | +end | ... | ... |
recipes/replicate_pri_addnodes.rb
0 → 100644
1 | +# | |
2 | +# Author:: Earth U (<sysadmin @ chromedia.com>) | |
3 | +# Cookbook Name:: cfe-mongodb | |
4 | +# Recipe:: replicate_pri_addnodes | |
5 | +# | |
6 | +# Copyright (C) 2016, Chromedia Far East, Inc. | |
7 | +# | |
8 | +# Licensed under the Apache License, Version 2.0 (the "License"); | |
9 | +# you may not use this file except in compliance with the License. | |
10 | +# You may obtain a copy of the License at | |
11 | +# | |
12 | +# http://www.apache.org/licenses/LICENSE-2.0 | |
13 | +# | |
14 | +# Unless required by applicable law or agreed to in writing, software | |
15 | +# distributed under the License is distributed on an "AS IS" BASIS, | |
16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
17 | +# See the License for the specific language governing permissions and | |
18 | +# limitations under the License. | |
19 | +# | |
20 | + | |
21 | +mongod = node['mongodb3']['config']['mongod'] | |
22 | +ip = (mongod['net']['bindIp'].split(','))[-1] | |
23 | +port = mongod['net']['port'] | |
24 | + | |
25 | +com = "sleep #{node[cookbook_name]['rs']['delay']} "\ | |
26 | + "&& mongo --host #{ip} --port #{port} "\ | |
27 | + "-u root -p #{node[cookbook_name]['db']['pass_root']} "\ | |
28 | + "--authenticationDatabase admin --eval 'rs.add<<1>>(\"<<2>>\")' "\ | |
29 | + ">> #{::File.dirname(mongod['systemLog']['path'])}/primary_addnodes" | |
30 | + | |
31 | +node[cookbook_name]['rs']['nodes']['secondary'].each do |sec| | |
32 | + execute com.sub('<<1>>', '').sub('<<2>>', sec) | |
33 | +end | |
34 | + | |
35 | +node[cookbook_name]['rs']['nodes']['arbiter'].each do |arb| | |
36 | + execute com.sub('<<1>>', 'Arb').sub('<<2>>', arb) | |
37 | +end | ... | ... |
recipes/replicate_pri_init.rb
0 → 100644
1 | +# | |
2 | +# Author:: Earth U (<sysadmin @ chromedia.com>) | |
3 | +# Cookbook Name:: cfe-mongodb | |
4 | +# Recipe:: replicate_pri_init | |
5 | +# | |
6 | +# Copyright (C) 2016, Chromedia Far East, Inc. | |
7 | +# | |
8 | +# Licensed under the Apache License, Version 2.0 (the "License"); | |
9 | +# you may not use this file except in compliance with the License. | |
10 | +# You may obtain a copy of the License at | |
11 | +# | |
12 | +# http://www.apache.org/licenses/LICENSE-2.0 | |
13 | +# | |
14 | +# Unless required by applicable law or agreed to in writing, software | |
15 | +# distributed under the License is distributed on an "AS IS" BASIS, | |
16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
17 | +# See the License for the specific language governing permissions and | |
18 | +# limitations under the License. | |
19 | +# | |
20 | + | |
21 | +# NOTE: | |
22 | +# The recipe cfe-mongodb::replicate must be run before this | |
23 | +# in order to restart the node beforehand. | |
24 | +# Otherwise, manually restart the node first because that seems to | |
25 | +# be the only way for an automated rs.initiate() to proceed. | |
26 | +# Doing it immediately after mongodb install does not work, | |
27 | +# nor does implementing delay, or even restarting the mongod service. | |
28 | + | |
29 | +# NOTE: | |
30 | +# It is recommended that this cookbook be run on secondary and | |
31 | +# arbiter nodes first, before finally running it on the primary node. | |
32 | + | |
33 | +log = node['mongodb3']['config']['mongod']['systemLog'] | |
34 | +port = node['mongodb3']['config']['mongod']['net']['port'] | |
35 | + | |
36 | +boot_mark = "#{::File.dirname(log['path'])}/primary_init_mark" | |
37 | +boot_log = "#{::File.dirname(log['path'])}/primary_init" | |
38 | +is_logrename = log['logRotate'] == 'rename' | |
39 | + | |
40 | +boot_script = "#{Chef::Config[:file_cache_path]}/mongodb_pri_bootstrap.js" | |
41 | + | |
42 | +template boot_script do | |
43 | + sensitive true | |
44 | + variables( | |
45 | + :db_map => node[cookbook_name]['db']['map'], | |
46 | + :db_pass_root => node[cookbook_name]['db']['pass_root'], | |
47 | + :db_pass_backup => node[cookbook_name]['db']['pass_backup'], | |
48 | + :is_logrename => is_logrename | |
49 | + ) | |
50 | +end | |
51 | + | |
52 | +script 'mongodb_pri_bootstrap' do | |
53 | + interpreter 'bash' | |
54 | + not_if { ::File.exist?(boot_mark) } | |
55 | + code <<-CODE | |
56 | + set -e | |
57 | + mongo --host 127.0.0.1 --port #{port} --eval "rs.initiate()" >> #{boot_log} | |
58 | + sleep #{node[cookbook_name]['rs']['delay']} | |
59 | + mongo #{boot_script} --host 127.0.0.1 --port #{port} >> #{boot_log} | |
60 | + date > #{boot_mark} | |
61 | + CODE | |
62 | +end | |
63 | + | |
64 | +file boot_script do | |
65 | + action :delete | |
66 | +end | ... | ... |
templates/default/.gitkeep
deleted
100644 → 0
templates/default/logrotate.erb
0 → 100644
templates/default/mongodb_backup2s3.erb
0 → 100644
1 | +#!/bin/bash | |
2 | +# | |
3 | +# This file was generated by CHEF. Changes will be overwritten! | |
4 | + | |
5 | +# Backup script for MongoDB in AWS EC2 instances. | |
6 | +# | |
7 | +# If this mongod instance is primary, then backup the | |
8 | +# designated database/s and upload it to S3. Encrypt first if needed. | |
9 | +# | |
10 | +# Depends on: | |
11 | +# MongoDB | |
12 | +# AWSCLI | |
13 | +# OpenSSL | |
14 | + | |
15 | +set -e | |
16 | + | |
17 | +suffix=.mongodb_backup2s3 | |
18 | +if [ -f /tmp/*"$suffix" ] ; then | |
19 | + ( >&2 echo "[ERROR] Another operation might still be in progress" ) | |
20 | + exit 200 | |
21 | +fi | |
22 | +tmp_file=$( mktemp --suffix "$suffix" ) | |
23 | + | |
24 | +bin_aws=<%= @bin_aws %> | |
25 | +bin_mongo=<%= @bin_mongo %> | |
26 | +bin_mongodump=<%= @bin_mongodump %> | |
27 | +bin_openssl=<%= @bin_openssl %> | |
28 | + | |
29 | +db_host=<%= @db_host %> | |
30 | +db_port=<%= @db_port %> | |
31 | +db_user='<%= @backup_user %>' | |
32 | +db_pass='<%= @backup_pass %>' | |
33 | +db_auth=<%= @backup_auth %> | |
34 | + | |
35 | +s3_bucket=<%= @s3_bucket %> | |
36 | +s3_region=<%= @s3_region %> | |
37 | + | |
38 | +pub_key=<%= @pub_key %> | |
39 | +<% bak_dir = "#{Chef::Config[:file_cache_path]}/mongodb_backup2s3" -%> | |
40 | +bak_dir=<%= bak_dir %> | |
41 | + | |
42 | +# Perform the actual mongodump | |
43 | +# Args: | |
44 | +# $1 = db name | |
45 | +# $2 = backup dump filename, e.g. 'mydb' | |
46 | +export_db() { | |
47 | + echo "$(date) : Export database ${1} to ${bak_dir}." | |
48 | + rm -f "${bak_dir}/${2}.gz" | |
49 | + | |
50 | + "$bin_mongodump" --host="${db_host}:${db_port}" \ | |
51 | + -u "$db_user" -p "$db_pass" --authenticationDatabase="${db_auth}" \ | |
52 | + --dumpDbUsersAndRoles --gzip -d "${1}" --archive="${bak_dir}/${1}.gz" | |
53 | +} | |
54 | + | |
55 | +# Encrypt the backup file with OpenSSL | |
56 | +# Args: | |
57 | +# $1 = compressed dump filename, e.g. 'mydb.gz' | |
58 | +encrypt_backup() { | |
59 | + echo "$(date) : Encrypt file ${1}." | |
60 | + rm -f "${bak_dir}/${1}.enc" | |
61 | + | |
62 | + "$bin_openssl" smime -encrypt -binary -text -aes256 \ | |
63 | + -in "${bak_dir}/${1}" -out "${bak_dir}/${1}.enc" \ | |
64 | + -outform DER "$pub_key" | |
65 | + rm "${bak_dir}/${1}" | |
66 | +} | |
67 | + | |
68 | +# Rotate the current backups in S3 | |
69 | +# Args: | |
70 | +# $1 = backup dump filename, e.g. 'mydb' | |
71 | +# $2 = max number of backup files to store at a time | |
72 | +rotate_backup() { | |
73 | + folder=$1 | |
74 | + max=$2 | |
75 | + | |
76 | + # Backups will stored inside subfolders (prefixes) | |
77 | + # in S3, so only look at 'PRE' objects. | |
78 | + baks=$( "$bin_aws" --output text --region "$s3_region" \ | |
79 | + s3 ls "s3://${s3_bucket}/" | grep '^\s*PRE' | \ | |
80 | + sed -e 's/^ *PRE //' -e 's/\/$//' | \ | |
81 | + grep "^${folder}" || echo "" ) | |
82 | + | |
83 | + echo "$(date) : Backup rotation for ${folder}." | |
84 | + start=$((max - 1)) | |
85 | + | |
86 | + for (( x=start ; x > 0 ; x-- )) ; do | |
87 | + if echo "$baks" | grep "^${folder}\\.${x}\$" ; then | |
88 | + newx=$((x + 1)) | |
89 | + if [[ $newx -lt $max ]] ; then | |
90 | + "$bin_aws" --region "$s3_region" \ | |
91 | + s3 mv --recursive "s3://${s3_bucket}/${folder}.${x}" \ | |
92 | + "s3://${s3_bucket}/${folder}.${newx}" | |
93 | + else | |
94 | + "$bin_aws" --region "$s3_region" \ | |
95 | + s3 rm --recursive "s3://${s3_bucket}/${folder}.${x}" | |
96 | + fi | |
97 | + fi | |
98 | + done | |
99 | + | |
100 | + if echo "$baks" | grep "^${folder}\$" ; then | |
101 | + if [[ $max -gt 1 ]] ; then | |
102 | + "$bin_aws" --region "$s3_region" \ | |
103 | + s3 mv --recursive "s3://${s3_bucket}/${folder}" \ | |
104 | + "s3://${s3_bucket}/${folder}.1" | |
105 | + else | |
106 | + "$bin_aws" --region "$s3_region" \ | |
107 | + s3 rm --recursive "s3://${s3_bucket}/${folder}" | |
108 | + fi | |
109 | + fi | |
110 | +} | |
111 | + | |
112 | +# Upload the compressed db backup file. It will be uploaded | |
113 | +# to this example location: ${s3_bucket}/dump_filename/dump_filename.gz.enc | |
114 | +# | |
115 | +# A timestamp file will also be uploaded to this location: | |
116 | +# ${s3_bucket}/dump_filename/YYYY-MM-DDThh:mm:ss.txt | |
117 | +# Args: | |
118 | +# $1 = backup dump filename, e.g. 'mydb' | |
119 | +# $2 = if file is encrypted or not (boolean) | |
120 | +upload_to_s3() { | |
121 | + keyname=$1 | |
122 | + if [ "$2" = true ] ; then | |
123 | + fname="${keyname}.gz.enc" | |
124 | + else | |
125 | + fname="${keyname}.gz" | |
126 | + fi | |
127 | + | |
128 | + echo "$(date) : Upload ${fname} to S3 bucket ${s3_bucket}." | |
129 | + | |
130 | + stamp=$( date +"%FT%T" ) | |
131 | + echo "Uploaded: ${stamp}" > "${bak_dir}/${stamp}.txt" | |
132 | + | |
133 | + "$bin_aws" --region "$s3_region" \ | |
134 | + s3 mv "${bak_dir}/${fname}" \ | |
135 | + "s3://${s3_bucket}/${keyname}/${fname}" | |
136 | + "$bin_aws" --region "$s3_region" \ | |
137 | + s3 mv "${bak_dir}/${stamp}.txt" \ | |
138 | + "s3://${s3_bucket}/${keyname}/${stamp}.txt" | |
139 | +} | |
140 | + | |
141 | +## Do the backup only if this node is the primary: | |
142 | +if "$bin_mongo" --host="${db_host}:${db_port}" -u "$db_user" \ | |
143 | + -p "$db_pass" --authenticationDatabase="${db_auth}" \ | |
144 | + --eval 'db.isMaster()["ismaster"]' | grep -q true ; then | |
145 | + | |
146 | + if [[ ! -d "$bak_dir" ]] ; then | |
147 | + mkdir -p "$bak_dir" | |
148 | + fi | |
149 | + | |
150 | +<% @db_map.each do |x| -%> | |
151 | +<% | |
152 | + if x.is_a?(Array) | |
153 | + db_name = x[0] | |
154 | + x = x[1] | |
155 | + else | |
156 | + db_name = x[:db_name] | |
157 | + end | |
158 | + | |
159 | + do_backup = x.has_key?(:backup) ? x[:backup] : true | |
160 | + is_enc = x.has_key?(:bak_encrypted) ? x[:bak_encrypted] : false | |
161 | + bak_filename = x[:bak_filename] || db_name | |
162 | + bak_maxcopies = x[:bak_maxcopies] || 30 | |
163 | +-%> | |
164 | +<% if do_backup -%> | |
165 | + # Database: <%= db_name %> | |
166 | + export_db <%= db_name %> <%= bak_filename %> | |
167 | +<% if is_enc -%> | |
168 | + encrypt_backup <%= bak_filename %>.gz | |
169 | +<% end -%> | |
170 | + rotate_backup <%= bak_filename %> <%= bak_maxcopies %> | |
171 | + upload_to_s3 <%= bak_filename %> <%= is_enc %> | |
172 | + | |
173 | +<% end -%> | |
174 | +<% end -%> | |
175 | + echo "$(date) : Done." | |
176 | +fi | |
177 | + | |
178 | +rm "$tmp_file" | ... | ... |
1 | +var db_pass_root = "<%= @db_pass_root %>"; | |
2 | +var db_pass_backup = "<%= @db_pass_backup %>"; | |
3 | + | |
4 | +db = db.getSiblingDB("admin"); | |
5 | +db.createUser( { user: "root", pwd: db_pass_root, roles: [ "root", "__system" ] } ); | |
6 | +db.auth("root", db_pass_root); | |
7 | + | |
8 | +<% if @is_logrename -%> | |
9 | +db.runCommand( { logRotate : 1 } ); | |
10 | + | |
11 | +<% end -%> | |
12 | +db.createUser( { user: "backup", pwd: db_pass_backup, roles: [ "backup" ] } ); | |
13 | + | |
14 | +<% @db_map.each do |x| -%> | |
15 | +<% | |
16 | + if x.is_a?(Array) | |
17 | + db_name = x[0] | |
18 | + x = x[1] | |
19 | + else | |
20 | + db_name = x[:db_name] | |
21 | + end | |
22 | + | |
23 | + db_auth = x.has_key?(:db_auth) ? x[:db_auth] : db_name | |
24 | +-%> | |
25 | +db = db.getSiblingDB("<%= db_auth %>"); | |
26 | +db.createUser( { user: "<%= x[:db_user] %>", pwd: "<%= x[:db_pass] %>", roles: [ { role: "readWrite", db: "<%= db_name %>" }, { role: "dbAdmin", db: "<%= db_name %>" } ] } ); | |
27 | + | |
28 | +<% end -%> | ... | ... |