Commit 9f0ea9b0837d8fe0cd6bddcfacd8979091b673cf
Committed by
Earth Ugat
1 parent
986ae38c
Version 0.1.0 release
Showing
13 changed files
with
631 additions
and
5 deletions
| @@ -13,6 +13,14 @@ driver: | @@ -13,6 +13,14 @@ driver: | ||
| 13 | provisioner: | 13 | provisioner: |
| 14 | name: chef_zero | 14 | name: chef_zero |
| 15 | 15 | ||
| 16 | +platforms: | ||
| 17 | + - name: ubuntu-14.04 | ||
| 18 | + driver: | ||
| 19 | + image_id: ami-xxx | ||
| 20 | + transport: | ||
| 21 | + username: ubuntu | ||
| 22 | + ssh_key: ~/.ssh/xxx.pem | ||
| 23 | + | ||
| 16 | suites: | 24 | suites: |
| 17 | - name: default | 25 | - name: default |
| 18 | run_list: | 26 | run_list: |
| @@ -4,7 +4,7 @@ Just a wrapper around [`mongodb3`](https://supermarket.chef.io/cookbooks/mongodb | @@ -4,7 +4,7 @@ Just a wrapper around [`mongodb3`](https://supermarket.chef.io/cookbooks/mongodb | ||
| 4 | 4 | ||
| 5 | ## Supported Platforms | 5 | ## Supported Platforms |
| 6 | 6 | ||
| 7 | -The cookbook `mongodb3` supports the most common Linux distros. | 7 | +The cookbook `mongodb3` supports the most common Linux distros, but this cookbook has some constants that might be Ubuntu-specific only. |
| 8 | 8 | ||
| 9 | ## Attributes | 9 | ## Attributes |
| 10 | 10 | ||
| @@ -16,10 +16,76 @@ The cookbook `mongodb3` supports the most common Linux distros. | @@ -16,10 +16,76 @@ The cookbook `mongodb3` supports the most common Linux distros. | ||
| 16 | <th>Default</th> | 16 | <th>Default</th> |
| 17 | </tr> | 17 | </tr> |
| 18 | <tr> | 18 | <tr> |
| 19 | - <td><tt>['cfe-mongodb']['']</tt></td> | ||
| 20 | - <td>Boolean</td> | ||
| 21 | - <td>Desctd> | ||
| 22 | - <td><tt>True</tt></td> | 19 | + <td><tt>['cfe-mongodb']['local_ipv4']</tt></td> |
| 20 | + <td>String</td> | ||
| 21 | + <td>The local IP where mongod should be listening. Should be at least a member of the mongod bind IPs. (The node's IP in a replication set.)<td> | ||
| 22 | + <td><tt>'127.0.0.1'</tt></td> | ||
| 23 | + </tr> | ||
| 24 | + <tr> | ||
| 25 | + <td><tt>['cfe-mongodb']['s3_region']</tt></td> | ||
| 26 | + <td>String</td> | ||
| 27 | + <td>S3 region if using the backup script.<td> | ||
| 28 | + <td><tt>'us-east-1'</tt></td> | ||
| 29 | + </tr> | ||
| 30 | + <tr> | ||
| 31 | + <td><tt>['cfe-mongodb']['s3_bucket']</tt></td> | ||
| 32 | + <td>String</td> | ||
| 33 | + <td>S3 bucket if using the backup script<td> | ||
| 34 | + <td><tt>'test-bucket'</tt></td> | ||
| 35 | + </tr> | ||
| 36 | + <tr> | ||
| 37 | + <td><tt>['cfe-mongodb']['db']['map']</tt></td> | ||
| 38 | + <td>Hash/Array</td> | ||
| 39 | + <td>Details of every database to set up, including users and passwords, etc. Please refer to the default attributes file.<td> | ||
| 40 | + <td><tt>{}</tt></td> | ||
| 41 | + </tr> | ||
| 42 | + <tr> | ||
| 43 | + <td><tt>['cfe-mongodb']['db']['pass_root']</tt></td> | ||
| 44 | + <td>String</td> | ||
| 45 | + <td>The mongodb password for user 'root'<td> | ||
| 46 | + <td><tt>'secret'</tt></td> | ||
| 47 | + </tr> | ||
| 48 | + <tr> | ||
| 49 | + <td><tt>['cfe-mongodb']['db']['pass_backup']</tt></td> | ||
| 50 | + <td>String</td> | ||
| 51 | + <td>The mongodb password for user 'backup', used for performing mongodumps during the backup script operation.<td> | ||
| 52 | + <td><tt>'secret'</tt></td> | ||
| 53 | + </tr> | ||
| 54 | + <tr> | ||
| 55 | + <td><tt>['cfe-mongodb']['rs']['key']</tt></td> | ||
| 56 | + <td>String</td> | ||
| 57 | + <td>Contents of replication key.<td> | ||
| 58 | + <td><tt>'xxxx'</tt></td> | ||
| 59 | + </tr> | ||
| 60 | + <tr> | ||
| 61 | + <td><tt>['cfe-mongodb']['rs']['nodes']['primary']</tt></td> | ||
| 62 | + <td>String</td> | ||
| 63 | + <td>The primary mongodb node during initial Chef run, e.g. '10.0.0.1:27017'.<td> | ||
| 64 | + <td><tt>''</tt></td> | ||
| 65 | + </tr> | ||
| 66 | + <tr> | ||
| 67 | + <td><tt>['cfe-mongodb']['rs']['nodes']['secondary']</tt></td> | ||
| 68 | + <td>Array</td> | ||
| 69 | + <td>Array of secondary mongodb nodes during initial Chef run, e.g. ['10.0.0.2:27017', '10.0.0.3:27017'].<td> | ||
| 70 | + <td><tt>[]</tt></td> | ||
| 71 | + </tr> | ||
| 72 | + <tr> | ||
| 73 | + <td><tt>['cfe-mongodb']['rs']['nodes']['arbiter']</tt></td> | ||
| 74 | + <td>Array</td> | ||
| 75 | + <td>Array of arbiter mongodb nodes during initial Chef run, e.g. ['10.0.0.4:27017'].<td> | ||
| 76 | + <td><tt>[]</tt></td> | ||
| 77 | + </tr> | ||
| 78 | + <tr> | ||
| 79 | + <td><tt>['cfe-mongodb']['install']['bak_sched']</tt></td> | ||
| 80 | + <td>String</td> | ||
| 81 | + <td>Crontab schedule for running the backup script.<td> | ||
| 82 | + <td><tt>'0 7 * * *'</tt></td> | ||
| 83 | + </tr> | ||
| 84 | + <tr> | ||
| 85 | + <td><tt>['cfe-mongodb']['encrypt']['pub_key']</tt></td> | ||
| 86 | + <td>String</td> | ||
| 87 | + <td>Encryption public key if at least one of the databases specified in the DB map is to be encrypted when it is automatically backed up.<td> | ||
| 88 | + <td><tt>nil</tt></td> | ||
| 23 | </tr> | 89 | </tr> |
| 24 | </table> | 90 | </table> |
| 25 | 91 | ||
| @@ -37,6 +103,19 @@ After setting proper node attributes, include `cfe-mongodb` in your node's `run_ | @@ -37,6 +103,19 @@ After setting proper node attributes, include `cfe-mongodb` in your node's `run_ | ||
| 37 | } | 103 | } |
| 38 | ``` | 104 | ``` |
| 39 | 105 | ||
| 106 | +### cfe-mongodb::backup2s3 | ||
| 107 | + | ||
| 108 | +Sets up an automated backup script that uploads DB backups into an S3 bucket: | ||
| 109 | + | ||
| 110 | +```json | ||
| 111 | +{ | ||
| 112 | + "run_list": [ | ||
| 113 | + "recipe[cfe-mongodb::default]", | ||
| 114 | + "recipe[cfe-mongodb::backup2s3]" | ||
| 115 | + ] | ||
| 116 | +} | ||
| 117 | +``` | ||
| 118 | + | ||
| 40 | ## License and Authors | 119 | ## License and Authors |
| 41 | 120 | ||
| 42 | Author:: Earth U. (<sysadmin @ chromedia.com>) | 121 | Author:: Earth U. (<sysadmin @ chromedia.com>) |
| @@ -17,3 +17,76 @@ | @@ -17,3 +17,76 @@ | ||
| 17 | # See the License for the specific language governing permissions and | 17 | # See the License for the specific language governing permissions and |
| 18 | # limitations under the License. | 18 | # limitations under the License. |
| 19 | # | 19 | # |
| 20 | + | ||
| 21 | +default['cfe-mongodb']['local_ipv4'] = '127.0.0.1' | ||
| 22 | + | ||
| 23 | +default['cfe-mongodb']['s3_region'] = 'us-east-1' | ||
| 24 | +default['cfe-mongodb']['s3_bucket'] = 'test-bucket' | ||
| 25 | + | ||
| 26 | +default['cfe-mongodb']['db']['map'] = { | ||
| 27 | + # The db_map format is: | ||
| 28 | + # 'example_db_name' => { | ||
| 29 | + # :db_user => '<name_of_custom_user>', | ||
| 30 | + # :db_pass => '<name_of_custom_pass>', | ||
| 31 | + # | ||
| 32 | + # Optional: | ||
| 33 | + # :db_auth => '<name_of_authentication_db>' # default is db_name | ||
| 34 | + # :backup => # default: true | ||
| 35 | + # :bak_encrypted => # default: false | ||
| 36 | + # :bak_filename => # default is db_name | ||
| 37 | + # :bak_maxcopies => # default: 30 | ||
| 38 | + # } | ||
| 39 | +} | ||
| 40 | +default['cfe-mongodb']['db']['pass_root'] = 'secret' | ||
| 41 | +default['cfe-mongodb']['db']['pass_backup'] = 'secret' | ||
| 42 | + | ||
| 43 | +# Create custom key contents with: | ||
| 44 | +# $ openssl rand -base64 756 | ||
| 45 | +default['cfe-mongodb']['rs']['key'] = 'supersecretkeyxxx' | ||
| 46 | +default['cfe-mongodb']['rs']['delay'] = 2 | ||
| 47 | +# Note: the following attributes for primary/secondary/arbiter | ||
| 48 | +# should all include the port number (e.g. '1.2.3.4:27017') | ||
| 49 | +default['cfe-mongodb']['rs']['nodes']['primary'] = '' | ||
| 50 | +default['cfe-mongodb']['rs']['nodes']['secondary'] = [] | ||
| 51 | +default['cfe-mongodb']['rs']['nodes']['arbiter'] = [] | ||
| 52 | + | ||
| 53 | +default['cfe-mongodb']['install']['priv_dir'] = '/opt/mongodb/priv' | ||
| 54 | +default['cfe-mongodb']['install']['bak_log_dir'] = '/var/log/mongodb_backup2s3' | ||
| 55 | +default['cfe-mongodb']['install']['bak_sched'] = '0 7 * * *' | ||
| 56 | + | ||
| 57 | +default['cfe-mongodb']['encrypt']['priv_key'] = nil | ||
| 58 | +default['cfe-mongodb']['encrypt']['pub_key'] = nil | ||
| 59 | + | ||
| 60 | +## Constant logrotate options if automated backups are used | ||
| 61 | + | ||
| 62 | +default['cfe-mongodb']['logrotate']['conf_dir'] = '/etc/logrotate.d' | ||
| 63 | +default['cfe-mongodb']['logrotate']['options'] = %w{ | ||
| 64 | + weekly | ||
| 65 | + rotate\ 12 | ||
| 66 | + missingok | ||
| 67 | + compress | ||
| 68 | + notifempty | ||
| 69 | +} | ||
| 70 | + | ||
| 71 | +## Constant location of binaries (at least for Ubuntu 14.04) | ||
| 72 | + | ||
| 73 | +default['cfe-mongodb']['bin']['aws'] = '/usr/local/bin/aws' | ||
| 74 | +default['cfe-mongodb']['bin']['mongo'] = '/usr/bin/mongo' | ||
| 75 | +default['cfe-mongodb']['bin']['mongodump'] = '/usr/bin/mongodump' | ||
| 76 | +default['cfe-mongodb']['bin']['openssl'] = '/usr/bin/openssl' | ||
| 77 | + | ||
| 78 | +## mongodb3 attributes | ||
| 79 | + | ||
| 80 | +default['mongodb3']['version'] = '3.2.8' | ||
| 81 | + | ||
| 82 | +default['mongodb3']['mongod']['disable-transparent-hugepages'] = true | ||
| 83 | + | ||
| 84 | +default['mongodb3']['config']['mongod']['net']['port'] = 27017 | ||
| 85 | +default['mongodb3']['config']['mongod']['net']['bindIp'] = '127.0.0.1' | ||
| 86 | + | ||
| 87 | +default['mongodb3']['config']['mongod']['security']['authorization'] = 'enabled' | ||
| 88 | +default['mongodb3']['config']['mongod']['security']['keyFile'] = | ||
| 89 | + "#{node['cfe-mongodb']['install']['priv_dir']}/mongod.rs.key" | ||
| 90 | + | ||
| 91 | +default['mongodb3']['config']['mongod']['replication']['replSetName'] = 'test' | ||
| 92 | +default['mongodb3']['config']['mongod']['replication']['oplogSizeMB'] = '1024' |
| @@ -5,3 +5,10 @@ license 'Apache License' | @@ -5,3 +5,10 @@ license 'Apache License' | ||
| 5 | description 'Simplifies setup of staging MongoDB servers' | 5 | description 'Simplifies setup of staging MongoDB servers' |
| 6 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) | 6 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) |
| 7 | version '0.1.0' | 7 | version '0.1.0' |
| 8 | + | ||
| 9 | +depends 'mongodb3', '~> 5.3.0' | ||
| 10 | +depends 'awscli', '~> 1.0.1' | ||
| 11 | +depends 'openssl', '~> 4.4.0' | ||
| 12 | +depends 'cron', '~> 1.7.4' | ||
| 13 | + | ||
| 14 | +supports 'ubuntu', '14.04' |
recipes/backup2s3.rb
0 → 100644
| 1 | +# | ||
| 2 | +# Author:: Earth U (<sysadmin @ chromedia.com>) | ||
| 3 | +# Cookbook Name:: cfe-mongodb | ||
| 4 | +# Recipe:: backup2s3 | ||
| 5 | +# | ||
| 6 | +# Copyright (C) 2016, Chromedia Far East, Inc. | ||
| 7 | +# | ||
| 8 | +# Licensed under the Apache License, Version 2.0 (the "License"); | ||
| 9 | +# you may not use this file except in compliance with the License. | ||
| 10 | +# You may obtain a copy of the License at | ||
| 11 | +# | ||
| 12 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 13 | +# | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and | ||
| 18 | +# limitations under the License. | ||
| 19 | +# | ||
| 20 | + | ||
| 21 | +package 'gzip' | ||
| 22 | +package 'logrotate' | ||
| 23 | +include_recipe 'openssl::upgrade' | ||
| 24 | +include_recipe 'awscli' | ||
| 25 | + | ||
| 26 | +pub_key = "#{node[cookbook_name]['install']['priv_dir']}/pub.key" | ||
| 27 | +bscript = "#{node[cookbook_name]['install']['priv_dir']}/mongodb_backup2s3" | ||
| 28 | + | ||
| 29 | +ip = (node['mongodb3']['config']['mongod']['net']['bindIp'].split(','))[-1] | ||
| 30 | + | ||
| 31 | +directory(node[cookbook_name]['install']['bak_log_dir']) { recursive true } | ||
| 32 | + | ||
| 33 | +is_any_enc = node[cookbook_name]['db']['map'].any? do |x| | ||
| 34 | + if x.is_a?(Array) | ||
| 35 | + x = x[1] | ||
| 36 | + end | ||
| 37 | + do_backup = x.has_key?(:backup) ? x[:backup] : true | ||
| 38 | + do_backup ? x[:bak_encrypted] : false | ||
| 39 | +end | ||
| 40 | +if !node[cookbook_name]['encrypt']['pub_key'] && is_any_enc | ||
| 41 | + Chef::Application.fatal!('No encryption public key contents supplied') | ||
| 42 | +end | ||
| 43 | + | ||
| 44 | +file pub_key do | ||
| 45 | + content node[cookbook_name]['encrypt']['pub_key'] | ||
| 46 | + mode 0600 | ||
| 47 | + owner 'root' | ||
| 48 | + group 'root' | ||
| 49 | + sensitive true | ||
| 50 | + only_if { is_any_enc } | ||
| 51 | +end | ||
| 52 | + | ||
| 53 | +template bscript do | ||
| 54 | + mode 0700 | ||
| 55 | + owner 'root' | ||
| 56 | + group 'root' | ||
| 57 | + sensitive true | ||
| 58 | + variables( | ||
| 59 | + :bin_aws => node[cookbook_name]['bin']['aws'], | ||
| 60 | + :bin_mongo => node[cookbook_name]['bin']['mongo'], | ||
| 61 | + :bin_mongodump => node[cookbook_name]['bin']['mongodump'], | ||
| 62 | + :bin_openssl => node[cookbook_name]['bin']['openssl'], | ||
| 63 | + | ||
| 64 | + :db_host => ip, | ||
| 65 | + :db_port => node['mongodb3']['config']['mongod']['net']['port'], | ||
| 66 | + :db_map => node[cookbook_name]['db']['map'], | ||
| 67 | + | ||
| 68 | + :backup_user => 'backup', | ||
| 69 | + :backup_pass => node[cookbook_name]['db']['pass_backup'], | ||
| 70 | + :backup_auth => 'admin', | ||
| 71 | + | ||
| 72 | + :s3_region => node[cookbook_name]['s3_region'], | ||
| 73 | + :s3_bucket => node[cookbook_name]['s3_bucket'], | ||
| 74 | + | ||
| 75 | + :pub_key => pub_key | ||
| 76 | + ) | ||
| 77 | +end | ||
| 78 | + | ||
| 79 | +sched = node[cookbook_name]['install']['bak_sched'].split(' ') | ||
| 80 | +cron_d 'mongodb_backup2s3' do | ||
| 81 | + command "bash #{bscript} >> #{node[cookbook_name]['install']['bak_log_dir']}"\ | ||
| 82 | + '/mongodb_backup2s3.log 2>&1' | ||
| 83 | + minute sched[0] | ||
| 84 | + hour sched[1] | ||
| 85 | + day sched[2] | ||
| 86 | + month sched[3] | ||
| 87 | + weekday sched[4] | ||
| 88 | + mailto "''" | ||
| 89 | + path '/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin' | ||
| 90 | +end | ||
| 91 | + | ||
| 92 | +template "#{node[cookbook_name]['logrotate']['conf_dir']}/mongodb_backup2s3" do | ||
| 93 | + source 'logrotate.erb' | ||
| 94 | + variables( | ||
| 95 | + :log_dir => node[cookbook_name]['install']['bak_log_dir'], | ||
| 96 | + :opts => node[cookbook_name]['logrotate']['options'] | ||
| 97 | + ) | ||
| 98 | +end |
| @@ -17,3 +17,6 @@ | @@ -17,3 +17,6 @@ | ||
| 17 | # See the License for the specific language governing permissions and | 17 | # See the License for the specific language governing permissions and |
| 18 | # limitations under the License. | 18 | # limitations under the License. |
| 19 | # | 19 | # |
| 20 | + | ||
| 21 | +include_recipe 'mongodb3' | ||
| 22 | +include_recipe "#{cookbook_name}::replicate" |
recipes/replicate.rb
0 → 100644
| 1 | +# | ||
| 2 | +# Author:: Earth U (<sysadmin @ chromedia.com>) | ||
| 3 | +# Cookbook Name:: cfe-mongodb | ||
| 4 | +# Recipe:: replicate | ||
| 5 | +# | ||
| 6 | +# Copyright (C) 2016, Chromedia Far East, Inc. | ||
| 7 | +# | ||
| 8 | +# Licensed under the Apache License, Version 2.0 (the "License"); | ||
| 9 | +# you may not use this file except in compliance with the License. | ||
| 10 | +# You may obtain a copy of the License at | ||
| 11 | +# | ||
| 12 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 13 | +# | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and | ||
| 18 | +# limitations under the License. | ||
| 19 | +# | ||
| 20 | + | ||
| 21 | +directory(node[cookbook_name]['install']['priv_dir']) { recursive true } | ||
| 22 | + | ||
| 23 | +file node['mongodb3']['config']['mongod']['security']['keyFile'] do | ||
| 24 | + content node[cookbook_name]['rs']['key'] | ||
| 25 | + owner node['mongodb3']['user'] | ||
| 26 | + group node['mongodb3']['group'] | ||
| 27 | + mode 0400 | ||
| 28 | + sensitive true | ||
| 29 | + notifies :restart, 'service[mongod]', :immediately | ||
| 30 | + notifies :reboot_now, 'reboot[reboot_before_replication]', :immediately | ||
| 31 | + only_if { node[cookbook_name]['rs']['key'] } | ||
| 32 | +end | ||
| 33 | + | ||
| 34 | +reboot 'reboot_before_replication' do | ||
| 35 | + action :nothing | ||
| 36 | + reason 'Node needs to restart before initiating MongoDB replication' | ||
| 37 | +end | ||
| 38 | + | ||
| 39 | +server = "#{node[cookbook_name]['local_ipv4']}"\ | ||
| 40 | + ":#{node['mongodb3']['config']['mongod']['net']['port']}" | ||
| 41 | +if node['cfe-mongodb']['rs']['nodes']['primary'] == server | ||
| 42 | + include_recipe "#{cookbook_name}::replicate_pri_init" | ||
| 43 | + include_recipe "#{cookbook_name}::replicate_pri_addnodes" | ||
| 44 | +end |
recipes/replicate_pri_addnodes.rb
0 → 100644
| 1 | +# | ||
| 2 | +# Author:: Earth U (<sysadmin @ chromedia.com>) | ||
| 3 | +# Cookbook Name:: cfe-mongodb | ||
| 4 | +# Recipe:: replicate_pri_addnodes | ||
| 5 | +# | ||
| 6 | +# Copyright (C) 2016, Chromedia Far East, Inc. | ||
| 7 | +# | ||
| 8 | +# Licensed under the Apache License, Version 2.0 (the "License"); | ||
| 9 | +# you may not use this file except in compliance with the License. | ||
| 10 | +# You may obtain a copy of the License at | ||
| 11 | +# | ||
| 12 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 13 | +# | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and | ||
| 18 | +# limitations under the License. | ||
| 19 | +# | ||
| 20 | + | ||
| 21 | +mongod = node['mongodb3']['config']['mongod'] | ||
| 22 | +ip = (mongod['net']['bindIp'].split(','))[-1] | ||
| 23 | +port = mongod['net']['port'] | ||
| 24 | + | ||
| 25 | +com = "sleep #{node[cookbook_name]['rs']['delay']} "\ | ||
| 26 | + "&& mongo --host #{ip} --port #{port} "\ | ||
| 27 | + "-u root -p #{node[cookbook_name]['db']['pass_root']} "\ | ||
| 28 | + "--authenticationDatabase admin --eval 'rs.add<<1>>(\"<<2>>\")' "\ | ||
| 29 | + ">> #{::File.dirname(mongod['systemLog']['path'])}/primary_addnodes" | ||
| 30 | + | ||
| 31 | +node[cookbook_name]['rs']['nodes']['secondary'].each do |sec| | ||
| 32 | + execute com.sub('<<1>>', '').sub('<<2>>', sec) | ||
| 33 | +end | ||
| 34 | + | ||
| 35 | +node[cookbook_name]['rs']['nodes']['arbiter'].each do |arb| | ||
| 36 | + execute com.sub('<<1>>', 'Arb').sub('<<2>>', arb) | ||
| 37 | +end |
recipes/replicate_pri_init.rb
0 → 100644
| 1 | +# | ||
| 2 | +# Author:: Earth U (<sysadmin @ chromedia.com>) | ||
| 3 | +# Cookbook Name:: cfe-mongodb | ||
| 4 | +# Recipe:: replicate_pri_init | ||
| 5 | +# | ||
| 6 | +# Copyright (C) 2016, Chromedia Far East, Inc. | ||
| 7 | +# | ||
| 8 | +# Licensed under the Apache License, Version 2.0 (the "License"); | ||
| 9 | +# you may not use this file except in compliance with the License. | ||
| 10 | +# You may obtain a copy of the License at | ||
| 11 | +# | ||
| 12 | +# http://www.apache.org/licenses/LICENSE-2.0 | ||
| 13 | +# | ||
| 14 | +# Unless required by applicable law or agreed to in writing, software | ||
| 15 | +# distributed under the License is distributed on an "AS IS" BASIS, | ||
| 16 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 17 | +# See the License for the specific language governing permissions and | ||
| 18 | +# limitations under the License. | ||
| 19 | +# | ||
| 20 | + | ||
| 21 | +# NOTE: | ||
| 22 | +# The recipe cfe-mongodb::replicate must be run before this | ||
| 23 | +# in order to restart the node beforehand. | ||
| 24 | +# Otherwise, manually restart the node first because that seems to | ||
| 25 | +# be the only way for an automated rs.initiate() to proceed. | ||
| 26 | +# Doing it immediately after mongodb install does not work, | ||
| 27 | +# nor does implementing delay, or even restarting the mongod service. | ||
| 28 | + | ||
| 29 | +# NOTE: | ||
| 30 | +# It is recommended that this cookbook be run on secondary and | ||
| 31 | +# arbiter nodes first, before finally running it on the primary node. | ||
| 32 | + | ||
| 33 | +log = node['mongodb3']['config']['mongod']['systemLog'] | ||
| 34 | +port = node['mongodb3']['config']['mongod']['net']['port'] | ||
| 35 | + | ||
| 36 | +boot_mark = "#{::File.dirname(log['path'])}/primary_init_mark" | ||
| 37 | +boot_log = "#{::File.dirname(log['path'])}/primary_init" | ||
| 38 | +is_logrename = log['logRotate'] == 'rename' | ||
| 39 | + | ||
| 40 | +boot_script = "#{Chef::Config[:file_cache_path]}/mongodb_pri_bootstrap.js" | ||
| 41 | + | ||
| 42 | +template boot_script do | ||
| 43 | + sensitive true | ||
| 44 | + variables( | ||
| 45 | + :db_map => node[cookbook_name]['db']['map'], | ||
| 46 | + :db_pass_root => node[cookbook_name]['db']['pass_root'], | ||
| 47 | + :db_pass_backup => node[cookbook_name]['db']['pass_backup'], | ||
| 48 | + :is_logrename => is_logrename | ||
| 49 | + ) | ||
| 50 | +end | ||
| 51 | + | ||
| 52 | +script 'mongodb_pri_bootstrap' do | ||
| 53 | + interpreter 'bash' | ||
| 54 | + not_if { ::File.exist?(boot_mark) } | ||
| 55 | + code <<-CODE | ||
| 56 | + set -e | ||
| 57 | + mongo --host 127.0.0.1 --port #{port} --eval "rs.initiate()" >> #{boot_log} | ||
| 58 | + sleep #{node[cookbook_name]['rs']['delay']} | ||
| 59 | + mongo #{boot_script} --host 127.0.0.1 --port #{port} >> #{boot_log} | ||
| 60 | + date > #{boot_mark} | ||
| 61 | + CODE | ||
| 62 | +end | ||
| 63 | + | ||
| 64 | +file boot_script do | ||
| 65 | + action :delete | ||
| 66 | +end |
templates/default/.gitkeep
deleted
100644 → 0
templates/default/logrotate.erb
0 → 100644
templates/default/mongodb_backup2s3.erb
0 → 100644
| 1 | +#!/bin/bash | ||
| 2 | +# | ||
| 3 | +# This file was generated by CHEF. Changes will be overwritten! | ||
| 4 | + | ||
| 5 | +# Backup script for MongoDB in AWS EC2 instances. | ||
| 6 | +# | ||
| 7 | +# If this mongod instance is primary, then backup the | ||
| 8 | +# designated database/s and upload it to S3. Encrypt first if needed. | ||
| 9 | +# | ||
| 10 | +# Depends on: | ||
| 11 | +# MongoDB | ||
| 12 | +# AWSCLI | ||
| 13 | +# OpenSSL | ||
| 14 | + | ||
| 15 | +set -e | ||
| 16 | + | ||
| 17 | +suffix=.mongodb_backup2s3 | ||
| 18 | +if [ -f /tmp/*"$suffix" ] ; then | ||
| 19 | + ( >&2 echo "[ERROR] Another operation might still be in progress" ) | ||
| 20 | + exit 200 | ||
| 21 | +fi | ||
| 22 | +tmp_file=$( mktemp --suffix "$suffix" ) | ||
| 23 | + | ||
| 24 | +bin_aws=<%= @bin_aws %> | ||
| 25 | +bin_mongo=<%= @bin_mongo %> | ||
| 26 | +bin_mongodump=<%= @bin_mongodump %> | ||
| 27 | +bin_openssl=<%= @bin_openssl %> | ||
| 28 | + | ||
| 29 | +db_host=<%= @db_host %> | ||
| 30 | +db_port=<%= @db_port %> | ||
| 31 | +db_user='<%= @backup_user %>' | ||
| 32 | +db_pass='<%= @backup_pass %>' | ||
| 33 | +db_auth=<%= @backup_auth %> | ||
| 34 | + | ||
| 35 | +s3_bucket=<%= @s3_bucket %> | ||
| 36 | +s3_region=<%= @s3_region %> | ||
| 37 | + | ||
| 38 | +pub_key=<%= @pub_key %> | ||
| 39 | +<% bak_dir = "#{Chef::Config[:file_cache_path]}/mongodb_backup2s3" -%> | ||
| 40 | +bak_dir=<%= bak_dir %> | ||
| 41 | + | ||
| 42 | +# Perform the actual mongodump | ||
| 43 | +# Args: | ||
| 44 | +# $1 = db name | ||
| 45 | +# $2 = backup dump filename, e.g. 'mydb' | ||
| 46 | +export_db() { | ||
| 47 | + echo "$(date) : Export database ${1} to ${bak_dir}." | ||
| 48 | + rm -f "${bak_dir}/${2}.gz" | ||
| 49 | + | ||
| 50 | + "$bin_mongodump" --host="${db_host}:${db_port}" \ | ||
| 51 | + -u "$db_user" -p "$db_pass" --authenticationDatabase="${db_auth}" \ | ||
| 52 | + --dumpDbUsersAndRoles --gzip -d "${1}" --archive="${bak_dir}/${1}.gz" | ||
| 53 | +} | ||
| 54 | + | ||
| 55 | +# Encrypt the backup file with OpenSSL | ||
| 56 | +# Args: | ||
| 57 | +# $1 = compressed dump filename, e.g. 'mydb.gz' | ||
| 58 | +encrypt_backup() { | ||
| 59 | + echo "$(date) : Encrypt file ${1}." | ||
| 60 | + rm -f "${bak_dir}/${1}.enc" | ||
| 61 | + | ||
| 62 | + "$bin_openssl" smime -encrypt -binary -text -aes256 \ | ||
| 63 | + -in "${bak_dir}/${1}" -out "${bak_dir}/${1}.enc" \ | ||
| 64 | + -outform DER "$pub_key" | ||
| 65 | + rm "${bak_dir}/${1}" | ||
| 66 | +} | ||
| 67 | + | ||
| 68 | +# Rotate the current backups in S3 | ||
| 69 | +# Args: | ||
| 70 | +# $1 = backup dump filename, e.g. 'mydb' | ||
| 71 | +# $2 = max number of backup files to store at a time | ||
| 72 | +rotate_backup() { | ||
| 73 | + folder=$1 | ||
| 74 | + max=$2 | ||
| 75 | + | ||
| 76 | + # Backups will stored inside subfolders (prefixes) | ||
| 77 | + # in S3, so only look at 'PRE' objects. | ||
| 78 | + baks=$( "$bin_aws" --output text --region "$s3_region" \ | ||
| 79 | + s3 ls "s3://${s3_bucket}/" | grep '^\s*PRE' | \ | ||
| 80 | + sed -e 's/^ *PRE //' -e 's/\/$//' | \ | ||
| 81 | + grep "^${folder}" || echo "" ) | ||
| 82 | + | ||
| 83 | + echo "$(date) : Backup rotation for ${folder}." | ||
| 84 | + start=$((max - 1)) | ||
| 85 | + | ||
| 86 | + for (( x=start ; x > 0 ; x-- )) ; do | ||
| 87 | + if echo "$baks" | grep "^${folder}\\.${x}\$" ; then | ||
| 88 | + newx=$((x + 1)) | ||
| 89 | + if [[ $newx -lt $max ]] ; then | ||
| 90 | + "$bin_aws" --region "$s3_region" \ | ||
| 91 | + s3 mv --recursive "s3://${s3_bucket}/${folder}.${x}" \ | ||
| 92 | + "s3://${s3_bucket}/${folder}.${newx}" | ||
| 93 | + else | ||
| 94 | + "$bin_aws" --region "$s3_region" \ | ||
| 95 | + s3 rm --recursive "s3://${s3_bucket}/${folder}.${x}" | ||
| 96 | + fi | ||
| 97 | + fi | ||
| 98 | + done | ||
| 99 | + | ||
| 100 | + if echo "$baks" | grep "^${folder}\$" ; then | ||
| 101 | + if [[ $max -gt 1 ]] ; then | ||
| 102 | + "$bin_aws" --region "$s3_region" \ | ||
| 103 | + s3 mv --recursive "s3://${s3_bucket}/${folder}" \ | ||
| 104 | + "s3://${s3_bucket}/${folder}.1" | ||
| 105 | + else | ||
| 106 | + "$bin_aws" --region "$s3_region" \ | ||
| 107 | + s3 rm --recursive "s3://${s3_bucket}/${folder}" | ||
| 108 | + fi | ||
| 109 | + fi | ||
| 110 | +} | ||
| 111 | + | ||
| 112 | +# Upload the compressed db backup file. It will be uploaded | ||
| 113 | +# to this example location: ${s3_bucket}/dump_filename/dump_filename.gz.enc | ||
| 114 | +# | ||
| 115 | +# A timestamp file will also be uploaded to this location: | ||
| 116 | +# ${s3_bucket}/dump_filename/YYYY-MM-DDThh:mm:ss.txt | ||
| 117 | +# Args: | ||
| 118 | +# $1 = backup dump filename, e.g. 'mydb' | ||
| 119 | +# $2 = if file is encrypted or not (boolean) | ||
| 120 | +upload_to_s3() { | ||
| 121 | + keyname=$1 | ||
| 122 | + if [ "$2" = true ] ; then | ||
| 123 | + fname="${keyname}.gz.enc" | ||
| 124 | + else | ||
| 125 | + fname="${keyname}.gz" | ||
| 126 | + fi | ||
| 127 | + | ||
| 128 | + echo "$(date) : Upload ${fname} to S3 bucket ${s3_bucket}." | ||
| 129 | + | ||
| 130 | + stamp=$( date +"%FT%T" ) | ||
| 131 | + echo "Uploaded: ${stamp}" > "${bak_dir}/${stamp}.txt" | ||
| 132 | + | ||
| 133 | + "$bin_aws" --region "$s3_region" \ | ||
| 134 | + s3 mv "${bak_dir}/${fname}" \ | ||
| 135 | + "s3://${s3_bucket}/${keyname}/${fname}" | ||
| 136 | + "$bin_aws" --region "$s3_region" \ | ||
| 137 | + s3 mv "${bak_dir}/${stamp}.txt" \ | ||
| 138 | + "s3://${s3_bucket}/${keyname}/${stamp}.txt" | ||
| 139 | +} | ||
| 140 | + | ||
| 141 | +## Do the backup only if this node is the primary: | ||
| 142 | +if "$bin_mongo" --host="${db_host}:${db_port}" -u "$db_user" \ | ||
| 143 | + -p "$db_pass" --authenticationDatabase="${db_auth}" \ | ||
| 144 | + --eval 'db.isMaster()["ismaster"]' | grep -q true ; then | ||
| 145 | + | ||
| 146 | + if [[ ! -d "$bak_dir" ]] ; then | ||
| 147 | + mkdir -p "$bak_dir" | ||
| 148 | + fi | ||
| 149 | + | ||
| 150 | +<% @db_map.each do |x| -%> | ||
| 151 | +<% | ||
| 152 | + if x.is_a?(Array) | ||
| 153 | + db_name = x[0] | ||
| 154 | + x = x[1] | ||
| 155 | + else | ||
| 156 | + db_name = x[:db_name] | ||
| 157 | + end | ||
| 158 | + | ||
| 159 | + do_backup = x.has_key?(:backup) ? x[:backup] : true | ||
| 160 | + is_enc = x.has_key?(:bak_encrypted) ? x[:bak_encrypted] : false | ||
| 161 | + bak_filename = x[:bak_filename] || db_name | ||
| 162 | + bak_maxcopies = x[:bak_maxcopies] || 30 | ||
| 163 | +-%> | ||
| 164 | +<% if do_backup -%> | ||
| 165 | + # Database: <%= db_name %> | ||
| 166 | + export_db <%= db_name %> <%= bak_filename %> | ||
| 167 | +<% if is_enc -%> | ||
| 168 | + encrypt_backup <%= bak_filename %>.gz | ||
| 169 | +<% end -%> | ||
| 170 | + rotate_backup <%= bak_filename %> <%= bak_maxcopies %> | ||
| 171 | + upload_to_s3 <%= bak_filename %> <%= is_enc %> | ||
| 172 | + | ||
| 173 | +<% end -%> | ||
| 174 | +<% end -%> | ||
| 175 | + echo "$(date) : Done." | ||
| 176 | +fi | ||
| 177 | + | ||
| 178 | +rm "$tmp_file" |
| 1 | +var db_pass_root = "<%= @db_pass_root %>"; | ||
| 2 | +var db_pass_backup = "<%= @db_pass_backup %>"; | ||
| 3 | + | ||
| 4 | +db = db.getSiblingDB("admin"); | ||
| 5 | +db.createUser( { user: "root", pwd: db_pass_root, roles: [ "root", "__system" ] } ); | ||
| 6 | +db.auth("root", db_pass_root); | ||
| 7 | + | ||
| 8 | +<% if @is_logrename -%> | ||
| 9 | +db.runCommand( { logRotate : 1 } ); | ||
| 10 | + | ||
| 11 | +<% end -%> | ||
| 12 | +db.createUser( { user: "backup", pwd: db_pass_backup, roles: [ "backup" ] } ); | ||
| 13 | + | ||
| 14 | +<% @db_map.each do |x| -%> | ||
| 15 | +<% | ||
| 16 | + if x.is_a?(Array) | ||
| 17 | + db_name = x[0] | ||
| 18 | + x = x[1] | ||
| 19 | + else | ||
| 20 | + db_name = x[:db_name] | ||
| 21 | + end | ||
| 22 | + | ||
| 23 | + db_auth = x.has_key?(:db_auth) ? x[:db_auth] : db_name | ||
| 24 | +-%> | ||
| 25 | +db = db.getSiblingDB("<%= db_auth %>"); | ||
| 26 | +db.createUser( { user: "<%= x[:db_user] %>", pwd: "<%= x[:db_pass] %>", roles: [ { role: "readWrite", db: "<%= db_name %>" }, { role: "dbAdmin", db: "<%= db_name %>" } ] } ); | ||
| 27 | + | ||
| 28 | +<% end -%> |