Merge Release 2.2.1

lukeseawalker · web-flow · commit 554f5b69f699 · 2019-02-28T09:10:39.000+01:00
Merge Release 2.2.1
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,28 @@ aws-parallelcluster-cookbook CHANGELOG
 
 This file is used to list changes made in each version of the AWS ParallelCluster cookbook.
 
+2.2.1
+-----
+
+**ENHANCEMENTS**
+- `FSx Lustre`: add support for FSx Lustre in Centos7. In case of custom AMI, FSx Lustre is
+  only supported with Centos 7.5 and Centos 7.6.
+
+**CHANGES**
+- `SGE`: allow users to force job deletion
+- `Centos7`: use official AMI as the base image when building ParallelCluster AMI
+
+**BUG FIXES**
+- `Torque`: wait for scheduler initialization before completing compute node setup
+- `EBS`: fix block device conversion to correctly attach ebs nvme volumes
+- `Packer`: retrieve `aws-cfn-bootstrap-latest` package from `cn-north-1` in all China regions.
+- `RAID`: automatically prepend `/` to `raid_shared_dir` when not specified in cluster config
+- `EBS`: remove extra leading `/` when mounting EBS volumes
+
+**TOOLING**
+- Add a script to simplify cookbook package upload when using `custom_chef_cookbook` option
+
+
 2.1.1
 -----
 
diff --git a/Gemfile b/Gemfile
@@ -11,6 +11,7 @@ end
 group :test do
   gem 'chefspec', '~> 4.0'
   gem 'kitchen-vagrant', '~> 1.1.0'
+  gem 'safe_yaml', '~> 1.0.5'
   gem 'test-kitchen', '~> 1.13.2'
 end
 
diff --git a/amis/packer_centos7.json b/amis/packer_centos7.json
@@ -25,10 +25,10 @@
       "source_ami_filter": {
         "filters": {
           "virtualization-type": "hvm",
-          "name": "CentOS 7.x x86_64 - minimal with cloud-init - *",
+          "name": "CentOS Linux 7 x86_64 HVM EBS ENA *",
           "root-device-type": "ebs"
         },
-        "owners": ["247102896272"],
+        "owners": ["410186602215"],
         "most_recent": true
       },
       "ami_regions" : "{{user `build_for`}}",
@@ -140,6 +140,14 @@
         "sudo /bin/sed -r -i -e 's/SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config"
       ]
     },
+    {
+      "type" : "shell",
+      "only": ["centos7"],
+      "inline" : [
+        "sudo /bin/sed -r -i -e 's/GRUB_CMDLINE_LINUX=\"(.*)\"/GRUB_CMDLINE_LINUX=\"\\1 rd.driver.blacklist=nouveau nouveau.modeset=0\"/' /etc/default/grub",
+        "sudo grub2-mkconfig -o /boot/grub2/grub.cfg"
+      ]
+    },
     {
       "type" : "shell",
       "inline" : [
diff --git a/amis/packer_variables.json b/amis/packer_variables.json
@@ -1,6 +1,6 @@
 {
-  "parallelcluster_version": "2.1.1",
-  "parallelcluster_cookbook_version": "2.1.1",
+  "parallelcluster_version": "2.2.1",
+  "parallelcluster_cookbook_version": "2.2.1",
   "chef_version": "14.2.0",
   "ridley_version": "5.1.1",
   "berkshelf_version": "7.0.4"
diff --git a/attributes/default.rb b/attributes/default.rb
@@ -19,8 +19,8 @@
 default['cfncluster']['scripts_dir'] = "#{node['cfncluster']['base_dir']}/scripts"
 default['cfncluster']['license_dir'] = "#{node['cfncluster']['base_dir']}/licenses"
 # Python packages
-default['cfncluster']['cfncluster-version'] = '2.1.1'
-default['cfncluster']['cfncluster-node-version'] = '2.1.1'
+default['cfncluster']['cfncluster-version'] = '2.2.1'
+default['cfncluster']['cfncluster-node-version'] = '2.2.1'
 default['cfncluster']['cfncluster-supervisor-version'] = '3.3.1'
 # URLs to software packages used during install recipes
 # Gridengine software
@@ -74,12 +74,29 @@
                                                 httpd boost-devel redhat-lsb mlocate mpich-devel openmpi-devel R atlas-devel
                                                 blas-devel fftw-devel libffi-devel openssl-devel dkms mysql-devel libedit-devel
                                                 libical-devel postgresql-devel postgresql-server sendmail mdadm]
+
+    # Lustre Drivers for Centos 6
+    default['cfncluster']['lustre']['version'] = '2.10.6'
+    default['cfncluster']['lustre']['kmod_url'] = 'https://downloads.whamcloud.com/public/lustre/lustre-2.10.6/el6/client/RPMS/x86_64/kmod-lustre-client-2.10.6-1.el6.x86_64.rpm'
+    default['cfncluster']['lustre']['client_url'] = 'https://downloads.whamcloud.com/public/lustre/lustre-2.10.6/el6/client/RPMS/x86_64/lustre-client-2.10.6-1.el6.x86_64.rpm'
+
     if node['platform_version'].to_i >= 7
       default['cfncluster']['base_packages'] = %w[vim ksh tcsh zsh openssl-devel ncurses-devel pam-devel net-tools openmotif-devel
                                                   libXmu-devel hwloc-devel libdb-devel tcl-devel automake autoconf pyparted libtool
                                                   httpd boost-devel redhat-lsb mlocate lvm2 mpich-devel openmpi-devel R atlas-devel
                                                   blas-devel fftw-devel libffi-devel openssl-devel dkms mariadb-devel libedit-devel
                                                   libical-devel postgresql-devel postgresql-server sendmail libxml2-devel libglvnd-devel mdadm]
+      if node['platform_version'].split('.')[1] == '6'
+        # Lustre Drivers for Centos 7.6
+        default['cfncluster']['lustre']['version'] = '2.10.6'
+        default['cfncluster']['lustre']['kmod_url'] = 'https://downloads.whamcloud.com/public/lustre/lustre-2.10.6/el7/client/RPMS/x86_64/kmod-lustre-client-2.10.6-1.el7.x86_64.rpm'
+        default['cfncluster']['lustre']['client_url'] = 'https://downloads.whamcloud.com/public/lustre/lustre-2.10.6/el7/client/RPMS/x86_64/lustre-client-2.10.6-1.el7.x86_64.rpm'
+      elsif node['platform_version'].split('.')[1] == '5'
+        # Lustre Drivers for Centos 7.5
+        default['cfncluster']['lustre']['version'] = '2.10.5'
+        default['cfncluster']['lustre']['kmod_url'] = 'https://downloads.whamcloud.com/public/lustre/lustre-2.10.5/el7.5.1804/client/RPMS/x86_64/kmod-lustre-client-2.10.5-1.el7.x86_64.rpm'
+        default['cfncluster']['lustre']['client_url'] = 'https://downloads.whamcloud.com/public/lustre/lustre-2.10.5/el7.5.1804/client/RPMS/x86_64/lustre-client-2.10.5-1.el7.x86_64.rpm'
+      end
     end
     default['cfncluster']['kernel_devel_pkg']['name'] = "kernel-lt-devel" if node['platform'] == 'centos' && node['platform_version'].to_i >= 6 && node['platform_version'].to_i < 7
     default['cfncluster']['rhel']['extra_repo'] = 'rhui-REGION-rhel-server-releases-optional' if node['platform'] == 'redhat' && node['platform_version'].to_i >= 6 && node['platform_version'].to_i < 7
@@ -157,7 +174,9 @@
 default['cfncluster']['cfn_node_type'] = nil
 default['cfncluster']['cfn_master'] = nil
 default['cfncluster']['cfn_cluster_user'] = 'ec2-user'
+default['cfncluster']['cfn_fsx_options'] = 'NONE'
+default['cfncluster']['cfn_fsx_fs_id'] = nil
 default['cfncluster']['custom_node_package'] = nil
 default['cfncluster']['custom_awsbatchcli_package'] = nil
-default['cfncluster']['cfn_raid_parameters'] = "NONE"
+default['cfncluster']['cfn_raid_parameters'] = 'NONE'
 default['cfncluster']['cfn_raid_vol_ids'] = nil
diff --git a/files/default/attachVolume.py b/files/default/attachVolume.py
@@ -15,7 +15,7 @@ def convert_dev(dev):
     # FIXME This approach could be broken in some OS variants, see
     # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/device_naming.html
     if '/nvme' in dev:
-        return '/dev/' + os.popen('sudo /usr/local/sbin/parallelcluster-ebsnvme-id -b ' + dev).read().strip()
+        return '/dev/' + os.popen('sudo /usr/local/sbin/parallelcluster-ebsnvme-id -u -b ' + dev).read().strip()
     elif '/hd' in dev:
         return dev.replace('hd', 'sd')
     elif '/xvd' in dev:
@@ -58,7 +58,7 @@ def main():
     if config.has_option('Boto', 'proxy') and config.has_option('Boto', 'proxy_port'):
         proxy = config.get('Boto', 'proxy')
         proxy_port = config.get('Boto', 'proxy_port')
-        proxy_config = Config(proxies={'https': "{}:{}".format(proxy, proxy_port)})
+        proxy_config = Config(proxies={'https': "{0}:{1}".format(proxy, proxy_port)})
 
     # Connect to AWS using boto
     ec2 = boto3.client('ec2', region_name=region, config=proxy_config)
diff --git a/files/default/ec2_dev_2_volid.py b/files/default/ec2_dev_2_volid.py
@@ -44,7 +44,7 @@ def main():
     if config.has_option('Boto', 'proxy') and config.has_option('Boto', 'proxy_port'):
         proxy = config.get('Boto', 'proxy')
         proxy_port = config.get('Boto', 'proxy_port')
-        proxy_config = Config(proxies={'https': "{}:{}".format(proxy, proxy_port)})
+        proxy_config = Config(proxies={'https': "{0}:{1}".format(proxy, proxy_port)})
 
     # Connect to AWS using boto
     ec2 = boto3.client('ec2', region_name=region, config=proxy_config)
diff --git a/files/default/qconf_scripts.tar.gz b/files/default/qconf_scripts.tar.gz
diff --git a/metadata.rb b/metadata.rb
@@ -7,7 +7,7 @@
 issues_url 'https://github.com/aws/aws-parallelcluster-cookbook/issues'
 source_url 'https://github.com/aws/aws-parallelcluster-cookbook'
 chef_version '14.2.0'
-version '2.1.1'
+version '2.2.1'
 
 supports 'amazon'
 supports 'centos', '= 6'
diff --git a/recipes/_compute_base_config.rb b/recipes/_compute_base_config.rb
@@ -33,6 +33,9 @@
 raid_shared_dir = node['cfncluster']['cfn_raid_parameters'].split(',')[0]
 
 if raid_shared_dir != "NONE"
+  # Path needs to be fully qualified, for example "shared/temp" becomes "/shared/temp"
+  raid_shared_dir = "/" + raid_shared_dir unless raid_shared_dir.start_with?("/")
+
   # Created RAID shared mount point
   directory raid_shared_dir do
     mode '1777'
@@ -85,7 +88,7 @@
 shared_dir_array = node['cfncluster']['cfn_shared_dir'].split(',')
 shared_dir_array.each_with_index do |dir, index|
   shared_dir_array[index] = dir.strip
-  shared_dir_array[index] = "/" + shared_dir_array[index]
+  shared_dir_array[index] = "/" + shared_dir_array[index] unless shared_dir_array[index].start_with?("/")
 end
 
 # Mount each volume with NFS
diff --git a/recipes/_lustre_install.rb b/recipes/_lustre_install.rb
@@ -0,0 +1,50 @@
+#
+# Cookbook Name:: aws-parallelcluster
+# Recipe:: lustre_install
+#
+# Copyright 2013-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the
+# License. A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Install only on Centos 7.6 and 7.5
+if node['platform'] == 'centos' && (5..6).cover?(node['platform_version'].split('.')[1].to_i)
+  lustre_kmod_rpm = "#{node['cfncluster']['sources_dir']}/kmod-lustre-client-#{node['cfncluster']['lustre']['version']}.x86_64.rpm"
+  lustre_client_rpm = "#{node['cfncluster']['sources_dir']}/lustre-client-#{node['cfncluster']['lustre']['version']}.x86_64.rpm"
+
+  # Get Lustre Kernel Module RPM
+  remote_file lustre_kmod_rpm do
+    source node['cfncluster']['lustre']['kmod_url']
+    mode '0644'
+    retries 3
+    retry_delay 5
+    not_if { ::File.exist?(lustre_kmod_rpm) }
+  end
+
+  # Get Lustre Client RPM
+  remote_file lustre_client_rpm do
+    source node['cfncluster']['lustre']['client_url']
+    mode '0644'
+    retries 3
+    retry_delay 5
+    not_if { ::File.exist?(lustre_client_rpm) }
+  end
+
+  # Install lustre mount drivers
+  yum_package 'lustre_kmod' do
+    source lustre_kmod_rpm
+  end
+
+  # Install lustre mount drivers
+  yum_package 'lustre_client' do
+    source lustre_client_rpm
+  end
+elsif node['platform'] == 'centos'
+  Chef::Log.warn("Unsupported version of Centos, #{node['platform_version']}, supported versions are 7.6 and 7.5")
+end
diff --git a/recipes/_master_base_config.rb b/recipes/_master_base_config.rb
@@ -35,7 +35,7 @@
 shared_dir_array = node['cfncluster']['cfn_shared_dir'].split(',')
 shared_dir_array.each_with_index do |dir, index|
   shared_dir_array[index] = dir.strip
-  shared_dir_array[index] = "/" + shared_dir_array[index]
+  shared_dir_array[index] = "/" + shared_dir_array[index] unless shared_dir_array[index].start_with?("/")
 end
 
 # Parse volume info into an arary
diff --git a/recipes/_master_sge_config.rb b/recipes/_master_sge_config.rb
@@ -65,3 +65,11 @@
     rm $TMPFILE
   SETAS
 end
+
+# Enable non-admin users to force delete a job
+bash "enable_forced_qdel" do
+  code <<-ENABLEFORCEDQDEL
+    . /opt/sge/default/common/settings.sh
+    /opt/sge/util/qconf_add_list_value -mconf qmaster_params ENABLE_FORCED_QDEL global
+  ENABLEFORCEDQDEL
+end
diff --git a/recipes/base_config.rb b/recipes/base_config.rb
@@ -58,3 +58,9 @@
   supports restart: true
   action %i[enable start]
 end
+
+# Only run FSx on centos for now
+if node['platform'] == 'centos'
+  # Mount FSx
+  include_recipe 'aws-parallelcluster::fsx_mount'
+end
diff --git a/recipes/base_install.rb b/recipes/base_install.rb
@@ -214,3 +214,6 @@
 
 # Install NVIDIA and CUDA
 include_recipe "aws-parallelcluster::_nvidia_install"
+
+# Install FSx options
+include_recipe "aws-parallelcluster::_lustre_install"
diff --git a/recipes/efs_mount.rb b/recipes/efs_mount.rb
@@ -14,10 +14,14 @@
 # limitations under the License.
 
 # Get shared_dir path and mount EFS filesystem
-efs_shared_dir = "/" + node['cfncluster']['cfn_efs_shared_dir'].split(',')[0]
+efs_shared_dir =  node['cfncluster']['cfn_efs_shared_dir'].split(',')[0]
 
 # Check to see if EFS is created
-if efs_shared_dir != "/NONE"
+if efs_shared_dir != "NONE"
+
+  # Path needs to be fully qualified, for example "shared/temp" becomes "/shared/temp"
+  efs_shared_dir = "/" + efs_shared_dir unless efs_shared_dir.start_with?("/")
+
   # Create the EFS shared directory
   directory efs_shared_dir do
     owner 'root'
diff --git a/recipes/fsx_mount.rb b/recipes/fsx_mount.rb
@@ -0,0 +1,48 @@
+#
+# Cookbook Name:: aws-parallelcluster
+# Recipe:: fsx_mount
+#
+# Copyright 2013-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the
+# License. A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+
+fsx_shared_dir = node['cfncluster']['cfn_fsx_options'].split(',')[0]
+
+# Check to see if FSx is created
+if fsx_shared_dir != "NONE"
+
+  # Path needs to be fully qualified, for example "shared/temp" becomes "/shared/temp"
+  fsx_shared_dir = "/" + fsx_shared_dir unless fsx_shared_dir.start_with?("/")
+
+  # Create the shared directories
+  directory fsx_shared_dir do
+    owner 'root'
+    group 'root'
+    mode '1777'
+    action :create
+  end
+
+  # Mount FSx over NFS
+  mount fsx_shared_dir do
+    device "#{node['cfncluster']['cfn_fsx_fs_id']}.fsx.#{node['cfncluster']['cfn_region']}.amazonaws.com@tcp:/fsx"
+    fstype 'lustre'
+    dump 0
+    pass 0
+    options %w[defaults _netdev]
+    action %i[mount enable]
+  end
+
+  # Make sure permission is correct
+  directory fsx_shared_dir do
+    owner 'root'
+    group 'root'
+    mode '1777'
+  end
+end
diff --git a/recipes/setup_raid_on_master.rb b/recipes/setup_raid_on_master.rb
diff --git a/recipes/sge_install.rb b/recipes/sge_install.rb
diff --git a/templates/default/torque.setup.erb b/templates/default/torque.setup.erb
diff --git a/util/bump-version.sh b/util/bump-version.sh
diff --git a/util/uploadCookbook.sh b/util/uploadCookbook.sh

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`		`- "parallelcluster_version": "2.1.1",`
`3`		`- "parallelcluster_cookbook_version": "2.1.1",`
	`2`	`+ "parallelcluster_version": "2.2.1",`
	`3`	`+ "parallelcluster_cookbook_version": "2.2.1",`
`4`	`4`	`"chef_version": "14.2.0",`
`5`	`5`	`"ridley_version": "5.1.1",`
`6`	`6`	`"berkshelf_version": "7.0.4"`