Release 1.5.2

Author: sean-smith

.kitchen.cloud.yml

@@ -15,7 +15,7 @@ driver_config:
 
 provisioner:
   name: chef_zero
-  require_chef_omnibus: 12.19.36
+  require_chef_omnibus: 14.2.0
   retry_on_exit_code:
     - 35 # 35 is the exit code signaling that the node is rebooting
   max_retries: 1
@@ -28,7 +28,7 @@ platforms:
   - name: amazon-linux-latest
     driver_plugin: ec2
     driver_config:
-      image_id: ami-55ef662f
+      image_id: <%= ENV['ALINUX_IMAGE_ID'] || "ami-55ef662f" %>
       block_device_mappings:
         - device_name: /dev/xvda
           ebs:
@@ -89,7 +89,7 @@ platforms:
   - name: centos-6-minimal
     driver_plugin: ec2
     driver_config:
-      image_id: ami-9ff841e5
+      image_id: <%= ENV['CENTOS6_IMAGE_ID'] || "ami-44f1aa3e" %>
       block_device_mappings:
         - device_name: /dev/sda1
           ebs:
@@ -150,7 +150,7 @@ platforms:
   - name: centos-7-minimal
     driver_plugin: ec2
     driver_config:
-      image_id: ami-01098c7b
+      image_id: <%= ENV['CENTOS7_IMAGE_ID'] || "ami-06fea57c" %>
       block_device_mappings:
         - device_name: /dev/sda1
           ebs:
@@ -211,7 +211,7 @@ platforms:
   - name: ubuntu-14-04-lts
     driver_plugin: ec2
     driver_config:
-      image_id: ami-c29e1cb8
+      image_id: <%= ENV['UBUNTU1404_IMAGE_ID'] || "ami-c29e1cb8" %>
       block_device_mappings:
         - device_name: /dev/sda1
           ebs:
@@ -272,7 +272,7 @@ platforms:
   - name: ubuntu-16-04-lts
     driver_plugin: ec2
     driver_config:
-      image_id: ami-aa2ea6d0
+      image_id: <%= ENV['UBUNTU1604_IMAGE_ID'] || "ami-aa2ea6d0" %>
       block_device_mappings:
         - device_name: /dev/sda1
           ebs:

.kitchen.yml

@@ -4,7 +4,7 @@ driver:
 
 provisioner:
   name: chef_zero
-  require_chef_omnibus: 12.19.36
+  require_chef_omnibus: 14.2.0
 
 platforms:
   - name: centos-6.6

CHANGELOG.md

@@ -3,10 +3,16 @@ cfncluster-cookbook CHANGELOG
 
 This file is used to list changes made in each version of the cfncluster cookbook.
 
+1.5.2
+-----
+
+- Fix bug that prevented c5d/m5d instances from working
+- Set CPU as a consumable resource in slurm config
+
 1.5.1
 -----
 
-Major new features/updates: 
+Major new features/updates:
 
   - Added parameter to specify custom cfncluster-node package
 
@@ -27,21 +33,21 @@ Major new features/updates:
 1.4.0
 -----
 
-Major new features/updates: 
+Major new features/updates:
 
   - Updated to Amazon Linux 2017.09.1
   - Applied patches to Ubuntu 16.04
   - Applied patches to Ubuntu 14.04
   - Updated to Centos 7.4
   - Upgraded Centos 6 AMI
-  - Updated to Nvidia driver 384 
+  - Updated to Nvidia driver 384
   - Updated to CUDA 9
-  - Updated to latest cfncluster-node 1.4.2 
+  - Updated to latest cfncluster-node 1.4.2
 
 Bug fixes/minor improvements:
 
   - Added support for NVMe-based instance store
-  - Fixed ganglia plotting issue on ubuntu 
+  - Fixed ganglia plotting issue on ubuntu
   - Fixed slow SLURM scaling times on systemd platforms.
 
 1.3.2

amis/build_ami.sh

@@ -25,7 +25,7 @@ public=$3
 build_date=$4
 
 available_os="centos6 centos7 alinux ubuntu1404 ubuntu1604"
-available_regions="eu-west-1,eu-west-2,eu-west-3,ap-southeast-1,ap-southeast-2,eu-central-1,ap-northeast-1,ap-northeast-2,ap-northeast-3,us-west-2,sa-east-1,us-west-1,us-east-2,ap-south-1,ca-central-1"
+available_regions="$(aws ec2 --region us-east-1 describe-regions --query Regions[].RegionName --output text | tr '\t' ',')"
 cwd="$(dirname $0)"
 export VENDOR_PATH="${cwd}/../../vendor/cookbooks"
 

amis/build_env_setup.sh

@@ -1,24 +1,21 @@
 #!/bin/bash
 
 # This script sets up environment for building cfncluster ami
-# Launch Ec2 machine with amazon linux and run this script
+# Launch EC2 machine with amazon linux and run this script
 
 # Get latest packages and Development tools
 sudo yum -y update
 sudo yum -y groupinstall 'Development Tools'
 
-# Get chefdk - As of now we use 1.4.3
-# Verify cookbook chef version in metadata.rb file
-# In case you see version mismatch, Please install
-# chefdk version referred in metadata file
-wget https://packages.chef.io/files/stable/chefdk/1.4.3/el/7/chefdk-1.4.3-1.el7.x86_64.rpm
-sudo rpm --install chefdk-1.4.3-1.el7.x86_64.rpm
+# Get chefdk
+# Verify cookbook chef version in packer_variables.json file
+wget https://packages.chef.io/files/stable/chefdk/3.0.36/el/7/chefdk-3.0.36-1.el7.x86_64.rpm
+sudo rpm --install chefdk-3.0.36-1.el7.x86_64.rpm
 chef -v
 
 # Get Packer - tool used to build ami
-# As of now we use 1.1.1
-wget https://releases.hashicorp.com/packer/1.1.1/packer_1.1.1_linux_amd64.zip
-unzip packer_1.1.1_linux_amd64.zip
+wget https://releases.hashicorp.com/packer/1.2.4/packer_1.2.4_linux_amd64.zip
+unzip packer_1.2.4_linux_amd64.zip
 # Copy packer to PATH
 sudo cp packer /usr/local/bin/
 packer --version
@@ -28,7 +25,7 @@ packer --version
 # git clone https://github.com/awslabs/cfncluster-cookbook.git
 # cd cfncluster-cookbook
 
-# Run ami builder script 
+# Run ami builder script
 # ./build_ami.sh <operating_system> <region> <public/private>
 # example
 # ./build_ami.sh all us-east-1 private

amis/centos-upgrade-first-stage.sh

@@ -76,14 +76,12 @@ set +e
 echo "Disabling SELinux"
 sudo /bin/sed -r -i -e 's/SELINUX=enforcing/SELINUX=permissive/' /etc/selinux/config
 
-# Updating GRUB to blacklist the neuvou driver (so the nVidia driver can
-# install) and turning off ifnames, because a bunch of the scripts expect
-# to find eth0.  Should handle the eth naming better, but that's todo.
+# Updating GRUB to blacklist the nouveau driver (so the nVidia driver can install)
 echo "Updating Grub command line"
 if test $is_centos6 -eq 1; then
-    sudo grubby --update-kernel=ALL --args="net.ifnames=0 rd.driver.blacklist=nouveau nouveau.modeset=0"
+    sudo grubby --update-kernel=ALL --args="rd.driver.blacklist=nouveau nouveau.modeset=0"
 else
-    sudo /bin/sed -r -i -e 's/GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 net.ifnames=0 rd.driver.blacklist=nouveau nouveau.modeset=0"/' /etc/default/grub
+    sudo /bin/sed -r -i -e 's/GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 rd.driver.blacklist=nouveau nouveau.modeset=0"/' /etc/default/grub
     sudo grub2-mkconfig -o /boot/grub2/grub.cfg
 fi
 

amis/packer_alinux.json

@@ -12,7 +12,8 @@
     "nvidia_enabled" : "{{env `NVIDIA_ENABLED`}}",
     "instance_type" : "{{env `AWS_FLAVOR_ID`}}",
     "subnet_id" : "{{env `AWS_SUBNET_ID`}}",
-    "vpc_id" : "{{env `AWS_VPC_ID`}}"
+    "vpc_id" : "{{env `AWS_VPC_ID`}}",
+    "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}"
   },
   "builders" : [
     {
@@ -24,7 +25,7 @@
       "instance_type" : "{{user `instance_type`}}",
       "ssh_username" : "ec2-user",
       "ssh_pty" : true,
-      "ami_name" : "cfncluster-{{user `cfncluster_version`}}-amzn-hvm-{{user `build_date`}}",
+      "ami_name" : "{{user `ami_name_prefix`}}cfncluster-{{user `cfncluster_version`}}-amzn-hvm-{{user `build_date`}}",
       "subnet_id" : "{{user `subnet_id`}}",
       "vpc_id" : "{{user `vpc_id`}}",
       "skip_region_validation" : true,

amis/packer_centos6.json

@@ -12,7 +12,8 @@
     "nvidia_enabled" : "{{env `NVIDIA_ENABLED`}}",
     "instance_type" : "{{env `AWS_FLAVOR_ID`}}",
     "subnet_id" : "{{env `AWS_SUBNET_ID`}}",
-    "vpc_id" : "{{env `AWS_VPC_ID`}}"
+    "vpc_id" : "{{env `AWS_VPC_ID`}}",
+    "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}"
   },
   "builders" : [
     {
@@ -24,7 +25,7 @@
       "instance_type" : "{{user `instance_type`}}",
       "ssh_username" : "centos",
       "ssh_pty" : true,
-      "ami_name" : "cfncluster-{{user `cfncluster_version`}}-centos6-hvm-{{user `build_date`}}",
+      "ami_name" : "{{user `ami_name_prefix`}}cfncluster-{{user `cfncluster_version`}}-centos6-hvm-{{user `build_date`}}",
       "subnet_id" : "{{user `subnet_id`}}",
       "vpc_id" : "{{user `vpc_id`}}",
       "skip_region_validation" : true,

amis/packer_centos7.json

@@ -12,7 +12,8 @@
     "nvidia_enabled" : "{{env `NVIDIA_ENABLED`}}",
     "instance_type" : "{{env `AWS_FLAVOR_ID`}}",
     "subnet_id" : "{{env `AWS_SUBNET_ID`}}",
-    "vpc_id" : "{{env `AWS_VPC_ID`}}"
+    "vpc_id" : "{{env `AWS_VPC_ID`}}",
+    "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}"
   },
   "builders" : [
     {
@@ -24,7 +25,7 @@
       "instance_type" : "{{user `instance_type`}}",
       "ssh_username" : "centos",
       "ssh_pty" : true,
-      "ami_name" : "cfncluster-{{user `cfncluster_version`}}-centos7-hvm-{{user `build_date`}}",
+      "ami_name" : "{{user `ami_name_prefix`}}cfncluster-{{user `cfncluster_version`}}-centos7-hvm-{{user `build_date`}}",
       "subnet_id" : "{{user `subnet_id`}}",
       "vpc_id" : "{{user `vpc_id`}}",
       "skip_region_validation" : true,

amis/packer_ubuntu1404.json

@@ -12,7 +12,8 @@
     "nvidia_enabled" : "{{env `NVIDIA_ENABLED`}}",
     "instance_type" : "{{env `AWS_FLAVOR_ID`}}",
     "subnet_id" : "{{env `AWS_SUBNET_ID`}}",
-    "vpc_id" : "{{env `AWS_VPC_ID`}}"
+    "vpc_id" : "{{env `AWS_VPC_ID`}}",
+    "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}"
   },
   "builders" : [
     {
@@ -24,7 +25,7 @@
       "instance_type" : "{{user `instance_type`}}",
       "ssh_username" : "ubuntu",
       "ssh_pty" : true,
-      "ami_name" : "cfncluster-{{user `cfncluster_version`}}-ubuntu-1404-lts-hvm-{{user `build_date`}}",
+      "ami_name" : "{{user `ami_name_prefix`}}cfncluster-{{user `cfncluster_version`}}-ubuntu-1404-lts-hvm-{{user `build_date`}}",
       "subnet_id" : "{{user `subnet_id`}}",
       "vpc_id" : "{{user `vpc_id`}}",
       "skip_region_validation" : true,