Release 1.5.4

sean-smith · web-flow · commit 72402cc8d7bb · 2018-08-30T07:39:55.000-07:00
Release 1.5.4
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,14 @@ cfncluster-cookbook CHANGELOG
 
 This file is used to list changes made in each version of the cfncluster cookbook.
 
+1.5.4
+-----
+
+- Set SGE Accounting summary to be true, this reports a single accounting record
+for a mpi job
+- Add option to disable ganglia `extra_json = { "cfncluster" : { "ganglia_enabled" : "no" } }`
+
+
 1.5.2
 -----
 
diff --git a/amis/build_ami.sh b/amis/build_ami.sh
@@ -13,19 +13,20 @@
 #
 # Usage: build_ami.sh <os> <region> [private|public] [<build-date>]
 #   os: the os to build (supported values: all|centos6|centos7|alinux|ubuntu1404|ubuntu1604)
-#   region: region to build (supported values: all|us-east-1|...)
+#   partition: partition to build in (supported values: commercial|govcloud)
+#   region: region to copy ami too (supported values: all|us-east-1|us-gov-west-1|...)
 #   private|public: specifies AMIs visibility (optional, default is private)
 #   build-date: timestamp to append to the AMIs names (optional)
 
 set -e
 
 os=$1
-region=$2
-public=$3
-build_date=$4
+partition=$2
+region=$3
+public=$4
+build_date=$5
 
 available_os="centos6 centos7 alinux ubuntu1404 ubuntu1604"
-available_regions="$(aws ec2 --region us-east-1 describe-regions --query Regions[].RegionName --output text | tr '\t' ',')"
 cwd="$(dirname $0)"
 export VENDOR_PATH="${cwd}/../../vendor/cookbooks"
 
@@ -36,17 +37,27 @@ if [ "x${os}" == "x" ]; then
 fi
 
 if [ "x${region}" == "x" ]; then
-  echo "Must provide AWS region to build for."
-  echo "Options: us-east-1 all"
+  echo "Must provide AWS region to copy ami into"
+  echo "Options: all us-east-1 us-gov-west-1 ..."
   exit 1
 fi
 
 if [ "${public}" == "public" ]; then
   export AMI_PERMS="all"
 fi
 
-# NOTE: the AMI is always built in us-east-1 and then copied to the specified regions
+if [ "${partition}" == "commercial" ]; then
+  export AWS_REGION="us-east-1"
+elif [ "${partition}" == "govcloud" ]; then
+  export AWS_REGION="us-gov-west-1"
+else
+  echo "Must provide AWS partition to build for."
+  echo "Options: commercial govcloud"
+  exit 1
+fi
+
 if [ "${region}" == "all" ]; then
+  available_regions="$(aws ec2 --region ${AWS_REGION} describe-regions --query Regions[].RegionName --output text | tr '\t' ',')"
   export BUILD_FOR=${available_regions}
 else
   export BUILD_FOR=${region}
diff --git a/amis/packer_alinux.json b/amis/packer_alinux.json
@@ -13,13 +13,22 @@
     "instance_type" : "{{env `AWS_FLAVOR_ID`}}",
     "subnet_id" : "{{env `AWS_SUBNET_ID`}}",
     "vpc_id" : "{{env `AWS_VPC_ID`}}",
-    "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}"
+    "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}",
+    "region" : "{{env `AWS_REGION`}}"
   },
   "builders" : [
     {
       "type" : "amazon-ebs",
-      "region" : "us-east-1",
-      "source_ami" : "ami-55ef662f",
+      "region" : "{{user `region`}}",
+      "source_ami_filter": {
+        "filters": {
+          "virtualization-type": "hvm",
+          "name": "amzn-ami-hvm-*.*.*.*-x86_64-gp2",
+          "root-device-type": "ebs",
+          "owner-alias": "amazon"
+        },
+        "most_recent": true
+      },
       "ami_regions" : "{{user `build_for`}}",
       "ami_groups" : "{{user `ami_perms`}}",
       "instance_type" : "{{user `instance_type`}}",
diff --git a/amis/packer_centos6.json b/amis/packer_centos6.json
@@ -13,13 +13,22 @@
     "instance_type" : "{{env `AWS_FLAVOR_ID`}}",
     "subnet_id" : "{{env `AWS_SUBNET_ID`}}",
     "vpc_id" : "{{env `AWS_VPC_ID`}}",
-    "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}"
+    "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}",
+    "region" : "{{env `AWS_REGION`}}"
   },
   "builders" : [
     {
       "type" : "amazon-ebs",
-      "region" : "us-east-1",
-      "source_ami" : "ami-44f1aa3e",
+      "region" : "{{user `region`}}",
+      "source_ami_filter": {
+        "filters": {
+          "virtualization-type": "hvm",
+          "name": "CentOS 6.x x86_64 - minimal with cloud-init - *",
+          "root-device-type": "ebs"
+        },
+        "owners": ["247102896272"],
+        "most_recent": true
+      },
       "ami_regions" : "{{user `build_for`}}",
       "ami_groups" : "{{user `ami_perms`}}",
       "instance_type" : "{{user `instance_type`}}",
diff --git a/amis/packer_centos7.json b/amis/packer_centos7.json
@@ -13,13 +13,22 @@
     "instance_type" : "{{env `AWS_FLAVOR_ID`}}",
     "subnet_id" : "{{env `AWS_SUBNET_ID`}}",
     "vpc_id" : "{{env `AWS_VPC_ID`}}",
-    "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}"
+    "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}",
+    "region" : "{{env `AWS_REGION`}}"
   },
   "builders" : [
     {
       "type" : "amazon-ebs",
-      "region" : "us-east-1",
-      "source_ami" : "ami-06fea57c",
+      "region" : "{{user `region`}}",
+      "source_ami_filter": {
+        "filters": {
+          "virtualization-type": "hvm",
+          "name": "CentOS 7.x x86_64 - minimal with cloud-init - *",
+          "root-device-type": "ebs"
+        },
+        "owners": ["247102896272"],
+        "most_recent": true
+      },
       "ami_regions" : "{{user `build_for`}}",
       "ami_groups" : "{{user `ami_perms`}}",
       "instance_type" : "{{user `instance_type`}}",
diff --git a/amis/packer_ubuntu1404.json b/amis/packer_ubuntu1404.json
@@ -13,15 +13,24 @@
     "instance_type" : "{{env `AWS_FLAVOR_ID`}}",
     "subnet_id" : "{{env `AWS_SUBNET_ID`}}",
     "vpc_id" : "{{env `AWS_VPC_ID`}}",
-    "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}"
+    "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}",
+    "region" : "{{env `AWS_REGION`}}"
   },
   "builders" : [
     {
       "type" : "amazon-ebs",
-      "region" : "us-east-1",
+      "region" : "{{user `region`}}",
+      "source_ami_filter": {
+        "filters": {
+          "virtualization-type": "hvm",
+          "name": "ubuntu/images/hvm-ssd/ubuntu-trusty-14.04-amd64-server-*",
+          "root-device-type": "ebs"
+        },
+        "owners": ["099720109477", "513442679011"],
+        "most_recent": true
+      },
       "ami_regions" : "{{user `build_for`}}",
       "ami_groups" : "{{user `ami_perms`}}",
-      "source_ami" : "ami-c29e1cb8",
       "instance_type" : "{{user `instance_type`}}",
       "ssh_username" : "ubuntu",
       "ssh_pty" : true,
diff --git a/amis/packer_ubuntu1604.json b/amis/packer_ubuntu1604.json
@@ -13,15 +13,24 @@
     "instance_type" : "{{env `AWS_FLAVOR_ID`}}",
     "subnet_id" : "{{env `AWS_SUBNET_ID`}}",
     "vpc_id" : "{{env `AWS_VPC_ID`}}",
-    "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}"
+    "ami_name_prefix" : "{{env `AMI_NAME_PREFIX`}}",
+    "region" : "{{env `AWS_REGION`}}"
   },
   "builders" : [
     {
       "type" : "amazon-ebs",
-      "region" : "us-east-1",
+      "region" : "{{user `region`}}",
+      "source_ami_filter": {
+        "filters": {
+          "virtualization-type": "hvm",
+          "name": "ubuntu/images/hvm-ssd/ubuntu-xenial-16.04-amd64-server-*",
+          "root-device-type": "ebs"
+        },
+        "owners": ["099720109477", "513442679011"],
+        "most_recent": true
+      },
       "ami_regions" : "{{user `build_for`}}",
       "ami_groups" : "{{user `ami_perms`}}",
-      "source_ami" : "ami-aa2ea6d0",
       "instance_type" : "{{user `instance_type`}}",
       "ssh_username" : "ubuntu",
       "ssh_pty" : true,
diff --git a/amis/packer_variables.json b/amis/packer_variables.json
@@ -1,6 +1,6 @@
 {
-  "cfncluster_version": "1.5.2",
-  "cfncluster_cookbook_version": "1.5.2",
+  "cfncluster_version": "1.5.4",
+  "cfncluster_cookbook_version": "1.5.4",
   "chef_version": "14.2.0",
   "ridley_version": "5.1.1",
   "berkshelf_version": "7.0.4"
diff --git a/attributes/default.rb b/attributes/default.rb
@@ -19,7 +19,7 @@
 default['cfncluster']['scripts_dir'] = "#{node['cfncluster']['base_dir']}/scripts"
 default['cfncluster']['license_dir'] = "#{node['cfncluster']['base_dir']}/licenses"
 # Python packages
-default['cfncluster']['cfncluster-node-version'] = '1.5.2'
+default['cfncluster']['cfncluster-node-version'] = '1.5.4'
 default['cfncluster']['cfncluster-supervisor-version'] = '3.3.1'
 # URLs to software packages used during install receipes
 # Gridengine software
@@ -35,6 +35,7 @@
 default['cfncluster']['munge']['munge_version'] = '0.5.12'
 default['cfncluster']['munge']['munge_url'] = 'https://github.com/dun/munge/archive/munge-0.5.12.tar.gz'
 # Ganglia
+default['cfncluster']['ganglia_enabled'] = 'yes'
 default['cfncluster']['ganglia']['version'] = '3.7.2'
 default['cfncluster']['ganglia']['url'] = 'https://github.com/ganglia/monitor-core/archive/3.7.2.tar.gz'
 default['cfncluster']['ganglia']['web_version'] = '3.7.2'
diff --git a/metadata.rb b/metadata.rb
@@ -6,7 +6,7 @@
 long_description 'Installs/Configures cfncluster'
 issues_url 'https://github.com/awslabs/cfncluster-cookbook/issues'
 source_url 'https://github.com/awslabs/cfncluster-cookbook'
-version '1.5.2'
+version '1.5.4'
 
 depends 'build-essential', '~> 8.1.1'
 depends 'poise-python', '~> 1.7.0'
diff --git a/recipes/_compute_base_config.rb b/recipes/_compute_base_config.rb
@@ -43,16 +43,18 @@
 end
 
 # Configure Ganglia
-template '/etc/ganglia/gmond.conf' do
-  source 'gmond.conf.erb'
-  owner 'root'
-  group 'root'
-  mode '0644'
-end
+if node['cfncluster']['ganglia_enabled'] == "yes"
+  template '/etc/ganglia/gmond.conf' do
+    source 'gmond.conf.erb'
+    owner 'root'
+    group 'root'
+    mode '0644'
+  end
 
-service node['cfncluster']['ganglia']['gmond_service'] do
-  supports restart: true
-  action %i[enable restart]
+  service node['cfncluster']['ganglia']['gmond_service'] do
+    supports restart: true
+    action %i[enable restart]
+  end
 end
 
 # Setup cluster user
diff --git a/recipes/_ganglia_install.rb b/recipes/_ganglia_install.rb
diff --git a/recipes/_master_base_config.rb b/recipes/_master_base_config.rb
diff --git a/recipes/_master_sge_config.rb b/recipes/_master_sge_config.rb
diff --git a/recipes/base_install.rb b/recipes/base_install.rb
diff --git a/recipes/torque_config.rb b/recipes/torque_config.rb

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`		`- "cfncluster_version": "1.5.2",`
`3`		`- "cfncluster_cookbook_version": "1.5.2",`
	`2`	`+ "cfncluster_version": "1.5.4",`
	`3`	`+ "cfncluster_cookbook_version": "1.5.4",`
`4`	`4`	`"chef_version": "14.2.0",`
`5`	`5`	`"ridley_version": "5.1.1",`
`6`	`6`	`"berkshelf_version": "7.0.4"`