Skip to content

Commit 2ed3203

Browse files
sean-smithsean-smith
authored
Release 1.6.0
Release 1.6.0
2 parents 72402cc + d96ea0e commit 2ed3203

33 files changed

+323
-281
lines changed

‎.kitchen.yml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,11 @@ suites:
7373
run_list:
7474
- recipe[cfncluster::_prep_env]
7575
- recipe[cfncluster::sge_config]
76+
- recipe[cfncluster::tests]
7677
attributes:
7778
cfncluster:
7879
cfn_node_type: 'MasterServer'
80+
cfn_scheduler: 'sge'
7981
stack_name: <%= ENV['AWS_STACK_NAME'] %>
8082
cfn_volume: <%= ENV['CFN_VOLUME'] %>
8183
cfn_region: <%= ENV['AWS_DEFAULT_REGION'] %>
@@ -90,9 +92,11 @@ suites:
9092
run_list:
9193
- recipe[cfncluster::_prep_env]
9294
- recipe[cfncluster::torque_config]
95+
- recipe[cfncluster::tests]
9396
attributes:
9497
cfncluster:
9598
cfn_node_type: 'MasterServer'
99+
cfn_scheduler: 'torque'
96100
stack_name: <%= ENV['AWS_STACK_NAME'] %>
97101
cfn_volume: <%= ENV['CFN_VOLUME'] %>
98102
cfn_region: <%= ENV['AWS_DEFAULT_REGION'] %>
@@ -107,9 +111,11 @@ suites:
107111
run_list:
108112
- recipe[cfncluster::_prep_env]
109113
- recipe[cfncluster::slurm_config]
114+
- recipe[cfncluster::tests]
110115
attributes:
111116
cfncluster:
112117
cfn_node_type: 'MasterServer'
118+
cfn_scheduler: 'slurm'
113119
stack_name: <%= ENV['AWS_STACK_NAME'] %>
114120
cfn_volume: <%= ENV['CFN_VOLUME'] %>
115121
cfn_region: <%= ENV['AWS_DEFAULT_REGION'] %>
@@ -124,9 +130,11 @@ suites:
124130
run_list:
125131
- recipe[cfncluster::_prep_env]
126132
- recipe[cfncluster::sge_config]
133+
- recipe[cfncluster::tests]
127134
attributes:
128135
cfncluster:
129136
cfn_node_type: 'ComputeFleet'
137+
cfn_scheduler: 'sge'
130138
stack_name: <%= ENV['AWS_STACK_NAME'] %>
131139
cfn_volume: <%= ENV['CFN_VOLUME'] %>
132140
cfn_region: <%= ENV['AWS_DEFAULT_REGION'] %>
@@ -141,9 +149,11 @@ suites:
141149
run_list:
142150
- recipe[cfncluster::_prep_env]
143151
- recipe[cfncluster::torque_config]
152+
- recipe[cfncluster::tests]
144153
attributes:
145154
cfncluster:
146155
cfn_node_type: 'ComputeFleet'
156+
cfn_scheduler: 'torque'
147157
stack_name: <%= ENV['AWS_STACK_NAME'] %>
148158
cfn_volume: <%= ENV['CFN_VOLUME'] %>
149159
cfn_region: <%= ENV['AWS_DEFAULT_REGION'] %>
@@ -158,9 +168,11 @@ suites:
158168
run_list:
159169
- recipe[cfncluster::_prep_env]
160170
- recipe[cfncluster::slurm_config]
171+
- recipe[cfncluster::tests]
161172
attributes:
162173
cfncluster:
163174
cfn_node_type: 'ComputeFleet'
175+
cfn_scheduler: 'slurm'
164176
stack_name: <%= ENV['AWS_STACK_NAME'] %>
165177
cfn_volume: <%= ENV['CFN_VOLUME'] %>
166178
cfn_region: <%= ENV['AWS_DEFAULT_REGION'] %>

‎Berksfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
source 'https://supermarket.getchef.com'
1+
source 'https://supermarket.chef.io'
22

33
metadata

‎CHANGELOG.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,14 @@ cfncluster-cookbook CHANGELOG
33

44
This file is used to list changes made in each version of the cfncluster cookbook.
55

6+
1.6.0
7+
-----
8+
9+
- Add `scaledown_idletime` to nodewatcher config
10+
- Add cookbook recipes for jobwatcher
11+
- Remove publish_pending scripts
12+
13+
614
1.5.4
715
-----
816

‎amis/packer_centos6.json

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,32 @@
115115
},
116116
{
117117
"type" : "chef-solo",
118+
"remote_cookbook_paths" : [
119+
"/etc/chef/cookbooks"
120+
],
121+
"skip_install" : "true",
122+
"execute_command" : "sudo chef-client -z --no-color -c {{.ConfigPath}} -j {{.JsonPath}}",
123+
"json" : {
124+
"cfncluster" : {
125+
"default_pre_reboot" : "false"
126+
}
127+
},
128+
"run_list" : [
129+
"cfncluster::_default_pre"
130+
]
131+
},
132+
{
133+
"type" : "shell",
134+
"expect_disconnect" : "true",
135+
"inline" : [
136+
"sudo /etc/init.d/sshd stop",
137+
"nohup sudo shutdown -r now < /dev/null > /dev/null 2>&1 &",
138+
"exit 0"
139+
]
140+
},
141+
{
142+
"type" : "chef-solo",
143+
"pause_before": "2m",
118144
"json" : {
119145
"cfncluster" : {
120146
"nvidia" : {

‎amis/packer_ubuntu1404.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,12 @@
7878
}
7979
],
8080
"provisioners" : [
81+
{
82+
"type": "shell",
83+
"inline": [
84+
"while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done"
85+
]
86+
},
8187
{
8288
"type" : "shell",
8389
"inline" : [

‎amis/packer_ubuntu1604.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,12 @@
7878
}
7979
],
8080
"provisioners" : [
81+
{
82+
"type": "shell",
83+
"inline": [
84+
"while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done"
85+
]
86+
},
8187
{
8288
"type" : "shell",
8389
"inline" : [

‎amis/packer_variables.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
2-
"cfncluster_version": "1.5.4",
3-
"cfncluster_cookbook_version": "1.5.4",
2+
"cfncluster_version": "1.6.0",
3+
"cfncluster_cookbook_version": "1.6.0",
44
"chef_version": "14.2.0",
55
"ridley_version": "5.1.1",
66
"berkshelf_version": "7.0.4"

‎attributes/default.rb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
default['cfncluster']['scripts_dir'] = "#{node['cfncluster']['base_dir']}/scripts"
2020
default['cfncluster']['license_dir'] = "#{node['cfncluster']['base_dir']}/licenses"
2121
# Python packages
22-
default['cfncluster']['cfncluster-node-version'] = '1.5.4'
22+
default['cfncluster']['cfncluster-node-version'] = '1.6.0'
2323
default['cfncluster']['cfncluster-supervisor-version'] = '3.3.1'
2424
# URLs to software packages used during install receipes
2525
# Gridengine software

‎files/default/attachVolume.py

Lines changed: 66 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -5,52 +5,71 @@
55
import urllib2
66
import boto3
77
import time
8+
import ConfigParser
9+
from botocore.config import Config
810

9-
# Get volumeId
10-
try:
11-
volumeId = str(sys.argv[1])
12-
except IndexError:
13-
print "Provide an EBS volume ID to attach i.e. vol-cc789ea5"
14-
sys.exit(1)
15-
16-
# Get instance ID
17-
instanceId = urllib2.urlopen("http://169.254.169.254/latest/meta-data/instance-id").read()
18-
19-
# Get region
20-
region = urllib2.urlopen("http://169.254.169.254/latest/meta-data/placement/availability-zone").read()
21-
region = region[:-1]
22-
23-
# Generate a list of system paths minus the root path
24-
paths = [ device.path for device in parted.getAllDevices() ]
25-
26-
# List of possible block devices
27-
blockDevices = [ '/dev/xvdb', '/dev/xvdc', '/dev/xvdd', '/dev/xvde', '/dev/xvdf', '/dev/xvdg', '/dev/xvdh', '/dev/xvdi', '/dev/xvdj', '/dev/xvdk', '/dev/xvdl', '/dev/xvdm', '/dev/xvdn', '/dev/xvdo', '/dev/xvdp', '/dev/xvdq', '/dev/xvdr', '/dev/xvds', '/dev/xvdt', '/dev/xvdu', '/dev/xvdv', '/dev/xvdw', '/dev/xvdx', '/dev/xvdy', '/dev/xvdz' ]
28-
29-
# List of available block devices after removing currently used block devices
30-
availableDevices = [a for a in blockDevices if a not in paths]
31-
32-
# Connect to AWS using boto
33-
ec2 = boto3.client('ec2', region_name=region)
34-
35-
# Attach the volume
36-
dev = availableDevices[0].replace('xvd', 'sd')
37-
response = ec2.attach_volume(VolumeId=volumeId, InstanceId=instanceId, Device=dev)
38-
39-
# Poll for volume to attach
40-
state = response.get("State")
41-
42-
x = 0
43-
while state != "attached":
44-
if x == 36:
45-
print "Volume %s failed to mount in 180 seconds." % (volumeId)
46-
exit(1)
47-
if state in ["busy" or "detached"]:
48-
print "Volume %s in bad state %s" % (volumeId, state)
49-
exit(1)
50-
print "Volume %s in state %s ... waiting to be 'attached'" % (volumeId, state)
51-
time.sleep(5)
52-
x += 1
11+
12+
def main():
13+
# Get EBS volume Id
5314
try:
54-
state = ec2.describe_volumes(VolumeIds=[volumeId]).get('Volumes')[0].get('Attachments')[0].get('State')
55-
except IndexError as e:
56-
continue
15+
volumeId = str(sys.argv[1])
16+
except IndexError:
17+
print "Provide an EBS volume ID to attach i.e. vol-cc789ea5"
18+
sys.exit(1)
19+
20+
# Get instance ID
21+
instanceId = urllib2.urlopen("http://169.254.169.254/latest/meta-data/instance-id").read()
22+
23+
# Get region
24+
region = urllib2.urlopen("http://169.254.169.254/latest/meta-data/placement/availability-zone").read()
25+
region = region[:-1]
26+
27+
# Generate a list of system paths minus the root path
28+
paths = [device.path for device in parted.getAllDevices()]
29+
30+
# List of possible block devices
31+
blockDevices = ['/dev/xvdb', '/dev/xvdc', '/dev/xvdd', '/dev/xvde', '/dev/xvdf', '/dev/xvdg', '/dev/xvdh',
32+
'/dev/xvdi','/dev/xvdj', '/dev/xvdk', '/dev/xvdl', '/dev/xvdm', '/dev/xvdn', '/dev/xvdo',
33+
'/dev/xvdp', '/dev/xvdq', '/dev/xvdr', '/dev/xvds', '/dev/xvdt', '/dev/xvdu', '/dev/xvdv',
34+
'/dev/xvdw', '/dev/xvdx', '/dev/xvdy', '/dev/xvdz' ]
35+
36+
# List of available block devices after removing currently used block devices
37+
availableDevices = [a for a in blockDevices if a not in paths]
38+
39+
# Parse configuration file to read proxy settings
40+
config = ConfigParser.RawConfigParser()
41+
config.read('/etc/boto.cfg')
42+
proxy_config = Config()
43+
if config.has_option('Boto', 'proxy') and config.has_option('Boto', 'proxy_port'):
44+
proxy = config.get('Boto', 'proxy')
45+
proxy_port = config.get('Boto', 'proxy_port')
46+
proxy_config = Config(proxies={'https': "{}:{}".format(proxy, proxy_port)})
47+
48+
# Connect to AWS using boto
49+
ec2 = boto3.client('ec2', region_name=region, config=proxy_config)
50+
51+
# Attach the volume
52+
dev = availableDevices[0].replace('xvd', 'sd')
53+
response = ec2.attach_volume(VolumeId=volumeId, InstanceId=instanceId, Device=dev)
54+
55+
# Poll for volume to attach
56+
state = response.get("State")
57+
x = 0
58+
while state != "attached":
59+
if x == 36:
60+
print "Volume %s failed to mount in 180 seconds." % volumeId
61+
exit(1)
62+
if state in ["busy" or "detached"]:
63+
print "Volume %s in bad state %s" % (volumeId, state)
64+
exit(1)
65+
print "Volume %s in state %s ... waiting to be 'attached'" % (volumeId, state)
66+
time.sleep(5)
67+
x += 1
68+
try:
69+
state = ec2.describe_volumes(VolumeIds=[volumeId]).get('Volumes')[0].get('Attachments')[0].get('State')
70+
except IndexError as e:
71+
continue
72+
73+
74+
if __name__ == '__main__':
75+
main()

‎files/default/ec2_dev_2_volid.py

Lines changed: 60 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -6,48 +6,67 @@
66
import syslog
77
import time
88
import boto3
9+
import ConfigParser
10+
from botocore.config import Config
911

10-
# Get dev
11-
try:
12-
dev = str(sys.argv[1])
13-
except IndexError:
14-
syslog.syslog(syslog.LOG_ERR, "Provide block device i.e. xvdf")
15-
sys.exit(1)
16-
17-
# Convert dev to mapping format
18-
if 'nvme' in dev:
19-
# For newer instances which expose EBS volumes as NVMe devices, translate the
20-
# device name so boto can discover it.
21-
output = os.popen('sudo /usr/local/sbin/cfncluster-ebsnvme-id -v /dev/' + dev).read().split(":")[1].strip()
22-
print(output)
23-
sys.exit(0)
24-
else:
25-
dev = dev.replace('xvd', 'sd')
26-
dev = '/dev/' + dev
27-
28-
# Get instance ID
29-
instanceId = urllib2.urlopen("http://169.254.169.254/latest/meta-data/instance-id").read()
30-
31-
# Get region
32-
region = urllib2.urlopen("http://169.254.169.254/latest/meta-data/placement/availability-zone").read()
33-
region = region[:-1]
34-
35-
# Connect to AWS using boto
36-
ec2 = boto3.client('ec2', region_name=region)
37-
38-
# Poll for blockdevicemapping
39-
devices = ec2.describe_instance_attribute(InstanceId=instanceId, Attribute='blockDeviceMapping').get('BlockDeviceMappings')
40-
devmap = dict((d.get('DeviceName'), d) for d in devices)
41-
x = 0
42-
while not devmap.has_key(dev):
43-
if x == 36:
44-
syslog.syslog("Dev %s did not appears in 180 seconds." % dev)
45-
sys.exit(1)
46-
syslog.syslog("Looking for dev %s in devmap %s" % (dev, devmap))
47-
time.sleep(5)
12+
13+
def main():
14+
syslog.syslog("Starting ec2_dev_2_volid.py script")
15+
# Get dev
16+
try:
17+
dev = str(sys.argv[1])
18+
syslog.syslog("Input block device is %s" % dev)
19+
except IndexError:
20+
syslog.syslog(syslog.LOG_ERR, "Provide block device i.e. xvdf")
21+
22+
# Convert dev to mapping format
23+
if 'nvme' in dev:
24+
# For newer instances which expose EBS volumes as NVMe devices, translate the
25+
# device name so boto can discover it.
26+
output = os.popen('sudo /usr/local/sbin/cfncluster-ebsnvme-id -v /dev/' + dev).read().split(":")[1].strip()
27+
print(output)
28+
sys.exit(0)
29+
else:
30+
dev = dev.replace('xvd', 'sd')
31+
dev = '/dev/' + dev
32+
33+
# Get instance ID
34+
instanceId = urllib2.urlopen("http://169.254.169.254/latest/meta-data/instance-id").read()
35+
36+
# Get region
37+
region = urllib2.urlopen("http://169.254.169.254/latest/meta-data/placement/availability-zone").read()
38+
region = region[:-1]
39+
40+
# Parse configuration file to read proxy settings
41+
config = ConfigParser.RawConfigParser()
42+
config.read('/etc/boto.cfg')
43+
proxy_config = Config()
44+
if config.has_option('Boto', 'proxy') and config.has_option('Boto', 'proxy_port'):
45+
proxy = config.get('Boto', 'proxy')
46+
proxy_port = config.get('Boto', 'proxy_port')
47+
proxy_config = Config(proxies={'https': "{}:{}".format(proxy, proxy_port)})
48+
49+
# Connect to AWS using boto
50+
ec2 = boto3.client('ec2', region_name=region, config=proxy_config)
51+
52+
# Poll for blockdevicemapping
4853
devices = ec2.describe_instance_attribute(InstanceId=instanceId, Attribute='blockDeviceMapping').get('BlockDeviceMappings')
4954
devmap = dict((d.get('DeviceName'), d) for d in devices)
50-
x += 1
55+
x = 0
56+
while not devmap.has_key(dev):
57+
if x == 36:
58+
syslog.syslog("Dev %s did not appears in 180 seconds." % dev)
59+
sys.exit(1)
60+
syslog.syslog("Looking for dev %s in devmap %s" % (dev, devmap))
61+
time.sleep(5)
62+
devices = ec2.describe_instance_attribute(InstanceId=instanceId, Attribute='blockDeviceMapping').get('BlockDeviceMappings')
63+
devmap = dict((d.get('DeviceName'), d) for d in devices)
64+
x += 1
65+
66+
# Return volumeId
67+
volumeId = devmap.get(dev).get('Ebs').get('VolumeId')
68+
print(volumeId)
69+
5170

52-
volumeId = devmap.get(dev).get('Ebs').get('VolumeId')
53-
print(volumeId)
71+
if __name__ == '__main__':
72+
main()

0 commit comments

Comments
 (0)