From 9e03506e9c8f3e4c5ddbd20287f84c643c2c5740 Mon Sep 17 00:00:00 2001 From: ericcalabretta Date: Fri, 15 May 2020 12:16:31 -0500 Subject: [PATCH 1/2] lots of cleanup & add roll-back demo --- docs/rollback_demo.md | 41 ++++++++++++++++++++++++ habitat/hooks/health-check | 30 +++++++++++++++++ habitat/hooks/init | 4 +++ habitat/hooks/run | 5 +-- habitat/plan.sh | 2 +- terraform/aws/national-parks.tf | 20 ++++++------ terraform/aws/templates.tf | 3 ++ terraform/aws/variables.tf | 23 +++++++++++++ terraform/azure/main.tf | 2 +- terraform/azure/templates.tf | 1 + terraform/azure/variables.tf | 7 ++++ terraform/chef-automate/aws/variables.tf | 3 +- terraform/templates/hab-sup.service | 4 +++ terraform/templates/peer-sup.service | 2 ++ 14 files changed, 131 insertions(+), 16 deletions(-) create mode 100644 docs/rollback_demo.md create mode 100644 habitat/hooks/health-check diff --git a/docs/rollback_demo.md b/docs/rollback_demo.md new file mode 100644 index 0000000..e006f8f --- /dev/null +++ b/docs/rollback_demo.md @@ -0,0 +1,41 @@ +# Rollback demo using National-Parks with VMs & Terraform + +With the new `--update_condition track-channel` option we can now easily demo roll-back with Habitat. + +The demo is set to configure services with the `track-channel` configuration. This gives the "rollback" behavior + +``` +variable "update_condition" { + default = "track-channel" +} +``` + +First we need to create a bad `.hart` so we can show a failure in Automate. The health-check uses curl to check our health. We want to create a `404` error so the failure registers as critcial in Chef Automate. + +You can comment out line the code in the `init` hook that copies the National-Parks `.war` file into the correct directory. It's line 11 at the time of this creation of this readme. + +``` +#NOTE comment the next line out to create/demo a bad package +cp {{pkg.path}}/*.war {{pkg.svc_var_path}}/tc/webapps +``` +Change the major package version number so you can distinguish good vs bad packages. + +Update `pkg_version=8.0.0` in your plan.sh. + +You can also create a different channel for the roll-back demo so you don't have a bad pacakge in `current` + +Upload your .hart & create the `rb_prod` channel + +`hab pkg upload eric-national-parks-8.0.0-20200509192449-x86_64-linux.hart rb_prod` + +To run the demo start with a working .hart, then promote the bad .hart +`hab pkg promote eric/national-parks/8.0.0/20200509192449 rb_prod` + +The national-park service will upgrade & you can show the failure in the Automate Applications tab. Once you click on the failed service the right pannel will display the error log. + +"Rollback" your change by demoting the bad package. The national-parks service will downgrade back to the previous version. You can show the new package number & the systems are now healthy in Chef Automate's Application tab. + +`hab pkg demote eric/national-parks/8.0.0/20200509192449 rb_prod` + +Included in the repo is Terraform code (Version `0.12`) for launching the application in AWS and Google Kubernetes Engine. Provision either AWS, GKE, or both, and then you can watch Habitat update across cloud deployments. + diff --git a/habitat/hooks/health-check b/habitat/hooks/health-check new file mode 100644 index 0000000..f7a504c --- /dev/null +++ b/habitat/hooks/health-check @@ -0,0 +1,30 @@ +#!/bin/sh + +#define default return code as 0 +rc=3 + +#curl local host - 200 code is "ok" +#variable cfg.server.port so port can be dynamic in health check +STATUS=$({{pkgPathFor "core/curl"}}/bin/curl -s -o /dev/null -w '%{http_code}' http://localhost:8080/national-parks/) + +echo "HTTP Status Code: $STATUS" + +case $STATUS in + 200) + rc=0 ;; + 1) + rc=1 ;; + 4**) + rc=2 ;; + *) + rc=3 ;; +esac + +# hab error codes +# 0- ok +# 1- warning +# 2- critical +# 3- unknown + +echo "healthcheck return code: $rc" +exit $rc \ No newline at end of file diff --git a/habitat/hooks/init b/habitat/hooks/init index 50c2169..d32053f 100644 --- a/habitat/hooks/init +++ b/habitat/hooks/init @@ -1,9 +1,13 @@ #!/bin/bash -x +echo "clean up after a previous pacakge" +rm -rf {{pkg.svc_var_path}}/tc + echo "Preparing TOMCAT_HOME..." # Move directories that ship in the package into place cp -a {{pkgPathFor "core/tomcat8"}}/tc {{pkg.svc_var_path}}/ +#NOTE comment the next line out to create/demo a bad package cp {{pkg.path}}/*.war {{pkg.svc_var_path}}/tc/webapps # Following lines are to copy project tomcat configs to run time tomcat8 directory diff --git a/habitat/hooks/run b/habitat/hooks/run index df08893..18738cf 100644 --- a/habitat/hooks/run +++ b/habitat/hooks/run @@ -1,4 +1,5 @@ #!/bin/bash +exec 2>&1 {{#if bind.database ~}} export CATALINA_OPTS="-DMONGODB_SERVICE_HOST={{bind.database.first.sys.ip}} @@ -7,6 +8,6 @@ export CATALINA_OPTS="-DMONGODB_SERVICE_HOST={{bind.database.first.sys.ip}} {{/if ~}} export JAVA_HOME="{{pkgPathFor "core/corretto"}}" -export TOMCAT_HOME="{{pkgPathFor "core/tomcat8"}}/tc" -cp {{pkg.path}}/*.war $TOMCAT_HOME/webapps +export TOMCAT_HOME="{{pkg.svc_var_path}}/tc" + exec ${TOMCAT_HOME}/bin/catalina.sh run \ No newline at end of file diff --git a/habitat/plan.sh b/habitat/plan.sh index 116e377..b9b88e2 100644 --- a/habitat/plan.sh +++ b/habitat/plan.sh @@ -4,7 +4,7 @@ pkg_origin=scottford pkg_version=7.0.0 pkg_maintainer="Scott Ford " pkg_license=('Apache-2.0') -pkg_deps=(core/tomcat8 core/corretto core/mongo-tools) +pkg_deps=(core/tomcat8 core/corretto core/mongo-tools core/curl) pkg_build_deps=(core/corretto core/maven) pkg_svc_user="root" pkg_binds=( diff --git a/terraform/aws/national-parks.tf b/terraform/aws/national-parks.tf index c736390..4eacc7c 100644 --- a/terraform/aws/national-parks.tf +++ b/terraform/aws/national-parks.tf @@ -67,8 +67,8 @@ resource "aws_instance" "permanent_peer" { "sudo /sbin/sysctl -w net.ipv4.conf.all.accept_redirects=0", "sudo cp /home/${var.aws_ami_user}/audit_linux.toml /hab/user/audit-baseline/config/user.toml", "sudo cp /home/${var.aws_ami_user}/config_linux.toml /hab/user/config-baseline/config/user.toml", - "sudo hab svc load effortless/config-baseline --group ${var.group} --strategy at-once --channel stable", - "sudo hab svc load effortless/audit-baseline --group ${var.group} --strategy at-once --channel stable", + "sudo hab svc load ${var.effortless_origin}/${var.effortless_infra} --group ${var.group} --strategy at-once --channel stable", + "sudo hab svc load ${var.effortless_origin}/${var.effortless_inspec} --group ${var.group} --strategy at-once --channel stable", ] } } @@ -152,8 +152,8 @@ resource "aws_instance" "mongodb" { "sudo cp /home/${var.aws_ami_user}/audit_linux.toml /hab/user/audit-baseline/config/user.toml", "sudo cp /home/${var.aws_ami_user}/config_linux.toml /hab/user/config-baseline/config/user.toml", "sudo cp /home/${var.aws_ami_user}/mongo.toml /hab/user/mongodb/config/user.toml", - "sudo hab svc load effortless/config-baseline --group ${var.group} --strategy at-once --channel stable", - "sudo hab svc load effortless/audit-baseline --group ${var.group} --strategy at-once --channel stable", + "sudo hab svc load ${var.effortless_origin}/${var.effortless_infra} --group ${var.group} --strategy at-once --channel stable", + "sudo hab svc load ${var.effortless_origin}/${var.effortless_inspec} --group ${var.group} --strategy at-once --channel stable", "sudo hab svc load core/mongodb/3.2.10/20171016003652 --group ${var.group}", ] } @@ -210,7 +210,7 @@ resource "aws_instance" "national_parks" { inline = [ "sudo rm -rf /etc/machine-id", "sudo systemd-machine-id-setup", - "sudo hostname national-parks-${var.node_count}", + "sudo hostname national-parks-${count.index}", "sudo groupadd hab", "sudo adduser hab -g hab", "chmod +x /tmp/install_hab.sh", @@ -230,9 +230,9 @@ resource "aws_instance" "national_parks" { "sudo /sbin/sysctl -w net.ipv4.conf.all.accept_redirects=0", "sudo cp /home/${var.aws_ami_user}/audit_linux.toml /hab/user/audit-baseline/config/user.toml", "sudo cp /home/${var.aws_ami_user}/config_linux.toml /hab/user/config-baseline/config/user.toml", - "sudo hab svc load effortless/config-baseline --group ${var.group} --strategy at-once --channel stable", - "sudo hab svc load effortless/audit-baseline --group ${var.group} --strategy at-once --channel stable", - "sudo hab svc load ${var.origin}/national-parks --group ${var.group} --channel ${var.prod_channel} --strategy ${var.update_strategy} --update-condition ${var.update_condition} --bind database:mongodb.${var.group}", + "sudo hab svc load ${var.effortless_origin}/${var.effortless_infra} --group ${var.group} --strategy at-once --channel stable", + "sudo hab svc load ${var.effortless_origin}/${var.effortless_inspec} --group ${var.group} --strategy at-once --channel stable", + "sudo hab svc load ${var.origin}/national-parks --group ${var.group} --channel ${var.prod_channel} --strategy ${var.update_strategy} --update-condition ${var.update_condition} --health-check-interval ${var.health_check_interval} --bind database:mongodb.${var.group}", ] } } @@ -312,8 +312,8 @@ resource "aws_instance" "haproxy" { "sudo cp /home/${var.aws_ami_user}/audit_linux.toml /hab/user/audit-baseline/config/user.toml", "sudo cp /home/${var.aws_ami_user}/config_linux.toml /hab/user/config-baseline/config/user.toml", "sudo cp /home/${var.aws_ami_user}/haproxy.toml /hab/user/haproxy/config/user.toml", - "sudo hab svc load effortless/config-baseline --group ${var.group} --strategy at-once --channel stable", - "sudo hab svc load effortless/audit-baseline --group ${var.group} --strategy at-once --channel stable", + "sudo hab svc load ${var.effortless_origin}/${var.effortless_infra} --group ${var.group} --strategy at-once --channel stable", + "sudo hab svc load ${var.effortless_origin}/${var.effortless_inspec} --group ${var.group} --strategy at-once --channel stable", "sudo hab svc load core/haproxy --group ${var.group} --bind backend:national-parks.${var.group}", ] } diff --git a/terraform/aws/templates.tf b/terraform/aws/templates.tf index 1f8faac..4963086 100644 --- a/terraform/aws/templates.tf +++ b/terraform/aws/templates.tf @@ -5,6 +5,7 @@ data "template_file" "permanent_peer" { template = file("${path.module}/../templates/peer-sup.service") vars = { + HAB_AUTH_TOKEN ="${var.HAB_AUTH_TOKEN}" flags = "--auto-update --listen-gossip 0.0.0.0:9638 --listen-http 0.0.0.0:9631 --permanent-peer --event-stream-application=${var.event-stream-application} --event-stream-environment=${var.event-stream-environment} --event-stream-site=${var.aws_region} --event-stream-url=${var.automate_ip}:4222 --event-stream-token=${var.automate_token}" } } @@ -13,6 +14,8 @@ data "template_file" "sup_service" { template = file("${path.module}/../templates/hab-sup.service") vars = { + HAB_AUTH_TOKEN ="${var.HAB_AUTH_TOKEN}" + UPDATE_STRATEGY_FREQUENCY_MS ="${var.UPDATE_STRATEGY_FREQUENCY_MS}" flags = "--auto-update --peer ${aws_instance.permanent_peer.private_ip} --listen-gossip 0.0.0.0:9638 --listen-http 0.0.0.0:9631 --event-stream-application=${var.event-stream-application} --event-stream-environment=${var.event-stream-environment} --event-stream-site=${var.aws_region} --event-stream-url=${var.automate_ip}:4222 --event-stream-token=${var.automate_token}" } } diff --git a/terraform/aws/variables.tf b/terraform/aws/variables.tf index d7346af..6d8cf91 100644 --- a/terraform/aws/variables.tf +++ b/terraform/aws/variables.tf @@ -100,6 +100,18 @@ variable "node_count" { variable "origin" { } +variable "effortless_origin" { + default = "effortless" +} + +variable "effortless_infra" { + default = "config-baseline" +} + +variable "effortless_inspec" { + default = "audit-baseline" +} + variable "group" { default = "default" } @@ -112,9 +124,20 @@ variable "update_condition" { default = "track-channel" } +variable "health_check_interval" { + default = "30" +} + variable "sleep" { default = "60" } +variable "UPDATE_STRATEGY_FREQUENCY_MS" { + default = "60000" +} + +variable "HAB_AUTH_TOKEN" { + default = "blankkey" +} //////////////////////////////// // Automate Info diff --git a/terraform/azure/main.tf b/terraform/azure/main.tf index 22af174..6f051c3 100644 --- a/terraform/azure/main.tf +++ b/terraform/azure/main.tf @@ -672,7 +672,7 @@ resource "azurerm_virtual_machine" "app" { "sudo cp /home/${var.azure_image_user}/config_linux.toml /hab/user/config-baseline/config/user.toml", "sudo hab svc load effortless/audit-baseline --channel stable --strategy at-once --group ${var.group}", "sudo hab svc load effortless/config-baseline --channel stable --strategy at-once --group ${var.group}", - "sudo hab svc load ${var.origin}/national-parks --group ${var.group} --channel ${var.channel} --strategy ${var.update_strategy} --update-condition ${var.update_condition} --bind database:mongodb.${var.group}", + "sudo hab svc load ${var.origin}/national-parks --group ${var.group} --channel ${var.channel} --strategy ${var.update_strategy} --update-condition ${var.update_condition} --health-check-interval ${var.health_check_interval} --bind database:mongodb.${var.group}", ] } diff --git a/terraform/azure/templates.tf b/terraform/azure/templates.tf index 53a0d20..36d3e74 100644 --- a/terraform/azure/templates.tf +++ b/terraform/azure/templates.tf @@ -13,6 +13,7 @@ data "template_file" "sup_service" { template = file("${path.module}/../templates/hab-sup.service") vars = { + UPDATE_STRATEGY_FREQUENCY_MS ="${var.UPDATE_STRATEGY_FREQUENCY_MS}" flags = "--auto-update --peer ${azurerm_public_ip.permanent-peer-pip.ip_address} --listen-gossip 0.0.0.0:9638 --listen-http 0.0.0.0:9631 --event-stream-application=${var.event-stream-application} --event-stream-environment=${var.event-stream-environment} --event-stream-site=${var.azure_region} --event-stream-url=${var.automate_ip}:4222 --event-stream-token=${var.automate_token}" } } diff --git a/terraform/azure/variables.tf b/terraform/azure/variables.tf index 1fa2ad4..afe09ba 100644 --- a/terraform/azure/variables.tf +++ b/terraform/azure/variables.tf @@ -48,6 +48,13 @@ variable "update_condition" { default = "track-channel" } +variable "health_check_interval" { + default = "30" +} +variable "UPDATE_STRATEGY_FREQUENCY_MS" { + default = "60000" +} + //////////////////////////////// // Tags diff --git a/terraform/chef-automate/aws/variables.tf b/terraform/chef-automate/aws/variables.tf index 67d841e..e2da290 100644 --- a/terraform/chef-automate/aws/variables.tf +++ b/terraform/chef-automate/aws/variables.tf @@ -136,8 +136,7 @@ variable "automate_password" { } variable "automate_token" { - default = "x9GfO534d4mpctuQlKq4TA6SpDE=" - description = "default token please change" + description = "no default token" } variable "automate_products" { diff --git a/terraform/templates/hab-sup.service b/terraform/templates/hab-sup.service index 65ce94a..beed98e 100644 --- a/terraform/templates/hab-sup.service +++ b/terraform/templates/hab-sup.service @@ -2,6 +2,10 @@ Description=Habitat Supervisor [Service] + +Environment=HAB_AUTH_TOKEN=${HAB_AUTH_TOKEN} +Environment=HAB_UPDATE_STRATEGY_FREQUENCY_BYPASS_CHECK=1 +Environment=HAB_UPDATE_STRATEGY_FREQUENCY_MS=${UPDATE_STRATEGY_FREQUENCY_MS} ExecStartPre=/bin/bash -c /bin/systemctl ExecStart=/bin/hab run ${flags} diff --git a/terraform/templates/peer-sup.service b/terraform/templates/peer-sup.service index 65ce94a..8ba13cf 100644 --- a/terraform/templates/peer-sup.service +++ b/terraform/templates/peer-sup.service @@ -2,6 +2,8 @@ Description=Habitat Supervisor [Service] + +Environment=HAB_AUTH_TOKEN=${HAB_AUTH_TOKEN} ExecStartPre=/bin/bash -c /bin/systemctl ExecStart=/bin/hab run ${flags} From 550be6f9d0b6131394725b8678b2915bb3952d6c Mon Sep 17 00:00:00 2001 From: ericcalabretta Date: Mon, 1 Jun 2020 18:50:32 -0500 Subject: [PATCH 2/2] add package variables --- terraform/aws/national-parks.tf | 16 ++++++++-------- terraform/aws/variables.tf | 8 ++++++++ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/terraform/aws/national-parks.tf b/terraform/aws/national-parks.tf index 4eacc7c..d117ee8 100644 --- a/terraform/aws/national-parks.tf +++ b/terraform/aws/national-parks.tf @@ -67,8 +67,8 @@ resource "aws_instance" "permanent_peer" { "sudo /sbin/sysctl -w net.ipv4.conf.all.accept_redirects=0", "sudo cp /home/${var.aws_ami_user}/audit_linux.toml /hab/user/audit-baseline/config/user.toml", "sudo cp /home/${var.aws_ami_user}/config_linux.toml /hab/user/config-baseline/config/user.toml", - "sudo hab svc load ${var.effortless_origin}/${var.effortless_infra} --group ${var.group} --strategy at-once --channel stable", - "sudo hab svc load ${var.effortless_origin}/${var.effortless_inspec} --group ${var.group} --strategy at-once --channel stable", + "sudo hab svc load ${var.effortless_origin}/${var.effortless_infra} --group ${var.group} --strategy at-once --channel ${var.effortless_infra_channel}", + "sudo hab svc load ${var.effortless_origin}/${var.effortless_inspec} --group ${var.group} --strategy at-once --channel ${var.effortless_inspec_channel}", ] } } @@ -152,8 +152,8 @@ resource "aws_instance" "mongodb" { "sudo cp /home/${var.aws_ami_user}/audit_linux.toml /hab/user/audit-baseline/config/user.toml", "sudo cp /home/${var.aws_ami_user}/config_linux.toml /hab/user/config-baseline/config/user.toml", "sudo cp /home/${var.aws_ami_user}/mongo.toml /hab/user/mongodb/config/user.toml", - "sudo hab svc load ${var.effortless_origin}/${var.effortless_infra} --group ${var.group} --strategy at-once --channel stable", - "sudo hab svc load ${var.effortless_origin}/${var.effortless_inspec} --group ${var.group} --strategy at-once --channel stable", + "sudo hab svc load ${var.effortless_origin}/${var.effortless_infra} --group ${var.group} --strategy at-once --channel ${var.effortless_infra_channel}", + "sudo hab svc load ${var.effortless_origin}/${var.effortless_inspec} --group ${var.group} --strategy at-once --channel ${var.effortless_inspec_channel}", "sudo hab svc load core/mongodb/3.2.10/20171016003652 --group ${var.group}", ] } @@ -230,8 +230,8 @@ resource "aws_instance" "national_parks" { "sudo /sbin/sysctl -w net.ipv4.conf.all.accept_redirects=0", "sudo cp /home/${var.aws_ami_user}/audit_linux.toml /hab/user/audit-baseline/config/user.toml", "sudo cp /home/${var.aws_ami_user}/config_linux.toml /hab/user/config-baseline/config/user.toml", - "sudo hab svc load ${var.effortless_origin}/${var.effortless_infra} --group ${var.group} --strategy at-once --channel stable", - "sudo hab svc load ${var.effortless_origin}/${var.effortless_inspec} --group ${var.group} --strategy at-once --channel stable", + "sudo hab svc load ${var.effortless_origin}/${var.effortless_infra} --group ${var.group} --strategy at-once --channel ${var.effortless_infra_channel}", + "sudo hab svc load ${var.effortless_origin}/${var.effortless_inspec} --group ${var.group} --strategy at-once --channel ${var.effortless_inspec_channel}", "sudo hab svc load ${var.origin}/national-parks --group ${var.group} --channel ${var.prod_channel} --strategy ${var.update_strategy} --update-condition ${var.update_condition} --health-check-interval ${var.health_check_interval} --bind database:mongodb.${var.group}", ] } @@ -312,8 +312,8 @@ resource "aws_instance" "haproxy" { "sudo cp /home/${var.aws_ami_user}/audit_linux.toml /hab/user/audit-baseline/config/user.toml", "sudo cp /home/${var.aws_ami_user}/config_linux.toml /hab/user/config-baseline/config/user.toml", "sudo cp /home/${var.aws_ami_user}/haproxy.toml /hab/user/haproxy/config/user.toml", - "sudo hab svc load ${var.effortless_origin}/${var.effortless_infra} --group ${var.group} --strategy at-once --channel stable", - "sudo hab svc load ${var.effortless_origin}/${var.effortless_inspec} --group ${var.group} --strategy at-once --channel stable", + "sudo hab svc load ${var.effortless_origin}/${var.effortless_infra} --group ${var.group} --strategy at-once --channel ${var.effortless_infra_channel}", + "sudo hab svc load ${var.effortless_origin}/${var.effortless_inspec} --group ${var.group} --strategy at-once --channel ${var.effortless_inspec_channel}", "sudo hab svc load core/haproxy --group ${var.group} --bind backend:national-parks.${var.group}", ] } diff --git a/terraform/aws/variables.tf b/terraform/aws/variables.tf index 6d8cf91..5a60c17 100644 --- a/terraform/aws/variables.tf +++ b/terraform/aws/variables.tf @@ -112,6 +112,14 @@ variable "effortless_inspec" { default = "audit-baseline" } +variable "effortless_infra_channel" { + default = "stable" +} + +variable "effortless_inspec_channel" { + default = "stable" +} + variable "group" { default = "default" }