From dd9e1f7c593a69fccabe154e9f2ca9390123e87d Mon Sep 17 00:00:00 2001 From: Ian Rose Date: Tue, 30 May 2023 09:35:40 -0700 Subject: [PATCH 1/4] Remove public key from terraform config --- docs/snowflake.md | 3 --- terraform/snowflake/environments/dev/main.tf | 20 +------------------ .../environments/dev/terraform.tfvars | 7 ++----- terraform/snowflake/environments/prd/main.tf | 20 +------------------ .../environments/prd/terraform.tfvars | 7 ++----- terraform/snowflake/modules/elt/users.tf | 4 ---- terraform/snowflake/modules/elt/variables.tf | 15 -------------- 7 files changed, 6 insertions(+), 70 deletions(-) diff --git a/docs/snowflake.md b/docs/snowflake.md index 8bf56f3c..bf6f7ee1 100644 --- a/docs/snowflake.md +++ b/docs/snowflake.md @@ -248,10 +248,7 @@ The **elt** module has the following configuration: | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [airflow\_public\_key](#input\_airflow\_public\_key) | Public key for Airflow service user | `string` | n/a | yes | -| [dbt\_public\_key](#input\_dbt\_public\_key) | Public key for dbt Cloud service user | `string` | n/a | yes | | [environment](#input\_environment) | Environment suffix | `string` | n/a | yes | -| [github\_ci\_public\_key](#input\_github\_ci\_public\_key) | Public key for GitHub CI service user | `string` | n/a | yes | ## Outputs diff --git a/terraform/snowflake/environments/dev/main.tf b/terraform/snowflake/environments/dev/main.tf index 29425be6..fe4c58eb 100644 --- a/terraform/snowflake/environments/dev/main.tf +++ b/terraform/snowflake/environments/dev/main.tf @@ -12,21 +12,6 @@ variable "locator" { type = string } -variable "airflow_public_key" { - description = "Public key for Airflow service user" - type = string -} - -variable "dbt_public_key" { - description = "Public key for dbt Cloud service user" - type = string -} - -variable "github_ci_public_key" { - description = "Public key for GitHub CI service user" - type = string -} - ############################ # Providers # ############################ @@ -86,8 +71,5 @@ module "elt" { snowflake.useradmin = snowflake.useradmin, } - environment = var.environment - airflow_public_key = var.airflow_public_key - dbt_public_key = var.dbt_public_key - github_ci_public_key = var.github_ci_public_key + environment = var.environment } diff --git a/terraform/snowflake/environments/dev/terraform.tfvars b/terraform/snowflake/environments/dev/terraform.tfvars index 44aaa6e4..433f69d1 100644 --- a/terraform/snowflake/environments/dev/terraform.tfvars +++ b/terraform/snowflake/environments/dev/terraform.tfvars @@ -1,5 +1,2 @@ -airflow_public_key = "" -dbt_public_key = "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxGn4yPVeOTBHFDCEf6idprUOLUyR12FICA8UAOtLzYDIqJdSHcQUhrHqqXtPn0Zp8YJbfSbUadNmP5van3F8Q0DcuY+SWOd0MeeSJYkoaib1YTARzLidVn3HSSiQofuSTw60lvc8POMH9Km9q2wLiVmOaGSSbgXBk3K22jb1J2QVoJeOT0awJRgZTAix9TOQEFiUmXZEBe23rPzP86yoERr0JCDlDYjB17S83FxF+gZdpv92Mjbi5s5SBXSPHwIPKUN6qOEAmL5fRheSD+J3TNPmZw8H6w4kYJlSxAQUflumhj7M7eeWwCqnB+OakaBxOVjbe3x80JaVZXPUTnFg0QIDAQAB" -github_ci_public_key = "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAj9d6NsUrmluQL87jpksnMTc+lVvbMBIjemtMYvnxlBYW+TyQhmwZDKD4ety05LBb7VPXDs0bovTtIgTBRmG7wmD11egsWKigglH4qgNF0FQbfJZ5zEx5kUtp3DBL/CMsa87Pz1FrDIISxvdcfkCIi05M3p1iseqW1nFvogfLM6fO2eVzXj7n/9thDRtVx2NJAKrbe1D2ePwUuZ71RT/C5pjoNdPHa/KqVwau9PPd+Ce6+nvnw6dVYG4PoJzud8R0FGk0W77AHlXeyRb4MiaukuCjSN+aFbolBY41lViP3X8daESzF+VOB5nv4q93T0HyUPu8TbUeJl1bX5/AOtQn6QIDAQAB" -locator = "heb41095" -environment = "DEV" +locator = "heb41095" +environment = "DEV" diff --git a/terraform/snowflake/environments/prd/main.tf b/terraform/snowflake/environments/prd/main.tf index 29425be6..fe4c58eb 100644 --- a/terraform/snowflake/environments/prd/main.tf +++ b/terraform/snowflake/environments/prd/main.tf @@ -12,21 +12,6 @@ variable "locator" { type = string } -variable "airflow_public_key" { - description = "Public key for Airflow service user" - type = string -} - -variable "dbt_public_key" { - description = "Public key for dbt Cloud service user" - type = string -} - -variable "github_ci_public_key" { - description = "Public key for GitHub CI service user" - type = string -} - ############################ # Providers # ############################ @@ -86,8 +71,5 @@ module "elt" { snowflake.useradmin = snowflake.useradmin, } - environment = var.environment - airflow_public_key = var.airflow_public_key - dbt_public_key = var.dbt_public_key - github_ci_public_key = var.github_ci_public_key + environment = var.environment } diff --git a/terraform/snowflake/environments/prd/terraform.tfvars b/terraform/snowflake/environments/prd/terraform.tfvars index fb6e9a9e..153b6ed7 100644 --- a/terraform/snowflake/environments/prd/terraform.tfvars +++ b/terraform/snowflake/environments/prd/terraform.tfvars @@ -1,5 +1,2 @@ -airflow_public_key = "" -dbt_public_key = "" -github_ci_public_key = "" -locator = "heb41095" -environment = "PRD" +locator = "heb41095" +environment = "PRD" diff --git a/terraform/snowflake/modules/elt/users.tf b/terraform/snowflake/modules/elt/users.tf index 9b8247ba..a419de29 100644 --- a/terraform/snowflake/modules/elt/users.tf +++ b/terraform/snowflake/modules/elt/users.tf @@ -11,7 +11,6 @@ resource "snowflake_user" "dbt" { default_role = snowflake_role.transformer.name must_change_password = false - rsa_public_key = var.dbt_public_key } @@ -24,8 +23,6 @@ resource "snowflake_user" "airflow" { default_role = snowflake_role.loader.name must_change_password = false - rsa_public_key = var.airflow_public_key - } resource "snowflake_user" "github_ci" { @@ -37,7 +34,6 @@ resource "snowflake_user" "github_ci" { default_role = snowflake_role.reader.name must_change_password = false - rsa_public_key = var.github_ci_public_key } ###################################### diff --git a/terraform/snowflake/modules/elt/variables.tf b/terraform/snowflake/modules/elt/variables.tf index 102e6d22..6ccbd061 100644 --- a/terraform/snowflake/modules/elt/variables.tf +++ b/terraform/snowflake/modules/elt/variables.tf @@ -2,18 +2,3 @@ variable "environment" { description = "Environment suffix" type = string } - -variable "airflow_public_key" { - description = "Public key for Airflow service user" - type = string -} - -variable "dbt_public_key" { - description = "Public key for dbt Cloud service user" - type = string -} - -variable "github_ci_public_key" { - description = "Public key for GitHub CI service user" - type = string -} From 4defb251b303dbb2ab5eba47a380d4ae21556307 Mon Sep 17 00:00:00 2001 From: Ian Rose Date: Tue, 30 May 2023 12:47:37 -0700 Subject: [PATCH 2/4] Update docs --- docs/snowflake.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/snowflake.md b/docs/snowflake.md index bf6f7ee1..8416305f 100644 --- a/docs/snowflake.md +++ b/docs/snowflake.md @@ -155,10 +155,12 @@ The following are steps for creating a new service account with key pair authent Most of the time, you should create a key pair with encryption enabled for the private key. 1. Add the private key to the CalData 1Password vault, along with the intended service account user name and passphrase (if applicable) 1. Create a new user in the Snowflake Terraform configuration (`users.tf`) and assign it the appropriate functional role. - The public key of the key pair should be attached to the user using the property `rsa_public_key`. + Once the user is created, add its public key in the Snowflake UI: + ```sql + ALTER USER SET RSA_PUBLIC_KEY='MII...' + ``` Note that we need to remove the header and trailer (i.e. `-- BEGIN PUBLIC KEY --`) as well as any line breaks in order for Snowflake to accept the public key as valid. - It is okay for this public key to be in version control. 1. Add the *private* key for the user to whatever system needs to access Snowflake. Service accounts should not be shared across different applications, From 045ce19fb1bae7db4e0f3a4bc5c1edf3e149ff24 Mon Sep 17 00:00:00 2001 From: Ian Rose Date: Wed, 31 May 2023 08:57:36 -0700 Subject: [PATCH 3/4] Use more canonical form of warehouse size. --- terraform/snowflake/modules/elt/warehouses.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/snowflake/modules/elt/warehouses.tf b/terraform/snowflake/modules/elt/warehouses.tf index caa8be15..a4d464b9 100644 --- a/terraform/snowflake/modules/elt/warehouses.tf +++ b/terraform/snowflake/modules/elt/warehouses.tf @@ -4,9 +4,9 @@ locals { sizes = { - "XS" = "x-small", - "XL" = "x-large", - "4XL" = "4x-large", + "XS" = "X-SMALL", + "XL" = "X-LARGE", + "4XL" = "4X-LARGE", } } From 4104836722dda9ef9ed41a9f597255b8a17c77b7 Mon Sep 17 00:00:00 2001 From: Ian Rose Date: Wed, 31 May 2023 09:23:40 -0700 Subject: [PATCH 4/4] Add fivetran user. --- docs/snowflake.md | 2 ++ terraform/snowflake/modules/elt/users.tf | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/docs/snowflake.md b/docs/snowflake.md index 8416305f..bb57baac 100644 --- a/docs/snowflake.md +++ b/docs/snowflake.md @@ -227,6 +227,7 @@ The **elt** module has the following configuration: | [snowflake_role_grants.analytics_r_to_reporter](https://registry.terraform.io/providers/Snowflake-Labs/snowflake/latest/docs/resources/role_grants) | resource | | [snowflake_role_grants.analytics_rwc_to_transformer](https://registry.terraform.io/providers/Snowflake-Labs/snowflake/latest/docs/resources/role_grants) | resource | | [snowflake_role_grants.loader_to_airflow](https://registry.terraform.io/providers/Snowflake-Labs/snowflake/latest/docs/resources/role_grants) | resource | +| [snowflake_role_grants.loader_to_fivetran](https://registry.terraform.io/providers/Snowflake-Labs/snowflake/latest/docs/resources/role_grants) | resource | | [snowflake_role_grants.loader_to_sysadmin](https://registry.terraform.io/providers/Snowflake-Labs/snowflake/latest/docs/resources/role_grants) | resource | | [snowflake_role_grants.loading_to_loader](https://registry.terraform.io/providers/Snowflake-Labs/snowflake/latest/docs/resources/role_grants) | resource | | [snowflake_role_grants.raw_r_to_reader](https://registry.terraform.io/providers/Snowflake-Labs/snowflake/latest/docs/resources/role_grants) | resource | @@ -244,6 +245,7 @@ The **elt** module has the following configuration: | [snowflake_role_grants.transforming_to_transformer](https://registry.terraform.io/providers/Snowflake-Labs/snowflake/latest/docs/resources/role_grants) | resource | | [snowflake_user.airflow](https://registry.terraform.io/providers/Snowflake-Labs/snowflake/latest/docs/resources/user) | resource | | [snowflake_user.dbt](https://registry.terraform.io/providers/Snowflake-Labs/snowflake/latest/docs/resources/user) | resource | +| [snowflake_user.fivetran](https://registry.terraform.io/providers/Snowflake-Labs/snowflake/latest/docs/resources/user) | resource | | [snowflake_user.github_ci](https://registry.terraform.io/providers/Snowflake-Labs/snowflake/latest/docs/resources/user) | resource | ## Inputs diff --git a/terraform/snowflake/modules/elt/users.tf b/terraform/snowflake/modules/elt/users.tf index a419de29..545e5e69 100644 --- a/terraform/snowflake/modules/elt/users.tf +++ b/terraform/snowflake/modules/elt/users.tf @@ -25,6 +25,17 @@ resource "snowflake_user" "airflow" { must_change_password = false } +resource "snowflake_user" "fivetran" { + provider = snowflake.useradmin + name = "FIVETRAN_SVC_USER_${var.environment}" + comment = "Service user for Fivetran" + + default_warehouse = module.loading["XS"].name + default_role = snowflake_role.loader.name + + must_change_password = false +} + resource "snowflake_user" "github_ci" { provider = snowflake.useradmin name = "GITHUB_ACTIONS_SVC_USER_${var.environment}" @@ -54,6 +65,13 @@ resource "snowflake_role_grants" "loader_to_airflow" { users = [snowflake_user.airflow.name] } +resource "snowflake_role_grants" "loader_to_fivetran" { + provider = snowflake.useradmin + role_name = snowflake_role.loader.name + enable_multiple_grants = true + users = [snowflake_user.fivetran.name] +} + resource "snowflake_role_grants" "reader_to_github_ci" { provider = snowflake.useradmin role_name = snowflake_role.reader.name