S3 bucket access via s3fs

Author: DmitryFomin1

README.md

@@ -20,7 +20,7 @@ Your source PostgreSQL database can be located anywhere, but DLE with other comp
     * Read/Write permissions on Route53
     * Read/Write permissions on Cloudwatch
 
-## How to use
+## Configuration overview
 - :construction: Currently, it is supposed that you run `terraform` commands on a Linux machine. MacOS and Windows support is not yet implemented (but planned).
 - It is recommended to clone this Git repository and adjust for your needs. Below we provide the detailed step-by-step instructions for quick start (see "Quick start") for a PoC setup
 - To configure parameters used by Terraform (and the Database Lab Engine itself), you will need to modify `terraform.tfvars` and create a file with secrets (`secret.tfvars`)
@@ -31,7 +31,7 @@ Your source PostgreSQL database can be located anywhere, but DLE with other comp
     - values passed on the command line
 - All variables starting with `postgres_` represent the source database connection information for the data (from that database) to be fetched by the DLE. That database must be accessible from the instance hosting the DLE (that one created by Terraform)
 
-## Quick start
+## How-to guide: using this Terraform module to set up DLE and its components
 The following steps were tested on Ubuntu 20.04 but supposed to be valid for other Linux distributions without significant modification.
 
 1. SSH to any machine with internet access, it will be used as deployment machine
@@ -65,12 +65,23 @@ The following steps were tested on Ubuntu 20.04 but supposed to be valid for oth
     aws_deploy_allow_ssh_from_cidrs = ["0.0.0.0/0"]
     aws_deploy_dns_api_subdomain = "tf-test" # subdomain in aws.postgres.ai, fqdn will be ${dns_api_subdomain}-engine.aws.postgres
 
+    # Source – two options. Choose one of two:
+    #    - direct connection to source DB
+    #    - dump stored on AWS S3
+
+    # option 1 – direct PG connection
+    source_type = "postgres" # source is working dome postgres database
     source_postgres_version = "13"
-    source_postgres_host = "ec2-3-215-57-87.compute-1.amazonaws.com"
+    source_postgres_host = "ec2-3-215-57-87.compute-1.amazonaws.com" # an example DB at Heroku
     source_postgres_port = "5432"
-    source_postgres_dbname = "d3dljqkrnopdvg" # this is an existing DB (Heroku example DB)
-    source_postgres_username = "postgres"
-
+    source_postgres_dbname = "d3dljqkrnopdvg" # an example DB at Heroku
+    source_postgres_username = "bfxuriuhcfpftt" # an example DB at Heroku
+    
+    # option 2 – dump on S3. Important: your AWS user has to be able to create IAM roles to work with S3 buckets in your AWS account
+    # source_type = 's3' # source is dump stored on demo s3 bucket
+    # source_pgdump_s3_bucket = "tf-demo-dump" # This is an example public bucket
+    # source_pgdump_path_on_s3_bucket = "heroku.dmp" # This is an example dump from demo database
+    
     dle_debug_mode = "true"
     dle_retrieval_refresh_timetable = "0 0 * * 0"
     postgres_config_shared_preload_libraries = "pg_stat_statements,logerrors" # DB Migration Checker requires logerrors extension
@@ -79,7 +90,7 @@ The following steps were tested on Ubuntu 20.04 but supposed to be valid for oth
     ```
 1. Create `secret.tfvars` containing `source_postgres_password`, `platform_access_token`, and `vcs_github_secret_token`. An example:
     ```config
-    source_postgres_password = "YOUR_DB_PASSWORD" # todo: put pwd for heroku example DB here
+    source_postgres_password = "dfe01cbd809a71efbaecafec5311a36b439460ace161627e5973e278dfe960b7" # an example DB at Heroku
     platform_access_token = "YOUR_ACCESS_TOKEN"   # to generate, open https://console.postgres.ai/, choose your organization,
                                                 # then "Access tokens" in the left menu
     vcs_github_secret_token = "vcs_secret_token"  # to generate, open https://github.com/settings/tokens/new
@@ -94,16 +105,16 @@ The following steps were tested on Ubuntu 20.04 but supposed to be valid for oth
     export AWS_SECRET_ACCESS_KEY = "accesskey"
     ```
 1. Deploy:
-    ```
+    ```shell
     terraform  apply -var-file="secret.tfvars" -auto-approve
     ```
 1. If everything goes well, you should get an output like this:
     ```config
     vcs_db_migration_checker_verification_token = "gsio7KmgaxECfJ80kUx2tUeIf4kEXZex"
     dle_verification_token = "zXPodd13LyQaKgVXGmSCeB8TUtnGNnIa"
-    ec2_public_dns = "ec2-18-118-126-25.us-east-2.compute.amazonaws.com"
-    ec2instance = "i-0b07738148950af25"
-    ip = "18.118.126.25"
+    ec2_public_dns = "ec2-11-111-111-11.us-east-2.compute.amazonaws.com"
+    ec2instance = "i-0000000000000"
+    ip = "11.111.111.11"
     platform_joe_signing_secret = "lG23qZbUh2kq0ULIBfW6TRwKzqGZu1aP"
     public_dns_name = "demo-api-engine.aws.postgres.ai"  # todo: this should be URL, not hostname – further we'll need URL, with protocol – `https://`
     ```

dle-logical-init.sh.tpl

@@ -99,34 +99,48 @@ for i in $${!disks[@]}; do
   $${disks[$i]}
 done
 
-#configure and start DLE
+# Adjust DLE config
 mkdir ~/.dblab
-#cp /home/ubuntu/.dblab/config.example.logical_generic.yml ~/.dblab/server.yml
 curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version_full}/configs/config.example.logical_generic.yml --output ~/.dblab/server.yml
 sed -ri "s/^(\s*)(debug:.*$)/\1debug: ${dle_debug_mode}/" ~/.dblab/server.yml
+sed -ri "s/^(\s*)(verificationToken:.*$)/\1verificationToken: ${dle_verification_token}/" ~/.dblab/server.yml
 sed -ri "s/^(\s*)(timetable:.*$)/\1timetable: \"${dle_retrieval_refresh_timetable}\"/" ~/.dblab/server.yml
 sed -ri "s/^(\s*)(forceInit:.*$)/\1forceInit: true/" ~/.dblab/server.yml
-sed -ri "s/^(\s*)(verificationToken:.*$)/\1verificationToken: ${dle_verification_token}/" ~/.dblab/server.yml
 sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: ${source_postgres_dbname}/" ~/.dblab/server.yml
-sed -ri "s/^(\s*)(host: 34.56.78.90$)/\1host: ${source_postgres_host}/" ~/.dblab/server.yml
-sed -ri "s/^(\s*)(port: 5432$)/\1port: ${source_postgres_port}/" ~/.dblab/server.yml
-sed -ri "s/^(\s*)(            username: postgres$)/\1            username: ${source_postgres_username}/" ~/.dblab/server.yml
-sed -ri "s/^(\s*)(password:.*$)/\1password: ${source_postgres_password}/" ~/.dblab/server.yml
-sed -ri "s/:13/:${source_postgres_version}/g"  ~/.dblab/server.yml
-#restore pg_dump via pipe -  without saving it on the disk
-sed -ri "s/^(\s*)(parallelJobs:.*$)/\1parallelJobs: 1/" ~/.dblab/server.yml
-sed -ri "s/^(\s*)(# immediateRestore:.*$)/\1immediateRestore: /" ~/.dblab/server.yml
-sed -ri "s/^(\s*)(#   forceInit: false.*$)/\1  forceInit: true /" ~/.dblab/server.yml
-sed -ri "s/^(\s*)(        #   configs:$)/\1          configs: /" ~/.dblab/server.yml
-sed -ri "s/^(\s*)(        #      shared_preload_libraries: .*$)/\1            shared_preload_libraries: '${postgres_config_shared_preload_libraries}'/" ~/.dblab/server.yml
-sed -ri "s/^(\s*)(          shared_preload_libraries:.*$)/\1          shared_preload_libraries: '${postgres_config_shared_preload_libraries}'/" ~/.dblab/server.yml
-sed -ri "s/^(\s*)(- logicalRestore.*$)/\1#- logicalRestore /" ~/.dblab/server.yml
 # Enable Platform
 sed -ri "s/^(\s*)(#platform:$)/\1platform: /" ~/.dblab/server.yml
 sed -ri "s/^(\s*)(#  url: \"https\\:\\/\\/postgres.ai\\/api\\/general\"$)/\1  url: \"https\\:\\/\\/postgres.ai\\/api\\/general\" /" ~/.dblab/server.yml
 sed -ri "s/^(\s*)(#  accessToken: \"platform_access_token\"$)/\1  accessToken: \"${platform_access_token}\"/" ~/.dblab/server.yml
 sed -ri "s/^(\s*)(#  enablePersonalTokens: true$)/\1  enablePersonalTokens: true/" ~/.dblab/server.yml
+sed -ri "s/:13/:${source_postgres_version}/g"  ~/.dblab/server.yml
 
+case "${source_type}" in 
+
+  postgres)
+  sed -ri "s/^(\s*)(host: 34.56.78.90$)/\1host: ${source_postgres_host}/" ~/.dblab/server.yml
+  sed -ri "s/^(\s*)(port: 5432$)/\1port: ${source_postgres_port}/" ~/.dblab/server.yml
+  sed -ri "s/^(\s*)(            username: postgres$)/\1            username: ${source_postgres_username}/" ~/.dblab/server.yml
+  sed -ri "s/^(\s*)(password:.*$)/\1password: ${source_postgres_password}/" ~/.dblab/server.yml
+  #restore pg_dump via pipe -  without saving it on the disk
+  sed -ri "s/^(\s*)(parallelJobs:.*$)/\1parallelJobs: 1/" ~/.dblab/server.yml
+  sed -ri "s/^(\s*)(# immediateRestore:.*$)/\1immediateRestore: /" ~/.dblab/server.yml
+  sed -ri "s/^(\s*)(#   forceInit: false.*$)/\1  forceInit: true /" ~/.dblab/server.yml
+  sed -ri "s/^(\s*)(        #   configs:$)/\1          configs: /" ~/.dblab/server.yml
+  sed -ri "s/^(\s*)(        #      shared_preload_libraries: .*$)/\1            shared_preload_libraries: '${postgres_config_shared_preload_libraries}'/" ~/.dblab/server.yml
+  sed -ri "s/^(\s*)(          shared_preload_libraries:.*$)/\1          shared_preload_libraries: '${postgres_config_shared_preload_libraries}'/" ~/.dblab/server.yml
+  sed -ri "s/^(\s*)(- logicalRestore.*$)/\1#- logicalRestore /" ~/.dblab/server.yml
+  ;;
+
+  s3)
+  # Mount S3 bucket if it's defined in Terraform variables
+  mkdir -p "${source_pgdump_s3_mount_point}"
+  s3fs ${source_pgdump_s3_bucket} ${source_pgdump_s3_mount_point} -o iam_role -o use_cache=/tmp -o allow_other
+  
+  sed -ri "s/^(\s*)(- logicalDump.*$)/\1#- logicalDump /" ~/.dblab/server.yml
+  sed -ri "s|^(\s*)(        dumpLocation:.*$)|\1        dumpLocation: ${source_pgdump_s3_mount_point}/${source_pgdump_path_on_s3_bucket}|" ~/.dblab/server.yml
+  ;;
+
+esac
 
 sudo docker run \
  --name dblab_server \

instance.tf

@@ -45,6 +45,10 @@ data "template_file" "init" {
     platform_joe_signing_secret = "${random_string.platform_joe_signing_secret.result}"
     vcs_db_migration_checker_verification_token = "${random_string.vcs_db_migration_checker_verification_token.result}"
     vcs_github_secret_token = "${var.vcs_github_secret_token}"
+    source_type = "${var.source_type}"
+    source_pgdump_s3_bucket = "${var.source_pgdump_s3_bucket}"
+    source_pgdump_s3_mount_point = "${var.source_pgdump_s3_mount_point}"
+    source_pgdump_path_on_s3_bucket = "${var.source_pgdump_path_on_s3_bucket}"
   }
 }
 
@@ -55,5 +59,6 @@ resource "aws_instance" "aws_ec2" {
   security_groups   = ["${aws_security_group.dle_instance_sg.name}"]
   key_name          = "${var.aws_keypair}"
   tags              = "${local.common_tags}"
+  iam_instance_profile = "${var.source_type == "s3" ? "${aws_iam_instance_profile.instance_profile[0].name}" : null}"
   user_data         = "${data.template_file.init.rendered}"
 }

role.tf

@@ -0,0 +1,42 @@
+resource "aws_iam_role" "db_lab_engine_role" {
+  count = "${var.source_type == "s3" ? 1 : 0}"
+  name               = "database_lab_engine"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Sid    = ""
+        Principal = {
+          Service = "ec2.amazonaws.com"
+        }
+      },
+    ]
+  })
+
+  inline_policy {
+    name = "pg_dump_access"
+
+    policy = jsonencode({
+      Version = "2012-10-17"
+      Statement = [
+        {
+          Action   = ["s3:ListBucket"]
+          Effect   = "Allow"
+          Resource = "arn:aws:s3:::${var.source_pgdump_s3_bucket}"
+        },
+        {
+          Action   = ["s3:GetObject"]
+          Effect   = "Allow"
+          Resource = "arn:aws:s3:::${var.source_pgdump_s3_bucket}/*" # Grant read access to entire bucket
+        }
+      ]
+    })
+  }
+}
+resource "aws_iam_instance_profile" "instance_profile" {
+    count = "${var.source_type == "s3" ? 1 : 0}"
+    name = "dle-instance-profile"
+    role = "${aws_iam_role.db_lab_engine_role[0].name}"
+}

variables.tf

@@ -147,3 +147,24 @@ variable "vcs_github_secret_token" {
 variable "postgres_config_shared_preload_libraries" {
   description = "shared_preload_libraries postgresql.conf parameter value for clones"
 }
+
+variable "source_type" {
+  description = "Type of data source used for DLE. For now it can be postgres,S3"
+  default = ""
+}
+
+variable "source_pgdump_s3_bucket" {
+  description = "S3 bucket name where a dump (created using pg_dump) is stored. This dump will be used as data source. Leave the value empty (default) to use a different source of data."
+  default = ""
+}
+
+variable "source_pgdump_path_on_s3_bucket" {
+  description = "relative path to pg_dump file or directory"
+  default = ""
+}
+
+variable "source_pgdump_s3_mount_point"{
+  description = "mount point on DLE EC2 instance where S3 bucket with the source dump file/directory is mounted to. If pgdump_s3_bucket is empty, pgdump_s3_mount_point is ignored"
+  default = "/s3/pg_dump"
+}
+