added support for preloading of docker images into instance pools

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Version changelog
 
+## 0.3.5
+
+* Fixed setting of permissions for SQLA endpoints ([#661](https://github.com/databrickslabs/terraform-provider-databricks/issues/661))
+* Added support for preloading of Docker images into instance pools ([#663](https://github.com/databrickslabs/terraform-provider-databricks/issues/663))
+* Added the `databricks_user` data source ([#648](https://github.com/databrickslabs/terraform-provider-databricks/pull/648))
+
 ## 0.3.4
 
 * Fixed state refresh bugs in `databricks_sql_permissions` ([#620](https://github.com/databrickslabs/terraform-provider-databricks/issues/620), [#619](https://github.com/databrickslabs/terraform-provider-databricks/issues/620))

compute/model.go

@@ -416,6 +416,7 @@ type InstancePool struct {
 	EnableElasticDisk                  bool                         `json:"enable_elastic_disk,omitempty"`
 	DiskSpec                           *InstancePoolDiskSpec        `json:"disk_spec,omitempty"`
 	PreloadedSparkVersions             []string                     `json:"preloaded_spark_versions,omitempty"`
+	PreloadedDockerImages              []DockerImage                `json:"preloaded_docker_images,omitempty" tf:"slice_set,alias:preloaded_docker_image"`
 }
 
 // InstancePoolStats contains the stats on a given pool
@@ -443,6 +444,7 @@ type InstancePoolAndStats struct {
 	PreloadedSparkVersions             []string                     `json:"preloaded_spark_versions,omitempty"`
 	State                              string                       `json:"state,omitempty"`
 	Stats                              *InstancePoolStats           `json:"stats,omitempty"`
+	PreloadedDockerImages              []DockerImage                `json:"preloaded_docker_images,omitempty" tf:"slice_set,alias:preloaded_docker_image"`
 }
 
 // InstancePoolList shows list of instance pools

compute/resource_instance_pool.go

@@ -72,6 +72,7 @@ func ResourceInstancePool() *schema.Resource {
 		s["node_type_id"].ForceNew = true
 		s["custom_tags"].ForceNew = true
 		s["preloaded_spark_versions"].ForceNew = true
+		s["preloaded_docker_image"].ForceNew = true
 		s["azure_attributes"].ForceNew = true
 		s["disk_spec"].ForceNew = true
 		s["enable_elastic_disk"].ForceNew = true
@@ -122,6 +123,15 @@ func ResourceInstancePool() *schema.Resource {
 				EbsVolumeTypeThroughputOptimizedHdd,
 			}, false)
 		}
+		if v, err := common.SchemaPath(s, "preloaded_docker_image", "url"); err == nil {
+			v.ForceNew = true
+		}
+		if v, err := common.SchemaPath(s, "preloaded_docker_image", "basic_auth", "username"); err == nil {
+			v.ForceNew = true
+		}
+		if v, err := common.SchemaPath(s, "preloaded_docker_image", "basic_auth", "password"); err == nil {
+			v.ForceNew = true
+		}
 		return s
 	})
 	return common.Resource{

docs/resources/cluster.md

@@ -120,7 +120,7 @@ resource "databricks_cluster" "single_node" {
 
 ### library Configuration Block
 
-To install libraries, one must specify each library in its configuration block. Each different type of library has a slightly different syntax. It's possible to set only one type of library within one config block. Otherwise, the plan will fail with an error.
+To install libraries, one must specify each library in a separate configuration block. Each different type of library has a slightly different syntax. It's possible to set only one type of library within one config block. Otherwise, the plan will fail with an error.
 
 Installing JAR artifacts on a cluster. Location can be anything, that is DBFS or mounted object store (s3, adls, ...)
 ```hcl

docs/resources/instance_pool.md

@@ -36,7 +36,7 @@ resource "databricks_instance_pool" "smallest_nodes" {
 
 ## Argument Reference
 
-The following arguments are required:
+The following arguments are supported:
 
 * `instance_pool_name` - (Required) (String) The name of the instance pool. This is required for create and edit operations. It must be unique, non-empty, and less than 100 characters.
 * `min_idle_instances` - (Optional) (Integer) The minimum number of idle instances maintained by the pool. This is in addition to any instances in use by active clusters.
@@ -83,6 +83,37 @@ For disk_spec make sure to use **ebs_volume_type** only on AWS deployment of Dat
   * Premium LRS (SSD): `1 - 1023` GiB
   * Standard LRS (HDD): `1- 1023` GiB
 
+## preloaded_docker_image sub_block
+
+[Databricks Container Services](https://docs.databricks.com/clusters/custom-containers.html) lets you specify a Docker image when you create a cluster.  You need to enable Container Services in *Admin Console /  Advanced* page in the user interface. By enabling this feature, you acknowledge and agree that your usage of this feature is subject to the [applicable additional terms](http://www.databricks.com/product-specific-terms). You can instruct instance pool to pre-download the Docker image onto the instances so when node is acquired for a cluster that requires a custom Docker image the setup process will be faster.
+
+`preloaded_docker_image` configuration block has the following attributes:
+
+* `url` - URL for the Docker image
+* `basic_auth` - (Optional) `basic_auth.username` and `basic_auth.password` for Docker repository. Docker registry credentials are encrypted when they are stored in Databricks internal storage and when they are passed to a registry upon fetching Docker images at cluster launch. However, other authenticated and authorized API users of this workspace can access the username and password.
+
+Example usage with [azurerm_container_registry](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/container_registry) and [docker_registry_image](https://registry.terraform.io/providers/kreuzwerker/docker/latest/docs/resources/registry_image), that you can adapt to your specific use-case:
+
+```hcl
+resource "docker_registry_image" "this" {
+  name = "${azurerm_container_registry.this.login_server}/sample:latest"
+  build {
+    # ...
+  }
+}
+
+resource "databricks_instance_pool" "this" {
+  # ...
+  preloaded_docker_image {
+    url = docker_registry_image.this.name
+    basic_auth {
+      username = azurerm_container_registry.this.admin_username
+      password = azurerm_container_registry.this.admin_password
+    }
+  }
+}
+```
+
 ## Attribute Reference
 
 In addition to all arguments above, the following attributes are exported: