feat: add http datasource (#11658)

command/plugin.go

@@ -17,6 +17,7 @@ import (
 	nullbuilder "github.com/hashicorp/packer/builder/null"
 	hcppackerimagedatasource "github.com/hashicorp/packer/datasource/hcp-packer-image"
 	hcppackeriterationdatasource "github.com/hashicorp/packer/datasource/hcp-packer-iteration"
+	httpdatasource "github.com/hashicorp/packer/datasource/http"
 	nulldatasource "github.com/hashicorp/packer/datasource/null"
 	artificepostprocessor "github.com/hashicorp/packer/post-processor/artifice"
 	checksumpostprocessor "github.com/hashicorp/packer/post-processor/checksum"
@@ -64,6 +65,7 @@ var PostProcessors = map[string]packersdk.PostProcessor{
 var Datasources = map[string]packersdk.Datasource{
 	"hcp-packer-image":     new(hcppackerimagedatasource.Datasource),
 	"hcp-packer-iteration": new(hcppackeriterationdatasource.Datasource),
+	"http":                 new(httpdatasource.Datasource),
 	"null":                 new(nulldatasource.Datasource),
 }
 

datasource/http/data.go

@@ -0,0 +1,157 @@
+//go:generate packer-sdc struct-markdown
+//go:generate packer-sdc mapstructure-to-hcl2 -type DatasourceOutput,Config
+package http
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"mime"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/hashicorp/hcl/v2/hcldec"
+	"github.com/hashicorp/packer-plugin-sdk/common"
+	"github.com/hashicorp/packer-plugin-sdk/hcl2helper"
+	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
+	"github.com/hashicorp/packer-plugin-sdk/template/config"
+	"github.com/zclconf/go-cty/cty"
+)
+
+type Config struct {
+	common.PackerConfig `mapstructure:",squash"`
+	// The URL to request data from. This URL must respond with a `200 OK` response and a `text/*` or `application/json` Content-Type
+	Url string `mapstructure:"url" required:"true"`
+	// A map of strings representing additional HTTP headers to include in the request.
+	Request_headers map[string]string `mapstructure:"request_headers" required:"false"`
+}
+
+type Datasource struct {
+	config Config
+}
+
+type DatasourceOutput struct {
+        // The URL the data was requested from. 
+	Url              string            `mapstructure:"url"`
+	// The raw body of the HTTP response.
+	Response_body    string            `mapstructure:"body"`
+	// A map of strings representing the response HTTP headers. 
+	// Duplicate headers are contatenated with , according to [RFC2616](https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2)
+	Response_headers map[string]string `mapstructure:"request_headers"`
+}
+
+func (d *Datasource) ConfigSpec() hcldec.ObjectSpec {
+	return d.config.FlatMapstructure().HCL2Spec()
+}
+
+func (d *Datasource) Configure(raws ...interface{}) error {
+	err := config.Decode(&d.config, nil, raws...)
+	if err != nil {
+		return err
+	}
+
+	var errs *packersdk.MultiError
+
+	if d.config.Url == "" {
+		errs = packersdk.MultiErrorAppend(
+			errs,
+			fmt.Errorf("the `url` must be specified"))
+	}
+
+	if errs != nil && len(errs.Errors) > 0 {
+		return errs
+	}
+	return nil
+}
+
+func (d *Datasource) OutputSpec() hcldec.ObjectSpec {
+	return (&DatasourceOutput{}).FlatMapstructure().HCL2Spec()
+}
+
+// This is to prevent potential issues w/ binary files
+// and generally unprintable characters
+// See https://github.com/hashicorp/terraform/pull/3858#issuecomment-156856738
+func isContentTypeText(contentType string) bool {
+
+	parsedType, params, err := mime.ParseMediaType(contentType)
+	if err != nil {
+		return false
+	}
+
+	allowedContentTypes := []*regexp.Regexp{
+		regexp.MustCompile("^text/.+"),
+		regexp.MustCompile("^application/json$"),
+		regexp.MustCompile("^application/samlmetadata\\+xml"),
+	}
+
+	for _, r := range allowedContentTypes {
+		if r.MatchString(parsedType) {
+			charset := strings.ToLower(params["charset"])
+			return charset == "" || charset == "utf-8" || charset == "us-ascii"
+		}
+	}
+
+	return false
+}
+
+// Most of this code comes from http terraform provider data source
+// https://github.com/hashicorp/terraform-provider-http/blob/main/internal/provider/data_source.go
+func (d *Datasource) Execute() (cty.Value, error) {
+	ctx := context.TODO()
+	url, headers := d.config.Url, d.config.Request_headers
+	client := &http.Client{}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+	// TODO: How to make a test case for this?
+	if err != nil {
+		fmt.Println("Error creating http request")
+		return cty.NullVal(cty.EmptyObject), err
+	}
+
+	for name, value := range headers {
+		req.Header.Set(name, value)
+	}
+
+	resp, err := client.Do(req)
+	// TODO: How to make test case for this
+	if err != nil {
+		fmt.Println("Error making performing http request")
+		return cty.NullVal(cty.EmptyObject), err
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return cty.NullVal(cty.EmptyObject), fmt.Errorf("HTTP request error. Response code: %d", resp.StatusCode)
+	}
+
+	contentType := resp.Header.Get("Content-Type")
+	if contentType == "" || isContentTypeText(contentType) == false {
+		fmt.Println(fmt.Sprintf(
+			"Content-Type is not recognized as a text type, got %q",
+			contentType))
+		fmt.Println("If the content is binary data, Packer may not properly handle the contents of the response.")
+	}
+
+	bytes, err := ioutil.ReadAll(resp.Body)
+	// TODO: How to make test case for this?
+	if err != nil {
+		fmt.Println("Error processing response body of call")
+		return cty.NullVal(cty.EmptyObject), err
+	}
+
+	responseHeaders := make(map[string]string)
+	for k, v := range resp.Header {
+		// Concatenate according to RFC2616
+		// cf. https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
+		responseHeaders[k] = strings.Join(v, ", ")
+	}
+
+	output := DatasourceOutput{
+		Url:              d.config.Url,
+		Response_headers: responseHeaders,
+		Response_body:    string(bytes),
+	}
+	return hcl2helper.HCL2ValueFromConfig(output, d.OutputSpec()), nil
+}

datasource/http/data_acc_test.go

@@ -0,0 +1,109 @@
+package http
+
+import (
+	_ "embed"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"regexp"
+	"testing"
+
+	"github.com/hashicorp/packer-plugin-sdk/acctest"
+)
+
+//go:embed test-fixtures/basic.pkr.hcl
+var testDatasourceBasic string
+
+//go:embed test-fixtures/empty_url.pkr.hcl
+var testDatasourceEmptyUrl string
+
+//go:embed test-fixtures/404_url.pkr.hcl
+var testDatasource404Url string
+
+func TestHttpDataSource(t *testing.T) {
+	tests := []struct {
+		Name    string
+		Path    string
+		Error   bool
+		Outputs map[string]string
+	}{
+		{
+			Name:  "basic_test",
+			Path:  testDatasourceBasic,
+			Error: false,
+			Outputs: map[string]string{
+				"url": "url is https://www.packer.io/",
+				// Check that body is not empty
+				"body": "body is true",
+			},
+		},
+		{
+			Name:  "url_is_empty",
+			Path:  testDatasourceEmptyUrl,
+			Error: true,
+			Outputs: map[string]string{
+				"error": "the `url` must be specified",
+			},
+		},
+		{
+			Name:  "404_url",
+			Path:  testDatasource404Url,
+			Error: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.Name, func(t *testing.T) {
+			testCase := &acctest.PluginTestCase{
+				Name: tt.Name,
+				Setup: func() error {
+					return nil
+				},
+				Teardown: func() error {
+					return nil
+				},
+				Template: tt.Path,
+				Type:     "http",
+				Check: func(buildCommand *exec.Cmd, logfile string) error {
+					if buildCommand.ProcessState != nil {
+						if buildCommand.ProcessState.ExitCode() != 0 && !tt.Error {
+							return fmt.Errorf("Bad exit code. Logfile: %s", logfile)
+						}
+						if tt.Error && buildCommand.ProcessState.ExitCode() == 0 {
+							return fmt.Errorf("Expected Bad exit code.")
+						}
+					}
+
+					if tt.Outputs != nil {
+						logs, err := os.Open(logfile)
+						if err != nil {
+							return fmt.Errorf("Unable find %s", logfile)
+						}
+						defer logs.Close()
+
+						logsBytes, err := ioutil.ReadAll(logs)
+						if err != nil {
+							return fmt.Errorf("Unable to read %s", logfile)
+						}
+						logsString := string(logsBytes)
+
+						for key, val := range tt.Outputs {
+							if matched, _ := regexp.MatchString(val+".*", logsString); !matched {
+								t.Fatalf(
+									"logs doesn't contain expected log %v with value %v in %q",
+									key,
+									val,
+									logsString)
+							}
+						}
+
+					}
+
+					return nil
+				},
+			}
+			acctest.TestPlugin(t, testCase)
+		})
+	}
+
+}

datasource/http/test-fixtures/404_url.pkr.hcl

@@ -0,0 +1,24 @@
+
+source "null" "example" {
+  communicator = "none"
+}
+
+data "http" "basic" {
+  url = "https://www.packer.io/thisWillFail"
+}
+
+locals {
+  url = "${data.http.basic.url}"
+}
+
+build {
+  name = "mybuild"
+  sources = [
+    "source.null.example"
+  ]
+  provisioner "shell-local" {
+    inline = [
+      "echo data is ${local.url}",
+    ]
+  }
+}