Merge pull request #90 from techpivot/main

Add new opt-in flag to specify the output_file_mode to produce more deterministic behavior across operating systems.

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 2.2.0 (Unreleased)
+
+ENHANCEMENTS:
+
+* New opt-in flag to specify the `output_file_mode` to produce more deterministic behavior across operating systems. ([#90](https://github.com/hashicorp/terraform-provider-archive/pull/90))
+
 ## 2.1.0 (February 19, 2021)
 
 Binary releases of this provider now include the darwin-arm64 platform. This version contains no further changes.

internal/provider/archiver.go

@@ -10,17 +10,18 @@ type Archiver interface {
 	ArchiveFile(infilename string) error
 	ArchiveDir(indirname string, excludes []string) error
 	ArchiveMultiple(content map[string][]byte) error
+	SetOutputFileMode(outputFileMode string)
 }
 
-type ArchiverBuilder func(filepath string) Archiver
+type ArchiverBuilder func(outputPath string) Archiver
 
 var archiverBuilders = map[string]ArchiverBuilder{
 	"zip": NewZipArchiver,
 }
 
-func getArchiver(archiveType string, filepath string) Archiver {
+func getArchiver(archiveType string, outputPath string) Archiver {
 	if builder, ok := archiverBuilders[archiveType]; ok {
-		return builder(filepath)
+		return builder(outputPath)
 	}
 	return nil
 }

internal/provider/data_source_archive_file.go

@@ -113,6 +113,12 @@ func dataSourceFile() *schema.Resource {
 				ForceNew:    true,
 				Description: "MD5 of output file",
 			},
+			"output_file_mode": {
+				Type:     schema.TypeString,
+				Optional: true,
+				Default:  "",
+				ForceNew: true,
+			},
 		},
 	}
 }
@@ -141,13 +147,12 @@ func dataSourceFileRead(d *schema.ResourceData, meta interface{}) error {
 
 	sha1, base64sha256, md5, err := genFileShas(outputPath)
 	if err != nil {
-
 		return fmt.Errorf("could not generate file checksum sha256: %s", err)
 	}
+
 	d.Set("output_sha", sha1)
 	d.Set("output_base64sha256", base64sha256)
 	d.Set("output_md5", md5)
-
 	d.Set("output_size", fi.Size())
 	d.SetId(d.Get("output_sha").(string))
 
@@ -171,6 +176,11 @@ func archive(d *schema.ResourceData) error {
 		return fmt.Errorf("archive type not supported: %s", archiveType)
 	}
 
+	outputFileMode := d.Get("output_file_mode").(string)
+	if outputFileMode != "" {
+		archiver.SetOutputFileMode(outputFileMode)
+	}
+
 	if dir, ok := d.GetOk("source_dir"); ok {
 		if excludes, ok := d.GetOk("excludes"); ok {
 			excludeList := expandStringList(excludes.(*schema.Set).List())

internal/provider/data_source_archive_file_test.go

@@ -5,7 +5,6 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"regexp"
 	"testing"
 
 	r "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
@@ -28,19 +27,14 @@ func TestAccArchiveFile_Basic(t *testing.T) {
 					testAccArchiveFileExists(f, &fileSize),
 					r.TestCheckResourceAttrPtr("data.archive_file.foo", "output_size", &fileSize),
 
-					// We just check the hashes for syntax rather than exact
-					// content since we don't want to break if the archive
-					// library starts generating different bytes that are
-					// functionally equivalent.
-					r.TestMatchResourceAttr(
-						"data.archive_file.foo", "output_base64sha256",
-						regexp.MustCompile(`^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$`),
+					r.TestCheckResourceAttr(
+						"data.archive_file.foo", "output_base64sha256", "P7VckxoEiUO411WN3nwuS/yOBL4zsbVWkQU9E1I5H6c=",
 					),
-					r.TestMatchResourceAttr(
-						"data.archive_file.foo", "output_md5", regexp.MustCompile(`^[0-9a-f]{32}$`),
+					r.TestCheckResourceAttr(
+						"data.archive_file.foo", "output_md5", "ea35f0444ea9a3d5641d8760bc2815cc",
 					),
-					r.TestMatchResourceAttr(
-						"data.archive_file.foo", "output_sha", regexp.MustCompile(`^[0-9a-f]{40}$`),
+					r.TestCheckResourceAttr(
+						"data.archive_file.foo", "output_sha", "019c79c4dc14dbe1edb3e467b2de6a6aad148717",
 					),
 				),
 			},
@@ -49,13 +43,31 @@ func TestAccArchiveFile_Basic(t *testing.T) {
 				Check: r.ComposeTestCheckFunc(
 					testAccArchiveFileExists(f, &fileSize),
 					r.TestCheckResourceAttrPtr("data.archive_file.foo", "output_size", &fileSize),
+					r.TestCheckResourceAttr(
+						"data.archive_file.foo", "output_base64sha256", "UTE4f5cWfaR6p0HfOrLILxgvF8UUwiJTjTRwjQTgdWs=",
+					),
+					r.TestCheckResourceAttr(
+						"data.archive_file.foo", "output_md5", "59fbc9e62af3cbc2f588f97498240dae",
+					),
+					r.TestCheckResourceAttr(
+						"data.archive_file.foo", "output_sha", "ce4ee1450ab93ac86e11446649e44cea907b6568",
+					),
 				),
 			},
 			{
 				Config: testAccArchiveFileDirConfig(f),
 				Check: r.ComposeTestCheckFunc(
 					testAccArchiveFileExists(f, &fileSize),
 					r.TestCheckResourceAttrPtr("data.archive_file.foo", "output_size", &fileSize),
+					r.TestCheckResourceAttr(
+						"data.archive_file.foo", "output_base64sha256", "ydB8wtq8nK9vQ77VH6YTwoHmyljK46jW+uIJSwCzNpo=",
+					),
+					r.TestCheckResourceAttr(
+						"data.archive_file.foo", "output_md5", "b73f64a383716070aa4a29563b8b14d4",
+					),
+					r.TestCheckResourceAttr(
+						"data.archive_file.foo", "output_sha", "76d20a402eefd1cfbdc47886abd4e0909616c191",
+					),
 				),
 			},
 			{
@@ -103,19 +115,21 @@ data "archive_file" "foo" {
 func testAccArchiveFileFileConfig(outputPath string) string {
 	return fmt.Sprintf(`
 data "archive_file" "foo" {
-  type        = "zip"
-  source_file = "test-fixtures/test-file.txt"
-  output_path = "%s"
+  type             = "zip"
+  source_file      = "test-fixtures/test-file.txt"
+  output_path      = "%s"
+  output_file_mode = "0666"
 }
 `, filepath.ToSlash(outputPath))
 }
 
 func testAccArchiveFileDirConfig(outputPath string) string {
 	return fmt.Sprintf(`
 data "archive_file" "foo" {
-  type        = "zip"
-  source_dir  = "test-fixtures/test-dir"
-  output_path = "%s"
+  type             = "zip"
+  source_dir       = "test-fixtures/test-dir"
+  output_path      = "%s"
+  output_file_mode = "0666"
 }
 `, filepath.ToSlash(outputPath))
 }

internal/provider/zip_archiver.go

@@ -7,13 +7,15 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"time"
 )
 
 type ZipArchiver struct {
-	filepath   string
-	filewriter *os.File
-	writer     *zip.Writer
+	filepath       string
+	outputFileMode string // Default value "" means unset
+	filewriter     *os.File
+	writer         *zip.Writer
 }
 
 func NewZipArchiver(filepath string) Archiver {
@@ -62,6 +64,14 @@ func (a *ZipArchiver) ArchiveFile(infilename string) error {
 	// fh.Modified alone isn't enough when using a zero value
 	fh.SetModTime(time.Time{})
 
+	if a.outputFileMode != "" {
+		filemode, err := strconv.ParseUint(a.outputFileMode, 0, 32)
+		if err != nil {
+			return fmt.Errorf("error parsing output_file_mode value: %s", a.outputFileMode)
+		}
+		fh.SetMode(os.FileMode(filemode))
+	}
+
 	f, err := a.writer.CreateHeader(fh)
 	if err != nil {
 		return fmt.Errorf("error creating file inside archive: %s", err)
@@ -137,6 +147,14 @@ func (a *ZipArchiver) ArchiveDir(indirname string, excludes []string) error {
 		// fh.Modified alone isn't enough when using a zero value
 		fh.SetModTime(time.Time{})
 
+		if a.outputFileMode != "" {
+			filemode, err := strconv.ParseUint(a.outputFileMode, 0, 32)
+			if err != nil {
+				return fmt.Errorf("error parsing output_file_mode value: %s", a.outputFileMode)
+			}
+			fh.SetMode(os.FileMode(filemode))
+		}
+
 		f, err := a.writer.CreateHeader(fh)
 		if err != nil {
 			return fmt.Errorf("error creating file inside archive: %s", err)
@@ -178,6 +196,10 @@ func (a *ZipArchiver) ArchiveMultiple(content map[string][]byte) error {
 	return nil
 }
 
+func (a *ZipArchiver) SetOutputFileMode(outputFileMode string) {
+	a.outputFileMode = outputFileMode
+}
+
 func (a *ZipArchiver) open() error {
 	f, err := os.Create(a.filepath)
 	if err != nil {

internal/provider/zip_archiver_test.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strconv"
 	"testing"
 	"time"
 )
@@ -33,6 +34,30 @@ func TestZipArchiver_File(t *testing.T) {
 		"test-file.txt": []byte("This is test content"),
 	})
 }
+
+func TestZipArchiver_FileMode(t *testing.T) {
+	file, err := ioutil.TempFile("", "archive-file-mode-test.zip")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var (
+		zipFilePath = file.Name()
+		toZipPath   = filepath.FromSlash("./test-fixtures/test-file.txt")
+	)
+
+	stringArray := [5]string{"0444", "0644", "0666", "0744", "0777"}
+	for _, element := range stringArray {
+		archiver := NewZipArchiver(zipFilePath)
+		archiver.SetOutputFileMode(element)
+		if err := archiver.ArchiveFile(toZipPath); err != nil {
+			t.Fatalf("unexpected error: %s", err)
+		}
+
+		ensureFileMode(t, zipFilePath, element)
+	}
+}
+
 func TestZipArchiver_FileModified(t *testing.T) {
 	var (
 		zipFilePath = filepath.FromSlash("archive-file.zip")
@@ -63,7 +88,7 @@ func TestZipArchiver_FileModified(t *testing.T) {
 
 	actualContents, err := ioutil.ReadFile(zipFilePath)
 	if err != nil {
-		t.Fatalf("unexpecte error: %s", err)
+		t.Fatalf("unexpected error: %s", err)
 	}
 
 	if !bytes.Equal(expectedContents, actualContents) {
@@ -126,10 +151,10 @@ func TestZipArchiver_Multiple(t *testing.T) {
 	}
 
 	ensureContents(t, zipfilepath, content)
-
 }
 
 func ensureContents(t *testing.T, zipfilepath string, wants map[string][]byte) {
+	t.Helper()
 	r, err := zip.OpenReader(zipfilepath)
 	if err != nil {
 		t.Fatalf("could not open zip file: %s", err)
@@ -145,6 +170,7 @@ func ensureContents(t *testing.T, zipfilepath string, wants map[string][]byte) {
 }
 
 func ensureContent(t *testing.T, wants map[string][]byte, got *zip.File) {
+	t.Helper()
 	want, ok := wants[got.Name]
 	if !ok {
 		t.Errorf("additional file in zip: %s", got.Name)
@@ -167,3 +193,28 @@ func ensureContent(t *testing.T, wants map[string][]byte, got *zip.File) {
 		t.Errorf("mismatched content\ngot\n%s\nwant\n%s", gotContent, wantContent)
 	}
 }
+
+func ensureFileMode(t *testing.T, zipfilepath string, outputFileMode string) {
+	t.Helper()
+	r, err := zip.OpenReader(zipfilepath)
+	if err != nil {
+		t.Fatalf("could not open zip file: %s", err)
+	}
+	defer r.Close()
+
+	filemode, err := strconv.ParseUint(outputFileMode, 0, 32)
+	if err != nil {
+		t.Fatalf("error parsing outputFileMode value: %s", outputFileMode)
+	}
+	var osfilemode = os.FileMode(filemode)
+
+	for _, cf := range r.File {
+		if cf.FileInfo().IsDir() {
+			continue
+		}
+
+		if cf.Mode() != osfilemode {
+			t.Fatalf("Expected filemode \"%s\" but was \"%s\"", osfilemode, cf.Mode())
+		}
+	}
+}

website/docs/d/archive_file.html.markdown

@@ -38,6 +38,16 @@ data "archive_file" "dotfiles" {
     filename = ".ssh/config"
   }
 }
+
+# Archive a file to be used with Lambda using consistent file mode
+
+data "archive_file" "lambda_my_function" {
+  type             = "zip"
+  source_file      = "${path.module}/../lambda/my-function/index.js"
+  output_file_mode = "0666"
+  output_path      = "${path.module}/files/lambda-my-function.js.zip"
+}
+
 ```
 
 ~> **Note regarding symbolic links**: Due to a bug, the `archive_file` data
@@ -58,6 +68,8 @@ NOTE: One of `source`, `source_content_filename` (with `source_content`), `sourc
 
 * `output_path` - (Required) The output of the archive file.
 
+* `output_file_mode` (Optional) String that specifies the octal file mode for all archived files. For example: `"0666"`. Setting this will ensure that cross platform usage of this module will not vary the modes of archived files (and ultimately checksums) resulting in more deterministic behavior.
+
 * `source_content` - (Optional) Add only this content to the archive with `source_content_filename` as the filename.
 
 * `source_content_filename` - (Optional) Set this as the filename when using `source_content`.