From 4de6b1e01605f5537d9d5f2f7374ca6e566d946a Mon Sep 17 00:00:00 2001 From: Selena Goods Date: Wed, 24 Jul 2024 14:42:45 -0400 Subject: [PATCH 1/8] Add glob pattern matching support for the `excludes` attribute. --- go.mod | 1 + go.sum | 2 + internal/provider/data_source_archive_file.go | 3 +- internal/provider/resource_archive_file.go | 3 +- internal/provider/zip_archiver.go | 24 +++++--- internal/provider/zip_archiver_test.go | 60 +++++++++++++++++++ 6 files changed, 82 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 3a150442..5ce46219 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 toolchain go1.21.3 require ( + github.com/bmatcuk/doublestar/v4 v4.6.1 github.com/hashicorp/terraform-plugin-framework v1.10.0 github.com/hashicorp/terraform-plugin-framework-validators v0.13.0 github.com/hashicorp/terraform-plugin-go v0.23.0 diff --git a/go.sum b/go.sum index b6a12c66..afd408fc 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/agext/levenshtein v1.2.2/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJEynmyUenKwP++x/+DdGV/Ec= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= +github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I= +github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= diff --git a/internal/provider/data_source_archive_file.go b/internal/provider/data_source_archive_file.go index 5a21511b..44f4c20a 100644 --- a/internal/provider/data_source_archive_file.go +++ b/internal/provider/data_source_archive_file.go @@ -134,7 +134,8 @@ func (d *archiveFileDataSource) Schema(ctx context.Context, req datasource.Schem }, }, "excludes": schema.SetAttribute{ - Description: "Specify files to ignore when reading the `source_dir`.", + Description: "Specify files/directories to ignore when reading the `source_dir`. " + + "Supports glob file matching patterns including doublestar/globstar (`**`) patterns", ElementType: types.StringType, Optional: true, Validators: []validator.Set{ diff --git a/internal/provider/resource_archive_file.go b/internal/provider/resource_archive_file.go index 9f2beb5d..04bcfdc8 100644 --- a/internal/provider/resource_archive_file.go +++ b/internal/provider/resource_archive_file.go @@ -154,7 +154,8 @@ func (d *archiveFileResource) Schema(ctx context.Context, req resource.SchemaReq }, }, "excludes": schema.SetAttribute{ - Description: "Specify files to ignore when reading the `source_dir`.", + Description: "Specify files/directories to ignore when reading the `source_dir`. " + + "Supports glob file matching patterns including doublestar/globstar (`**`) patterns", ElementType: types.StringType, Optional: true, Validators: []validator.Set{ diff --git a/internal/provider/zip_archiver.go b/internal/provider/zip_archiver.go index d4c00121..80dc47ce 100644 --- a/internal/provider/zip_archiver.go +++ b/internal/provider/zip_archiver.go @@ -11,6 +11,8 @@ import ( "sort" "strconv" "time" + + "github.com/bmatcuk/doublestar/v4" ) type ZipArchiver struct { @@ -83,17 +85,22 @@ func (a *ZipArchiver) ArchiveFile(infilename string) error { return err } -func checkMatch(fileName string, excludes []string) (value bool) { +func checkMatch(fileName string, excludes []string) (value bool, err error) { for _, exclude := range excludes { if exclude == "" { continue } - if exclude == fileName { - return true + match, err := doublestar.Match(exclude, fileName) + if err != nil { + return false, err + } + + if match { + return true, nil } } - return false + return false, nil } func (a *ZipArchiver) ArchiveDir(indirname string, opts ArchiveDirOpts) error { @@ -141,7 +148,10 @@ func (a *ZipArchiver) createWalkFunc(basePath, indirname string, opts ArchiveDir archivePath := filepath.Join(basePath, relname) - isMatch := checkMatch(archivePath, opts.Excludes) + isMatch, err := checkMatch(archivePath, opts.Excludes) + if err != nil { + return fmt.Errorf("error checking excludes matches: %s", err) + } if info.IsDir() { if isMatch { @@ -154,10 +164,6 @@ func (a *ZipArchiver) createWalkFunc(basePath, indirname string, opts ArchiveDir return nil } - if err != nil { - return err - } - if info.Mode()&os.ModeSymlink == os.ModeSymlink { realPath, err := filepath.EvalSymlinks(path) if err != nil { diff --git a/internal/provider/zip_archiver_test.go b/internal/provider/zip_archiver_test.go index a46de0d8..d881e5b9 100644 --- a/internal/provider/zip_archiver_test.go +++ b/internal/provider/zip_archiver_test.go @@ -254,6 +254,32 @@ func TestZipArchiver_Dir_Exclude_DoNotExcludeSymlinkDirectories(t *testing.T) { }) } +func TestZipArchiver_Dir_Exclude_Glob_DoNotExcludeSymlinkDirectories(t *testing.T) { + zipFilePath := filepath.Join(t.TempDir(), "archive-dir-with-symlink-dir.zip") + + archiver := NewZipArchiver(zipFilePath) + if err := archiver.ArchiveDir("./test-fixtures", ArchiveDirOpts{ + Excludes: []string{ + "**/file1.txt", + "**/file2.*", + "test-dir-with-symlink-dir/test-symlink-dir", + "test-symlink-dir-with-symlink-file/test-symlink.txt", + }, + }); err != nil { + t.Fatalf("unexpected error: %s", err) + } + + ensureContents(t, zipFilePath, map[string][]byte{ + "test-dir/test-dir1/file3.txt": []byte("This is file 3"), + "test-dir/test-dir2/file3.txt": []byte("This is file 3"), + "test-dir/test-file.txt": []byte("This is test content"), + "test-dir-with-symlink-file/test-file.txt": []byte("This is test content"), + "test-dir-with-symlink-file/test-symlink.txt": []byte("This is test content"), + "test-symlink-dir/file3.txt": []byte("This is file 3"), + "test-symlink-dir-with-symlink-file/test-file.txt": []byte("This is test content"), + }) +} + func TestZipArchiver_Dir_Exclude_ExcludeSymlinkDirectories(t *testing.T) { zipFilePath := filepath.Join(t.TempDir(), "archive-dir-with-symlink-dir.zip") @@ -269,6 +295,40 @@ func TestZipArchiver_Dir_Exclude_ExcludeSymlinkDirectories(t *testing.T) { if err != nil { t.Errorf("expected no error: %s", err) } + + ensureContents(t, zipFilePath, map[string][]byte{ + "test-dir/test-dir1/file2.txt": []byte("This is file 2"), + "test-dir/test-dir1/file3.txt": []byte("This is file 3"), + "test-dir/test-dir2/file1.txt": []byte("This is file 1"), + "test-dir/test-dir2/file2.txt": []byte("This is file 2"), + "test-dir/test-dir2/file3.txt": []byte("This is file 3"), + "test-dir/test-file.txt": []byte("This is test content"), + "test-dir-with-symlink-file/test-file.txt": []byte("This is test content"), + "test-dir-with-symlink-file/test-symlink.txt": []byte("This is test content"), + }) +} + +func TestZipArchiver_Dir_Exclude_Glob_ExcludeSymlinkDirectories(t *testing.T) { + zipFilePath := filepath.Join(t.TempDir(), "archive-dir-with-symlink-dir.zip") + + archiver := NewZipArchiver(zipFilePath) + err := archiver.ArchiveDir("./test-fixtures", ArchiveDirOpts{ + Excludes: []string{ + "test-dir/test-dir1/file1.txt", + "**/file[2-3].txt", + "test-dir-with-symlink-file", + }, + ExcludeSymlinkDirectories: true, + }) + + if err != nil { + t.Errorf("expected no error: %s", err) + } + + ensureContents(t, zipFilePath, map[string][]byte{ + "test-dir/test-dir2/file1.txt": []byte("This is file 1"), + "test-dir/test-file.txt": []byte("This is test content"), + }) } func ensureContents(t *testing.T, zipfilepath string, wants map[string][]byte) { From fc5b8aa48e0e3eb7567a6681db8691e0123a4318 Mon Sep 17 00:00:00 2001 From: Selena Goods Date: Wed, 24 Jul 2024 15:30:43 -0400 Subject: [PATCH 2/8] Regenerate provider documentation --- docs/data-sources/file.md | 2 +- docs/resources/file.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/data-sources/file.md b/docs/data-sources/file.md index 19bde89d..c58dd024 100644 --- a/docs/data-sources/file.md +++ b/docs/data-sources/file.md @@ -63,7 +63,7 @@ data "archive_file" "lambda_my_function" { ### Optional - `exclude_symlink_directories` (Boolean) Boolean flag indicating whether symbolically linked directories should be excluded during the creation of the archive. Defaults to `false`. -- `excludes` (Set of String) Specify files to ignore when reading the `source_dir`. +- `excludes` (Set of String) Specify files/directories to ignore when reading the `source_dir`. Supports glob file matching patterns including doublestar/globstar (`**`) patterns - `output_file_mode` (String) String that specifies the octal file mode for all archived files. For example: `"0666"`. Setting this will ensure that cross platform usage of this module will not vary the modes of archived files (and ultimately checksums) resulting in more deterministic behavior. - `source` (Block Set) Specifies attributes of a single source file to include into the archive. One and only one of `source`, `source_content_filename` (with `source_content`), `source_file`, or `source_dir` must be specified. (see [below for nested schema](#nestedblock--source)) - `source_content` (String) Add only this content to the archive with `source_content_filename` as the filename. One and only one of `source`, `source_content_filename` (with `source_content`), `source_file`, or `source_dir` must be specified. diff --git a/docs/resources/file.md b/docs/resources/file.md index 00000a8f..bbbe85f6 100644 --- a/docs/resources/file.md +++ b/docs/resources/file.md @@ -23,7 +23,7 @@ description: |- ### Optional - `exclude_symlink_directories` (Boolean) Boolean flag indicating whether symbolically linked directories should be excluded during the creation of the archive. Defaults to `false`. -- `excludes` (Set of String) Specify files to ignore when reading the `source_dir`. +- `excludes` (Set of String) Specify files/directories to ignore when reading the `source_dir`. Supports glob file matching patterns including doublestar/globstar (`**`) patterns - `output_file_mode` (String) String that specifies the octal file mode for all archived files. For example: `"0666"`. Setting this will ensure that cross platform usage of this module will not vary the modes of archived files (and ultimately checksums) resulting in more deterministic behavior. - `source` (Block Set) Specifies attributes of a single source file to include into the archive. One and only one of `source`, `source_content_filename` (with `source_content`), `source_file`, or `source_dir` must be specified. (see [below for nested schema](#nestedblock--source)) - `source_content` (String) Add only this content to the archive with `source_content_filename` as the filename. One and only one of `source`, `source_content_filename` (with `source_content`), `source_file`, or `source_dir` must be specified. From 75294243f27c2ffd1b4d66fed3c63aaefc5a3784 Mon Sep 17 00:00:00 2001 From: Selena Goods Date: Wed, 24 Jul 2024 15:58:24 -0400 Subject: [PATCH 3/8] Switch from `Match()` to `PathMatch()` to be OS agnostic. --- internal/provider/zip_archiver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/provider/zip_archiver.go b/internal/provider/zip_archiver.go index 80dc47ce..0a0ae7cd 100644 --- a/internal/provider/zip_archiver.go +++ b/internal/provider/zip_archiver.go @@ -91,7 +91,7 @@ func checkMatch(fileName string, excludes []string) (value bool, err error) { continue } - match, err := doublestar.Match(exclude, fileName) + match, err := doublestar.PathMatch(exclude, fileName) if err != nil { return false, err } From c315257cfd729c1345ab86b1cbf2ac14aaf050c2 Mon Sep 17 00:00:00 2001 From: Selena Goods Date: Wed, 24 Jul 2024 16:08:48 -0400 Subject: [PATCH 4/8] Add changelog entries --- .changes/unreleased/ENHANCEMENTS-20240724-160724.yaml | 5 +++++ .changes/unreleased/ENHANCEMENTS-20240724-160830.yaml | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .changes/unreleased/ENHANCEMENTS-20240724-160724.yaml create mode 100644 .changes/unreleased/ENHANCEMENTS-20240724-160830.yaml diff --git a/.changes/unreleased/ENHANCEMENTS-20240724-160724.yaml b/.changes/unreleased/ENHANCEMENTS-20240724-160724.yaml new file mode 100644 index 00000000..88a63740 --- /dev/null +++ b/.changes/unreleased/ENHANCEMENTS-20240724-160724.yaml @@ -0,0 +1,5 @@ +kind: ENHANCEMENTS +body: 'data-source/archive_file: Add glob pattern matching support to the `excludes` attribute.' +time: 2024-07-24T16:07:24.058378-04:00 +custom: + Issue: "354" diff --git a/.changes/unreleased/ENHANCEMENTS-20240724-160830.yaml b/.changes/unreleased/ENHANCEMENTS-20240724-160830.yaml new file mode 100644 index 00000000..123816d4 --- /dev/null +++ b/.changes/unreleased/ENHANCEMENTS-20240724-160830.yaml @@ -0,0 +1,5 @@ +kind: ENHANCEMENTS +body: 'resource/archive_file: Add glob pattern matching support to the `excludes` attribute.' +time: 2024-07-24T16:08:30.211413-04:00 +custom: + Issue: "354" From e35bc52061d2f1bd5c68e764f87e56b23b1f1f2b Mon Sep 17 00:00:00 2001 From: Selena Goods Date: Wed, 24 Jul 2024 16:11:53 -0400 Subject: [PATCH 5/8] Add missing punctuation. --- internal/provider/data_source_archive_file.go | 2 +- internal/provider/resource_archive_file.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/provider/data_source_archive_file.go b/internal/provider/data_source_archive_file.go index 44f4c20a..507d1dd5 100644 --- a/internal/provider/data_source_archive_file.go +++ b/internal/provider/data_source_archive_file.go @@ -135,7 +135,7 @@ func (d *archiveFileDataSource) Schema(ctx context.Context, req datasource.Schem }, "excludes": schema.SetAttribute{ Description: "Specify files/directories to ignore when reading the `source_dir`. " + - "Supports glob file matching patterns including doublestar/globstar (`**`) patterns", + "Supports glob file matching patterns including doublestar/globstar (`**`) patterns.", ElementType: types.StringType, Optional: true, Validators: []validator.Set{ diff --git a/internal/provider/resource_archive_file.go b/internal/provider/resource_archive_file.go index 04bcfdc8..50e215c2 100644 --- a/internal/provider/resource_archive_file.go +++ b/internal/provider/resource_archive_file.go @@ -155,7 +155,7 @@ func (d *archiveFileResource) Schema(ctx context.Context, req resource.SchemaReq }, "excludes": schema.SetAttribute{ Description: "Specify files/directories to ignore when reading the `source_dir`. " + - "Supports glob file matching patterns including doublestar/globstar (`**`) patterns", + "Supports glob file matching patterns including doublestar/globstar (`**`) patterns.", ElementType: types.StringType, Optional: true, Validators: []validator.Set{ From 83e59b0edfd3feed9744f68885e67966d89d469a Mon Sep 17 00:00:00 2001 From: Selena Goods Date: Wed, 24 Jul 2024 16:14:01 -0400 Subject: [PATCH 6/8] Regenerate provider documentation. --- docs/data-sources/file.md | 2 +- docs/resources/file.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/data-sources/file.md b/docs/data-sources/file.md index c58dd024..f390d0c9 100644 --- a/docs/data-sources/file.md +++ b/docs/data-sources/file.md @@ -63,7 +63,7 @@ data "archive_file" "lambda_my_function" { ### Optional - `exclude_symlink_directories` (Boolean) Boolean flag indicating whether symbolically linked directories should be excluded during the creation of the archive. Defaults to `false`. -- `excludes` (Set of String) Specify files/directories to ignore when reading the `source_dir`. Supports glob file matching patterns including doublestar/globstar (`**`) patterns +- `excludes` (Set of String) Specify files/directories to ignore when reading the `source_dir`. Supports glob file matching patterns including doublestar/globstar (`**`) patterns. - `output_file_mode` (String) String that specifies the octal file mode for all archived files. For example: `"0666"`. Setting this will ensure that cross platform usage of this module will not vary the modes of archived files (and ultimately checksums) resulting in more deterministic behavior. - `source` (Block Set) Specifies attributes of a single source file to include into the archive. One and only one of `source`, `source_content_filename` (with `source_content`), `source_file`, or `source_dir` must be specified. (see [below for nested schema](#nestedblock--source)) - `source_content` (String) Add only this content to the archive with `source_content_filename` as the filename. One and only one of `source`, `source_content_filename` (with `source_content`), `source_file`, or `source_dir` must be specified. diff --git a/docs/resources/file.md b/docs/resources/file.md index bbbe85f6..dcee9d05 100644 --- a/docs/resources/file.md +++ b/docs/resources/file.md @@ -23,7 +23,7 @@ description: |- ### Optional - `exclude_symlink_directories` (Boolean) Boolean flag indicating whether symbolically linked directories should be excluded during the creation of the archive. Defaults to `false`. -- `excludes` (Set of String) Specify files/directories to ignore when reading the `source_dir`. Supports glob file matching patterns including doublestar/globstar (`**`) patterns +- `excludes` (Set of String) Specify files/directories to ignore when reading the `source_dir`. Supports glob file matching patterns including doublestar/globstar (`**`) patterns. - `output_file_mode` (String) String that specifies the octal file mode for all archived files. For example: `"0666"`. Setting this will ensure that cross platform usage of this module will not vary the modes of archived files (and ultimately checksums) resulting in more deterministic behavior. - `source` (Block Set) Specifies attributes of a single source file to include into the archive. One and only one of `source`, `source_content_filename` (with `source_content`), `source_file`, or `source_dir` must be specified. (see [below for nested schema](#nestedblock--source)) - `source_content` (String) Add only this content to the archive with `source_content_filename` as the filename. One and only one of `source`, `source_content_filename` (with `source_content`), `source_file`, or `source_dir` must be specified. From 133b6727f744accce4ccc125e6578c87f81c18cd Mon Sep 17 00:00:00 2001 From: Selena Goods Date: Tue, 30 Jul 2024 14:49:06 -0400 Subject: [PATCH 7/8] Update internal/provider/zip_archiver.go Co-authored-by: Austin Valle --- internal/provider/zip_archiver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/provider/zip_archiver.go b/internal/provider/zip_archiver.go index 0a0ae7cd..303fa465 100644 --- a/internal/provider/zip_archiver.go +++ b/internal/provider/zip_archiver.go @@ -150,7 +150,7 @@ func (a *ZipArchiver) createWalkFunc(basePath, indirname string, opts ArchiveDir isMatch, err := checkMatch(archivePath, opts.Excludes) if err != nil { - return fmt.Errorf("error checking excludes matches: %s", err) + return fmt.Errorf("error checking excludes matches: %w", err) } if info.IsDir() { From 51c00cc511e520daf5e088ee7ca197be3d55da5f Mon Sep 17 00:00:00 2001 From: Selena Goods Date: Tue, 30 Jul 2024 15:04:59 -0400 Subject: [PATCH 8/8] Add acceptance tests --- .../provider/data_source_archive_file_test.go | 18 ++++++++++++++++++ .../provider/resource_archive_file_test.go | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/internal/provider/data_source_archive_file_test.go b/internal/provider/data_source_archive_file_test.go index 3a78a59d..d9afe27f 100644 --- a/internal/provider/data_source_archive_file_test.go +++ b/internal/provider/data_source_archive_file_test.go @@ -105,6 +105,13 @@ func TestAccArchiveFile_Basic(t *testing.T) { r.TestCheckResourceAttrPtr("data.archive_file.foo", "output_size", &fileSize), ), }, + { + Config: testAccArchiveFileDirExcludesGlobConfig(f), + Check: r.ComposeTestCheckFunc( + testAccArchiveFileSize(f, &fileSize), + r.TestCheckResourceAttrPtr("data.archive_file.foo", "output_size", &fileSize), + ), + }, { Config: testAccArchiveFileMultiSourceConfig(f), Check: r.ComposeTestCheckFunc( @@ -1389,6 +1396,17 @@ data "archive_file" "foo" { `, filepath.ToSlash(outputPath)) } +func testAccArchiveFileDirExcludesGlobConfig(outputPath string) string { + return fmt.Sprintf(` +data "archive_file" "foo" { + type = "zip" + source_dir = "test-fixtures/test-dir/test-dir1" + excludes = ["test-fixtures/test-dir/test-dir1/file2.txt", "**/file[2-3].txt"] + output_path = "%s" +} +`, filepath.ToSlash(outputPath)) +} + func testAccArchiveFileMultiSourceConfig(outputPath string) string { return fmt.Sprintf(` data "archive_file" "foo" { diff --git a/internal/provider/resource_archive_file_test.go b/internal/provider/resource_archive_file_test.go index 84f6c71d..8c815006 100644 --- a/internal/provider/resource_archive_file_test.go +++ b/internal/provider/resource_archive_file_test.go @@ -104,6 +104,13 @@ func TestAccArchiveFile_Resource_Basic(t *testing.T) { r.TestCheckResourceAttrPtr("archive_file.foo", "output_size", &fileSize), ), }, + { + Config: testAccArchiveFileResourceDirExcludesGlobConfig(f), + Check: r.ComposeTestCheckFunc( + testAccArchiveFileSize(f, &fileSize), + r.TestCheckResourceAttrPtr("archive_file.foo", "output_size", &fileSize), + ), + }, { Config: testAccArchiveFileResourceMultiSourceConfig(f), Check: r.ComposeTestCheckFunc( @@ -1485,6 +1492,17 @@ resource "archive_file" "foo" { `, filepath.ToSlash(outputPath)) } +func testAccArchiveFileResourceDirExcludesGlobConfig(outputPath string) string { + return fmt.Sprintf(` +resource "archive_file" "foo" { + type = "zip" + source_dir = "test-fixtures/test-dir" + excludes = ["test-fixtures/test-dir/file2.txt", "**/file[2-3].txt"] + output_path = "%s" +} +`, filepath.ToSlash(outputPath)) +} + func testAccArchiveFileResourceMultiSourceConfig(outputPath string) string { return fmt.Sprintf(` resource "archive_file" "foo" {