From cb367027d756676f3e320ccf92bdbd526f62b03c Mon Sep 17 00:00:00 2001 From: Selena Goods Date: Tue, 30 Jul 2024 15:05:58 -0400 Subject: [PATCH] Add glob file pattern matching support for the `excludes` attribute. (#354) * Add glob pattern matching support for the `excludes` attribute. * Regenerate provider documentation * Switch from `Match()` to `PathMatch()` to be OS agnostic. * Add changelog entries * Add missing punctuation. * Regenerate provider documentation. * Update internal/provider/zip_archiver.go Co-authored-by: Austin Valle * Add acceptance tests --------- Co-authored-by: Austin Valle --- .../ENHANCEMENTS-20240724-160724.yaml | 5 ++ .../ENHANCEMENTS-20240724-160830.yaml | 5 ++ docs/data-sources/file.md | 2 +- docs/resources/file.md | 2 +- go.mod | 1 + go.sum | 2 + internal/provider/data_source_archive_file.go | 3 +- .../provider/data_source_archive_file_test.go | 18 ++++++ internal/provider/resource_archive_file.go | 3 +- .../provider/resource_archive_file_test.go | 18 ++++++ internal/provider/zip_archiver.go | 24 +++++--- internal/provider/zip_archiver_test.go | 60 +++++++++++++++++++ 12 files changed, 130 insertions(+), 13 deletions(-) create mode 100644 .changes/unreleased/ENHANCEMENTS-20240724-160724.yaml create mode 100644 .changes/unreleased/ENHANCEMENTS-20240724-160830.yaml diff --git a/.changes/unreleased/ENHANCEMENTS-20240724-160724.yaml b/.changes/unreleased/ENHANCEMENTS-20240724-160724.yaml new file mode 100644 index 00000000..88a63740 --- /dev/null +++ b/.changes/unreleased/ENHANCEMENTS-20240724-160724.yaml @@ -0,0 +1,5 @@ +kind: ENHANCEMENTS +body: 'data-source/archive_file: Add glob pattern matching support to the `excludes` attribute.' +time: 2024-07-24T16:07:24.058378-04:00 +custom: + Issue: "354" diff --git a/.changes/unreleased/ENHANCEMENTS-20240724-160830.yaml b/.changes/unreleased/ENHANCEMENTS-20240724-160830.yaml new file mode 100644 index 00000000..123816d4 --- /dev/null +++ b/.changes/unreleased/ENHANCEMENTS-20240724-160830.yaml @@ -0,0 +1,5 @@ +kind: ENHANCEMENTS +body: 'resource/archive_file: Add glob pattern matching support to the `excludes` attribute.' +time: 2024-07-24T16:08:30.211413-04:00 +custom: + Issue: "354" diff --git a/docs/data-sources/file.md b/docs/data-sources/file.md index 19bde89d..f390d0c9 100644 --- a/docs/data-sources/file.md +++ b/docs/data-sources/file.md @@ -63,7 +63,7 @@ data "archive_file" "lambda_my_function" { ### Optional - `exclude_symlink_directories` (Boolean) Boolean flag indicating whether symbolically linked directories should be excluded during the creation of the archive. Defaults to `false`. -- `excludes` (Set of String) Specify files to ignore when reading the `source_dir`. +- `excludes` (Set of String) Specify files/directories to ignore when reading the `source_dir`. Supports glob file matching patterns including doublestar/globstar (`**`) patterns. - `output_file_mode` (String) String that specifies the octal file mode for all archived files. For example: `"0666"`. Setting this will ensure that cross platform usage of this module will not vary the modes of archived files (and ultimately checksums) resulting in more deterministic behavior. - `source` (Block Set) Specifies attributes of a single source file to include into the archive. One and only one of `source`, `source_content_filename` (with `source_content`), `source_file`, or `source_dir` must be specified. (see [below for nested schema](#nestedblock--source)) - `source_content` (String) Add only this content to the archive with `source_content_filename` as the filename. One and only one of `source`, `source_content_filename` (with `source_content`), `source_file`, or `source_dir` must be specified. diff --git a/docs/resources/file.md b/docs/resources/file.md index 00000a8f..dcee9d05 100644 --- a/docs/resources/file.md +++ b/docs/resources/file.md @@ -23,7 +23,7 @@ description: |- ### Optional - `exclude_symlink_directories` (Boolean) Boolean flag indicating whether symbolically linked directories should be excluded during the creation of the archive. Defaults to `false`. -- `excludes` (Set of String) Specify files to ignore when reading the `source_dir`. +- `excludes` (Set of String) Specify files/directories to ignore when reading the `source_dir`. Supports glob file matching patterns including doublestar/globstar (`**`) patterns. - `output_file_mode` (String) String that specifies the octal file mode for all archived files. For example: `"0666"`. Setting this will ensure that cross platform usage of this module will not vary the modes of archived files (and ultimately checksums) resulting in more deterministic behavior. - `source` (Block Set) Specifies attributes of a single source file to include into the archive. One and only one of `source`, `source_content_filename` (with `source_content`), `source_file`, or `source_dir` must be specified. (see [below for nested schema](#nestedblock--source)) - `source_content` (String) Add only this content to the archive with `source_content_filename` as the filename. One and only one of `source`, `source_content_filename` (with `source_content`), `source_file`, or `source_dir` must be specified. diff --git a/go.mod b/go.mod index 3a150442..5ce46219 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 toolchain go1.21.3 require ( + github.com/bmatcuk/doublestar/v4 v4.6.1 github.com/hashicorp/terraform-plugin-framework v1.10.0 github.com/hashicorp/terraform-plugin-framework-validators v0.13.0 github.com/hashicorp/terraform-plugin-go v0.23.0 diff --git a/go.sum b/go.sum index b6a12c66..afd408fc 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/agext/levenshtein v1.2.2/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJEynmyUenKwP++x/+DdGV/Ec= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= +github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I= +github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= diff --git a/internal/provider/data_source_archive_file.go b/internal/provider/data_source_archive_file.go index 5a21511b..507d1dd5 100644 --- a/internal/provider/data_source_archive_file.go +++ b/internal/provider/data_source_archive_file.go @@ -134,7 +134,8 @@ func (d *archiveFileDataSource) Schema(ctx context.Context, req datasource.Schem }, }, "excludes": schema.SetAttribute{ - Description: "Specify files to ignore when reading the `source_dir`.", + Description: "Specify files/directories to ignore when reading the `source_dir`. " + + "Supports glob file matching patterns including doublestar/globstar (`**`) patterns.", ElementType: types.StringType, Optional: true, Validators: []validator.Set{ diff --git a/internal/provider/data_source_archive_file_test.go b/internal/provider/data_source_archive_file_test.go index 3a78a59d..d9afe27f 100644 --- a/internal/provider/data_source_archive_file_test.go +++ b/internal/provider/data_source_archive_file_test.go @@ -105,6 +105,13 @@ func TestAccArchiveFile_Basic(t *testing.T) { r.TestCheckResourceAttrPtr("data.archive_file.foo", "output_size", &fileSize), ), }, + { + Config: testAccArchiveFileDirExcludesGlobConfig(f), + Check: r.ComposeTestCheckFunc( + testAccArchiveFileSize(f, &fileSize), + r.TestCheckResourceAttrPtr("data.archive_file.foo", "output_size", &fileSize), + ), + }, { Config: testAccArchiveFileMultiSourceConfig(f), Check: r.ComposeTestCheckFunc( @@ -1389,6 +1396,17 @@ data "archive_file" "foo" { `, filepath.ToSlash(outputPath)) } +func testAccArchiveFileDirExcludesGlobConfig(outputPath string) string { + return fmt.Sprintf(` +data "archive_file" "foo" { + type = "zip" + source_dir = "test-fixtures/test-dir/test-dir1" + excludes = ["test-fixtures/test-dir/test-dir1/file2.txt", "**/file[2-3].txt"] + output_path = "%s" +} +`, filepath.ToSlash(outputPath)) +} + func testAccArchiveFileMultiSourceConfig(outputPath string) string { return fmt.Sprintf(` data "archive_file" "foo" { diff --git a/internal/provider/resource_archive_file.go b/internal/provider/resource_archive_file.go index 9f2beb5d..50e215c2 100644 --- a/internal/provider/resource_archive_file.go +++ b/internal/provider/resource_archive_file.go @@ -154,7 +154,8 @@ func (d *archiveFileResource) Schema(ctx context.Context, req resource.SchemaReq }, }, "excludes": schema.SetAttribute{ - Description: "Specify files to ignore when reading the `source_dir`.", + Description: "Specify files/directories to ignore when reading the `source_dir`. " + + "Supports glob file matching patterns including doublestar/globstar (`**`) patterns.", ElementType: types.StringType, Optional: true, Validators: []validator.Set{ diff --git a/internal/provider/resource_archive_file_test.go b/internal/provider/resource_archive_file_test.go index 84f6c71d..8c815006 100644 --- a/internal/provider/resource_archive_file_test.go +++ b/internal/provider/resource_archive_file_test.go @@ -104,6 +104,13 @@ func TestAccArchiveFile_Resource_Basic(t *testing.T) { r.TestCheckResourceAttrPtr("archive_file.foo", "output_size", &fileSize), ), }, + { + Config: testAccArchiveFileResourceDirExcludesGlobConfig(f), + Check: r.ComposeTestCheckFunc( + testAccArchiveFileSize(f, &fileSize), + r.TestCheckResourceAttrPtr("archive_file.foo", "output_size", &fileSize), + ), + }, { Config: testAccArchiveFileResourceMultiSourceConfig(f), Check: r.ComposeTestCheckFunc( @@ -1485,6 +1492,17 @@ resource "archive_file" "foo" { `, filepath.ToSlash(outputPath)) } +func testAccArchiveFileResourceDirExcludesGlobConfig(outputPath string) string { + return fmt.Sprintf(` +resource "archive_file" "foo" { + type = "zip" + source_dir = "test-fixtures/test-dir" + excludes = ["test-fixtures/test-dir/file2.txt", "**/file[2-3].txt"] + output_path = "%s" +} +`, filepath.ToSlash(outputPath)) +} + func testAccArchiveFileResourceMultiSourceConfig(outputPath string) string { return fmt.Sprintf(` resource "archive_file" "foo" { diff --git a/internal/provider/zip_archiver.go b/internal/provider/zip_archiver.go index d4c00121..303fa465 100644 --- a/internal/provider/zip_archiver.go +++ b/internal/provider/zip_archiver.go @@ -11,6 +11,8 @@ import ( "sort" "strconv" "time" + + "github.com/bmatcuk/doublestar/v4" ) type ZipArchiver struct { @@ -83,17 +85,22 @@ func (a *ZipArchiver) ArchiveFile(infilename string) error { return err } -func checkMatch(fileName string, excludes []string) (value bool) { +func checkMatch(fileName string, excludes []string) (value bool, err error) { for _, exclude := range excludes { if exclude == "" { continue } - if exclude == fileName { - return true + match, err := doublestar.PathMatch(exclude, fileName) + if err != nil { + return false, err + } + + if match { + return true, nil } } - return false + return false, nil } func (a *ZipArchiver) ArchiveDir(indirname string, opts ArchiveDirOpts) error { @@ -141,7 +148,10 @@ func (a *ZipArchiver) createWalkFunc(basePath, indirname string, opts ArchiveDir archivePath := filepath.Join(basePath, relname) - isMatch := checkMatch(archivePath, opts.Excludes) + isMatch, err := checkMatch(archivePath, opts.Excludes) + if err != nil { + return fmt.Errorf("error checking excludes matches: %w", err) + } if info.IsDir() { if isMatch { @@ -154,10 +164,6 @@ func (a *ZipArchiver) createWalkFunc(basePath, indirname string, opts ArchiveDir return nil } - if err != nil { - return err - } - if info.Mode()&os.ModeSymlink == os.ModeSymlink { realPath, err := filepath.EvalSymlinks(path) if err != nil { diff --git a/internal/provider/zip_archiver_test.go b/internal/provider/zip_archiver_test.go index a46de0d8..d881e5b9 100644 --- a/internal/provider/zip_archiver_test.go +++ b/internal/provider/zip_archiver_test.go @@ -254,6 +254,32 @@ func TestZipArchiver_Dir_Exclude_DoNotExcludeSymlinkDirectories(t *testing.T) { }) } +func TestZipArchiver_Dir_Exclude_Glob_DoNotExcludeSymlinkDirectories(t *testing.T) { + zipFilePath := filepath.Join(t.TempDir(), "archive-dir-with-symlink-dir.zip") + + archiver := NewZipArchiver(zipFilePath) + if err := archiver.ArchiveDir("./test-fixtures", ArchiveDirOpts{ + Excludes: []string{ + "**/file1.txt", + "**/file2.*", + "test-dir-with-symlink-dir/test-symlink-dir", + "test-symlink-dir-with-symlink-file/test-symlink.txt", + }, + }); err != nil { + t.Fatalf("unexpected error: %s", err) + } + + ensureContents(t, zipFilePath, map[string][]byte{ + "test-dir/test-dir1/file3.txt": []byte("This is file 3"), + "test-dir/test-dir2/file3.txt": []byte("This is file 3"), + "test-dir/test-file.txt": []byte("This is test content"), + "test-dir-with-symlink-file/test-file.txt": []byte("This is test content"), + "test-dir-with-symlink-file/test-symlink.txt": []byte("This is test content"), + "test-symlink-dir/file3.txt": []byte("This is file 3"), + "test-symlink-dir-with-symlink-file/test-file.txt": []byte("This is test content"), + }) +} + func TestZipArchiver_Dir_Exclude_ExcludeSymlinkDirectories(t *testing.T) { zipFilePath := filepath.Join(t.TempDir(), "archive-dir-with-symlink-dir.zip") @@ -269,6 +295,40 @@ func TestZipArchiver_Dir_Exclude_ExcludeSymlinkDirectories(t *testing.T) { if err != nil { t.Errorf("expected no error: %s", err) } + + ensureContents(t, zipFilePath, map[string][]byte{ + "test-dir/test-dir1/file2.txt": []byte("This is file 2"), + "test-dir/test-dir1/file3.txt": []byte("This is file 3"), + "test-dir/test-dir2/file1.txt": []byte("This is file 1"), + "test-dir/test-dir2/file2.txt": []byte("This is file 2"), + "test-dir/test-dir2/file3.txt": []byte("This is file 3"), + "test-dir/test-file.txt": []byte("This is test content"), + "test-dir-with-symlink-file/test-file.txt": []byte("This is test content"), + "test-dir-with-symlink-file/test-symlink.txt": []byte("This is test content"), + }) +} + +func TestZipArchiver_Dir_Exclude_Glob_ExcludeSymlinkDirectories(t *testing.T) { + zipFilePath := filepath.Join(t.TempDir(), "archive-dir-with-symlink-dir.zip") + + archiver := NewZipArchiver(zipFilePath) + err := archiver.ArchiveDir("./test-fixtures", ArchiveDirOpts{ + Excludes: []string{ + "test-dir/test-dir1/file1.txt", + "**/file[2-3].txt", + "test-dir-with-symlink-file", + }, + ExcludeSymlinkDirectories: true, + }) + + if err != nil { + t.Errorf("expected no error: %s", err) + } + + ensureContents(t, zipFilePath, map[string][]byte{ + "test-dir/test-dir2/file1.txt": []byte("This is file 1"), + "test-dir/test-file.txt": []byte("This is test content"), + }) } func ensureContents(t *testing.T, zipfilepath string, wants map[string][]byte) {