diff --git a/archive/archiver.go b/archive/archiver.go deleted file mode 100644 index 7491f98b..00000000 --- a/archive/archiver.go +++ /dev/null @@ -1,48 +0,0 @@ -package archive - -import ( - "fmt" - "os" -) - -type Archiver interface { - ArchiveContent(content []byte, infilename string) error - ArchiveFile(infilename string) error - ArchiveDir(indirname string, excludes []string) error - ArchiveMultiple(content map[string][]byte) error -} - -type ArchiverBuilder func(filepath string) Archiver - -var archiverBuilders = map[string]ArchiverBuilder{ - "zip": NewZipArchiver, -} - -func getArchiver(archiveType string, filepath string) Archiver { - if builder, ok := archiverBuilders[archiveType]; ok { - return builder(filepath) - } - return nil -} - -func assertValidFile(infilename string) (os.FileInfo, error) { - fi, err := os.Stat(infilename) - if err != nil && os.IsNotExist(err) { - return fi, fmt.Errorf("could not archive missing file: %s", infilename) - } - return fi, err -} - -func assertValidDir(indirname string) (os.FileInfo, error) { - fi, err := os.Stat(indirname) - if err != nil { - if os.IsNotExist(err) { - return fi, fmt.Errorf("could not archive missing directory: %s", indirname) - } - return fi, err - } - if !fi.IsDir() { - return fi, fmt.Errorf("could not archive directory that is a file: %s", indirname) - } - return fi, nil -} diff --git a/archive/data_source_archive_file.go b/archive/data_source_archive_file.go index a53fbdce..5b6379af 100644 --- a/archive/data_source_archive_file.go +++ b/archive/data_source_archive_file.go @@ -11,9 +11,13 @@ import ( "io/ioutil" "os" "path" + "path/filepath" + "reflect" "github.com/hashicorp/terraform/helper/hashcode" "github.com/hashicorp/terraform/helper/schema" + "github.com/karrick/godirwalk" + "github.com/mholt/archiver" ) func dataSourceFile() *schema.Resource { @@ -22,14 +26,13 @@ func dataSourceFile() *schema.Resource { Schema: map[string]*schema.Schema{ "type": { - Type: schema.TypeString, - Required: true, - ForceNew: true, + Type: schema.TypeString, + Optional: true, + Deprecated: "Archive type is now determined from the output_path's file extenstion.", }, "source": { Type: schema.TypeSet, Optional: true, - Computed: true, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "content": { @@ -89,6 +92,7 @@ func dataSourceFile() *schema.Resource { "output_path": { Type: schema.TypeString, Required: true, + ForceNew: true, }, "output_size": { Type: schema.TypeInt, @@ -162,36 +166,77 @@ func expandStringList(configured []interface{}) []string { return vs } +func checkMatch(fileName string, excludes []string) (value bool) { + for _, exclude := range excludes { + if exclude == "" { + continue + } + + if exclude == fileName { + return true + } + } + return false +} + +func getFileList(dirName string, excludes []string) ([]string, error) { + var files []string + + err := godirwalk.Walk(dirName, &godirwalk.Options{ + Callback: func(osPathname string, de *godirwalk.Dirent) error { + relname, err := filepath.Rel(dirName, osPathname) + if err != nil { + return nil + } + + if checkMatch(relname, []string{".", ".."}) { + return nil + } + + shouldExclude := checkMatch(relname, excludes) + fullName := filepath.FromSlash(fmt.Sprintf("%s/%s", dirName, relname)) + + if de.IsDir() { + if shouldExclude { + return filepath.SkipDir + } + } else if shouldExclude { + return nil + } + + files = append(files, fullName) + return nil + }, + }) + + return files, err +} + func archive(d *schema.ResourceData) error { - archiveType := d.Get("type").(string) outputPath := d.Get("output_path").(string) + var filesToArchive []string - archiver := getArchiver(archiveType, outputPath) - if archiver == nil { - return fmt.Errorf("archive type not supported: %s", archiveType) + compressor := archiver.MatchingFormat(outputPath) + if compressor == nil { + return fmt.Errorf("cannot compress unsupported file type: %s", outputPath) } + var err error + if dir, ok := d.GetOk("source_dir"); ok { + var excludeList []string if excludes, ok := d.GetOk("excludes"); ok { - excludeList := expandStringList(excludes.(*schema.Set).List()) + excludeList = expandStringList(excludes.(*schema.Set).List()) + } - if err := archiver.ArchiveDir(dir.(string), excludeList); err != nil { - return fmt.Errorf("error archiving directory: %s", err) - } - } else { - if err := archiver.ArchiveDir(dir.(string), []string{""}); err != nil { - return fmt.Errorf("error archiving directory: %s", err) - } + filesToArchive, err = getFileList(dir.(string), excludeList) + if err != nil { + return fmt.Errorf("could not walk dir: %s", dir.(string)) } } else if file, ok := d.GetOk("source_file"); ok { - if err := archiver.ArchiveFile(file.(string)); err != nil { - return fmt.Errorf("error archiving file: %s", err) - } - } else if filename, ok := d.GetOk("source_content_filename"); ok { - content := d.Get("source_content").(string) - if err := archiver.ArchiveContent([]byte(content), filename.(string)); err != nil { - return fmt.Errorf("error archiving content: %s", err) - } + filesToArchive = append(filesToArchive, file.(string)) + } else if fileName, ok := d.GetOk("source_content_filename"); ok { + return fmt.Errorf("source_content not supported for %s", fileName) } else if v, ok := d.GetOk("source"); ok { vL := v.(*schema.Set).List() content := make(map[string][]byte) @@ -199,12 +244,18 @@ func archive(d *schema.ResourceData) error { src := v.(map[string]interface{}) content[src["filename"].(string)] = []byte(src["content"].(string)) } - if err := archiver.ArchiveMultiple(content); err != nil { - return fmt.Errorf("error archiving content: %s", err) - } + + keys := reflect.ValueOf(content).MapKeys() + return fmt.Errorf("cannot compress %d source blocks", len(keys)) } else { return fmt.Errorf("one of 'source_dir', 'source_file', 'source_content_filename' must be specified") } + + check := compressor.Make(outputPath, filesToArchive) + if check != nil { + return fmt.Errorf("could not archive to %s: %s", outputPath, check) + } + return nil } diff --git a/archive/data_source_archive_file_test.go b/archive/data_source_archive_file_test.go index 3f7c7115..27e8ce1d 100644 --- a/archive/data_source_archive_file_test.go +++ b/archive/data_source_archive_file_test.go @@ -10,13 +10,13 @@ import ( "github.com/hashicorp/terraform/terraform" ) -func TestAccArchiveFile_Basic(t *testing.T) { +func TestAccArchiveFile_SourceContent(t *testing.T) { var fileSize string r.Test(t, r.TestCase{ Providers: testProviders, Steps: []r.TestStep{ { - Config: testAccArchiveFileContentConfig, + Config: testAccArchiveFileOutputPath, Check: r.ComposeTestCheckFunc( testAccArchiveFileExists("zip_file_acc_test.zip", &fileSize), r.TestCheckResourceAttrPtr("data.archive_file.foo", "output_size", &fileSize), @@ -37,27 +37,54 @@ func TestAccArchiveFile_Basic(t *testing.T) { ), ), }, + }, + }) +} + +func TestAccArchiveFile_SourceDir(t *testing.T) { + var fileSize string + r.Test(t, r.TestCase{ + Providers: testProviders, + Steps: []r.TestStep{ { - Config: testAccArchiveFileFileConfig, + Config: testAccArchiveFileDirConfig, Check: r.ComposeTestCheckFunc( testAccArchiveFileExists("zip_file_acc_test.zip", &fileSize), r.TestCheckResourceAttrPtr("data.archive_file.foo", "output_size", &fileSize), ), }, { - Config: testAccArchiveFileDirConfig, + Config: testAccArchiveFileDirExcludesConfig, Check: r.ComposeTestCheckFunc( testAccArchiveFileExists("zip_file_acc_test.zip", &fileSize), r.TestCheckResourceAttrPtr("data.archive_file.foo", "output_size", &fileSize), ), }, + }, + }) +} + +func TestAccArchiveFile_SourceFile(t *testing.T) { + var fileSize string + r.Test(t, r.TestCase{ + Providers: testProviders, + Steps: []r.TestStep{ { - Config: testAccArchiveFileDirExcludesConfig, + Config: testAccArchiveFileFileConfig, Check: r.ComposeTestCheckFunc( testAccArchiveFileExists("zip_file_acc_test.zip", &fileSize), r.TestCheckResourceAttrPtr("data.archive_file.foo", "output_size", &fileSize), ), }, + }, + }) +} + +func TestAccArchiveFile_SourceContentBlocks(t *testing.T) { + var fileSize string + r.Test(t, r.TestCase{ + Providers: testProviders, + Steps: []r.TestStep{ { Config: testAccArchiveFileMultiConfig, Check: r.ComposeTestCheckFunc( @@ -65,12 +92,6 @@ func TestAccArchiveFile_Basic(t *testing.T) { r.TestCheckResourceAttrPtr("data.archive_file.foo", "output_size", &fileSize), ), }, - { - Config: testAccArchiveFileOutputPath, - Check: r.ComposeTestCheckFunc( - testAccArchiveFileExists(fmt.Sprintf("%s/test.zip", tmpDir), &fileSize), - ), - }, }, }) } @@ -87,28 +108,16 @@ func testAccArchiveFileExists(filename string, fileSize *string) r.TestCheckFunc } } -var testAccArchiveFileContentConfig = ` +var testAccArchiveFileOutputPath = ` data "archive_file" "foo" { - type = "zip" source_content = "This is some content" source_content_filename = "content.txt" output_path = "zip_file_acc_test.zip" } ` -var tmpDir = os.TempDir() + "/test" -var testAccArchiveFileOutputPath = fmt.Sprintf(` -data "archive_file" "foo" { - type = "zip" - source_content = "This is some content" - source_content_filename = "content.txt" - output_path = "%s/test.zip" -} -`, tmpDir) - var testAccArchiveFileFileConfig = ` data "archive_file" "foo" { - type = "zip" source_file = "test-fixtures/test-file.txt" output_path = "zip_file_acc_test.zip" } @@ -116,7 +125,6 @@ data "archive_file" "foo" { var testAccArchiveFileDirConfig = ` data "archive_file" "foo" { - type = "zip" source_dir = "test-fixtures/test-dir" output_path = "zip_file_acc_test.zip" } @@ -124,8 +132,7 @@ data "archive_file" "foo" { var testAccArchiveFileDirExcludesConfig = ` data "archive_file" "foo" { - type = "zip" - source_dir = "../archive/test-fixtures/../test-fixtures/test-dir" + source_dir = "test-fixtures/test-dir" excludes = ["test-fixtures/test-dir/file2.txt"] output_path = "zip_file_acc_test.zip" } @@ -133,11 +140,10 @@ data "archive_file" "foo" { var testAccArchiveFileMultiConfig = ` data "archive_file" "foo" { - type = "zip" + output_path = "zip_file_acc_test.zip" source { - filename = "content.txt" - content = "This is some content" - } - output_path = "zip_file_acc_test.zip" + filename = "content.txt" + content = "This is some content" + } } ` diff --git a/archive/zip_archiver.go b/archive/zip_archiver.go deleted file mode 100644 index 1ed341f9..00000000 --- a/archive/zip_archiver.go +++ /dev/null @@ -1,195 +0,0 @@ -package archive - -import ( - "archive/zip" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "sort" - "time" -) - -type ZipArchiver struct { - filepath string - filewriter *os.File - writer *zip.Writer -} - -func NewZipArchiver(filepath string) Archiver { - return &ZipArchiver{ - filepath: filepath, - } -} - -func (a *ZipArchiver) ArchiveContent(content []byte, infilename string) error { - if err := a.open(); err != nil { - return err - } - defer a.close() - - f, err := a.writer.Create(filepath.ToSlash(infilename)) - if err != nil { - return err - } - - _, err = f.Write(content) - return err -} - -func (a *ZipArchiver) ArchiveFile(infilename string) error { - fi, err := assertValidFile(infilename) - if err != nil { - return err - } - - content, err := ioutil.ReadFile(infilename) - if err != nil { - return err - } - - if err := a.open(); err != nil { - return err - } - defer a.close() - - fh, err := zip.FileInfoHeader(fi) - if err != nil { - return fmt.Errorf("error creating file header: %s", err) - } - fh.Name = filepath.ToSlash(fi.Name()) - fh.Method = zip.Deflate - // fh.Modified alone isn't enough when using a zero value - fh.SetModTime(time.Time{}) - - f, err := a.writer.CreateHeader(fh) - if err != nil { - return fmt.Errorf("error creating file inside archive: %s", err) - } - - _, err = f.Write(content) - return err -} - -func checkMatch(fileName string, excludes []string) (value bool) { - for _, exclude := range excludes { - if exclude == "" { - continue - } - - if exclude == fileName { - return true - } - } - return false -} - -func (a *ZipArchiver) ArchiveDir(indirname string, excludes []string) error { - _, err := assertValidDir(indirname) - if err != nil { - return err - } - - if err := a.open(); err != nil { - return err - } - defer a.close() - - return filepath.Walk(indirname, func(path string, info os.FileInfo, err error) error { - - if err != nil { - return fmt.Errorf("error encountered during file walk: %s", err) - } - - relname, err := filepath.Rel(indirname, path) - if err != nil { - return fmt.Errorf("error relativizing file for archival: %s", err) - } - - isMatch := checkMatch(relname, excludes) - - if info.IsDir() { - if isMatch { - return filepath.SkipDir - } - return nil - } - - if isMatch { - return nil - } - - if err != nil { - return err - } - - fh, err := zip.FileInfoHeader(info) - if err != nil { - return fmt.Errorf("error creating file header: %s", err) - } - fh.Name = filepath.ToSlash(relname) - fh.Method = zip.Deflate - // fh.Modified alone isn't enough when using a zero value - fh.SetModTime(time.Time{}) - - f, err := a.writer.CreateHeader(fh) - if err != nil { - return fmt.Errorf("error creating file inside archive: %s", err) - } - content, err := ioutil.ReadFile(path) - if err != nil { - return fmt.Errorf("error reading file for archival: %s", err) - } - _, err = f.Write(content) - return err - }) -} - -func (a *ZipArchiver) ArchiveMultiple(content map[string][]byte) error { - if err := a.open(); err != nil { - return err - } - defer a.close() - - // Ensure files are processed in the same order so hashes don't change - keys := make([]string, len(content)) - i := 0 - for k := range content { - keys[i] = k - i++ - } - sort.Strings(keys) - - for _, filename := range keys { - f, err := a.writer.Create(filepath.ToSlash(filename)) - if err != nil { - return err - } - _, err = f.Write(content[filename]) - if err != nil { - return err - } - } - return nil -} - -func (a *ZipArchiver) open() error { - f, err := os.Create(a.filepath) - if err != nil { - return err - } - a.filewriter = f - a.writer = zip.NewWriter(f) - return nil -} - -func (a *ZipArchiver) close() { - if a.writer != nil { - a.writer.Close() - a.writer = nil - } - if a.filewriter != nil { - a.filewriter.Close() - a.filewriter = nil - } -} diff --git a/archive/zip_archiver_test.go b/archive/zip_archiver_test.go deleted file mode 100644 index 5429096c..00000000 --- a/archive/zip_archiver_test.go +++ /dev/null @@ -1,169 +0,0 @@ -package archive - -import ( - "archive/zip" - "bytes" - "io/ioutil" - "os" - "path/filepath" - "testing" - "time" -) - -func TestZipArchiver_Content(t *testing.T) { - zipfilepath := "archive-content.zip" - archiver := NewZipArchiver(zipfilepath) - if err := archiver.ArchiveContent([]byte("This is some content"), "content.txt"); err != nil { - t.Fatalf("unexpected error: %s", err) - } - - ensureContents(t, zipfilepath, map[string][]byte{ - "content.txt": []byte("This is some content"), - }) -} - -func TestZipArchiver_File(t *testing.T) { - zipfilepath := "archive-file.zip" - archiver := NewZipArchiver(zipfilepath) - if err := archiver.ArchiveFile("./test-fixtures/test-file.txt"); err != nil { - t.Fatalf("unexpected error: %s", err) - } - - ensureContents(t, zipfilepath, map[string][]byte{ - "test-file.txt": []byte("This is test content"), - }) -} -func TestZipArchiver_FileModified(t *testing.T) { - var ( - zipFilePath = filepath.FromSlash("archive-file.zip") - toZipPath = filepath.FromSlash("./test-fixtures/test-file.txt") - ) - - var zip = func() { - archiver := NewZipArchiver(zipFilePath) - if err := archiver.ArchiveFile(toZipPath); err != nil { - t.Fatalf("unexpected error: %s", err) - } - } - - zip() - - expectedContents, err := ioutil.ReadFile(zipFilePath) - if err != nil { - t.Fatalf("unexpected error: %s", err) - } - - //touch file modified, in the future just in case of weird race issues - newTime := time.Now().Add(1 * time.Hour) - if err := os.Chtimes(toZipPath, newTime, newTime); err != nil { - t.Fatalf("unexpected error: %s", err) - } - - zip() - - actualContents, err := ioutil.ReadFile(zipFilePath) - if err != nil { - t.Fatalf("unexpecte error: %s", err) - } - - if !bytes.Equal(expectedContents, actualContents) { - t.Fatalf("zip contents do not match, potentially a modified time issue") - } -} - -func TestZipArchiver_Dir(t *testing.T) { - zipfilepath := "archive-dir.zip" - archiver := NewZipArchiver(zipfilepath) - if err := archiver.ArchiveDir("./test-fixtures/test-dir", []string{""}); err != nil { - t.Fatalf("unexpected error: %s", err) - } - - ensureContents(t, zipfilepath, map[string][]byte{ - "file1.txt": []byte("This is file 1"), - "file2.txt": []byte("This is file 2"), - "file3.txt": []byte("This is file 3"), - }) -} - -func TestZipArchiver_Dir_Exclude(t *testing.T) { - zipfilepath := "archive-dir.zip" - archiver := NewZipArchiver(zipfilepath) - if err := archiver.ArchiveDir("./test-fixtures/test-dir", []string{"file2.txt"}); err != nil { - t.Fatalf("unexpected error: %s", err) - } - - ensureContents(t, zipfilepath, map[string][]byte{ - "file1.txt": []byte("This is file 1"), - "file3.txt": []byte("This is file 3"), - }) -} - -func TestZipArchiver_Dir_Exclude_With_Directory(t *testing.T) { - zipfilepath := "archive-dir.zip" - archiver := NewZipArchiver(zipfilepath) - if err := archiver.ArchiveDir("./test-fixtures/", []string{"test-dir", "test-dir2/file2.txt"}); err != nil { - t.Fatalf("unexpected error: %s", err) - } - - ensureContents(t, zipfilepath, map[string][]byte{ - "test-dir2/file1.txt": []byte("This is file 1"), - "test-dir2/file3.txt": []byte("This is file 3"), - "test-file.txt": []byte("This is test content"), - }) -} - -func TestZipArchiver_Multiple(t *testing.T) { - zipfilepath := "archive-content.zip" - content := map[string][]byte{ - "file1.txt": []byte("This is file 1"), - "file2.txt": []byte("This is file 2"), - "file3.txt": []byte("This is file 3"), - } - - archiver := NewZipArchiver(zipfilepath) - if err := archiver.ArchiveMultiple(content); err != nil { - t.Fatalf("unexpected error: %s", err) - } - - ensureContents(t, zipfilepath, content) - -} - -func ensureContents(t *testing.T, zipfilepath string, wants map[string][]byte) { - r, err := zip.OpenReader(zipfilepath) - if err != nil { - t.Fatalf("could not open zip file: %s", err) - } - defer r.Close() - - if len(r.File) != len(wants) { - t.Errorf("mismatched file count, got %d, want %d", len(r.File), len(wants)) - } - for _, cf := range r.File { - ensureContent(t, wants, cf) - } -} - -func ensureContent(t *testing.T, wants map[string][]byte, got *zip.File) { - want, ok := wants[got.Name] - if !ok { - t.Errorf("additional file in zip: %s", got.Name) - return - } - - r, err := got.Open() - if err != nil { - t.Errorf("could not open file: %s", err) - } - defer r.Close() - gotContentBytes, err := ioutil.ReadAll(r) - if err != nil { - t.Errorf("could not read file: %s", err) - } - - wantContent := string(want) - gotContent := string(gotContentBytes) - if gotContent != wantContent { - t.Errorf("mismatched content\ngot\n%s\nwant\n%s", gotContent, wantContent) - } -}