From 064f38f6e5a54c33bc3767a792a4b61d41e5af15 Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Thu, 13 May 2021 18:05:29 +0530 Subject: [PATCH 1/4] first cut of output checking --- fbt_lib/src/dir_diff.rs | 174 +++++++++--------- fbt_lib/src/run.rs | 35 ++++ tests/03_failing_fbt_tests/cmd.p1 | 17 +- .../input/tests/02_output_mismatch/cmd.p1 | 1 - .../tests/02_output_mismatch/output/foo.txt | 2 +- tests/07_valid_output/cmd.p1 | 2 + tests/07_valid_output/output/foo.txt | 1 + 7 files changed, 140 insertions(+), 92 deletions(-) create mode 100644 tests/07_valid_output/cmd.p1 create mode 100644 tests/07_valid_output/output/foo.txt diff --git a/fbt_lib/src/dir_diff.rs b/fbt_lib/src/dir_diff.rs index 5323bd3..bf27047 100644 --- a/fbt_lib/src/dir_diff.rs +++ b/fbt_lib/src/dir_diff.rs @@ -37,98 +37,94 @@ pub enum DirDiff { } pub(crate) fn diff, B: AsRef>( - _a_base: A, - _b_base: B, + a_base: A, + b_base: B, ) -> Result, DirDiffError> { - todo!("not yet implemented") - - // let mut a_walker = walk_dir(a_base)?; - // let mut b_walker = walk_dir(b_base)?; - // - // loop { - // match (a_walker.next(), b_walker.next()) { - // (Some(a), Some(b)) => { - // // first lets check the depth: - // // a > b: UnexpectedFileFound or UnexpectedFolderFound else - // // b > a: ExpectedFileMissing or ExpectedFolderMissing - // - // // if file names dont match how to find if we got a new entry - // // on left or extra entry on right? how do people actually - // // calculate diff? - // - // // then check file type - // - // // finally check file content if its a file - // - // // TODO: this is dummy code to test stuff - // let a = a?; - // let b = b?; - // eprintln!("a: {:?}", a); - // eprintln!("b: {:?}", b); - // - // // TODO: this is fully buggy! loop never loops. - // let found: std::path::PathBuf = b.path().into(); - // return Ok(Some(if found.is_dir() { - // DirDiff::UnexpectedFolderFound { found } - // } else { - // DirDiff::UnexpectedFileFound { found } - // })); - // } - // (None, Some(b)) => { - // // we have something in b, but a is done, lets iterate over all - // // entries in b, and put them in UnexpectedFileFound and - // // UnexpectedFolderFound - // let found: std::path::PathBuf = b?.path().into(); - // return Ok(Some(if found.is_dir() { - // DirDiff::UnexpectedFolderFound { found } - // } else { - // DirDiff::UnexpectedFileFound { found } - // })); - // } - // (Some(a), None) => { - // // we have something in a, but b is done, lets iterate over all - // // entries in a, and put them in ExpectedFileMissing and - // // ExpectedFolderMissing - // let expected: std::path::PathBuf = a?.path().into(); - // return Ok(Some(if expected.is_dir() { - // DirDiff::ExpectedFolderMissing { expected } - // } else { - // DirDiff::ExpectedFileMissing { expected } - // })); - // } - // (None, None) => break, - // } - // } - // - // Ok(None) + let mut a_walker = walk_dir(a_base)?; + let mut b_walker = walk_dir(b_base)?; + + loop { + match (a_walker.next(), b_walker.next()) { + (Some(a), Some(b)) => { + // first lets check the depth: + // a > b: UnexpectedFileFound or UnexpectedFolderFound else + // b > a: ExpectedFileMissing or ExpectedFolderMissing + + // if file names dont match how to find if we got a new entry + // on left or extra entry on right? how do people actually + // calculate diff? + + // then check file type + + // finally check file content if its a file + + // TODO: this is dummy code to test stuff + let a = a?; + let b = b?; + + let found: std::path::PathBuf = b.path().into(); + + if a.file_name() != b.file_name() { + return Ok(Some(if found.is_dir() { + DirDiff::UnexpectedFolderFound { found } + } else { + DirDiff::UnexpectedFileFound { found } + })); + } + + let a_content = std::fs::read_to_string(a.path())?; + let b_content = std::fs::read_to_string(b.path())?; + if a_content != b_content { + return Ok(Some(DirDiff::ContentMismatch { + expected: a_content, + found: b_content, + file: found, + })); + } + } + (None, Some(b)) => { + // we have something in b, but a is done, lets iterate over all + // entries in b, and put them in UnexpectedFileFound and + // UnexpectedFolderFound + let found: std::path::PathBuf = b?.path().into(); + return Ok(Some(if found.is_dir() { + DirDiff::UnexpectedFolderFound { found } + } else { + DirDiff::UnexpectedFileFound { found } + })); + } + (Some(a), None) => { + // we have something in a, but b is done, lets iterate over all + // entries in a, and put them in ExpectedFileMissing and + // ExpectedFolderMissing + let expected: std::path::PathBuf = a?.path().into(); + return Ok(Some(if expected.is_dir() { + DirDiff::ExpectedFolderMissing { expected } + } else { + DirDiff::ExpectedFileMissing { expected } + })); + } + (None, None) => break, + } + } + + Ok(None) } -// fn walk_dir>(path: P) -> Result { -// let mut walkdir = walkdir::WalkDir::new(path) -// .sort_by(compare_by_file_name) -// .into_iter(); -// if let Some(Err(e)) = walkdir.next() { -// Err(e.into()) -// } else { -// Ok(walkdir) -// } -// } - -// fn compare_by_file_name(a: &walkdir::DirEntry, b: &walkdir::DirEntry) -> std::cmp::Ordering { -// a.file_name().cmp(&b.file_name()) -// } - -// fn read_to_vec>(file: P) -> Result, std::io::Error> { -// use std::fs::File; -// use std::io::Read; -// -// let mut data = Vec::new(); -// let mut file = File::open(file.as_ref())?; -// -// file.read_to_end(&mut data)?; -// -// Ok(data) -// } +fn walk_dir>(path: P) -> Result { + let mut walkdir = walkdir::WalkDir::new(path) + .sort_by(compare_by_file_name) + .into_iter(); + if let Some(Err(e)) = walkdir.next() { + Err(e.into()) + } else { + Ok(walkdir) + } +} + +fn compare_by_file_name(a: &walkdir::DirEntry, b: &walkdir::DirEntry) -> std::cmp::Ordering { + a.file_name().cmp(&b.file_name()) +} impl From for DirDiffError { fn from(e: std::io::Error) -> DirDiffError { diff --git a/fbt_lib/src/run.rs b/fbt_lib/src/run.rs index 0a5fa19..d4c09cb 100644 --- a/fbt_lib/src/run.rs +++ b/fbt_lib/src/run.rs @@ -106,6 +106,41 @@ pub fn main() -> Option { ) ); } + Err(crate::Failure::OutputMismatch { diff }) => { + any_failed = true; + match diff { + crate::DirDiff::ContentMismatch { + found, + expected, + file, + } => { + println!( + "{}: {} {} (output content mismatch: {})", + case.id.blue(), + "FAILED".red(), + duration, + file.to_str().unwrap_or("cant-read-filename"), + ); + println!("found:\n\n{}\n", found.as_str()); + println!( + "diff:\n\n{}\n", + diffy::create_patch( + (expected.to_owned() + "\n").as_str(), + (found.to_owned() + "\n").as_str() + ) + ); + } + _ => { + println!( + "{}: {} {} (output mismatch: {:?})", + case.id.blue(), + "FAILED".red(), + duration, + diff + ); + } + } + } Err(e) => { any_failed = true; println!( diff --git a/tests/03_failing_fbt_tests/cmd.p1 b/tests/03_failing_fbt_tests/cmd.p1 index fa2a908..1d3de91 100644 --- a/tests/03_failing_fbt_tests/cmd.p1 +++ b/tests/03_failing_fbt_tests/cmd.p1 @@ -20,7 +20,22 @@ diff: +hello there -02_output_mismatch: SKIPPED (output not yet implemented) +02_output_mismatch: FAILED (output content mismatch: ./tests/02_output_mismatch/output/foo.txt) +found: + +hello world2 + + +diff: + +\--- original ++++ modified +@@ -1,2 +1,2 @@ +-hello world ++hello world2 + + + 03_stderr_mismatch: FAILED (stderr mismatch) stderr: diff --git a/tests/03_failing_fbt_tests/input/tests/02_output_mismatch/cmd.p1 b/tests/03_failing_fbt_tests/input/tests/02_output_mismatch/cmd.p1 index e1c6f9c..a589b9f 100644 --- a/tests/03_failing_fbt_tests/input/tests/02_output_mismatch/cmd.p1 +++ b/tests/03_failing_fbt_tests/input/tests/02_output_mismatch/cmd.p1 @@ -1,3 +1,2 @@ -- fbt: cmd: echo hello world > foo.txt -skip: output not yet implemented \ No newline at end of file diff --git a/tests/03_failing_fbt_tests/input/tests/02_output_mismatch/output/foo.txt b/tests/03_failing_fbt_tests/input/tests/02_output_mismatch/output/foo.txt index 3b18e51..1142904 100644 --- a/tests/03_failing_fbt_tests/input/tests/02_output_mismatch/output/foo.txt +++ b/tests/03_failing_fbt_tests/input/tests/02_output_mismatch/output/foo.txt @@ -1 +1 @@ -hello world +hello world2 diff --git a/tests/07_valid_output/cmd.p1 b/tests/07_valid_output/cmd.p1 new file mode 100644 index 0000000..a589b9f --- /dev/null +++ b/tests/07_valid_output/cmd.p1 @@ -0,0 +1,2 @@ +-- fbt: +cmd: echo hello world > foo.txt diff --git a/tests/07_valid_output/output/foo.txt b/tests/07_valid_output/output/foo.txt new file mode 100644 index 0000000..3b18e51 --- /dev/null +++ b/tests/07_valid_output/output/foo.txt @@ -0,0 +1 @@ +hello world From 504b71f44304195a882b167d29404e0a816dfa34 Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Thu, 13 May 2021 18:35:36 +0530 Subject: [PATCH 2/4] extra file found --- fbt_lib/src/run.rs | 9 +++++++++ tests/03_failing_fbt_tests/cmd.p1 | 1 + .../input/tests/04_extra_file/cmd.p1 | 2 ++ .../input/tests/04_extra_file/output/bar.txt | 1 + .../input/tests/04_extra_file/output/foo.txt | 1 + 5 files changed, 14 insertions(+) create mode 100644 tests/03_failing_fbt_tests/input/tests/04_extra_file/cmd.p1 create mode 100644 tests/03_failing_fbt_tests/input/tests/04_extra_file/output/bar.txt create mode 100644 tests/03_failing_fbt_tests/input/tests/04_extra_file/output/foo.txt diff --git a/fbt_lib/src/run.rs b/fbt_lib/src/run.rs index d4c09cb..05c36ec 100644 --- a/fbt_lib/src/run.rs +++ b/fbt_lib/src/run.rs @@ -130,6 +130,15 @@ pub fn main() -> Option { ) ); } + crate::DirDiff::UnexpectedFileFound { found } => { + println!( + "{}: {} {} (extra file found: {})", + case.id.blue(), + "FAILED".red(), + duration, + found.to_str().unwrap_or("cant-read-filename"), + ); + } _ => { println!( "{}: {} {} (output mismatch: {:?})", diff --git a/tests/03_failing_fbt_tests/cmd.p1 b/tests/03_failing_fbt_tests/cmd.p1 index 1d3de91..6efb52d 100644 --- a/tests/03_failing_fbt_tests/cmd.p1 +++ b/tests/03_failing_fbt_tests/cmd.p1 @@ -50,4 +50,5 @@ diff: +hello there +04_extra_file: FAILED (extra file found: ./tests/04_extra_file/output/bar.txt) diff --git a/tests/03_failing_fbt_tests/input/tests/04_extra_file/cmd.p1 b/tests/03_failing_fbt_tests/input/tests/04_extra_file/cmd.p1 new file mode 100644 index 0000000..a589b9f --- /dev/null +++ b/tests/03_failing_fbt_tests/input/tests/04_extra_file/cmd.p1 @@ -0,0 +1,2 @@ +-- fbt: +cmd: echo hello world > foo.txt diff --git a/tests/03_failing_fbt_tests/input/tests/04_extra_file/output/bar.txt b/tests/03_failing_fbt_tests/input/tests/04_extra_file/output/bar.txt new file mode 100644 index 0000000..7999d9d --- /dev/null +++ b/tests/03_failing_fbt_tests/input/tests/04_extra_file/output/bar.txt @@ -0,0 +1 @@ +bar bar dekho \ No newline at end of file diff --git a/tests/03_failing_fbt_tests/input/tests/04_extra_file/output/foo.txt b/tests/03_failing_fbt_tests/input/tests/04_extra_file/output/foo.txt new file mode 100644 index 0000000..3b18e51 --- /dev/null +++ b/tests/03_failing_fbt_tests/input/tests/04_extra_file/output/foo.txt @@ -0,0 +1 @@ +hello world From c59f46b8623bad085c89ee3d1573d588e07cb8dc Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Thu, 13 May 2021 19:02:02 +0530 Subject: [PATCH 3/4] more testing, but output is incorrect --- tests/03_failing_fbt_tests/cmd.p1 | 2 +- .../input/tests/05_expected_file_missing/cmd.p1 | 3 +++ .../input/tests/05_expected_file_missing/output/foo.txt | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 tests/03_failing_fbt_tests/input/tests/05_expected_file_missing/cmd.p1 create mode 100644 tests/03_failing_fbt_tests/input/tests/05_expected_file_missing/output/foo.txt diff --git a/tests/03_failing_fbt_tests/cmd.p1 b/tests/03_failing_fbt_tests/cmd.p1 index 6efb52d..9b75080 100644 --- a/tests/03_failing_fbt_tests/cmd.p1 +++ b/tests/03_failing_fbt_tests/cmd.p1 @@ -51,4 +51,4 @@ diff: 04_extra_file: FAILED (extra file found: ./tests/04_extra_file/output/bar.txt) - +05_expected_file_missing: SKIPPED (the output is incorrect) \ No newline at end of file diff --git a/tests/03_failing_fbt_tests/input/tests/05_expected_file_missing/cmd.p1 b/tests/03_failing_fbt_tests/input/tests/05_expected_file_missing/cmd.p1 new file mode 100644 index 0000000..0ec413b --- /dev/null +++ b/tests/03_failing_fbt_tests/input/tests/05_expected_file_missing/cmd.p1 @@ -0,0 +1,3 @@ +-- fbt: +cmd: echo hello world > foo.txt && echo bar bar dekho > bar.txt +skip: the output is incorrect \ No newline at end of file diff --git a/tests/03_failing_fbt_tests/input/tests/05_expected_file_missing/output/foo.txt b/tests/03_failing_fbt_tests/input/tests/05_expected_file_missing/output/foo.txt new file mode 100644 index 0000000..3b18e51 --- /dev/null +++ b/tests/03_failing_fbt_tests/input/tests/05_expected_file_missing/output/foo.txt @@ -0,0 +1 @@ +hello world From 010290fef2089ccfe6e5844a84513d6b79d68f48 Mon Sep 17 00:00:00 2001 From: Amit Upadhyay Date: Thu, 13 May 2021 19:06:49 +0530 Subject: [PATCH 4/4] version bump: 0.1.5 --- Cargo.lock | 4 ++-- Cargo.toml | 4 ++-- fbt_lib/Cargo.toml | 2 +- tests/06_version/cmd.p1 | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e9c716a..1bd9edc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -524,7 +524,7 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "fbt" -version = "0.1.4" +version = "0.1.5" dependencies = [ "fbt-lib", "rusty-hook", @@ -532,7 +532,7 @@ dependencies = [ [[package]] name = "fbt-lib" -version = "0.1.4" +version = "0.1.5" dependencies = [ "colored 2.0.0", "diffy", diff --git a/Cargo.toml b/Cargo.toml index d735684..588ee63 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fbt" -version = "0.1.4" +version = "0.1.5" authors = ["Sitesh ", "Amit Upadhyay "] edition = "2018" description = "folder based testing tool (library)" @@ -11,7 +11,7 @@ homepage = "https://www.fifthtry.com/fifthtry/fbt/" members = ["fbt_lib"] [dependencies] -# fbt-lib = { version = "0.1.4" } +# fbt-lib = { version = "0.1.5" } fbt-lib = { path = "fbt_lib" } [dev-dependencies] diff --git a/fbt_lib/Cargo.toml b/fbt_lib/Cargo.toml index 82d1c7c..7ef2235 100644 --- a/fbt_lib/Cargo.toml +++ b/fbt_lib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fbt-lib" -version = "0.1.4" +version = "0.1.5" authors = ["Sitesh ", "Amit Upadhyay "] edition = "2018" description = "folder based testing tool (library)" diff --git a/tests/06_version/cmd.p1 b/tests/06_version/cmd.p1 index 9d5429e..3ab2b2b 100644 --- a/tests/06_version/cmd.p1 +++ b/tests/06_version/cmd.p1 @@ -7,5 +7,5 @@ We will have to keep updating this file every time we release a new version. -- stdout: -fbt: 0.1.4 -fbt: 0.1.4 +fbt: 0.1.5 +fbt: 0.1.5