From eb9d199b86fc04ce35f12cf2cac63a4beeb18643 Mon Sep 17 00:00:00 2001 From: Anthony Hung Date: Tue, 27 Dec 2022 02:04:19 +0800 Subject: [PATCH 1/7] Add `rayon` crate --- Cargo.lock | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 3 +- 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index c2ab0a9..b890c5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -226,6 +226,7 @@ dependencies = [ "lazy_static", "liquid", "rand 0.7.3", + "rayon", "regex", "reqwest", "serde", @@ -288,6 +289,49 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -938,6 +982,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -1402,6 +1455,28 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rayon" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -1528,6 +1603,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "scratch" version = "1.0.2" diff --git a/Cargo.toml b/Cargo.toml index 2fd7e57..1716c5f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ shell-words = "^0.1.0" chrono = "^0.4.10" serde_json = "^1.0.0" walkdir = "^2.3.1" +rayon = "^1.6.1" regex = "^1.3.5" url = "^2.1.1" @@ -68,4 +69,4 @@ liquid = "^0.19.0" lazy_static = "^1.4.0" [profile.release] -lto = true +lto = true \ No newline at end of file From da13c028be80313834788f2869150a5f6304c375 Mon Sep 17 00:00:00 2001 From: Anthony Hung Date: Tue, 27 Dec 2022 02:43:52 +0800 Subject: [PATCH 2/7] Use `rayon` on `WalkDir` in `id_dates` to utilize more threads. --- src/message/saver.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/message/saver.rs b/src/message/saver.rs index ef8f41b..009315e 100644 --- a/src/message/saver.rs +++ b/src/message/saver.rs @@ -1,6 +1,7 @@ use std::fs; use std::path::{PathBuf}; +use rayon::prelude::*; use regex::Regex; use walkdir::{WalkDir, DirEntry}; use chrono::NaiveDateTime; @@ -175,6 +176,7 @@ impl<'b, C: SHNClient> Saver<'b, C> { fn id_dates(&self, dir_buf: &PathBuf) -> Vec { let mut result = WalkDir::new(dir_buf) .into_iter() + .par_bridge() .filter(|r| !r.as_ref().unwrap().path().is_dir()) .map(|r| { let dir_entry = r.unwrap(); From e5b32487427467293c654b30395cbac560cc3df2 Mon Sep 17 00:00:00 2001 From: Anthony Hung Date: Tue, 27 Dec 2022 02:48:01 +0800 Subject: [PATCH 3/7] Move regex in `dir_entry_to_id_date` to lazy_static. * Reduce the cost of Regex compilation. --- src/message/saver.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/message/saver.rs b/src/message/saver.rs index 009315e..5a3fad9 100644 --- a/src/message/saver.rs +++ b/src/message/saver.rs @@ -12,6 +12,10 @@ use crate::{ message::file::{Text, Picture, SaveToFile, Video, Voice}, }; +lazy_static! { + static ref ID_DATE_REGEX: Regex = Regex::new(r"(?x)(?P\d+)_\d_(?P\d+)").unwrap(); +} + pub struct Saver<'a, C: SHNClient> { config: &'a Config<'a, C>, } @@ -217,7 +221,7 @@ struct IdDate { } fn dir_entry_to_id_date(filename: &DirEntry) -> Option { - let re = Regex::new(r"(?x)(?P\d+)_\d_(?P\d+)").unwrap(); + let re = ID_DATE_REGEX.clone(); re.captures(filename.file_name().to_str().unwrap()) .and_then(|cap|Some(IdDate { id: cap["id"].parse::().unwrap(), From 18da205d8069e06c803bcc33134d6ed8a042b631 Mon Sep 17 00:00:00 2001 From: Anthony Hung Date: Tue, 27 Dec 2022 03:24:21 +0800 Subject: [PATCH 4/7] Initialize `member_identifier_vec` with the same size as `group` --- src/message/saver.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/message/saver.rs b/src/message/saver.rs index 5a3fad9..13d750c 100644 --- a/src/message/saver.rs +++ b/src/message/saver.rs @@ -51,7 +51,7 @@ impl<'b, C: SHNClient> Saver<'b, C> { } fn create_member_identifier_list(&self, group: &Vec, tags: &Vec) -> Vec { - let mut member_identifier_vec = Vec::new(); + let mut member_identifier_vec = Vec::with_capacity(group.len()); group.iter().for_each(|g| { // もっといい書き方があるはず let mut group = "".to_string(); let mut gen = "".to_string(); From 424dbac7ab6cd10dc5ffd337d35f1b61d5c660e9 Mon Sep 17 00:00:00 2001 From: proshunsuke Date: Wed, 28 Dec 2022 23:52:17 +0900 Subject: [PATCH 5/7] v3.0.3 From 29f4f55c26e5df30c712f8bae1b021aed7d9645a Mon Sep 17 00:00:00 2001 From: proshunsuke Date: Wed, 28 Dec 2022 23:57:48 +0900 Subject: [PATCH 6/7] update version --- Cargo.lock | 2 +- Cargo.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b890c5e..a785d7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -212,7 +212,7 @@ dependencies = [ [[package]] name = "colmsg" -version = "3.0.2" +version = "3.0.3" dependencies = [ "ansi_colours", "ansi_term", diff --git a/Cargo.toml b/Cargo.toml index 1716c5f..b43399f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "colmsg" -version = "3.0.2" +version = "3.0.3" authors = ["proshunsuke "] categories = ["command-line-utilities"] description="Save the messages of '櫻坂46メッセージ', '日向坂46メッセージ' and '乃木坂46メッセージ' apps to the local." @@ -69,4 +69,4 @@ liquid = "^0.19.0" lazy_static = "^1.4.0" [profile.release] -lto = true \ No newline at end of file +lto = true From ab0b6aedebf4487474c015d7b9e323228e10cdb3 Mon Sep 17 00:00:00 2001 From: proshunsuke Date: Wed, 28 Dec 2022 23:57:56 +0900 Subject: [PATCH 7/7] update CHANGELOG.md --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90bdc77..2ca3bf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ ## Packaging +# v3.0.3 + +https://github.com/proshunsuke/colmsg/pull/88 + +## Fix + +* Use [rayon](https://crates.io/crates/rayon) to speed up internal processing + # v3.0.2 https://github.com/proshunsuke/colmsg/pull/86