diff --git a/CHANGELOG.md b/CHANGELOG.md index 90bdc77..2ca3bf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ ## Packaging +# v3.0.3 + +https://github.com/proshunsuke/colmsg/pull/88 + +## Fix + +* Use [rayon](https://crates.io/crates/rayon) to speed up internal processing + # v3.0.2 https://github.com/proshunsuke/colmsg/pull/86 diff --git a/Cargo.lock b/Cargo.lock index c2ab0a9..a785d7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -212,7 +212,7 @@ dependencies = [ [[package]] name = "colmsg" -version = "3.0.2" +version = "3.0.3" dependencies = [ "ansi_colours", "ansi_term", @@ -226,6 +226,7 @@ dependencies = [ "lazy_static", "liquid", "rand 0.7.3", + "rayon", "regex", "reqwest", "serde", @@ -288,6 +289,49 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -938,6 +982,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -1402,6 +1455,28 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rayon" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -1528,6 +1603,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "scratch" version = "1.0.2" diff --git a/Cargo.toml b/Cargo.toml index 2fd7e57..b43399f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "colmsg" -version = "3.0.2" +version = "3.0.3" authors = ["proshunsuke "] categories = ["command-line-utilities"] description="Save the messages of '櫻坂46メッセージ', '日向坂46メッセージ' and '乃木坂46メッセージ' apps to the local." @@ -25,6 +25,7 @@ shell-words = "^0.1.0" chrono = "^0.4.10" serde_json = "^1.0.0" walkdir = "^2.3.1" +rayon = "^1.6.1" regex = "^1.3.5" url = "^2.1.1" diff --git a/src/message/saver.rs b/src/message/saver.rs index ef8f41b..13d750c 100644 --- a/src/message/saver.rs +++ b/src/message/saver.rs @@ -1,6 +1,7 @@ use std::fs; use std::path::{PathBuf}; +use rayon::prelude::*; use regex::Regex; use walkdir::{WalkDir, DirEntry}; use chrono::NaiveDateTime; @@ -11,6 +12,10 @@ use crate::{ message::file::{Text, Picture, SaveToFile, Video, Voice}, }; +lazy_static! { + static ref ID_DATE_REGEX: Regex = Regex::new(r"(?x)(?P\d+)_\d_(?P\d+)").unwrap(); +} + pub struct Saver<'a, C: SHNClient> { config: &'a Config<'a, C>, } @@ -46,7 +51,7 @@ impl<'b, C: SHNClient> Saver<'b, C> { } fn create_member_identifier_list(&self, group: &Vec, tags: &Vec) -> Vec { - let mut member_identifier_vec = Vec::new(); + let mut member_identifier_vec = Vec::with_capacity(group.len()); group.iter().for_each(|g| { // もっといい書き方があるはず let mut group = "".to_string(); let mut gen = "".to_string(); @@ -175,6 +180,7 @@ impl<'b, C: SHNClient> Saver<'b, C> { fn id_dates(&self, dir_buf: &PathBuf) -> Vec { let mut result = WalkDir::new(dir_buf) .into_iter() + .par_bridge() .filter(|r| !r.as_ref().unwrap().path().is_dir()) .map(|r| { let dir_entry = r.unwrap(); @@ -215,7 +221,7 @@ struct IdDate { } fn dir_entry_to_id_date(filename: &DirEntry) -> Option { - let re = Regex::new(r"(?x)(?P\d+)_\d_(?P\d+)").unwrap(); + let re = ID_DATE_REGEX.clone(); re.captures(filename.file_name().to_str().unwrap()) .and_then(|cap|Some(IdDate { id: cap["id"].parse::().unwrap(),