Add multithreading for batch conversion

This commit is contained in:
Simon Bruder 2020-07-10 14:07:25 +02:00
parent dad3a7ff13
commit 38180cfd0a
No known key found for this signature in database
GPG key ID: 6F03E0000CC5B62F
3 changed files with 105 additions and 6 deletions

94
Cargo.lock generated
View file

@ -61,6 +61,7 @@ dependencies = [
"quick-xml", "quick-xml",
"quickcheck", "quickcheck",
"quickcheck_macros", "quickcheck_macros",
"rayon",
"serde", "serde",
"tabwriter", "tabwriter",
"thiserror", "thiserror",
@ -130,6 +131,43 @@ dependencies = [
"maybe-uninit", "maybe-uninit",
] ]
[[package]]
name = "crossbeam-deque"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
"maybe-uninit",
]
[[package]]
name = "crossbeam-epoch"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"lazy_static",
"maybe-uninit",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-queue"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570"
dependencies = [
"cfg-if",
"crossbeam-utils",
"maybe-uninit",
]
[[package]] [[package]]
name = "crossbeam-utils" name = "crossbeam-utils"
version = "0.7.2" version = "0.7.2"
@ -152,6 +190,12 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "either"
version = "1.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
[[package]] [[package]]
name = "env_logger" name = "env_logger"
version = "0.7.1" version = "0.7.1"
@ -262,6 +306,15 @@ version = "2.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"
[[package]]
name = "memoffset"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c198b026e1bbf08a937e94c6c60f9ec4a2267f5b0d2eec9c1b21b061ce2be55f"
dependencies = [
"autocfg",
]
[[package]] [[package]]
name = "miniz_oxide" name = "miniz_oxide"
version = "0.4.0" version = "0.4.0"
@ -291,6 +344,16 @@ dependencies = [
"autocfg", "autocfg",
] ]
[[package]]
name = "num_cpus"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
dependencies = [
"hermit-abi",
"libc",
]
[[package]] [[package]]
name = "os_str_bytes" name = "os_str_bytes"
version = "2.3.1" version = "2.3.1"
@ -455,6 +518,31 @@ dependencies = [
"rand_core", "rand_core",
] ]
[[package]]
name = "rayon"
version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62f02856753d04e03e26929f820d0a0a337ebe71f849801eea335d464b349080"
dependencies = [
"autocfg",
"crossbeam-deque",
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e92e15d89083484e11353891f1af602cc661426deb9564c298b270c726973280"
dependencies = [
"crossbeam-deque",
"crossbeam-queue",
"crossbeam-utils",
"lazy_static",
"num_cpus",
]
[[package]] [[package]]
name = "regex" name = "regex"
version = "1.3.9" version = "1.3.9"
@ -473,6 +561,12 @@ version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8"
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]] [[package]]
name = "serde" name = "serde"
version = "1.0.114" version = "1.0.114"

View file

@ -16,6 +16,7 @@ num-traits = "0.2"
pbr = "1.0.3" pbr = "1.0.3"
pretty_env_logger = "0.4" pretty_env_logger = "0.4"
quick-xml = { version = "0.18", features = [ "serialize" ] } quick-xml = { version = "0.18", features = [ "serialize" ] }
rayon = "1"
serde = { version = "1.0", features = [ "derive" ] } serde = { version = "1.0", features = [ "derive" ] }
tabwriter = "1.2.1" tabwriter = "1.2.1"
thiserror = "1.0.20" thiserror = "1.0.20"

View file

@ -3,11 +3,13 @@ use std::fs;
use std::io; use std::io;
use std::io::Write; use std::io::Write;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use anyhow::{anyhow, Context, Result}; use anyhow::{anyhow, Context, Result};
use clap::Clap; use clap::Clap;
use log::{debug, error, info, warn}; use log::{debug, error, info, warn};
use pbr::ProgressBar; use pbr::ProgressBar;
use rayon::prelude::*;
use tabwriter::TabWriter; use tabwriter::TabWriter;
use brd::converter; use brd::converter;
@ -415,10 +417,12 @@ fn main() -> Result<()> {
fs::create_dir_all(&opts.out_dir)?; fs::create_dir_all(&opts.out_dir)?;
let mut pb = ProgressBar::new(musicdb.music.len().try_into()?); let pb = Arc::new(Mutex::new(ProgressBar::new(
for entry in musicdb.music { musicdb.music.len().try_into()?,
pb.message(&format!("{} ", entry.basename)); )));
pb.tick(); musicdb.music.into_par_iter().for_each(|entry| {
pb.lock().unwrap().message(&format!("{} ", entry.basename));
pb.lock().unwrap().tick();
let mut ssq_file = opts.ssq_dir.clone(); let mut ssq_file = opts.ssq_dir.clone();
ssq_file.push(&entry.basename); ssq_file.push(&entry.basename);
@ -449,8 +453,8 @@ fn main() -> Result<()> {
) )
}); });
pb.inc(); pb.lock().unwrap().inc();
} })
} }
} }
Ok(()) Ok(())