simon/brd
Archived
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add multithreading for batch conversion

Browse source
This commit is contained in:
Simon Bruder 2020-07-10 14:07:25 +02:00
parent dad3a7ff13
commit 38180cfd0a
No known key found for this signature in database
GPG key ID: 6F03E0000CC5B62F
3 changed files with 105 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

94
Cargo.lock generated
View file

@ -61,6 +61,7 @@ dependencies = [
"quick-xml",
"quickcheck",
"quickcheck_macros",
"rayon",
"serde",
"tabwriter",
"thiserror",
@ -130,6 +131,43 @@ dependencies = [
"maybe-uninit",
]
[[package]]
name = "crossbeam-deque"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
"maybe-uninit",
]
[[package]]
name = "crossbeam-epoch"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"lazy_static",
"maybe-uninit",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-queue"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570"
dependencies = [
"cfg-if",
"crossbeam-utils",
"maybe-uninit",
]
[[package]]
name = "crossbeam-utils"
version = "0.7.2"
@ -152,6 +190,12 @@ dependencies = [
"syn",
]
[[package]]
name = "either"
version = "1.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
[[package]]
name = "env_logger"
version = "0.7.1"
@ -262,6 +306,15 @@ version = "2.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"
[[package]]
name = "memoffset"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c198b026e1bbf08a937e94c6c60f9ec4a2267f5b0d2eec9c1b21b061ce2be55f"
dependencies = [
"autocfg",
]
[[package]]
name = "miniz_oxide"
version = "0.4.0"
@ -291,6 +344,16 @@ dependencies = [
"autocfg",
]
[[package]]
name = "num_cpus"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
dependencies = [
"hermit-abi",
"libc",
]
[[package]]
name = "os_str_bytes"
version = "2.3.1"
@ -455,6 +518,31 @@ dependencies = [
"rand_core",
]
[[package]]
name = "rayon"
version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62f02856753d04e03e26929f820d0a0a337ebe71f849801eea335d464b349080"
dependencies = [
"autocfg",
"crossbeam-deque",
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e92e15d89083484e11353891f1af602cc661426deb9564c298b270c726973280"
dependencies = [
"crossbeam-deque",
"crossbeam-queue",
"crossbeam-utils",
"lazy_static",
"num_cpus",
]
[[package]]
name = "regex"
version = "1.3.9"
@ -473,6 +561,12 @@ version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8"
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "serde"
version = "1.0.114"

1
Cargo.toml
View file

@ -16,6 +16,7 @@ num-traits = "0.2"
pbr = "1.0.3"
pretty_env_logger = "0.4"
quick-xml = { version = "0.18", features = [ "serialize" ] }
rayon = "1"
serde = { version = "1.0", features = [ "derive" ] }
tabwriter = "1.2.1"
thiserror = "1.0.20"

16
src/main.rs
View file

@ -3,11 +3,13 @@ use std::fs;
use std::io;
use std::io::Write;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use anyhow::{anyhow, Context, Result};
use clap::Clap;
use log::{debug, error, info, warn};
use pbr::ProgressBar;
use rayon::prelude::*;
use tabwriter::TabWriter;
use brd::converter;
@ -415,10 +417,12 @@ fn main() -> Result<()> {
fs::create_dir_all(&opts.out_dir)?;
let mut pb = ProgressBar::new(musicdb.music.len().try_into()?);
for entry in musicdb.music {
pb.message(&format!("{} ", entry.basename));
pb.tick();
let pb = Arc::new(Mutex::new(ProgressBar::new(
musicdb.music.len().try_into()?,
)));
musicdb.music.into_par_iter().for_each(|entry| {
pb.lock().unwrap().message(&format!("{} ", entry.basename));
pb.lock().unwrap().tick();
let mut ssq_file = opts.ssq_dir.clone();
ssq_file.push(&entry.basename);
@ -449,8 +453,8 @@ fn main() -> Result<()> {
)
});
pb.inc();
}
pb.lock().unwrap().inc();
})
}
}
Ok(())