diff --git a/Cargo.lock b/Cargo.lock index f47ad58..2e73da9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -211,6 +211,7 @@ dependencies = [ "ctrlc", "dirs", "env_logger", + "filetime", "hhmmss", "if_chain", "indicatif", @@ -587,6 +588,18 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "filetime" +version = "0.2.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.60.2", +] + [[package]] name = "flate2" version = "1.0.34" @@ -850,6 +863,17 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" +[[package]] +name = "libredox" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" +dependencies = [ + "bitflags 2.9.0", + "libc", + "redox_syscall 0.5.17", +] + [[package]] name = "libsqlite3-sys" version = "0.32.0" @@ -1236,6 +1260,15 @@ version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" +[[package]] +name = "redox_syscall" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" +dependencies = [ + "bitflags 2.9.0", +] + [[package]] name = "redox_users" version = "0.3.5" @@ -1243,7 +1276,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" dependencies = [ "getrandom 0.1.16", - "redox_syscall", + "redox_syscall 0.1.57", "rust-argon2", ] diff --git a/Cargo.toml b/Cargo.toml index 8e6b48b..ce6263d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ log = "0.4.14" env_logger = "0.11.7" indicatif = "0.16.2" lofty = "0.22.2" +filetime = "0.2" dirs = "1" chrono = "0.4.40" regex = "1" diff --git a/ChangeLog b/ChangeLog index 2cc8333..f6aa878 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,6 +5,8 @@ 3. Don't crash if file has no tags. 4. Set log level for tag reader library. 5. Multi-threaded read of existing analysis tags. +6. Store file failures in DB so don't re-analyse unless timestamp of file + changes. 0.4.0 ----- diff --git a/src/analyse.rs b/src/analyse.rs index f74eee8..1940cd1 100644 --- a/src/analyse.rs +++ b/src/analyse.rs @@ -19,6 +19,8 @@ use indicatif::{ProgressBar, ProgressStyle}; #[cfg(not(feature = "ffmpeg"))] use std::collections::HashSet; use std::convert::TryInto; +use filetime::FileTime; +use std::fs; use std::fs::DirEntry; use std::num::{NonZero, NonZeroUsize}; use std::path::{Path, PathBuf}; @@ -151,8 +153,23 @@ fn check_dir_entry(db: &mut db::Db, mpath: &Path, entry: DirEntry, track_paths: } else { if let Ok(id) = db.get_rowid(&sname) { if id<=0 { - track_paths.push(String::from(pb.to_string_lossy())); - *file_count+=1; + // Check this file is not in Failures table + let ts = db.get_failure_timestamp(&sname); + if ts<=0 { + // ...nope, not in there so analyse + track_paths.push(String::from(pb.to_string_lossy())); + *file_count+=1; + } else { + let path = String::from(pb.to_string_lossy()); + let metadata = fs::metadata(&path).unwrap(); + let mtime = FileTime::from_last_modification_time(&metadata).unix_seconds(); + if mtime!=ts { + // ...was in failures table, but timestamp has changed + db.remove_from_failures(&sname); + track_paths.push(path); + *file_count+=1; + } + } } } } @@ -372,7 +389,12 @@ fn analyse_new_files(db: &db::Db, mpath: &PathBuf, track_paths: Vec, max } analysed += 1; } - Err(e) => { failed.push(format!("{} - {}", sname, e)); } + Err(e) => { + failed.push(format!("{} - {}", sname, e)); + let metadata = fs::metadata(path).unwrap(); + let mtime = FileTime::from_last_modification_time(&metadata); + db.add_to_failures(&sname, mtime.unix_seconds(), &format!("{}", e)); + } }; if inc_progress { diff --git a/src/db.rs b/src/db.rs index 8b7204f..f03e500 100644 --- a/src/db.rs +++ b/src/db.rs @@ -139,14 +139,35 @@ impl Db { ); if cmd.is_err() { - log::error!("Failed to create DB table"); + log::error!("Failed to create Files table"); process::exit(-1); } let cmd = self.conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS TracksV2_idx ON TracksV2(File)", []); if cmd.is_err() { - log::error!("Failed to create DB index"); + log::error!("Failed to create Files index"); + process::exit(-1); + } + + let cmd = self.conn.execute( + "CREATE TABLE IF NOT EXISTS Failures ( + File text primary key, + Timestamp integer, + Reason text + );", + [], + ); + + if cmd.is_err() { + log::error!("Failed to create Failures table"); + process::exit(-1); + } + + let cmd = self.conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS Failures_idx ON Failures(File)", []); + + if cmd.is_err() { + log::error!("Failed to create Failures index"); process::exit(-1); } } @@ -185,7 +206,7 @@ impl Db { analysis[AnalysisIndex::Chroma1], analysis[AnalysisIndex::Chroma2], analysis[AnalysisIndex::Chroma3], analysis[AnalysisIndex::Chroma4], analysis[AnalysisIndex::Chroma5], analysis[AnalysisIndex::Chroma6], analysis[AnalysisIndex::Chroma7], analysis[AnalysisIndex::Chroma8], analysis[AnalysisIndex::Chroma9], analysis[AnalysisIndex::Chroma10], analysis[AnalysisIndex::Chroma11], analysis[AnalysisIndex::Chroma12], analysis[AnalysisIndex::Chroma13]]) { - Ok(_) => { } + Ok(_) => { self.remove_from_failures(path); } Err(e) => { log::error!("Failed to insert '{}' into database. {}", path, e); } } } else { @@ -195,7 +216,7 @@ impl Db { analysis[AnalysisIndex::StdDeviationSpectralRolloff], analysis[AnalysisIndex::MeanSpectralFlatness], analysis[AnalysisIndex::StdDeviationSpectralFlatness], analysis[AnalysisIndex::MeanLoudness], analysis[AnalysisIndex::StdDeviationLoudness], analysis[AnalysisIndex::Chroma1], analysis[AnalysisIndex::Chroma2], analysis[AnalysisIndex::Chroma3], analysis[AnalysisIndex::Chroma4], analysis[AnalysisIndex::Chroma5], analysis[AnalysisIndex::Chroma6], analysis[AnalysisIndex::Chroma7], analysis[AnalysisIndex::Chroma8], analysis[AnalysisIndex::Chroma9], analysis[AnalysisIndex::Chroma10], id]) { - Ok(_) => { } + Ok(_) => { self.remove_from_failures(path); } Err(e) => { log::error!("Failed to update '{}' in database. {}", path, e); } } } @@ -204,9 +225,38 @@ impl Db { } } + pub fn add_to_failures(&self, path: &String, timestamp:i64, reason: &String) { + self.remove_from_failures(path); + let _ = self.conn.execute("INSERT INTO Failures (File, Timestamp, Reason) VALUES (?, ?, ?);", params![path, timestamp, reason]); + } + + pub fn remove_from_failures(&self, path: &String) { + let _ = self.conn.execute("DELETE FROM Failures WHERE File = ?;", params![path]); + } + pub fn remove_old(&self, mpaths: &Vec, dry_run: bool) { + self.remove_old_from_table(mpaths, dry_run, "TracksV2"); + self.remove_old_from_table(mpaths, dry_run, "Failures"); + } + + pub fn get_failure_timestamp(&self, path: &String) -> i64 { + let mut ts:i64 = 0; + let mut db_path = path.to_string(); + if cfg!(windows) { + db_path = db_path.replace("\\", "/"); + } + let mut stmt = self.conn.prepare("SELECT Timestamp FROM Failures WHERE File=:path;").unwrap(); + let track_iter = stmt.query_map(&[(":path", &db_path)], |row| Ok(row.get(0)?)).unwrap(); + for tr in track_iter { + ts = tr.unwrap(); + break; + } + ts + } + + fn remove_old_from_table(&self, mpaths: &Vec, dry_run: bool, table: &str) { log::info!("Looking for non-existent tracks"); - let mut stmt = self.conn.prepare("SELECT File FROM TracksV2;").unwrap(); + let mut stmt = self.conn.prepare(format!("SELECT File FROM {};", table).as_str()).unwrap(); let track_iter = stmt.query_map([], |row| Ok((row.get(0)?,))).unwrap(); let mut to_remove: Vec = Vec::new(); for tr in track_iter { @@ -250,7 +300,7 @@ impl Db { let count_before = self.get_track_count(); for t in to_remove { //log::debug!("Remove '{}'", t); - let cmd = self.conn.execute("DELETE FROM TracksV2 WHERE File = ?;", params![t]); + let cmd = self.conn.execute(format!("DELETE FROM {} WHERE File = ?;", table).as_str(), params![t]); if let Err(e) = cmd { log::error!("Failed to remove '{}' - {}", t, e)