Store file failures in DB so don't re-analyse unless timestamp of file changes.

Cargo.lock

@@ -211,6 +211,7 @@ dependencies = [
  "ctrlc",
  "dirs",
  "env_logger",
+ "filetime",
  "hhmmss",
  "if_chain",
  "indicatif",
@@ -587,6 +588,18 @@ dependencies = [
  "vcpkg",
 ]
 
+[[package]]
+name = "filetime"
+version = "0.2.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "libredox",
+ "windows-sys 0.60.2",
+]
+
 [[package]]
 name = "flate2"
 version = "1.0.34"
@@ -850,6 +863,17 @@ version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa"
 
+[[package]]
+name = "libredox"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb"
+dependencies = [
+ "bitflags 2.9.0",
+ "libc",
+ "redox_syscall 0.5.17",
+]
+
 [[package]]
 name = "libsqlite3-sys"
 version = "0.32.0"
@@ -1236,6 +1260,15 @@ version = "0.1.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
 
+[[package]]
+name = "redox_syscall"
+version = "0.5.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77"
+dependencies = [
+ "bitflags 2.9.0",
+]
+
 [[package]]
 name = "redox_users"
 version = "0.3.5"
@@ -1243,7 +1276,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d"
 dependencies = [
  "getrandom 0.1.16",
- "redox_syscall",
+ "redox_syscall 0.1.57",
  "rust-argon2",
 ]
 

Cargo.toml

@@ -20,6 +20,7 @@ log = "0.4.14"
 env_logger = "0.11.7"
 indicatif = "0.16.2"
 lofty = "0.22.2"
+filetime = "0.2"
 dirs = "1"
 chrono = "0.4.40"
 regex = "1"

ChangeLog

@@ -5,6 +5,8 @@
 3. Don't crash if file has no tags.
 4. Set log level for tag reader library.
 5. Multi-threaded read of existing analysis tags.
+6. Store file failures in DB so don't re-analyse unless timestamp of file
+   changes.
 
 0.4.0
 -----

src/analyse.rs

@@ -19,6 +19,8 @@ use indicatif::{ProgressBar, ProgressStyle};
 #[cfg(not(feature = "ffmpeg"))]
 use std::collections::HashSet;
 use std::convert::TryInto;
+use filetime::FileTime;
+use std::fs;
 use std::fs::DirEntry;
 use std::num::{NonZero, NonZeroUsize};
 use std::path::{Path, PathBuf};
@@ -151,8 +153,23 @@ fn check_dir_entry(db: &mut db::Db, mpath: &Path, entry: DirEntry, track_paths:
                 } else {
                     if let Ok(id) = db.get_rowid(&sname) {
                         if id<=0 {
-                            track_paths.push(String::from(pb.to_string_lossy()));
-                            *file_count+=1;
+                            // Check this file is not in Failures table
+                            let ts = db.get_failure_timestamp(&sname);
+                            if ts<=0 {
+                                // ...nope, not in there so analyse
+                                track_paths.push(String::from(pb.to_string_lossy()));
+                                *file_count+=1;
+                            } else {
+                                let path = String::from(pb.to_string_lossy());
+                                let metadata = fs::metadata(&path).unwrap();
+                                let mtime = FileTime::from_last_modification_time(&metadata).unix_seconds();
+                                if mtime!=ts {
+                                    // ...was in failures table, but timestamp has changed
+                                    db.remove_from_failures(&sname);
+                                    track_paths.push(path);
+                                    *file_count+=1;
+                                }
+                            }
                         }
                     }
                 }
@@ -372,7 +389,12 @@ fn analyse_new_files(db: &db::Db, mpath: &PathBuf, track_paths: Vec<String>, max
                 }
                 analysed += 1;
             }
-            Err(e) => { failed.push(format!("{} - {}", sname, e)); }
+            Err(e) => {
+                failed.push(format!("{} - {}", sname, e));
+                let metadata = fs::metadata(path).unwrap();
+                let mtime = FileTime::from_last_modification_time(&metadata);
+                db.add_to_failures(&sname, mtime.unix_seconds(), &format!("{}", e));
+            }
         };
 
         if inc_progress {

src/db.rs

@@ -139,14 +139,35 @@ impl Db {
         );
 
         if cmd.is_err() {
-            log::error!("Failed to create DB table");
+            log::error!("Failed to create Files table");
             process::exit(-1);
         }
 
         let cmd = self.conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS TracksV2_idx ON TracksV2(File)", []);
 
         if cmd.is_err() {
-            log::error!("Failed to create DB index");
+            log::error!("Failed to create Files index");
+            process::exit(-1);
+        }
+
+        let cmd = self.conn.execute(
+            "CREATE TABLE IF NOT EXISTS Failures (
+                File text primary key,
+                Timestamp integer,
+                Reason text
+            );",
+            [],
+        );
+
+        if cmd.is_err() {
+            log::error!("Failed to create Failures table");
+            process::exit(-1);
+        }
+
+        let cmd = self.conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS Failures_idx ON Failures(File)", []);
+
+        if cmd.is_err() {
+            log::error!("Failed to create Failures index");
             process::exit(-1);
         }
     }
@@ -185,7 +206,7 @@ impl Db {
                             analysis[AnalysisIndex::Chroma1], analysis[AnalysisIndex::Chroma2], analysis[AnalysisIndex::Chroma3], analysis[AnalysisIndex::Chroma4], analysis[AnalysisIndex::Chroma5],
                             analysis[AnalysisIndex::Chroma6], analysis[AnalysisIndex::Chroma7], analysis[AnalysisIndex::Chroma8], analysis[AnalysisIndex::Chroma9], analysis[AnalysisIndex::Chroma10],
                             analysis[AnalysisIndex::Chroma11], analysis[AnalysisIndex::Chroma12], analysis[AnalysisIndex::Chroma13]]) {
-                        Ok(_) => { }
+                        Ok(_) => { self.remove_from_failures(path); }
                         Err(e) => { log::error!("Failed to insert '{}' into database. {}", path, e); }
                     }
                 } else {
@@ -195,7 +216,7 @@ impl Db {
                             analysis[AnalysisIndex::StdDeviationSpectralRolloff], analysis[AnalysisIndex::MeanSpectralFlatness], analysis[AnalysisIndex::StdDeviationSpectralFlatness], analysis[AnalysisIndex::MeanLoudness], analysis[AnalysisIndex::StdDeviationLoudness],
                             analysis[AnalysisIndex::Chroma1], analysis[AnalysisIndex::Chroma2], analysis[AnalysisIndex::Chroma3], analysis[AnalysisIndex::Chroma4], analysis[AnalysisIndex::Chroma5],
                             analysis[AnalysisIndex::Chroma6], analysis[AnalysisIndex::Chroma7], analysis[AnalysisIndex::Chroma8], analysis[AnalysisIndex::Chroma9], analysis[AnalysisIndex::Chroma10], id]) {
-                        Ok(_) => { }
+                        Ok(_) => { self.remove_from_failures(path); }
                         Err(e) => { log::error!("Failed to update '{}' in database. {}", path, e); }
                     }
                 }
@@ -204,9 +225,38 @@ impl Db {
         }
     }
 
+    pub fn add_to_failures(&self, path: &String, timestamp:i64, reason: &String) {
+        self.remove_from_failures(path);
+        let _ = self.conn.execute("INSERT INTO Failures (File, Timestamp, Reason) VALUES (?, ?, ?);", params![path, timestamp, reason]);
+    }
+
+    pub fn remove_from_failures(&self, path: &String) {
+        let _ = self.conn.execute("DELETE FROM Failures WHERE File = ?;", params![path]);
+    }
+
     pub fn remove_old(&self, mpaths: &Vec<PathBuf>, dry_run: bool) {
+        self.remove_old_from_table(mpaths, dry_run, "TracksV2");
+        self.remove_old_from_table(mpaths, dry_run, "Failures");
+    }
+
+    pub fn get_failure_timestamp(&self, path: &String) -> i64 {
+        let mut ts:i64 = 0;
+        let mut db_path = path.to_string();
+        if cfg!(windows) {
+            db_path = db_path.replace("\\", "/");
+        }
+        let mut stmt = self.conn.prepare("SELECT Timestamp FROM Failures WHERE File=:path;").unwrap();
+        let track_iter = stmt.query_map(&[(":path", &db_path)], |row| Ok(row.get(0)?)).unwrap();
+        for tr in track_iter {
+            ts = tr.unwrap();
+            break;
+        }
+        ts
+    }
+
+    fn remove_old_from_table(&self, mpaths: &Vec<PathBuf>, dry_run: bool, table: &str) {
         log::info!("Looking for non-existent tracks");
-        let mut stmt = self.conn.prepare("SELECT File FROM TracksV2;").unwrap();
+        let mut stmt = self.conn.prepare(format!("SELECT File FROM {};", table).as_str()).unwrap();
         let track_iter = stmt.query_map([], |row| Ok((row.get(0)?,))).unwrap();
         let mut to_remove: Vec<String> = Vec::new();
         for tr in track_iter {
@@ -250,7 +300,7 @@ impl Db {
                 let count_before = self.get_track_count();
                 for t in to_remove {
                     //log::debug!("Remove '{}'", t);
-                    let cmd = self.conn.execute("DELETE FROM TracksV2 WHERE File = ?;", params![t]);
+                    let cmd = self.conn.execute(format!("DELETE FROM {} WHERE File = ?;", table).as_str(), params![t]);
 
                     if let Err(e) = cmd {
                         log::error!("Failed to remove '{}' - {}", t, e)