1use crate::document::file_kind;
2use crate::fl;
3use crate::helpers::{Fingerprint, Fp, IsHidden};
4use crate::library::db::Db as LibraryDb;
5use crate::metadata::{FileInfo, Info, extract_metadata_from_document};
6use crate::settings::ImportSettings;
7use crate::task::ShutdownSignal;
8use crate::view::{Event, NotificationEvent, ViewId};
9use fxhash::FxHashMap;
10use std::path::{Path, PathBuf};
11use std::sync::mpsc::Sender;
12use tracing::{debug, error, info};
13use walkdir::{DirEntry, WalkDir};
14
15enum PendingRelocation {
16 FingerprintChanged {
17 new_fp: Fp,
18 old_fp: Fp,
19 file_size: u64,
20 },
21}
22
23impl PendingRelocation {
24 fn old_fp(&self) -> Fp {
25 match self {
26 PendingRelocation::FingerprintChanged { old_fp, .. } => *old_fp,
27 }
28 }
29}
30
31struct ScanContext<'a> {
32 hub: &'a Sender<Event>,
33 notif_id: ViewId,
34 shutdown: &'a ShutdownSignal,
35}
36
37struct ScanResult {
38 books_to_insert: Vec<(Fp, Info)>,
39 path_updates: Vec<(Fp, PathBuf, PathBuf)>,
40 books_to_delete: Vec<Fp>,
41 pending_relocations: Vec<PendingRelocation>,
42 thumbnails_to_delete: Vec<Fp>,
43}
44
45#[cfg(feature = "emulator")]
46const IGNORED_TOP_LEVEL_DIRS: &[&str] = &["target", "node_modules", "thirdparty"];
47
48#[cfg_attr(feature = "tracing", tracing::instrument(skip(home)))]
49fn walk_files(home: &Path) -> Vec<DirEntry> {
50 WalkDir::new(home)
51 .min_depth(1)
52 .into_iter()
53 .filter_entry(|e| {
54 if e.is_hidden() {
55 return false;
56 }
57 #[cfg(feature = "emulator")]
58 if e.depth() == 1 && e.file_type().is_dir() {
59 if let Some(name) = e.file_name().to_str() {
60 if IGNORED_TOP_LEVEL_DIRS.contains(&name) {
61 return false;
62 }
63 }
64 }
65 true
66 })
67 .filter_map(|e| e.ok())
68 .filter(|e| !e.file_type().is_dir())
69 .collect()
70}
71
72#[cfg_attr(
73 feature = "tracing",
74 tracing::instrument(
75 skip(home, settings, ctx, handles_by_fp, handles_by_path),
76 fields(total)
77 )
78)]
79fn scan_entries(
80 home: &Path,
81 entries: &[DirEntry],
82 settings: &ImportSettings,
83 ctx: &ScanContext<'_>,
84 handles_by_fp: &mut FxHashMap<Fp, PathBuf>,
85 handles_by_path: &mut FxHashMap<PathBuf, Fp>,
86) -> Option<ScanResult> {
87 let total = entries.len();
88 tracing::Span::current().record("total", total);
89
90 let mut books_to_insert: Vec<(Fp, Info)> = Vec::new();
91 let mut path_updates: Vec<(Fp, PathBuf, PathBuf)> = Vec::new();
92 let mut books_to_delete: Vec<Fp> = Vec::new();
93 let mut pending_relocations: Vec<PendingRelocation> = Vec::new();
94 let mut thumbnails_to_delete: Vec<Fp> = Vec::new();
95
96 for (idx, entry) in entries.iter().enumerate() {
97 if ctx.shutdown.should_stop() {
98 tracing::info!("import scan interrupted by shutdown");
99 return None;
100 }
101
102 let path = entry.path();
103 let relat = path.strip_prefix(home).unwrap_or(path);
104
105 let kind = file_kind(path);
106 let is_known_to_db = handles_by_path.contains_key(relat);
107 let allowed_kind = kind.filter(|k| settings.is_kind_allowed(*k));
108
109 if !is_known_to_db && allowed_kind.is_none() {
110 send_progress(ctx.hub, ctx.notif_id, idx, total);
111 continue;
112 }
113
114 let fp = match path.fingerprint() {
115 Ok(fp) => fp,
116 Err(e) => {
117 error!(path = ?path, error = %e, "failed to compute fingerprint, skipping");
118 send_progress(ctx.hub, ctx.notif_id, idx, total);
119 continue;
120 }
121 };
122
123 if handles_by_fp.contains_key(&fp) {
124 if relat != handles_by_fp[&fp] {
125 debug!(
126 fp = %fp,
127 old_path = %handles_by_fp[&fp].display(),
128 new_path = %relat.display(),
129 "updated book path"
130 );
131 let old_path = handles_by_fp.remove(&fp).unwrap();
132 handles_by_path.remove(&old_path);
133 handles_by_fp.insert(fp, relat.to_path_buf());
134 handles_by_path.insert(relat.to_path_buf(), fp);
135 path_updates.push((fp, relat.to_path_buf(), path.to_path_buf()));
136 }
137 send_progress(ctx.hub, ctx.notif_id, idx, total);
138 continue;
139 }
140
141 if let Some(old_fp) = handles_by_path.get(relat).cloned() {
142 debug!(
143 path = %relat.display(),
144 old_fp = %old_fp,
145 new_fp = %fp,
146 "updated book fingerprint"
147 );
148
149 handles_by_fp.remove(&old_fp);
150 handles_by_path.remove(relat);
151 handles_by_fp.insert(fp, relat.to_path_buf());
152 handles_by_path.insert(relat.to_path_buf(), fp);
153 books_to_delete.push(old_fp);
154
155 pending_relocations.push(PendingRelocation::FingerprintChanged {
156 new_fp: fp,
157 old_fp,
158 file_size: entry.metadata().map(|m| m.len()).unwrap_or(0),
159 });
160
161 thumbnails_to_delete.push(old_fp);
162 send_progress(ctx.hub, ctx.notif_id, idx, total);
163 continue;
164 }
165
166 if let Some(kind) = allowed_kind {
167 info!(fp = %fp, path = %relat.display(), "added new entry");
168 let size = entry.metadata().map(|m| m.len()).unwrap_or(0);
169 let mut info = Info {
170 file: FileInfo {
171 path: relat.to_path_buf(),
172 absolute_path: path.to_path_buf(),
173 kind: kind.as_str().to_owned(),
174 size,
175 },
176 ..Default::default()
177 };
178 if settings.metadata_kinds.contains(&info.file.kind) {
179 extract_metadata_from_document(home, &mut info);
180 }
181 handles_by_fp.insert(fp, relat.to_path_buf());
182 handles_by_path.insert(relat.to_path_buf(), fp);
183 books_to_insert.push((fp, info));
184 }
185
186 send_progress(ctx.hub, ctx.notif_id, idx, total);
187 }
188
189 Some(ScanResult {
190 books_to_insert,
191 path_updates,
192 books_to_delete,
193 pending_relocations,
194 thumbnails_to_delete,
195 })
196}
197
198fn send_progress(hub: &Sender<Event>, notif_id: ViewId, idx: usize, total: usize) {
199 let Some(current_percent) = ((idx + 1) * 100).checked_div(total) else {
200 return;
201 };
202 let Some(previous_percent) = (idx * 100).checked_div(total) else {
203 return;
204 };
205
206 let current_bucket = current_percent / 5;
207 if current_bucket == previous_percent / 5 {
208 return;
209 }
210
211 let percent = (current_bucket * 5).min(100) as u8;
212 debug!(percent, "import progress");
213 hub.send(Event::Notification(NotificationEvent::UpdateProgress(
214 notif_id, percent,
215 )))
216 .ok();
217}
218
219#[cfg_attr(
220 feature = "tracing",
221 tracing::instrument(skip(db, home, settings, pending_relocations, books_to_insert))
222)]
223fn resolve_relocations(
224 db: &LibraryDb,
225 library_id: i64,
226 home: &Path,
227 settings: &ImportSettings,
228 pending_relocations: Vec<PendingRelocation>,
229 books_to_insert: &mut Vec<(Fp, Info)>,
230) {
231 let old_fps: Vec<Fp> = pending_relocations
232 .iter()
233 .map(PendingRelocation::old_fp)
234 .collect();
235
236 let mut fetched = db
237 .batch_get_books_by_fingerprints(library_id, &old_fps)
238 .unwrap_or_default();
239
240 for relocation in pending_relocations {
241 match relocation {
242 PendingRelocation::FingerprintChanged {
243 new_fp,
244 old_fp,
245 file_size,
246 } => {
247 if let Some(mut info) = fetched.remove(&old_fp) {
248 if settings.sync_metadata && settings.metadata_kinds.contains(&info.file.kind) {
249 extract_metadata_from_document(home, &mut info);
250 }
251 info.file.size = file_size;
252 books_to_insert.push((new_fp, info));
253 }
254 }
255 }
256 }
257}
258
259#[cfg_attr(feature = "tracing", tracing::instrument(skip(handles_by_fp, home)))]
260fn find_deleted_books(handles_by_fp: &FxHashMap<Fp, PathBuf>, home: &Path) -> Vec<Fp> {
261 handles_by_fp
262 .iter()
263 .filter(|(_, relat)| relat.as_os_str().is_empty() || !home.join(relat).exists())
264 .map(|(fp, relat)| {
265 info!(fp = %fp, path = %relat.display(), "removing deleted entry");
266 *fp
267 })
268 .collect()
269}
270
271#[cfg_attr(
272 feature = "tracing",
273 tracing::instrument(skip(
274 db,
275 books_to_insert,
276 path_updates,
277 books_to_delete,
278 thumbnails_to_delete
279 ))
280)]
281fn flush_to_db(
282 db: &LibraryDb,
283 library_id: i64,
284 books_to_insert: Vec<(Fp, Info)>,
285 path_updates: Vec<(Fp, PathBuf, PathBuf)>,
286 books_to_delete: Vec<Fp>,
287 thumbnails_to_delete: Vec<Fp>,
288) {
289 if let Err(e) = db.batch_delete_thumbnails(&thumbnails_to_delete) {
290 error!(
291 error = %e,
292 count = thumbnails_to_delete.len(),
293 "batch delete thumbnails failed"
294 );
295 }
296
297 if !books_to_insert.is_empty() {
298 let book_refs: Vec<(Fp, &Info)> = books_to_insert
299 .iter()
300 .map(|(fp, info)| (*fp, info))
301 .collect();
302 if let Err(e) = db.batch_insert_books(library_id, &book_refs) {
303 error!(error = %e, count = book_refs.len(), "batch insert failed");
304 }
305 }
306
307 if let Err(e) = db.batch_update_book_paths(library_id, &path_updates) {
308 error!(
309 error = %e,
310 count = path_updates.len(),
311 "batch update book paths failed"
312 );
313 }
314
315 if !books_to_delete.is_empty() {
316 if let Err(e) = db.batch_delete_books(library_id, &books_to_delete) {
317 error!(error = %e, count = books_to_delete.len(), "batch delete failed");
318 }
319 }
320
321 if let Err(e) = db.compute_sort_keys(library_id) {
322 error!(error = %e, library_id, "failed to compute sort keys");
323 }
324}
325
326#[cfg_attr(
332 feature = "tracing",
333 tracing::instrument(skip(db, settings, hub, notif_id, shutdown))
334)]
335pub fn run(
336 db: &LibraryDb,
337 library_id: i64,
338 home: &Path,
339 settings: &ImportSettings,
340 hub: &Sender<Event>,
341 notif_id: ViewId,
342 shutdown: &ShutdownSignal,
343) {
344 hub.send(Event::Notification(NotificationEvent::ShowPinned(
345 notif_id,
346 fl!("importer-importing-library"),
347 )))
348 .ok();
349
350 let handles = match db.list_book_handles(library_id) {
351 Ok(h) => h,
352 Err(e) => {
353 error!(error = %e, "failed to load book handles for import");
354 hub.send(Event::Close(notif_id)).ok();
355 return;
356 }
357 };
358
359 let mut handles_by_fp: FxHashMap<Fp, PathBuf> = handles.iter().cloned().collect();
360 let mut handles_by_path: FxHashMap<PathBuf, Fp> =
361 handles.into_iter().map(|(fp, p)| (p, fp)).collect();
362
363 let purged_fps = db
364 .delete_books_with_disallowed_kinds(library_id, &settings.allowed_kinds)
365 .unwrap_or_else(|e| {
366 error!(error = %e, "failed to purge disallowed books");
367 Vec::new()
368 });
369
370 for fp in &purged_fps {
371 if let Some(path) = handles_by_fp.remove(fp) {
372 handles_by_path.remove(&path);
373 }
374 }
375
376 if !purged_fps.is_empty() {
377 if let Err(e) = db.batch_delete_thumbnails(&purged_fps) {
378 error!(error = %e, count = purged_fps.len(), "failed to delete thumbnails for purged books");
379 }
380 }
381
382 let entries = walk_files(home);
383
384 let ctx = ScanContext {
385 hub,
386 notif_id,
387 shutdown,
388 };
389
390 let Some(mut result) = scan_entries(
391 home,
392 &entries,
393 settings,
394 &ctx,
395 &mut handles_by_fp,
396 &mut handles_by_path,
397 ) else {
398 hub.send(Event::Close(notif_id)).ok();
399 return;
400 };
401
402 let mut deleted = find_deleted_books(&handles_by_fp, home);
403 result.books_to_delete.append(&mut deleted);
404
405 if !result.pending_relocations.is_empty() {
406 resolve_relocations(
407 db,
408 library_id,
409 home,
410 settings,
411 result.pending_relocations,
412 &mut result.books_to_insert,
413 );
414 }
415
416 flush_to_db(
417 db,
418 library_id,
419 result.books_to_insert,
420 result.path_updates,
421 result.books_to_delete,
422 result.thumbnails_to_delete,
423 );
424
425 hub.send(Event::Close(notif_id)).ok();
426}
427
428#[cfg(test)]
429mod tests {
430 use super::*;
431 use crate::db::Database;
432 use crate::library::Library;
433 use crate::metadata::{FileInfo, Info};
434 use crate::settings::ImportSettings;
435 use crate::task::ShutdownSignal;
436 use crate::view::ViewId;
437 use std::sync::mpsc;
438
439 fn create_migrated_db() -> Database {
440 let db = Database::new(":memory:").expect("in-memory db");
441 db.migrate().expect("migrations");
442 db
443 }
444
445 fn run_import(dir: &Path, db: &Database, shutdown: &ShutdownSignal) -> Vec<Event> {
446 let lib = Library::new(dir, db, "test").expect("failed to create library");
447 let (tx, rx) = mpsc::channel();
448 let notif_id = ViewId::MessageNotif(0);
449 run(
450 &lib.db,
451 lib.library_id,
452 dir,
453 &ImportSettings::default(),
454 &tx,
455 notif_id,
456 shutdown,
457 );
458 drop(tx);
459 rx.try_iter().collect()
460 }
461
462 #[test]
463 fn imports_files_when_not_shutdown() {
464 let dir = tempfile::tempdir().expect("tempdir");
465 let db = create_migrated_db();
466 std::fs::write(dir.path().join("book.epub"), b"epub content").expect("write");
467
468 let shutdown = ShutdownSignal::never();
469 let events = run_import(dir.path(), &db, &shutdown);
470
471 assert!(
472 events.iter().any(|e| matches!(e, Event::Close(_))),
473 "expected Close event on normal completion"
474 );
475 assert!(
476 !events.iter().any(|e| matches!(
477 e,
478 Event::Notification(crate::view::NotificationEvent::UpdateProgress(_, 0))
479 )),
480 "progress should advance past 0"
481 );
482 }
483
484 #[test]
485 fn stops_early_when_shutdown_requested() {
486 let dir = tempfile::tempdir().expect("tempdir");
487 let db = create_migrated_db();
488
489 for i in 0..20 {
490 std::fs::write(dir.path().join(format!("book{i}.epub")), b"epub content")
491 .expect("write");
492 }
493
494 let (shutdown_tx, shutdown_rx) = mpsc::channel();
495 let shutdown = ShutdownSignal::new_for_test(shutdown_rx);
496
497 shutdown_tx.send(()).expect("send shutdown");
499
500 let lib = Library::new(dir.path(), &db, "test").expect("library");
501 let (tx, rx) = mpsc::channel();
502 let notif_id = ViewId::MessageNotif(0);
503 run(
504 &lib.db,
505 lib.library_id,
506 dir.path(),
507 &ImportSettings::default(),
508 &tx,
509 notif_id,
510 &shutdown,
511 );
512 drop(tx);
513 let events: Vec<Event> = rx.try_iter().collect();
514
515 assert!(
516 events.iter().any(|e| matches!(e, Event::Close(_))),
517 "notif must be closed even on early exit"
518 );
519
520 let progress_events: Vec<_> = events
521 .iter()
522 .filter(|e| {
523 matches!(
524 e,
525 Event::Notification(crate::view::NotificationEvent::UpdateProgress(_, _))
526 )
527 })
528 .collect();
529 assert!(
530 progress_events.len() < 20,
531 "shutdown should have cut the scan short (got {} progress events)",
532 progress_events.len()
533 );
534 }
535
536 #[test]
537 fn sends_progress_in_five_percent_steps() {
538 let (tx, rx) = mpsc::channel();
539 let notif_id = ViewId::MessageNotif(0);
540
541 for idx in 0..72 {
542 send_progress(&tx, notif_id, idx, 72);
543 }
544
545 drop(tx);
546 let percents: Vec<u8> = rx
547 .try_iter()
548 .filter_map(|event| match event {
549 Event::Notification(NotificationEvent::UpdateProgress(_, percent)) => Some(percent),
550 _ => None,
551 })
552 .collect();
553
554 assert_eq!(
555 percents,
556 (5..=100).step_by(5).collect::<Vec<_>>(),
557 "progress should emit each 5% step exactly once"
558 );
559 }
560
561 #[test]
562 fn finds_deleted_books_when_file_path_is_empty() {
563 let dir = tempfile::tempdir().expect("tempdir");
564 let db = create_migrated_db();
565 let lib = Library::new(dir.path(), &db, "test").expect("library");
566 let fp = Fp::from_u64(1);
567 let info = Info {
568 title: "test".to_string(),
569 file: FileInfo {
570 path: PathBuf::new(),
571 absolute_path: dir.path().join("missing.epub"),
572 kind: "epub".to_string(),
573 size: 1,
574 },
575 ..Default::default()
576 };
577
578 lib.db
579 .batch_insert_books(lib.library_id, &[(fp, &info)])
580 .expect("insert library book");
581
582 let handles = lib.db.list_book_handles(lib.library_id).expect("handles");
583 let handles_by_fp: FxHashMap<Fp, PathBuf> = handles.into_iter().collect();
584
585 assert_eq!(find_deleted_books(&handles_by_fp, dir.path()), vec![fp]);
586 }
587
588 #[test]
589 fn skips_fingerprinting_disallowed_new_files() {
590 use crate::settings::FileExtension;
591 use fxhash::FxHashSet;
592
593 let dir = tempfile::tempdir().expect("tempdir");
594 let db = create_migrated_db();
595
596 std::fs::write(dir.path().join("book.epub"), b"epub content").expect("write epub");
597 std::fs::write(dir.path().join("ignore.xyz"), b"unsupported content").expect("write xyz");
598
599 let mut allowed: FxHashSet<FileExtension> = FxHashSet::default();
600 allowed.insert(FileExtension::Epub);
601
602 let settings = ImportSettings {
603 allowed_kinds: allowed,
604 ..ImportSettings::default()
605 };
606
607 let lib = Library::new(dir.path(), &db, "test").expect("library");
608 let (tx, rx) = std::sync::mpsc::channel();
609 let notif_id = ViewId::MessageNotif(0);
610 let shutdown = ShutdownSignal::never();
611
612 run(
613 &lib.db,
614 lib.library_id,
615 dir.path(),
616 &settings,
617 &tx,
618 notif_id,
619 &shutdown,
620 );
621 drop(tx);
622 let _events: Vec<Event> = rx.try_iter().collect();
623
624 let handles = lib.db.list_book_handles(lib.library_id).expect("handles");
625 let paths: Vec<_> = handles.iter().map(|(_, p)| p.clone()).collect();
626
627 assert!(
628 paths.iter().any(|p| p.ends_with("book.epub")),
629 "epub should be imported"
630 );
631 assert!(
632 !paths.iter().any(|p| p.ends_with("ignore.xyz")),
633 "unsupported kind should not be imported"
634 );
635 }
636
637 #[test]
638 fn purges_disallowed_books_on_import() {
639 use crate::settings::FileExtension;
640 use fxhash::FxHashSet;
641
642 let dir = tempfile::tempdir().expect("tempdir");
643 let db = create_migrated_db();
644
645 std::fs::write(dir.path().join("book.epub"), b"epub content").expect("write epub");
646 std::fs::write(dir.path().join("doc.pdf"), b"pdf content").expect("write pdf");
647
648 let lib = Library::new(dir.path(), &db, "test").expect("library");
649 let (tx, rx) = std::sync::mpsc::channel();
650 let notif_id = ViewId::MessageNotif(0);
651 let shutdown = ShutdownSignal::never();
652
653 run(
654 &lib.db,
655 lib.library_id,
656 dir.path(),
657 &ImportSettings::default(),
658 &tx,
659 notif_id,
660 &shutdown,
661 );
662 drop(tx);
663 let _: Vec<Event> = rx.try_iter().collect();
664
665 let handles = lib.db.list_book_handles(lib.library_id).expect("handles");
666 assert_eq!(handles.len(), 2, "both files should be imported initially");
667
668 let mut epub_only: FxHashSet<FileExtension> = FxHashSet::default();
669 epub_only.insert(FileExtension::Epub);
670
671 let settings = ImportSettings {
672 allowed_kinds: epub_only,
673 ..ImportSettings::default()
674 };
675
676 let (tx2, rx2) = std::sync::mpsc::channel();
677 run(
678 &lib.db,
679 lib.library_id,
680 dir.path(),
681 &settings,
682 &tx2,
683 notif_id,
684 &shutdown,
685 );
686 drop(tx2);
687 let _: Vec<Event> = rx2.try_iter().collect();
688
689 let handles = lib
690 .db
691 .list_book_handles(lib.library_id)
692 .expect("handles after purge");
693 let paths: Vec<_> = handles.iter().map(|(_, p)| p.clone()).collect();
694
695 assert_eq!(handles.len(), 1, "only epub should remain after purge");
696 assert!(
697 paths.iter().any(|p| p.ends_with("book.epub")),
698 "epub should still be present"
699 );
700 }
701}