STRLCPY/deduplicator

Merge pull request #32 from sreedevk/performance/glob_improvements
```
Performance/glob improvements
```
Sreedev Kodichath committed with GitHub 1 year ago

53d28abb

2 parents
c0042fc9
7b275d40

■ ■ ■ ■ ■ ■

src/output.rs

		skipped 39 lines
40	40		Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
41	41		}
42	42
43		-	fn print_meta_info() {
44		-	println!("Deduplicator v{}", std::env!("CARGO_PKG_VERSION"));
45		-	}
46		-
47	43		fn scan_group_instruction() -> Result<String> {
48	44		println!("\nEnter the indices of the files you want to delete.");
49	45		println!("You can enter multiple files using commas to seperate file indices.");
		skipped 65 lines
115	111		}
116	112
117	113		pub fn interactive(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
118		-	print_meta_info();
119		-
120	114		if duplicates.is_empty() {
121	115		println!(
122	116		"\n{}",
		skipped 5 lines
128	122		duplicates
129	123		.clone()
130	124		.into_iter()
131		-	.sorted_unstable_by_key(\|f\| {
132		-	-(f.1.first().and_then(\|ff\| ff.size).unwrap_or_default() as i64)
	125	+	.sorted_unstable_by_key(\|(_, f)\| {
	126	+	-(f.first().and_then(\|ff\| ff.size).unwrap_or_default() as i64)
133	127		}) // sort by descending file size in interactive mode
134	128		.enumerate()
135	129		.for_each(\|(gindex, (_, group))\| {
		skipped 4 lines
140	134		itable.add_row(row![
141	135		index,
142	136		format_path(&file.path, opts).unwrap_or_default().blue(),
143		-	file_size(&file).unwrap_or_default().red(),
	137	+	file_size(file).unwrap_or_default().red(),
144	138		modified_time(&file.path).unwrap_or_default().yellow()
145	139		]);
146	140		});
		skipped 3 lines
150	144		}
151	145
152	146		pub fn print(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
153		-	print_meta_info();
154		-
155	147		if duplicates.is_empty() {
156	148		println!(
157	149		"\n{}",
		skipped 6 lines
164	156		output_table.set_titles(row!["hash", "duplicates"]);
165	157		duplicates
166	158		.into_iter()
167		-	.sorted_unstable_by_key(\|f\| f.1.first().and_then(\|ff\| ff.size).unwrap_or_default()) // sort by ascending size
	159	+	.sorted_unstable_by_key(\|(_, f)\| f.first().and_then(\|ff\| ff.size).unwrap_or_default())
168	160		.for_each(\|(hash, group)\| {
169	161		let mut inner_table = Table::new();
170	162		inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
171	163		group.iter().for_each(\|file\| {
172	164		inner_table.add_row(row![
173	165		format_path(&file.path, opts).unwrap_or_default().blue(),
174		-	file_size(&file).unwrap_or_default().red(),
	166	+	file_size(file).unwrap_or_default().red(),
175	167		modified_time(&file.path).unwrap_or_default().yellow()
176	168		]);
177	169		});
		skipped 6 lines

■ ■ ■ ■ ■ ■

src/params.rs

		skipped 32 lines
33	33		pub fn get_directory(&self) -> Result<String> {
34	34		let dir_pathbuf: PathBuf = self
35	35		.dir
36		-	.clone()
37		-	.unwrap_or(std::env::current_dir()?)
	36	+	.as_ref()
	37	+	.unwrap_or(&std::env::current_dir()?)
38	38		.as_os_str()
39	39		.into();
40	40
		skipped 6 lines
47	47		Ok(dir)
48	48		}
49	49
50		-	pub fn get_glob_patterns(&self) -> Vec<PathBuf> {
51		-	self.types
52		-	.clone()
53		-	.unwrap_or_else(\|\| String::from("*"))
54		-	.split(',')
55		-	.map(\|filetype\| format!("*.{}", filetype))
56		-	.map(\|filetype\| {
57		-	vec![self.get_directory().unwrap(), String::from("**"), filetype]
58		-	.iter()
59		-	.collect()
60		-	})
61		-	.collect()
	50	+	pub fn get_glob_patterns(&self) -> PathBuf {
	51	+	match self.types.as_ref() {
	52	+	Some(filetypes) => vec![
	53	+	self.get_directory().unwrap(),
	54	+	String::from("**"),
	55	+	format!("{{{}}}", filetypes),
	56	+	]
	57	+	.iter()
	58	+	.collect::<PathBuf>(),
	59	+	None => vec![self.get_directory().unwrap().as_str(), "*", ""]
	60	+	.iter()
	61	+	.collect::<PathBuf>(),
	62	+	}
62	63		}
63	64		}
64	65

■ ■ ■ ■ ■ ■

src/scanner.rs

		skipped 39 lines
40	40		}
41	41
42	42		fn scan(app_opts: &Params) -> Result<Vec<File>> {
43		-	let glob_patterns: Vec<PathBuf> = app_opts.get_glob_patterns();
44		-	let files: Vec<File> = glob_patterns
45		-	.par_iter()
	43	+	let glob_patterns = app_opts.get_glob_patterns().display().to_string();
	44	+	let glob_iter = glob(&glob_patterns)?;
	45	+	let files = glob_iter
	46	+	.filter(Result::is_ok)
	47	+	.map(\|file\| file.unwrap())
	48	+	.filter(\|fpath\| fpath.is_file())
	49	+	.collect::<Vec<PathBuf>>()
	50	+	.into_par_iter()
46	51		.progress_with_style(ProgressStyle::with_template(
47		-	"{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
	52	+	"{spinner:.green} [processing scan results] [{wide_bar:.cyan/blue}] {pos}/{len} files",
48	53		)?)
49		-	.filter_map(\|glob_pattern\| glob(glob_pattern.as_os_str().to_str()?).ok())
50		-	.flat_map(\|file_vec\| {
51		-	file_vec
52		-	.filter_map(\|x\| Some(x.ok()?.as_os_str().to_str()?.to_string()))
53		-	.filter(\|glob_result\| {
54		-	fs::metadata(glob_result)
55		-	.map(\|f\| f.is_file())
56		-	.unwrap_or(false)
57		-	})
58		-	.collect::<Vec<String>>()
59		-	})
60		-	.map(\|file_path\| File {
61		-	path: file_path.clone(),
	54	+	.map(\|fpath\| fpath.display().to_string())
	55	+	.map(\|fpath\| File {
	56	+	path: fpath.clone(),
62	57		hash: None,
63		-	size: Some(fs::metadata(file_path).unwrap().len()),
	58	+	size: Some(fs::metadata(fpath).unwrap().len()),
64	59		})
65	60		.filter(\|file\| filters::is_file_gt_minsize(app_opts, file))
66	61		.collect();
67	62
68	63		Ok(files)
69		-	}
70		-
71		-	fn process_file_hash_index(file: &File) -> Result<File> {
72		-	Ok(File {
73		-	path: file.path.clone(),
74		-	size: file.size,
75		-	hash: Some(hash_file(&file.path).unwrap_or_default()),
76		-	})
77	64		}
78	65
79	66		fn process_file_index(
80		-	file: File,
	67	+	mut file: File,
81	68		store: &DashMap<String, Vec<File>>,
82	69		index_criteria: IndexCritera,
83	70		) {
		skipped 5 lines
89	76		.or_insert_with(\|\| vec![file]);
90	77		}
91	78		IndexCritera::Hash => {
92		-	let processed_file = process_file_hash_index(&file).unwrap();
93		-	let indexhash = processed_file.clone().hash.unwrap_or_default();
94		-
	79	+	file.hash = Some(hash_file(&file.path).unwrap_or_default());
95	80		store
96		-	.entry(indexhash)
97		-	.and_modify(\|fileset\| fileset.push(processed_file.clone()))
98		-	.or_insert_with(\|\| vec![processed_file]);
	81	+	.entry(file.clone().hash.unwrap())
	82	+	.and_modify(\|fileset\| fileset.push(file.clone()))
	83	+	.or_insert_with(\|\| vec![file]);
99	84		}
100	85		}
101	86		}
		skipped 42 lines

Merge pull request #32 from sreedevk/performance/glob_improvements