STRLCPY/deduplicator

■ ■ ■ ■ ■ ■

.github/workflows/build_and_release.yml

1	+	name: Build and release
2	+
3	+	on:
4	+	push:
5	+	branches:
6	+	- main
7	+	release:
8	+	types: [created]
9	+
10	+	jobs:
11	+	build:
12	+	runs-on: ubuntu-latest
13	+	steps:
14	+	- name: Checkout code
15	+	uses: actions/checkout@v2
16	+
17	+	- name: Setup Rust
18	+	uses: actions-rs/toolchain@v1
19	+	with:
20	+	toolchain: stable
21	+	profile: minimal
22	+	override: true
23	+
24	+	- name: Build for Windows
25	+	run: \|
26	+	cargo build --release --target x86_64-pc-windows-gnu
27	+
28	+	- name: Build for Linux
29	+	run: \|
30	+	cargo build --release --target x86_64-unknown-linux-gnu
31	+
32	+	- name: Build for MacOS
33	+	run: \|
34	+	cargo build --release --target x86_64-apple-darwin
35	+
36	+	- name: Create release
37	+	uses: actions/create-release@v2
38	+	env:
39	+	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
40	+	with:
41	+	tag_name: ${{ github.ref }}
42	+	release_name: Release ${{ github.ref }}
43	+	draft: false
44	+	prerelease: false
45	+

■ ■ ■ ■ ■ ■

.github/workflows/rust.yml

1	-	name: Rust
2	-
3	-	on:
4	-	push:
5	-	branches: [ "main" ]
6	-	pull_request:
7	-	branches: [ "main" ]
8	-
9	-	env:
10	-	CARGO_TERM_COLOR: always
11	-
12	-	jobs:
13	-	build:
14	-	runs-on: ubuntu-latest
15	-	steps:
16	-	- uses: actions/checkout@v3
17	-	- name: Build
18	-	run: cargo build --verbose
19	-	- name: Run tests
20	-	run: cargo test --verbose
21	-

■ ■ ■ ■ ■ ■

Cargo.lock

		skipped 304 lines
305	305
306	306		[[package]]
307	307		name = "deduplicator"
308		-	version = "0.1.2"
	308	+	version = "0.1.3"
309	309		dependencies = [
310	310		"anyhow",
311	311		"bytesize",
		skipped 620 lines
932	932
933	933		[[package]]
934	934		name = "tokio"
935		-	version = "1.23.0"
	935	+	version = "1.23.1"
936	936		source = "registry+https://github.com/rust-lang/crates.io-index"
937		-	checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46"
	937	+	checksum = "38a54aca0c15d014013256222ba0ebed095673f89345dd79119d912eb561b7a8"
938	938		dependencies = [
939	939		"autocfg",
940	940		"bytes",
		skipped 201 lines

■ ■ ■ ■ ■ ■

Cargo.toml

1	1		[package]
2	2		name = "deduplicator"
3		-	version = "0.1.2"
	3	+	version = "0.1.3"
4	4		edition = "2021"
5	5		description = "find,filter,delete Duplicates"
6	6		license = "MIT"
		skipped 16 lines
23	23		prettytable-rs = "0.10.0"
24	24		rayon = "1.6.1"
25	25		thiserror = "1.0.38"
26		-	tokio = { version = "1.23.0", features = ["full"] }
	26	+	tokio = { version = "1.23.1", features = ["full"] }
27	27		unicode-segmentation = "1.10.0"
28	28

■ ■ ■ ■ ■ ■

src/output.rs

		skipped 39 lines
40	40		Ok(modified_time.format("%Y-%m-%d %H:%M:%S").to_string())
41	41		}
42	42
43		-	fn print_meta_info() {
44		-	println!("Deduplicator v{}", std::env!("CARGO_PKG_VERSION"));
45		-	}
46		-
47	43		fn scan_group_instruction() -> Result<String> {
48	44		println!("\nEnter the indices of the files you want to delete.");
49	45		println!("You can enter multiple files using commas to seperate file indices.");
		skipped 65 lines
115	111		}
116	112
117	113		pub fn interactive(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
118		-	print_meta_info();
119		-
120	114		if duplicates.is_empty() {
121	115		println!(
122	116		"\n{}",
		skipped 5 lines
128	122		duplicates
129	123		.clone()
130	124		.into_iter()
131		-	.sorted_unstable_by_key(\|f\| {
132		-	-(f.1.first().and_then(\|ff\| ff.size).unwrap_or_default() as i64)
	125	+	.sorted_unstable_by_key(\|(_, f)\| {
	126	+	-(f.first().and_then(\|ff\| ff.size).unwrap_or_default() as i64)
133	127		}) // sort by descending file size in interactive mode
134	128		.enumerate()
135	129		.for_each(\|(gindex, (_, group))\| {
		skipped 4 lines
140	134		itable.add_row(row![
141	135		index,
142	136		format_path(&file.path, opts).unwrap_or_default().blue(),
143		-	file_size(&file).unwrap_or_default().red(),
	137	+	file_size(file).unwrap_or_default().red(),
144	138		modified_time(&file.path).unwrap_or_default().yellow()
145	139		]);
146	140		});
		skipped 3 lines
150	144		}
151	145
152	146		pub fn print(duplicates: DashMap<String, Vec<File>>, opts: &Params) {
153		-	print_meta_info();
154		-
155	147		if duplicates.is_empty() {
156	148		println!(
157	149		"\n{}",
		skipped 6 lines
164	156		output_table.set_titles(row!["hash", "duplicates"]);
165	157		duplicates
166	158		.into_iter()
167		-	.sorted_unstable_by_key(\|f\| f.1.first().and_then(\|ff\| ff.size).unwrap_or_default()) // sort by ascending size
	159	+	.sorted_unstable_by_key(\|(_, f)\| f.first().and_then(\|ff\| ff.size).unwrap_or_default())
168	160		.for_each(\|(hash, group)\| {
169	161		let mut inner_table = Table::new();
170	162		inner_table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
171	163		group.iter().for_each(\|file\| {
172	164		inner_table.add_row(row![
173	165		format_path(&file.path, opts).unwrap_or_default().blue(),
174		-	file_size(&file).unwrap_or_default().red(),
	166	+	file_size(file).unwrap_or_default().red(),
175	167		modified_time(&file.path).unwrap_or_default().yellow()
176	168		]);
177	169		});
		skipped 6 lines

■ ■ ■ ■ ■ ■

src/params.rs

		skipped 32 lines
33	33		pub fn get_directory(&self) -> Result<String> {
34	34		let dir_pathbuf: PathBuf = self
35	35		.dir
36		-	.clone()
37		-	.unwrap_or(std::env::current_dir()?)
	36	+	.as_ref()
	37	+	.unwrap_or(&std::env::current_dir()?)
38	38		.as_os_str()
39	39		.into();
40	40
		skipped 6 lines
47	47		Ok(dir)
48	48		}
49	49
50		-	pub fn get_glob_patterns(&self) -> Vec<PathBuf> {
51		-	self.types
52		-	.clone()
53		-	.unwrap_or_else(\|\| String::from("*"))
54		-	.split(',')
55		-	.map(\|filetype\| format!("*.{}", filetype))
56		-	.map(\|filetype\| {
57		-	vec![self.get_directory().unwrap(), String::from("**"), filetype]
58		-	.iter()
59		-	.collect()
60		-	})
61		-	.collect()
	50	+	pub fn get_glob_patterns(&self) -> PathBuf {
	51	+	match self.types.as_ref() {
	52	+	Some(filetypes) => vec![
	53	+	self.get_directory().unwrap(),
	54	+	String::from("**"),
	55	+	format!("{{{}}}", filetypes),
	56	+	]
	57	+	.iter()
	58	+	.collect::<PathBuf>(),
	59	+	None => vec![self.get_directory().unwrap().as_str(), "*", ""]
	60	+	.iter()
	61	+	.collect::<PathBuf>(),
	62	+	}
62	63		}
63	64		}
64	65

■ ■ ■ ■ ■ ■

src/scanner.rs

		skipped 39 lines
40	40		}
41	41
42	42		fn scan(app_opts: &Params) -> Result<Vec<File>> {
43		-	let glob_patterns: Vec<PathBuf> = app_opts.get_glob_patterns();
44		-	let files: Vec<File> = glob_patterns
45		-	.par_iter()
	43	+	let glob_patterns = app_opts.get_glob_patterns().display().to_string();
	44	+	let glob_iter = glob(&glob_patterns)?;
	45	+	let files = glob_iter
	46	+	.filter(Result::is_ok)
	47	+	.map(\|file\| file.unwrap())
	48	+	.filter(\|fpath\| fpath.is_file())
	49	+	.collect::<Vec<PathBuf>>()
	50	+	.into_par_iter()
46	51		.progress_with_style(ProgressStyle::with_template(
47		-	"{spinner:.green} [scanning files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
	52	+	"{spinner:.green} [processing scan results] [{wide_bar:.cyan/blue}] {pos}/{len} files",
48	53		)?)
49		-	.filter_map(\|glob_pattern\| glob(glob_pattern.as_os_str().to_str()?).ok())
50		-	.flat_map(\|file_vec\| {
51		-	file_vec
52		-	.filter_map(\|x\| Some(x.ok()?.as_os_str().to_str()?.to_string()))
53		-	.filter(\|glob_result\| {
54		-	fs::metadata(glob_result)
55		-	.map(\|f\| f.is_file())
56		-	.unwrap_or(false)
57		-	})
58		-	.collect::<Vec<String>>()
59		-	})
60		-	.map(\|file_path\| File {
61		-	path: file_path.clone(),
	54	+	.map(\|fpath\| fpath.display().to_string())
	55	+	.map(\|fpath\| File {
	56	+	path: fpath.clone(),
62	57		hash: None,
63		-	size: Some(fs::metadata(file_path).unwrap().len()),
	58	+	size: Some(fs::metadata(fpath).unwrap().len()),
64	59		})
65	60		.filter(\|file\| filters::is_file_gt_minsize(app_opts, file))
66	61		.collect();
67	62
68	63		Ok(files)
69		-	}
70		-
71		-	fn process_file_hash_index(file: &File) -> Result<File> {
72		-	Ok(File {
73		-	path: file.path.clone(),
74		-	size: file.size,
75		-	hash: Some(hash_file(&file.path).unwrap_or_default()),
76		-	})
77	64		}
78	65
79	66		fn process_file_index(
80		-	file: File,
	67	+	mut file: File,
81	68		store: &DashMap<String, Vec<File>>,
82	69		index_criteria: IndexCritera,
83	70		) {
		skipped 5 lines
89	76		.or_insert_with(\|\| vec![file]);
90	77		}
91	78		IndexCritera::Hash => {
92		-	let processed_file = process_file_hash_index(&file).unwrap();
93		-	let indexhash = processed_file.clone().hash.unwrap_or_default();
94		-
	79	+	file.hash = Some(hash_file(&file.path).unwrap_or_default());
95	80		store
96		-	.entry(indexhash)
97		-	.and_modify(\|fileset\| fileset.push(processed_file.clone()))
98		-	.or_insert_with(\|\| vec![processed_file]);
	81	+	.entry(file.clone().hash.unwrap())
	82	+	.and_modify(\|fileset\| fileset.push(file.clone()))
	83	+	.or_insert_with(\|\| vec![file]);
99	84		}
100	85		}
101	86		}
		skipped 42 lines

Merge pull request #33 from sreedevk/development