STRLCPY/deduplicator

Merge pull request #49 from sreedevk/development
```
Version 0.1.6
```
Sreedev Kodichath committed with GitHub 1 year ago

7bd88f16

2 parents
ab44d1ed
a65e2226

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

■ ■ ■ ■ ■ ■

.github/workflows/release.yml

	1	+	# CI that:
	2	+	#
	3	+	# * checks for a Git Tag that looks like a release ("v1.2.0")
	4	+	# * creates a Github Release™️
	5	+	# * builds binaries/packages with cargo-dist
	6	+	# * uploads those packages to the Github Release™️
	7	+	#
	8	+	# Note that the Github Release™️ will be created before the packages,
	9	+	# so there will be a few minutes where the release has no packages
	10	+	# and then they will slowly trickle in, possibly failing. To make
	11	+	# this more pleasant we mark the release as a "draft" until all
	12	+	# artifacts have been successfully uploaded. This allows you to
	13	+	# choose what to do with partial successes and avoids spamming
	14	+	# anyone with notifications before the release is actually ready.
1	15		name: Release
2	16
3		-	env:
4		-	PROJECT_NAME: deduplicator
5		-	PROJECT_DESC: "Filter, Sort & Delete Duplicate Files Recursively"
6		-	PROJECT_AUTH: "sreedevk"
	17	+	permissions:
	18	+	contents: write
7	19
	20	+	# This task will run whenever you push a git tag that looks like
	21	+	# a version number. We just look for `v` followed by at least one number
	22	+	# and then whatever. so `v1`, `v1.0.0`, and `v1.0.0-prerelease` all work.
	23	+	#
	24	+	# If there's a prerelease-style suffix to the version then the Github Release™️
	25	+	# will be marked as a prerelease (handled by taiki-e/create-gh-release-action).
	26	+	#
	27	+	# Note that when generating links to uploaded artifacts, cargo-dist will currently
	28	+	# assume that your git tag is always v{VERSION} where VERSION is the version in
	29	+	# the published package's Cargo.toml (this is the default behaviour of cargo-release).
	30	+	# In the future this may be made more robust/configurable.
8	31		on:
9		-	release:
10		-	types:
11		-	- created
	32	+	push:
	33	+	tags:
	34	+	- v[0-9]+.*
	35	+
	36	+	env:
	37	+	ALL_CARGO_DIST_TARGET_ARGS: --target=x86_64-unknown-linux-gnu --target=x86_64-apple-darwin --target=x86_64-pc-windows-msvc
	38	+	ALL_CARGO_DIST_INSTALLER_ARGS:
12	39
13	40		jobs:
14		-	upload-assets:
	41	+	# Create the Github Release™️ so the packages have something to be uploaded to
	42	+	create-release:
	43	+	runs-on: ubuntu-latest
	44	+	outputs:
	45	+	tag: ${{ steps.create-gh-release.outputs.computed-prefix }}${{ steps.create-gh-release.outputs.version }}
	46	+	steps:
	47	+	- uses: actions/checkout@v3
	48	+	- id: create-gh-release
	49	+	uses: taiki-e/create-gh-release-action@v1
	50	+	with:
	51	+	draft: true
	52	+	# (required) GitHub token for creating GitHub Releases.
	53	+	token: ${{ secrets.GITHUB_TOKEN }}
	54	+
	55	+
	56	+	# Build and packages all the things
	57	+	upload-artifacts:
	58	+	needs: create-release
15	59		strategy:
16	60		matrix:
17		-	os:
18		-	- ubuntu-latest
19		-	- macos-latest
20		-	- windows-latest
	61	+	# For these target platforms
	62	+	include:
	63	+	- target: x86_64-unknown-linux-gnu
	64	+	os: ubuntu-20.04
	65	+	install-dist: curl --proto '=https' --tlsv1.2 -L -sSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.2/installer.sh \| sh
	66	+	- target: x86_64-apple-darwin
	67	+	os: macos-11
	68	+	install-dist: curl --proto '=https' --tlsv1.2 -L -sSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.2/installer.sh \| sh
	69	+	- target: x86_64-pc-windows-msvc
	70	+	os: windows-2019
	71	+	install-dist: irm 'https://github.com/axodotdev/cargo-dist/releases/download/v0.0.2/installer.ps1' \| iex
21	72		runs-on: ${{ matrix.os }}
	73	+	env:
	74	+	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
22	75		steps:
23	76		- uses: actions/checkout@v3
24		-	- uses: taiki-e/upload-rust-binary-action@v1
25		-	with:
26		-	bin: deduplicator
27		-	tar: unix
28		-	zip: windows
29		-	token: ${{ secrets.GITHUB_TOKEN }}
30		-	env:
31		-	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	77	+	- name: Install Rust
	78	+	run: rustup update stable && rustup default stable
	79	+	- name: Install cargo-dist
	80	+	run: ${{ matrix.install-dist }}
	81	+	- name: Run cargo-dist
	82	+	# This logic is a bit janky because it's trying to be a polyglot between
	83	+	# powershell and bash since this will run on windows, macos, and linux!
	84	+	# The two platforms don't agree on how to talk about env vars but they
	85	+	# do agree on 'cat' and '$()' so we use that to marshal values between commmands.
	86	+	run: \|
	87	+	# Actually do builds and make zips and whatnot
	88	+	cargo dist --target=${{ matrix.target }} --output-format=json > dist-manifest.json
	89	+	echo "dist ran successfully"
	90	+	cat dist-manifest.json
	91	+	# Parse out what we just built and upload it to the Github Release™️
	92	+	cat dist-manifest.json \| jq --raw-output ".releases[].artifacts[].path" > uploads.txt
	93	+	echo "uploading..."
	94	+	cat uploads.txt
	95	+	gh release upload ${{ needs.create-release.outputs.tag }} $(cat uploads.txt)
	96	+	echo "uploaded!"
	97	+
	98	+	# Compute and upload the manifest for everything
	99	+	upload-manifest:
	100	+	needs: create-release
	101	+	runs-on: ubuntu-latest
	102	+	env:
	103	+	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	104	+	steps:
	105	+	- uses: actions/checkout@v3
	106	+	- name: Install Rust
	107	+	run: rustup update stable && rustup default stable
	108	+	- name: Install cargo-dist
	109	+	run: curl --proto '=https' --tlsv1.2 -L -sSf https://github.com/axodotdev/cargo-dist/releases/download/v0.0.2/installer.sh \| sh
	110	+	- name: Run cargo-dist manifest
	111	+	run: \|
	112	+	# Generate a manifest describing everything
	113	+	cargo dist manifest --no-local-paths --output-format=json $ALL_CARGO_DIST_TARGET_ARGS $ALL_CARGO_DIST_INSTALLER_ARGS > dist-manifest.json
	114	+	echo "dist manifest ran successfully"
	115	+	cat dist-manifest.json
	116	+	# Upload the manifest to the Github Release™️
	117	+	gh release upload ${{ needs.create-release.outputs.tag }} dist-manifest.json
	118	+	echo "uploaded manifest!"
	119	+	# Edit the Github Release™️ title/body to match what cargo-dist thinks it should be
	120	+	CHANGELOG_TITLE=$(cat dist-manifest.json \| jq --raw-output ".releases[].changelog_title")
	121	+	cat dist-manifest.json \| jq --raw-output ".releases[].changelog_body" > new_dist_changelog.md
	122	+	gh release edit ${{ needs.create-release.outputs.tag }} --title="$CHANGELOG_TITLE" --notes-file=new_dist_changelog.md
	123	+	echo "updated release notes!"
	124	+
	125	+	# Mark the Github Release™️ as a non-draft now that everything has succeeded!
	126	+	publish-release:
	127	+	needs: [create-release, upload-artifacts, upload-manifest]
	128	+	runs-on: ubuntu-latest
	129	+	env:
	130	+	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	131	+	steps:
	132	+	- uses: actions/checkout@v3
	133	+	- name: mark release as non-draft
	134	+	run: \|
	135	+	gh release edit ${{ needs.create-release.outputs.tag }} --draft=false
	136	+
32	137

■ ■ ■ ■ ■ ■

Cargo.lock

		skipped 72 lines
73	73
74	74		[[package]]
75	75		name = "bumpalo"
76		-	version = "3.11.1"
	76	+	version = "3.12.0"
77	77		source = "registry+https://github.com/rust-lang/crates.io-index"
78		-	checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"
	78	+	checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
79	79
80	80		[[package]]
81	81		name = "byteorder"
82	82		version = "1.4.3"
83	83		source = "registry+https://github.com/rust-lang/crates.io-index"
84	84		checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
85		-
86		-	[[package]]
87		-	name = "bytes"
88		-	version = "1.3.0"
89		-	source = "registry+https://github.com/rust-lang/crates.io-index"
90		-	checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c"
91	85
92	86		[[package]]
93	87		name = "bytesize"
		skipped 30 lines
124	118
125	119		[[package]]
126	120		name = "clap"
127		-	version = "4.0.32"
	121	+	version = "4.1.4"
128	122		source = "registry+https://github.com/rust-lang/crates.io-index"
129		-	checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39"
	123	+	checksum = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76"
130	124		dependencies = [
131	125		"bitflags",
132	126		"clap_derive",
		skipped 6 lines
139	133
140	134		[[package]]
141	135		name = "clap_derive"
142		-	version = "4.0.21"
	136	+	version = "4.1.0"
143	137		source = "registry+https://github.com/rust-lang/crates.io-index"
144		-	checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014"
	138	+	checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8"
145	139		dependencies = [
146	140		"heck",
147	141		"proc-macro-error",
		skipped 4 lines
152	146
153	147		[[package]]
154	148		name = "clap_lex"
155		-	version = "0.3.0"
	149	+	version = "0.3.1"
156	150		source = "registry+https://github.com/rust-lang/crates.io-index"
157		-	checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8"
	151	+	checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade"
158	152		dependencies = [
159	153		"os_str_bytes",
160	154		]
		skipped 21 lines
182	176
183	177		[[package]]
184	178		name = "console"
185		-	version = "0.15.4"
	179	+	version = "0.15.5"
186	180		source = "registry+https://github.com/rust-lang/crates.io-index"
187		-	checksum = "c9b6515d269224923b26b5febea2ed42b2d5f2ce37284a4dd670fedd6cb8347a"
	181	+	checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60"
188	182		dependencies = [
189	183		"encode_unicode 0.3.6",
190	184		"lazy_static",
		skipped 75 lines
266	260
267	261		[[package]]
268	262		name = "cxx"
269		-	version = "1.0.85"
	263	+	version = "1.0.87"
270	264		source = "registry+https://github.com/rust-lang/crates.io-index"
271		-	checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd"
	265	+	checksum = "b61a7545f753a88bcbe0a70de1fcc0221e10bfc752f576754fa91e663db1622e"
272	266		dependencies = [
273	267		"cc",
274	268		"cxxbridge-flags",
		skipped 3 lines
278	272
279	273		[[package]]
280	274		name = "cxx-build"
281		-	version = "1.0.85"
	275	+	version = "1.0.87"
282	276		source = "registry+https://github.com/rust-lang/crates.io-index"
283		-	checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0"
	277	+	checksum = "f464457d494b5ed6905c63b0c4704842aba319084a0a3561cdc1359536b53200"
284	278		dependencies = [
285	279		"cc",
286	280		"codespan-reporting",
		skipped 6 lines
293	287
294	288		[[package]]
295	289		name = "cxxbridge-flags"
296		-	version = "1.0.85"
	290	+	version = "1.0.87"
297	291		source = "registry+https://github.com/rust-lang/crates.io-index"
298		-	checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59"
	292	+	checksum = "43c7119ce3a3701ed81aca8410b9acf6fc399d2629d057b87e2efa4e63a3aaea"
299	293
300	294		[[package]]
301	295		name = "cxxbridge-macro"
302		-	version = "1.0.85"
	296	+	version = "1.0.87"
303	297		source = "registry+https://github.com/rust-lang/crates.io-index"
304		-	checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6"
	298	+	checksum = "65e07508b90551e610910fa648a1878991d367064997a596135b86df30daf07e"
305	299		dependencies = [
306	300		"proc-macro2",
307	301		"quote",
		skipped 16 lines
324	318
325	319		[[package]]
326	320		name = "deduplicator"
327		-	version = "0.1.5"
	321	+	version = "0.1.6"
328	322		dependencies = [
329	323		"anyhow",
330	324		"bytesize",
		skipped 8 lines
339	333		"memmap2",
340	334		"prettytable-rs",
341	335		"rayon",
342		-	"thiserror",
343		-	"tokio",
344	336		"unicode-segmentation",
345	337		]
346	338
		skipped 180 lines
527	519
528	520		[[package]]
529	521		name = "indicatif"
530		-	version = "0.17.2"
	522	+	version = "0.17.3"
531	523		source = "registry+https://github.com/rust-lang/crates.io-index"
532		-	checksum = "4295cbb7573c16d310e99e713cf9e75101eb190ab31fccd35f2d2691b4352b19"
	524	+	checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729"
533	525		dependencies = [
534	526		"console",
535	527		"number_prefix",
536	528		"portable-atomic",
537	529		"rayon",
538		-	"tokio",
539	530		"unicode-width",
540	531		]
541	532
542	533		[[package]]
543	534		name = "io-lifetimes"
544		-	version = "1.0.3"
	535	+	version = "1.0.4"
545	536		source = "registry+https://github.com/rust-lang/crates.io-index"
546		-	checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c"
	537	+	checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e"
547	538		dependencies = [
548	539		"libc",
549	540		"windows-sys",
		skipped 106 lines
656	647		]
657	648
658	649		[[package]]
659		-	name = "mio"
660		-	version = "0.8.5"
661		-	source = "registry+https://github.com/rust-lang/crates.io-index"
662		-	checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de"
663		-	dependencies = [
664		-	"libc",
665		-	"log",
666		-	"wasi 0.11.0+wasi-snapshot-preview1",
667		-	"windows-sys",
668		-	]
669		-
670		-	[[package]]
671	650		name = "num-integer"
672	651		version = "0.1.45"
673	652		source = "registry+https://github.com/rust-lang/crates.io-index"
		skipped 30 lines
704	683
705	684		[[package]]
706	685		name = "once_cell"
707		-	version = "1.16.0"
	686	+	version = "1.17.0"
708	687		source = "registry+https://github.com/rust-lang/crates.io-index"
709		-	checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
	688	+	checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
710	689
711	690		[[package]]
712	691		name = "os_str_bytes"
713	692		version = "6.4.1"
714	693		source = "registry+https://github.com/rust-lang/crates.io-index"
715	694		checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee"
716		-
717		-	[[package]]
718		-	name = "parking_lot"
719		-	version = "0.12.1"
720		-	source = "registry+https://github.com/rust-lang/crates.io-index"
721		-	checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
722		-	dependencies = [
723		-	"lock_api",
724		-	"parking_lot_core",
725		-	]
726	695
727	696		[[package]]
728	697		name = "parking_lot_core"
729		-	version = "0.9.5"
	698	+	version = "0.9.6"
730	699		source = "registry+https://github.com/rust-lang/crates.io-index"
731		-	checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba"
	700	+	checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf"
732	701		dependencies = [
733	702		"cfg-if",
734	703		"libc",
		skipped 1 lines
736	705		"smallvec",
737	706		"windows-sys",
738	707		]
739		-
740		-	[[package]]
741		-	name = "pin-project-lite"
742		-	version = "0.2.9"
743		-	source = "registry+https://github.com/rust-lang/crates.io-index"
744		-	checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
745	708
746	709		[[package]]
747	710		name = "portable-atomic"
		skipped 41 lines
789	752
790	753		[[package]]
791	754		name = "proc-macro2"
792		-	version = "1.0.49"
	755	+	version = "1.0.50"
793	756		source = "registry+https://github.com/rust-lang/crates.io-index"
794		-	checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5"
	757	+	checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2"
795	758		dependencies = [
796	759		"unicode-ident",
797	760		]
		skipped 19 lines
817	780
818	781		[[package]]
819	782		name = "rayon-core"
820		-	version = "1.10.1"
	783	+	version = "1.10.2"
821	784		source = "registry+https://github.com/rust-lang/crates.io-index"
822		-	checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3"
	785	+	checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b"
823	786		dependencies = [
824	787		"crossbeam-channel",
825	788		"crossbeam-deque",
		skipped 46 lines
872	835
873	836		[[package]]
874	837		name = "rustix"
875		-	version = "0.36.5"
	838	+	version = "0.36.7"
876	839		source = "registry+https://github.com/rust-lang/crates.io-index"
877		-	checksum = "a3807b5d10909833d3e9acd1eb5fb988f79376ff10fce42937de71a449c4c588"
	840	+	checksum = "d4fdebc4b395b7fbb9ab11e462e20ed9051e7b16e42d24042c776eca0ac81b03"
878	841		dependencies = [
879	842		"bitflags",
880	843		"errno",
		skipped 43 lines
924	887		checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
925	888
926	889		[[package]]
927		-	name = "signal-hook-registry"
928		-	version = "1.4.0"
929		-	source = "registry+https://github.com/rust-lang/crates.io-index"
930		-	checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0"
931		-	dependencies = [
932		-	"libc",
933		-	]
934		-
935		-	[[package]]
936	890		name = "smallvec"
937	891		version = "1.10.0"
938	892		source = "registry+https://github.com/rust-lang/crates.io-index"
939	893		checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
940	894
941	895		[[package]]
942		-	name = "socket2"
943		-	version = "0.4.7"
944		-	source = "registry+https://github.com/rust-lang/crates.io-index"
945		-	checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd"
946		-	dependencies = [
947		-	"libc",
948		-	"winapi",
949		-	]
950		-
951		-	[[package]]
952	896		name = "strsim"
953	897		version = "0.10.0"
954	898		source = "registry+https://github.com/rust-lang/crates.io-index"
		skipped 23 lines
978	922
979	923		[[package]]
980	924		name = "termcolor"
981		-	version = "1.1.3"
	925	+	version = "1.2.0"
982	926		source = "registry+https://github.com/rust-lang/crates.io-index"
983		-	checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
	927	+	checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
984	928		dependencies = [
985	929		"winapi-util",
986	930		]
		skipped 39 lines
1026	970		]
1027	971
1028	972		[[package]]
1029		-	name = "tokio"
1030		-	version = "1.23.1"
1031		-	source = "registry+https://github.com/rust-lang/crates.io-index"
1032		-	checksum = "38a54aca0c15d014013256222ba0ebed095673f89345dd79119d912eb561b7a8"
1033		-	dependencies = [
1034		-	"autocfg",
1035		-	"bytes",
1036		-	"libc",
1037		-	"memchr",
1038		-	"mio",
1039		-	"num_cpus",
1040		-	"parking_lot",
1041		-	"pin-project-lite",
1042		-	"signal-hook-registry",
1043		-	"socket2",
1044		-	"tokio-macros",
1045		-	"windows-sys",
1046		-	]
1047		-
1048		-	[[package]]
1049		-	name = "tokio-macros"
1050		-	version = "1.8.2"
1051		-	source = "registry+https://github.com/rust-lang/crates.io-index"
1052		-	checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8"
1053		-	dependencies = [
1054		-	"proc-macro2",
1055		-	"quote",
1056		-	"syn",
1057		-	]
1058		-
1059		-	[[package]]
1060	973		name = "unicode-ident"
1061	974		version = "1.0.6"
1062	975		source = "registry+https://github.com/rust-lang/crates.io-index"
		skipped 142 lines
1205	1118
1206	1119		[[package]]
1207	1120		name = "windows_aarch64_gnullvm"
1208		-	version = "0.42.0"
	1121	+	version = "0.42.1"
1209	1122		source = "registry+https://github.com/rust-lang/crates.io-index"
1210		-	checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e"
	1123	+	checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
1211	1124
1212	1125		[[package]]
1213	1126		name = "windows_aarch64_msvc"
1214		-	version = "0.42.0"
	1127	+	version = "0.42.1"
1215	1128		source = "registry+https://github.com/rust-lang/crates.io-index"
1216		-	checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4"
	1129	+	checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
1217	1130
1218	1131		[[package]]
1219	1132		name = "windows_i686_gnu"
1220		-	version = "0.42.0"
	1133	+	version = "0.42.1"
1221	1134		source = "registry+https://github.com/rust-lang/crates.io-index"
1222		-	checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7"
	1135	+	checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
1223	1136
1224	1137		[[package]]
1225	1138		name = "windows_i686_msvc"
1226		-	version = "0.42.0"
	1139	+	version = "0.42.1"
1227	1140		source = "registry+https://github.com/rust-lang/crates.io-index"
1228		-	checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246"
	1141	+	checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
1229	1142
1230	1143		[[package]]
1231	1144		name = "windows_x86_64_gnu"
1232		-	version = "0.42.0"
	1145	+	version = "0.42.1"
1233	1146		source = "registry+https://github.com/rust-lang/crates.io-index"
1234		-	checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed"
	1147	+	checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
1235	1148
1236	1149		[[package]]
1237	1150		name = "windows_x86_64_gnullvm"
1238		-	version = "0.42.0"
	1151	+	version = "0.42.1"
1239	1152		source = "registry+https://github.com/rust-lang/crates.io-index"
1240		-	checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028"
	1153	+	checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
1241	1154
1242	1155		[[package]]
1243	1156		name = "windows_x86_64_msvc"
1244		-	version = "0.42.0"
	1157	+	version = "0.42.1"
1245	1158		source = "registry+https://github.com/rust-lang/crates.io-index"
1246		-	checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5"
	1159	+	checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
1247	1160

■ ■ ■ ■ ■ ■

Cargo.toml

1	1		[package]
2	2		name = "deduplicator"
3		-	version = "0.1.5"
	3	+	version = "0.1.6"
4	4		edition = "2021"
5	5		description = "find,filter,delete Duplicates"
6	6		license = "MIT"
		skipped 14 lines
21	21		dashmap = { version = "5.4.0", features = ["rayon"] }
22	22		fxhash = "0.2.1"
23	23		globwalk = "0.8.1"
24		-	indicatif = { version = "0.17.2", features = ["rayon", "tokio"] }
	24	+	indicatif = { version = "0.17.2", features = ["rayon"] }
25	25		itertools = "0.10.5"
26	26		memmap2 = "0.5.8"
27	27		prettytable-rs = "0.10.0"
28	28		rayon = "1.6.1"
29		-	thiserror = "1.0.38"
30		-	tokio = { version = "1.23.1", features = ["full"] }
31	29		unicode-segmentation = "1.10.0"
32	30
	31	+	# generated by 'cargo dist init'
	32	+	[profile.dist]
	33	+	inherits = "release"
	34	+	debug = true
	35	+	split-debuginfo = "packed"
	36	+
	37	+

■ ■ ■ ■ ■ ■

README.md

		skipped 93 lines
94	94
95	95		## benchmarks
96	96
97		-	\| Command \| Dirsize \| Mean [ms] \| Min [ms] \| Max [ms] \| Relative \|
98		-	\|:---\|:---\|---:\|---:\|---:\|---:\|
99		-	\| `deduplicator --dir ~/Data/tmp` \| (~120G) \| 27.5 ± 1.0 \| 26.0 \| 32.1 \| 1.70 ± 0.09 \|
100		-	\| `deduplicator --dir ~/Data/books` \| (~8.6G) \| 21.8 ± 0.7 \| 20.5 \| 24.4 \| 1.35 ± 0.07 \|
101		-	\| `deduplicator --dir ~/Data/books --min-size 10M` \| (~8.6G) \| 16.1 ± 0.6 \| 14.9 \| 18.8 \| 1.00 \|
102		-	\| `deduplicator --dir ~/Data/ --types pdf,jpg,png,jpeg` \| (~290G) \| 1857.4 ± 24.5 \| 1817.0 \| 1895.5 \| 115.07 ± 4.64 \|
	97	+	\| Command \| Dirsize \| Filecount \| Mean [ms] \| Min [ms] \| Max [ms] \| Relative \|
	98	+	\|:---\|:---\|---:\|---:\|---:\|---:\|---:\|
	99	+	\| `deduplicator ~/Data/tmp` \| (~120G) \| 721 files \| 33.5 ± 28.6 \| 25.3 \| 151.5 \| 1.87 ± 1.60 \|
	100	+	\| `deduplicator ~/Data/books` \| (~8.6G) \| 1419 files \| 24.5 ± 1.0 \| 22.9 \| 28.1 \| 1.37 ± 0.08 \|
	101	+	\| `deduplicator ~/Data/books --min-size 10M` \| (~8.6G) \| 1419 files \| 17.9 ± 0.7 \| 16.8 \| 20.0 \| 1.00 \|
	102	+	\| `deduplicator ~/Data/ --types pdf,jpg,png,jpeg` \| (~290G) \| 104222 files \| 1207.2 ± 37.0 \| 1172.2 \| 1287.7 \| 67.27 ± 3.33 \|
103	103
104	104		* The last entry is lower because of the number of files deduplicator had to go through (~660895 Files). The average size of the files rarely affect the performance of deduplicator.
105	105
		skipped 19 lines

■ ■ ■ ■ ■ ■

src/file_manager.rs

1	1		use anyhow::Result;
2	2		use colored::Colorize;
	3	+	use std::path::PathBuf;
3	4
4	5		#[derive(Debug, Clone)]
5	6		pub struct File {
6		-	pub path: String,
	7	+	pub path: PathBuf,
7	8		pub size: Option<u64>,
8	9		pub hash: Option<String>,
9	10		}
		skipped 1 lines
11	12		pub fn delete_files(files: Vec<File>) -> Result<()> {
12	13		files.into_iter().for_each(\|file\| {
13	14		match std::fs::remove_file(file.path.clone()) {
14		-	Ok(_) => println!("{}: {}", "DELETED".green(), file.path),
15		-	Err(_) => println!("{}: {}", "FAILED".red(), file.path)
	15	+	Ok(_) => println!("{}: {}", "DELETED".green(), file.path.display()),
	16	+	Err(_) => println!("{}: {}", "FAILED".red(), file.path.display())
16	17		}
17	18		});
18	19
		skipped 3 lines

■ ■ ■ ■ ■ ■

src/main.rs

		skipped 8 lines
9	9		use app::App;
10	10		use clap::Parser;
11	11
12		-	#[tokio::main]
13		-	async fn main() -> Result<()> {
	12	+	fn main() -> Result<()> {
14	13		App::init(&params::Params::parse())
15	14		}
16	15

■ ■ ■ ■ ■ ■

src/output.rs

		skipped 8 lines
9	9		use itertools::Itertools;
10	10		use prettytable::{format, row, Table};
11	11		use std::io::Write;
	12	+	use std::path::Path;
	13	+	use std::time::Duration;
12	14		use std::{fs, io};
13	15		use unicode_segmentation::UnicodeSegmentation;
14	16
15		-	fn format_path(path: &str, opts: &Params) -> Result<String> {
16		-	let display_path = path.replace(opts.get_directory()?.to_string_lossy().as_ref(), "");
	17	+	fn format_path(path: &Path, opts: &Params) -> Result<String> {
	18	+	let display_path = path
	19	+	.to_string_lossy()
	20	+	.replace(opts.get_directory()?.to_string_lossy().as_ref(), "");
17	21		let display_range = if display_path.chars().count() > 32 {
18	22		display_path
19	23		.graphemes(true)
		skipped 14 lines
34	38		Ok(format!("{:>12}", bytesize::ByteSize::b(file.size.unwrap())))
35	39		}
36	40
37		-	fn modified_time(path: &String) -> Result<String> {
	41	+	fn modified_time(path: &Path) -> Result<String> {
38	42		let mdata = fs::metadata(path)?;
39	43		let modified_time: DateTime<Utc> = mdata.modified()?.into();
40	44
		skipped 33 lines
74	78		.split(',')
75	79		.filter(\|element\| !element.is_empty())
76	80		.map(\|index\| index.parse::<usize>().unwrap_or_default())
77		-	.collect_vec();
	81	+	.collect::<Vec<usize>>();
78	82
79	83		if parsed_file_indices
80	84		.clone()
		skipped 19 lines
100	104		.clone()
101	105		.enumerate()
102	106		.for_each(\|(index, file)\| {
103		-	println!("{}: {}", index.to_string().blue(), file.path);
	107	+	println!("{}: {}", index.to_string().blue(), file.path.display());
104	108		});
105	109
106	110		match scan_group_confirmation().unwrap() {
107	111		true => {
108		-	file_manager::delete_files(files_to_delete.collect_vec()).ok();
	112	+	file_manager::delete_files(files_to_delete.collect::<Vec<File>>()).ok();
109	113		}
110	114		false => println!("{}", "\nCancelled Delete Operation.".red()),
111	115		}
		skipped 43 lines
155	159
156	160		let mut output_table = Table::new();
157	161		let progress_bar = ProgressBar::new(duplicates.len() as u64);
	162	+	progress_bar.enable_steady_tick(Duration::from_millis(50));
158	163		let progress_style = ProgressStyle::default_bar()
159	164		.template("{spinner:.green} [generating output] [{wide_bar:.cyan/blue}] {pos}/{len} files")
160	165		.unwrap();
		skipped 21 lines
182	187		output_table.printstd();
183	188		}
184	189
	190	+	#[allow(unused)]
	191	+	pub fn raw(duplicates: DashMap<String, Vec<File>>, opts: &Params) -> Result<()> {
	192	+	if duplicates.is_empty() {
	193	+	println!(
	194	+	"\n{}",
	195	+	"No duplicates found matching your search criteria.".green()
	196	+	);
	197	+	return Ok(());
	198	+	}
	199	+
	200	+	duplicates
	201	+	.into_iter()
	202	+	.sorted_unstable_by_key(\|(_, f)\| f.first().and_then(\|ff\| ff.size).unwrap_or_default())
	203	+	.for_each(\|(_hash, group)\| {
	204	+	group.iter().for_each(\|file\| {
	205	+	println!(
	206	+	"{}\t{}\t{}",
	207	+	format_path(&file.path, opts).unwrap_or_default().blue(),
	208	+	file_size(file).unwrap_or_default().red(),
	209	+	modified_time(&file.path).unwrap_or_default().yellow()
	210	+	)
	211	+	});
	212	+
	213	+	println!("---");
	214	+	});
	215	+
	216	+	Ok(())
	217	+	}
	218	+

■ ■ ■ ■ ■ ■

src/scanner.rs

		skipped 6 lines
7	7		use rayon::prelude::*;
8	8		use std::hash::Hasher;
9	9		use std::time::Duration;
10		-	use std::{fs, path::PathBuf};
	10	+	use std::{
	11	+	fs,
	12	+	path::{Path, PathBuf},
	13	+	};
11	14
12	15		#[derive(Clone, Copy)]
13	16		enum IndexCritera {
		skipped 31 lines
45	48		let progress_style =
46	49		ProgressStyle::with_template("{spinner:.green} [mapping paths] {pos} paths")?;
47	50		progress.set_style(progress_style);
48		-	progress.enable_steady_tick(Duration::from_millis(100));
	51	+	progress.enable_steady_tick(Duration::from_millis(50));
49	52
50	53		let files = walker
51	54		.progress_with(progress)
52	55		.filter_map(Result::ok)
53	56		.map(\|file\| file.into_path())
54	57		.filter(\|fpath\| fpath.is_file())
55		-	.collect::<Vec<PathBuf>>()
	58	+	.collect::<Vec<PathBuf>>();
	59	+
	60	+	let scan_progress = ProgressBar::new(files.len() as u64);
	61	+	let scan_progress_style = ProgressStyle::with_template(
	62	+	"{spinner:.green} [processing mapped paths] [{wide_bar:.cyan/blue}] {pos}/{len} files",
	63	+	)?;
	64	+	scan_progress.set_style(scan_progress_style);
	65	+	scan_progress.enable_steady_tick(Duration::from_millis(50));
	66	+
	67	+	let scan_results = files
56	68		.into_par_iter()
57		-	.progress_with_style(ProgressStyle::with_template(
58		-	"{spinner:.green} [processing mapped paths] [{wide_bar:.cyan/blue}] {pos}/{len} files",
59		-	)?)
60		-	.map(\|fpath\| fpath.display().to_string())
	69	+	.progress_with(scan_progress)
61	70		.map(\|fpath\| File {
62	71		path: fpath.clone(),
63	72		hash: None,
64		-	size: Some(fs::metadata(fpath).unwrap().len()),
	73	+	size: Some(
	74	+	fs::metadata(fpath)
	75	+	.map(\|metadata\| metadata.len())
	76	+	.unwrap_or_default(),
	77	+	),
65	78		})
66	79		.filter(\|file\| filters::is_file_gt_min_size(app_opts, file))
67	80		.collect();
68	81
69		-	Ok(files)
	82	+	Ok(scan_results)
70	83		}
71	84
72	85		fn process_file_index(
		skipped 11 lines
84	97		IndexCritera::Hash => {
85	98		file.hash = Some(hash_file(&file.path).unwrap_or_default());
86	99		store
87		-	.entry(file.clone().hash.unwrap())
	100	+	.entry(file.clone().hash.unwrap_or_default())
88	101		.and_modify(\|fileset\| fileset.push(file.clone()))
89	102		.or_insert_with(\|\| vec![file]);
90	103		}
		skipped 5 lines
96	109		index_criteria: IndexCritera,
97	110		) -> Result<DashMap<String, Vec<File>>> {
98	111		let store: DashMap<String, Vec<File>> = DashMap::new();
	112	+	let index_progress = ProgressBar::new(files.len() as u64);
	113	+	let index_progress_style = ProgressStyle::with_template(
	114	+	"{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
	115	+	)?;
	116	+	index_progress.set_style(index_progress_style);
	117	+	index_progress.enable_steady_tick(Duration::from_millis(50));
	118	+
99	119		files
100	120		.into_par_iter()
101		-	.progress_with_style(ProgressStyle::with_template(
102		-	"{spinner:.green} [indexing files] [{wide_bar:.cyan/blue}] {pos}/{len} files",
103		-	)?)
	121	+	.progress_with(index_progress)
104	122		.for_each(\|file\| process_file_index(file, &store, index_criteria));
105	123
106	124		Ok(store)
107	125		}
108	126
109		-	fn incremental_hashing(filepath: &str) -> Result<String> {
	127	+	fn incremental_hashing(filepath: &Path) -> Result<String> {
110	128		let file = fs::File::open(filepath)?;
111	129		let fmap = unsafe { Mmap::map(&file)? };
112	130		let mut inchasher = fxhash::FxHasher::default();
		skipped 4 lines
117	135		Ok(format!("{}", inchasher.finish()))
118	136		}
119	137
120		-	fn standard_hashing(filepath: &str) -> Result<String> {
	138	+	fn standard_hashing(filepath: &Path) -> Result<String> {
121	139		let file = fs::read(filepath)?;
122	140		Ok(hasher(&*file).to_string())
123	141		}
124	142
125		-	fn hash_file(filepath: &str) -> Result<String> {
	143	+	fn hash_file(filepath: &Path) -> Result<String> {
126	144		let filemeta = fs::metadata(filepath)?;
127	145
128	146		// NOTE: USE INCREMENTAL HASHING ONLY FOR FILES > 100MB
		skipped 6 lines

Merge pull request #49 from sreedevk/development