Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

add option: check_content

  • Loading branch information...
commit 8a4785501cb3988ca5ac827e4f6567420e354f6a 1 parent ccd41d3
Steven Haryanto (on PC) authored
Showing with 29 additions and 7 deletions.
  1. +24 −7 lib/App/UniqFiles.pm
  2. +5 −0 t/01-basics.t
31 lib/App/UniqFiles.pm
View
@@ -75,6 +75,17 @@ _
arg_aliases => {
},
}],
+ check_content => [bool => {
+ summary => "Whether to check file content ",
+ description => <<'_',
+
+If set to 0, uniqueness will be determined solely from file size. This can be
+quicker but might generate a false positive when two files of the same size are
+deemed as duplicate even though their content are different.
+
+_
+ default => 1,
+ }],
count => [bool => {
summary => "Whether to return each file content's ".
"number of occurence",
@@ -95,6 +106,7 @@ sub uniq_files {
return [400, "Please specify files"] if !$files || !@$files;
my $report_unique = $args{report_unique} // 1;
my $report_duplicate = $args{report_duplicate} // 2;
+ my $check_content = $args{check_content} // 1;
my $count = $args{count} // 0;
# get sizes of all files
@@ -117,14 +129,19 @@ sub uniq_files {
for my $f (@$files) {
next unless defined $file_sizes{$f};
next if $size_counts{ $file_sizes{$f} } == 1;
- my $fh;
- unless (open $fh, "<", $f) {
- $log->error("Can't open file `$f`: $!, skipped");
- next;
+ my $digest;
+ if ($check_content) {
+ my $fh;
+ unless (open $fh, "<", $f) {
+ $log->error("Can't open file `$f`: $!, skipped");
+ next;
+ }
+ my $ctx = Digest::MD5->new;
+ $ctx->addfile($fh);
+ $digest = $ctx->hexdigest;
+ } else {
+ $digest = "";
}
- my $ctx = Digest::MD5->new;
- $ctx->addfile($fh);
- my $digest = $ctx->hexdigest;
$digest_counts{$digest}++;
$digest_files{$digest} //= [];
push @{$digest_files{$digest}}, $f;
5 t/01-basics.t
View
@@ -43,5 +43,10 @@ $res = uniq_files(files => \@f, count=>1);
is_deeply($res->[2], {f1=>3, f2=>3, f3=>1, f4=>1, f5=>3}, "count")
or diag explain $res;
+# check_content=0
+$res = uniq_files(files => \@f, count=>1, check_content=>0);
+is_deeply($res->[2], {f1=>4, f2=>4, f3=>4, f4=>1, f5=>4}, "check_content=0")
+ or diag explain $res;
+
chdir "/" if Test::More->builder->is_passing;
done_testing();
Please sign in to comment.
Something went wrong with that request. Please try again.