aparler/RosettaCodeData
1
0
Fork 0
Code Issues 12 Pull requests 3 Projects Releases Packages Wiki Activity Actions

Data commit

Browse source
This commit is contained in:
Ingy döt Net 2023-07-01 11:58:00 -04:00
parent 7387c8f97b
commit cb5bb5e222
199093 changed files with 3378972 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Task/Find-duplicate-files/00-META.yaml Normal file
View file

@ -0,0 +1,2 @@
---
from: http://rosettacode.org/wiki/Find_duplicate_files

15
Task/Find-duplicate-files/00-TASK.txt Normal file
View file

@ -0,0 +1,15 @@
In a large directory structure it is easy to inadvertently leave unnecessary copies of files around, which can use considerable disk space and create confusion.
;Task:
Create a program which, given a minimum size and a folder/directory, will find all files of at least ''size'' bytes with duplicate contents under the directory and output or show the sets of duplicate files in order of decreasing size.
The program may be command-line or graphical, and duplicate content may be determined by direct comparison or by calculating a hash of the data.
Specify which filesystems or operating systems your program works with if it has any filesystem- or OS-specific requirements.
Identify hard links (filenames referencing the same content) in the output if applicable for the filesystem.
For extra points, detect when whole directory sub-trees are identical, or optionally remove or link identical files.
<br><br>

146
Task/Find-duplicate-files/C++/find-duplicate-files.cpp Normal file
View file

@ -0,0 +1,146 @@
#include<iostream>
#include<string>
#include<boost/filesystem.hpp>
#include<boost/format.hpp>
#include<boost/iostreams/device/mapped_file.hpp>
#include<optional>
#include<algorithm>
#include<iterator>
#include<execution>
#include"dependencies/xxhash.hpp" // https://github.com/RedSpah/xxhash_cpp
/**
* Find ranges (neighbouring elements) of the same value within [begin, end[ and
* call callback for each such range
* @param begin start of container
* @param end end of container (1 beyond last element)
* @param function returns value for each iterator V(*T&)
* @param callback void(start, end, value)
* @return number of range
*/
template<typename T, typename V, typename F>
size_t for_each_adjacent_range(T begin, T end, V getvalue, F callback) {
size_t partitions = 0;
while (begin != end) {
auto const& value = getvalue(*begin);
auto current = begin;
while (++current != end && getvalue(*current) == value);
callback(begin, current, value);
++partitions;
begin = current;
}
return partitions;
}
namespace bi = boost::iostreams;
namespace fs = boost::filesystem;
struct file_entry {
public:
explicit file_entry(fs::directory_entry const & entry)
: path_{entry.path()}, size_{fs::file_size(entry)}
{}
auto size() const { return size_; }
auto const& path() const { return path_; }
auto get_hash() {
if (!hash_)
hash_ = compute_hash();
return *hash_;
}
private:
xxh::hash64_t compute_hash() {
bi::mapped_file_source source;
source.open<fs::wpath>(this->path());
if (!source.is_open()) {
std::cerr << "Cannot open " << path() << std::endl;
throw std::runtime_error("Cannot open file");
}
xxh::hash_state64_t hash_stream;
hash_stream.update(source.data(), size_);
return hash_stream.digest();
}
private:
fs::wpath path_;
uintmax_t size_;
std::optional<xxh::hash64_t> hash_;
};
using vector_type = std::vector<file_entry>;
using iterator_type = vector_type::iterator;
auto find_files_in_dir(fs::wpath const& path, vector_type& file_vector, uintmax_t min_size = 1) {
size_t found = 0, ignored = 0;
if (!fs::is_directory(path)) {
std::cerr << path << " is not a directory!" << std::endl;
}
else {
std::cerr << "Searching " << path << std::endl;
for (auto& e : fs::recursive_directory_iterator(path)) {
++found;
if (fs::is_regular_file(e) && fs::file_size(e) >= min_size)
file_vector.emplace_back(e);
else ++ignored;
}
}
return std::make_tuple(found, ignored);
}
int main(int argn, char* argv[])
{
vector_type files;
for (auto i = 1; i < argn; ++i) {
fs::wpath path(argv[i]);
auto [found, ignored] = find_files_in_dir(path, files);
std::cerr << boost::format{
" %1$6d files found\n"
" %2$6d files ignored\n"
" %3$6d files added\n" } % found % ignored % (found - ignored)
<< std::endl;
}
std::cerr << "Found " << files.size() << " regular files" << std::endl;
// sort files in descending order by file size
std::sort(std::execution::par_unseq, files.begin(), files.end()
, [](auto const& a, auto const& b) { return a.size() > b.size(); }
);
for_each_adjacent_range(
std::begin(files)
, std::end(files)
, [](vector_type::value_type const& f) { return f.size(); }
, [](auto start, auto end, auto file_size) {
// Files with same size
size_t nr_of_files = std::distance(start, end);
if (nr_of_files > 1) {
// sort range start-end by hash
std::sort(start, end, [](auto& a, auto& b) {
auto const& ha = a.get_hash();
auto const& hb = b.get_hash();
auto const& pa = a.path();
auto const& pb = b.path();
return std::tie(ha, pa) < std::tie(hb, pb);
});
for_each_adjacent_range(
start
, end
, [](vector_type::value_type& f) { return f.get_hash(); }
, [file_size](auto hstart, auto hend, auto hash) {
// Files with same size and same hash are assumed to be identical
// could resort to compare files byte-by-byte now
size_t hnr_of_files = std::distance(hstart, hend);
if (hnr_of_files > 1) {
std::cout << boost::format{ "%1$3d files with hash %3$016x and size %2$d\n" }
% hnr_of_files % file_size % hash;
std::for_each(hstart, hend, [hash, file_size](auto& e) {
std::cout << '\t' << e.path() << '\n';
}
);
}
}
);
}
}
);
return 0;
}

22
Task/Find-duplicate-files/Elixir/find-duplicate-files.elixir Normal file
View file

@ -0,0 +1,22 @@
defmodule Files do
def find_duplicate_files(dir) do
IO.puts "\nDirectory : #{dir}"
File.cd!(dir, fn ->
Enum.filter(File.ls!, fn fname -> File.regular?(fname) end)
|> Enum.group_by(fn file -> File.stat!(file).size end)
|> Enum.filter(fn {_, files} -> length(files)>1 end)
|> Enum.each(fn {size, files} ->
Enum.group_by(files, fn file -> :erlang.md5(File.read!(file)) end)
|> Enum.filter(fn {_, files} -> length(files)>1 end)
|> Enum.each(fn {_md5, fs} ->
IO.puts " --------------------------------------------"
Enum.each(fs, fn file ->
IO.puts " #{inspect File.stat!(file).mtime}\t#{size} #{file}"
end)
end)
end)
end)
end
end
hd(System.argv) |> Files.find_duplicate_files

71
Task/Find-duplicate-files/Go/find-duplicate-files.go Normal file
View file

@ -0,0 +1,71 @@
package main
import (
"fmt"
"crypto/md5"
"io/ioutil"
"log"
"os"
"path/filepath"
"sort"
"time"
)
type fileData struct {
filePath string
info os.FileInfo
}
type hash [16]byte
func check(err error) {
if err != nil {
log.Fatal(err)
}
}
func checksum(filePath string) hash {
bytes, err := ioutil.ReadFile(filePath)
check(err)
return hash(md5.Sum(bytes))
}
func findDuplicates(dirPath string, minSize int64) [][2]fileData {
var dups [][2]fileData
m := make(map[hash]fileData)
werr := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() && info.Size() >= minSize {
h := checksum(path)
fd, ok := m[h]
fd2 := fileData{path, info}
if !ok {
m[h] = fd2
} else {
dups = append(dups, [2]fileData{fd, fd2})
}
}
return nil
})
check(werr)
return dups
}
func main() {
dups := findDuplicates(".", 1)
fmt.Println("The following pairs of files have the same size and the same hash:\n")
fmt.Println("File name Size Date last modified")
fmt.Println("==========================================================")
sort.Slice(dups, func(i, j int) bool {
return dups[i][0].info.Size() > dups[j][0].info.Size() // in order of decreasing size
})
for _, dup := range dups {
for i := 0; i < 2; i++ {
d := dup[i]
fmt.Printf("%-20s %8d %v\n", d.filePath, d.info.Size(), d.info.ModTime().Format(time.ANSIC))
}
fmt.Println()
}
}

62
Task/Find-duplicate-files/Haskell/find-duplicate-files.hs Normal file
View file

@ -0,0 +1,62 @@
import Crypto.Hash.MD5 (hash)
import Data.ByteString as BS (readFile, ByteString())
import System.Environment (getArgs, getProgName)
import System.Directory (doesDirectoryExist, getDirectoryContents)
import System.FilePath.Posix ((</>))
import Control.Monad (forM)
import Text.Printf (printf)
import System.IO (withFile, IOMode(ReadMode), hFileSize)
type File = (BS.ByteString, -- md5hash
FilePath) -- filepath
type FileSize = Integer
getRecursiveContents :: FilePath -> FileSize -> IO [File]
getRecursiveContents curDir maxsize = do
names <- getDirectoryContents curDir
let dirs = filter (`notElem` [".", ".."]) names
files <- forM dirs $ \path -> do
let path' = curDir </> path
exists <- doesDirectoryExist path'
if exists
then getRecursiveContents path' maxsize
else genFileHash path' maxsize
return $ concat files
genFileHash :: FilePath -> FileSize -> IO [File]
genFileHash path maxsize = do
size <- withFile path ReadMode hFileSize
if size <= maxsize
then BS.readFile path >>= \bs -> return [(hash bs, path)]
else return []
findDuplicates :: FilePath -> FileSize -> IO ()
findDuplicates dir bytes = do
exists <- doesDirectoryExist dir
if exists
then getRecursiveContents dir bytes >>= findSameHashes
else printf "Sorry, the directory \"%s\" does not exist...\n" dir
findSameHashes :: [File] -> IO ()
findSameHashes [] = return ()
findSameHashes ((hash, fp):xs) = do
case lookup hash xs of
(Just dupFile) -> printf "===========================\n\
\Found duplicate:\n\
\=> %s \n\
\=> %s \n\n" fp dupFile
>> findSameHashes xs
(_) -> findSameHashes xs
main :: IO ()
main = do
args <- getArgs
case args of
[dir, mbytes] | [(bytes ,"")] <- reads mbytes
, bytes >= 1 -> findDuplicates dir bytes
(_) -> do
name <- getProgName
printf "Something went wrong - please use ./%s <dir> <bytes>\n" name

138
Task/Find-duplicate-files/Java/find-duplicate-files.java Normal file
View file

@ -0,0 +1,138 @@
import java.io.*;
import java.nio.*;
import java.nio.file.*;
import java.nio.file.attribute.*;
import java.security.*;
import java.util.*;
public class DuplicateFiles {
public static void main(String[] args) {
if (args.length != 2) {
System.err.println("Directory name and minimum file size are required.");
System.exit(1);
}
try {
findDuplicateFiles(args[0], Long.parseLong(args[1]));
} catch (Exception e) {
e.printStackTrace();
}
}
private static void findDuplicateFiles(String directory, long minimumSize)
throws IOException, NoSuchAlgorithmException {
System.out.println("Directory: '" + directory + "', minimum size: " + minimumSize + " bytes.");
Path path = FileSystems.getDefault().getPath(directory);
FileVisitor visitor = new FileVisitor(path, minimumSize);
Files.walkFileTree(path, visitor);
System.out.println("The following sets of files have the same size and checksum:");
for (Map.Entry<FileKey, Map<Object, List<String>>> e : visitor.fileMap_.entrySet()) {
Map<Object, List<String>> map = e.getValue();
if (!containsDuplicates(map))
continue;
List<List<String>> fileSets = new ArrayList<>(map.values());
for (List<String> files : fileSets)
Collections.sort(files);
Collections.sort(fileSets, new StringListComparator());
FileKey key = e.getKey();
System.out.println();
System.out.println("Size: " + key.size_ + " bytes");
for (List<String> files : fileSets) {
for (int i = 0, n = files.size(); i < n; ++i) {
if (i > 0)
System.out.print(" = ");
System.out.print(files.get(i));
}
System.out.println();
}
}
}
private static class StringListComparator implements Comparator<List<String>> {
public int compare(List<String> a, List<String> b) {
int len1 = a.size(), len2 = b.size();
for (int i = 0; i < len1 && i < len2; ++i) {
int c = a.get(i).compareTo(b.get(i));
if (c != 0)
return c;
}
return Integer.compare(len1, len2);
}
}
private static boolean containsDuplicates(Map<Object, List<String>> map) {
if (map.size() > 1)
return true;
for (List<String> files : map.values()) {
if (files.size() > 1)
return true;
}
return false;
}
private static class FileVisitor extends SimpleFileVisitor<Path> {
private MessageDigest digest_;
private Path directory_;
private long minimumSize_;
private Map<FileKey, Map<Object, List<String>>> fileMap_ = new TreeMap<>();
private FileVisitor(Path directory, long minimumSize) throws NoSuchAlgorithmException {
directory_ = directory;
minimumSize_ = minimumSize;
digest_ = MessageDigest.getInstance("MD5");
}
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
if (attrs.size() >= minimumSize_) {
FileKey key = new FileKey(file, attrs, getMD5Sum(file));
Map<Object, List<String>> map = fileMap_.get(key);
if (map == null)
fileMap_.put(key, map = new HashMap<>());
List<String> files = map.get(attrs.fileKey());
if (files == null)
map.put(attrs.fileKey(), files = new ArrayList<>());
Path relative = directory_.relativize(file);
files.add(relative.toString());
}
return FileVisitResult.CONTINUE;
}
private byte[] getMD5Sum(Path file) throws IOException {
digest_.reset();
try (InputStream in = new FileInputStream(file.toString())) {
byte[] buffer = new byte[8192];
int bytes;
while ((bytes = in.read(buffer)) != -1) {
digest_.update(buffer, 0, bytes);
}
}
return digest_.digest();
}
}
private static class FileKey implements Comparable<FileKey> {
private byte[] hash_;
private long size_;
private FileKey(Path file, BasicFileAttributes attrs, byte[] hash) throws IOException {
size_ = attrs.size();
hash_ = hash;
}
public int compareTo(FileKey other) {
int c = Long.compare(other.size_, size_);
if (c == 0)
c = hashCompare(hash_, other.hash_);
return c;
}
}
private static int hashCompare(byte[] a, byte[] b) {
int len1 = a.length, len2 = b.length;
for (int i = 0; i < len1 && i < len2; ++i) {
int c = Byte.compare(a[i], b[i]);
if (c != 0)
return c;
}
return Integer.compare(len1, len2);
}
}

45
Task/Find-duplicate-files/Julia/find-duplicate-files.julia Normal file
View file

@ -0,0 +1,45 @@
using Printf, Nettle
function find_duplicates(path::String, minsize::Int = 0)
filesdict = Dict{String,Array{NamedTuple}}()
for (root, dirs, files) in walkdir(path), fn in files
filepath = joinpath(root, fn)
filestats = stat(filepath)
filestats.size > minsize || continue
hash = open(f -> hexdigest("md5", read(f)), filepath)
if haskey(filesdict, hash)
push!(filesdict[hash], (path = filepath, stats = filestats))
else
filesdict[hash] = [(path = filepath, stats = filestats)]
end
end
# Get duplicates
dups = [tups for tups in values(filesdict) if length(tups) > 1]
return dups
end
function main()
path = "."
println("Finding duplicates in \"$path\"")
dups = find_duplicates(".", 1)
println("The following group of files have the same size and the same hash:\n")
println("File name Size last modified")
println("="^76)
for files in sort(dups, by = tups -> tups[1].stats.size, rev = true)
for (path, stats) in sort(files, by = tup -> tup.path, rev = true)
@printf("%-44s%8d %s\n", path, stats.size, Libc.strftime(stats.mtime))
end
println()
end
end
main()

5
Task/Find-duplicate-files/Mathematica/find-duplicate-files.math Normal file
View file

@ -0,0 +1,5 @@
hash="SHA256";
minSize=Quantity[1,"Megabytes"];
allfiles=Once@Select[FileNames["*","",∞],!Once@DirectoryQ[#]&&Once@FileSize[#]>minSize&];
data={#,Once[FileHash[#,hash,All,"HexString"]]}&/@allfiles[[;;5]];
Grid[Select[GatherBy[data,Last],Length[#]>1&][[All,All,1]]]

111
Task/Find-duplicate-files/Nim/find-duplicate-files.nim Normal file
View file

@ -0,0 +1,111 @@
import algorithm
import os
import strformat
import strutils
import tables
import std/sha1
import times
type
# Mapping "size" -> "list of paths".
PathsFromSizes = Table[BiggestInt, seq[string]]
# Mapping "hash" -> "list fo paths".
PathsFromHashes = Table[string, seq[string]]
# Information data.
Info = tuple[size: BiggestInt; paths: seq[string]]
#---------------------------------------------------------------------------------------------------
proc processCmdLine(): tuple[dirpath: string; minsize: Natural] =
## Process the command line. Extra parameters are ignored.
if paramCount() == 0:
quit fmt"Usage: {getAppFileName().splitPath()[1]} folder minsize"
result.dirpath = paramStr(1)
if not result.dirpath.dirExists():
quit fmt"Wrong directory path: {result.dirpath}"
if paramCount() >= 2:
try:
result.minsize = parseInt(paramStr(2))
except ValueError:
quit fmt"Wrong minimum size: {paramStr(2)}"
#---------------------------------------------------------------------------------------------------
proc initPathsFromSize(dirpath: string; minsize: Natural): PathsFromSizes =
## Retrieve the files in directory "dirpath" with minimal size "minsize"
## and build the mapping from size to paths.
for path in dirpath.walkDirRec():
if not path.fileExists():
continue # Not a regular file.
let size = path.getFileSize()
if size >= minSize:
# Store path in "size to paths" table.
result.mgetOrPut(size, @[]).add(path)
#---------------------------------------------------------------------------------------------------
proc initPathsFromHashes(pathsFromSizes: PathsFromSizes): PathsFromHashes =
## Compute hashes for files whose size is not unique and build the mapping
## from hash to paths.
for size, paths in pathsFromSizes.pairs:
if paths.len > 1:
for path in paths:
# Store path in "digest to paths" table.
result.mgetOrPut($path.secureHashFile(), @[]).add(path)
#---------------------------------------------------------------------------------------------------
proc cmp(x, y: Info): int =
## Compare two information tuples. Used to sort the list of duplicates files.
result = cmp(x.size, y.size)
if result == 0:
# Same size. Compare the first paths (we are sure that they are different).
result = cmp(x.paths[0], y.paths[0])
#---------------------------------------------------------------------------------------------------
proc displayDuplicates(dirpath: string; pathsFromHashes: PathsFromHashes) =
## Display duplicates files in directory "dirpath".
echo "Files with same size and same SHA1 hash value in directory: ", dirpath
echo ""
# Build list of duplicates.
var duplicates: seq[Info]
for paths in pathsFromHashes.values:
if paths.len > 1:
duplicates.add((paths[0].getFileSize(), sorted(paths)))
if duplicates.len == 0:
echo "No files"
return
duplicates.sort(cmp, Descending)
# Display duplicates.
echo fmt"""{"Size":>10} {"Last date modified":^19} {"Inode":>8} HL File name"""
echo repeat('=', 80)
for (size, paths) in duplicates:
echo ""
for path in paths:
let mtime = path.getLastModificationTime().format("YYYY-MM-dd HH:mm:ss")
let info = path.getFileInfo()
let inode = info.id.file
let hardlink = if info.linkCount == 1: " " else: "*"
echo fmt"{size:>10} {mtime:>23} {inode:>12} {hardlink:<5} {path.relativePath(dirpath)}"
#———————————————————————————————————————————————————————————————————————————————————————————————————
let (dirpath, minsize) = processCmdLine()
let pathsFromSizes = initPathsFromSize(dirpath, minsize)
let pathsFromHashes = initPathsFromHashes(pathsFromSizes)
dirpath.displayDuplicates(pathsFromHashes)

67
Task/Find-duplicate-files/OCaml/find-duplicate-files.ocaml Normal file
View file

@ -0,0 +1,67 @@
let readdir_or_empty dir =
try Sys.readdir dir
with Sys_error e ->
prerr_endline ("Could not read dir " ^ dir ^ ": " ^ e);
[||]
let directory_walk root func =
let rec aux dir =
readdir_or_empty dir
|> Array.iter (fun filename ->
let path = Filename.concat dir filename in
let open Unix in
let stat = lstat path in
match stat.st_kind with
| S_DIR -> aux path
| S_REG -> func path stat
| _ -> ())
in
aux root
let rec input_retry ic buf pos len =
let count = input ic buf pos len in
if count = 0 || count = len then count + pos
else input_retry ic buf (pos + count) (len - count)
let with_file_in_bin fn f =
let fh = open_in_bin fn in
Fun.protect ~finally:(fun () -> close_in fh) (fun () -> f fh)
let is_really_same_file fn1 fn2 =
with_file_in_bin fn1 (fun fh1 ->
with_file_in_bin fn2 (fun fh2 ->
let len = 2048 in
let buf1 = Bytes.create len in
let buf2 = Bytes.create len in
let rec aux () =
let read1 = input_retry fh1 buf1 0 len in
let read2 = input_retry fh2 buf2 0 len in
if read1 <> read2 || buf1 <> buf2 then false
else if read1 = 0 then true
else aux ()
in
aux ()))
let () =
let tbl = Hashtbl.create 128 in
let seen = Hashtbl.create 128 in
let min_size = int_of_string Sys.argv.(2) in
directory_walk Sys.argv.(1) (fun path stat ->
try
let identity_tuple = (stat.st_dev, stat.st_ino) in
match Hashtbl.find_opt seen identity_tuple with
| Some existing ->
print_endline
("File " ^ existing ^ " is the same hard link as " ^ path)
| None -> (
Hashtbl.add seen identity_tuple path;
let size = stat.st_size in
if size >= min_size then
let digest = Digest.file path in
Hashtbl.find_all tbl digest
|> List.find_opt (is_really_same_file path)
|> function
| Some existing ->
print_endline ("File " ^ existing ^ " matches " ^ path)
| None -> Hashtbl.add tbl digest path)
with Sys_error e -> prerr_endline ("Could not hash " ^ path ^ ": " ^ e))

86
Task/Find-duplicate-files/Objeck/find-duplicate-files.objeck Normal file
View file

@ -0,0 +1,86 @@
use System.IO.File;
use System.Time;
use Collection;
class Duplicate {
function : Main(args : String[]) ~ Nil {
if(args->Size() = 2) {
file_sets := SortDups(GetDups(args[0], args[1]->ToInt()));
each(i : file_sets) {
file_set := file_sets->Get(i)->As(Vector);
if(file_set->Size() > 1) {
"Duplicates:"->PrintLine();
"----"->PrintLine();
each(j : file_set) {
file_set->Get(j)->As(FileMeta)->ToString()->PrintLine();
};
};
'\n'->Print();
};
};
}
function : SortDups(unsorted : Vector) ~ Vector {
sorted := IntMap->New();
each(i : unsorted) {
value := unsorted->Get(i)->As(Vector);
key := value->Get(0)->As(FileMeta)->GetSize();
sorted->Insert(key, value);
};
return sorted->GetValues();
}
function : GetDups(dir : String, size : Int) ~ Vector {
duplicates := StringMap->New();
files := Directory->List(dir);
each(i : files) {
file_name := String->New(dir);
file_name += '/';
file_name += files[i];
file_size := File->Size(file_name);
if(file_size >= size) {
file_date := File->ModifiedTime(file_name);
file_hash := file_size->ToString();
file_hash += ':';
file_hash += Encryption.Hash->MD5(FileReader->ReadBinaryFile(file_name))->ToString();
file_meta := FileMeta->New(file_name, file_size, file_date, file_hash);
file_set := duplicates->Find(file_hash)->As(Vector);
if(file_set = Nil) {
file_set := Vector->New();
duplicates->Insert(file_hash, file_set);
};
file_set->AddBack(file_meta);
};
};
return duplicates->GetValues();
}
}
class FileMeta {
@name : String;
@size : Int;
@date : Date;
@hash : String;
New(name : String, size : Int, date : Date, hash : String) {
@name := name;
@size := size;
@date := date;
@hash := hash;
}
method : public : GetSize() ~ Int {
return @size;
}
method : public : ToString() ~ String {
date_str := @date->ToShortString();
return "{$@name}, {$@size}, {$date_str}";
}
}

39
Task/Find-duplicate-files/Perl/find-duplicate-files.pl Normal file
View file

@ -0,0 +1,39 @@
use File::Find qw(find);
use File::Compare qw(compare);
use Sort::Naturally;
use Getopt::Std qw(getopts);
my %opts;
$opts{s} = 1;
getopts("s:", \%opts);
sub find_dups {
my($dir) = @_;
my @results;
my %files;
find {
no_chdir => 1,
wanted => sub { lstat; -f _ && (-s >= $opt{s} ) && push @{$files{-s _}}, $_ }
} => $dir;
foreach my $files (values %files) {
next unless @$files;
my %dups;
foreach my $a (0 .. @$files - 1) {
for (my $b = $a + 1 ; $b < @$files ; $b++) {
next if compare(@$files[$a], @$files[$b]);
push @{$dups{ @$files[$a] }}, splice @$files, $b--, 1;
}
}
while (my ($original, $clones) = each %dups) {
push @results, sprintf "%8d %s\n", (stat($original))[7], join ', ', sort $original, @$clones;
}
}
reverse nsort @results;
}
print for find_dups(@ARGV);

58
Task/Find-duplicate-files/Phix/find-duplicate-files.phix Normal file
View file

@ -0,0 +1,58 @@
(notonline)-->
<span style="color: #008080;">without</span> <span style="color: #008080;">js</span> <span style="color: #000080;font-style:italic;">-- file i/o</span>
<span style="color: #004080;">integer</span> <span style="color: #000000;">min_size</span><span style="color: #0000FF;">=</span><span style="color: #000000;">1</span>
<span style="color: #004080;">sequence</span> <span style="color: #000000;">res</span> <span style="color: #0000FF;">=</span> <span style="color: #0000FF;">{}</span>
<span style="color: #004080;">atom</span> <span style="color: #000000;">t1</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">time</span><span style="color: #0000FF;">()+</span><span style="color: #000000;">1</span>
<span style="color: #008080;">function</span> <span style="color: #000000;">store_res</span><span style="color: #0000FF;">(</span><span style="color: #004080;">string</span> <span style="color: #000000;">filepath</span><span style="color: #0000FF;">,</span> <span style="color: #004080;">sequence</span> <span style="color: #000000;">dir_entry</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">if</span> <span style="color: #008080;">not</span> <span style="color: #7060A8;">match</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"backup"</span><span style="color: #0000FF;">,</span><span style="color: #000000;">filepath</span><span style="color: #0000FF;">)</span> <span style="color: #000080;font-style:italic;">-- (example filter)</span>
<span style="color: #008080;">and</span> <span style="color: #008080;">not</span> <span style="color: #7060A8;">find</span><span style="color: #0000FF;">(</span><span style="color: #008000;">'d'</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">dir_entry</span><span style="color: #0000FF;">[</span><span style="color: #004600;">D_ATTRIBUTES</span><span style="color: #0000FF;">])</span> <span style="color: #008080;">then</span>
<span style="color: #004080;">atom</span> <span style="color: #000000;">size</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">dir_entry</span><span style="color: #0000FF;">[</span><span style="color: #004600;">D_SIZE</span><span style="color: #0000FF;">]</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">size</span><span style="color: #0000FF;">>=</span><span style="color: #000000;">min_size</span> <span style="color: #008080;">then</span>
<span style="color: #000000;">res</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">append</span><span style="color: #0000FF;">(</span><span style="color: #000000;">res</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">size</span><span style="color: #0000FF;">,</span><span style="color: #000000;">filepath</span><span style="color: #0000FF;">,</span><span style="color: #000000;">dir_entry</span><span style="color: #0000FF;">})</span>
<span style="color: #008080;">if</span> <span style="color: #7060A8;">time</span><span style="color: #0000FF;">()></span><span style="color: #000000;">t1</span> <span style="color: #008080;">then</span>
<span style="color: #7060A8;">printf</span><span style="color: #0000FF;">(</span><span style="color: #000000;">1</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"%d files found\r"</span><span style="color: #0000FF;">,</span><span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">res</span><span style="color: #0000FF;">))</span>
<span style="color: #000000;">t1</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">time</span><span style="color: #0000FF;">()+</span><span style="color: #000000;">1</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">return</span> <span style="color: #000000;">0</span> <span style="color: #000080;font-style:italic;">-- keep going</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">function</span>
<span style="color: #004080;">integer</span> <span style="color: #000000;">exit_code</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">walk_dir</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"demo\\clocks\\love"</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">store_res</span><span style="color: #0000FF;">,</span> <span style="color: #004600;">true</span><span style="color: #0000FF;">)</span>
<span style="color: #000000;">res</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">sort</span><span style="color: #0000FF;">(</span><span style="color: #000000;">res</span><span style="color: #0000FF;">,</span><span style="color: #000000;">DESCENDING</span><span style="color: #0000FF;">)</span>
<span style="color: #7060A8;">printf</span><span style="color: #0000FF;">(</span><span style="color: #000000;">1</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"%d files found\n"</span><span style="color: #0000FF;">,</span><span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">res</span><span style="color: #0000FF;">))</span>
<span style="color: #004080;">integer</span> <span style="color: #000000;">duplicates</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">0</span>
<span style="color: #008080;">for</span> <span style="color: #000000;">i</span><span style="color: #0000FF;">=</span><span style="color: #000000;">1</span> <span style="color: #008080;">to</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">res</span><span style="color: #0000FF;">)-</span><span style="color: #000000;">1</span> <span style="color: #008080;">do</span>
<span style="color: #008080;">for</span> <span style="color: #000000;">j</span><span style="color: #0000FF;">=</span><span style="color: #000000;">i</span><span style="color: #0000FF;">+</span><span style="color: #000000;">1</span> <span style="color: #008080;">to</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">res</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">do</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">res</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">][</span><span style="color: #000000;">1</span><span style="color: #0000FF;">]!=</span><span style="color: #000000;">res</span><span style="color: #0000FF;">[</span><span style="color: #000000;">j</span><span style="color: #0000FF;">][</span><span style="color: #000000;">1</span><span style="color: #0000FF;">]</span> <span style="color: #008080;">then</span> <span style="color: #008080;">exit</span> <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #004080;">string</span> <span style="color: #000000;">si</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">join_path</span><span style="color: #0000FF;">({</span><span style="color: #000000;">res</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">][</span><span style="color: #000000;">2</span><span style="color: #0000FF;">],</span><span style="color: #000000;">res</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">][</span><span style="color: #000000;">3</span><span style="color: #0000FF;">][</span><span style="color: #004600;">D_NAME</span><span style="color: #0000FF;">]}),</span>
<span style="color: #000000;">sj</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">join_path</span><span style="color: #0000FF;">({</span><span style="color: #000000;">res</span><span style="color: #0000FF;">[</span><span style="color: #000000;">j</span><span style="color: #0000FF;">][</span><span style="color: #000000;">2</span><span style="color: #0000FF;">],</span><span style="color: #000000;">res</span><span style="color: #0000FF;">[</span><span style="color: #000000;">j</span><span style="color: #0000FF;">][</span><span style="color: #000000;">3</span><span style="color: #0000FF;">][</span><span style="color: #004600;">D_NAME</span><span style="color: #0000FF;">]})</span>
<span style="color: #004080;">integer</span> <span style="color: #000000;">fni</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">open</span><span style="color: #0000FF;">(</span><span style="color: #000000;">si</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"rb"</span><span style="color: #0000FF;">),</span>
<span style="color: #000000;">fnj</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">open</span><span style="color: #0000FF;">(</span><span style="color: #000000;">sj</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"rb"</span><span style="color: #0000FF;">),</span>
<span style="color: #000000;">size</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">res</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">][</span><span style="color: #000000;">1</span><span style="color: #0000FF;">]</span>
<span style="color: #004080;">bool</span> <span style="color: #000000;">same</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">true</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">fni</span><span style="color: #0000FF;">=-</span><span style="color: #000000;">1</span> <span style="color: #008080;">or</span> <span style="color: #000000;">fnj</span><span style="color: #0000FF;">=-</span><span style="color: #000000;">1</span> <span style="color: #008080;">then</span> <span style="color: #0000FF;">?</span><span style="color: #000000;">9</span><span style="color: #0000FF;">/</span><span style="color: #000000;">0</span> <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">for</span> <span style="color: #000000;">k</span><span style="color: #0000FF;">=</span><span style="color: #000000;">1</span> <span style="color: #008080;">to</span> <span style="color: #000000;">size</span><span style="color: #0000FF;">+</span><span style="color: #000000;">1</span> <span style="color: #008080;">do</span> <span style="color: #000080;font-style:italic;">-- (check eof as well)</span>
<span style="color: #008080;">if</span> <span style="color: #7060A8;">getc</span><span style="color: #0000FF;">(</span><span style="color: #000000;">fni</span><span style="color: #0000FF;">)!=</span><span style="color: #7060A8;">getc</span><span style="color: #0000FF;">(</span><span style="color: #000000;">fnj</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">then</span>
<span style="color: #000000;">same</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">false</span>
<span style="color: #008080;">exit</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">for</span>
<span style="color: #7060A8;">close</span><span style="color: #0000FF;">(</span><span style="color: #000000;">fni</span><span style="color: #0000FF;">)</span>
<span style="color: #7060A8;">close</span><span style="color: #0000FF;">(</span><span style="color: #000000;">fnj</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">same</span> <span style="color: #008080;">then</span>
<span style="color: #000080;font-style:italic;">-- prettifying the output left as an exercise...</span>
<span style="color: #0000FF;">?</span><span style="color: #000000;">res</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">]</span>
<span style="color: #0000FF;">?</span><span style="color: #000000;">res</span><span style="color: #0000FF;">[</span><span style="color: #000000;">j</span><span style="color: #0000FF;">]</span>
<span style="color: #000000;">duplicates</span> <span style="color: #0000FF;">+=</span> <span style="color: #000000;">1</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">for</span>
<span style="color: #008080;">if</span> <span style="color: #7060A8;">time</span><span style="color: #0000FF;">()></span><span style="color: #000000;">t1</span> <span style="color: #008080;">then</span>
<span style="color: #7060A8;">printf</span><span style="color: #0000FF;">(</span><span style="color: #000000;">1</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"processing %d/%d...\r"</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">i</span><span style="color: #0000FF;">,</span><span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">res</span><span style="color: #0000FF;">)})</span>
<span style="color: #000000;">t1</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">time</span><span style="color: #0000FF;">()+</span><span style="color: #000000;">1</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">for</span>
<span style="color: #7060A8;">printf</span><span style="color: #0000FF;">(</span><span style="color: #000000;">1</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"%d duplicates found\n"</span><span style="color: #0000FF;">,</span><span style="color: #000000;">duplicates</span><span style="color: #0000FF;">)</span>
<!--

34
Task/Find-duplicate-files/PicoLisp/find-duplicate-files.l Normal file
View file

@ -0,0 +1,34 @@
`(== 64 64)
(de mmap (L F)
(native "@" "mmap" 'N 0 L 1 2 F 0) )
(de munmap (A L)
(native "@" "munmap" 'N A L) )
(de xxh64 (M S)
(let
(R (native "libxxhash.so" "XXH64" 'N M S 0)
P `(** 2 64) )
(if (lt0 R)
(& (+ R P) (dec P))
R ) ) )
(de walk (Dir)
(recur (Dir)
(for F (dir Dir)
(let (Path (pack Dir "/" F) Info (info Path T))
(when (car Info)
(if (=T (car Info))
(recurse Path)
(if (lup D (car Info))
(push (cdr @) Path)
(idx 'D (list (car Info) (cons Path)) T) ) ) ) ) ) ) )
(off D)
(walk "/bin")
(for Lst (filter cdadr (idx 'D))
(let L
(by
'((F)
(let (M (mmap (car Lst) (open F T))
S (car Lst) )
(prog1 (xxh64 M S) (munmap M S)) ) )
group
(cadr Lst) )
(and (filter cdr L) (println (car Lst) @)) ) )

51
Task/Find-duplicate-files/Python/find-duplicate-files.py Normal file
View file

@ -0,0 +1,51 @@
from __future__ import print_function
import os
import hashlib
import datetime
def FindDuplicateFiles(pth, minSize = 0, hashName = "md5"):
knownFiles = {}
#Analyse files
for root, dirs, files in os.walk(pth):
for fina in files:
fullFina = os.path.join(root, fina)
isSymLink = os.path.islink(fullFina)
if isSymLink:
continue # Skip symlinks
si = os.path.getsize(fullFina)
if si < minSize:
continue
if si not in knownFiles:
knownFiles[si] = {}
h = hashlib.new(hashName)
h.update(open(fullFina, "rb").read())
hashed = h.digest()
if hashed in knownFiles[si]:
fileRec = knownFiles[si][hashed]
fileRec.append(fullFina)
else:
knownFiles[si][hashed] = [fullFina]
#Print result
sizeList = list(knownFiles.keys())
sizeList.sort(reverse=True)
for si in sizeList:
filesAtThisSize = knownFiles[si]
for hashVal in filesAtThisSize:
if len(filesAtThisSize[hashVal]) < 2:
continue
fullFinaLi = filesAtThisSize[hashVal]
print ("=======Duplicate=======")
for fullFina in fullFinaLi:
st = os.stat(fullFina)
isHardLink = st.st_nlink > 1
infoStr = []
if isHardLink:
infoStr.append("(Hard linked)")
fmtModTime = datetime.datetime.utcfromtimestamp(st.st_mtime).strftime('%Y-%m-%dT%H:%M:%SZ')
print (fmtModTime, si, os.path.relpath(fullFina, pth), " ".join(infoStr))
if __name__=="__main__":
FindDuplicateFiles('/home/tim/Dropbox', 1024*1024)

26
Task/Find-duplicate-files/REXX/find-duplicate-files-1.rexx Normal file
View file

@ -0,0 +1,26 @@
/*REXX program to reads a (DOS) directory and finds and displays files that identical.*/
sep=center(' files are identical in size and content: ',79,"═") /*define the header. */
tFID= 'c:\TEMP\FINDDUP.TMP' /*use this as a temporary FileID. */
arg maxSize aDir /*obtain optional arguments from the CL*/
if maxSize='' | maxSize="," then maxSize=1000000 /*filesize limit (in bytes) [1 million]*/
aDir=strip(aDir) /*remove any leading or trailing blanks*/
if right(aDir,1)\=='\' then aDir=aDir"\" /*possibly add a trailing backslash [\]*/
"DIR" aDir '/a-d-s-h /oS /s | FIND "/" >' tFID /*the (DOS) DIR output ───► temp file. */
pFN= /*the previous filename and filesize. */
pSZ=; do j=0 while lines(tFID)\==0 /*process each of the files in the list*/
aLine=linein(tFID) /*obtain (DOS) DIR's output about a FID*/
parse var aLine . . sz fn /*obtain the filesize and its fileID. */
sz=space(translate(sz,,','),0) /*elide any commas from the size number*/
if sz>maxSize then leave /*Is the file > maximum? Ignore file. */
/* [↓] files identical? (1st million)*/
if sz==pSZ then if charin(aDir||pFN,1,sz)==charin(aDir||FN,1,sz) then do
say sep
say pLine
say aLine
say
end
pSZ=sz; pFN=FN; pLine=aLine /*remember the previous stuff for later*/
end /*j*/
if lines(tFID)\==0 then 'ERASE' tFID /*do housecleaning (delete temp file).*/
/*stick a fork in it, we're all done. */

57
Task/Find-duplicate-files/REXX/find-duplicate-files-2.rexx Normal file
View file

@ -0,0 +1,57 @@
/*REXX program to reads a (DOS) directory and finds and displays files that identical.*/
sep=center(' files are identical in size and content: ',79,"═") /*define the header. */
parse arg !; if !all(arg()) then exit /*boilerplate HELP(?)*/
signal on halt; signal on novalue; signal on syntax /*handle exceptions, */
if \!dos then call err 'this program requires the DOS [environment].'
call getTFID /*defines a temporary File ID for DOS.*/
arg maxSize aDir /*obtain optional arguments from the CL*/
if maxSize='' | maxSize="," then maxSize=1000000 /*filesize limit (in bytes) [1 million]*/
if \isInt(maxSize) then call err "maxSize isn't an integer:" maxSize
if maxSize<0 then call err "maxSize can't be negative:" maxSize
if maxSize=0 then call err "maxSize can't be zero:" maxSize
aDir=strip(aDir) /*remove any leading or trailing blanks*/
if right(aDir,3)=='*.*' then aDir=substr(aDir,1,length(aDir)-3) /*adjust the dir name.*/
if right(aDir,1)\=='\' then aDir=aDir"\" /*possibly add a trailing backslash [\]*/
@dir = 'DIR' /*literal for the (DOS) DIR command. */
@dirNots= '/a-d-s-h' /*ignore DIRs, SYSTEM, and HIDDEN files*/
@dirOpts= '/oS /s' /*sort DIR's (+ subdirs) files by size.*/
@filter = '| FIND "/"' /*the "lines" must have a slash [/]. */
@erase = 'ERASE' /*literal for the (DOS) ERASE command.*/
@dir aDir @dirNots @dirOpts @filter '>' tFID /*(DOS) DIR output ──► temporary file.*/
pFN= /*the previous filename and filesize. */
pSZ=; do j=0 while lines(tFID)\==0 /*process each of the files in the list*/
aLine=linein(tFID) /*obtain (DOS) DIR's output about a FID*/
parse var aLine . . sz fn /*obtain the filesize and its fileID. */
sz=space(translate(sz,,','),0) /*elide any commas from the size number*/
if sz>maxSize then leave /*Is the file > maximum? Ignore file. */
/* [↓] files identical? (1st million)*/
if sz==pSZ then if charin(aDir||pFN,1,sz)==charin(aDir||FN,1,sz) then do
say sep
say pLine
say aLine
say
end
pSZ=sz; pFN=FN; pLine=aLine /*remember the previous stuff for later*/
end /*j*/
say j 'file's(j) "examined in" aDir /*show information to the screen.*/
if lines(tFID)\==0 then 'ERASE' tFID /*do housecleaning (delete temp file).*/
exit /*stick a fork in it, we're all done. */
/*═════════════════════════════general 1─line subs══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════════*/
!all: !!=!;!=space(!);upper !;call !fid;!nt=right(!var('OS'),2)=="NT";!cls=word('CLS VMFCLEAR CLRSCREEN',1+!cms+!tso*2);if arg(1)\==1 then return 0;if wordpos(!,"? ?SAMPLES ?AUTHOR ?FLOW")==0 then return 0;!call=']$H';call "$H" !fn !;!call=;return 1
!cal: if symbol('!CALL')\=="VAR" then !call=; return !call
!env: !env='ENVIRONMENT'; if !sys=="MSDOS" | !brexx | !r4 | !roo then !env='SYSTEM'; if !os2 then !env="OS2"!env; !ebcdic=1=='f1'x; return
!fid: parse upper source !sys !fun !fid . 1 . . !fn !ft !fm .; call !sys; if !dos then do; _=lastpos('\',!fn); !fm=left(!fn,_); !fn=substr(!fn,_+1); parse var !fn !fn "." !ft; end; return word(0 !fn !ft !fm, 1+('0'arg(1)))
!rex: parse upper version !ver !vernum !verdate .; !brexx='BY'==!vernum; !kexx="KEXX"==!ver; !pcrexx='REXX/PERSONAL'==!ver | "REXX/PC"==!ver; !r4='REXX-R4'==!ver; !regina="REXX-REGINA"==left(!ver,11); !roo='REXX-ROO'==!ver; call !env; return
!sys: !cms=!sys=='CMS'; !os2=!sys=="OS2"; !tso=!sys=='TSO' | !sys=="MVS"; !vse=!sys=='VSE'; !dos=pos("DOS",!sys)\==0|pos('WIN',!sys)\==0|!sys=="CMD"; call !rex; return
!var: call !fid; if !kexx then return space(dosenv(arg(1))); return space(value(arg(1),,!env))
err: say; say; say center(' error! ', 60, "*"); say; do j=1 for arg(); say arg(j); say; end; say; exit 13
getdTFID: tfid=p(!var("TMP") !var('TEMP') homedrive()"\"); if substr(tfid,2,1)==':'&substr(tfid,3,1)\=="\" then tfid=insert('\',t,2); return strip(tfid,"T",'\')"\"arg(1)'.'arg(2)
getTFID: if symbol('TFID')=="LIT" then tfid=; if tfid\=='' then return tfid; gfn=word(arg(1) !fn,1);gft=word(arg(2) "TMP",1); tfid='TEMP';if !tso then tfid=gfn"."gft;if !cms then tfid=gfn','gft",A4";if !dos then tfid=getdTFID(gfn,gft);return tfid
halt: call err 'program has been halted.'
homedrive: if symbol('HOMEDRIVE')\=="VAR" then homedrive=p(!var('HOMEDRIVE') "C:"); return homedrive
isint: return datatype(arg(1),'W')
novalue: syntax: call err 'REXX program' condition("C") 'error',condition("D"),'REXX source statement (line' sigl"):",sourceline(sigl)
p: return word(arg(1),1)
s: if arg(1)==1 then return arg(3); return word(arg(2) 's',1)

39
Task/Find-duplicate-files/Racket/find-duplicate-files.rkt Normal file
View file

@ -0,0 +1,39 @@
#lang racket
(struct F (name id size [links #:mutable]))
(require openssl/sha1)
(define (find-duplicate-files path size)
(define Fs
(sort
(fold-files
(λ(path type acc)
(define s (and (eq? 'file type) (file-size path)))
(define i (and s (<= size s) (file-or-directory-identity path)))
(define ln (and i (findf (λ(x) (equal? i (F-id x))) acc)))
(when ln (set-F-links! ln (cons (path->string path) (F-links ln))))
(if (and i (not ln)) (cons (F path i s '()) acc) acc))
'() path #f)
> #:key F-size))
(define (find-duplicates Fs)
(define t (make-hash))
(for ([F Fs])
(define cksum (call-with-input-file (F-name F) sha1))
(hash-set! t cksum (cons F (hash-ref t cksum '()))))
(for/list ([(n Fs) (in-hash t)] #:unless (null? (cdr Fs))) Fs))
(let loop ([Fs Fs])
(if (null? Fs) '()
(let-values ([(Fs Rs)
(splitf-at Fs (λ(F) (= (F-size F) (F-size (car Fs)))))])
(append (find-duplicates Fs)
(loop Rs))))))
(define (show-duplicates path size)
(for ([Fs (find-duplicate-files path size)])
(define (links F)
(if (null? (F-links F)) ""
(format " also linked at ~a" (string-join (F-links F) ", "))))
(printf "~a (~a)~a\n" (F-name (car Fs)) (F-size (car Fs)) (links (car Fs)))
(for ([F (cdr Fs)]) (printf " ~a~a\n" (F-name F) (links F)))))
(show-duplicates (find-system-path 'home-dir) 1024)

32
Task/Find-duplicate-files/Raku/find-duplicate-files.raku Normal file
View file

@ -0,0 +1,32 @@
use Digest::SHA256::Native;
sub MAIN( $dir = '.', :$minsize = 5, :$recurse = True ) {
my %files;
my @dirs = $dir.IO.absolute.IO;
while @dirs {
my @files = @dirs.pop;
while @files {
for @files.pop.dir -> $path {
%files{ $path.s }.push: $path if $path.f and $path.s >= $minsize;
@dirs.push: $path if $path.d and $path.r and $recurse
}
}
}
for %files.sort( +*.key ).grep( *.value.elems > 1)».kv -> ($size, @list) {
my %dups;
@list.map: { %dups{ sha256-hex( ($_.slurp :bin).decode ) }.push: $_.Str };
for %dups.grep( *.value.elems > 1)».value -> @dups {
say sprintf("%9s : ", scale $size ), @dups.join(', ');
}
}
}
sub scale ($bytes) {
given $bytes {
when $_ < 2**10 { $bytes ~ ' B' }
when $_ < 2**20 { ($bytes / 2**10).round(.1) ~ ' KB' }
when $_ < 2**30 { ($bytes / 2**20).round(.1) ~ ' MB' }
default { ($bytes / 2**30).round(.1) ~ ' GB' }
}
}

44
Task/Find-duplicate-files/Ring/find-duplicate-files.ring Normal file
View file

@ -0,0 +1,44 @@
# Project : Find duplicate files
d = "/Windows/System32"
chdir(d)
dir = dir(d)
dirlist = []
for n = 1 to len(dir)
if dir[n][2] = 0
str = read(dir[n][1])
lenstr = len(str)
add(dirlist,[lenstr,dir[n][1]])
ok
next
see "Directory : " + d + nl
see "--------------------------------------------" + nl
dirlist = sortfirst(dirlist)
line = 0
for n = 1 to len(dirlist)-1
if dirlist[n][1] = dirlist[n+1][1]
see "" + dirlist[n][1] + " " + dirlist[n][2] + nl
see "" + dirlist[n+1][1] + " " + dirlist[n+1][2] + nl
if n < len(dirlist)-2 and dirlist[n+1][1] != dirlist[n+2][1]
line = 1
ok
else
line = 0
ok
if line = 1
see "--------------------------------------------" + nl
ok
next
func sortfirst(alist)
for n = 1 to len(alist) - 1
for m = n + 1 to len(alist)
if alist[m][1] < alist[n][1]
swap(alist,m,n)
ok
if alist[m][1] = alist[n][1] and strcmp(alist[m][2],alist[n][2]) < 0
swap(alist,m,n)
ok
next
next
return alist

18
Task/Find-duplicate-files/Ruby/find-duplicate-files.rb Normal file
View file

@ -0,0 +1,18 @@
require 'digest/md5'
def find_duplicate_files(dir)
puts "\nDirectory : #{dir}"
Dir.chdir(dir) do
file_size = Dir.foreach('.').select{|f| FileTest.file?(f)}.group_by{|f| File.size(f)}
file_size.each do |size, files|
next if files.size==1
files.group_by{|f| Digest::MD5.file(f).to_s}.each do |md5,fs|
next if fs.size==1
puts " --------------------------------------------"
fs.each{|file| puts " #{File.mtime(file)} #{size} #{file}"}
end
end
end
end
find_duplicate_files("/Windows/System32")

97
Task/Find-duplicate-files/Rust/find-duplicate-files.rust Normal file
View file

@ -0,0 +1,97 @@
use std::{
collections::BTreeMap,
fs::{read_dir, File},
hash::Hasher,
io::Read,
path::{Path, PathBuf},
};
type Duplicates = BTreeMap<(u64, u64), Vec<PathBuf>>;
struct DuplicateFinder {
found: Duplicates,
min_size: u64,
}
impl DuplicateFinder {
fn search(path: impl AsRef<Path>, min_size: u64) -> std::io::Result<Duplicates> {
let mut result = Self {
found: BTreeMap::new(),
min_size,
};
result.walk(path)?;
Ok(result.found)
}
fn walk(&mut self, path: impl AsRef<Path>) -> std::io::Result<()> {
let listing = read_dir(path.as_ref())?;
for entry in listing {
let entry = entry?;
let path = entry.path();
if path.is_dir() {
self.walk(path)?;
} else {
self.compute_digest(&path)?;
}
}
Ok(())
}
fn compute_digest(&mut self, file: &Path) -> std::io::Result<()> {
let size = file.metadata()?.len();
if size < self.min_size {
return Ok(());
}
// This hasher is weak, we could otherwise use an external crate
let mut hasher = std::collections::hash_map::DefaultHasher::default();
let mut bytes = [0u8; 8182];
let mut f = File::open(file)?;
loop {
let n = f.read(&mut bytes[..])?;
hasher.write(&bytes[..n]);
if n == 0 {
break;
}
}
let hash = hasher.finish();
self.found
.entry((size, hash))
.or_insert_with(Vec::new)
.push(file.to_owned());
Ok(())
}
}
fn main() -> std::io::Result<()> {
let mut args = std::env::args();
args.next(); // Skip the executable name
let dir = args.next().unwrap_or_else(|| ".".to_owned());
let min_size = args
.next()
.and_then(|arg| arg.parse::<u64>().ok())
.unwrap_or(0u64);
DuplicateFinder::search(dir, min_size)?
.iter()
.rev()
.filter(|(_, files)| files.len() > 1)
.for_each(|((size, _), files)| {
println!("Size: {}", size);
files
.iter()
.for_each(|file| println!("{}", file.to_string_lossy()));
println!();
});
Ok(())
}

49
Task/Find-duplicate-files/Sidef/find-duplicate-files.sidef Normal file
View file

@ -0,0 +1,49 @@
# usage: sidef fdf.sf [size] [dir1] [...]
require('File::Find')
func find_duplicate_files(Block code, size_min=0, *dirs) {
var files = Hash()
%S<File::Find>.find(
Hash(
no_chdir => true,
wanted => func(arg) {
var file = File(arg)
file.is_file || return()
file.is_link && return()
var size = file.size
size >= size_min || return()
files{size} := [] << file
},
) => dirs...
)
files.values.each { |set|
set.len > 1 || next
var dups = Hash()
for i in (^set.end) {
for (var j = set.end; j > i; --j) {
if (set[i].compare(set[j]) == 0) {
dups{set[i]} := [] << set.pop_at(j++)
}
}
}
dups.each{ |k,v| code(k.to_file, v...) }
}
return()
}
var duplicates = Hash()
func collect(*files) {
duplicates{files[0].size} := [] << files
}
find_duplicate_files(collect, Num(ARGV.shift), ARGV...)
for k,v in (duplicates.sort_by { |k| -k.to_i }) {
say "=> Size: #{k}\n#{'~'*80}"
for files in v {
say "#{files.sort.join(%Q[\n])}\n#{'-'*80}"
}
}

57
Task/Find-duplicate-files/Tcl/find-duplicate-files.tcl Normal file
View file

@ -0,0 +1,57 @@
package require fileutil
package require md5
proc finddupfiles {dir {minsize 1}} {
foreach fn [fileutil::find $dir] {
file lstat $fn stat
if {$stat(size) < $minsize} continue
dict lappend byino $stat(dev),$stat(ino) $fn
if {$stat(type) ne "file"} continue
set f [open $fn "rb"]
set content [read $f]
close $f
set md5 [md5::md5 -hex $content]
dict lappend byhash $md5 $fn
}
set groups {}
foreach group [dict values $byino] {
if {[llength $group] <= 1} continue
set gs [lsort $group]
dict set groups [lindex $gs 0] $gs
}
foreach group [dict values $byhash] {
if {[llength $group] <= 1} continue
foreach f $group {
if {[dict exists $groups $f]} {
dict set groups $f [lsort -unique \
[concat [dict get $groups $f] $group]]
unset group
break
}
}
if {[info exist group]} {
set gs [lsort $group]
dict set groups [lindex $gs 0] $gs
}
}
set masters {}
dict for {n g} $groups {
lappend masters [list $n [llength $g],$n]
}
set result {}
foreach p [lsort -decreasing -index 1 -dictionary $masters] {
set n [lindex $p 0]
lappend result $n [dict get $groups $n]
}
return $result
}
foreach {leader dupes} [finddupfiles {*}$argv] {
puts "$leader has duplicates"
set n 0
foreach d $dupes {
if {$d ne $leader} {
puts " dupe #[incr n]: $d"
}
}
}

40
Task/Find-duplicate-files/Wren/find-duplicate-files.wren Normal file
View file

@ -0,0 +1,40 @@
import "io" for Directory, File, Stat
import "/crypto" for Sha1
import "/sort" for Sort
var findDuplicates = Fn.new { |dir, minSize|
if (!Directory.exists(dir)) Fiber.abort("Directory does not exist.")
var files = Directory.list(dir).where { |f| Stat.path("%(dir)/%(f)").size >= minSize }
var hashMap = {}
for (file in files) {
var path = "%(dir)/%(file)"
if (Stat.path(path).isDirectory) continue
var contents = File.read(path)
var hash = Sha1.digest(contents)
var exists = hashMap.containsKey(hash)
if (exists) {
hashMap[hash].add(file)
} else {
hashMap[hash] = [file]
}
}
var duplicates = []
for (key in hashMap.keys) {
if (hashMap[key].count > 1) {
var files = hashMap[key]
var path = "%(dir)/%(files[0])"
var size = Stat.path(path).size
duplicates.add([size, files])
}
}
var cmp = Fn.new { |i, j| (j[0] - i[0]).sign } // by decreasing size
Sort.insertion(duplicates, cmp)
System.print("The sets of duplicate files are:\n")
for (dup in duplicates) {
System.print("Size %(dup[0]) bytes:")
System.print(dup[1].join("\n"))
System.print()
}
}
findDuplicates.call("./", 1000)

54
Task/Find-duplicate-files/Zkl/find-duplicate-files.zkl Normal file
View file

@ -0,0 +1,54 @@
include(zkl.h.zkl);
const FLAGS=FILE.GLOB.IGNORE_CASE + FILE.GLOB.NO_DIRS;
var [const] MsgHash=Import("zklMsgHash");
var recurse=False, fileSpec, minSz=0, maxSz=(0).MAX;
argh:=Utils.Argh(
T("+R","R","Recurse into subdirectories, starting at <arg>",
fcn(arg){ recurse=arg }),
T("+minSz","","Only consider files larger than <arg>",
fcn(arg){ minSz=arg.toInt() }),
T("+maxSz","","Only consider files less than <arg>",
fcn(arg){ maxSz=arg.toInt() }),
);
argh.parse(vm.arglist);
try { fileSpec=argh.loners[0]; }
catch{
argh.usage("Find duplicate files");
System.exit(1);
}
fnames:=Data(0,String);
if (recurse) File.globular(recurse,fileSpec,True,FLAGS,fnames);
else File.glob(fileSpec,FLAGS).pump(fnames);
files:=Dictionary(); // (len:(name,name...), ...)
foreach fname in (fnames){
sz:=File.len(fname);
if(minSz<=sz<=maxSz) files.appendV(File.len(fname),fname);
}
//////////////////////// group files by size
files=files.pump(List,Void.Xplode,fcn(k,v){ v.len()>1 and v or Void.Skip });
println("Found %d groups of same sized files, %d files total.".fmt(files.len(),
files.apply("len").sum(0)));
if(not files) System.exit(); // no files found
buffer:=Data(0d100_000); // we'll resuse this buffer for hashing
hashes:=files.pump(List,'wrap(fnames){ // get the MD5 hash for each file
fnames.pump(List,'wrap(fname){
file,hash := File(fname,"rb"), MsgHash.toSink("MD5");
file.pump(buffer,hash); file.close();
return(hash.close(),fname); // -->( (hash,name), (hash,name) ... )
})
},T(Void.Write,Void.Write)); // flatten list of lists of lists to above
// Hash the file hashes, then scoop out the files with the same hash
buffer:=Dictionary();
files:=hashes.pump(Void,Void.Xplode,buffer.appendV)
.pump(List,Void.Xplode,fcn(k,v){ v.len()>1 and v or Void.Skip });
println("Found %d duplicate files:".fmt(files.apply("len").sum(0)));
foreach group in (files){ println(" ",group.concat(", ")) }