def check_for_duplicates(paths, hash=hashlib.sha1):
    hashes_by_size = defaultdict(list)  # dict of size_in_bytes
    hashes_on_1k = defaultdict(list)  # dict of (hash1k, size_in_bytes)
    hashes_full = {}

for path in paths:
    for dirpath, dirnames, filenames in os.walk(path):
        for filename in filenames:
            full_path = os.path.join(dirpath, filename)

full_path = os.path.realpath(full_path)
file_size = os.path.getsize(full_path)
hashes_by_size[file_size].append(full_path)

for size_in_bytes, files in hashes_by_size.items():
    if len(files) < 2:
        continue  # File size is unique, no need to spend CPU cycles on it

for __, files_list in hashes_on_1k.items():
    if len(files_list) < 2:
        continue    # Hash of fist 1k file bytes is unique, no need to spend cpy cycles on it

for filename in files_list:
    try: 
        full_hash = get_hash(filename, first_chunk_only=False)
        duplicate = hashes_full.get(full_hash)
        if duplicate:
            print("Duplicate found: {} and {}".format(filename, duplicate))
        else:
            hashes_full[full_hash] = filename

#!/usr/bin/python

#Find duplicate files in directory tree.

import sys
import os
import hashlib

def chunk_reader(fobj, chunk_size=1024):
    while True:
        chunk = fobj.read(chunk_size)
        if not chunk:
            return
        yield chunk

def check_for_duplicates(paths, hash=hashlib.sha1):
    hashes = {}
    for path in paths:
        for dirpath, dirnames, filenames in os.walk(path):
            for filename in filenames:
                full_path = os.path.join(dirpath, filename)
                hashobj = hash()
                try:
                    for chunk in chunk_reader(open(full_path, 'rb')):
                        hashobj.update(chunk)
                    file_id = (hashobj.digest(), os.path.getsize(full_path))
                    duplicate = hashes.get(file_id, None)
                    with open('Duplog.txt', 'a') as f:
                        if duplicate:
                            f.write(full_path + ' <-AND-> ' + duplicate + '\n')
                            print("Duplicate found: %s <-AND-> %s" % (full_path, duplicate))
                        else:
                            hashes[file_id] = full_path
                except (OSError,):
                    continue

if sys.argv[1:]:
    check_for_duplicates(sys.argv[1:])
else:
    print("Please pass the paths to check as parameters to the script")

% time fileinfo -c scans 1> /dev/null

real	0m2.408s
user	0m1.547s
sys	0m0.861s

% time sha1sum `find scans -type f` 1> /dev/null

real	0m18.818s
user	0m17.947s
sys	0m0.872s

% time md5sum `find scans -type f` 1> /dev/null

real	0m15.041s
user	0m14.071s
sys	0m0.970s