find -not -empty -type f -printf "%s\n" | sort -rn | uniq -d | xargs -I{} -n1 find -type f -size {}c -print0 | xargs -0 md5sum | sort | uniq -w32 --all-repeated=separate

find -not -empty -type f -printf "%s\n" | sort -rn | uniq -d |  xargs -I{} -n1 find -type f -size {}c -print0 | xargs -0 md5sum | sort | uniq -w32 --all-repeated=separate | cut -f3-100 -d ' ' | tr '\n.' '\t.' | sed 's/\t\t/\n/g' | cut -f2-100 | tr '\t' '\n' | perl -i -pe 's/([ (){}-])/\\$1/g' | perl -i -pe 's/'\''/\\'\''/g' | xargs -pr rm -v

find -not -empty -type f -printf "%s\n" | sort -rn | uniq -d | xargs -I{} -n1 find -type f -size {}c -print0 \
| xargs -0 md5sum | sort | uniq -w32 --all-repeated=separate

md5deep -r /some/path|sort|uniq -w32 --all-repeated=separate

#!/usr/bin/env python

from hashes.simhash import simhash
import Levenshtein
from scipy.spatial.distance import pdist
from scipy.cluster.hierarchy import fcluster
from fastcluster import linkage
from pprint import pprint
import sys, glob, os

def make_clusters(cl, files):
    clusters = { i:[] for i in xrange(1, max(cl)+1)}
    for i,n in enumerate(cl):
        clusters[n] += [ files[i] ]
    return [ c for c in clusters.values() if len(c)>1 ]

class File:
    simhash = None
    size = None
    def __init__(self, name):
        self.name = name
    def __repr__(self):
        s = '<' + self.name
        if self.size:
            s += '; sz='+ str(self.size)
        if self.simhash:
            s += '; hash='+ str(self.simhash)
        s += '>'
        return s
    def get_size(self):
        if not self.size:
            self.size = os.stat(self.name).st_size
        return self.size
    def get_simhash(self):
        if not self.simhash:
            s = self.get_str()
            self.simhash = simhash(s)
            del s
        return self.simhash
    def get_str(self):
        # may unzip etc here
        return open(self.name).read()

def cluster_on_size(files, th=1e4):
    if len(files) < 2: return []
    sz = [ [f.get_size()] for f in files]
    dm = pdist( sz, 'minkowski', 1)
    link = linkage(dm, preserve_input=False, method='complete')
    cl = fcluster(link, t=th, criterion='distance')
    return make_clusters(cl, files)

def cluster_on_simhash(files, th=0.2):
    if len(files) < 2: return []

    def sh_dist(i, j):
        h1 = files[int(i)].get_simhash()
        h2 = files[int(j)].get_simhash()
        # dist b/w similar files is small:
        return 1 - h1.similarity(h2)

    sh = [ [i] for i in range( len(files) ) ]
    dm = pdist( sh, sh_dist )

    link = linkage( dm, method='complete')
    cl = fcluster(link, t=th, criterion='distance')
    return make_clusters(cl, files)

def main(argv):
    if len(argv) == 1:
        print "Usage:", argv[0], "glob1 ..."
        return 1
    
    sz_th  = 1e3
    sim_th = 0.2
    ratio_th = 0.8

    files = [ File(fn) for g in argv[1:] for fn in glob.glob(g) if os.path.isfile(fn)]
    print "Processing", len(files), 'files'
    sz_clusters = cluster_on_size(files, sz_th)
    print "{} size-clusters formed (sz_th={} bytes, sim_th={}%, ratio_th={}%)".\
            format(len(sz_clusters), sz_th, sim_th*100, ratio_th*100)

    res = []
    for n, cl in enumerate(sz_clusters):
        print "cluster {} ({} files)".format(n, len(cl) )
        res += cluster_on_simhash(cl, sim_th)

    # sort clusters in ascending order of file sizes
    res = sorted(res, key=lambda c: c[0].size)

    pprint(res)

    print 'Compute pairwise similarity. This may take... eternity'

    for c in res:
        for n,a in enumerate(c):
            for b in c[n+1:]:
                #print 'checking', a.name, b.name
                r = Levenshtein.ratio(a.get_str(), b.get_str())
                if r >= ratio_th:
                    print '{:0.3f} {} {}'.format(r, a.name, b.name)


if __name__ == "__main__":
    sys.exit(main(sys.argv))