I needed this today and since link.box.sk and newdata.box.sk are not online (all those tools link to them) I wrote a simple script. have fun
#!/usr/bin/perl -w
#
# dupes.pl
#
# Removes dupes from wordlists. Each word must use
# its own line.
die "Usage: $0 wordlist" if (@ARGV!=1);
open(OUTPUT, ">duped.txt" );
open(INPUT, $ARGV[0] ) || die "wordlist not found\n";
while(<INPUT>){
push(@words, $_)
}
close(INPUT);
foreach $word( @words ) {
@parts = split( "\n", $word );
$dupe = $parts[0];
unless( $seen{$dupe} ) {
print OUTPUT "$dupe\n";
$seen{$dupe} = 1;
}
}
close(OUTPUT);
