Python script to clean clones out of a MAME romset

dugan · 02-16-2024, 08:51 PM

Here's a Python script to remove clones from a MAME romset. Not always advisable, but if you want to, here's how you do it. Personally, my use case was to build a split FBNEO romset with clrmamepro, remove clones from that, and then build a RetroArch playlist.

It takes two parameters. The first is the output of "mame -listxml", redirected to a file. The second is the directory of ROMs that you want to clean. The output is a list of files to delete (which you for-loop or xargs over).

Code:

#!/usr/bin/env python3


import argparse
from lxml import etree
import pathlib

def main():
    parser = argparse.ArgumentParser("romclean", description="Cleans an arcade romset")
    parser.add_argument("dat")
    parser.add_argument("romdir")
    args = parser.parse_args()

    killer = set()

    with open(args.dat) as f:
        mame = etree.parse(args.dat).getroot()
        for machine in mame:
            if machine.get("cloneof") is not None:
                continue
            if machine.get("ismechanical") == "yes":
                continue
            if machine.get("isbios") == "yes":
                continue
            if not machine.get("name"):
                continue
            killer.add(machine.get("name"))

    for rom in (x for x in pathlib.Path(args.romdir).iterdir() if x.is_file()):
        if rom.stem not in killer:
            print(rom.name)

if __name__ == '__main__':
    main()

dugan · 05-09-2024, 07:47 PM

Too late to edit this, but I just noticed this mistake:

Code:

    with open(args.dat) as f:
        mame = etree.parse(args.dat).getroot()

dugan · 08-16-2024, 05:03 PM

With the mistake fixed:

Code:

#!/usr/bin/env python3


import argparse
from lxml import etree
import pathlib

def main():
    parser = argparse.ArgumentParser("romclean", description="Cleans an arcade romset")
    parser.add_argument("dat")
    parser.add_argument("romdir")
    args = parser.parse_args()

    killer = set()

    with open(args.dat) as f:
        mame = etree.parse(f).getroot()
        for machine in mame:
            if machine.get("cloneof") is not None:
                continue
            if machine.get("ismechanical") == "yes":
                continue
            if machine.get("isbios") == "yes":
                continue
            if not machine.get("name"):
                continue
            killer.add(machine.get("name"))

    for rom in (x for x in pathlib.Path(args.romdir).iterdir() if x.is_file()):
        if rom.stem not in killer:
            print(rom.name)

if __name__ == '__main__':
    main()

dugan · 09-05-2024, 05:00 PM

Might as well just update this thread, because it's so related. It's a script to list ROMs that EmulationStation-DE didn't scrape. Run it in the ROM directory; the argument is the path to the ROM directory's gamelist.xml file.

For example, this is my actual command-line:

Code:

/Emulation/ES-DE/ROMs/gba ❯ ~/Documents/list_esde_unscraped.py ~/Emulation/ES-DE/gamelists/gba/gamelist.xml

Here it is:

Code:

#!/usr/bin/env python3


import argparse
from lxml import etree
import os
import fnmatch
import pathlib

def main():
    parser = argparse.ArgumentParser('list_esde_unscraped', description="Lists ROMs that didn't scrape. Run it in the ROM directory")
    parser.add_argument("gamelist", help="Path to the ROM directory's gamelist.xml")
    args = parser.parse_args()

    scraped = set()

    with open(args.gamelist) as f:
         for element in etree.parse(f).getroot().iter():
             if element.tag == 'path':
                scraped.add(element.text)

    for rom in pathlib.Path('.').iterdir():

        # My ROM directory is intentionally flat.
        if not rom.is_file():
            continue

        if rom.name == "systeminfo.txt":
            continue

        # I expect the directory to be sprinkled with soft patches
        if fnmatch.fnmatch(rom.name, '*.ips*'):
            continue
        
        if os.path.join(".", rom) not in scraped:
            print(rom)

if __name__ == '__main__':
    main()

And then, because I'm using FISH, I can automatically move these unscraped ROMs to another directory. This is also an actual command-line:

Code:

❯ for x in (~/Documents/list_esde_unscraped.py ~/Emulation/ES-DE/gamelists/gba/gamelist.xml)
      mv -nv $x ~/Emulation/roms_misc/gba/
  end

dugan · 09-08-2024, 01:55 PM

Next script. I want my EmulationStation-DE to have a 1G1R collection, so here's a script to detect duplicates.

Code:

#!/usr/bin/env python3


import argparse
from lxml import etree
import collections

def main():
    parser = argparse.ArgumentParser('list_esde_dupes', description="Lists duplicate games in an ES-DE gamelist")
    parser.add_argument("gamelist", help="Path to an ES-DE gamelist.xml")
    args = parser.parse_args()

    names = collections.defaultdict(list)

    with open(args.gamelist) as f:
        for game in etree.parse(f).getroot():
            names[game.find('name').text].append(game)
    
    for name, elements in names.items():
        if len(elements) > 1:
            print(name)
            for element in elements:
                path = element.find('path').text
                print(f"\t{path}")

if __name__ == '__main__':
    main()

Actual run:

Code:

❯ ./list_esde_dupes.py ~/Emulation/ES-DE/gamelists/gba/gamelist.xml
Advance Wars
        ./Advance Wars (USA).gba
        ./Advance Wars (USA) (Rev 1).gba
Pokémon LeafGreen Version
        ./Pokemon - LeafGreen Version (USA, Europe).gba
        ./Pokemon - LeafGreen Version (USA, Europe) (Rev 1).gba
Spyro Orange: The Cortex Conspiracy
        ./Spyro Orange - The Cortex Conspiracy (USA).gba
        ./Spyro Orange - The Cortex Conspiracy (USA) (Rev 1).gba
Super Dodge Ball Advance
        ./Super Dodge Ball Advance (USA).gba
        ./Super Dodge Ball Advance (USA) (Rev 1).gba
Super Mario Advance 4 : Super Mario Bros. 3
        ./Super Mario Advance 4 - Super Mario Bros. 3 (USA) (Rev 1) (Virtual Console).gba
        ./Super Mario Advance 4 - Super Mario Bros. 3 (USA, Australia) (Rev 1).gba
Super Puzzle Fighter II
        ./Super Puzzle Fighter II (USA) (Rev 1).gba
        ./Super Puzzle Fighter II Turbo (USA).gba

You run ES-DE's scraper once to generate the gamelist.xml file, clean the directory with the help of these two scripts, and then you run ES-DE's "Utilities" (from its menus) to clean out orphaned data. Makes it so much easier to get a fully-scraped 1G1R (one game one ROM, for those who don't know) collection.

Note: both of these scripts will fail if you've set an "Alternative Emulator". That causes Python's lxml library to see the gamelist.xml file as invalid (because it doesn't have a root node). I'm just going to document that and leave it.

dugan · 09-23-2024, 07:57 PM

Updated list_esde_unscraped.py to account for MSU1.

Code:

#!/usr/bin/env python3


import argparse
from lxml import etree
import os
import fnmatch
import pathlib

def main():
    parser = argparse.ArgumentParser('list_esde_unscraped', description="Lists ROMs that didn't scrape. Run it in the ROM directory")
    parser.add_argument("gamelist", help="Path to the ROM directory's gamelist.xml")
    args = parser.parse_args()

    scraped = set()

    with open(args.gamelist) as f:
         for element in etree.parse(f).getroot().iter():
             if element.tag == 'path':
                scraped.add(element.text)

    for rom in pathlib.Path('.').iterdir():

        # My ROM directory is intentionally flat.
        if not rom.is_file():
            continue

        if rom.name == "systeminfo.txt":
            continue

        # I expect the directory to be sprinkled with soft patches
        if fnmatch.fnmatch(rom.name, '*.ips*'):
            continue

        # I haven't used msu1, but let's account for it.

        if fnmatch.fnmatch(rom.name, '*.pcm'):
            continue

        if fnmatch.fnmatch(rom.name, '*.msu'):
            continue

        if os.path.join(".", rom) not in scraped:
            print(rom)

if __name__ == '__main__':
    main()

dugan · 11-24-2024, 02:45 PM

Updating list_esde_unscraped.py to support other patch formats:

Code:

#!/usr/bin/env python3


import argparse
from lxml import etree
import os
import fnmatch
import pathlib

def main():
    parser = argparse.ArgumentParser('list_esde_unscraped', description="Lists ROMs that didn't scrape. Run it in the ROM directory")
    parser.add_argument("gamelist", help="Path to the ROM directory's gamelist.xml")
    args = parser.parse_args()

    scraped = set()

    with open(args.gamelist) as f:
         for element in etree.parse(f).getroot().iter():
             if element.tag == 'path':
                scraped.add(element.text)

    for rom in pathlib.Path('.').iterdir():

        # My ROM directory is intentionally flat.
        if not rom.is_file():
            continue

        if rom.name == "systeminfo.txt":
            continue

        # I expect the directory to be sprinkled with soft patches

        if fnmatch.fnmatch(rom.name, '*.ips*'):
            continue

        if fnmatch.fnmatch(rom.name, '*.bps*'):
            continue

        if fnmatch.fnmatch(rom.name, '*.ups*'):
            continue

        # I haven't used msu1, but let's account for it.

        if fnmatch.fnmatch(rom.name, '*.pcm'):
            continue

        if fnmatch.fnmatch(rom.name, '*.msu'):
            continue

        if os.path.join(".", rom) not in scraped:
            print(rom)

if __name__ == '__main__':
    main()

dugan · 12-08-2024, 03:07 PM

Just adding this to my ROM-management scripts.

The MiSTer expects CD-ROM games to have a different directory structure. This time I didn't bother with a script: I just did the whole thing from an interactive FISH shell. A test, where the only thing that needs to change is the value of $root:

Code:

❯❯ tree -a
.
├── Bonk III - Bonk's Big Adventure.chd
├── Einhaender (USA).chd
├── Lunar - Silver Star Story Complete (USA) [Un-Worked Design by Supper v0 (09-04-2017)].m3u
└── .multidisc
    ├── Lunar - Silver Star Story Complete (USA) (Disc 1) [Un-Worked Design by Supper v0 (09-04-2017)].chd
    └── Lunar - Silver Star Story Complete (USA) (Disc 2) [Un-Worked Design by Supper v0 (09-04-2017)].chd

2 directories, 5 files
❯ set root ../misterporttteest
❯ for game in *
      set gamename (basename $game)
      set gamename (basename $gamename .chd)
      set gamename (basename $gamename .m3u)
      set gamename (echo $gamename | awk -F'(' '{print $1}')
      set gamename $(echo $gamename | sed 's/^[   ]*//;s/[    ]*$//')
      set dst $root/{$gamename}
      mkdir -p $dst
      if [ (basename $game .m3u) != $game ]
          for chd in (cat $game)
              cp -Lnv $chd $dst;
          end
      else
          cp -Lnv $game $dst
      end
  end
"Bonk III - Bonk's Big Adventure.chd" -> "../misterporttteest/Bonk III - Bonk's Big Adventure/Bonk III - Bonk's Big Adventure.chd"
'Einhaender (USA).chd' -> '../misterporttteest/Einhaender/Einhaender (USA).chd'
'./.multidisc/Lunar - Silver Star Story Complete (USA) (Disc 1) [Un-Worked Design by Supper v0 (09-04-2017)].chd' -> '../misterporttteest/Lunar - Silver Star Story Complete/Lunar - Silver Star Story Complete (USA) (Disc 1) [Un-Worked Design by Supper v0 (09-04-2017)].chd'
'./.multidisc/Lunar - Silver Star Story Complete (USA) (Disc 2) [Un-Worked Design by Supper v0 (09-04-2017)].chd' -> '../misterporttteest/Lunar - Silver Star Story Complete/Lunar - Silver Star Story Complete (USA) (Disc 2) [Un-Worked Design by Supper v0 (09-04-2017)].chd'
❯ tree -a ../misterporttteest/
../misterporttteest/
├── Bonk III - Bonk's Big Adventure
│** └── Bonk III - Bonk's Big Adventure.chd
├── Einhaender
│** └── Einhaender (USA).chd
├── Lunar - Silver Star Story Complete
│** ├── Lunar - Silver Star Story Complete (USA) (Disc 1) [Un-Worked Design by Supper v0 (09-04-2017)].chd
│** └── Lunar - Silver Star Story Complete (USA) (Disc 2) [Un-Worked Design by Supper v0 (09-04-2017)].chd
└── .multidisc
    ├── Lunar - Silver Star Story Complete (USA) (Disc 1) [Un-Worked Design by Supper v0 (09-04-2017)].chd
    └── Lunar - Silver Star Story Complete (USA) (Disc 2) [Un-Worked Design by Supper v0 (09-04-2017)].chd

5 directories, 6 files
~/Documents/mister_psx ❯

dugan · 02-01-2026, 12:00 PM

list_esde_unscraped.py updated to reject games that didn't fully scrape:

Code:

#!/usr/bin/env python3


import argparse
from lxml import etree
import os
import fnmatch
import pathlib


def main():
    parser = argparse.ArgumentParser(
        "list_esde_unscraped",
        description="Lists ROMs that didn't scrape. Run it in the ROM directory",
    )
    parser.add_argument("gamelist", help="Path to the ROM directory's gamelist.xml")
    args = parser.parse_args()

    scraped = set()

    with open(args.gamelist) as f:
        for game in etree.parse(f).getroot():
            path = None
            desc = None
            for child in game:
                if child.tag == "path":
                    path = child.text
                if child.tag == "desc":
                    desc = child.text
            if path and desc:
                scraped.add(path)

    for rom in pathlib.Path(".").iterdir():

        # My ROM directory is intentionally flat.
        if not rom.is_file():
            continue

        if rom.name == "systeminfo.txt":
            continue

        # I expect the directory to be sprinkled with soft patches
        if fnmatch.fnmatch(rom.name, "*.ips*"):
            continue

        if os.path.join(".", rom) not in scraped:
            print(rom)


if __name__ == "__main__":
    main()

dugan · 02-01-2026, 01:02 PM

list_esde_dupes.py has been updated to handle different games with the same name:

Code:

#!/usr/bin/env python3


import argparse
from lxml import etree
import collections


def main():
    parser = argparse.ArgumentParser(
        "list_esde_dupes", description="Lists duplicate games in an ES-DE gamelist"
    )
    parser.add_argument("gamelist", help="Path to an ES-DE gamelist.xml")
    args = parser.parse_args()

    Game = collections.namedtuple("Game", ["name", "desc"])
    names = collections.defaultdict(list)

    with open(args.gamelist) as f:
        for element in etree.parse(f).getroot():
            game = Game(name=element.find("name").text, desc=element.find("desc").text)
            names[game].append(element.find("path").text)

    for game, paths in names.items():
        if len(paths) > 1:
            print(game.name)
            for path in paths:
                print(f"\t{path}")


if __name__ == "__main__":
    main()