Merge pull request #40 from shawngmc/develop

v0.1.4

.vscode/launch.json

@@ -72,6 +72,25 @@
                 "\"${workspaceFolder}\\out\\sf30ac\""
             ]
         },
+        {
+            "name": "Python: Toolbox - Testing SF30AC (Verbose)",
+            "type": "python",
+            "request": "launch",
+            "program": "toolbox.py",
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}/src/gex",
+            "justMyCode": true,
+            "args": [
+                "tasks", 
+                "extract", 
+                "--task", 
+                "sf30ac", 
+                "--destdir", 
+                "\"${workspaceFolder}\\out\\sf30ac\"",
+                "-v",
+                "DEBUG"
+            ]
+        },
         {
             "name": "Python: Toolbox - Profile SF30AC",
             "type": "python",
@@ -127,9 +146,7 @@
                 "--destdir", 
                 "\"${workspaceFolder}\\out\\snk40\"",
                 "--srcdir", 
-                "\"C:\\Program Files (x86)\\Amazon Games\\Library\\SNK 40th Anniversary Collection\"",
-                "--prop",
-                "include-partials=True"
+                "\"C:\\Program Files (x86)\\Amazon Games\\Library\\SNK 40th Anniversary Collection\""
             ]
         },
         {
@@ -308,7 +325,7 @@
             ]
         },
         {
-            "name": "Python: Toolbox - Testing Disney Classics Aladding and Lion King w/ DLC",
+            "name": "Python: Toolbox - Testing Disney Classics Aladdin and Lion King w/ DLC",
             "type": "python",
             "request": "launch",
             "program": "toolbox.py",
@@ -377,6 +394,74 @@
                 "include-arcade-partials=True"
             ]
         },
+        {
+            "name": "Python: Toolbox - Testing AGS Galaga",
+            "type": "python",
+            "request": "launch",
+            "program": "toolbox.py",
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}/src/gex",
+            "justMyCode": true,
+            "args": [
+                "tasks", 
+                "extract", 
+                "--task", 
+                "ags_galaga", 
+                "--destdir", 
+                "\"${workspaceFolder}\\out\\ags_galaga\""
+            ]
+        },
+        {
+            "name": "Python: Toolbox - Testing AGS Dig Dug",
+            "type": "python",
+            "request": "launch",
+            "program": "toolbox.py",
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}/src/gex",
+            "justMyCode": true,
+            "args": [
+                "tasks", 
+                "extract", 
+                "--task", 
+                "ags_digdug", 
+                "--destdir", 
+                "\"${workspaceFolder}\\out\\ags_digdug\""
+            ]
+        },
+        {
+            "name": "Python: Toolbox - Testing AGS Pac-Man",
+            "type": "python",
+            "request": "launch",
+            "program": "toolbox.py",
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}/src/gex",
+            "justMyCode": true,
+            "args": [
+                "tasks", 
+                "extract", 
+                "--task", 
+                "ags_pacman", 
+                "--destdir", 
+                "\"${workspaceFolder}\\out\\ags_pacman\""
+            ]
+        },
+        {
+            "name": "Python: Toolbox - Testing AGS Ms. Pac-Man",
+            "type": "python",
+            "request": "launch",
+            "program": "toolbox.py",
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}/src/gex",
+            "justMyCode": true,
+            "args": [
+                "tasks", 
+                "extract", 
+                "--task", 
+                "ags_mspacman", 
+                "--destdir", 
+                "\"${workspaceFolder}\\out\\ags_mspacman\""
+            ]
+        },
         {
             "name": "Python: Toolbox - Testing AGS",
             "type": "python",
@@ -484,9 +569,7 @@
                 "--destdir", 
                 "\"${workspaceFolder}\\out\\cas1\"",
                 "--prop",
-                "include-partials=True",
-                "--verbosity",
-                "DEBUG"
+                "include-partials=True"
             ]
         },
         {
@@ -526,6 +609,19 @@
                 "--prop",
                 "include-partials=True"
             ]
+        },
+        {
+            "name": "Python: Toolbox - List Tasks",
+            "type": "python",
+            "request": "launch",
+            "program": "toolbox.py",
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}/src/gex",
+            "justMyCode": true,
+            "args": [
+                "tasks", 
+                "list"
+            ]
         }
     ]
 }

CHANGELOG.md

@@ -1,3 +1,16 @@
+# 0.1.4
+- Updated Collections:
+  - Genesis and Mega Drive Collection
+    - Fixed mislabeled Super Thunder Blade ROM
+  - IREM Arcade Classics
+    - Fix bug preventing details output
+  - SNK 40th Anniversary Edition
+    - Bermuda Triangle: Completed extraction on actual Bermuda Triangle (#37)
+- Improvements:
+  - Added input/output file verification to tasks (#36)
+- Internal:
+  - Made a few reusable tasks - 'CopyTask', 'ZipSpliceTask', 'SpliceTask'
+
 # 0.1.3
 - Improvements:
   - Fixed forgotten version number roll

README.md

@@ -73,6 +73,10 @@ I prefer not to be artificially locked to specific platforms - why play a 25 yea
 ### Why not just download the ROMs?
 It is illegal. One can discuss economic moral implications all day, but at the end of the day, it's theft. As a software developer myself, I don't want my code stolen - so I can't claim a moral high ground there. Equally importantly, showing classic game owners that the market will support rereleases increases the liklihood of rereleases, and I want people who can't/won't download ROMs to enjoy these titles as well.
 
+### What artificial limitations does this tool have to prevent copyright claims?
+* This tool will not hold decryption keys.
+* This tool will not hold any 'whole' files, nor substantial parts of files. A limited 'patch' capability is allowed for less than 1% of a file, but this should be very limited.
+
 ### This MAME ROM is actually XXXXYY, not XXXXZZ - why do you have it that way?
 The goal of this tool is not to create a perfect ROM set for the latest MAME, because it can't. A few factors go into what version of MAME we target:
 
@@ -108,30 +112,33 @@ These are noted in the documentation for each script. If you think a ROM is misi
 
 ### Playable
 
-**Collection**                             | **Status**     | **Notes**           
+**Collection**                             | **Status**     | **Verification** | **Notes**           
 --------------------------------------------|-------------|---------------------------------------------------------------------  
- **Arcade Collection Anniversary Classics**     | 75%    | A couple games are good extractions so far...
- **Atari Vault**                                | 90%    | Some arcade ROMs from this collection are incomplete.  
- **Blizzard Arcade Collection**                 | 100%   |   
- **Bubsy Two-Fur**                              | 100%   |   
- **Capcom Arcade Stadium 1 (via Depot)**        | 95%    | Requires Steam depot downloading, a couple shaky ROMs... 
- **Capcom Beat 'Em Up Bundle**                  | 95%    | 6/7 playable on some version of MAME, but wof/wofj missing audiocpu data  
- **Capcom Fighting Collection**                 | 90%    | CPS2 is semi-standard. No Enc keys present. CP3 game is a curveball!  
- **Collection of SaGa/Final Fantasy Legend**    | 100%   |  
- **Disney Afternoon Collection**                | 100%   |  
- **Disney Classics Aladdin & Lion King w/DLC**  | 100%   | Includes Jungle Book DLC
- **Double Dragon Trilogy**                      | 100%   |  
- **IREM Arcade Classics**                       | 100%   |  
- **Mega Man Legacy Collection 1**               | 100%   |  
- **Mega Man X Legacy Collection 1**             | 75%    | X4 doesn't appear to be ROM based  
- **Namco Arcade Game Series**                   | 75%    | Ms. Pac-Man cannot be cleanly extracted
- **Pac Man Museum Plus**                        | 40%    | Some progress, but there are a lot of non-extractable titles.
- **Sega Genesis and Mega Drive Collection**     | 90%    | Some compressed variants not yet extracted  
- **Sonic Adventure DX (Hidden Game Gear games)**| 100%   | This is only the Game Gear games - SADX itself can not be made into a ROM/ISO!
- **SNK 40th Anniversary Collection**            | 100%   | All games supported by an emulator are extracted!
- **Street Fighter 30th Anniversary Collection** | 90%    | Now includes all playable international versions.
- **Wonder Boy: The Dragon's Trap**              | 100%   |  
- **Zombies Ate My Neighbors and Ghoul Patrol**  | 100%   |  
+ **Arcade Collection Anniversary Classics**     | 75%    | Y | A couple games are good extractions so far...
+ **Atari Vault**                                | 90%    | Y | Some arcade ROMs from this collection are incomplete.  
+ **Blizzard Arcade Collection**                 | 100%   | Y |   
+ **Bubsy Two-Fur**                              | 100%   | Y |   
+ **Capcom Arcade Stadium 1 (via Depot)**        | 95%    | N | Requires Steam depot downloading, a couple shaky ROMs... 
+ **Capcom Beat 'Em Up Bundle**                  | 95%    | Y | 6/7 playable on some version of MAME, but wof/wofj missing audiocpu data  
+ **Capcom Fighting Collection**                 | 90%    | Y | CPS2 is semi-standard. No Enc keys present. CP3 game is a curveball!  
+ **Collection of SaGa/Final Fantasy Legend**    | 100%   | Y |  
+ **Disney Afternoon Collection**                | 100%   | Y |  
+ **Disney Classics Aladdin & Lion King w/DLC**  | 100%   | Y | Includes Jungle Book DLC
+ **Double Dragon Trilogy**                      | 100%   | Y |  
+ **IREM Arcade Classics**                       | 100%   | Y |  
+ **Mega Man Legacy Collection 1**               | 100%   | Y |  
+ **Mega Man X Legacy Collection 1**             | 75%    | Y | X4 doesn't appear to be ROM based  
+ **Namco Arcade Game Series: Dig Dug**          | 100%   | Y |   
+ **Namco Arcade Game Series: Galaga**           | 100%   | Y |   
+ **Namco Arcade Game Series: Ms. Pac-Man**      | 100%   | Y |   
+ **Namco Arcade Game Series: Pacman**           | 100%   | Y |   
+ **Pac Man Museum Plus**                        | 40%    | Y | Some progress, but there are a lot of non-extractable titles.
+ **Sega Genesis and Mega Drive Collection**     | 90%    | Y | Some compressed variants not yet extracted  
+ **Sonic Adventure DX (Hidden Game Gear games)**| 100%   | Y | This is only the Game Gear games - SADX itself can not be made into a ROM/ISO!
+ **SNK 40th Anniversary Collection**            | 100%   | Y | All games supported by an emulator are extracted!
+ **Street Fighter 30th Anniversary Collection** | 90%    | Y | Now includes all playable international versions.
+ **Wonder Boy: The Dragon's Trap**              | 100%   | Y |  
+ **Zombies Ate My Neighbors and Ghoul Patrol**  | 100%   | Y |  
 
 
 ### Completely Unplayable
@@ -155,4 +162,4 @@ These are noted in the documentation for each script. If you think a ROM is misi
  * The ROM has missing files that can be filled with placeholders or left out.
  * Files are present that work as normal, but have bad CRCs. This may be due to copyright changes, etc., but ultimately does not match a known good source.
  * Publishers may offer modified ROMs - such as increased difficulty, theming, or new game modes - which are often not publicly tracked.
- * It may only work in older versions of emulators due to odd dumps, missing files, etc.
+ * It may only work in older versions of emulators due to odd dumps, missing files, etc.

ROADMAP.md

@@ -4,6 +4,10 @@
 - Retro Classix by Data East
   In title\title_Data\StreamingAssets folder
 - Brave Battle Saga - The Legend of The Magic Warrior
+- Activision Anthology Remix PC
+  https://forums.atariage.com/topic/336118-how-to-extract-roms-from-the-activision-anthology-for-pc-and-a-request-for-info-on-the-included-roms/
+- Sega Mega Drive & Genesis Classics
+  Compressed ROMS: https://github.com/farmerbb/RED-Project/wiki/Sega-Mega-Drive-&-Genesis-Classics
 
 ### Uses BPList/Mbundle
 - Samuari Shodown Collection
@@ -30,6 +34,7 @@
 ## Owned / Under Investigation
 - Teenage Mutant Ninja Turtles: The Cowabunga Collection (August 30th, 2022)
   Assets are jammed into one file, but that file appears to be encrypted or compressed. Notes in #17.
+  https://github.com/Masquerade64/Cowabunga
 - Capcom Arcade Stadium 1
   Current versions have an odd archive format (unlike the CAS1_Old ZIP version).
   PLACEHOLDER ADDED. See #18.
@@ -42,6 +47,11 @@
 - Arcade Collection Anniversary Classics
   Mostly working, but 2 titles remain...
 - Psiyko Shooter Collector's Bundle https://store.steampowered.com/bundle/18805/PSIKYO_SHOOTER_Collectors_Bundle/
+- PSP: EA Replay
+  \PSP_GAME\USRDIR\data\roms.viv is an EA BIG4 archive that can be extracted via QuickBMS
+  It has sensible ROM filenames, but the files are too small and not in an expected format.
+  They don't appear to have REFPACK, zlib or gzip headers, nor use RAW DEFLATE.
+
 
 
 ## Not Yet Owned / Future Investigation
@@ -60,35 +70,29 @@
 - CAVE: Deathsmiles I + II
   Recommended in #15.
 - Darius Cozmic Collection Arcade
+- Taito Legends 1 and 2
+- Saturn Cotton Collection
+  https://gbatemp.net/threads/saturn-emulation-using-cotton-guardian-force-testing-and-debug.600756/
 
 ## Upcoming Releases
 - Atari 50th: Anniversary Collection (Winter 2022)
 
+## Unlikely
+- Williams Arcade Classics (PC CD-ROM, 1995)
+  Initial research shows that this is likely not ROM based.
+
 ## Not Complete ROMs (may add partial extraction later)
 - Mortal Kombat Kollection
   Audio ROMs were replaced with a different audio solution.
 - Dungeons & Dragons: Chronicles of Mystara
   Maincpu and Gfx ROMs are there, but audio is completely reworked.
-- Midway Arcade Treasures Vol. 1 (PS2)
-  Audio ROMs replaced. For example, Smash TV has all the game ROMs, but none of the sound ROMs.
-  Klax
-    - Missing OKI sound roms
-    - Missing Pals
-    - pfrom.klx
-      split
-      0x00000 0x1000 136075-2010.17x
-      0x10000 0x1000 136075-2012.12x
-      0x20000 0x1000 136075-2014.17y
-      0x30000 0x1000 136075-2009.17u
-      0x40000 0x1000 136075-2011.12u
-      0x50000 0x1000 136075-2013.17w
-    - fixedcpu.klx
-      cut in half
-      deinterleave each, 1 byte
-      136075-6006.3k
-      136075-6005.1k
-      136075-6008.3k
-      136075-6007.1k
+
+### .SR Releases
+The SR archives can be extracted via QuickBMS script: https://forum.xentax.com/viewtopic.php?t=13718
+However, these seem to always use rebuilt audio, and some of the games have been rebuilt as native ELF executables.
+- PSP: Capcom Classics Collection: Remixed
+- PS2: Midway Arcade Treasures Vol. 1
+- PS2: Midway Arcade Treasures Vol. 2
 
 ## Not ROMs
 - Phoenix Wright Ace Attorney Trilogy
@@ -99,4 +103,7 @@
   Windows Version
 - Earthworm Jim 1 and 2 (Steam, GOG)
   DOS Version
+- Disney's Hercules
+  https://www.gog.com/en/game/disneys_hercules
+  Native PC Game
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "game-extraction-toolbox"
-version = "0.1.3"
+version = "0.1.4"
 authors = [
   { name="Shawn McNaughton", email="[email protected]" },
 ]

setup.cfg

@@ -8,4 +8,8 @@ package_dir =
 packages = find_namespace:
 
 [options.packages.find]
-where = src
+where = src
+
+[options.package_data]
+* =
+    *.json

src/gex/commands/archive/archive.py

@@ -2,10 +2,12 @@
 
 from .arc.arc import arc
 from .kpka.kpka import kpka
+from .zip.zip import zip_cli
 
 @click.group()
 def archive():
     """Tools for working with various types of archives"""
 
 archive.add_command(arc)
 archive.add_command(kpka)
+archive.add_command(zip_cli)