2021-02-12 21:57:06 +00:00
#!/usr/bin/env python3
2022-02-06 20:30:09 +00:00
import argparse , os , shutil , time
from itertools import repeat
2022-02-02 14:36:04 +00:00
from pathlib import Path
2022-02-06 20:30:09 +00:00
from concurrent . futures import ThreadPoolExecutor
2022-02-06 23:29:28 +00:00
from tqdm import tqdm
2022-02-17 21:15:00 +00:00
import shutil
from oot import *
2022-02-02 14:36:04 +00:00
2022-02-06 20:30:09 +00:00
def ExtractFile ( xmlPath : Path , outputPath : Path , outputSourcePath : Path , unaccounted : bool ) :
# Create target folder recursivly, ignore error if it already exists
2022-02-17 21:15:00 +00:00
createDir ( outputSourcePath )
zapd = Path ( zapdBinary ( ) )
zapdConfig = Path ( romPath ( " zapd/Config.xml " ) )
baserom = assetPath ( ' baserom ' )
externalXmlFolder = assetPath ( ' xml ' )
execStr = f " { zapd } e -i { xmlPath } -b { baserom } -o { outputPath } -osf { outputSourcePath } -gsf 1 -rconf { zapdConfig } --externalXmlFolder { externalXmlFolder } "
2022-02-02 14:36:04 +00:00
2022-02-06 20:30:09 +00:00
# Use error handler only if non-windows environment, because it's not supported in windows
if os . name != ' nt ' :
execStr + = " -eh "
2021-05-23 00:00:10 +00:00
2022-02-06 20:30:09 +00:00
if " overlays " in str ( xmlPath ) :
2021-10-17 11:32:09 +00:00
execStr + = " --static "
2022-02-06 20:30:09 +00:00
if unaccounted == True :
execStr + = " --warn-unaccounted "
2021-10-17 11:32:09 +00:00
2022-02-06 20:30:09 +00:00
# print(f"Extracting {xmlPath.stem}")
# print(execStr)
2020-03-17 04:31:30 +00:00
2021-05-23 00:00:10 +00:00
exitValue = os . system ( execStr )
2022-02-17 21:15:00 +00:00
2021-05-23 00:00:10 +00:00
if exitValue != 0 :
2022-02-06 20:30:09 +00:00
return ( " failed " , f " Error extracting { xmlPath } : { os . sys . stderr } " )
else :
return ( " success " , f " Successfully extracted { xmlPath } " )
def ExtractFunc ( fullPath : Path , force : bool , unaccounted : bool ) :
# Ensure that file exists. It can happen that files don't exist if paths are manually passed
if fullPath . exists ( ) == False :
return ( " failed " , f " File cannot be found: { fullPath } " )
# Remove xml directory and suffix in path to use as output directory
# Example: assets/xml/objects/object_example.xml -> assets/objects/object_bdoor
2022-02-17 21:15:00 +00:00
rel = relPath ( fullPath , romPath ( ) ) [ 4 : - 4 ]
outPath = Path ( assetPath ( rel ) )
2022-02-06 20:30:09 +00:00
# If output is more recent than source file: skip extraction
# Don't skip if force == True was passed
if force == False and outPath . is_dir ( ) and outPath . stat ( ) . st_mtime > fullPath . stat ( ) . st_mtime :
return ( " skipped " , f " Skipped extraction of { fullPath } " )
# Delete target folder before extraction, ignore errors in case there is no folder to delete
shutil . rmtree ( outPath , ignore_errors = True )
return ExtractFile ( fullPath , outPath , outPath , unaccounted )
def ExtractText ( force : bool ) :
2022-02-17 21:15:00 +00:00
extract_text_path = Path ( assetPath ( " text/message_data.h " ) )
extract_staff_text_path = Path ( assetPath ( " text/message_data_staff.h " ) )
2022-02-06 20:30:09 +00:00
# Ensure target folder exists
2022-02-17 21:15:00 +00:00
createDir ( extract_text_path . parent )
2022-02-06 20:30:09 +00:00
# Always extract if force flag was passed
# otherwise check if target already exists and skip (by setting it to None) if it does
if force == False :
if extract_text_path . exists ( ) :
extract_text_path = None
if extract_staff_text_path . exists ( ) :
extract_staff_text_path = None
if extract_text_path is not None or extract_staff_text_path is not None :
print ( " Start Extracting text " )
2022-02-18 05:09:37 +00:00
import msgdis
2022-02-06 20:30:09 +00:00
msgdis . extract_all_text ( extract_text_path , extract_staff_text_path )
print ( " Finished extracting text " )
2022-02-06 21:48:52 +00:00
def ExtractXMLAssets ( xmlFiles , force : bool , unaccounted : bool ) :
thread_count = os . cpu_count ( ) or 1
print ( f " Start extracting { len ( xmlFiles ) } assets with %d threads " % thread_count )
2022-02-06 20:30:09 +00:00
success = skipped = failed = 0
errors = [ ]
# Multithreading instead of multiprocessing, because of IO heavy operation
2022-02-06 21:48:52 +00:00
with ThreadPoolExecutor ( max_workers = thread_count ) as executor :
2022-02-06 23:29:28 +00:00
with tqdm ( total = len ( xmlFiles ) ) as progress :
for result in executor . map ( ExtractFunc , xmlFiles , repeat ( force ) , repeat ( unaccounted ) ) :
progress . update ( )
# Parsing of results
status = result [ 0 ]
message = result [ 1 ]
if status == " success " :
success + = 1
elif status == " skipped " :
skipped + = 1
else :
failed + = 1
errors . append ( message )
2022-02-06 20:30:09 +00:00
print ( f " Extracted: { success } , Skipped: { skipped } , Failed: { failed } " )
if errors :
print ( f " Errors: { errors } " )
2020-05-26 16:53:53 +00:00
2020-12-28 23:37:52 +00:00
def main ( ) :
2022-02-06 20:30:09 +00:00
# Command Line Interface
2021-02-12 21:57:06 +00:00
parser = argparse . ArgumentParser ( description = " baserom asset extractor " )
2022-02-06 20:30:09 +00:00
parser . add_argument ( " assets " , help = " asset path(s) relative to assets/xml/, e.g. objects/gameplay_keep. Passing nothing will extract entire assets/xml/ tree " , nargs = " * " , default = None )
parser . add_argument ( " -f " , " --force " , help = " Force the extraction of every asset instead of only recently modified " , action = " store_true " , default = False )
parser . add_argument ( " -u " , " --unaccounted " , help = " Enables ZAPD unaccounted detector warning system. " , action = " store_true " , default = False )
2021-02-12 21:57:06 +00:00
args = parser . parse_args ( )
2022-02-17 21:15:00 +00:00
shutil . rmtree ( Path ( assetPath ( ' xml ' ) ) , ignore_errors = True )
2022-03-24 00:46:26 +00:00
shutil . copytree ( Path ( romPath ( ' xml ' ) ) , Path ( assetPath ( ' xml ' ) ) )
2022-02-17 21:15:00 +00:00
2022-02-06 20:30:09 +00:00
# List of xml files for extraction
xmlFiles = None
2022-02-17 21:15:00 +00:00
if args . assets and len ( args . assets ) > 1 :
2022-02-06 20:30:09 +00:00
# Generate list of Paths by transforming asset inputs into full Path structure
# Example: objects/gameplay_keep --> assets/xml/objects/gameplay_keep.xml
2022-02-17 21:15:00 +00:00
xmlFiles = [ Path ( assetPath ( ' xml ' ) ) . joinpath ( file ) . with_suffix ( ' .xml ' ) for file in args . assets [ 1 : ] ]
2021-02-12 21:57:06 +00:00
else :
2022-02-06 20:30:09 +00:00
# Get list of all xml files in assets/xml/ subdirectories recursivly
2022-02-17 21:15:00 +00:00
xmlFiles = sorted ( Path ( romPath ( ) ) . glob ( ' xml/**/*.xml ' ) )
2022-02-06 20:30:09 +00:00
start = time . perf_counter ( )
ExtractText ( args . force )
ExtractXMLAssets ( xmlFiles , args . force , args . unaccounted )
finish = time . perf_counter ( )
print ( f " Finished extracting assets in { round ( finish - start , 2 ) } seconds " )
2020-12-28 23:37:52 +00:00
if __name__ == " __main__ " :
z_message_PAL, message_data_static and surrounding doc (#996)
* Initial progress on z_message_PAL, very messy
* Fix merge
* Some more progress
* Fix merge
* More z_message_PAL
* Small progress
* More small progress
* message_data_static files OK
* Prepare z_message_tables
* Matched another function, small updates
* Attempt to use asm-processor static-symbols branch
* Refactor text id declarations
* Begin large text codes parser function
* Fix merge
* Refactor done
* Build OK, add color and highscore names
* Remove encoded text headers and automatically encode during build
* Fix kanfont
* Various cleanups
* DISP macros
* Another match aside data
* Further progress
* Small improvements
* Deduplicate magic values for text control codes, small improvements
* Tiny progress
* Minor cleanups
* Clean up z_message_PAL comment
* Progress on large functions
* Further progress on large functions
* Changes to mkldscript to link .data in the .rodata section
* data OK
* Few improvements
* Use gDPLoadTextureBlock macros where appropriate
* rm z_message_tables, progress on large functions
* 2 more matches
* Improvements
* Small progress
* More progress on big function
* progress
* match func_80107980
* match Message_Update
* match func_8010BED8
* done
* Progress on remaining large functions
* Small progress on largest function
* Another match, extract text and move to assets, improve text build system
* Small nonmatchings improvements
* docs wip
* Largest function maybe equivalent
* Fix merge
* Document do_action values, largest function is almost instruction-matching
* Rename NAVI do_action to NONE, as that appears to be how that value is used in practice
* Fix merge
* one match
* Last function is instruction-matching
* Fix
* Improvements thanks to engineer124
* Stack matched thanks to petrie911, now just a/v/low t regalloc issues, some cleanup
* More variables labeled, use text state enum everywhere
* More labels and names
* Fix
* Actor_IsTalking -> Actor_TalkRequested
* Match func_8010C39C and remove unused asm
* More docs
* Mostly ocarina related docs
* All msgModes named
* Fix assetclean
* Cleanup
* Extraction fixes and headers
* Suggestions
* Review suggestions
* Change text extraction again, only extract if the headers do not already exist
* Fix
* Use ast for charmap, fix assetclean for real this time
* Review suggestions
* BGM ids and ran formatter
* Review comments
* rename include_readonly to include_data_with_rodata
* Remove leading 0s in number directives
* Review suggestions for message_data_static
* textbox pos enum comments, rename several enum names from Message to TextBox
Co-authored-by: Thar0 <maximilianc64@gmail.com>
Co-authored-by: Zelllll <56516451+Zelllll@users.noreply.github.com>
Co-authored-by: petrie911 <pmontag@DESKTOP-LG8A167.localdomain>
Co-authored-by: Roman971 <romanlasnier@hotmail.com>
2021-11-23 01:20:30 +00:00
main ( )