Owr actions update (#19)

* Go for Broke

* Let it fire

* Add PipLine

* Create the dir if it doesn't exist

* Install Setuptools

* Track Test Action's files

* Fix Calling Job

* Track Build Action files

* Install Distutils, rename filenames

* Fix Fail conditions

* Make Build scripts smarter

* Add file

* Concat DLLs lists

* Try to fail if Error DLLs

* Try to make the fail smarter

* Moar verbosity

* Print the stuff first

* Print outputs objects

* See if this skips failure

* Use py instead

* Print error list

* Don't ValueError

* Try checking a different way

* Try something else

* Bleh, spell filename correctly

* Update excluded_dlls.json

* Ugh, gotta compare old to new somehow

* Compare to old list

* Condense build script

* Moar verbosity

* Update the global version

* Update Excluded DLLs list

* Actually use the bad DLLs list

* Make a version number

* Fix version number building

* Fix version number building again

* Fix Diagnostics

* Try REST API stuff

* Try REST API again

* Moar REST

* await

* Get SHA

* Try it all together

* Del test workflow

* Add Perms

* Use a Token

* Try this Token

* Try different Token

* Try different Token

* Create App Version earlier

* See this error again

* Don't fail if App Version not made yet

* Use New Secret

* Print whole response

* Documentation for Tagger

* Update CI Instructions

* Update CI

* List References

* Find latest tag

Fix App Version getter

* Fix commas

* Check returned data

* Update Build Script

* Fix substring

* Fix Git tag

* Fix tag again

* Visual indicators

* Use encoding

* Remove an indicator

* Update CI

* Update Project Name

* PyInstaller Spec Template file

* Update Build Script

* Fix Tagger

* Update CI

* Download AppVersion during build

* Test job can fail

* Upload Logs instead of printing them

* Change from Reusable Workflow to Action

* Change ref to token

* Compare to string

* Use PAT

* Use String literal

* Remove Reusable Workflow

* Update CI Scripts

* Go for Broke

* Let it fire

* Add PipLine

* Create the dir if it doesn't exist

* Install Setuptools

* Track Test Action's files

* Fix Calling Job

* Track Build Action files

* Install Distutils, rename filenames

* Fix Fail conditions

* Make Build scripts smarter

* Add file

* Concat DLLs lists

* Try to fail if Error DLLs

* Try to make the fail smarter

* Moar verbosity

* Print the stuff first

* Print outputs objects

* See if this skips failure

* Use py instead

* Print error list

* Don't ValueError

* Try checking a different way

* Try something else

* Bleh, spell filename correctly

* Update excluded_dlls.json

* Ugh, gotta compare old to new somehow

* Compare to old list

* Condense build script

* Moar verbosity

* Update the global version

* Update Excluded DLLs list

* Actually use the bad DLLs list

* Make a version number

* Fix version number building

* Fix version number building again

* Fix Diagnostics

* Try REST API stuff

* Try REST API again

* Moar REST

* await

* Get SHA

* Try it all together

* Del test workflow

* Add Perms

* Use a Token

* Try this Token

* Try different Token

* Try different Token

* Create App Version earlier

* See this error again

* Don't fail if App Version not made yet

* Use New Secret

* Print whole response

* Documentation for Tagger

* Update CI Instructions

* Update CI

* List References

* Find latest tag

Fix App Version getter

* Fix commas

* Check returned data

* Update Build Script

* Fix substring

* Fix Git tag

* Fix tag again

* Visual indicators

* Use encoding

* Remove an indicator

* Update CI

* Update Project Name

* PyInstaller Spec Template file

* Update Build Script

* Fix Tagger

* Update CI

* Download AppVersion during build

* Test job can fail

* Upload Logs instead of printing them

* Change from Reusable Workflow to Action

* Change ref to token

* Compare to string

* Use PAT

* Use String literal

* Remove Reusable Workflow

* Update CI Scripts

---------

Co-authored-by: Minnie A. Trethewey (Mike) <minnietrethewey@gmail.com>
This commit is contained in:
codemann8
2024-05-23 19:29:39 -05:00
committed by GitHub
parent 9eb5293a3c
commit b55c3700c0
39 changed files with 2076 additions and 588 deletions

View File

@@ -1,68 +0,0 @@
# -*- mode: python -*-
import sys
block_cipher = None
console = True # <--- change this to True to enable command prompt when the app runs
if sys.platform.find("mac") or sys.platform.find("osx"):
console = False
BINARY_SLUG = "DungeonRandomizer"
def recurse_for_py_files(names_so_far):
returnvalue = []
for name in os.listdir(os.path.join(*names_so_far)):
if name != "__pycache__":
subdir_name = os.path.join(*names_so_far, name)
if os.path.isdir(subdir_name):
new_name_list = names_so_far + [name]
for filename in os.listdir(os.path.join(*new_name_list)):
base_file,file_extension = os.path.splitext(filename)
if file_extension == ".py":
new_name = ".".join(new_name_list+[base_file])
if not new_name in returnvalue:
returnvalue.append(new_name)
returnvalue.extend(recurse_for_py_files(new_name_list))
returnvalue.append("PIL._tkinter_finder") #Linux needs this
return returnvalue
hiddenimports = []
binaries = []
a = Analysis([f"../{BINARY_SLUG}.py"],
pathex=[],
binaries=binaries,
datas=[('../data/', 'data/')],
hiddenimports=hiddenimports,
hookspath=[],
runtime_hooks=[],
excludes=[],
win_no_prefer_redirects=False,
win_private_assemblies=False,
cipher=block_cipher,
noarchive=False)
# https://stackoverflow.com/questions/17034434/how-to-remove-exclude-modules-and-files-from-pyinstaller
excluded_binaries = [
'VCRUNTIME140.dll',
'ucrtbase.dll',
'msvcp140.dll',
'mfc140u.dll']
a.binaries = TOC([x for x in a.binaries if x[0] not in excluded_binaries])
pyz = PYZ(a.pure, a.zipped_data,
cipher=block_cipher)
exe = EXE(pyz,
a.scripts,
a.binaries,
a.zipfiles,
a.datas,
[],
name=BINARY_SLUG,
debug=False,
bootloader_ignore_signals=False,
strip=False,
upx=True,
runtime_tmpdir=None,
console=console )

View File

@@ -1,69 +0,0 @@
# -*- mode: python -*-
import sys
block_cipher = None
console = True # <--- change this to True to enable command prompt when the app runs
if sys.platform.find("mac") or sys.platform.find("osx"):
console = False
BINARY_SLUG = "Gui"
def recurse_for_py_files(names_so_far):
returnvalue = []
for name in os.listdir(os.path.join(*names_so_far)):
if name != "__pycache__":
subdir_name = os.path.join(*names_so_far, name)
if os.path.isdir(subdir_name):
new_name_list = names_so_far + [name]
for filename in os.listdir(os.path.join(*new_name_list)):
base_file,file_extension = os.path.splitext(filename)
if file_extension == ".py":
new_name = ".".join(new_name_list+[base_file])
if not new_name in returnvalue:
returnvalue.append(new_name)
returnvalue.extend(recurse_for_py_files(new_name_list))
returnvalue.append("PIL._tkinter_finder") #Linux needs this
return returnvalue
hiddenimports = []
binaries = []
a = Analysis([f"../{BINARY_SLUG}.py"],
pathex=[],
binaries=binaries,
datas=[('../data/', 'data/')],
hiddenimports=hiddenimports,
hookspath=[],
runtime_hooks=[],
excludes=[],
win_no_prefer_redirects=False,
win_private_assemblies=False,
cipher=block_cipher,
noarchive=False)
# https://stackoverflow.com/questions/17034434/how-to-remove-exclude-modules-and-files-from-pyinstaller
excluded_binaries = [
'VCRUNTIME140.dll',
'ucrtbase.dll',
'msvcp140.dll',
'mfc140u.dll']
a.binaries = TOC([x for x in a.binaries if x[0] not in excluded_binaries])
pyz = PYZ(a.pure, a.zipped_data,
cipher=block_cipher)
exe = EXE(pyz,
a.scripts,
a.binaries,
a.zipfiles,
a.datas,
[],
name=BINARY_SLUG,
debug=False,
bootloader_ignore_signals=False,
icon='../data/ER.ico',
strip=False,
upx=True,
runtime_tmpdir=None,
console=console )

98
source/Template.spec Normal file
View File

@@ -0,0 +1,98 @@
# -*- mode: python -*-
import json
import os
import sys
from json.decoder import JSONDecodeError
from PyInstaller.utils.hooks import collect_submodules
block_cipher = None
console = False # <--- change this to True to enable command prompt when the app runs
if sys.platform.find("mac") or sys.platform.find("osx"):
console = True
BINARY_SLUG = "<BINARY_SLUG>"
def recurse_for_py_files(names_so_far):
# get py files
returnvalue = []
for name in os.listdir(os.path.join(*names_so_far)):
# ignore __pycache__
if name != "__pycache__":
subdir_name = os.path.join(*names_so_far, name)
if os.path.isdir(subdir_name):
new_name_list = names_so_far + [name]
for filename in os.listdir(os.path.join(*new_name_list)):
base_file, file_extension = os.path.splitext(filename)
# if it's a .py
if file_extension == ".py":
new_name = ".".join(new_name_list+[base_file])
if not new_name in returnvalue:
returnvalue.append(new_name)
returnvalue.extend(recurse_for_py_files(new_name_list))
return returnvalue
hiddenimports = recurse_for_py_files(["source"])
for hidden in (collect_submodules("pkg_resources")):
hiddenimports.append(hidden)
a = Analysis(
[f"../{BINARY_SLUG}.py"],
pathex=[],
binaries=[],
datas=[('../data/', 'data/')],
hiddenimports=hiddenimports,
hookspath=[],
runtime_hooks=[],
excludes=[],
win_no_prefer_redirects=False,
win_private_assemblies=False,
cipher=block_cipher,
noarchive=False
)
# https://stackoverflow.com/questions/17034434/how-to-remove-exclude-modules-and-files-from-pyinstaller
excluded_binaries = [
'mfc140u.dll',
'msvcp140.dll',
'ucrtbase.dll',
'VCRUNTIME140.dll'
]
# win is temperamental
with open(os.path.join(".","resources","app","meta","manifests","excluded_dlls.json")) as dllsManifest:
dlls = []
try:
dlls = json.load(dllsManifest)
except JSONDecodeError as e:
raise ValueError("Windows DLLs manifest malformed!")
for dll in dlls:
for submod in ["core", "crt"]:
for ver in ["1-1-0", "1-1-1", "1-2-0", "2-1-0"]:
excluded_binaries.append(f"api-ms-win-{submod}-{dll}-l{ver}.dll")
a.binaries = TOC([x for x in a.binaries if x[0] not in excluded_binaries])
pyz = PYZ(
a.pure,
a.zipped_data,
cipher=block_cipher
)
exe = EXE(
pyz,
a.scripts,
a.binaries,
a.zipfiles,
a.datas,
[],
name=BINARY_SLUG,
debug=False,
bootloader_ignore_signals=False,
strip=False,
upx=True,
runtime_tmpdir=None,
console=console
)

View File

@@ -0,0 +1,17 @@
import os
from OverworldShuffle import __version__
OWR_VERSION = __version__
def write_appversion():
APP_VERSION = OWR_VERSION
if "-" in APP_VERSION:
APP_VERSION = APP_VERSION[:APP_VERSION.find("-")]
APP_VERSION_FILE = os.path.join(".","resources","app","meta","manifests","app_version.txt")
with open(APP_VERSION_FILE,"w") as f:
f.seek(0)
f.truncate()
f.write(APP_VERSION)
if __name__ == "__main__":
write_appversion()

View File

@@ -1,16 +1,28 @@
import platform, sys, os, subprocess
import pkg_resources
from datetime import datetime
try:
import pkg_resources
except ModuleNotFoundError as e:
pass
import datetime
from Main import __version__
DR_VERSION = __version__
from OverworldShuffle import __version__
OWR_VERSION = __version__
PROJECT_NAME = "ALttP Overworld Randomizer"
def diagpad(str):
return str.ljust(len("ALttP Door Randomizer Version") + 5,'.')
return str.ljust(len(f"{PROJECT_NAME} Version") + 5,'.')
def output(APP_VERSION):
def output():
lines = [
"ALttP Door Randomizer Diagnostics",
f"{PROJECT_NAME} Diagnostics",
"=================================",
diagpad("UTC Time") + str(datetime.utcnow())[:19],
diagpad("ALttP Door Randomizer Version") + APP_VERSION,
diagpad("UTC Time") + str(datetime.datetime.now(datetime.UTC))[:19],
diagpad("ALttP Door Randomizer Version") + DR_VERSION,
diagpad(f"{PROJECT_NAME} Version") + OWR_VERSION,
diagpad("Python Version") + platform.python_version()
]
lines.append(diagpad("OS Version") + "%s %s" % (platform.system(), platform.release()))
@@ -35,6 +47,7 @@ def output(APP_VERSION):
pkg = pkg.split("==")
lines.append(diagpad(pkg[0]) + pkg[1])
'''
installed_packages = []
installed_packages = [str(d) for d in pkg_resources.working_set] #this doesn't work from the .exe either, but it doesn't crash the program
installed_packages.sort()
for pkg in installed_packages:

View File

@@ -154,7 +154,7 @@ def generation_page(parent,settings):
diag.geometry(str(dims["window"]["width"]) + 'x' + str(dims["window"]["height"]))
text = Text(diag, width=dims["textarea.characters"]["width"], height=dims["textarea.characters"]["height"])
text.pack()
text.insert(INSERT,"\n".join(diagnostics.output(__version__)))
text.insert(INSERT,"\n".join(diagnostics.output()))
# dialog button
self.widgets[widget].pieces["button"] = Button(self.widgets[widget].pieces["frame"], text='Run Diagnostics', command=partial(diags))

View File

@@ -1,27 +0,0 @@
import subprocess
import os
import shutil
import sys
# Spec file
SPEC_FILE = os.path.join(".", "source", "DungeonRandomizer.spec")
# Destination is current dir
DEST_DIRECTORY = '.'
# Check for UPX
if os.path.isdir("upx"):
upx_string = "--upx-dir=upx"
else:
upx_string = ""
if os.path.isdir("build") and not sys.platform.find("mac") and not sys.platform.find("osx"):
shutil.rmtree("build")
# Run pyinstaller for DungeonRandomizer
subprocess.run(" ".join([f"pyinstaller {SPEC_FILE} ",
upx_string,
"-y ",
f"--distpath {DEST_DIRECTORY} ",
]),
shell=True)

View File

@@ -1,27 +0,0 @@
import subprocess
import os
import shutil
import sys
# Spec file
SPEC_FILE = os.path.join(".", "source", "Gui.spec")
# Destination is current dir
DEST_DIRECTORY = '.'
# Check for UPX
if os.path.isdir("upx"):
upx_string = "--upx-dir=upx"
else:
upx_string = ""
if os.path.isdir("build") and not sys.platform.find("mac") and not sys.platform.find("osx"):
shutil.rmtree("build")
# Run pyinstaller for Gui
subprocess.run(" ".join([f"pyinstaller {SPEC_FILE} ",
upx_string,
"-y ",
f"--distpath {DEST_DIRECTORY} ",
]),
shell=True)

155
source/meta/build.py Normal file
View File

@@ -0,0 +1,155 @@
'''
Build Entrypoints
'''
import json
import platform
import os # for checking for dirs
import re
from json.decoder import JSONDecodeError
from subprocess import Popen, PIPE, STDOUT, CalledProcessError
DEST_DIRECTORY = "."
# UPX greatly reduces the filesize. You can get this utility from https://upx.github.io/
# just place it in a subdirectory named "upx" and this script will find it
UPX_DIR = "upx"
if os.path.isdir(os.path.join(".", UPX_DIR)):
upx_string = f"--upx-dir={UPX_DIR}"
else:
upx_string = ""
GO = True
DIFF_DLLS = False
# set a global var for Actions to try to read
def set_output(name, value):
with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
print(f'{name}={value}', file=fh)
# build the thing
def run_build(slug):
global GO
global DIFF_DLLS
print(f"Building '{slug}' via Python {platform.python_version()}")
# get template, mod to do the thing
specTemplateFile = open(os.path.join(".","source","Template.spec"))
specTemplate = specTemplateFile.read()
specTemplateFile.close()
with(open(os.path.join(".","source",f"{slug}.spec"), "w")) as specFile:
print(f"Writing '{slug}' PyInstaller spec file")
thisTemplate = specTemplate.replace("<BINARY_SLUG>", slug)
specFile.write(thisTemplate)
PYINST_EXECUTABLE = "pyinstaller"
args = [
os.path.join("source", f"{slug}.spec").replace(os.sep, os.sep * 2),
upx_string,
"-y",
f"--distpath={DEST_DIRECTORY}"
]
errs = []
strs = []
print("PyInstaller args: %s" % " ".join(args))
cmd = [
PYINST_EXECUTABLE,
*args
]
ret = {
"stdout": [],
"stderr": []
}
with Popen(cmd, stdout=PIPE, stderr=STDOUT, bufsize=1, universal_newlines=True) as p:
for line in p.stdout:
ret["stdout"].append(line)
print(line, end='')
# if p.stderr:
# for line in p.stderr:
# ret["stderr"].append(line)
# print(line, end='')
# if p.returncode != 0:
# raise CalledProcessError(p.returncode, p.args)
# check stdout & stderr
for key in ["stdout","stderr"]:
if len(ret[key]) > 0:
for line in ret[key]:
# UPX can't compress this file
if "NotCompressibleException" in line.strip():
print(line)
errs.append(line.strip())
# print UPX messages
if "UPX" in line:
print(line)
# try to get DLL filename
elif "NotCompressibleException" in line.strip():
matches = re.search(r'api-ms-win-(?:[^-]*)-([^-]*)', line.strip())
if matches:
strAdd = matches.group(1)
strs.append(strAdd)
errs.append(line.strip())
# print collected errors
if len(errs) > 0:
print("=" * 10)
print("| ERRORS |")
print("=" * 10)
print("\n".join(errs))
else:
GO = False
# if we identified DLLs to ignore
if len(strs) > 0:
# read DLLs manifest that we've already got saved
with open(os.path.join(".","resources","app","meta","manifests","excluded_dlls.json"), "w+", encoding="utf-8") as dllsManifest:
oldDLLs = []
try:
oldDLLs = json.load(dllsManifest)
except JSONDecodeError as e:
oldDLLs = []
# raise ValueError("Windows DLLs manifest malformed!")
# bucket for new list
newDLLs = sorted(list(set(oldDLLs)))
# items to add
addDLLs = sorted(list(set(strs)))
# add items
newDLLs += addDLLs
newDLLs = sorted(list(set(newDLLs)))
# if the lists differ, we've gotta update the included list
diffDLLs = newDLLs != oldDLLs
if diffDLLs:
DIFF_DLLS = True
dllsManifest.seek(0)
dllsManifest.truncate()
dllsManifest.write(json.dumps(sorted(newDLLs), indent=2))
print(f"Old DLLs: {json.dumps(sorted(oldDLLs))}")
print(f"Add DLLs: {json.dumps(sorted(addDLLs))}")
print(f"New DLLs: {json.dumps(sorted(newDLLs))}")
print(f"Diff DLLs: {DIFF_DLLS}")
print("")
def go_build(slug):
slug = slug or ""
if slug != "":
GO = True
while GO:
run_build(slug)
GO = False
if __name__ == "__main__":
binary_slugs = []
#TODO: Make sure we've got the proper binaries that we need
with open(os.path.join(".","resources","app","meta","manifests","binaries.json")) as binariesFile:
binary_slugs = json.load(binariesFile)
for file_slug in binary_slugs:
go_build(file_slug)
if DIFF_DLLS:
print("🔴Had to update Error DLLs list!")
exit(1)

View File

@@ -0,0 +1,10 @@
import json
import os
error_dlls_path = os.path.join(".","resources","app","meta","manifests","excluded_dlls.json")
if os.path.isfile(error_dlls_path):
with open(error_dlls_path, "r") as error_dlls_file:
error_dlls_json = json.load(error_dlls_file)
if len(error_dlls_json) > 0 and error_dlls_json[0].strip() != "":
print(error_dlls_json)
# exit(1)

10
source/meta/run_diags.py Normal file
View File

@@ -0,0 +1,10 @@
from source.classes import diags as diags
global VERBOSE
VERBOSE = True
if __name__ == "__main__":
if VERBOSE:
print("DIAGNOSTICS")
print('.' * 70)
print("\n".join(diags.output()))