build-release.py 79 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556
  1. #!/usr/bin/env python3
  2. """
  3. This script is shared between SDL2, SDL3, and all satellite libraries.
  4. Don't specialize this script for doing project-specific modifications.
  5. Rather, modify release-info.json.
  6. """
  7. import argparse
  8. import collections
  9. import dataclasses
  10. from collections.abc import Callable
  11. import contextlib
  12. import datetime
  13. import fnmatch
  14. import glob
  15. import io
  16. import json
  17. import logging
  18. import multiprocessing
  19. import os
  20. from pathlib import Path
  21. import platform
  22. import re
  23. import shlex
  24. import shutil
  25. import subprocess
  26. import sys
  27. import tarfile
  28. import tempfile
  29. import textwrap
  30. import typing
  31. import zipfile
  32. logger = logging.getLogger(__name__)
  33. GIT_HASH_FILENAME = ".git-hash"
  34. REVISION_TXT = "REVISION.txt"
  35. RE_ILLEGAL_MINGW_LIBRARIES = re.compile(r"(?:lib)?(?:gcc|(?:std)?c[+][+]|(?:win)?pthread).*", flags=re.I)
  36. def safe_isotime_to_datetime(str_isotime: str) -> datetime.datetime:
  37. try:
  38. return datetime.datetime.fromisoformat(str_isotime)
  39. except ValueError:
  40. pass
  41. logger.warning("Invalid iso time: %s", str_isotime)
  42. if str_isotime[-6:-5] in ("+", "-"):
  43. # Commits can have isotime with invalid timezone offset (e.g. "2021-07-04T20:01:40+32:00")
  44. modified_str_isotime = str_isotime[:-6] + "+00:00"
  45. try:
  46. return datetime.datetime.fromisoformat(modified_str_isotime)
  47. except ValueError:
  48. pass
  49. raise ValueError(f"Invalid isotime: {str_isotime}")
  50. def arc_join(*parts: list[str]) -> str:
  51. assert all(p[:1] != "/" and p[-1:] != "/" for p in parts), f"None of {parts} may start or end with '/'"
  52. return "/".join(p for p in parts if p)
  53. @dataclasses.dataclass(frozen=True)
  54. class VsArchPlatformConfig:
  55. arch: str
  56. configuration: str
  57. platform: str
  58. def extra_context(self):
  59. return {
  60. "ARCH": self.arch,
  61. "CONFIGURATION": self.configuration,
  62. "PLATFORM": self.platform,
  63. }
  64. @contextlib.contextmanager
  65. def chdir(path):
  66. original_cwd = os.getcwd()
  67. try:
  68. os.chdir(path)
  69. yield
  70. finally:
  71. os.chdir(original_cwd)
  72. class Executer:
  73. def __init__(self, root: Path, dry: bool=False):
  74. self.root = root
  75. self.dry = dry
  76. def run(self, cmd, cwd=None, env=None):
  77. logger.info("Executing args=%r", cmd)
  78. sys.stdout.flush()
  79. if not self.dry:
  80. subprocess.check_call(cmd, cwd=cwd or self.root, env=env, text=True)
  81. def check_output(self, cmd, cwd=None, dry_out=None, env=None, text=True):
  82. logger.info("Executing args=%r", cmd)
  83. sys.stdout.flush()
  84. if self.dry:
  85. return dry_out
  86. return subprocess.check_output(cmd, cwd=cwd or self.root, env=env, text=text)
  87. class SectionPrinter:
  88. @contextlib.contextmanager
  89. def group(self, title: str):
  90. print(f"{title}:")
  91. yield
  92. class GitHubSectionPrinter(SectionPrinter):
  93. def __init__(self):
  94. super().__init__()
  95. self.in_group = False
  96. @contextlib.contextmanager
  97. def group(self, title: str):
  98. print(f"::group::{title}")
  99. assert not self.in_group, "Can enter a group only once"
  100. self.in_group = True
  101. yield
  102. self.in_group = False
  103. print("::endgroup::")
  104. class VisualStudio:
  105. def __init__(self, executer: Executer, year: typing.Optional[str]=None):
  106. self.executer = executer
  107. self.vsdevcmd = self.find_vsdevcmd(year)
  108. self.msbuild = self.find_msbuild()
  109. @property
  110. def dry(self) -> bool:
  111. return self.executer.dry
  112. VS_YEAR_TO_VERSION = {
  113. "2022": 17,
  114. "2019": 16,
  115. "2017": 15,
  116. "2015": 14,
  117. "2013": 12,
  118. }
  119. def find_vsdevcmd(self, year: typing.Optional[str]=None) -> typing.Optional[Path]:
  120. vswhere_spec = ["-latest"]
  121. if year is not None:
  122. try:
  123. version = self.VS_YEAR_TO_VERSION[year]
  124. except KeyError:
  125. logger.error("Invalid Visual Studio year")
  126. return None
  127. vswhere_spec.extend(["-version", f"[{version},{version+1})"])
  128. vswhere_cmd = ["vswhere"] + vswhere_spec + ["-property", "installationPath"]
  129. vs_install_path = Path(self.executer.check_output(vswhere_cmd, dry_out="/tmp").strip())
  130. logger.info("VS install_path = %s", vs_install_path)
  131. assert vs_install_path.is_dir(), "VS installation path does not exist"
  132. vsdevcmd_path = vs_install_path / "Common7/Tools/vsdevcmd.bat"
  133. logger.info("vsdevcmd path = %s", vsdevcmd_path)
  134. if self.dry:
  135. vsdevcmd_path.parent.mkdir(parents=True, exist_ok=True)
  136. vsdevcmd_path.touch(exist_ok=True)
  137. assert vsdevcmd_path.is_file(), "vsdevcmd.bat batch file does not exist"
  138. return vsdevcmd_path
  139. def find_msbuild(self) -> typing.Optional[Path]:
  140. vswhere_cmd = ["vswhere", "-latest", "-requires", "Microsoft.Component.MSBuild", "-find", r"MSBuild\**\Bin\MSBuild.exe"]
  141. msbuild_path = Path(self.executer.check_output(vswhere_cmd, dry_out="/tmp/MSBuild.exe").strip())
  142. logger.info("MSBuild path = %s", msbuild_path)
  143. if self.dry:
  144. msbuild_path.parent.mkdir(parents=True, exist_ok=True)
  145. msbuild_path.touch(exist_ok=True)
  146. assert msbuild_path.is_file(), "MSBuild.exe does not exist"
  147. return msbuild_path
  148. def build(self, arch_platform: VsArchPlatformConfig, projects: list[Path]):
  149. assert projects, "Need at least one project to build"
  150. vsdev_cmd_str = f"\"{self.vsdevcmd}\" -arch={arch_platform.arch}"
  151. msbuild_cmd_str = " && ".join([f"\"{self.msbuild}\" \"{project}\" /m /p:BuildInParallel=true /p:Platform={arch_platform.platform} /p:Configuration={arch_platform.configuration}" for project in projects])
  152. bat_contents = f"{vsdev_cmd_str} && {msbuild_cmd_str}\n"
  153. bat_path = Path(tempfile.gettempdir()) / "cmd.bat"
  154. with bat_path.open("w") as f:
  155. f.write(bat_contents)
  156. logger.info("Running cmd.exe script (%s): %s", bat_path, bat_contents)
  157. cmd = ["cmd.exe", "/D", "/E:ON", "/V:OFF", "/S", "/C", f"CALL {str(bat_path)}"]
  158. self.executer.run(cmd)
  159. class Archiver:
  160. def __init__(self, zip_path: typing.Optional[Path]=None, tgz_path: typing.Optional[Path]=None, txz_path: typing.Optional[Path]=None):
  161. self._zip_files = []
  162. self._tar_files = []
  163. self._added_files = set()
  164. if zip_path:
  165. self._zip_files.append(zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED))
  166. if tgz_path:
  167. self._tar_files.append(tarfile.open(tgz_path, "w:gz"))
  168. if txz_path:
  169. self._tar_files.append(tarfile.open(txz_path, "w:xz"))
  170. @property
  171. def added_files(self) -> set[str]:
  172. return self._added_files
  173. def add_file_data(self, arcpath: str, data: bytes, mode: int, time: datetime.datetime):
  174. for zf in self._zip_files:
  175. file_data_time = (time.year, time.month, time.day, time.hour, time.minute, time.second)
  176. zip_info = zipfile.ZipInfo(filename=arcpath, date_time=file_data_time)
  177. zip_info.external_attr = mode << 16
  178. zip_info.compress_type = zipfile.ZIP_DEFLATED
  179. zf.writestr(zip_info, data=data)
  180. for tf in self._tar_files:
  181. tar_info = tarfile.TarInfo(arcpath)
  182. tar_info.type = tarfile.REGTYPE
  183. tar_info.mode = mode
  184. tar_info.size = len(data)
  185. tar_info.mtime = int(time.timestamp())
  186. tf.addfile(tar_info, fileobj=io.BytesIO(data))
  187. self._added_files.add(arcpath)
  188. def add_symlink(self, arcpath: str, target: str, time: datetime.datetime, files_for_zip):
  189. logger.debug("Adding symlink (target=%r) -> %s", target, arcpath)
  190. for zf in self._zip_files:
  191. file_data_time = (time.year, time.month, time.day, time.hour, time.minute, time.second)
  192. for f in files_for_zip:
  193. zip_info = zipfile.ZipInfo(filename=f["arcpath"], date_time=file_data_time)
  194. zip_info.external_attr = f["mode"] << 16
  195. zip_info.compress_type = zipfile.ZIP_DEFLATED
  196. zf.writestr(zip_info, data=f["data"])
  197. for tf in self._tar_files:
  198. tar_info = tarfile.TarInfo(arcpath)
  199. tar_info.type = tarfile.SYMTYPE
  200. tar_info.mode = 0o777
  201. tar_info.mtime = int(time.timestamp())
  202. tar_info.linkname = target
  203. tf.addfile(tar_info)
  204. self._added_files.update(f["arcpath"] for f in files_for_zip)
  205. def add_git_hash(self, arcdir: str, commit: str, time: datetime.datetime):
  206. arcpath = arc_join(arcdir, GIT_HASH_FILENAME)
  207. data = f"{commit}\n".encode()
  208. self.add_file_data(arcpath=arcpath, data=data, mode=0o100644, time=time)
  209. def add_file_path(self, arcpath: str, path: Path):
  210. assert path.is_file(), f"{path} should be a file"
  211. logger.debug("Adding %s -> %s", path, arcpath)
  212. for zf in self._zip_files:
  213. zf.write(path, arcname=arcpath)
  214. for tf in self._tar_files:
  215. tf.add(path, arcname=arcpath)
  216. def add_file_directory(self, arcdirpath: str, dirpath: Path):
  217. assert dirpath.is_dir()
  218. if arcdirpath and arcdirpath[-1:] != "/":
  219. arcdirpath += "/"
  220. for f in dirpath.iterdir():
  221. if f.is_file():
  222. arcpath = f"{arcdirpath}{f.name}"
  223. logger.debug("Adding %s to %s", f, arcpath)
  224. self.add_file_path(arcpath=arcpath, path=f)
  225. def close(self):
  226. # Archiver is intentionally made invalid after this function
  227. del self._zip_files
  228. self._zip_files = None
  229. del self._tar_files
  230. self._tar_files = None
  231. def __enter__(self):
  232. return self
  233. def __exit__(self, type, value, traceback):
  234. self.close()
  235. class NodeInArchive:
  236. def __init__(self, arcpath: str, path: typing.Optional[Path]=None, data: typing.Optional[bytes]=None, mode: typing.Optional[int]=None, symtarget: typing.Optional[str]=None, time: typing.Optional[datetime.datetime]=None, directory: bool=False):
  237. self.arcpath = arcpath
  238. self.path = path
  239. self.data = data
  240. self.mode = mode
  241. self.symtarget = symtarget
  242. self.time = time
  243. self.directory = directory
  244. @classmethod
  245. def from_fs(cls, arcpath: str, path: Path, mode: int=0o100644, time: typing.Optional[datetime.datetime]=None) -> "NodeInArchive":
  246. if time is None:
  247. time = datetime.datetime.fromtimestamp(os.stat(path).st_mtime)
  248. return cls(arcpath=arcpath, path=path, mode=mode)
  249. @classmethod
  250. def from_data(cls, arcpath: str, data: bytes, time: datetime.datetime) -> "NodeInArchive":
  251. return cls(arcpath=arcpath, data=data, time=time, mode=0o100644)
  252. @classmethod
  253. def from_text(cls, arcpath: str, text: str, time: datetime.datetime) -> "NodeInArchive":
  254. return cls.from_data(arcpath=arcpath, data=text.encode(), time=time)
  255. @classmethod
  256. def from_symlink(cls, arcpath: str, symtarget: str) -> "NodeInArchive":
  257. return cls(arcpath=arcpath, symtarget=symtarget)
  258. @classmethod
  259. def from_directory(cls, arcpath: str) -> "NodeInArchive":
  260. return cls(arcpath=arcpath, directory=True)
  261. def __repr__(self) -> str:
  262. return f"<{type(self).__name__}:arcpath={self.arcpath},path='{str(self.path)}',len(data)={len(self.data) if self.data else 'n/a'},directory={self.directory},symtarget={self.symtarget}>"
  263. def configure_file(path: Path, context: dict[str, str]) -> bytes:
  264. text = path.read_text()
  265. return configure_text(text, context=context).encode()
  266. def configure_text(text: str, context: dict[str, str]) -> str:
  267. original_text = text
  268. for txt, repl in context.items():
  269. text = text.replace(f"@<@{txt}@>@", repl)
  270. success = all(thing not in text for thing in ("@<@", "@>@"))
  271. if not success:
  272. raise ValueError(f"Failed to configure {repr(original_text)}")
  273. return text
  274. def configure_text_list(text_list: list[str], context: dict[str, str]) -> list[str]:
  275. return [configure_text(text=e, context=context) for e in text_list]
  276. class ArchiveFileTree:
  277. def __init__(self):
  278. self._tree: dict[str, NodeInArchive] = {}
  279. def add_file(self, file: NodeInArchive):
  280. self._tree[file.arcpath] = file
  281. def __iter__(self) -> typing.Iterable[NodeInArchive]:
  282. yield from self._tree.values()
  283. def __contains__(self, value: str) -> bool:
  284. return value in self._tree
  285. def get_latest_mod_time(self) -> datetime.datetime:
  286. return max(item.time for item in self._tree.values() if item.time)
  287. def add_to_archiver(self, archive_base: str, archiver: Archiver):
  288. remaining_symlinks = set()
  289. added_files = dict()
  290. def calculate_symlink_target(s: NodeInArchive) -> str:
  291. dest_dir = os.path.dirname(s.arcpath)
  292. if dest_dir:
  293. dest_dir += "/"
  294. target = dest_dir + s.symtarget
  295. while True:
  296. new_target, n = re.subn(r"([^/]+/+[.]{2}/)", "", target)
  297. target = new_target
  298. if not n:
  299. break
  300. return target
  301. # Add files in first pass
  302. for arcpath, node in self._tree.items():
  303. assert node is not None, f"{arcpath} -> node"
  304. if node.data is not None:
  305. archiver.add_file_data(arcpath=arc_join(archive_base, arcpath), data=node.data, time=node.time, mode=node.mode)
  306. assert node.arcpath is not None, f"{node=}"
  307. added_files[node.arcpath] = node
  308. elif node.path is not None:
  309. archiver.add_file_path(arcpath=arc_join(archive_base, arcpath), path=node.path)
  310. assert node.arcpath is not None, f"{node=}"
  311. added_files[node.arcpath] = node
  312. elif node.symtarget is not None:
  313. remaining_symlinks.add(node)
  314. elif node.directory:
  315. pass
  316. else:
  317. raise ValueError(f"Invalid Archive Node: {repr(node)}")
  318. assert None not in added_files
  319. # Resolve symlinks in second pass: zipfile does not support symlinks, so add files to zip archive
  320. while True:
  321. if not remaining_symlinks:
  322. break
  323. symlinks_this_time = set()
  324. extra_added_files = {}
  325. for symlink in remaining_symlinks:
  326. symlink_files_for_zip = {}
  327. symlink_target_path = calculate_symlink_target(symlink)
  328. if symlink_target_path in added_files:
  329. symlink_files_for_zip[symlink.arcpath] = added_files[symlink_target_path]
  330. else:
  331. symlink_target_path_slash = symlink_target_path + "/"
  332. for added_file in added_files:
  333. if added_file.startswith(symlink_target_path_slash):
  334. path_in_symlink = symlink.arcpath + "/" + added_file.removeprefix(symlink_target_path_slash)
  335. symlink_files_for_zip[path_in_symlink] = added_files[added_file]
  336. if symlink_files_for_zip:
  337. symlinks_this_time.add(symlink)
  338. extra_added_files.update(symlink_files_for_zip)
  339. files_for_zip = [{"arcpath": f"{archive_base}/{sym_path}", "data": sym_info.data, "mode": sym_info.mode} for sym_path, sym_info in symlink_files_for_zip.items()]
  340. archiver.add_symlink(arcpath=f"{archive_base}/{symlink.arcpath}", target=symlink.symtarget, time=symlink.time, files_for_zip=files_for_zip)
  341. # if not symlinks_this_time:
  342. # logger.info("files added: %r", set(path for path in added_files.keys()))
  343. assert symlinks_this_time, f"No targets found for symlinks: {remaining_symlinks}"
  344. remaining_symlinks.difference_update(symlinks_this_time)
  345. added_files.update(extra_added_files)
  346. def add_directory_tree(self, arc_dir: str, path: Path, time: datetime.datetime):
  347. assert path.is_dir()
  348. for files_dir, _, filenames in os.walk(path):
  349. files_dir_path = Path(files_dir)
  350. rel_files_path = files_dir_path.relative_to(path)
  351. for filename in filenames:
  352. self.add_file(NodeInArchive.from_fs(arcpath=arc_join(arc_dir, str(rel_files_path), filename), path=files_dir_path / filename, time=time))
  353. def _add_files_recursively(self, arc_dir: str, paths: list[Path], time: datetime.datetime):
  354. logger.debug(f"_add_files_recursively({arc_dir=} {paths=})")
  355. for path in paths:
  356. arcpath = arc_join(arc_dir, path.name)
  357. if path.is_file():
  358. logger.debug("Adding %s as %s", path, arcpath)
  359. self.add_file(NodeInArchive.from_fs(arcpath=arcpath, path=path, time=time))
  360. elif path.is_dir():
  361. self._add_files_recursively(arc_dir=arc_join(arc_dir, path.name), paths=list(path.iterdir()), time=time)
  362. else:
  363. raise ValueError(f"Unsupported file type to add recursively: {path}")
  364. def add_file_mapping(self, arc_dir: str, file_mapping: dict[str, list[str]], file_mapping_root: Path, context: dict[str, str], time: datetime.datetime):
  365. for meta_rel_destdir, meta_file_globs in file_mapping.items():
  366. rel_destdir = configure_text(meta_rel_destdir, context=context)
  367. assert "@" not in rel_destdir, f"archive destination should not contain an @ after configuration ({repr(meta_rel_destdir)}->{repr(rel_destdir)})"
  368. for meta_file_glob in meta_file_globs:
  369. file_glob = configure_text(meta_file_glob, context=context)
  370. assert "@" not in rel_destdir, f"archive glob should not contain an @ after configuration ({repr(meta_file_glob)}->{repr(file_glob)})"
  371. if ":" in file_glob:
  372. original_path, new_filename = file_glob.rsplit(":", 1)
  373. assert ":" not in original_path, f"Too many ':' in {repr(file_glob)}"
  374. assert "/" not in new_filename, f"New filename cannot contain a '/' in {repr(file_glob)}"
  375. path = file_mapping_root / original_path
  376. arcpath = arc_join(arc_dir, rel_destdir, new_filename)
  377. if path.suffix == ".in":
  378. data = configure_file(path, context=context)
  379. logger.debug("Adding processed %s -> %s", path, arcpath)
  380. self.add_file(NodeInArchive.from_data(arcpath=arcpath, data=data, time=time))
  381. else:
  382. logger.debug("Adding %s -> %s", path, arcpath)
  383. self.add_file(NodeInArchive.from_fs(arcpath=arcpath, path=path, time=time))
  384. else:
  385. relative_file_paths = glob.glob(file_glob, root_dir=file_mapping_root)
  386. assert relative_file_paths, f"Glob '{file_glob}' does not match any file"
  387. self._add_files_recursively(arc_dir=arc_join(arc_dir, rel_destdir), paths=[file_mapping_root / p for p in relative_file_paths], time=time)
  388. class SourceCollector:
  389. # TreeItem = collections.namedtuple("TreeItem", ("path", "mode", "data", "symtarget", "directory", "time"))
  390. def __init__(self, root: Path, commit: str, filter: typing.Optional[Callable[[str], bool]], executer: Executer):
  391. self.root = root
  392. self.commit = commit
  393. self.filter = filter
  394. self.executer = executer
  395. def get_archive_file_tree(self) -> ArchiveFileTree:
  396. git_archive_args = ["git", "archive", "--format=tar.gz", self.commit, "-o", "/dev/stdout"]
  397. logger.info("Executing args=%r", git_archive_args)
  398. contents_tgz = subprocess.check_output(git_archive_args, cwd=self.root, text=False)
  399. tar_archive = tarfile.open(fileobj=io.BytesIO(contents_tgz), mode="r:gz")
  400. filenames = tuple(m.name for m in tar_archive if (m.isfile() or m.issym()))
  401. file_times = self._get_file_times(paths=filenames)
  402. git_contents = ArchiveFileTree()
  403. for ti in tar_archive:
  404. if self.filter and not self.filter(ti.name):
  405. continue
  406. data = None
  407. symtarget = None
  408. directory = False
  409. file_time = None
  410. if ti.isfile():
  411. contents_file = tar_archive.extractfile(ti.name)
  412. data = contents_file.read()
  413. file_time = file_times[ti.name]
  414. elif ti.issym():
  415. symtarget = ti.linkname
  416. file_time = file_times[ti.name]
  417. elif ti.isdir():
  418. directory = True
  419. else:
  420. raise ValueError(f"{ti.name}: unknown type")
  421. node = NodeInArchive(arcpath=ti.name, data=data, mode=ti.mode, symtarget=symtarget, time=file_time, directory=directory)
  422. git_contents.add_file(node)
  423. return git_contents
  424. def _get_file_times(self, paths: tuple[str, ...]) -> dict[str, datetime.datetime]:
  425. dry_out = textwrap.dedent("""\
  426. time=2024-03-14T15:40:25-07:00
  427. M\tCMakeLists.txt
  428. """)
  429. git_log_out = self.executer.check_output(["git", "log", "--name-status", '--pretty=time=%cI', self.commit], dry_out=dry_out, cwd=self.root).splitlines(keepends=False)
  430. current_time = None
  431. set_paths = set(paths)
  432. path_times: dict[str, datetime.datetime] = {}
  433. for line in git_log_out:
  434. if not line:
  435. continue
  436. if line.startswith("time="):
  437. current_time = safe_isotime_to_datetime(line.removeprefix("time="))
  438. continue
  439. mod_type, file_paths = line.split(maxsplit=1)
  440. assert current_time is not None
  441. for file_path in file_paths.split("\t"):
  442. if file_path in set_paths and file_path not in path_times:
  443. path_times[file_path] = current_time
  444. # FIXME: find out why some files are not shown in "git log"
  445. # assert set(path_times.keys()) == set_paths
  446. if set(path_times.keys()) != set_paths:
  447. found_times = set(path_times.keys())
  448. paths_without_times = set_paths.difference(found_times)
  449. logger.warning("No times found for these paths: %s", paths_without_times)
  450. max_time = max(time for time in path_times.values())
  451. for path in paths_without_times:
  452. path_times[path] = max_time
  453. return path_times
  454. class AndroidApiVersion:
  455. def __init__(self, name: str, ints: tuple[int, ...]):
  456. self.name = name
  457. self.ints = ints
  458. def __repr__(self) -> str:
  459. return f"<{self.name} ({'.'.join(str(v) for v in self.ints)})>"
  460. class Releaser:
  461. def __init__(self, release_info: dict, commit: str, revision: str, root: Path, dist_path: Path, section_printer: SectionPrinter, executer: Executer, cmake_generator: str, deps_path: Path, overwrite: bool, github: bool, fast: bool):
  462. self.release_info = release_info
  463. self.project = release_info["name"]
  464. self.version = self.extract_sdl_version(root=root, release_info=release_info)
  465. self.root = root
  466. self.commit = commit
  467. self.revision = revision
  468. self.dist_path = dist_path
  469. self.section_printer = section_printer
  470. self.executer = executer
  471. self.cmake_generator = cmake_generator
  472. self.cpu_count = multiprocessing.cpu_count()
  473. self.deps_path = deps_path
  474. self.overwrite = overwrite
  475. self.github = github
  476. self.fast = fast
  477. self.arc_time = datetime.datetime.now()
  478. self.artifacts: dict[str, Path] = {}
  479. def get_context(self, extra_context: typing.Optional[dict[str, str]]=None) -> dict[str, str]:
  480. ctx = {
  481. "PROJECT_NAME": self.project,
  482. "PROJECT_VERSION": self.version,
  483. "PROJECT_COMMIT": self.commit,
  484. "PROJECT_REVISION": self.revision,
  485. "PROJECT_ROOT": str(self.root),
  486. }
  487. if extra_context:
  488. ctx.update(extra_context)
  489. return ctx
  490. @property
  491. def dry(self) -> bool:
  492. return self.executer.dry
  493. def prepare(self):
  494. logger.debug("Creating dist folder")
  495. self.dist_path.mkdir(parents=True, exist_ok=True)
  496. @classmethod
  497. def _path_filter(cls, path: str) -> bool:
  498. if ".gitmodules" in path:
  499. return True
  500. if path.startswith(".git"):
  501. return False
  502. return True
  503. @classmethod
  504. def _external_repo_path_filter(cls, path: str) -> bool:
  505. if not cls._path_filter(path):
  506. return False
  507. if path.startswith("test/") or path.startswith("tests/"):
  508. return False
  509. return True
  510. def create_source_archives(self) -> None:
  511. source_collector = SourceCollector(root=self.root, commit=self.commit, executer=self.executer, filter=self._path_filter)
  512. print(f"Collecting sources of {self.project}...")
  513. archive_tree: ArchiveFileTree = source_collector.get_archive_file_tree()
  514. latest_mod_time = archive_tree.get_latest_mod_time()
  515. archive_tree.add_file(NodeInArchive.from_text(arcpath=REVISION_TXT, text=f"{self.revision}\n", time=latest_mod_time))
  516. archive_tree.add_file(NodeInArchive.from_text(arcpath=f"{GIT_HASH_FILENAME}", text=f"{self.commit}\n", time=latest_mod_time))
  517. archive_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["source"].get("files", {}), file_mapping_root=self.root, context=self.get_context(), time=latest_mod_time)
  518. if "Makefile.am" in archive_tree:
  519. patched_time = latest_mod_time + datetime.timedelta(minutes=1)
  520. print(f"Makefile.am detected -> touching aclocal.m4, */Makefile.in, configure")
  521. for node_data in archive_tree:
  522. arc_name = os.path.basename(node_data.arcpath)
  523. arc_name_we, arc_name_ext = os.path.splitext(arc_name)
  524. if arc_name in ("aclocal.m4", "configure", "Makefile.in"):
  525. print(f"Bumping time of {node_data.arcpath}")
  526. node_data.time = patched_time
  527. archive_base = f"{self.project}-{self.version}"
  528. zip_path = self.dist_path / f"{archive_base}.zip"
  529. tgz_path = self.dist_path / f"{archive_base}.tar.gz"
  530. txz_path = self.dist_path / f"{archive_base}.tar.xz"
  531. logger.info("Creating zip/tgz/txz source archives ...")
  532. if self.dry:
  533. zip_path.touch()
  534. tgz_path.touch()
  535. txz_path.touch()
  536. else:
  537. with Archiver(zip_path=zip_path, tgz_path=tgz_path, txz_path=txz_path) as archiver:
  538. print(f"Adding source files of {self.project}...")
  539. archive_tree.add_to_archiver(archive_base=archive_base, archiver=archiver)
  540. for extra_repo in self.release_info["source"].get("extra-repos", []):
  541. extra_repo_root = self.root / extra_repo
  542. assert (extra_repo_root / ".git").exists(), f"{extra_repo_root} must be a git repo"
  543. extra_repo_commit = self.executer.check_output(["git", "rev-parse", "HEAD"], dry_out=f"gitsha-extra-repo-{extra_repo}", cwd=extra_repo_root).strip()
  544. extra_repo_source_collector = SourceCollector(root=extra_repo_root, commit=extra_repo_commit, executer=self.executer, filter=self._external_repo_path_filter)
  545. print(f"Collecting sources of {extra_repo} ...")
  546. extra_repo_archive_tree = extra_repo_source_collector.get_archive_file_tree()
  547. print(f"Adding source files of {extra_repo} ...")
  548. extra_repo_archive_tree.add_to_archiver(archive_base=f"{archive_base}/{extra_repo}", archiver=archiver)
  549. for file in self.release_info["source"]["checks"]:
  550. assert f"{archive_base}/{file}" in archiver.added_files, f"'{archive_base}/{file}' must exist"
  551. logger.info("... done")
  552. self.artifacts["src-zip"] = zip_path
  553. self.artifacts["src-tar-gz"] = tgz_path
  554. self.artifacts["src-tar-xz"] = txz_path
  555. if not self.dry:
  556. with tgz_path.open("r+b") as f:
  557. # Zero the embedded timestamp in the gzip'ed tarball
  558. f.seek(4, 0)
  559. f.write(b"\x00\x00\x00\x00")
  560. def create_dmg(self, configuration: str="Release") -> None:
  561. dmg_in = self.root / self.release_info["dmg"]["path"]
  562. xcode_project = self.root / self.release_info["dmg"]["project"]
  563. assert xcode_project.is_dir(), f"{xcode_project} must be a directory"
  564. assert (xcode_project / "project.pbxproj").is_file, f"{xcode_project} must contain project.pbxproj"
  565. if not self.fast:
  566. dmg_in.unlink(missing_ok=True)
  567. build_xcconfig = self.release_info["dmg"].get("build-xcconfig")
  568. if build_xcconfig:
  569. shutil.copy(self.root / build_xcconfig, xcode_project.parent / "build.xcconfig")
  570. xcode_scheme = self.release_info["dmg"].get("scheme")
  571. xcode_target = self.release_info["dmg"].get("target")
  572. assert xcode_scheme or xcode_target, "dmg needs scheme or target"
  573. assert not (xcode_scheme and xcode_target), "dmg cannot have both scheme and target set"
  574. if xcode_scheme:
  575. scheme_or_target = "-scheme"
  576. target_like = xcode_scheme
  577. else:
  578. scheme_or_target = "-target"
  579. target_like = xcode_target
  580. self.executer.run(["xcodebuild", "ONLY_ACTIVE_ARCH=NO", "-project", xcode_project, scheme_or_target, target_like, "-configuration", configuration])
  581. if self.dry:
  582. dmg_in.parent.mkdir(parents=True, exist_ok=True)
  583. dmg_in.touch()
  584. assert dmg_in.is_file(), f"{self.project}.dmg was not created by xcodebuild"
  585. dmg_out = self.dist_path / f"{self.project}-{self.version}.dmg"
  586. shutil.copy(dmg_in, dmg_out)
  587. self.artifacts["dmg"] = dmg_out
  588. @property
  589. def git_hash_data(self) -> bytes:
  590. return f"{self.commit}\n".encode()
  591. def verify_mingw_library(self, triplet: str, path: Path):
  592. objdump_output = self.executer.check_output([f"{triplet}-objdump", "-p", str(path)])
  593. libraries = re.findall(r"DLL Name: ([^\n]+)", objdump_output)
  594. logger.info("%s (%s) libraries: %r", path, triplet, libraries)
  595. illegal_libraries = list(filter(RE_ILLEGAL_MINGW_LIBRARIES.match, libraries))
  596. logger.error("Detected 'illegal' libraries: %r", illegal_libraries)
  597. if illegal_libraries:
  598. raise Exception(f"{path} links to illegal libraries: {illegal_libraries}")
  599. def create_mingw_archives(self) -> None:
  600. build_type = "Release"
  601. build_parent_dir = self.root / "build-mingw"
  602. ARCH_TO_GNU_ARCH = {
  603. # "arm64": "aarch64",
  604. "x86": "i686",
  605. "x64": "x86_64",
  606. }
  607. ARCH_TO_TRIPLET = {
  608. # "arm64": "aarch64-w64-mingw32",
  609. "x86": "i686-w64-mingw32",
  610. "x64": "x86_64-w64-mingw32",
  611. }
  612. new_env = dict(os.environ)
  613. cmake_prefix_paths = []
  614. mingw_deps_path = self.deps_path / "mingw-deps"
  615. if "dependencies" in self.release_info["mingw"]:
  616. shutil.rmtree(mingw_deps_path, ignore_errors=True)
  617. mingw_deps_path.mkdir()
  618. for triplet in ARCH_TO_TRIPLET.values():
  619. (mingw_deps_path / triplet).mkdir()
  620. def extract_filter(member: tarfile.TarInfo, path: str, /):
  621. if member.name.startswith("SDL"):
  622. member.name = "/".join(Path(member.name).parts[1:])
  623. return member
  624. for dep in self.release_info.get("dependencies", {}):
  625. extract_path = mingw_deps_path / f"extract-{dep}"
  626. extract_path.mkdir()
  627. with chdir(extract_path):
  628. tar_path = self.deps_path / glob.glob(self.release_info["mingw"]["dependencies"][dep]["artifact"], root_dir=self.deps_path)[0]
  629. logger.info("Extracting %s to %s", tar_path, mingw_deps_path)
  630. assert tar_path.suffix in (".gz", ".xz")
  631. with tarfile.open(tar_path, mode=f"r:{tar_path.suffix.strip('.')}") as tarf:
  632. tarf.extractall(filter=extract_filter)
  633. for arch, triplet in ARCH_TO_TRIPLET.items():
  634. install_cmd = self.release_info["mingw"]["dependencies"][dep]["install-command"]
  635. extra_configure_data = {
  636. "ARCH": ARCH_TO_GNU_ARCH[arch],
  637. "TRIPLET": triplet,
  638. "PREFIX": str(mingw_deps_path / triplet),
  639. }
  640. install_cmd = configure_text(install_cmd, context=self.get_context(extra_configure_data))
  641. self.executer.run(shlex.split(install_cmd), cwd=str(extract_path))
  642. dep_binpath = mingw_deps_path / triplet / "bin"
  643. assert dep_binpath.is_dir(), f"{dep_binpath} for PATH should exist"
  644. dep_pkgconfig = mingw_deps_path / triplet / "lib/pkgconfig"
  645. assert dep_pkgconfig.is_dir(), f"{dep_pkgconfig} for PKG_CONFIG_PATH should exist"
  646. new_env["PATH"] = os.pathsep.join([str(dep_binpath), new_env["PATH"]])
  647. new_env["PKG_CONFIG_PATH"] = str(dep_pkgconfig)
  648. cmake_prefix_paths.append(mingw_deps_path)
  649. new_env["CFLAGS"] = f"-O2 -ffile-prefix-map={self.root}=/src/{self.project}"
  650. new_env["CXXFLAGS"] = f"-O2 -ffile-prefix-map={self.root}=/src/{self.project}"
  651. assert any(system in self.release_info["mingw"] for system in ("autotools", "cmake"))
  652. assert not all(system in self.release_info["mingw"] for system in ("autotools", "cmake"))
  653. mingw_archs = set()
  654. arc_root = f"{self.project}-{self.version}"
  655. archive_file_tree = ArchiveFileTree()
  656. if "autotools" in self.release_info["mingw"]:
  657. for arch in self.release_info["mingw"]["autotools"]["archs"]:
  658. triplet = ARCH_TO_TRIPLET[arch]
  659. new_env["CC"] = f"{triplet}-gcc"
  660. new_env["CXX"] = f"{triplet}-g++"
  661. new_env["RC"] = f"{triplet}-windres"
  662. assert arch not in mingw_archs
  663. mingw_archs.add(arch)
  664. build_path = build_parent_dir / f"build-{triplet}"
  665. install_path = build_parent_dir / f"install-{triplet}"
  666. shutil.rmtree(install_path, ignore_errors=True)
  667. build_path.mkdir(parents=True, exist_ok=True)
  668. context = self.get_context({
  669. "ARCH": arch,
  670. "DEP_PREFIX": str(mingw_deps_path / triplet),
  671. })
  672. extra_args = configure_text_list(text_list=self.release_info["mingw"]["autotools"]["args"], context=context)
  673. with self.section_printer.group(f"Configuring MinGW {triplet} (autotools)"):
  674. assert "@" not in " ".join(extra_args), f"@ should not be present in extra arguments ({extra_args})"
  675. self.executer.run([
  676. self.root / "configure",
  677. f"--prefix={install_path}",
  678. f"--includedir=${{prefix}}/include",
  679. f"--libdir=${{prefix}}/lib",
  680. f"--bindir=${{prefix}}/bin",
  681. f"--host={triplet}",
  682. f"--build=x86_64-none-linux-gnu",
  683. "CFLAGS=-O2",
  684. "CXXFLAGS=-O2",
  685. "LDFLAGS=-Wl,-s",
  686. ] + extra_args, cwd=build_path, env=new_env)
  687. with self.section_printer.group(f"Build MinGW {triplet} (autotools)"):
  688. self.executer.run(["make", f"-j{self.cpu_count}"], cwd=build_path, env=new_env)
  689. with self.section_printer.group(f"Install MinGW {triplet} (autotools)"):
  690. self.executer.run(["make", "install"], cwd=build_path, env=new_env)
  691. self.verify_mingw_library(triplet=ARCH_TO_TRIPLET[arch], path=install_path / "bin" / f"{self.project}.dll")
  692. archive_file_tree.add_directory_tree(arc_dir=arc_join(arc_root, triplet), path=install_path, time=self.arc_time)
  693. print("Recording arch-dependent extra files for MinGW development archive ...")
  694. extra_context = {
  695. "TRIPLET": ARCH_TO_TRIPLET[arch],
  696. }
  697. archive_file_tree.add_file_mapping(arc_dir=arc_root, file_mapping=self.release_info["mingw"]["autotools"].get("files", {}), file_mapping_root=self.root, context=self.get_context(extra_context=extra_context), time=self.arc_time)
  698. if "cmake" in self.release_info["mingw"]:
  699. assert self.release_info["mingw"]["cmake"]["shared-static"] in ("args", "both")
  700. for arch in self.release_info["mingw"]["cmake"]["archs"]:
  701. triplet = ARCH_TO_TRIPLET[arch]
  702. new_env["CC"] = f"{triplet}-gcc"
  703. new_env["CXX"] = f"{triplet}-g++"
  704. new_env["RC"] = f"{triplet}-windres"
  705. assert arch not in mingw_archs
  706. mingw_archs.add(arch)
  707. context = self.get_context({
  708. "ARCH": arch,
  709. "DEP_PREFIX": str(mingw_deps_path / triplet),
  710. })
  711. extra_args = configure_text_list(text_list=self.release_info["mingw"]["cmake"]["args"], context=context)
  712. build_path = build_parent_dir / f"build-{triplet}"
  713. install_path = build_parent_dir / f"install-{triplet}"
  714. shutil.rmtree(install_path, ignore_errors=True)
  715. build_path.mkdir(parents=True, exist_ok=True)
  716. if self.release_info["mingw"]["cmake"]["shared-static"] == "args":
  717. args_for_shared_static = ([], )
  718. elif self.release_info["mingw"]["cmake"]["shared-static"] == "both":
  719. args_for_shared_static = (["-DBUILD_SHARED_LIBS=ON"], ["-DBUILD_SHARED_LIBS=OFF"])
  720. for arg_for_shared_static in args_for_shared_static:
  721. with self.section_printer.group(f"Configuring MinGW {triplet} (CMake)"):
  722. assert "@" not in " ".join(extra_args), f"@ should not be present in extra arguments ({extra_args})"
  723. self.executer.run([
  724. f"cmake",
  725. f"-S", str(self.root), "-B", str(build_path),
  726. f"-DCMAKE_BUILD_TYPE={build_type}",
  727. f'''-DCMAKE_C_FLAGS="-ffile-prefix-map={self.root}=/src/{self.project}"''',
  728. f'''-DCMAKE_CXX_FLAGS="-ffile-prefix-map={self.root}=/src/{self.project}"''',
  729. f"-DCMAKE_PREFIX_PATH={mingw_deps_path / triplet}",
  730. f"-DCMAKE_INSTALL_PREFIX={install_path}",
  731. f"-DCMAKE_INSTALL_INCLUDEDIR=include",
  732. f"-DCMAKE_INSTALL_LIBDIR=lib",
  733. f"-DCMAKE_INSTALL_BINDIR=bin",
  734. f"-DCMAKE_INSTALL_DATAROOTDIR=share",
  735. f"-DCMAKE_TOOLCHAIN_FILE={self.root}/build-scripts/cmake-toolchain-mingw64-{ARCH_TO_GNU_ARCH[arch]}.cmake",
  736. f"-G{self.cmake_generator}",
  737. ] + extra_args + ([] if self.fast else ["--fresh"]) + arg_for_shared_static, cwd=build_path, env=new_env)
  738. with self.section_printer.group(f"Build MinGW {triplet} (CMake)"):
  739. self.executer.run(["cmake", "--build", str(build_path), "--verbose", "--config", build_type], cwd=build_path, env=new_env)
  740. with self.section_printer.group(f"Install MinGW {triplet} (CMake)"):
  741. self.executer.run(["cmake", "--install", str(build_path)], cwd=build_path, env=new_env)
  742. self.verify_mingw_library(triplet=ARCH_TO_TRIPLET[arch], path=install_path / "bin" / f"{self.project}.dll")
  743. archive_file_tree.add_directory_tree(arc_dir=arc_join(arc_root, triplet), path=install_path, time=self.arc_time)
  744. print("Recording arch-dependent extra files for MinGW development archive ...")
  745. extra_context = {
  746. "TRIPLET": ARCH_TO_TRIPLET[arch],
  747. }
  748. archive_file_tree.add_file_mapping(arc_dir=arc_root, file_mapping=self.release_info["mingw"]["cmake"].get("files", {}), file_mapping_root=self.root, context=self.get_context(extra_context=extra_context), time=self.arc_time)
  749. print("... done")
  750. print("Recording extra files for MinGW development archive ...")
  751. archive_file_tree.add_file_mapping(arc_dir=arc_root, file_mapping=self.release_info["mingw"].get("files", {}), file_mapping_root=self.root, context=self.get_context(), time=self.arc_time)
  752. print("... done")
  753. print("Creating zip/tgz/txz development archives ...")
  754. zip_path = self.dist_path / f"{self.project}-devel-{self.version}-mingw.zip"
  755. tgz_path = self.dist_path / f"{self.project}-devel-{self.version}-mingw.tar.gz"
  756. txz_path = self.dist_path / f"{self.project}-devel-{self.version}-mingw.tar.xz"
  757. with Archiver(zip_path=zip_path, tgz_path=tgz_path, txz_path=txz_path) as archiver:
  758. archive_file_tree.add_to_archiver(archive_base="", archiver=archiver)
  759. archiver.add_git_hash(arcdir=arc_root, commit=self.commit, time=self.arc_time)
  760. print("... done")
  761. self.artifacts["mingw-devel-zip"] = zip_path
  762. self.artifacts["mingw-devel-tar-gz"] = tgz_path
  763. self.artifacts["mingw-devel-tar-xz"] = txz_path
  764. def _detect_android_api(self, android_home: str) -> typing.Optional[AndroidApiVersion]:
  765. platform_dirs = list(Path(p) for p in glob.glob(f"{android_home}/platforms/android-*"))
  766. re_platform = re.compile("^android-([0-9]+)(?:-ext([0-9]+))?$")
  767. platform_versions: list[AndroidApiVersion] = []
  768. for platform_dir in platform_dirs:
  769. logger.debug("Found Android Platform SDK: %s", platform_dir)
  770. if not (platform_dir / "android.jar").is_file():
  771. logger.debug("Skipping SDK, missing android.jar")
  772. continue
  773. if m:= re_platform.match(platform_dir.name):
  774. platform_versions.append(AndroidApiVersion(name=platform_dir.name, ints=(int(m.group(1)), int(m.group(2) or 0))))
  775. platform_versions.sort(key=lambda v: v.ints)
  776. logger.info("Available platform versions: %s", platform_versions)
  777. platform_versions = list(filter(lambda v: v.ints >= self._android_api_minimum.ints, platform_versions))
  778. logger.info("Valid platform versions (>=%s): %s", self._android_api_minimum.ints, platform_versions)
  779. if not platform_versions:
  780. return None
  781. android_api = platform_versions[0]
  782. logger.info("Selected API version %s", android_api)
  783. return android_api
  784. def _get_prefab_json_text(self) -> str:
  785. return textwrap.dedent(f"""\
  786. {{
  787. "schema_version": 2,
  788. "name": "{self.project}",
  789. "version": "{self.version}",
  790. "dependencies": []
  791. }}
  792. """)
  793. def _get_prefab_module_json_text(self, library_name: typing.Optional[str], export_libraries: list[str]) -> str:
  794. for lib in export_libraries:
  795. assert isinstance(lib, str), f"{lib} must be a string"
  796. module_json_dict = {
  797. "export_libraries": export_libraries,
  798. }
  799. if library_name:
  800. module_json_dict["library_name"] = f"lib{library_name}"
  801. return json.dumps(module_json_dict, indent=4)
  802. @property
  803. def _android_api_minimum(self) -> AndroidApiVersion:
  804. value = self.release_info["android"]["api-minimum"]
  805. if isinstance(value, int):
  806. ints = (value, )
  807. elif isinstance(value, str):
  808. ints = tuple(split("."))
  809. else:
  810. raise ValueError("Invalid android.api-minimum: must be X or X.Y")
  811. match len(ints):
  812. case 1: name = f"android-{ints[0]}"
  813. case 2: name = f"android-{ints[0]}-ext-{ints[1]}"
  814. case _: raise ValueError("Invalid android.api-minimum: must be X or X.Y")
  815. return AndroidApiVersion(name=name, ints=ints)
  816. @property
  817. def _android_api_target(self):
  818. return self.release_info["android"]["api-target"]
  819. @property
  820. def _android_ndk_minimum(self):
  821. return self.release_info["android"]["ndk-minimum"]
  822. def _get_prefab_abi_json_text(self, abi: str, cpp: bool, shared: bool) -> str:
  823. abi_json_dict = {
  824. "abi": abi,
  825. "api": self._android_api_minimum.ints[0],
  826. "ndk": self._android_ndk_minimum,
  827. "stl": "c++_shared" if cpp else "none",
  828. "static": not shared,
  829. }
  830. return json.dumps(abi_json_dict, indent=4)
  831. def _get_android_manifest_text(self) -> str:
  832. return textwrap.dedent(f"""\
  833. <manifest
  834. xmlns:android="http://schemas.android.com/apk/res/android"
  835. package="org.libsdl.android.{self.project}" android:versionCode="1"
  836. android:versionName="1.0">
  837. <uses-sdk android:minSdkVersion="{self._android_api_minimum.ints[0]}"
  838. android:targetSdkVersion="{self._android_api_target}" />
  839. </manifest>
  840. """)
  841. def create_android_archives(self, android_api: int, android_home: Path, android_ndk_home: Path) -> None:
  842. cmake_toolchain_file = Path(android_ndk_home) / "build/cmake/android.toolchain.cmake"
  843. if not cmake_toolchain_file.exists():
  844. logger.error("CMake toolchain file does not exist (%s)", cmake_toolchain_file)
  845. raise SystemExit(1)
  846. aar_path = self.root / "build-android" / f"{self.project}-{self.version}.aar"
  847. android_dist_path = self.dist_path / f"{self.project}-devel-{self.version}-android.zip"
  848. android_abis = self.release_info["android"]["abis"]
  849. java_jars_added = False
  850. module_data_added = False
  851. android_deps_path = self.deps_path / "android-deps"
  852. shutil.rmtree(android_deps_path, ignore_errors=True)
  853. for dep, depinfo in self.release_info["android"].get("dependencies", {}).items():
  854. dep_devel_zip = self.deps_path / glob.glob(depinfo["artifact"], root_dir=self.deps_path)[0]
  855. dep_extract_path = self.deps_path / f"extract/android/{dep}"
  856. shutil.rmtree(dep_extract_path, ignore_errors=True)
  857. dep_extract_path.mkdir(parents=True, exist_ok=True)
  858. with self.section_printer.group(f"Extracting Android dependency {dep} ({dep_devel_zip})"):
  859. with zipfile.ZipFile(dep_devel_zip, "r") as zf:
  860. zf.extractall(dep_extract_path)
  861. dep_devel_aar = dep_extract_path / glob.glob("*.aar", root_dir=dep_extract_path)[0]
  862. self.executer.run([sys.executable, str(dep_devel_aar), "-o", str(android_deps_path)])
  863. for module_name, module_info in self.release_info["android"]["modules"].items():
  864. assert "type" in module_info and module_info["type"] in ("interface", "library"), f"module {module_name} must have a valid type"
  865. aar_file_tree = ArchiveFileTree()
  866. android_devel_file_tree = ArchiveFileTree()
  867. for android_abi in android_abis:
  868. with self.section_printer.group(f"Building for Android {android_api} {android_abi}"):
  869. build_dir = self.root / "build-android" / f"{android_abi}-build"
  870. install_dir = self.root / "install-android" / f"{android_abi}-install"
  871. shutil.rmtree(install_dir, ignore_errors=True)
  872. assert not install_dir.is_dir(), f"{install_dir} should not exist prior to build"
  873. build_type = "Release"
  874. cmake_args = [
  875. "cmake",
  876. "-S", str(self.root),
  877. "-B", str(build_dir),
  878. f'''-DCMAKE_C_FLAGS="-ffile-prefix-map={self.root}=/src/{self.project}"''',
  879. f'''-DCMAKE_CXX_FLAGS="-ffile-prefix-map={self.root}=/src/{self.project}"''',
  880. f"-DCMAKE_TOOLCHAIN_FILE={cmake_toolchain_file}",
  881. f"-DCMAKE_PREFIX_PATH={str(android_deps_path)}",
  882. f"-DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH",
  883. f"-DANDROID_HOME={android_home}",
  884. f"-DANDROID_PLATFORM={android_api}",
  885. f"-DANDROID_ABI={android_abi}",
  886. "-DCMAKE_POSITION_INDEPENDENT_CODE=ON",
  887. f"-DCMAKE_INSTALL_PREFIX={install_dir}",
  888. "-DCMAKE_INSTALL_INCLUDEDIR=include ",
  889. "-DCMAKE_INSTALL_LIBDIR=lib",
  890. "-DCMAKE_INSTALL_DATAROOTDIR=share",
  891. f"-DCMAKE_BUILD_TYPE={build_type}",
  892. f"-G{self.cmake_generator}",
  893. ] + self.release_info["android"]["cmake"]["args"] + ([] if self.fast else ["--fresh"])
  894. build_args = [
  895. "cmake",
  896. "--build", str(build_dir),
  897. "--verbose",
  898. "--config", build_type,
  899. ]
  900. install_args = [
  901. "cmake",
  902. "--install", str(build_dir),
  903. "--config", build_type,
  904. ]
  905. self.executer.run(cmake_args)
  906. self.executer.run(build_args)
  907. self.executer.run(install_args)
  908. for module_name, module_info in self.release_info["android"]["modules"].items():
  909. arcdir_prefab_module = f"prefab/modules/{module_name}"
  910. if module_info["type"] == "library":
  911. library = install_dir / module_info["library"]
  912. assert library.suffix in (".so", ".a")
  913. assert library.is_file(), f"CMake should have built library '{library}' for module {module_name}"
  914. arcdir_prefab_libs = f"{arcdir_prefab_module}/libs/android.{android_abi}"
  915. aar_file_tree.add_file(NodeInArchive.from_fs(arcpath=f"{arcdir_prefab_libs}/{library.name}", path=library, time=self.arc_time))
  916. aar_file_tree.add_file(NodeInArchive.from_text(arcpath=f"{arcdir_prefab_libs}/abi.json", text=self._get_prefab_abi_json_text(abi=android_abi, cpp=False, shared=library.suffix == ".so"), time=self.arc_time))
  917. if not module_data_added:
  918. library_name = None
  919. if module_info["type"] == "library":
  920. library_name = Path(module_info["library"]).stem.removeprefix("lib")
  921. export_libraries = module_info.get("export-libraries", [])
  922. aar_file_tree.add_file(NodeInArchive.from_text(arcpath=arc_join(arcdir_prefab_module, "module.json"), text=self._get_prefab_module_json_text(library_name=library_name, export_libraries=export_libraries), time=self.arc_time))
  923. arcdir_prefab_include = f"prefab/modules/{module_name}/include"
  924. if "includes" in module_info:
  925. aar_file_tree.add_file_mapping(arc_dir=arcdir_prefab_include, file_mapping=module_info["includes"], file_mapping_root=install_dir, context=self.get_context(), time=self.arc_time)
  926. else:
  927. aar_file_tree.add_file(NodeInArchive.from_text(arcpath=arc_join(arcdir_prefab_include, ".keep"), text="\n", time=self.arc_time))
  928. module_data_added = True
  929. if not java_jars_added:
  930. java_jars_added = True
  931. if "jars" in self.release_info["android"]:
  932. classes_jar_path = install_dir / configure_text(text=self.release_info["android"]["jars"]["classes"], context=self.get_context())
  933. sources_jar_path = install_dir / configure_text(text=self.release_info["android"]["jars"]["sources"], context=self.get_context())
  934. doc_jar_path = install_dir / configure_text(text=self.release_info["android"]["jars"]["doc"], context=self.get_context())
  935. assert classes_jar_path.is_file(), f"CMake should have compiled the java sources and archived them into a JAR ({classes_jar_path})"
  936. assert sources_jar_path.is_file(), f"CMake should have archived the java sources into a JAR ({sources_jar_path})"
  937. assert doc_jar_path.is_file(), f"CMake should have archived javadoc into a JAR ({doc_jar_path})"
  938. aar_file_tree.add_file(NodeInArchive.from_fs(arcpath="classes.jar", path=classes_jar_path, time=self.arc_time))
  939. aar_file_tree.add_file(NodeInArchive.from_fs(arcpath="classes-sources.jar", path=sources_jar_path, time=self.arc_time))
  940. aar_file_tree.add_file(NodeInArchive.from_fs(arcpath="classes-doc.jar", path=doc_jar_path, time=self.arc_time))
  941. assert ("jars" in self.release_info["android"] and java_jars_added) or "jars" not in self.release_info["android"], "Must have archived java JAR archives"
  942. aar_file_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["android"]["aar-files"], file_mapping_root=self.root, context=self.get_context(), time=self.arc_time)
  943. aar_file_tree.add_file(NodeInArchive.from_text(arcpath="prefab/prefab.json", text=self._get_prefab_json_text(), time=self.arc_time))
  944. aar_file_tree.add_file(NodeInArchive.from_text(arcpath="AndroidManifest.xml", text=self._get_android_manifest_text(), time=self.arc_time))
  945. with Archiver(zip_path=aar_path) as archiver:
  946. aar_file_tree.add_to_archiver(archive_base="", archiver=archiver)
  947. archiver.add_git_hash(arcdir="", commit=self.commit, time=self.arc_time)
  948. android_devel_file_tree.add_file(NodeInArchive.from_fs(arcpath=aar_path.name, path=aar_path))
  949. android_devel_file_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["android"]["files"], file_mapping_root=self.root, context=self.get_context(), time=self.arc_time)
  950. with Archiver(zip_path=android_dist_path) as archiver:
  951. android_devel_file_tree.add_to_archiver(archive_base="", archiver=archiver)
  952. archiver.add_git_hash(arcdir="", commit=self.commit, time=self.arc_time)
  953. self.artifacts[f"android-aar"] = android_dist_path
  954. def download_dependencies(self):
  955. shutil.rmtree(self.deps_path, ignore_errors=True)
  956. self.deps_path.mkdir(parents=True)
  957. if self.github:
  958. with open(os.environ["GITHUB_OUTPUT"], "a") as f:
  959. f.write(f"dep-path={self.deps_path.absolute()}\n")
  960. for dep, depinfo in self.release_info.get("dependencies", {}).items():
  961. startswith = depinfo["startswith"]
  962. dep_repo = depinfo["repo"]
  963. # FIXME: dropped "--exclude-pre-releases"
  964. dep_string_data = self.executer.check_output(["gh", "-R", dep_repo, "release", "list", "--exclude-drafts", "--json", "name,createdAt,tagName", "--jq", f'[.[]|select(.name|startswith("{startswith}"))]|max_by(.createdAt)']).strip()
  965. dep_data = json.loads(dep_string_data)
  966. dep_tag = dep_data["tagName"]
  967. dep_version = dep_data["name"]
  968. logger.info("Download dependency %s version %s (tag=%s) ", dep, dep_version, dep_tag)
  969. self.executer.run(["gh", "-R", dep_repo, "release", "download", dep_tag], cwd=self.deps_path)
  970. if self.github:
  971. with open(os.environ["GITHUB_OUTPUT"], "a") as f:
  972. f.write(f"dep-{dep.lower()}-version={dep_version}\n")
  973. def verify_dependencies(self):
  974. for dep, depinfo in self.release_info.get("dependencies", {}).items():
  975. if "mingw" in self.release_info:
  976. mingw_matches = glob.glob(self.release_info["mingw"]["dependencies"][dep]["artifact"], root_dir=self.deps_path)
  977. assert len(mingw_matches) == 1, f"Exactly one archive matches mingw {dep} dependency: {mingw_matches}"
  978. if "dmg" in self.release_info:
  979. dmg_matches = glob.glob(self.release_info["dmg"]["dependencies"][dep]["artifact"], root_dir=self.deps_path)
  980. assert len(dmg_matches) == 1, f"Exactly one archive matches dmg {dep} dependency: {dmg_matches}"
  981. if "msvc" in self.release_info:
  982. msvc_matches = glob.glob(self.release_info["msvc"]["dependencies"][dep]["artifact"], root_dir=self.deps_path)
  983. assert len(msvc_matches) == 1, f"Exactly one archive matches msvc {dep} dependency: {msvc_matches}"
  984. if "android" in self.release_info:
  985. android_matches = glob.glob(self.release_info["android"]["dependencies"][dep]["artifact"], root_dir=self.deps_path)
  986. assert len(android_matches) == 1, f"Exactly one archive matches msvc {dep} dependency: {android_matches}"
  987. @staticmethod
  988. def _arch_to_vs_platform(arch: str, configuration: str="Release") -> VsArchPlatformConfig:
  989. ARCH_TO_VS_PLATFORM = {
  990. "x86": VsArchPlatformConfig(arch="x86", platform="Win32", configuration=configuration),
  991. "x64": VsArchPlatformConfig(arch="x64", platform="x64", configuration=configuration),
  992. "arm64": VsArchPlatformConfig(arch="arm64", platform="ARM64", configuration=configuration),
  993. }
  994. return ARCH_TO_VS_PLATFORM[arch]
  995. def build_msvc(self):
  996. with self.section_printer.group("Find Visual Studio"):
  997. vs = VisualStudio(executer=self.executer)
  998. for arch in self.release_info["msvc"].get("msbuild", {}).get("archs", []):
  999. self._build_msvc_msbuild(arch_platform=self._arch_to_vs_platform(arch=arch), vs=vs)
  1000. if "cmake" in self.release_info["msvc"]:
  1001. deps_path = self.root / "msvc-deps"
  1002. shutil.rmtree(deps_path, ignore_errors=True)
  1003. dep_roots = []
  1004. for dep, depinfo in self.release_info["msvc"].get("dependencies", {}).items():
  1005. dep_extract_path = deps_path / f"extract-{dep}"
  1006. msvc_zip = self.deps_path / glob.glob(depinfo["artifact"], root_dir=self.deps_path)[0]
  1007. with zipfile.ZipFile(msvc_zip, "r") as zf:
  1008. zf.extractall(dep_extract_path)
  1009. contents_msvc_zip = glob.glob(str(dep_extract_path / "*"))
  1010. assert len(contents_msvc_zip) == 1, f"There must be exactly one root item in the root directory of {dep}"
  1011. dep_roots.append(contents_msvc_zip[0])
  1012. for arch in self.release_info["msvc"].get("cmake", {}).get("archs", []):
  1013. self._build_msvc_cmake(arch_platform=self._arch_to_vs_platform(arch=arch), dep_roots=dep_roots)
  1014. with self.section_printer.group("Create SDL VC development zip"):
  1015. self._build_msvc_devel()
  1016. def _build_msvc_msbuild(self, arch_platform: VsArchPlatformConfig, vs: VisualStudio):
  1017. platform_context = self.get_context(arch_platform.extra_context())
  1018. for dep, depinfo in self.release_info["msvc"].get("dependencies", {}).items():
  1019. msvc_zip = self.deps_path / glob.glob(depinfo["artifact"], root_dir=self.deps_path)[0]
  1020. src_globs = [configure_text(instr["src"], context=platform_context) for instr in depinfo["copy"]]
  1021. with zipfile.ZipFile(msvc_zip, "r") as zf:
  1022. for member in zf.namelist():
  1023. member_path = "/".join(Path(member).parts[1:])
  1024. for src_i, src_glob in enumerate(src_globs):
  1025. if fnmatch.fnmatch(member_path, src_glob):
  1026. dst = (self.root / configure_text(depinfo["copy"][src_i]["dst"], context=platform_context)).resolve() / Path(member_path).name
  1027. zip_data = zf.read(member)
  1028. if dst.exists():
  1029. identical = False
  1030. if dst.is_file():
  1031. orig_bytes = dst.read_bytes()
  1032. if orig_bytes == zip_data:
  1033. identical = True
  1034. if not identical:
  1035. logger.warning("Extracting dependency %s, will cause %s to be overwritten", dep, dst)
  1036. if not self.overwrite:
  1037. raise RuntimeError("Run with --overwrite to allow overwriting")
  1038. logger.debug("Extracting %s -> %s", member, dst)
  1039. dst.parent.mkdir(exist_ok=True, parents=True)
  1040. dst.write_bytes(zip_data)
  1041. prebuilt_paths = set(self.root / full_prebuilt_path for prebuilt_path in self.release_info["msvc"]["msbuild"].get("prebuilt", []) for full_prebuilt_path in glob.glob(configure_text(prebuilt_path, context=platform_context), root_dir=self.root))
  1042. msbuild_paths = set(self.root / configure_text(f, context=platform_context) for file_mapping in (self.release_info["msvc"]["msbuild"]["files-lib"], self.release_info["msvc"]["msbuild"]["files-devel"]) for files_list in file_mapping.values() for f in files_list)
  1043. assert prebuilt_paths.issubset(msbuild_paths), f"msvc.msbuild.prebuilt must be a subset of (msvc.msbuild.files-lib, msvc.msbuild.files-devel)"
  1044. built_paths = msbuild_paths.difference(prebuilt_paths)
  1045. logger.info("MSbuild builds these files, to be included in the package: %s", built_paths)
  1046. if not self.fast:
  1047. for b in built_paths:
  1048. b.unlink(missing_ok=True)
  1049. rel_projects: list[str] = self.release_info["msvc"]["msbuild"]["projects"]
  1050. projects = list(self.root / p for p in rel_projects)
  1051. directory_build_props_src_relpath = self.release_info["msvc"]["msbuild"].get("directory-build-props")
  1052. for project in projects:
  1053. dir_b_props = project.parent / "Directory.Build.props"
  1054. dir_b_props.unlink(missing_ok = True)
  1055. if directory_build_props_src_relpath:
  1056. src = self.root / directory_build_props_src_relpath
  1057. logger.debug("Copying %s -> %s", src, dir_b_props)
  1058. shutil.copy(src=src, dst=dir_b_props)
  1059. with self.section_printer.group(f"Build {arch_platform.arch} VS binary"):
  1060. vs.build(arch_platform=arch_platform, projects=projects)
  1061. if self.dry:
  1062. for b in built_paths:
  1063. b.parent.mkdir(parents=True, exist_ok=True)
  1064. b.touch()
  1065. for b in built_paths:
  1066. assert b.is_file(), f"{b} has not been created"
  1067. b.parent.mkdir(parents=True, exist_ok=True)
  1068. b.touch()
  1069. zip_path = self.dist_path / f"{self.project}-{self.version}-win32-{arch_platform.arch}.zip"
  1070. zip_path.unlink(missing_ok=True)
  1071. logger.info("Collecting files...")
  1072. archive_file_tree = ArchiveFileTree()
  1073. archive_file_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["msvc"]["msbuild"]["files-lib"], file_mapping_root=self.root, context=platform_context, time=self.arc_time)
  1074. archive_file_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["msvc"]["files-lib"], file_mapping_root=self.root, context=platform_context, time=self.arc_time)
  1075. logger.info("Writing to %s", zip_path)
  1076. with Archiver(zip_path=zip_path) as archiver:
  1077. arc_root = f""
  1078. archive_file_tree.add_to_archiver(archive_base=arc_root, archiver=archiver)
  1079. archiver.add_git_hash(arcdir=arc_root, commit=self.commit, time=self.arc_time)
  1080. self.artifacts[f"VC-{arch_platform.arch}"] = zip_path
  1081. for p in built_paths:
  1082. assert p.is_file(), f"{p} should exist"
  1083. def _arch_platform_to_build_path(self, arch_platform: VsArchPlatformConfig) -> Path:
  1084. return self.root / f"build-vs-{arch_platform.arch}"
  1085. def _arch_platform_to_install_path(self, arch_platform: VsArchPlatformConfig) -> Path:
  1086. return self._arch_platform_to_build_path(arch_platform) / "prefix"
  1087. def _build_msvc_cmake(self, arch_platform: VsArchPlatformConfig, dep_roots: list[Path]):
  1088. build_path = self._arch_platform_to_build_path(arch_platform)
  1089. install_path = self._arch_platform_to_install_path(arch_platform)
  1090. platform_context = self.get_context(extra_context=arch_platform.extra_context())
  1091. build_type = "Release"
  1092. extra_context = {
  1093. "ARCH": arch_platform.arch,
  1094. "PLATFORM": arch_platform.platform,
  1095. }
  1096. built_paths = set(install_path / configure_text(f, context=platform_context) for file_mapping in (self.release_info["msvc"]["cmake"]["files-lib"], self.release_info["msvc"]["cmake"]["files-devel"]) for files_list in file_mapping.values() for f in files_list)
  1097. logger.info("CMake builds these files, to be included in the package: %s", built_paths)
  1098. if not self.fast:
  1099. for b in built_paths:
  1100. b.unlink(missing_ok=True)
  1101. shutil.rmtree(install_path, ignore_errors=True)
  1102. build_path.mkdir(parents=True, exist_ok=True)
  1103. with self.section_printer.group(f"Configure VC CMake project for {arch_platform.arch}"):
  1104. self.executer.run([
  1105. "cmake", "-S", str(self.root), "-B", str(build_path),
  1106. "-A", arch_platform.platform,
  1107. "-DCMAKE_INSTALL_BINDIR=bin",
  1108. "-DCMAKE_INSTALL_DATAROOTDIR=share",
  1109. "-DCMAKE_INSTALL_INCLUDEDIR=include",
  1110. "-DCMAKE_INSTALL_LIBDIR=lib",
  1111. f"-DCMAKE_BUILD_TYPE={build_type}",
  1112. f"-DCMAKE_INSTALL_PREFIX={install_path}",
  1113. # MSVC debug information format flags are selected by an abstraction
  1114. "-DCMAKE_POLICY_DEFAULT_CMP0141=NEW",
  1115. # MSVC debug information format
  1116. "-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=ProgramDatabase",
  1117. # Linker flags for executables
  1118. "-DCMAKE_EXE_LINKER_FLAGS=-INCREMENTAL:NO -DEBUG -OPT:REF -OPT:ICF",
  1119. # Linker flag for shared libraries
  1120. "-DCMAKE_SHARED_LINKER_FLAGS=-INCREMENTAL:NO -DEBUG -OPT:REF -OPT:ICF",
  1121. # MSVC runtime library flags are selected by an abstraction
  1122. "-DCMAKE_POLICY_DEFAULT_CMP0091=NEW",
  1123. # Use statically linked runtime (-MT) (ideally, should be "MultiThreaded$<$<CONFIG:Debug>:Debug>")
  1124. "-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded",
  1125. f"-DCMAKE_PREFIX_PATH={';'.join(str(s) for s in dep_roots)}",
  1126. ] + self.release_info["msvc"]["cmake"]["args"] + ([] if self.fast else ["--fresh"]))
  1127. with self.section_printer.group(f"Build VC CMake project for {arch_platform.arch}"):
  1128. self.executer.run(["cmake", "--build", str(build_path), "--verbose", "--config", build_type])
  1129. with self.section_printer.group(f"Install VC CMake project for {arch_platform.arch}"):
  1130. self.executer.run(["cmake", "--install", str(build_path), "--config", build_type])
  1131. if self.dry:
  1132. for b in built_paths:
  1133. b.parent.mkdir(parents=True, exist_ok=True)
  1134. b.touch()
  1135. zip_path = self.dist_path / f"{self.project}-{self.version}-win32-{arch_platform.arch}.zip"
  1136. zip_path.unlink(missing_ok=True)
  1137. logger.info("Collecting files...")
  1138. archive_file_tree = ArchiveFileTree()
  1139. archive_file_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["msvc"]["cmake"]["files-lib"], file_mapping_root=install_path, context=platform_context, time=self.arc_time)
  1140. archive_file_tree.add_file_mapping(arc_dir="", file_mapping=self.release_info["msvc"]["files-lib"], file_mapping_root=self.root, context=self.get_context(extra_context=extra_context), time=self.arc_time)
  1141. logger.info("Creating %s", zip_path)
  1142. with Archiver(zip_path=zip_path) as archiver:
  1143. arc_root = f""
  1144. archive_file_tree.add_to_archiver(archive_base=arc_root, archiver=archiver)
  1145. archiver.add_git_hash(arcdir=arc_root, commit=self.commit, time=self.arc_time)
  1146. for p in built_paths:
  1147. assert p.is_file(), f"{p} should exist"
  1148. def _build_msvc_devel(self) -> None:
  1149. zip_path = self.dist_path / f"{self.project}-devel-{self.version}-VC.zip"
  1150. arc_root = f"{self.project}-{self.version}"
  1151. def copy_files_devel(ctx):
  1152. archive_file_tree.add_file_mapping(arc_dir=arc_root, file_mapping=self.release_info["msvc"]["files-devel"], file_mapping_root=self.root, context=ctx, time=self.arc_time)
  1153. logger.info("Collecting files...")
  1154. archive_file_tree = ArchiveFileTree()
  1155. if "msbuild" in self.release_info["msvc"]:
  1156. for arch in self.release_info["msvc"]["msbuild"]["archs"]:
  1157. arch_platform = self._arch_to_vs_platform(arch=arch)
  1158. platform_context = self.get_context(arch_platform.extra_context())
  1159. archive_file_tree.add_file_mapping(arc_dir=arc_root, file_mapping=self.release_info["msvc"]["msbuild"]["files-devel"], file_mapping_root=self.root, context=platform_context, time=self.arc_time)
  1160. copy_files_devel(ctx=platform_context)
  1161. if "cmake" in self.release_info["msvc"]:
  1162. for arch in self.release_info["msvc"]["cmake"]["archs"]:
  1163. arch_platform = self._arch_to_vs_platform(arch=arch)
  1164. platform_context = self.get_context(arch_platform.extra_context())
  1165. archive_file_tree.add_file_mapping(arc_dir=arc_root, file_mapping=self.release_info["msvc"]["cmake"]["files-devel"], file_mapping_root=self._arch_platform_to_install_path(arch_platform), context=platform_context, time=self.arc_time)
  1166. copy_files_devel(ctx=platform_context)
  1167. with Archiver(zip_path=zip_path) as archiver:
  1168. archive_file_tree.add_to_archiver(archive_base="", archiver=archiver)
  1169. archiver.add_git_hash(arcdir=arc_root, commit=self.commit, time=self.arc_time)
  1170. self.artifacts["VC-devel"] = zip_path
  1171. @classmethod
  1172. def extract_sdl_version(cls, root: Path, release_info: dict) -> str:
  1173. with open(root / release_info["version"]["file"], "r") as f:
  1174. text = f.read()
  1175. major = next(re.finditer(release_info["version"]["re_major"], text, flags=re.M)).group(1)
  1176. minor = next(re.finditer(release_info["version"]["re_minor"], text, flags=re.M)).group(1)
  1177. micro = next(re.finditer(release_info["version"]["re_micro"], text, flags=re.M)).group(1)
  1178. return f"{major}.{minor}.{micro}"
  1179. def main(argv=None) -> int:
  1180. if sys.version_info < (3, 11):
  1181. logger.error("This script needs at least python 3.11")
  1182. return 1
  1183. parser = argparse.ArgumentParser(allow_abbrev=False, description="Create SDL release artifacts")
  1184. parser.add_argument("--root", metavar="DIR", type=Path, default=Path(__file__).absolute().parents[1], help="Root of project")
  1185. parser.add_argument("--release-info", metavar="JSON", dest="path_release_info", type=Path, default=Path(__file__).absolute().parent / "release-info.json", help="Path of release-info.json")
  1186. parser.add_argument("--dependency-folder", metavar="FOLDER", dest="deps_path", type=Path, default="deps", help="Directory containing pre-built archives of dependencies (will be removed when downloading archives)")
  1187. parser.add_argument("--out", "-o", metavar="DIR", dest="dist_path", type=Path, default="dist", help="Output directory")
  1188. parser.add_argument("--github", action="store_true", help="Script is running on a GitHub runner")
  1189. parser.add_argument("--commit", default="HEAD", help="Git commit/tag of which a release should be created")
  1190. parser.add_argument("--actions", choices=["download", "source", "android", "mingw", "msvc", "dmg"], required=True, nargs="+", dest="actions", help="What to do?")
  1191. parser.set_defaults(loglevel=logging.INFO)
  1192. parser.add_argument('--vs-year', dest="vs_year", help="Visual Studio year")
  1193. parser.add_argument('--android-api', dest="android_api", help="Android API version")
  1194. parser.add_argument('--android-home', dest="android_home", default=os.environ.get("ANDROID_HOME"), help="Android Home folder")
  1195. parser.add_argument('--android-ndk-home', dest="android_ndk_home", default=os.environ.get("ANDROID_NDK_HOME"), help="Android NDK Home folder")
  1196. parser.add_argument('--cmake-generator', dest="cmake_generator", default="Ninja", help="CMake Generator")
  1197. parser.add_argument('--debug', action='store_const', const=logging.DEBUG, dest="loglevel", help="Print script debug information")
  1198. parser.add_argument('--dry-run', action='store_true', dest="dry", help="Don't execute anything")
  1199. parser.add_argument('--force', action='store_true', dest="force", help="Ignore a non-clean git tree")
  1200. parser.add_argument('--overwrite', action='store_true', dest="overwrite", help="Allow potentially overwriting other projects")
  1201. parser.add_argument('--fast', action='store_true', dest="fast", help="Don't do a rebuild")
  1202. args = parser.parse_args(argv)
  1203. logging.basicConfig(level=args.loglevel, format='[%(levelname)s] %(message)s')
  1204. args.deps_path = args.deps_path.absolute()
  1205. args.dist_path = args.dist_path.absolute()
  1206. args.root = args.root.absolute()
  1207. args.dist_path = args.dist_path.absolute()
  1208. if args.dry:
  1209. args.dist_path = args.dist_path / "dry"
  1210. if args.github:
  1211. section_printer: SectionPrinter = GitHubSectionPrinter()
  1212. else:
  1213. section_printer = SectionPrinter()
  1214. if args.github and "GITHUB_OUTPUT" not in os.environ:
  1215. os.environ["GITHUB_OUTPUT"] = "/tmp/github_output.txt"
  1216. executer = Executer(root=args.root, dry=args.dry)
  1217. root_git_hash_path = args.root / GIT_HASH_FILENAME
  1218. root_is_maybe_archive = root_git_hash_path.is_file()
  1219. if root_is_maybe_archive:
  1220. logger.warning("%s detected: Building from archive", GIT_HASH_FILENAME)
  1221. archive_commit = root_git_hash_path.read_text().strip()
  1222. if args.commit != archive_commit:
  1223. logger.warning("Commit argument is %s, but archive commit is %s. Using %s.", args.commit, archive_commit, archive_commit)
  1224. args.commit = archive_commit
  1225. revision = (args.root / REVISION_TXT).read_text().strip()
  1226. else:
  1227. args.commit = executer.check_output(["git", "rev-parse", args.commit], dry_out="e5812a9fd2cda317b503325a702ba3c1c37861d9").strip()
  1228. revision = executer.check_output(["git", "describe", "--always", "--tags", "--long", args.commit], dry_out="preview-3.1.3-96-g9512f2144").strip()
  1229. logger.info("Using commit %s", args.commit)
  1230. try:
  1231. with args.path_release_info.open() as f:
  1232. release_info = json.load(f)
  1233. except FileNotFoundError:
  1234. logger.error(f"Could not find {args.path_release_info}")
  1235. releaser = Releaser(
  1236. release_info=release_info,
  1237. commit=args.commit,
  1238. revision=revision,
  1239. root=args.root,
  1240. dist_path=args.dist_path,
  1241. executer=executer,
  1242. section_printer=section_printer,
  1243. cmake_generator=args.cmake_generator,
  1244. deps_path=args.deps_path,
  1245. overwrite=args.overwrite,
  1246. github=args.github,
  1247. fast=args.fast,
  1248. )
  1249. if root_is_maybe_archive:
  1250. logger.warning("Building from archive. Skipping clean git tree check.")
  1251. else:
  1252. porcelain_status = executer.check_output(["git", "status", "--ignored", "--porcelain"], dry_out="\n").strip()
  1253. if porcelain_status:
  1254. print(porcelain_status)
  1255. logger.warning("The tree is dirty! Do not publish any generated artifacts!")
  1256. if not args.force:
  1257. raise Exception("The git repo contains modified and/or non-committed files. Run with --force to ignore.")
  1258. if args.fast:
  1259. logger.warning("Doing fast build! Do not publish generated artifacts!")
  1260. with section_printer.group("Arguments"):
  1261. print(f"project = {releaser.project}")
  1262. print(f"version = {releaser.version}")
  1263. print(f"revision = {revision}")
  1264. print(f"commit = {args.commit}")
  1265. print(f"out = {args.dist_path}")
  1266. print(f"actions = {args.actions}")
  1267. print(f"dry = {args.dry}")
  1268. print(f"force = {args.force}")
  1269. print(f"overwrite = {args.overwrite}")
  1270. print(f"cmake_generator = {args.cmake_generator}")
  1271. releaser.prepare()
  1272. if "download" in args.actions:
  1273. releaser.download_dependencies()
  1274. if set(args.actions).intersection({"msvc", "mingw", "android"}):
  1275. print("Verifying presence of dependencies (run 'download' action to download) ...")
  1276. releaser.verify_dependencies()
  1277. print("... done")
  1278. if "source" in args.actions:
  1279. if root_is_maybe_archive:
  1280. raise Exception("Cannot build source archive from source archive")
  1281. with section_printer.group("Create source archives"):
  1282. releaser.create_source_archives()
  1283. if "dmg" in args.actions:
  1284. if platform.system() != "Darwin" and not args.dry:
  1285. parser.error("framework artifact(s) can only be built on Darwin")
  1286. releaser.create_dmg()
  1287. if "msvc" in args.actions:
  1288. if platform.system() != "Windows" and not args.dry:
  1289. parser.error("msvc artifact(s) can only be built on Windows")
  1290. releaser.build_msvc()
  1291. if "mingw" in args.actions:
  1292. releaser.create_mingw_archives()
  1293. if "android" in args.actions:
  1294. if args.android_home is None or not Path(args.android_home).is_dir():
  1295. parser.error("Invalid $ANDROID_HOME or --android-home: must be a directory containing the Android SDK")
  1296. if args.android_ndk_home is None or not Path(args.android_ndk_home).is_dir():
  1297. parser.error("Invalid $ANDROID_NDK_HOME or --android_ndk_home: must be a directory containing the Android NDK")
  1298. if args.android_api is None:
  1299. with section_printer.group("Detect Android APIS"):
  1300. args.android_api = releaser._detect_android_api(android_home=args.android_home)
  1301. else:
  1302. try:
  1303. android_api_ints = tuple(int(v) for v in args.android_api.split("."))
  1304. match len(android_api_ints):
  1305. case 1: android_api_name = f"android-{android_api_ints[0]}"
  1306. case 2: android_api_name = f"android-{android_api_ints[0]}-ext-{android_api_ints[1]}"
  1307. case _: raise ValueError
  1308. except ValueError:
  1309. logger.error("Invalid --android-api, must be a 'X' or 'X.Y' version")
  1310. args.android_api = AndroidApiVersion(ints=android_api_ints, name=android_api_name)
  1311. if args.android_api is None:
  1312. parser.error("Invalid --android-api, and/or could not be detected")
  1313. android_api_path = Path(args.android_home) / f"platforms/{args.android_api.name}"
  1314. if not android_api_path.is_dir():
  1315. parser.error(f"Android API directory does not exist ({android_api_path})")
  1316. with section_printer.group("Android arguments"):
  1317. print(f"android_home = {args.android_home}")
  1318. print(f"android_ndk_home = {args.android_ndk_home}")
  1319. print(f"android_api = {args.android_api}")
  1320. releaser.create_android_archives(
  1321. android_api=args.android_api.ints[0],
  1322. android_home=args.android_home,
  1323. android_ndk_home=args.android_ndk_home,
  1324. )
  1325. with section_printer.group("Summary"):
  1326. print(f"artifacts = {releaser.artifacts}")
  1327. if args.github:
  1328. with open(os.environ["GITHUB_OUTPUT"], "a") as f:
  1329. f.write(f"project={releaser.project}\n")
  1330. f.write(f"version={releaser.version}\n")
  1331. for k, v in releaser.artifacts.items():
  1332. f.write(f"{k}={v.name}\n")
  1333. return 0
  1334. if __name__ == "__main__":
  1335. raise SystemExit(main())