[Tech] Download helper binaries instead of storing them in the repo

.github/actions/install-deps/action.yml

@@ -1,5 +1,5 @@
 name: 'Prerequisite setup'
-description: 'Installs dependencies (Node itself, node_modules, node-gyp)'
+description: 'Installs dependencies (Node itself, node_modules, node-gyp, helper binaries)'
 runs:
   using: 'composite'
   steps:
@@ -18,3 +18,6 @@ runs:
       shell: bash
       env:
         PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1'
+    - name: Download helper binaries
+      run: pnpm download-helper-binaries
+      shell: bash

.husky/post-checkout

@@ -1,2 +1,3 @@
 #!/bin/bash
 pnpm i
+pnpm download-helper-binaries

electron-builder.yml

@@ -15,6 +15,8 @@ asarUnpack:
   - build/icon-dark.png
   - build/icon-light.png
   - build/webviewPreload.js
+  - build/bin/**/*
+  - '!build/bin/legendary.LICENSE'
 
 electronDownload:
   mirror: https://github.com/castlabs/electron-releases/releases/download/v
@@ -27,11 +29,7 @@ protocols:
 win:
   artifactName: ${productName}-${version}-Setup-${arch}.${ext}
   icon: build/win_icon.ico
-  asarUnpack:
-    - build/bin/win32/legendary.exe
-    - build/bin/win32/gogdl.exe
-    - build/bin/win32/nile.exe
-  files: build/bin/win32/*
+  files: build/bin/*/win32/*
 
 portable:
   artifactName: ${productName}-${version}-Portable-${arch}.${ext}
@@ -46,12 +44,7 @@ mac:
     teamId: DLB2RYLUDX
   extendInfo:
     com.apple.security.cs.allow-jit: true
-  asarUnpack:
-    - build/bin/darwin/legendary
-    - build/bin/darwin/gogdl
-    - build/bin/darwin/nile
-  files:
-    - build/bin/darwin/*
+  files: build/bin/*/darwin/*
 
 dmg:
   background: public/dmg.png
@@ -74,13 +67,7 @@ linux:
   desktop:
     Name: Heroic Games Launcher
     Comment[de]: Ein Open Source Spielelauncher for GOG und Epic Games
-  asarUnpack:
-    - build/bin/linux/legendary
-    - build/bin/linux/gogdl
-    - build/bin/linux/nile
-    - build/bin/linux/vulkan-helper
-  files:
-    - build/bin/linux/*
+  files: build/bin/*/linux/*
 
 snap:
   base: core20

meta/downloadHelperBinaries.ts

@@ -0,0 +1,193 @@
+import { createWriteStream } from 'fs'
+import { chmod, stat, mkdir, readFile, writeFile } from 'fs/promises'
+import { dirname, join } from 'path'
+import { Readable } from 'stream'
+import { finished } from 'stream/promises'
+
+type SupportedPlatform = 'win32' | 'darwin' | 'linux'
+type DownloadedBinary = 'legendary' | 'gogdl' | 'nile'
+
+const RELEASE_TAGS = {
+  legendary: '0.20.35',
+  gogdl: 'v1.1.1',
+  nile: 'v1.1.0'
+} as const satisfies Record<DownloadedBinary, string>
+
+const pathExists = async (path: string): Promise<boolean> =>
+  stat(path).then(
+    () => true,
+    () => false
+  )
+
+async function downloadFile(url: string, dst: string) {
+  const response = await fetch(url, {
+    headers: {
+      'User-Agent': 'HeroicBinaryUpdater/1.0'
+    }
+  })
+  await mkdir(dirname(dst), { recursive: true })
+  const fileStream = createWriteStream(dst, { flags: 'w' })
+  await finished(Readable.fromWeb(response.body).pipe(fileStream))
+}
+
+async function downloadAsset(
+  binaryName: string,
+  repo: string,
+  tag_name: string,
+  arch: string,
+  platform: SupportedPlatform,
+  filename: string
+) {
+  const url = `https://github.com/${repo}/releases/download/${tag_name}/${filename}`
+  console.log('Downloading', binaryName, 'for', platform, arch, 'from', url)
+
+  const exeFilename = binaryName + (platform === 'win32' ? '.exe' : '')
+  const exePath = join('public', 'bin', arch, platform, exeFilename)
+  await downloadFile(url, exePath)
+
+  console.log('Done downloading', binaryName, 'for', platform, arch)
+
+  if (platform !== 'win32') {
+    await chmod(exePath, '755')
+  }
+}
+
+/**
+ * Downloads assets uploaded to a GitHub release
+ * @param binaryName The binary which was built & uploaded. Also used to get the final folder path
+ * @param repo The repo to download from
+ * @param tagName The GitHub Release tag which produced the binaries
+ * @param assetNames The name(s) of the assets which were uploaded, mapped to platforms
+ */
+async function downloadGithubAssets(
+  binaryName: string,
+  repo: string,
+  tagName: string,
+  assetNames: Record<
+    'x64' | 'arm64',
+    Partial<Record<SupportedPlatform, string>>
+  >
+) {
+  const downloadPromises = Object.entries(assetNames).map(
+    async ([arch, platformFilenameMap]) =>
+      Promise.all(
+        Object.entries(platformFilenameMap).map(([platform, filename]) => {
+          if (!filename) return
+          return downloadAsset(
+            binaryName,
+            repo,
+            tagName,
+            arch,
+            platform as keyof typeof platformFilenameMap,
+            filename
+          )
+        })
+      )
+  )
+
+  return Promise.all(downloadPromises)
+}
+
+async function downloadLegendary() {
+  return downloadGithubAssets(
+    'legendary',
+    'Heroic-Games-Launcher/legendary',
+    RELEASE_TAGS['legendary'],
+    {
+      x64: {
+        linux: 'legendary_linux_x86_64',
+        darwin: 'legendary_macOS_x86_64',
+        win32: 'legendary_windows_x86_64.exe'
+      },
+      arm64: {
+        darwin: 'legendary_macOS_arm64'
+      }
+    }
+  )
+}
+
+async function downloadGogdl() {
+  return downloadGithubAssets(
+    'gogdl',
+    'Heroic-Games-Launcher/heroic-gogdl',
+    RELEASE_TAGS['gogdl'],
+    {
+      x64: {
+        linux: 'gogdl_linux_x86_64',
+        darwin: 'gogdl_macOS_x86_64',
+        win32: 'gogdl_windows_x86_64.exe'
+      },
+      arm64: {
+        darwin: 'gogdl_macOS_arm64'
+      }
+    }
+  )
+}
+
+async function downloadNile() {
+  return downloadGithubAssets('nile', 'imLinguin/nile', RELEASE_TAGS['nile'], {
+    x64: {
+      linux: 'nile_linux_x86_64',
+      darwin: 'nile_macOS_x86_64',
+      win32: 'nile_windows_x86_64.exe'
+    },
+    arm64: {
+      darwin: 'nile_macOS_arm64'
+    }
+  })
+}
+
+/**
+ * Finds out which binaries need to be downloaded by comparing
+ * `public/bin/.release_tags` to RELEASE_TAGS
+ */
+async function compareDownloadedTags(): Promise<DownloadedBinary[]> {
+  const storedTagsText = await readFile(
+    'public/bin/.release_tags',
+    'utf-8'
+  ).catch(() => '{}')
+  let storedTagsParsed: Partial<Record<DownloadedBinary, string>>
+  try {
+    storedTagsParsed = JSON.parse(storedTagsText)
+  } catch {
+    return ['legendary', 'gogdl', 'nile']
+  }
+  const binariesToDownload: DownloadedBinary[] = []
+  for (const [runner, currentTag] of Object.entries(RELEASE_TAGS)) {
+    if (storedTagsParsed[runner] !== currentTag)
+      binariesToDownload.push(runner as keyof typeof RELEASE_TAGS)
+  }
+  return binariesToDownload
+}
+
+async function storeDownloadedTags() {
+  await writeFile('public/bin/.release_tags', JSON.stringify(RELEASE_TAGS))
+}
+
+async function main() {
+  if (!(await pathExists('public/bin'))) {
+    console.error('public/bin not found, are you in the source root?')
+    return
+  }
+
+  const binariesToDownload = await compareDownloadedTags()
+  if (!binariesToDownload.length) {
+    console.log('Nothing to download, binaries are up-to-date')
+    return
+  }
+
+  console.log('Downloading:', binariesToDownload)
+  const promisesToAwait: Promise<unknown>[] = []
+
+  if (binariesToDownload.includes('legendary'))
+    promisesToAwait.push(downloadLegendary())
+  if (binariesToDownload.includes('gogdl'))
+    promisesToAwait.push(downloadGogdl())
+  if (binariesToDownload.includes('nile')) promisesToAwait.push(downloadNile())
+
+  await Promise.all(promisesToAwait)
+
+  await storeDownloadedTags()
+}
+
+void main()

package.json

@@ -47,7 +47,8 @@
     "i18n": "i18next --silent",
     "prepare": "husky install",
     "prettier": "prettier --check .",
-    "prettier-fix": "prettier --write ."
+    "prettier-fix": "prettier --write .",
+    "download-helper-binaries": "esbuild --bundle --platform=node --target=node21 meta/downloadHelperBinaries.ts | node"
   },
   "dependencies": {
     "@emotion/react": "11.10.6",
@@ -116,7 +117,7 @@
     "@types/i18next-fs-backend": "1.1.4",
     "@types/ini": "1.3.31",
     "@types/jest": "29.4.0",
-    "@types/node": "18.15.0",
+    "@types/node": "20.14.9",
     "@types/plist": "3.0.2",
     "@types/react": "18.2.34",
     "@types/react-beautiful-dnd": "13.1.4",