From 9bc03a08cf312f99077cad1be30efeca6b69748c Mon Sep 17 00:00:00 2001 From: Vishwas Siravara Date: Tue, 17 Oct 2023 20:34:34 +0000 Subject: [PATCH] feat: add windows targets in makefile (#192) Issue #, if available: *Description of changes:* finch-core changes for windev. - Makefile changes - rootfs dockerfile change - lima submodule updates *Testing done:* Yes - [X] I've reviewed the guidance in CONTRIBUTING.md #### License Acceptance By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. --------- Signed-off-by: Vishwas Siravara Signed-off-by: Justin Alvarez Signed-off-by: Gavin Inglis Co-authored-by: Justin Co-authored-by: Gavin Inglis <43075615+ginglis13@users.noreply.github.com> --- .github/workflows/rootfs.yaml | 3 +- .gitignore | 1 + Dockerfile | 5 ++ Makefile | 127 ++++++++++++++++++++++++++++++---- bin/verify_hash.ps1 | 18 +++++ 5 files changed, 139 insertions(+), 15 deletions(-) create mode 100644 bin/verify_hash.ps1 diff --git a/.github/workflows/rootfs.yaml b/.github/workflows/rootfs.yaml index f5c0568..810d3a3 100644 --- a/.github/workflows/rootfs.yaml +++ b/.github/workflows/rootfs.yaml @@ -19,7 +19,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - arch: ['amd64', 'arm64'] + # finch only supports amd64 for windows + arch: ['amd64'] platform: ['common'] steps: - name: Generate Timestamp diff --git a/.gitignore b/.gitignore index b0df895..78f5ef5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ _output/ downloads/ +wingit-temp/ *.idea *.DS_Store *.bak diff --git a/Dockerfile b/Dockerfile index dab2ce9..c4721cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,11 @@ RUN dnf install -y \ iptables \ fuse-sshfs +# install cosign +RUN curl -L -O https://github.com/sigstore/cosign/releases/download/v2.0.1/cosign-2.0.1.x86_64.rpm && \ + sudo rpm -ivh cosign-2.0.1.x86_64.rpm && \ + rm -rf cosign-2.0.1.x86_64.rpm + RUN systemctl enable cloud-init cloud-init-local cloud-config cloud-final # enable systemd diff --git a/Makefile b/Makefile index 8ef1caa..2f7db92 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,8 @@ HASH_DIR ?= $(CURDIR)/hashes DOWNLOAD_DIR := $(CURDIR)/downloads OS_DOWNLOAD_DIR := $(DOWNLOAD_DIR)/os LIMA_DOWNLOAD_DIR := $(DOWNLOAD_DIR)/dependencies -ROOTFS_DOWNLOAD_DIR := $(DOWNLOAD_DIR)/rootfs +LIMA_OUTDIR ?= $(OUTDIR)/lima +ROOTFS_DOWNLOAD_DIR := $(DOWNLOAD_DIR)/os DEPENDENCIES_DOWNLOAD_DIR := $(DOWNLOAD_DIR)/dependencies SOCKET_VMNET_TEMP_PREFIX ?= $(OUTDIR)/dependencies/lima-socket_vmnet/opt/finch UNAME := $(shell uname -m) @@ -22,36 +23,48 @@ FINCH_OS_x86_DIGEST := $(or $(FINCH_OS_x86_DIGEST),"sha256:214cce00ce5f6ac402a0a FINCH_OS_AARCH64_URL := $(or $(FINCH_OS_AARCH64_URL),https://deps.runfinch.com/Fedora-Cloud-Base-38-1.6.aarch64-20230918164937.qcow2) FINCH_OS_AARCH64_DIGEST := $(or $(FINCH_OS_AARCH64_DIGEST),"sha256:ad4c2fa3f80736cb6ea8e46f1a6ccf1f5f578e56de462bb60fcbc241786478d2") +FINCH_ROOTFS_x86_URL := $(or $(FINCH_ROOTFS_x86_URL),https://deps.runfinch.com/common/x86-64/finch-rootfs-production-amd64-1696963702.tar.gz) +FINCH_ROOTFS_x86_DIGEST := $(or $(FINCH_ROOTFS_x86_DIGEST),"sha256:ed36fb7f4819644efaf409a3417456fe8378c4f4bcff0bd1e0e520954b10ccf5") + LIMA_DEPENDENCY_FILE_NAME ?= lima-and-qemu.tar.gz .DEFAULT_GOAL := all +WINGIT_TEMP_DIR := $(CURDIR)/wingit-temp +WINGIT_x86_URL := $(or $(WINGIT_x86_URL),https://github.com/git-for-windows/git/releases/download/v2.42.0.windows.2/Git-2.42.0.2-64-bit.tar.bz2) +WINGIT_x86_BASENAME ?= $(notdir $(WINGIT_x86_URL)) +WINGIT_x86_HASH := $(or $(WINGIT_x86_HASH),"sha256:c192e56f8ed3d364acc87ad04d1f5aa6ae03c23b32b67bf65fcc6f9b8f032e65") + ifneq (,$(findstring arm64,$(ARCH))) LIMA_ARCH = aarch64 LIMA_URL ?= https://deps.runfinch.com/aarch64/lima-and-qemu.macos-aarch64.1695247723.tar.gz FINCH_OS_BASENAME := $(notdir $(FINCH_OS_AARCH64_URL)) FINCH_OS_IMAGE_URL := $(FINCH_OS_AARCH64_URL) FINCH_OS_DIGEST ?= $(FINCH_OS_AARCH64_DIGEST) + # TODO: Use Finch rootfs in Finch on Windows testing + FINCH_ROOTFS_BASENAME := $(notdir $(FINCH_ROOTFS_AARCH64_URL)) + FINCH_ROOTFS_URL ?= $(FINCH_ROOTFS_AARCH64_URL) + FINCH_ROOTFS_DIGEST ?= $(FINCH_ROOTFS_AARCH64_DIGEST) HOMEBREW_PREFIX ?= /opt/homebrew - # TODO: Use Finch rootfs in Finch on Windows testing - FINCH_ROOTFS_URL ?= https://deps.runfinch.com/common/aarch64/finch-rootfs-production-arm64-1690920104.tar.zst - FINCH_ROOTFS_BASENAME := $(notdir $(FINCH_ROOTFS_URL)) else ifneq (,$(findstring x86_64,$(ARCH))) LIMA_ARCH = x86_64 LIMA_URL ?= https://deps.runfinch.com/x86-64/lima-and-qemu.macos-x86_64.1695247723.tar.gz FINCH_OS_BASENAME := $(notdir $(FINCH_OS_x86_URL)) FINCH_OS_IMAGE_URL := $(FINCH_OS_x86_URL) FINCH_OS_DIGEST ?= $(FINCH_OS_x86_DIGEST) + # TODO: Use Finch rootfs in Finch on Windows testing + FINCH_ROOTFS_BASENAME := $(notdir $(FINCH_ROOTFS_x86_URL)) + FINCH_ROOTFS_URL ?= $(FINCH_ROOTFS_x86_URL) + FINCH_ROOTFS_DIGEST ?= $(FINCH_ROOTFS_x86_DIGEST) HOMEBREW_PREFIX ?= /usr/local - # TODO: Use Finch rootfs in Finch on Windows testing - FINCH_ROOTFS_URL ?= https://deps.runfinch.com/common/x86-64/finch-rootfs-production-amd64-1690920103.tar.zst - FINCH_ROOTFS_BASENAME := $(notdir $(FINCH_ROOTFS_URL)) endif FINCH_OS_IMAGE_LOCATION ?= $(OUTDIR)/os/$(FINCH_OS_BASENAME) FINCH_OS_IMAGE_INSTALLATION_LOCATION ?= $(DEST)/os/$(FINCH_OS_BASENAME) +FINCH_ROOTFS_LOCATION ?= $(OUTDIR)/os/$(FINCH_ROOTFS_BASENAME) + .PHONY: all all: binaries @@ -59,14 +72,22 @@ all: binaries .PHONY: download # Rootfs required for Windows, require full OS for Linux and Mac - +FINCH_IMAGE_LOCATION ?= +FINCH_IMAGE_DIGEST ?= +FEDORA_YAML ?= BUILD_OS ?= $(OS) ifeq ($(BUILD_OS), Windows_NT) -binaries: rootfs lima-template +binaries: rootfs lima download: download.rootfs -else +lima: lima-exe install.lima-dependencies-wsl2 +FINCH_IMAGE_LOCATION := $(FINCH_ROOTFS_LOCATION) +FINCH_IMAGE_DIGEST := $(FINCH_ROOTFS_DIGEST) +else binaries: os lima-socket-vmnet lima-template download: download.os +FINCH_IMAGE_LOCATION := $(FINCH_OS_IMAGE_LOCATION) +FINCH_IMAGE_DIGEST := $(FINCH_OS_DIGEST) +FEDORA_YAML := fedora.yaml endif $(OS_DOWNLOAD_DIR)/$(FINCH_OS_BASENAME): @@ -76,13 +97,17 @@ $(OS_DOWNLOAD_DIR)/$(FINCH_OS_BASENAME): $(ROOTFS_DOWNLOAD_DIR)/$(FINCH_ROOTFS_BASENAME): mkdir -p $(ROOTFS_DOWNLOAD_DIR) + mkdir -p $(OUTDIR)/os curl -L --fail $(FINCH_ROOTFS_URL) > "$(ROOTFS_DOWNLOAD_DIR)/$(FINCH_ROOTFS_BASENAME)" + cp $(ROOTFS_DOWNLOAD_DIR)/$(FINCH_ROOTFS_BASENAME) $(OUTDIR)/os + .PHONY: download.os download.os: $(OS_DOWNLOAD_DIR)/$(FINCH_OS_BASENAME) .PHONY: download.rootfs download.rootfs: $(ROOTFS_DOWNLOAD_DIR)/$(FINCH_ROOTFS_BASENAME) + $(eval FINCH_ROOTFS_DIGEST := "sha256:$(sha256 $(ROOTFS_DOWNLOAD_DIR)/$(FINCH_ROOTFS_BASENAME))") $(LIMA_DOWNLOAD_DIR)/$(LIMA_DEPENDENCY_FILE_NAME): mkdir -p $(DEPENDENCIES_DOWNLOAD_DIR) @@ -93,18 +118,83 @@ $(LIMA_DOWNLOAD_DIR)/$(LIMA_DEPENDENCY_FILE_NAME): .PHONY: download.lima-dependencies download.lima-dependencies: $(LIMA_DOWNLOAD_DIR)/$(LIMA_DEPENDENCY_FILE_NAME) -.PHONE: install.lima-dependencies +.PHONY: install.lima-dependencies install.lima-dependencies: download.lima-dependencies +# Only redownload/extract if this file is missing (there's no particular reason for choosing this file instead of any other) +$(LIMA_OUTDIR)/bin/ssh.exe: + mkdir -p $(DEPENDENCIES_DOWNLOAD_DIR) + mkdir -p $(OUTDIR)/bin + + curl -L --fail $(WINGIT_x86_URL) > $(DEPENDENCIES_DOWNLOAD_DIR)/$(WINGIT_x86_BASENAME) + pwsh.exe -NoLogo -NoProfile -c ./bin/verify_hash.ps1 "$(DEPENDENCIES_DOWNLOAD_DIR)\$(WINGIT_x86_BASENAME)" $(WINGIT_x86_HASH) + mkdir -p $(WINGIT_TEMP_DIR) + # this takes a long time because of an almost 4:1 compression ratio and needing to extract many small files + tar --force-local -xvjf "$(DEPENDENCIES_DOWNLOAD_DIR)\$(WINGIT_x86_BASENAME)" -C $(WINGIT_TEMP_DIR) + + # Lima runtime dependencies + mkdir -p $(LIMA_OUTDIR)/bin + + # From https://packages.msys2.org/package/gzip?repo=msys&variant=x86_64 + cp $(WINGIT_TEMP_DIR)/usr/bin/gzip.exe $(LIMA_OUTDIR)/bin/ + # From https://packages.msys2.org/package/msys2-runtime?repo=msys&variant=x86_64 + cp $(WINGIT_TEMP_DIR)/usr/bin/cygpath.exe $(LIMA_OUTDIR)/bin/ + # From https://packages.msys2.org/package/tar?repo=msys&variant=x86_64 + cp $(WINGIT_TEMP_DIR)/usr/bin/tar.exe $(LIMA_OUTDIR)/bin/ + # From https://packages.msys2.org/package/openssh?repo=msys&variant=x86_64 + cp $(WINGIT_TEMP_DIR)/usr/bin/ssh.exe $(LIMA_OUTDIR)/bin/ + # From https://packages.msys2.org/package/openssh?repo=msys&variant=x86_64 + cp $(WINGIT_TEMP_DIR)/usr/bin/ssh-keygen.exe $(LIMA_OUTDIR)/bin/ + + # Dependency DLLs, extracted with https://github.com/lucasg/Dependencies + # Dependencies.exe -chain $(WINGIT_TEMP_DIR)\usr\bin\ssh.exe -depth 3 -json + # Depth 3 is only needed for ssh.exe, everything else only needs depth 1 + # TODO: Automate + + # Required by all MSYS2 programs, from https://github.com/msys2/msys2-runtime + cp $(WINGIT_TEMP_DIR)/usr/bin/msys-2.0.dll $(LIMA_OUTDIR)/bin/ + # Required by tar.exe, from https://packages.msys2.org/package/libiconv?repo=msys&variant=x86_64 + cp $(WINGIT_TEMP_DIR)/usr/bin/msys-iconv-2.dll $(LIMA_OUTDIR)/bin/ + # Required by msys-iconv-2.dll, from https://packages.msys2.org/package/libintl?repo=msys&variant=x86_64 + cp $(WINGIT_TEMP_DIR)/usr/bin/msys-intl-8.dll $(LIMA_OUTDIR)/bin/ + # GCC exception handling, required for all programs that throw exceptions, from https://packages.msys2.org/package/gcc-libs?repo=msys&variant=x86_64 + cp $(WINGIT_TEMP_DIR)/usr/bin/msys-gcc_s-seh-1.dll $(LIMA_OUTDIR)/bin/ + + # Required by ssh.exe, from https://packages.msys2.org/package/libopenssl?repo=msys&variant=x86_64 + cp $(WINGIT_TEMP_DIR)/usr/bin/msys-crypto-3.dll $(LIMA_OUTDIR)/bin/ + # Required by ssh.exe, from https://packages.msys2.org/package/zlib-devel?repo=msys&variant=x86_64 + cp $(WINGIT_TEMP_DIR)/usr/bin/msys-z.dll $(LIMA_OUTDIR)/bin/ + # Required by ssh.exe, from https://packages.msys2.org/package/libcrypt?repo=msys&variant=x86_64 + cp $(WINGIT_TEMP_DIR)/usr/bin/msys-crypt-0.dll $(LIMA_OUTDIR)/bin/ + # Required by heimdal-libs, from https://packages.msys2.org/package/libsqlite?repo=msys&variant=x86_64 + cp $(WINGIT_TEMP_DIR)/usr/bin/msys-sqlite3-0.dll $(LIMA_OUTDIR)/bin/ + + # Required by ssh.exe, from https://packages.msys2.org/package/heimdal-libs?repo=msys&variant=x86_64 + cp $(WINGIT_TEMP_DIR)/usr/bin/msys-asn1-8.dll $(LIMA_OUTDIR)/bin/ + cp $(WINGIT_TEMP_DIR)/usr/bin/msys-com_err-1.dll $(LIMA_OUTDIR)/bin/ + cp $(WINGIT_TEMP_DIR)/usr/bin/msys-gssapi-3.dll $(LIMA_OUTDIR)/bin/ + cp $(WINGIT_TEMP_DIR)/usr/bin/msys-hcrypto-4.dll $(LIMA_OUTDIR)/bin/ + cp $(WINGIT_TEMP_DIR)/usr/bin/msys-heimbase-1.dll $(LIMA_OUTDIR)/bin/ + cp $(WINGIT_TEMP_DIR)/usr/bin/msys-heimntlm-0.dll $(LIMA_OUTDIR)/bin/ + cp $(WINGIT_TEMP_DIR)/usr/bin/msys-hx509-5.dll $(LIMA_OUTDIR)/bin/ + cp $(WINGIT_TEMP_DIR)/usr/bin/msys-krb5-26.dll $(LIMA_OUTDIR)/bin/ + cp $(WINGIT_TEMP_DIR)/usr/bin/msys-roken-18.dll $(LIMA_OUTDIR)/bin/ + cp $(WINGIT_TEMP_DIR)/usr/bin/msys-wind-0.dll $(LIMA_OUTDIR)/bin/ + + -@rm -rf $(WINGIT_TEMP_DIR) + +.PHONY: install.lima-dependencies-wsl2 +install.lima-dependencies-wsl2: $(LIMA_OUTDIR)/bin/ssh.exe + .PHONY: lima-template lima-template: download mkdir -p $(OUTDIR)/lima-template cp lima-template/fedora.yaml $(OUTDIR)/lima-template # using -i.bak is very intentional, it allows the following commands to succeed for both GNU / BSD sed # this sed command uses the alternative separator of "|" because the image location uses "/" - sed -i.bak -e "s||$(FINCH_OS_IMAGE_LOCATION)|g" $(OUTDIR)/lima-template/fedora.yaml + sed -i.bak -e "s||$(FINCH_IMAGE_LOCATION)|g" $(OUTDIR)/lima-template/fedora.yaml sed -i.bak -e "s//$(LIMA_ARCH)/g" $(OUTDIR)/lima-template/fedora.yaml - sed -i.bak -e "s//$(FINCH_OS_DIGEST)/g" $(OUTDIR)/lima-template/fedora.yaml + sed -i.bak -e "s//$(FINCH_IMAGE_DIGEST)/g" $(OUTDIR)/lima-template/fedora.yaml rm $(OUTDIR)/lima-template/*.yaml.bak .PHONY: lima-socket-vmnet @@ -113,6 +203,13 @@ lima-socket-vmnet: cd src/socket_vmnet && git clean -f -d cd src/socket_vmnet && PREFIX=$(SOCKET_VMNET_TEMP_PREFIX) "$(MAKE)" install.bin +.PHONY: lima lima-exe +lima-exe: + cd src/lima && \ + "$(MAKE)" exe _output/share/lima/lima-guestagent.Linux-x86_64 + mkdir -p ${OUTDIR}/lima + cp -r src/lima/_output/* ${OUTDIR}/lima + .PHONY: download-sources download-sources: ./bin/download-sources.pl @@ -124,12 +221,14 @@ os: download .PHONY: rootfs rootfs: download + mkdir -p $(OUTDIR)/os + cp $(DOWNLOAD_DIR)/os/$(FINCH_ROOTFS_BASENAME) "$(OUTDIR)/os/$(FINCH_ROOTFS_BASENAME)" .PHONY: install install: uninstall mkdir -p $(DEST) (cd _output && tar c * | tar Cvx $(DEST) ) - sed -i.bak -e "s|${FINCH_OS_IMAGE_LOCATION}|$(FINCH_OS_IMAGE_LOCATION)|g" $(DEST)/lima-template/fedora.yaml + sed -i.bak -e "s|${FINCH_OS_IMAGE_LOCATION}|$(FINCH_IMAGE_LOCATION)|g" $(DEST)/lima-template/fedora.yaml rm $(DEST)/lima-template/*.yaml.bak .PHONY: uninstall diff --git a/bin/verify_hash.ps1 b/bin/verify_hash.ps1 new file mode 100644 index 0000000..eb2dfc4 --- /dev/null +++ b/bin/verify_hash.ps1 @@ -0,0 +1,18 @@ +<# + Verify file hashes using pure PowerShell +#> +param ( + # Dependency file path + [Parameter(Mandatory=$true)] + [string]$DependencyFilePath, + + # Dependency hash + [Parameter(Mandatory=$true)] + [string]$DependencyHash = 'out.png' +) + +if (!(Get-FileHash -Algorithm SHA256 "$DependencyFilePath").Hash -eq $DependencyHash) { + $host.SetShouldExit(-1); exit +} else { + Write-Output "Verified $DependencyFilePath" +}