Skip to content

Commit

Permalink
Merge pull request #4266 from snyk/chore/HMMR-612_extcli_in_ts
Browse files Browse the repository at this point in the history
Introduce TS Wrapper for Golang CLI
  • Loading branch information
PeterSchafer committed Jan 16, 2023
2 parents 5219680 + 575cb36 commit 5323088
Show file tree
Hide file tree
Showing 13 changed files with 844 additions and 52 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ local.log
/dist-docker
/pysrc
binary-releases
ts-cli-binaries
prepack
tmp
.DS_Store
!/test/fixtures/**/package-lock.json
Expand Down
139 changes: 94 additions & 45 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@
#

PKG := npx pkg ./ --compress Brotli
BINARY_WRAPPER_DIR = ts-binary-wrapper
EXTENSIBLE_CLI_DIR = cliv2
BINARY_RELEASES_FOLDER_TS_CLI = binary-releases
BINARY_OUTPUT_FOLDER = binary-releases

.DEFAULT: help
.PHONY: help
Expand All @@ -15,11 +19,16 @@ help:
@echo 'This Makefile is currently only for building release artifacts.'
@echo 'Use `npm run` for CLIv1 scripts.'

binary-releases:
mkdir binary-releases
$(BINARY_RELEASES_FOLDER_TS_CLI):
@mkdir $(BINARY_RELEASES_FOLDER_TS_CLI)

binary-releases/version: | binary-releases
./release-scripts/next-version.sh > binary-releases/version
$(BINARY_RELEASES_FOLDER_TS_CLI)/version: | $(BINARY_RELEASES_FOLDER_TS_CLI)
./release-scripts/next-version.sh > $(BINARY_RELEASES_FOLDER_TS_CLI)/version

ifneq ($(BINARY_OUTPUT_FOLDER), $(BINARY_RELEASES_FOLDER_TS_CLI))
$(BINARY_OUTPUT_FOLDER)/version: $(BINARY_RELEASES_FOLDER_TS_CLI)/version
@cp $(BINARY_RELEASES_FOLDER_TS_CLI)/version $(BINARY_OUTPUT_FOLDER)/version
endif

# prepack is not a typical target.
# It modifies package.json files rather than only creating new files.
Expand All @@ -29,85 +38,125 @@ binary-releases/version: | binary-releases
# Only removing "prepack" is not enough. We need to do additional cleanup (see clean-prepack).
.INTERMEDIATE: prepack
.SECONDARY: prepack
prepack: binary-releases/version
prepack: $(BINARY_RELEASES_FOLDER_TS_CLI)/version
@echo "'make prepack' was run. Run 'make clean-prepack' to rollback your package.json changes and this file." > prepack
npm version "$(shell cat binary-releases/version)" --no-git-tag-version --workspaces --include-workspace-root
npm version "$(shell cat $(BINARY_RELEASES_FOLDER_TS_CLI)/version)" --no-git-tag-version --workspaces --include-workspace-root
npx ts-node ./release-scripts/prune-dependencies-in-packagejson.ts

.PHONY: clean-prepack
clean-prepack:
git checkout package.json package-lock.json packages/*/package.json packages/*/package-lock.json
rm -f prepack

.PHONY: clean
clean: clean-prepack
.PHONY: clean-ts
clean-ts:
npm run clean
rm -f -r binary-releases
rm -f -r $(BINARY_RELEASES_FOLDER_TS_CLI)


binary-releases/sha256sums.txt.asc: $(wildcard binary-releases/*.sha256)
$(BINARY_OUTPUT_FOLDER)/sha256sums.txt.asc: $(wildcard $(BINARY_OUTPUT_FOLDER)/*.sha256)
./release-scripts/sha256sums.txt.asc.sh

binary-releases/release.json: binary-releases/version $(wildcard binary-releases/*.sha256)
$(BINARY_OUTPUT_FOLDER)/release.json: $(BINARY_OUTPUT_FOLDER)/version $(wildcard $(BINARY_OUTPUT_FOLDER)/*.sha256)
./release-scripts/release.json.sh

# --commit-path is forwarded to `git log <path>`.
# We're using this to remove CLIv2 changes in v1's changelogs.
# :(exclude) syntax: https://git-scm.com/docs/gitglossary.html#Documentation/gitglossary.txt-exclude
# Release notes uses version from package.json so we need to prepack beforehand.
binary-releases/RELEASE_NOTES.md: prepack | binary-releases
npx conventional-changelog-cli -p angular -l -r 1 --commit-path ':(exclude)cliv2' > binary-releases/RELEASE_NOTES.md
$(BINARY_OUTPUT_FOLDER)/RELEASE_NOTES.md: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI)
npx conventional-changelog-cli -p angular -l -r 1 --commit-path ':(exclude)cliv2' > $(BINARY_OUTPUT_FOLDER)/RELEASE_NOTES.md

# Generates a shasum of a target with the same name.
# See "Automatic Variables" in GNU Make docs (linked at the top)
%.sha256: %
cd $(@D); shasum -a 256 $(<F) > $(@F); shasum -a 256 -c $(@F)

binary-releases/snyk.tgz: prepack | binary-releases
mv $(shell npm pack) binary-releases/snyk.tgz
$(MAKE) binary-releases/snyk.tgz.sha256
$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk.tgz: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI)
mv $(shell npm pack) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk.tgz
$(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk.tgz.sha256

binary-releases/snyk-fix.tgz: prepack | binary-releases
mv $(shell npm pack --workspace '@snyk/fix') binary-releases/snyk-fix.tgz
$(MAKE) binary-releases/snyk-fix.tgz.sha256
$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-fix.tgz: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI)
mv $(shell npm pack --workspace '@snyk/fix') $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-fix.tgz
$(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-fix.tgz.sha256

binary-releases/snyk-protect.tgz: prepack | binary-releases
mv $(shell npm pack --workspace '@snyk/protect') binary-releases/snyk-protect.tgz
$(MAKE) binary-releases/snyk-protect.tgz.sha256
$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-protect.tgz: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI)
mv $(shell npm pack --workspace '@snyk/protect') $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-protect.tgz
$(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-protect.tgz.sha256

binary-releases/snyk-alpine: prepack | binary-releases
$(PKG) -t node16-alpine-x64 -o binary-releases/snyk-alpine
$(MAKE) binary-releases/snyk-alpine.sha256
$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-alpine: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI)
$(PKG) -t node16-alpine-x64 -o $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-alpine
$(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-alpine.sha256

binary-releases/snyk-linux: prepack | binary-releases
$(PKG) -t node16-linux-x64 -o binary-releases/snyk-linux
$(MAKE) binary-releases/snyk-linux.sha256
$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-linux: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI)
$(PKG) -t node16-linux-x64 -o $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-linux
$(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-linux.sha256

# Why `--no-bytecode` for Linux/arm64:
# arm64 bytecode generation requires various build tools on an x64 build
# environment. So disabling until we can support it. It's an optimisation.
# https://github.com/vercel/pkg#targets
binary-releases/snyk-linux-arm64: prepack | binary-releases
$(PKG) -t node16-linux-arm64 -o binary-releases/snyk-linux-arm64 --no-bytecode
$(MAKE) binary-releases/snyk-linux-arm64.sha256
$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-linux-arm64: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI)
$(PKG) -t node16-linux-arm64 -o $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-linux-arm64 --no-bytecode
$(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-linux-arm64.sha256

binary-releases/snyk-macos: prepack | binary-releases
$(PKG) -t node16-macos-x64 -o binary-releases/snyk-macos
$(MAKE) binary-releases/snyk-macos.sha256
$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-macos: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI)
$(PKG) -t node16-macos-x64 -o $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-macos
$(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-macos.sha256

binary-releases/snyk-win.exe: prepack | binary-releases
$(PKG) -t node16-win-x64 -o binary-releases/snyk-win.exe
./cliv2/scripts/sign_windows.sh binary-releases snyk-win.exe
$(MAKE) binary-releases/snyk-win.exe.sha256
$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-win.exe: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI)
$(PKG) -t node16-win-x64 -o $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-win.exe
./cliv2/scripts/sign_windows.sh $(BINARY_RELEASES_FOLDER_TS_CLI) snyk-win.exe
$(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-win.exe.sha256

binary-releases/snyk-for-docker-desktop-darwin-x64.tar.gz: prepack | binary-releases
$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-for-docker-desktop-darwin-x64.tar.gz: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI)
./docker-desktop/build.sh darwin x64
$(MAKE) binary-releases/snyk-for-docker-desktop-darwin-x64.tar.gz.sha256
$(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-for-docker-desktop-darwin-x64.tar.gz.sha256

binary-releases/snyk-for-docker-desktop-darwin-arm64.tar.gz: prepack | binary-releases
$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-for-docker-desktop-darwin-arm64.tar.gz: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI)
./docker-desktop/build.sh darwin arm64
$(MAKE) binary-releases/snyk-for-docker-desktop-darwin-arm64.tar.gz.sha256
$(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-for-docker-desktop-darwin-arm64.tar.gz.sha256

binary-releases/docker-mac-signed-bundle.tar.gz: prepack | binary-releases
$(BINARY_RELEASES_FOLDER_TS_CLI)/docker-mac-signed-bundle.tar.gz: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI)
./release-scripts/docker-desktop-release.sh
$(MAKE) binary-releases/docker-mac-signed-bundle.tar.gz.sha256
$(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/docker-mac-signed-bundle.tar.gz.sha256

# targets responsible for the Wrapper CLI (TS around Golang)
$(BINARY_WRAPPER_DIR)/src/generated:
@mkdir $(BINARY_WRAPPER_DIR)/src/generated/

$(BINARY_WRAPPER_DIR)/src/generated/version: $(BINARY_WRAPPER_DIR)/src/generated $(BINARY_RELEASES_FOLDER_TS_CLI)/version
@cp $(BINARY_RELEASES_FOLDER_TS_CLI)/version $(BINARY_WRAPPER_DIR)/src/generated/version

$(BINARY_WRAPPER_DIR)/src/generated/sha256sums.txt:
@echo "-- Generating $(@F)"
@cat $(BINARY_OUTPUT_FOLDER)/*.sha256 > $(BINARY_WRAPPER_DIR)/src/generated/sha256sums.txt

.PHONY: build-binary-wrapper
build-binary-wrapper: $(BINARY_WRAPPER_DIR)/src/generated/version $(BINARY_WRAPPER_DIR)/src/generated/sha256sums.txt
@echo "-- Building Typescript Binary Wrapper ($(BINARY_WRAPPER_DIR)/dist/)"
@cd $(BINARY_WRAPPER_DIR) && npm run build

.PHONY: clean-binary-wrapper
clean-binary-wrapper:
@cd $(BINARY_WRAPPER_DIR) && npm run clean

.PHONY: pack-binary-wrapper
pack-binary-wrapper: build-binary-wrapper
@echo "-- Packaging tarball ($(BINARY_OUTPUT_FOLDER)/snyk.tgz)"
@mv $(BINARY_WRAPPER_DIR)/$(shell cd $(BINARY_WRAPPER_DIR) && npm pack) $(BINARY_OUTPUT_FOLDER)/snyk.tgz

.PHONY: test-binary-wrapper
test-binary-wrapper:
@echo "-- Testing binary wrapper"
@cd $(BINARY_WRAPPER_DIR) && npm run test


# targets responsible for the complete CLI build
.PHONY: build
build:
@cd $(EXTENSIBLE_CLI_DIR) && $(MAKE) build-full install bindir=$(CURDIR)/$(BINARY_OUTPUT_FOLDER) USE_LEGACY_EXECUTABLE_NAME=1

.PHONY: clean
clean:
@cd $(EXTENSIBLE_CLI_DIR) && $(MAKE) clean-full
$(MAKE) clean-prepack
22 changes: 15 additions & 7 deletions cliv2/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ HASH_ALGORITHM = 256
CLI_V2_VERSION_TAG =
CLI_V1_VERSION_TAG =
CLI_V1_LOCATION =
USE_LEGACY_EXECUTABLE_NAME =

# Make directories per convention
prefix = /usr/local
Expand Down Expand Up @@ -70,6 +71,11 @@ APPLICATION_NAME = snyk
TEST_NAME = $(APPLICATION_NAME)$(_SEPARATOR)tests
V2_PLATFORM_STRING = $(GOOS)$(_SEPARATOR)$(GOARCH)
V2_EXECUTABLE_NAME = $(APPLICATION_NAME)$(_SEPARATOR)$(V2_PLATFORM_STRING)$(_EXE_POSTFIX)

ifneq ($(USE_LEGACY_EXECUTABLE_NAME), $(_EMPTY))
V2_EXECUTABLE_NAME = $(V1_EXECUTABLE_NAME)
endif

V1_EXECUTABLE_NAME = $(APPLICATION_NAME)-$(V1_PLATFORM_STING)$(_EXE_POSTFIX)
V2_DIRECTORY = $(WORKING_DIR)/internal/cliv2
V1_DIRECTORY = $(WORKING_DIR)/internal/embedded/cliv1
Expand All @@ -78,13 +84,15 @@ V1_EMBEDDED_FILE_TEMPLATE = $(V1_DIRECTORY)/embedded_binary_template.txt
V1_EMBEDDED_FILE_OUTPUT = embedded$(_SEPARATOR)$(V2_PLATFORM_STRING).go
V1_WORKING_DIR = $(WORKING_DIR)/..
V1_BUILD_TYPE = build
V1_BINARY_FOLDER = ts-cli-binaries
HASH_STRING = $(HASH)$(HASH_ALGORITHM)
TEST_SNYK_EXECUTABLE_PATH=$(BUILD_DIR)/$(V2_EXECUTABLE_NAME)
TEST_EXECUTABLE_NAME = $(TEST_NAME)$(_SEPARATOR)$(V2_PLATFORM_STRING)$(_EXE_POSTFIX)
SIGN_SCRIPT = sign_$(_GO_OS).sh
ISSIGNED_SCRIPT = issigned_$(_GO_OS).sh
EMBEDDED_DATA_DIR = $(WORKING_DIR)/internal/embedded/_data


# some make file variables
LOG_PREFIX = --

Expand Down Expand Up @@ -225,25 +233,25 @@ sign: _cleanup_sha_v2 $(SIGN_SCRIPT) $(BUILD_DIR)/$(V2_EXECUTABLE_NAME).$(HASH_S
test-signature: $(ISSIGNED_SCRIPT)

# Typescript CLI targets
$(V1_WORKING_DIR)/binary-releases/$(V1_EXECUTABLE_NAME):
$(V1_WORKING_DIR)/$(V1_BINARY_FOLDER)/$(V1_EXECUTABLE_NAME):
@echo "$(LOG_PREFIX) Building legacy CLI"
@cd $(V1_WORKING_DIR) && npm ci && npm run $(V1_BUILD_TYPE)
@$(MAKE) -C $(V1_WORKING_DIR) binary-releases/$(V1_EXECUTABLE_NAME)
@$(MAKE) -C $(V1_WORKING_DIR) $(V1_BINARY_FOLDER)/$(V1_EXECUTABLE_NAME) BINARY_RELEASES_FOLDER_TS_CLI=$(V1_BINARY_FOLDER)

.PHONY: build-ts-cli
build-ts-cli: $(V1_WORKING_DIR)/binary-releases/$(V1_EXECUTABLE_NAME)
$(eval CLI_V1_VERSION_TAG := $(shell cat $(V1_WORKING_DIR)/binary-releases/version))
$(eval CLI_V1_LOCATION := $(V1_WORKING_DIR)/binary-releases/)
build-ts-cli: $(V1_WORKING_DIR)/$(V1_BINARY_FOLDER)/$(V1_EXECUTABLE_NAME)
$(eval CLI_V1_VERSION_TAG := $(shell cat $(V1_WORKING_DIR)/$(V1_BINARY_FOLDER)/version))
$(eval CLI_V1_LOCATION := $(V1_WORKING_DIR)/$(V1_BINARY_FOLDER)/)

.PHONY: clean-ts-cli
clean-ts-cli:
@echo "$(LOG_PREFIX) Cleaning legacy CLI"
@$(MAKE) -C $(V1_WORKING_DIR) clean
@$(MAKE) -C $(V1_WORKING_DIR) clean-ts BINARY_RELEASES_FOLDER_TS_CLI=$(V1_BINARY_FOLDER)

# build the full CLI (Typescript+Golang)
.PHONY: build-full
build-full: | build-ts-cli
@$(MAKE) build CLI_V1_VERSION_TAG=$(CLI_V1_VERSION_TAG) CLI_V1_LOCATION="$(CLI_V1_LOCATION)"
@$(MAKE) build build-test CLI_V1_VERSION_TAG=$(CLI_V1_VERSION_TAG) CLI_V1_LOCATION="$(CLI_V1_LOCATION)"

# clean the full CLI (Typescript+Golang)
.PHONY: clean-full
Expand Down
3 changes: 3 additions & 0 deletions ts-binary-wrapper/jest.config.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
const { createJestConfig } = require('../test/createJestConfig');

module.exports = createJestConfig({ displayName: 'ts-binary-wrapper' });
20 changes: 20 additions & 0 deletions ts-binary-wrapper/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

39 changes: 39 additions & 0 deletions ts-binary-wrapper/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{
"name": "ts-binary-wrapper",
"version": "1.0.0",
"description": "Wrapper for Snyk's Golang based Extensible CLI.",
"main": "dist/index.js",
"directories": {
"lib": "src",
"test": "test"
},
"bin": {
"snyk": "dist/index.js"
},
"engines": {
"node": ">=16"
},
"scripts": {
"clean": "npx rimraf dist tsconfig.tsbuildinfo src/generated",
"build": "tsc && cp -R src/generated dist/",
"test": "npx jest test/*",
"postinstall": "node dist/bootstrap.js exec"
},
"keywords": [
"security",
"vulnerabilities",
"advisories",
"audit",
"snyk",
"scan",
"docker",
"container",
"scanning"
],
"author": "snyk.io",
"license": "Apache-2.0",
"repository": {
"type": "git",
"url": "https://github.com/snyk/snyk.git"
}
}
18 changes: 18 additions & 0 deletions ts-binary-wrapper/src/bootstrap.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import * as common from './common';
import * as process from 'process';

const config = common.getCurrentConfiguration();
export const executable = config.getLocalLocation();

if (process.argv.includes('exec')) {
const filenameShasum = config.getShasumFile();
const downloadUrl = config.getDownloadLocation();

common
.downloadExecutable(downloadUrl, executable, filenameShasum)
.then(process.exit)
.catch((err) => {
console.error(err);
process.exit(1);
});
}
Loading

0 comments on commit 5323088

Please sign in to comment.