From 374674b89687d338ba50c038206839c759d67103 Mon Sep 17 00:00:00 2001 From: Keegan Carruthers-Smith Date: Mon, 24 Aug 2020 21:17:28 +0200 Subject: [PATCH 1/4] dev: remove nginx We already have Caddy running for HTTPS support. We can use just Caddy to remove the need to have nginx in our dev environments. This does mean we won't be using the same reverse proxy as we do in production and for customers. However, I do not think that changes much in practice. --- .gitignore | 3 --- dev/.gitignore | 1 - dev/Caddyfile | 20 ++++++++++++++---- dev/Procfile | 1 - dev/caddy.sh | 7 +------ dev/nginx.conf | 40 ------------------------------------ dev/nginx/body/README | 1 - dev/start.sh | 2 +- doc/dev/local_development.md | 7 +++---- enterprise/dev/Procfile | 1 - 10 files changed, 21 insertions(+), 62 deletions(-) delete mode 100644 dev/nginx.conf delete mode 100644 dev/nginx/body/README diff --git a/.gitignore b/.gitignore index 7e4959e8133bc..ca9480855805f 100644 --- a/.gitignore +++ b/.gitignore @@ -132,9 +132,6 @@ storybook-static/ *.pem *.crt -# Custom dev-specific nginx config -/dev/*.nginx.conf - # tilt files /dev/tilt/generated-cluster /dev/tilt/tilt-watch-targets diff --git a/dev/.gitignore b/dev/.gitignore index 0243b7226f824..fe505ce380a77 100644 --- a/dev/.gitignore +++ b/dev/.gitignore @@ -2,4 +2,3 @@ /config_base.json /config_combined.json /config_backup.json -/nginx.pid diff --git a/dev/Caddyfile b/dev/Caddyfile index b9d7a434da2f3..7d2f49f89cacd 100644 --- a/dev/Caddyfile +++ b/dev/Caddyfile @@ -1,6 +1,18 @@ { - http_port 3088 + http_port 3081 + auto_https disable_redirects +} + +# A bit of monstrosity, since we need to reverse proxy via webpack which then +# reverse proxies to us on HTTP. +# +# Caddy (tls :3443) -> webpack (:3080) -> Caddy (:3081) -> sourcegraph-frontend (:3082) +{$SOURCEGRAPH_HTTPS_DOMAIN}:{$SOURCEGRAPH_HTTPS_PORT} { + tls internal + reverse_proxy localhost:3080 +} + +# Caddy (:3081) -> sourcegraph-frontend (:3082) +*:3081 { + reverse_proxy localhost:3082 } -{$SOURCEGRAPH_HTTPS_DOMAIN}:{$SOURCEGRAPH_HTTPS_PORT} -tls internal -reverse_proxy localhost:3080 diff --git a/dev/Procfile b/dev/Procfile index 777b9bf549cfe..6b16472bf26ea 100644 --- a/dev/Procfile +++ b/dev/Procfile @@ -6,7 +6,6 @@ symbols: symbols github-proxy: github-proxy frontend: env CONFIGURATION_MODE=server SITE_CONFIG_ESCAPE_HATCH_PATH=$HOME/.sourcegraph/site-config.json frontend watch: ./dev/changewatch.sh -nginx: nginx -p . -g 'daemon off;' -c $PWD/dev/nginx.conf 2>&1 | grep -v 'could not open error log file' caddy: ./dev/caddy.sh run --watch --config=dev/Caddyfile web: ./node_modules/.bin/gulp --color watch syntect_server: ./dev/syntect_server.sh diff --git a/dev/caddy.sh b/dev/caddy.sh index 0c7c34dbbf0ea..8792ebb30ba1a 100755 --- a/dev/caddy.sh +++ b/dev/caddy.sh @@ -4,14 +4,9 @@ set -euf -o pipefail pushd "$(dirname "${BASH_SOURCE[0]}")/.." >/dev/null -if [ -n "${NO_CADDY:-}" ]; then - echo Not using Caddy because NO_CADDY is set. SSH support through Caddy will not work. - exit 0 -fi - mkdir -p .bin -version="2.0.0" +version="2.1.1" case "$(go env GOOS)" in linux) os="linux" diff --git a/dev/nginx.conf b/dev/nginx.conf deleted file mode 100644 index 63c7ee559197d..0000000000000 --- a/dev/nginx.conf +++ /dev/null @@ -1,40 +0,0 @@ -error_log stderr; -pid dev/nginx.pid; - -events { -} - -http { - server_tokens off; - - proxy_buffers 8 16k; # Buffer pool = 8 buffers of 16k - proxy_buffer_size 16k; # 16k of buffers from pool used for headers - - # We can upload large extensions - client_max_body_size 150M; - - # Don't timeout websockets quickly. Default is 60s. This is the timeout - # between reads/writes, not the full session timeout. - proxy_send_timeout 1h; - proxy_read_timeout 1h; - - access_log off; - upstream backend { - server localhost:3082 max_fails=0; - } - - client_body_temp_path dev/nginx/body; - - server { - listen 3081; - location / { - proxy_pass http://backend; - proxy_set_header Host $http_host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - } - - # Allow dev-specific gitignored custom nginx config (e.g. custom.nginx.conf) - include ./*.nginx.conf; -} diff --git a/dev/nginx/body/README b/dev/nginx/body/README deleted file mode 100644 index e6c8e1506b7bc..0000000000000 --- a/dev/nginx/body/README +++ /dev/null @@ -1 +0,0 @@ -This directory contains temporary files for nginx POST bodies (`client_body_temp_path`). diff --git a/dev/start.sh b/dev/start.sh index 8e5166a75b6ee..fe0db556176e4 100755 --- a/dev/start.sh +++ b/dev/start.sh @@ -91,7 +91,7 @@ export SOURCEGRAPH_HTTPS_PORT="${SOURCEGRAPH_HTTPS_PORT:-"3443"}" [ -n "${DISABLE_SEARCH_SHARDING-}" ] || export INDEXED_SEARCH_SERVERS="localhost:3070 localhost:3071" # webpack-dev-server is a proxy running on port 3080 that (1) serves assets, waiting to respond -# until they are (re)built and (2) otherwise proxies to nginx running on port 3081 (which proxies to +# until they are (re)built and (2) otherwise proxies to Caddy running on port 3081 (which proxies to # Sourcegraph running on port 3082). That is why Sourcegraph listens on 3082 despite the externalURL # having port 3080. export SRC_HTTP_ADDR=":3082" diff --git a/doc/dev/local_development.md b/doc/dev/local_development.md index 49102fb819fca..c7364aca020c0 100644 --- a/doc/dev/local_development.md +++ b/doc/dev/local_development.md @@ -74,7 +74,6 @@ Sourcegraph has the following dependencies: - [PostgreSQL](https://wiki.postgresql.org/wiki/Detailed_installation_guides) (v11 or higher) - [Redis](http://redis.io/) (v5.0.7 or higher) - [Yarn](https://yarnpkg.com) (v1.10.1 or higher) -- [NGINX](https://docs.nginx.com/nginx/admin-guide/installing-nginx/installing-nginx-open-source/) (v1.14 or higher) - [SQLite](https://www.sqlite.org/index.html) tools - [Golang Migrate](https://github.com/golang-migrate/migrate/) (v4.7.0 or higher) - [Comby](https://github.com/comby-tools/comby/) (v0.11.3 or higher) @@ -93,10 +92,10 @@ The following are two recommendations for installing these dependencies: brew cask install docker ``` -3. Install Go, Node Version Manager, PostgreSQL, Redis, Git, NGINX, golang-migrate, Comby, SQLite tools, and jq with the following command: +3. Install Go, Node Version Manager, PostgreSQL, Redis, Git, golang-migrate, Comby, SQLite tools, and jq with the following command: ``` - brew install go yarn redis postgresql git gnu-sed nginx golang-migrate comby sqlite pcre FiloSottile/musl-cross/musl-cross jq watchman + brew install go yarn redis postgresql git gnu-sed golang-migrate comby sqlite pcre FiloSottile/musl-cross/musl-cross jq watchman ``` 4. Install the Node Version Manager (`nvm`) using: @@ -194,7 +193,7 @@ The following are two recommendations for installing these dependencies: 3. Install dependencies: ``` - sudo apt install -y make git-all postgresql postgresql-contrib redis-server nginx libpcre3-dev libsqlite3-dev pkg-config golang-go musl-tools docker-ce docker-ce-cli containerd.io yarn jq + sudo apt install -y make git-all postgresql postgresql-contrib redis-server libpcre3-dev libsqlite3-dev pkg-config golang-go musl-tools docker-ce docker-ce-cli containerd.io yarn jq # install golang-migrate (you must rename the extracted binary to `golang-migrate` and move the binary into your $PATH) curl -L https://github.com/golang-migrate/migrate/releases/download/v4.7.0/migrate.linux-amd64.tar.gz | tar xvz diff --git a/enterprise/dev/Procfile b/enterprise/dev/Procfile index f8595245f3899..6b685e972ff99 100644 --- a/enterprise/dev/Procfile +++ b/enterprise/dev/Procfile @@ -6,7 +6,6 @@ symbols: symbols github-proxy: github-proxy frontend: env CONFIGURATION_MODE=server SITE_CONFIG_ESCAPE_HATCH_PATH=$HOME/.sourcegraph/site-config.json frontend watch: ./dev/changewatch.sh -nginx: nginx -p . -g 'daemon off;' -c $PWD/dev/nginx.conf 2>&1 | grep -v 'could not open error log file' caddy: ./dev/caddy.sh run --watch --config=dev/Caddyfile web: ./node_modules/.bin/gulp --color watch syntect_server: ./dev/syntect_server.sh From a17c1af654422bd4f99b929c37e3a593a66e1edc Mon Sep 17 00:00:00 2001 From: Keegan Carruthers-Smith Date: Tue, 16 Feb 2021 14:02:27 +0200 Subject: [PATCH 2/4] dev: use caddy 2.3.0 --- dev/caddy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/caddy.sh b/dev/caddy.sh index 0c7c34dbbf0ea..58bea8353af41 100755 --- a/dev/caddy.sh +++ b/dev/caddy.sh @@ -11,7 +11,7 @@ fi mkdir -p .bin -version="2.0.0" +version="2.3.0" case "$(go env GOOS)" in linux) os="linux" From 97c9b9b62a1ad319e437f17c28b891ebac3f20b0 Mon Sep 17 00:00:00 2001 From: Keegan Carruthers-Smith Date: Tue, 16 Feb 2021 16:08:29 +0200 Subject: [PATCH 3/4] caddyfile works on 2.3.0 --- dev/Caddyfile | 2 +- doc/dev/how-to/troubleshooting_local_development.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/Caddyfile b/dev/Caddyfile index 7d2f49f89cacd..af42ec6b512cf 100644 --- a/dev/Caddyfile +++ b/dev/Caddyfile @@ -13,6 +13,6 @@ } # Caddy (:3081) -> sourcegraph-frontend (:3082) -*:3081 { +:3081 { reverse_proxy localhost:3082 } diff --git a/doc/dev/how-to/troubleshooting_local_development.md b/doc/dev/how-to/troubleshooting_local_development.md index 5fbf78d0664fb..6d354089db157 100644 --- a/doc/dev/how-to/troubleshooting_local_development.md +++ b/doc/dev/how-to/troubleshooting_local_development.md @@ -89,7 +89,7 @@ If you ever need to wipe your local database and Redis, run the following comman We use Caddy 2 to setup HTTPS for local development. It creates self-signed certificates and uses that to serve the local Sourcegraph instance. If your browser complains about the certificate, check the following: -1. The first time that Caddy 2 reverse-proxies your Sourcegraph instance, it needs to add its certificate authority to your local certificate store. This may require elevated permissions on your machine. If you haven't done so already, try running `caddy reverse-proxy --to localhost:3080` and enter your password if prompted. You may also need to run that command as the `root` user. +1. The first time that Caddy 2 reverse-proxies your Sourcegraph instance, it needs to add its certificate authority to your local certificate store. This may require elevated permissions on your machine. If you haven't done so already, try running `./dev/caddy.sh reverse-proxy --to localhost:3080` and enter your password if prompted. You may also need to run that command as the `root` user. 1. If you have completed the previous step and your browser still complains about the certificate, try restarting your browser or your local machine. From 2cf5028d9a41b4907ffaf772d677b4c2d137922e Mon Sep 17 00:00:00 2001 From: Keegan Carruthers-Smith Date: Thu, 18 Feb 2021 16:33:14 +0200 Subject: [PATCH 4/4] doc: remove more references to nginx in dev --- README.md | 1 - .../getting-started/quickstart_1_install_dependencies.md | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 97980391379fc..0019dc4ba8226 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,6 @@ - Node.js (version 8 or 10) - Redis - Yarn -- Nginx For a detailed guide to installing prerequisites, see [these instructions](doc/dev/getting-started/quickstart_1_install_dependencies.md). diff --git a/doc/dev/getting-started/quickstart_1_install_dependencies.md b/doc/dev/getting-started/quickstart_1_install_dependencies.md index 9bf8f652e108c..d9e7516ea8ddd 100644 --- a/doc/dev/getting-started/quickstart_1_install_dependencies.md +++ b/doc/dev/getting-started/quickstart_1_install_dependencies.md @@ -12,7 +12,6 @@ Sourcegraph has the following dependencies: - [PostgreSQL](https://wiki.postgresql.org/wiki/Detailed_installation_guides) (v11 or higher) - [Redis](http://redis.io/) (v5.0.7 or higher) - [Yarn](https://yarnpkg.com) (v1.10.1 or higher) -- [NGINX](https://docs.nginx.com/nginx/admin-guide/installing-nginx/installing-nginx-open-source/) (v1.14 or higher) - [SQLite](https://www.sqlite.org/index.html) tools - [Golang Migrate](https://github.com/golang-migrate/migrate/) (v4.7.0 or higher) - [Comby](https://github.com/comby-tools/comby/) (v0.11.3 or higher) @@ -31,10 +30,10 @@ The following are two recommendations for installing these dependencies: brew cask install docker ``` -3. Install Go, Node Version Manager, PostgreSQL, Redis, Git, NGINX, golang-migrate, Comby, SQLite tools, and jq with the following command: +3. Install Go, Node Version Manager, PostgreSQL, Redis, Git, golang-migrate, Comby, SQLite tools, and jq with the following command: ``` - brew install go yarn redis postgresql git gnu-sed nginx golang-migrate comby sqlite pcre FiloSottile/musl-cross/musl-cross jq watchman + brew install go yarn redis postgresql git gnu-sed golang-migrate comby sqlite pcre FiloSottile/musl-cross/musl-cross jq watchman ``` 4. Install the Node Version Manager (`nvm`) using: @@ -132,7 +131,7 @@ The following are two recommendations for installing these dependencies: 3. Install dependencies: ``` - sudo apt install -y make git-all postgresql postgresql-contrib redis-server nginx libpcre3-dev libsqlite3-dev pkg-config golang-go musl-tools docker-ce docker-ce-cli containerd.io yarn jq + sudo apt install -y make git-all postgresql postgresql-contrib redis-server libpcre3-dev libsqlite3-dev pkg-config golang-go musl-tools docker-ce docker-ce-cli containerd.io yarn jq # install golang-migrate (you must rename the extracted binary to `golang-migrate` and move the binary into your $PATH) curl -L https://github.com/golang-migrate/migrate/releases/download/v4.7.0/migrate.linux-amd64.tar.gz | tar xvz