REVIEW

.github/dependabot.yml

@@ -0,0 +1,13 @@
+version: 2
+updates:
+
+- package-ecosystem: github-actions
+  directory: "/"
+  schedule:
+    interval: daily
+    time: '00:00'
+    timezone: UTC
+  open-pull-requests-limit: 10
+  commit-message:
+    prefix: "Dependabot"
+    include: "scope"

.github/workflows/cron-flakes.yml

@@ -0,0 +1,69 @@
+name: "Flakes: Hourly import to Elasticsearch"
+
+on:
+
+  schedule:
+  - cron: '0 * * * *'
+
+jobs:
+
+  hourly-import-channel:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: true
+      matrix:
+        group:
+          - "manual"
+
+    env:
+      RUST_LOG: debug
+      FI_ES_EXISTS_STRATEGY: recreate
+      FI_ES_URL: ${{ secrets.ELASTICSEARCH_URL }}
+
+    steps:
+
+    - name: Checking out the repository
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Installing Nix
+      uses: cachix/install-nix-action@v13
+      with:
+        install_url: https://nixos-nix-install-tests.cachix.org/serve/i6laym9jw3wg9mw6ncyrk6gjx4l34vvx/install
+        install_options: '--tarball-url-prefix https://nixos-nix-install-tests.cachix.org/serve'
+        extra_nix_config: |
+          experimental-features = nix-command flakes
+          access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
+
+    - uses: cachix/cachix-action@v10
+      with:
+        name: nixos-search
+        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
+
+    - name: Install unstable channel
+      run: |
+        nix-channel --add https://nixos.org/channels/nixpkgs-unstable
+        nix-channel --update
+
+    - name: Installing jq
+      run: |
+        nix-env -iA nixpkgs.nixFlakes nixpkgs.jq
+
+    - name: Building import_scripts
+      run: |
+        nix build ./#packages.x86_64-linux.flake_info
+
+    - name: Import ${{ matrix.group }} group
+      run: |
+        ./result/bin/flake-info --push --elastic-schema-version=$(cat ./VERSION) group ./flakes/${{ matrix.group }}.toml ${{ matrix.group }}
+      if: github.repository == 'NixOS/nixos-search'
+
+    - name: Warmup ${{ matrix.group }} channel
+      run: |
+        curl ${{ secrets.ELASTICSEARCH_URL }}/$(cat VERSION)-${{ matrix.group }}/_search | jq '.took'
+        curl ${{ secrets.ELASTICSEARCH_URL }}/$(cat VERSION)-${{ matrix.group }}/_search | jq '.took'
+        curl ${{ secrets.ELASTICSEARCH_URL }}/$(cat VERSION)-${{ matrix.group }}/_search | jq '.took'
+      if: github.repository == 'NixOS/nixos-search'

.github/workflows/cron-nixpkgs.yml

@@ -0,0 +1,72 @@
+name: "Nixpkgs: Hourly import to Elasticsearch"
+
+on:
+
+  schedule:
+  - cron: '0 * * * *'
+  pull_request:
+
+jobs:
+
+  import-channel:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: true
+      matrix:
+        channel:
+          - unstable
+          - 21.05
+          - 20.09
+
+    env:
+      RUST_LOG: debug
+      FI_ES_EXISTS_STRATEGY: abort
+      FI_ES_URL: ${{ secrets.ELASTICSEARCH_URL }}
+
+    steps:
+
+    - name: Checking out the repository
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Installing Nix
+      uses: cachix/install-nix-action@v13
+      with:
+        install_url: https://nixos-nix-install-tests.cachix.org/serve/i6laym9jw3wg9mw6ncyrk6gjx4l34vvx/install
+        install_options: '--tarball-url-prefix https://nixos-nix-install-tests.cachix.org/serve'
+        extra_nix_config: |
+          experimental-features = nix-command flakes
+          access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
+
+    - uses: cachix/cachix-action@v10
+      with:
+        name: nixos-search
+        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
+
+    - name: Install unstable channel
+      run: |
+        nix-channel --add https://nixos.org/channels/nixpkgs-unstable
+        nix-channel --update
+
+    - name: Installing jq
+      run: |
+        nix-env -iA nixpkgs.nixFlakes nixpkgs.jq
+
+    - name: Building import_scripts
+      run: |
+        nix build ./#packages.x86_64-linux.flake_info
+
+    - name: Import ${{ matrix.channel }} channel
+      run: |
+        ./result/bin/flake-info --push --elastic-schema-version=$(cat ./VERSION) nixpkgs ${{ matrix.channel }}
+      if: github.repository == 'NixOS/nixos-search'
+
+    - name: Warmup ${{ matrix.channel }} channel
+      run: |
+        curl ${{ secrets.ELASTICSEARCH_URL }}/latest-$(cat VERSION)-nixpkgs-${{ matrix.channel }}/_search | jq '.took'
+        curl ${{ secrets.ELASTICSEARCH_URL }}/latest-$(cat VERSION)-nixpkgs-${{ matrix.channel }}/_search | jq '.took'
+        curl ${{ secrets.ELASTICSEARCH_URL }}/latest-$(cat VERSION)-nixpkgs-${{ matrix.channel }}/_search | jq '.took'
+      if: github.repository == 'NixOS/nixos-search'

.github/workflows/main.yml

@@ -0,0 +1,58 @@
+name: "Frontend: Build & Deploy to Netlify"
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checking out the repository
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Installing Nix
+      uses: cachix/install-nix-action@v13
+
+    - uses: cachix/cachix-action@v10
+      with:
+        name: nixos-search
+        signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
+
+    - name: Install unstable channel
+      run: |
+        nix-channel --add https://nixos.org/channels/nixpkgs-unstable
+        nix-channel --update
+
+    - name: Installing NixFlakes
+      run: |
+        nix-env -iA nixpkgs.nixFlakes
+        echo 'experimental-features = nix-command flakes' | sudo tee -a /etc/nix/nix.conf
+        nix --version
+        cat /etc/nix/nix.conf
+        echo "$HOME/.nix-profile/bin" >> $GITHUB_PATH
+
+    - name: Building search.nixos.org
+      run: |
+        nix build ./#packages.x86_64-linux.frontend
+        mkdir ./dist
+        cp -RL ./result/* ./dist/
+
+    - name: Deploy to Netlify
+      uses: nwtgck/actions-netlify@v1.2
+      env:
+        NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+        NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+      with:
+        production-branch: 'main'
+        production-deploy: ${{ github.event_name == 'push' }}
+        publish-dir: './dist'
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        deploy-message: 'Deploy from GitHub Actions'
+        enable-pull-request-comment: true
+        enable-commit-comment: true
+        enable-commit-status: true
+        overwrites-pull-request-comment: false
+      if: github.repository == 'NixOS/nixos-search'

.gitignore

@@ -0,0 +1,22 @@
+*.log
+.DS_Store
+.cache
+.idea
+.vscode
+.node_repl_history
+.npm
+target/
+build/Release
+dist
+elm-stuff/
+eval-*
+ignore
+import-scripts/import_scripts/__pycache__/
+import-scripts/tests/__pycache__/
+logs
+node_modules
+npm-debug.log*
+package-lock.json
+repl-temp-*
+result
+src-url

README.md

@@ -0,0 +1,60 @@
+# search.nixos.org
+
+This repository contains the scripts and the web application for
+`search.nixos.org`. 
+
+
+## How this project came to be
+
+Initial idea was to replace NixOS packages and options search which was
+fetching one JSON file which contained all packages (or options). This approach
+is good for its simple setup, but started to show its problems when packages
+number was getting bigger and bigger. I'm sure we could optimize it further,
+but ideas what all could we do if there would be some database in the back were
+to tempting not to try.
+
+For backend we are using Elasticsearch instance which is kindly sponsored by
+[bonsai.io](https://bonsai.io). On the frontend we are using
+[Elm](https://elm-lang.org).
+
+
+## How search works?
+
+The use case we want to solve is that a visitor want to see if a package
+exists or to look up certain package's details.
+
+A user wants to converge to a single result if possible. The more characters
+are added to a search query the more narrow is search is and we should show
+less results.
+
+Very important is also ranking of search results. This will bring more relevant
+search results to the top, since a lot of times it is hard to produce search
+query that will output only one result item.
+
+A less important, but providing better user experience. are suggestions for
+writing better search query. Suggesting feature should guide user to write
+better queries which in turn will produce better results.
+
+
+## Development
+
+To start developing open a terminal and run:
+
+```
+$ nix-shell --run "yarn dev"
+```
+
+You can point your browser to `http://localhost:3000` and start developing.
+Any changes to source files (`./src`) will trigger a hot reload of an
+application.
+
+
+## Deploying
+
+- On each commit to `main` branch a GitHub Action is trigger.
+- GitHub Action then builds production version of the web application using
+  `yarn prod` command.
+- The built web application (in `./dist`) is then deployed to Netlify.
+- GitHub Action can also be triggered via Pull Request, which if Pull Request
+  was created from a non-forked repo's branch, will provide a preview url in a
+  comment.

VERSION

@@ -0,0 +1 @@
+22

default.nix

@@ -0,0 +1,88 @@
+{ pkgs ? import <nixpkgs> { }
+, version ? pkgs.lib.removeSuffix "\n" (builtins.readFile ./VERSION)
+}:
+let
+  package = builtins.fromJSON (builtins.readFile ./package.json);
+  yarnPkg = pkgs.yarn2nix-moretea.mkYarnPackage rec {
+    name = "${package.name}-yarn-${package.version}";
+    src = null;
+    dontUnpack = true;
+    packageJSON = ./package.json;
+    yarnLock = ./yarn.lock;
+    preConfigure = ''
+      mkdir ${package.name}
+      cd ${package.name}
+      ln -s ${packageJSON} ./package.json
+      ln -s ${yarnLock} ./yarn.lock
+    '';
+    yarnPreBuild = ''
+      mkdir -p $HOME/.node-gyp/${pkgs.nodejs.version}
+      echo 9 > $HOME/.node-gyp/${pkgs.nodejs.version}/installVersion
+      ln -sfv ${pkgs.nodejs}/include $HOME/.node-gyp/${pkgs.nodejs.version}
+    '';
+    publishBinsFor =
+      [
+        "webpack"
+        "webpack-dev-server"
+      ];
+  };
+in
+pkgs.stdenv.mkDerivation {
+  name = "${package.name}-${package.version}";
+  src = pkgs.lib.cleanSource ./.;
+
+  preferLocalBuild = true;
+
+  buildInputs =
+    [
+      yarnPkg
+    ] ++
+    (with pkgs; [
+      nodejs
+      elm2nix
+    ]) ++
+    (with pkgs.nodePackages; [
+      yarn
+    ]) ++
+    (with pkgs.elmPackages; [
+      elm
+      elm-format
+      elm-language-server
+      elm-test
+      elm-analyse
+    ]);
+
+  configurePhase = pkgs.elmPackages.fetchElmDeps {
+    elmPackages = import ./elm-srcs.nix;
+    elmVersion = pkgs.elmPackages.elm.version;
+    registryDat = ./registry.dat;
+  };
+
+  patchPhase = ''
+    rm -rf node_modules
+    ln -sf ${yarnPkg}/libexec/${package.name}/node_modules .
+  '';
+
+  buildPhase = ''
+    # Yarn writes cache directories etc to $HOME.
+    export HOME=$PWD/yarn_home
+    sed -i -e "s|process.env.ELASTICSEARCH_MAPPING_SCHEMA_VERSION|${version}|" src/index.js
+    yarn prod
+  '';
+
+  installPhase = ''
+    mkdir -p $out
+    cp -R ./dist/* $out/
+    cp netlify.toml $out/
+  '';
+  shellHook = ''
+    rm -rf node_modules
+    ln -sf ${yarnPkg}/libexec/${package.name}/node_modules .
+    export PATH=$PWD/node_modules/.bin:$PATH
+    export ELASTICSEARCH_MAPPING_SCHEMA_VERSION=${version}
+    echo "============================"
+    echo "= To develop run: yarn dev ="
+    echo "============================"
+  '';
+
+}