Remove ctags (BloopAI#375)

* Remove from tauri

* Remove from github

* Remove from server

* Remove from flake

* Remove exclude_langs

* Remove overlay

* Remove the aarch64-darwin target for onnx

* Add common deps

.github/workflows/dependencies.yml

@@ -9,11 +9,11 @@ on:
       - ".github/workflows/dependencies.yml"
 
 jobs:
-  ctags-onnx-nix:
+  onnx-nix:
     strategy:
       fail-fast: false
       matrix:
-        package: [ ctags, onnxruntime ]
+        package: [ onnxruntime ]
         target: [
           x86_64-unknown-linux-gnu,
           x86_64-darwin,
@@ -29,16 +29,6 @@ jobs:
             os: macos-11
             cross: false
 
-          - target: aarch64-darwin
-            os: macos-11
-            cross: true
-
-        # Requires python for the build, which in a proper mess
-        # Just build this on a real machine instead of cross
-        exclude:
-          - package: onnxruntime
-            target: aarch64-darwin
-
     runs-on: ${{ matrix.os }}
     steps:
       - name: Setting variables
@@ -74,36 +64,6 @@ jobs:
           mkdir ${{ env.PACKAGE_NAME }}_bundle
           cp -r $package_path/* ${{ env.PACKAGE_NAME }}_bundle
 
-      - name: Copy macOS dynamic dependencies
-        if: matrix.os == 'macos-11' && ( matrix.package == 'ctags' )
-        env:
-          PACKAGE_NAME: ${{ steps.var.outputs.package-name }}
-        run: |
-          copy_deps() {
-              local lib=$2/$(basename $1)
-
-              cp -n $1 $lib || true
-              chmod +w $lib
-
-              local deps="$(otool -L $lib |awk '/\/nix\/store\// { gsub(":", ""); print $1 }' | tr '\n' ' ')"
-              for dep in $(echo $deps);
-              do
-                if [ ! -f "$2/$(basename $dep)" ]
-                then
-                  copy_deps $dep $2 || true
-                fi
-              done
-
-              for dep in $(echo $deps);
-              do
-                echo install_name_tool -change $dep @executable_path/$(basename $dep) $lib
-                install_name_tool -change $dep @executable_path/$(basename $dep) $lib
-              done
-          }
-
-          copy_deps ${{ env.package_path }}/bin/${{ matrix.package }}  ${{ env.PACKAGE_NAME }}_bundle
-          ls ${{ env.PACKAGE_NAME }}_bundle
-
       - name: Upload binaries
         env:
           PACKAGE_NAME: ${{ steps.var.outputs.package-name }}

.github/workflows/tauri-release.yml

@@ -76,18 +76,6 @@ jobs:
         run: |
           cargo --locked test --verbose --release -p bloop 
 
-      - name: Download binaries (windows) (windows is disabled)
-        if: matrix.name == 'windows-latest'
-        shell: bash
-        run: |
-          dirname=2022-12-05%2Fp5.9.20221204.0-5-gf9d21e7
-          filename=ctags-2022-12-05_p5.9.20221204.0-5-gf9d21e7-clang-x64.zip
-          curl -LO https://github.com/universal-ctags/ctags-win32/releases/download/$dirname/$filename
-          echo "7c7b7e2645347d092474a942f988988ee12e045d1d8e65ae7ced68ecaf672130 $filename" |sha256sum -c - 
-
-          unzip $filename || true
-          mv ctags.exe ${{ env.TAURI_BIN_PATH }}/ctags-$(rustc -Vv |grep host |cut -d\  -f2).exe
-
       - name: Check if Segment key exists
         env:
           segment_key: ${{ secrets.ANALYTICS_FE_WRITE_KEY_PROD }}

apps/desktop/src-tauri/src/backend.rs

@@ -5,7 +5,7 @@ use once_cell::sync::OnceCell;
 use sentry::ClientInitGuard;
 use tracing::info;
 
-use super::{plugin, relative_command_path, App, Manager, Payload, Runtime};
+use super::{plugin, App, Manager, Payload, Runtime};
 
 // a hack to get server/bleep/tests/desktop to run correctly
 #[cfg(not(test))]
@@ -35,7 +35,6 @@ where
         Configuration::from_cli().unwrap(),
     );
     configuration.qdrant_url = Some("http://127.0.0.1:6334".into());
-    configuration.ctags_path = relative_command_path("ctags");
     configuration.max_threads = bleep::default_parallelism() / 2;
     configuration.model_dir = app
         .path_resolver()

apps/desktop/src-tauri/tauri.conf.json

@@ -45,7 +45,6 @@
         "depends": []
       },
       "externalBin": [
-        "bin/ctags",
         "bin/qdrant"
       ],
       "icon": [

flake.nix

@@ -4,19 +4,12 @@
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
     flake-utils.url = "github:numtide/flake-utils";
-
-    rust-overlay = {
-      url = "github:oxalica/rust-overlay";
-      inputs.flake-utils.follows = "flake-utils";
-      inputs.nixpkgs.follows = "nixpkgs";
-    };
   };
 
-  outputs = { self, nixpkgs, flake-utils, rust-overlay }:
+  outputs = { self, nixpkgs, flake-utils }:
     flake-utils.lib.eachDefaultSystem (system:
       let
-        overlays = [ (import rust-overlay) ];
-        pkgs = import nixpkgs { inherit system overlays; };
+        pkgs = import nixpkgs { inherit system; };
         pkgsStatic = pkgs.pkgsStatic;
         lib = pkgs.lib;
 
@@ -25,14 +18,13 @@
         libclang = llvm.libclang;
         stdenv = llvm.stdenv;
 
-        rust = pkgs.rust-bin.stable.latest.default;
         rustPlatform = pkgs.makeRustPlatform {
-          cargo = rust;
-          rustc = rust;
+          cargo = pkgs.cargo;
+          rustc = pkgs.rustc;
         };
 
         runtimeDeps =
-          with pkgs; ([ openssl_1_1.out rocksdb universal-ctags git zlib ]);
+          with pkgs; ([ openssl_1_1.out rocksdb git zlib ]);
 
         buildDeps = with pkgs;
           ([
@@ -42,8 +34,6 @@
             openssl_1_1.out
             openssl_1_1.dev
 
-            rust
-
             protobuf
             onnxruntime-static
           ] ++ lib.optionals stdenv.isDarwin [
@@ -134,26 +124,12 @@
 
           };
 
-          ctags-static = pkgsStatic.universal-ctags.overrideAttrs (old: {
-            nativeBuildInputs = with pkgs; [ autoreconfHook perl pkg-config ];
-            doCheck = false;
-            checkFlags = [ ];
-          });
-
-          ctags-static-aarch64-darwin =
-            pkgs.pkgsCross.aarch64-darwin.pkgsStatic.universal-ctags.overrideAttrs
-            (old: {
-              nativeBuildInputs = with pkgs; [ autoreconfHook perl pkg-config ];
-              doCheck = false;
-              checkFlags = [ ];
-            });
-
           onnxruntime-static = onnxruntime-static;
         };
 
         devShell = (pkgs.mkShell {
           buildInputs = buildDeps ++ runtimeDeps ++ guiDeps
-            ++ (with pkgs; [ git-lfs ]);
+            ++ (with pkgs; [ git-lfs cargo rustc rustfmt clippy ]);
         }).overrideAttrs (old: envVars);
 
       });

server/bleep/benches/indexes.rs

@@ -1,7 +1,8 @@
 use bleep::{
     indexes::{reader::ContentReader, DocumentRead, File},
+    intelligence::TreeSitterFile,
     semantic::Semantic,
-    symbol::{Symbol, SymbolLocations},
+    symbol::SymbolLocations,
     Application, Configuration, Environment,
 };
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
@@ -10,16 +11,13 @@ use std::{path::Path, sync::Arc};
 use tantivy::doc;
 use tempdir::TempDir;
 
-async fn get_symbols() -> Vec<Symbol> {
-    let syms = bleep::ctags::get_symbols(&Path::new(".").canonicalize().unwrap(), &[]).await;
+async fn get_symbols() -> SymbolLocations {
+    let file = std::fs::read_to_string("./js-sample-big-symbols.js").unwrap();
+    let scope_graph = TreeSitterFile::try_build(file.as_bytes(), "javascript")
+        .and_then(TreeSitterFile::scope_graph)
+        .unwrap();
 
-    syms.into_iter()
-        .find(|(k, _)| {
-            k.components()
-                .any(|c| c.as_os_str() == "js-sample-big-symbols.js")
-        })
-        .map(|(_, v)| v)
-        .unwrap()
+    SymbolLocations::TreeSitter(scope_graph)
 }
 
 async fn index(index_dir: &Path) {
@@ -83,7 +81,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {
                     file.lang => &b"JavaScript"[..],
                     file.avg_line_length => 42.0,
                     file.last_commit_unix_seconds => 42.0,
-                    file.symbol_locations => bincode::serialize(&SymbolLocations::Ctags(symbols.clone())).unwrap(),
+                    file.symbol_locations => bincode::serialize(&symbols).unwrap(),
                 }
             },
             |tantivy_doc| ContentReader.read_document(black_box(&file), black_box(tantivy_doc)),