Language Server (carbon-language#3112)

Add a language server for carbon as part of GSoC.

This currently does code outline using toolchain parser.

See development steps in utils/vscode/README.md for running and using
language server.

.pre-commit-config.yaml

@@ -215,4 +215,5 @@ exclude: |
       bazel/patches/.*\.patch|
       third_party/examples/.*/carbon/.*|
       third_party/llvm-project/.*|
+      third_party/clangd/.*|
   )$

WORKSPACE

@@ -123,6 +123,7 @@ http_archive(
         "@carbon//bazel/patches/llvm:0001_Patch_for_mallinfo2_when_using_Bazel_build_system.patch",
         "@carbon//bazel/patches/llvm:0002_Added_Bazel_build_for_compiler_rt_fuzzer.patch",
         "@carbon//bazel/patches/llvm:0003_Modernize_py_binary_rule_for_lit.patch",
+        "@carbon//bazel/patches/llvm:0004_Add_library_for_clangd.patch",
     ],
     sha256 = "03a8eb4b243846ee037d700b048ec48a87eeef480cb129ab56aa7e0537172b98",
     strip_prefix = "llvm-project-{0}".format(llvm_version),

bazel/patches/llvm/0004_Add_library_for_clangd.patch

@@ -0,0 +1,77 @@
+From 2625e497fef1429f2041922a739d841050eda909 Mon Sep 17 00:00:00 2001
+From: maan2003 <manmeetmann2003@gmail.com>
+Date: Sat, 19 Aug 2023 02:32:03 +0530
+Subject: [PATCH] Add library for clangd
+
+This exports clangd language server protocol helpers publically.
+Feature.h needs Feature.inc which is generated during build process, so we
+disable all features for now.
+---
+ clang-tools-extra/clangd/BUILD.bazel  | 36 +++++++++++++++++++++++++++
+ clang-tools-extra/clangd/Protocol.cpp | 15 +++++++----
+ clang-tools-extra/clangd/Protocol.h   | 12 +++++++--
+ clang-tools-extra/clangd/Transport.h  |  1 -
+ 4 files changed, 56 insertions(+), 8 deletions(-)
+ create mode 100644 clang-tools-extra/clangd/BUILD.bazel
+
+diff --git a/clang-tools-extra/clangd/BUILD.bazel b/clang-tools-extra/clangd/BUILD.bazel
+new file mode 100644
+index 000000000..9f3f93f24
+--- /dev/null
++++ b/clang-tools-extra/clangd/BUILD.bazel
+@@ -0,0 +1,39 @@
++package(default_visibility = ["//visibility:public"])
++
++cc_library(
++    name = "clangd_library",
++    srcs = [
++        "JSONTransport.cpp",
++        "Protocol.cpp",
++        "URI.cpp",
++        "index/SymbolID.cpp",
++        "support/Logger.cpp",
++        "support/Trace.cpp",
++        "support/MemoryTree.cpp",
++        "support/Context.cpp",
++        "support/Cancellation.cpp",
++        "support/ThreadCrashReporter.cpp",
++        "support/Shutdown.cpp",
++    ],
++    hdrs = [
++        "Transport.h",
++        "Protocol.h",
++        "URI.h",
++        "LSPBinder.h",
++        "index/SymbolID.h",
++        "support/Function.h",
++        "support/Cancellation.h",
++        "support/ThreadCrashReporter.h",
++        "support/Logger.h",
++        "support/Trace.h",
++        "support/MemoryTree.h",
++        "support/Context.h",
++        "support/Shutdown.h",
++    ],
++    includes = ["."],
++    deps = [
++        "//llvm:Support",
++        "//clang:basic",
++        "//clang:index",
++    ],
++)
+diff --git a/clang-tools-extra/clangd/Transport.h b/clang-tools-extra/clangd/Transport.h
+index 4e80ea95b..f17441cfc 100644
+--- a/clang-tools-extra/clangd/Transport.h
++++ b/clang-tools-extra/clangd/Transport.h
+@@ -17,9 +17,8 @@
+
+ #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_TRANSPORT_H
+ #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_TRANSPORT_H
+
+-#include "Feature.h"
+ #include "llvm/ADT/StringRef.h"
+ #include "llvm/Support/JSON.h"
+ #include "llvm/Support/raw_ostream.h"
+
+--
+2.41.0

language_server/BUILD

@@ -0,0 +1,28 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+package(default_visibility = [
+    "//language_server:__subpackages__",
+])
+
+cc_binary(
+    name = "language_server",
+    srcs = [
+        "language_server.cpp",
+        "language_server.h",
+        "main.cpp",
+    ],
+    # Some parameters are unused in clangd headers.
+    copts = ["-Wno-unused-parameter"],
+    deps = [
+        "//common:error",
+        "//toolchain/diagnostics:null_diagnostics",
+        "//toolchain/lexer:tokenized_buffer",
+        "//toolchain/parser:parse_node_kind",
+        "//toolchain/parser:parse_tree",
+        "//toolchain/source:source_buffer",
+        "@llvm-project//clang-tools-extra/clangd:clangd_library",
+        "@llvm-project//llvm:Support",
+    ],
+)

language_server/language_server.cpp

@@ -0,0 +1,155 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "language_server/language_server.h"
+
+#include "clang-tools-extra/clangd/Protocol.h"
+#include "toolchain/diagnostics/null_diagnostics.h"
+#include "toolchain/lexer/tokenized_buffer.h"
+#include "toolchain/parser/parse_node_kind.h"
+#include "toolchain/parser/parse_tree.h"
+#include "toolchain/source/source_buffer.h"
+
+namespace Carbon::LS {
+
+void LanguageServer::OnDidOpenTextDocument(
+    clang::clangd::DidOpenTextDocumentParams const& params) {
+  files_.emplace(params.textDocument.uri.file(), params.textDocument.text);
+}
+
+void LanguageServer::OnDidChangeTextDocument(
+    clang::clangd::DidChangeTextDocumentParams const& params) {
+  // full text is sent if full sync is specified in capabilities.
+  assert(params.contentChanges.size() == 1);
+  std::string file = params.textDocument.uri.file().str();
+  files_[file] = params.contentChanges[0].text;
+}
+
+void LanguageServer::OnInitialize(
+    clang::clangd::NoParams const& client_capabilities,
+    clang::clangd::Callback<llvm::json::Object> cb) {
+  llvm::json::Object capabilities{{"documentSymbolProvider", true},
+                                  {"textDocumentSync", /*Full=*/1}};
+
+  llvm::json::Object reply{{"capabilities", std::move(capabilities)}};
+  cb(reply);
+};
+
+auto LanguageServer::onNotify(llvm::StringRef method, llvm::json::Value value)
+    -> bool {
+  if (method == "exit") {
+    return false;
+  }
+  if (auto handler = handlers_.NotificationHandlers.find(method);
+      handler != handlers_.NotificationHandlers.end()) {
+    handler->second(std::move(value));
+  } else {
+    clang::clangd::log("unhandled notification {0}", method);
+  }
+
+  return true;
+}
+
+auto LanguageServer::onCall(llvm::StringRef method, llvm::json::Value params,
+                            llvm::json::Value id) -> bool {
+  if (auto handler = handlers_.MethodHandlers.find(method);
+      handler != handlers_.MethodHandlers.end()) {
+    // TODO: improve this if add threads
+    handler->second(std::move(params),
+                    [&](llvm::Expected<llvm::json::Value> reply) {
+                      transport_->reply(id, std::move(reply));
+                    });
+  } else {
+    transport_->reply(
+        id, llvm::make_error<clang::clangd::LSPError>(
+                "method not found", clang::clangd::ErrorCode::MethodNotFound));
+  }
+
+  return true;
+}
+
+auto LanguageServer::onReply(llvm::json::Value /*id*/,
+                             llvm::Expected<llvm::json::Value> /*result*/)
+    -> bool {
+  return true;
+}
+
+// Returns the text of first child of kind ParseNodeKind::Name.
+static auto getName(ParseTree& p, ParseTree::Node node)
+    -> std::optional<llvm::StringRef> {
+  for (auto ch : p.children(node)) {
+    if (p.node_kind(ch) == ParseNodeKind::Name) {
+      return p.GetNodeText(ch);
+    }
+  }
+  return std::nullopt;
+}
+
+void LanguageServer::OnDocumentSymbol(
+    clang::clangd::DocumentSymbolParams const& params,
+    clang::clangd::Callback<std::vector<clang::clangd::DocumentSymbol>> cb) {
+  llvm::vfs::InMemoryFileSystem vfs;
+  auto file = params.textDocument.uri.file().str();
+  vfs.addFile(file, /*mtime=*/0,
+              llvm::MemoryBuffer::getMemBufferCopy(files_.at(file)));
+
+  auto buf = SourceBuffer::CreateFromFile(vfs, file);
+  auto lexed = TokenizedBuffer::Lex(*buf, NullDiagnosticConsumer());
+  auto parsed = ParseTree::Parse(lexed, NullDiagnosticConsumer(), nullptr);
+  std::vector<clang::clangd::DocumentSymbol> result;
+  for (const auto& node : parsed.postorder()) {
+    clang::clangd::SymbolKind symbol_kind;
+    switch (parsed.node_kind(node)) {
+      case ParseNodeKind::FunctionDeclaration:
+      case ParseNodeKind::FunctionDefinitionStart:
+        symbol_kind = clang::clangd::SymbolKind::Function;
+        break;
+      case ParseNodeKind::Namespace:
+        symbol_kind = clang::clangd::SymbolKind::Namespace;
+        break;
+      case ParseNodeKind::InterfaceDefinitionStart:
+      case ParseNodeKind::NamedConstraintDefinitionStart:
+        symbol_kind = clang::clangd::SymbolKind::Interface;
+        break;
+      case ParseNodeKind::ClassDefinitionStart:
+        symbol_kind = clang::clangd::SymbolKind::Class;
+        break;
+      default:
+        continue;
+    }
+
+    if (auto name = getName(parsed, node)) {
+      auto tok = parsed.node_token(node);
+      clang::clangd::Position pos{lexed.GetLineNumber(tok) - 1,
+                                  lexed.GetColumnNumber(tok) - 1};
+
+      clang::clangd::DocumentSymbol symbol{
+          .name = std::string(*name),
+          .kind = symbol_kind,
+          .range = {.start = pos, .end = pos},
+          .selectionRange = {.start = pos, .end = pos},
+      };
+
+      result.push_back(symbol);
+    }
+  }
+  cb(result);
+}
+
+void LanguageServer::Start() {
+  auto transport =
+      clang::clangd::newJSONTransport(stdin, llvm::outs(), nullptr, true);
+  LanguageServer ls(std::move(transport));
+  clang::clangd::LSPBinder binder(ls.handlers_, ls);
+  binder.notification("textDocument/didOpen", &ls,
+                      &LanguageServer::OnDidOpenTextDocument);
+  binder.notification("textDocument/didChange", &ls,
+                      &LanguageServer::OnDidChangeTextDocument);
+  binder.method("initialize", &ls, &LanguageServer::OnInitialize);
+  binder.method("textDocument/documentSymbol", &ls,
+                &LanguageServer::OnDocumentSymbol);
+  auto error = ls.transport_->loop(ls);
+  llvm::errs() << "Error: " << error << "\n";
+}
+}  // namespace Carbon::LS

language_server/language_server.h

@@ -0,0 +1,83 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef CARBON_LANGUAGE_SERVER_LANGUAGE_SERVER_H_
+#define CARBON_LANGUAGE_SERVER_LANGUAGE_SERVER_H_
+#include <unordered_map>
+#include <vector>
+
+#include "clang-tools-extra/clangd/LSPBinder.h"
+#include "clang-tools-extra/clangd/Protocol.h"
+#include "clang-tools-extra/clangd/Transport.h"
+#include "clang-tools-extra/clangd/support/Function.h"
+#include "toolchain/lexer/tokenized_buffer.h"
+#include "toolchain/parser/parse_tree.h"
+#include "toolchain/source/source_buffer.h"
+
+namespace Carbon::LS {
+class LanguageServer : public clang::clangd::Transport::MessageHandler,
+                       public clang::clangd::LSPBinder::RawOutgoing {
+ public:
+  // Start the language server.
+  static void Start();
+
+  // Transport::MessageHandler
+  // Handlers returns true to keep processing messages, or false to shut down.
+
+  // Handler called on notification by client.
+  auto onNotify(llvm::StringRef method, llvm::json::Value value)
+      -> bool override;
+  // Handler called on method call by client.
+  auto onCall(llvm::StringRef method, llvm::json::Value params,
+              llvm::json::Value id) -> bool override;
+  // Handler called on response of Transport::call.
+  auto onReply(llvm::json::Value id, llvm::Expected<llvm::json::Value> result)
+      -> bool override;
+
+  // LSPBinder::RawOutgoing
+
+  // Send method call to client
+  void callMethod(llvm::StringRef Method, llvm::json::Value Params,
+                  clang::clangd::Callback<llvm::json::Value> Reply) override {
+    // TODO: implement when needed
+  }
+
+  // Send notification to client
+  void notify(llvm::StringRef method, llvm::json::Value params) override {
+    transport_->notify(method, params);
+  }
+
+ private:
+  const std::unique_ptr<clang::clangd::Transport> transport_;
+  // content of files managed by the language client.
+  std::unordered_map<std::string, std::string> files_;
+  // handlers for client methods and notifications
+  clang::clangd::LSPBinder::RawHandlers handlers_;
+
+  explicit LanguageServer(std::unique_ptr<clang::clangd::Transport> transport)
+      : transport_(std::move(transport)) {}
+
+  // Typed handlers for notifications and method calls by client.
+
+  // Client opened a document.
+  void OnDidOpenTextDocument(
+      clang::clangd::DidOpenTextDocumentParams const& params);
+
+  // Client updated content of a document.
+  void OnDidChangeTextDocument(
+      clang::clangd::DidChangeTextDocumentParams const& params);
+
+  // Capabilities negotiation
+  void OnInitialize(clang::clangd::NoParams const& client_capabilities,
+                    clang::clangd::Callback<llvm::json::Object> cb);
+
+  // Code outline
+  void OnDocumentSymbol(
+      clang::clangd::DocumentSymbolParams const& params,
+      clang::clangd::Callback<std::vector<clang::clangd::DocumentSymbol>> cb);
+};
+
+}  // namespace Carbon::LS
+
+#endif  // CARBON_LANGUAGE_SERVER_LANGUAGE_SERVER_H_

language_server/main.cpp

@@ -0,0 +1,10 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "language_server/language_server.h"
+
+auto main(int /*argc*/, char** /*argv*/) -> int {
+  Carbon::LS::LanguageServer::Start();
+  return 0;
+}