🍞 fix: Minor fixes and improved Bun support (danny-avila#1916)

* fix(bun): fix bun compatibility to allow gzip header: oven-sh/bun#267 (comment)

* chore: update custom config examples

* fix(OpenAIClient.chatCompletion): remove redundant call of stream.controller.abort() as `break` aborts the request and prevents abort errors when not called redundantly

* chore: bump bun.lockb

* fix: remove result-thinking class when message is no longer streaming

* fix(bun): improve Bun support by forcing use of old method in bun env, also update old methods with new customizable params

* fix(ci): pass tests

api/app/clients/ChatGPTClient.js

@@ -1,9 +1,16 @@
-const crypto = require('crypto');
 const Keyv = require('keyv');
+const crypto = require('crypto');
+const {
+  EModelEndpoint,
+  resolveHeaders,
+  mapModelToAzureConfig,
+} = require('librechat-data-provider');
 const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken');
 const { fetchEventSource } = require('@waylaidwanderer/fetch-event-source');
 const { Agent, ProxyAgent } = require('undici');
 const BaseClient = require('./BaseClient');
+const { logger } = require('~/config');
+const { extractBaseURL, constructAzureURL, genAzureChatCompletion } = require('~/utils');
 
 const CHATGPT_MODEL = 'gpt-3.5-turbo';
 const tokenizersCache = {};
@@ -144,7 +151,8 @@ class ChatGPTClient extends BaseClient {
     if (!abortController) {
       abortController = new AbortController();
     }
-    const modelOptions = { ...this.modelOptions };
+
+    let modelOptions = { ...this.modelOptions };
     if (typeof onProgress === 'function') {
       modelOptions.stream = true;
     }
@@ -159,56 +167,171 @@ class ChatGPTClient extends BaseClient {
     }
 
     const { debug } = this.options;
-    const url = this.completionsUrl;
+    let baseURL = this.completionsUrl;
     if (debug) {
       console.debug();
-      console.debug(url);
+      console.debug(baseURL);
       console.debug(modelOptions);
       console.debug();
     }
 
-    if (this.azure || this.options.azure) {
-      // Azure does not accept `model` in the body, so we need to remove it.
-      delete modelOptions.model;
-    }
-
     const opts = {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
       },
-      body: JSON.stringify(modelOptions),
       dispatcher: new Agent({
         bodyTimeout: 0,
         headersTimeout: 0,
       }),
     };
 
-    if (this.apiKey && this.options.azure) {
-      opts.headers['api-key'] = this.apiKey;
+    if (this.isVisionModel) {
+      modelOptions.max_tokens = 4000;
+    }
+
+    /** @type {TAzureConfig | undefined} */
+    const azureConfig = this.options?.req?.app?.locals?.[EModelEndpoint.azureOpenAI];
+
+    const isAzure = this.azure || this.options.azure;
+    if (
+      (isAzure && this.isVisionModel && azureConfig) ||
+      (azureConfig && this.isVisionModel && this.options.endpoint === EModelEndpoint.azureOpenAI)
+    ) {
+      const { modelGroupMap, groupMap } = azureConfig;
+      const {
+        azureOptions,
+        baseURL,
+        headers = {},
+        serverless,
+      } = mapModelToAzureConfig({
+        modelName: modelOptions.model,
+        modelGroupMap,
+        groupMap,
+      });
+      opts.headers = resolveHeaders(headers);
+      this.langchainProxy = extractBaseURL(baseURL);
+      this.apiKey = azureOptions.azureOpenAIApiKey;
+
+      const groupName = modelGroupMap[modelOptions.model].group;
+      this.options.addParams = azureConfig.groupMap[groupName].addParams;
+      this.options.dropParams = azureConfig.groupMap[groupName].dropParams;
+      // Note: `forcePrompt` not re-assigned as only chat models are vision models
+
+      this.azure = !serverless && azureOptions;
+      this.azureEndpoint =
+        !serverless && genAzureChatCompletion(this.azure, modelOptions.model, this);
+    }
+
+    if (this.options.headers) {
+      opts.headers = { ...opts.headers, ...this.options.headers };
+    }
+
+    if (isAzure) {
+      // Azure does not accept `model` in the body, so we need to remove it.
+      delete modelOptions.model;
+
+      baseURL = this.langchainProxy
+        ? constructAzureURL({
+          baseURL: this.langchainProxy,
+          azure: this.azure,
+        })
+        : this.azureEndpoint.split(/\/(chat|completion)/)[0];
+
+      if (this.options.forcePrompt) {
+        baseURL += '/completions';
+      } else {
+        baseURL += '/chat/completions';
+      }
+
+      opts.defaultQuery = { 'api-version': this.azure.azureOpenAIApiVersion };
+      opts.headers = { ...opts.headers, 'api-key': this.apiKey };
     } else if (this.apiKey) {
       opts.headers.Authorization = `Bearer ${this.apiKey}`;
     }
 
+    if (process.env.OPENAI_ORGANIZATION) {
+      opts.headers['OpenAI-Organization'] = process.env.OPENAI_ORGANIZATION;
+    }
+
     if (this.useOpenRouter) {
       opts.headers['HTTP-Referer'] = 'https://librechat.ai';
       opts.headers['X-Title'] = 'LibreChat';
     }
 
-    if (this.options.headers) {
-      opts.headers = { ...opts.headers, ...this.options.headers };
-    }
-
     if (this.options.proxy) {
       opts.dispatcher = new ProxyAgent(this.options.proxy);
     }
 
+    /* hacky fixes for Mistral AI API:
+      - Re-orders system message to the top of the messages payload, as not allowed anywhere else
+      - If there is only one message and it's a system message, change the role to user
+      */
+    if (baseURL.includes('https://api.mistral.ai/v1') && modelOptions.messages) {
+      const { messages } = modelOptions;
+
+      const systemMessageIndex = messages.findIndex((msg) => msg.role === 'system');
+
+      if (systemMessageIndex > 0) {
+        const [systemMessage] = messages.splice(systemMessageIndex, 1);
+        messages.unshift(systemMessage);
+      }
+
+      modelOptions.messages = messages;
+
+      if (messages.length === 1 && messages[0].role === 'system') {
+        modelOptions.messages[0].role = 'user';
+      }
+    }
+
+    if (this.options.addParams && typeof this.options.addParams === 'object') {
+      modelOptions = {
+        ...modelOptions,
+        ...this.options.addParams,
+      };
+      logger.debug('[ChatGPTClient] chatCompletion: added params', {
+        addParams: this.options.addParams,
+        modelOptions,
+      });
+    }
+
+    if (this.options.dropParams && Array.isArray(this.options.dropParams)) {
+      this.options.dropParams.forEach((param) => {
+        delete modelOptions[param];
+      });
+      logger.debug('[ChatGPTClient] chatCompletion: dropped params', {
+        dropParams: this.options.dropParams,
+        modelOptions,
+      });
+    }
+
+    if (baseURL.includes('v1') && !baseURL.includes('/completions') && !this.isChatCompletion) {
+      baseURL = baseURL.split('v1')[0] + 'v1/completions';
+    } else if (
+      baseURL.includes('v1') &&
+      !baseURL.includes('/chat/completions') &&
+      this.isChatCompletion
+    ) {
+      baseURL = baseURL.split('v1')[0] + 'v1/chat/completions';
+    }
+
+    const BASE_URL = new URL(baseURL);
+    if (opts.defaultQuery) {
+      Object.entries(opts.defaultQuery).forEach(([key, value]) => {
+        BASE_URL.searchParams.append(key, value);
+      });
+      delete opts.defaultQuery;
+    }
+
+    const completionsURL = BASE_URL.toString();
+    opts.body = JSON.stringify(modelOptions);
+
     if (modelOptions.stream) {
       // eslint-disable-next-line no-async-promise-executor
       return new Promise(async (resolve, reject) => {
         try {
           let done = false;
-          await fetchEventSource(url, {
+          await fetchEventSource(completionsURL, {
             ...opts,
             signal: abortController.signal,
             async onopen(response) {
@@ -236,7 +359,6 @@ class ChatGPTClient extends BaseClient {
               // workaround for private API not sending [DONE] event
               if (!done) {
                 onProgress('[DONE]');
-                abortController.abort();
                 resolve();
               }
             },
@@ -249,14 +371,13 @@ class ChatGPTClient extends BaseClient {
             },
             onmessage(message) {
               if (debug) {
-                // console.debug(message);
+                console.debug(message);
               }
               if (!message.data || message.event === 'ping') {
                 return;
               }
               if (message.data === '[DONE]') {
                 onProgress('[DONE]');
-                abortController.abort();
                 resolve();
                 done = true;
                 return;
@@ -269,7 +390,7 @@ class ChatGPTClient extends BaseClient {
         }
       });
     }
-    const response = await fetch(url, {
+    const response = await fetch(completionsURL, {
       ...opts,
       signal: abortController.signal,
     });

api/app/clients/OpenAIClient.js

@@ -560,7 +560,7 @@ class OpenAIClient extends BaseClient {
     let streamResult = null;
     this.modelOptions.user = this.user;
     const invalidBaseUrl = this.completionsUrl && extractBaseURL(this.completionsUrl) === null;
-    const useOldMethod = !!(invalidBaseUrl || !this.isChatCompletion);
+    const useOldMethod = !!(invalidBaseUrl || !this.isChatCompletion || typeof Bun !== 'undefined');
     if (typeof opts.onProgress === 'function' && useOldMethod) {
       await this.getCompletion(
         payload,

api/server/index.js

@@ -2,6 +2,7 @@ require('dotenv').config();
 const path = require('path');
 require('module-alias')({ base: path.resolve(__dirname, '..') });
 const cors = require('cors');
+const axios = require('axios');
 const express = require('express');
 const passport = require('passport');
 const mongoSanitize = require('express-mongo-sanitize');
@@ -22,6 +23,9 @@ const port = Number(PORT) || 3080;
 const host = HOST || 'localhost';
 
 const startServer = async () => {
+  if (typeof Bun !== 'undefined') {
+    axios.defaults.headers.common['Accept-Encoding'] = 'gzip';
+  }
   await connectDb();
   logger.info('Connected to MongoDB');
   await indexSync();

client/src/components/Chat/Messages/Content/Markdown.tsx

@@ -10,7 +10,7 @@ import rehypeHighlight from 'rehype-highlight';
 import type { TMessage } from 'librechat-data-provider';
 import type { PluggableList } from 'unified';
 import CodeBlock from '~/components/Messages/Content/CodeBlock';
-import { langSubset, validateIframe, processLaTeX } from '~/utils';
+import { cn, langSubset, validateIframe, processLaTeX } from '~/utils';
 import { useChatContext } from '~/Providers';
 import store from '~/store';
 
@@ -75,7 +75,7 @@ const Markdown = memo(({ content, message, showCursor }: TContentProps) => {
     return (
       <div className="absolute">
         <p className="relative">
-          <span className="result-thinking" />
+          <span className={cn(isSubmitting ? 'result-thinking' : '')} />
         </p>
       </div>
     );

docs/install/configuration/ai_endpoints.md

@@ -35,7 +35,6 @@ Some of the endpoints are marked as **Known,** which means they might have speci
           ]
         fetch: false
       titleConvo: true
-      titleMethod: "completion"
       titleModel: "mixtral-8x7b-32768"
       modelDisplayLabel: "groq"
       iconURL: "https://raw.githubusercontent.com/fuegovic/lc-config-yaml/main/icons/groq.png"
@@ -64,7 +63,6 @@ Some of the endpoints are marked as **Known,** which means they might have speci
         default: ["mistral-tiny", "mistral-small", "mistral-medium", "mistral-large-latest"]
         fetch: true
       titleConvo: true
-      titleMethod: "completion"
       titleModel: "mistral-tiny"
       modelDisplayLabel: "Mistral"
       # Drop Default params parameters from the request. See default params in guide linked below.
@@ -81,7 +79,7 @@ Some of the endpoints are marked as **Known,** which means they might have speci
 
 - **Known:** icon provided, fetching list of models is recommended as API token rates and pricing used for token credit balances when models are fetched.
 
-- API may be strict for some models, and may not allow fields like `stop`, in which case, you should use [`dropParams`.](./custom_config.md#dropparams)
+- It's recommended, and for some models required, to use [`dropParams`](./custom_config.md#dropparams) to drop the `stop` as Openrouter models use a variety of stop tokens.
 
 - Known issue: you should not use `OPENROUTER_API_KEY` as it will then override the `openAI` endpoint to use OpenRouter as well.
 
@@ -95,9 +93,10 @@ Some of the endpoints are marked as **Known,** which means they might have speci
         default: ["gpt-3.5-turbo"]
         fetch: true
       titleConvo: true
-      titleMethod: "completion"
       titleModel: "gpt-3.5-turbo" # change to your preferred model
       modelDisplayLabel: "OpenRouter"
+      # Recommended: Drop the stop parameter from the request as Openrouter models use a variety of stop tokens.
+      dropParams: ["stop"]
 ```
 
 ![image](https://github.com/danny-avila/LibreChat/assets/110412045/c4a0415e-732c-46af-82a6-3598663b7f42)