diff --git a/server/http_handler.go b/server/http_handler.go
index 5efbda887bab6..86a5c9c2fe2e5 100644
--- a/server/http_handler.go
+++ b/server/http_handler.go
@@ -2220,12 +2220,15 @@ func (h labelHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 
 	if len(labels) > 0 {
 		cfg := *config.GetGlobalConfig()
-		if cfg.Labels == nil {
-			cfg.Labels = make(map[string]string, len(labels))
-		}
-		for k, v := range labels {
-			cfg.Labels[k] = v
+		// Be careful of data race. The key & value of cfg.Labels must not be changed.
+		if cfg.Labels != nil {
+			for k, v := range cfg.Labels {
+				if _, found := labels[k]; !found {
+					labels[k] = v
+				}
+			}
 		}
+		cfg.Labels = labels
 		config.StoreGlobalConfig(&cfg)
 		logutil.BgLogger().Info("update server labels", zap.Any("labels", cfg.Labels))
 	}
diff --git a/server/http_handler_test.go b/server/http_handler_test.go
index 1253399898e81..03c3a846669f0 100644
--- a/server/http_handler_test.go
+++ b/server/http_handler_test.go
@@ -24,6 +24,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"math/rand"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -1199,3 +1200,44 @@ func TestSetLabels(t *testing.T) {
 	// reset the global variable
 	config.GetGlobalConfig().Labels = map[string]string{}
 }
+
+func TestSetLabelsConcurrentWithGetLabel(t *testing.T) {
+	ts := createBasicHTTPHandlerTestSuite()
+
+	ts.startServer(t)
+	defer ts.stopServer(t)
+
+	testUpdateLabels := func() {
+		labels := map[string]string{}
+		labels["zone"] = fmt.Sprintf("z-%v", rand.Intn(100000))
+		buffer := bytes.NewBuffer([]byte{})
+		require.Nil(t, json.NewEncoder(buffer).Encode(labels))
+		resp, err := ts.postStatus("/labels", "application/json", buffer)
+		require.NoError(t, err)
+		require.NotNil(t, resp)
+		defer func() {
+			require.NoError(t, resp.Body.Close())
+		}()
+		require.Equal(t, http.StatusOK, resp.StatusCode)
+		newLabels := config.GetGlobalConfig().Labels
+		require.Equal(t, newLabels, labels)
+	}
+	done := make(chan struct{})
+	go func() {
+		for {
+			select {
+			case <-done:
+				return
+			default:
+				config.GetGlobalConfig().GetTiKVConfig()
+			}
+		}
+	}()
+	for i := 0; i < 100; i++ {
+		testUpdateLabels()
+	}
+	close(done)
+
+	// reset the global variable
+	config.GetGlobalConfig().Labels = map[string]string{}
+}