[Backport release-v0.7] fix: cache the server version info of kuberne…

…tes (openyurtio#977)

* fix: cache the server version info of kubernetes

(cherry picked from commit 1112bf3)

* fix: fix the errors according to the comments

(cherry picked from commit 4f79e7e)

* fix: fix the bug of cache the non-resource info update

(cherry picked from commit 41292c2)

* fix: delete the change before

(cherry picked from commit ec2880c)

* fix: update the change

(cherry picked from commit c9bf2e1)

* fix: update the change

(cherry picked from commit c90db6b)

* fix: update the unit test

(cherry picked from commit 0656362)

* fix: fix the go pipeline error

(cherry picked from commit e2ebb76)

* fix: fix the errors

(cherry picked from commit 7bab3ab)

Co-authored-by: Yuxuan <sodawyx@126.com>

cmd/yurthub/app/start.go

@@ -184,7 +184,7 @@ func Run(cfg *config.YurtHubConfiguration, stopCh <-chan struct{}) error {
 	cfg.YurtSharedFactory.Start(stopCh)
 
 	klog.Infof("%d. new %s server and begin to serve, proxy server: %s, secure proxy server: %s, hub server: %s", trace, projectinfo.GetHubName(), cfg.YurtHubProxyServerAddr, cfg.YurtHubProxyServerSecureAddr, cfg.YurtHubServerAddr)
-	s, err := server.NewYurtHubServer(cfg, certManager, yurtProxyHandler)
+	s, err := server.NewYurtHubServer(cfg, certManager, yurtProxyHandler, restConfigMgr)
 	if err != nil {
 		return fmt.Errorf("could not create hub server, %w", err)
 	}

pkg/yurthub/cachemanager/cache_manager.go

@@ -561,6 +561,7 @@ func isCreate(ctx context.Context) bool {
 // 4. csr resource request
 func (cm *cacheManager) CanCacheFor(req *http.Request) bool {
 	ctx := req.Context()
+
 	comp, ok := util.ClientComponentFrom(ctx)
 	if !ok || len(comp) == 0 {
 		return false

pkg/yurthub/server/nonresource.go

@@ -0,0 +1,142 @@
+/*
+Copyright 2020 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"github.com/gorilla/mux"
+	"github.com/pkg/errors"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+
+	"github.com/openyurtio/openyurt/cmd/yurthub/app/config"
+	"github.com/openyurtio/openyurt/pkg/yurthub/cachemanager"
+)
+
+var nonResourceReqPaths = []string{
+	"/version",
+	"/apis/discovery.k8s.io/v1",
+	"/apis/discovery.k8s.io/v1beta1",
+}
+
+var cfg *config.YurtHubConfiguration
+var clientSet *kubernetes.Clientset
+
+func wrapNonResourceHandler(proxyHandler http.Handler, config *config.YurtHubConfiguration, cs *kubernetes.Clientset) http.Handler {
+	wrapMux := mux.NewRouter()
+	cfg = config
+	clientSet = cs
+	// register handler for non resource requests
+	for i := range nonResourceReqPaths {
+		wrapMux.HandleFunc(nonResourceReqPaths[i], withNonResourceRequest).Methods("GET")
+	}
+
+	// register handler for other requests
+	wrapMux.PathPrefix("/").Handler(proxyHandler)
+	return wrapMux
+}
+
+func withNonResourceRequest(w http.ResponseWriter, req *http.Request) {
+	for i := range nonResourceReqPaths {
+		if req.URL.Path == nonResourceReqPaths[i] {
+			cacheNonResourceInfo(w, req, cfg.StorageWrapper, fmt.Sprintf("non-reosurce-info%s", nonResourceReqPaths[i]), nonResourceReqPaths[i], false)
+			break
+		}
+	}
+
+}
+
+func cacheNonResourceInfo(w http.ResponseWriter, req *http.Request, sw cachemanager.StorageWrapper, key string, path string, isFake bool) {
+	nonResourceInfo, err := getClientSetQueryInfo(path, isFake)
+	copyHeader(w.Header(), req.Header)
+	if err == nil {
+		_, err = w.Write(nonResourceInfo)
+		if err != nil {
+			klog.Errorf("failed to write the non-resource info, the error is: %v", err)
+			ErrNonResource(err, w, req)
+			return
+		}
+
+		klog.Infof("success to query the cache non-resource info: %s", key)
+		w.WriteHeader(http.StatusOK)
+		sw.UpdateRaw(key, nonResourceInfo)
+	} else {
+		infoCache, err := sw.GetRaw(key)
+		if err != nil {
+			klog.Errorf("the non-resource info cannot be acquired, the error is: %v", err)
+			ErrNonResource(err, w, req)
+			return
+		}
+		_, err = w.Write(infoCache)
+		if err != nil {
+			klog.Errorf("failed to write the non-resource info, the error is: %v", err)
+			ErrNonResource(err, w, req)
+			return
+		}
+
+		klog.Infof("success to non-resource info: %s", key)
+		w.WriteHeader(http.StatusOK)
+	}
+
+}
+
+func getClientSetQueryInfo(path string, isFake bool) ([]byte, error) {
+	if clientSet == nil {
+		// used for unit test
+		if isFake {
+			return []byte(fmt.Sprintf("fake-non-resource-info-%s", path)), nil
+		}
+		klog.Errorf("the kubernetes clientSet is nil, and return the fake value for test")
+		return nil, errors.New("the kubernetes clientSet is nil")
+	}
+	nonResourceInfo, err := clientSet.RESTClient().Get().AbsPath(path).Do(context.TODO()).Raw()
+	if err != nil {
+		return nil, err
+	}
+	return nonResourceInfo, nil
+}
+
+func copyHeader(dst, src http.Header) {
+	for k, vv := range src {
+		if k == "Content-Type" || k == "Content-Length" {
+			for _, v := range vv {
+				dst.Add(k, v)
+			}
+		}
+	}
+}
+
+func ErrNonResource(err error, w http.ResponseWriter, req *http.Request) {
+	status := responsewriters.ErrorToAPIStatus(err)
+	code := int(status.Code)
+	// when writing an error, check to see if the status indicates a retry after period
+	if status.Details != nil && status.Details.RetryAfterSeconds > 0 {
+		delay := strconv.Itoa(int(status.Details.RetryAfterSeconds))
+		w.Header().Set("Retry-After", delay)
+	}
+
+	if code == http.StatusNoContent {
+		w.WriteHeader(code)
+	}
+	klog.Errorf("%v counter the error %v", req.URL, err)
+
+}

pkg/yurthub/server/nonresource_test.go

@@ -0,0 +1,120 @@
+/*
+Copyright 2020 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/openyurtio/openyurt/pkg/yurthub/cachemanager"
+	"github.com/openyurtio/openyurt/pkg/yurthub/storage/disk"
+)
+
+var rootDir = "/tmp/cache-local"
+
+func TestNonResourceInfoCache(t *testing.T) {
+	dStorage, err := disk.NewDiskStorage(rootDir)
+	if err != nil {
+		t.Errorf("disk initialize error: %v", err)
+	}
+
+	sw := cachemanager.NewStorageWrapper(dStorage)
+
+	testcases := map[string]struct {
+		Accept      string
+		Verb        string
+		Path        string
+		Code        int
+		ContentType string
+		Response    string
+	}{
+		"version info": {
+			Accept:      "application/json",
+			Verb:        "GET",
+			Path:        "/version",
+			Code:        http.StatusOK,
+			ContentType: "application/json",
+			Response:    "fake-non-resource-info-/version",
+		},
+
+		"discovery v1": {
+			Accept:      "application/json",
+			Verb:        "GET",
+			Path:        "/apis/discovery.k8s.io/v1",
+			Code:        http.StatusOK,
+			ContentType: "application/json",
+			Response:    "fake-non-resource-info-/apis/discovery.k8s.io/v1",
+		},
+		"discovery v1beta1": {
+			Accept:      "application/json",
+			Verb:        "GET",
+			Path:        "/apis/discovery.k8s.io/v1beta1",
+			Code:        http.StatusOK,
+			ContentType: "application/json",
+			Response:    "fake-non-resource-info-/apis/discovery.k8s.io/v1beta1",
+		},
+	}
+
+	for k, tt := range testcases {
+		//test for caching the response
+		t.Run(k, func(t *testing.T) {
+			var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+				cacheNonResourceInfo(w, req, sw, fmt.Sprintf("non-reosurce-info%s", req.URL.Path), req.URL.Path, true)
+			})
+			req, _ := http.NewRequest(tt.Verb, tt.Path, nil)
+			if len(tt.Accept) != 0 {
+				req.Header.Set("Accept", tt.Accept)
+			}
+			resp := httptest.NewRecorder()
+			handler.ServeHTTP(resp, req)
+			result := resp.Result()
+			if result.StatusCode != tt.Code {
+				t.Errorf("got status code %d, but expect %d", result.StatusCode, tt.Code)
+			}
+			if string(resp.Body.Bytes()) != tt.Response {
+				t.Errorf("got response %s, but expect %s", string(resp.Body.Bytes()), tt.Response)
+			}
+
+		})
+
+		//test for query the cache
+		t.Run(k, func(t *testing.T) {
+			var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+				cacheNonResourceInfo(w, req, sw, fmt.Sprintf("non-reosurce-info%s", req.URL.Path), req.URL.Path, false)
+			})
+			req, _ := http.NewRequest(tt.Verb, tt.Path, nil)
+			if len(tt.Accept) != 0 {
+				req.Header.Set("Accept", tt.Accept)
+			}
+			resp := httptest.NewRecorder()
+			handler.ServeHTTP(resp, req)
+			result := resp.Result()
+			if result.StatusCode != tt.Code {
+				t.Errorf("got status code %d, but expect %d", result.StatusCode, tt.Code)
+			}
+			if string(resp.Body.Bytes()) != tt.Response {
+				t.Errorf("got response %s, but expect %s", string(resp.Body.Bytes()), tt.Response)
+			}
+
+		})
+	}
+
+	os.RemoveAll(rootDir)
+}

pkg/yurthub/server/server.go

@@ -56,9 +56,16 @@ type yurtHubServer struct {
 // NewYurtHubServer creates a Server object
 func NewYurtHubServer(cfg *config.YurtHubConfiguration,
 	certificateMgr interfaces.YurtCertificateManager,
-	proxyHandler http.Handler) (Server, error) {
+	proxyHandler http.Handler,
+	rest *rest.RestConfigManager) (Server, error) {
 	hubMux := mux.NewRouter()
 	registerHandlers(hubMux, cfg, certificateMgr)
+	restCfg := rest.GetRestConfig(false)
+	clientSet, err := kubernetes.NewForConfig(restCfg)
+	if err != nil {
+		klog.Errorf("cannot create the client set: %v", err)
+		return nil, err
+	}
 	hubServer := &http.Server{
 		Addr:           cfg.YurtHubServerAddr,
 		Handler:        hubMux,
@@ -67,7 +74,7 @@ func NewYurtHubServer(cfg *config.YurtHubConfiguration,
 
 	proxyServer := &http.Server{
 		Addr:    cfg.YurtHubProxyServerAddr,
-		Handler: proxyHandler,
+		Handler: wrapNonResourceHandler(proxyHandler, cfg, clientSet),
 	}
 
 	secureProxyServer := &http.Server{