[ws-manager-mk2] support storage quotas (gitpod-io#17606)

* [ws-manager-mk2] add support for storage quotas

This way, on workspace create, `ws-daemon` can set XFS limits for `/workspace`

* [preview] set smaller /workspace limits

This way we don't have to spend more on preview environments.

* [ws-daemon] warn when xfs is missing

* Partial revert of "Revert "[ws-daemon] Restart IWS if ws-daemon is restarted (gitpod-io#17552)" (gitpod-io#17645)"

This reverts commit e082b7f.

It avoids reverts on notify.go and workspace_provider.go.

* [ws-daemon] log when handling running workspaces

* [test] add test for xfs quotas

components/ws-daemon/pkg/content/hooks.go

@@ -112,16 +112,18 @@ func hookInstallQuota(xfs *quota.XFS, isHard bool) session.WorkspaceLivecycleHoo
 		defer tracing.FinishSpan(span, &err)
 
 		if xfs == nil {
+			log.WithFields(ws.OWI()).Warn("no xfs definition")
 			return nil
 		}
 
 		if ws.StorageQuota == 0 {
+			log.WithFields(ws.OWI()).Warn("no storage quota defined")
 			return nil
 		}
 
 		size := quota.Size(ws.StorageQuota)
 
-		log.WithFields(ws.OWI()).WithField("size", size).WithField("directory", ws.Location).Debug("setting disk quota")
+		log.WithFields(ws.OWI()).WithField("isHard", isHard).WithField("size", size).WithField("directory", ws.Location).Debug("setting disk quota")
 
 		var (
 			prj int

components/ws-daemon/pkg/controller/workspace_controller.go

@@ -132,6 +132,11 @@ func (wsc *WorkspaceController) Reconcile(ctx context.Context, req ctrl.Request)
 		return result, err
 	}
 
+	if workspace.Status.Phase == workspacev1.WorkspacePhaseRunning {
+		result, err = wsc.handleWorkspaceRunning(ctx, &workspace, req)
+		return result, err
+	}
+
 	if workspace.Status.Phase == workspacev1.WorkspacePhaseStopping {
 
 		result, err = wsc.handleWorkspaceStop(ctx, &workspace, req)
@@ -178,8 +183,9 @@ func (wsc *WorkspaceController) handleWorkspaceInit(ctx context.Context, ws *wor
 				WorkspaceID: ws.Spec.Ownership.WorkspaceID,
 				InstanceID:  ws.Name,
 			},
-			Initializer: init,
-			Headless:    ws.IsHeadless(),
+			Initializer:  init,
+			Headless:     ws.IsHeadless(),
+			StorageQuota: ws.Spec.StorageQuota,
 		})
 
 		err = retry.RetryOnConflict(retryParams, func() error {
@@ -210,6 +216,16 @@ func (wsc *WorkspaceController) handleWorkspaceInit(ctx context.Context, ws *wor
 	return ctrl.Result{}, nil
 }
 
+func (wsc *WorkspaceController) handleWorkspaceRunning(ctx context.Context, ws *workspacev1.Workspace, req ctrl.Request) (result ctrl.Result, err error) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, "handleWorkspaceRunning")
+	defer tracing.FinishSpan(span, &err)
+
+	log := log.FromContext(ctx)
+	log.Info("handling running workspace")
+
+	return ctrl.Result{}, wsc.operations.SetupWorkspace(ctx, ws.Name)
+}
+
 func (wsc *WorkspaceController) handleWorkspaceStop(ctx context.Context, ws *workspacev1.Workspace, req ctrl.Request) (result ctrl.Result, err error) {
 	log := log.FromContext(ctx)
 	span, ctx := opentracing.StartSpanFromContext(ctx, "handleWorkspaceStop")

components/ws-daemon/pkg/controller/workspace_operations.go

@@ -28,7 +28,7 @@ import (
 	"golang.org/x/xerrors"
 )
 
-//go:generate mockgen -destination=mock.go -package=controller . WorkspaceOperations
+//go:generate sh -c "go install github.com/golang/mock/mockgen@v1.6.0 && mockgen -destination=mock.go -package=controller . WorkspaceOperations"
 type WorkspaceOperations interface {
 	// InitWorkspace initializes the workspace content
 	InitWorkspace(ctx context.Context, options InitOptions) (string, error)
@@ -40,6 +40,8 @@ type WorkspaceOperations interface {
 	SnapshotIDs(ctx context.Context, instanceID string) (snapshotUrl, snapshotName string, err error)
 	// Snapshot takes a snapshot of the workspace
 	Snapshot(ctx context.Context, instanceID, snapshotName string) (err error)
+	// Setup ensures that the workspace has been setup
+	SetupWorkspace(ctx context.Context, instanceID string) error
 }
 
 type DefaultWorkspaceOperations struct {
@@ -58,9 +60,10 @@ type WorkspaceMeta struct {
 }
 
 type InitOptions struct {
-	Meta        WorkspaceMeta
-	Initializer *csapi.WorkspaceInitializer
-	Headless    bool
+	Meta         WorkspaceMeta
+	Initializer  *csapi.WorkspaceInitializer
+	Headless     bool
+	StorageQuota int
 }
 
 type BackupOptions struct {
@@ -91,7 +94,7 @@ func NewWorkspaceOperations(config content.Config, provider *WorkspaceProvider,
 
 func (wso *DefaultWorkspaceOperations) InitWorkspace(ctx context.Context, options InitOptions) (string, error) {
 	ws, err := wso.provider.Create(ctx, options.Meta.InstanceID, filepath.Join(wso.provider.Location, options.Meta.InstanceID),
-		wso.creator(options.Meta.Owner, options.Meta.WorkspaceID, options.Meta.InstanceID, options.Initializer, false))
+		wso.creator(options.Meta.Owner, options.Meta.WorkspaceID, options.Meta.InstanceID, options.Initializer, false, options.StorageQuota))
 
 	if err != nil {
 		return "bug: cannot add workspace to store", xerrors.Errorf("cannot add workspace to store: %w", err)
@@ -153,7 +156,7 @@ func (wso *DefaultWorkspaceOperations) InitWorkspace(ctx context.Context, option
 	return "", nil
 }
 
-func (wso *DefaultWorkspaceOperations) creator(owner, workspaceID, instanceID string, init *csapi.WorkspaceInitializer, storageDisabled bool) session.WorkspaceFactory {
+func (wso *DefaultWorkspaceOperations) creator(owner, workspaceID, instanceID string, init *csapi.WorkspaceInitializer, storageDisabled bool, storageQuota int) session.WorkspaceFactory {
 	var checkoutLocation string
 	allLocations := csapi.GetCheckoutLocationsFromInitializer(init)
 	if len(allLocations) > 0 {
@@ -173,13 +176,23 @@ func (wso *DefaultWorkspaceOperations) creator(owner, workspaceID, instanceID st
 			PersistentVolumeClaim: false,
 			RemoteStorageDisabled: storageDisabled,
 			IsMk2:                 true,
+			StorageQuota:          storageQuota,
 
 			ServiceLocDaemon: filepath.Join(wso.config.WorkingArea, serviceDirName),
 			ServiceLocNode:   filepath.Join(wso.config.WorkingAreaNode, serviceDirName),
 		}, nil
 	}
 }
 
+func (wso *DefaultWorkspaceOperations) SetupWorkspace(ctx context.Context, instanceID string) error {
+	_, err := wso.provider.Get(ctx, instanceID)
+	if err != nil {
+		return fmt.Errorf("cannot setup workspace %s: %w", instanceID, err)
+	}
+
+	return nil
+}
+
 func (wso *DefaultWorkspaceOperations) BackupWorkspace(ctx context.Context, opts BackupOptions) (*csapi.GitStatus, error) {
 	ws, err := wso.provider.Get(ctx, opts.Meta.InstanceID)
 	if err != nil {

components/ws-manager-api/go/crd/v1/workspace_types.go

@@ -55,6 +55,11 @@ type WorkspaceSpec struct {
 	Ports []PortSpec `json:"ports"`
 
 	SshPublicKeys []string `json:"sshPublicKeys,omitempty"`
+
+	// TODO: make StorageQuota Required in the future, avoid for now to avoid runtime failures for existing workspaces
+
+	// the XFS quota to enforce on the workspace's /workspace folder
+	StorageQuota int `json:"storageQuota,omitempty"`
 }
 
 type Ownership struct {

components/ws-manager-mk2/config/crd/bases/workspace.gitpod.io_workspaces.yaml

@@ -161,6 +161,10 @@ spec:
                 items:
                   type: string
                 type: array
+              storageQuota:
+                description: the XFS quota to enforce on the workspace's /workspace
+                  folder
+                type: integer
               sysEnvVars:
                 items:
                   description: EnvVar represents an environment variable present in
@@ -412,8 +416,8 @@ spec:
                   description: "Condition contains details for one aspect of the current
                     state of this API Resource. --- This struct is intended for direct
                     use as an array at the field path .status.conditions.  For example,
-                    type FooStatus struct{ // Represents the observations of a foo's
-                    current state. // Known .status.conditions.type are: \"Available\",
+                    \n type FooStatus struct{ // Represents the observations of a
+                    foo's current state. // Known .status.conditions.type are: \"Available\",
                     \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge
                     // +listType=map // +listMapKey=type Conditions []metav1.Condition
                     `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\"

components/ws-manager-mk2/service/manager.go

@@ -195,6 +195,12 @@ func (wsm *WorkspaceManagerServer) StartWorkspace(ctx context.Context, req *wsma
 		return nil, status.Errorf(codes.InvalidArgument, "workspace class \"%s\" is unknown", req.Spec.Class)
 	}
 
+	storage, err := class.Container.Limits.StorageQuantity()
+	if err != nil {
+		msg := fmt.Sprintf("workspace class %s has invalid storage quantity: %v", class.Name, err)
+		return nil, status.Errorf(codes.InvalidArgument, msg)
+	}
+
 	annotations := make(map[string]string)
 	for k, v := range req.Metadata.Annotations {
 		annotations[k] = v
@@ -277,6 +283,7 @@ func (wsm *WorkspaceManagerServer) StartWorkspace(ctx context.Context, req *wsma
 			},
 			Ports:         ports,
 			SshPublicKeys: req.Spec.SshPublicKeys,
+			StorageQuota:  int(storage.Value()),
 		},
 	}
 	controllerutil.AddFinalizer(&ws, workspacev1.GitpodFinalizerName)

dev/preview/workflow/preview/deploy-gitpod.sh

@@ -261,15 +261,15 @@ yq w -i "${INSTALLER_CONFIG_PATH}" workspace.resources.requests.memory "256Mi"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.webapp.workspaceClasses[+].id "default"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.webapp.workspaceClasses[0].category "GENERAL PURPOSE"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.webapp.workspaceClasses[0].displayName "Default"
-yq w -i "${INSTALLER_CONFIG_PATH}" experimental.webapp.workspaceClasses[0].description "Default workspace class (30GB disk)"
+yq w -i "${INSTALLER_CONFIG_PATH}" experimental.webapp.workspaceClasses[0].description "Default workspace class (10GB disk)"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.webapp.workspaceClasses[0].powerups "1"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.webapp.workspaceClasses[0].isDefault "true"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.webapp.workspaceClasses[0].credits.perMinute "0.3333333333"
 
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.webapp.workspaceClasses[+].id "small"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.webapp.workspaceClasses[1].category "GENERAL PURPOSE"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.webapp.workspaceClasses[1].displayName "Small"
-yq w -i "${INSTALLER_CONFIG_PATH}" experimental.webapp.workspaceClasses[1].description "Small workspace class (20GB disk)"
+yq w -i "${INSTALLER_CONFIG_PATH}" experimental.webapp.workspaceClasses[1].description "Small workspace class (5GB disk)"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.webapp.workspaceClasses[1].powerups "2"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.webapp.workspaceClasses[1].credits.perMinute "0.1666666667"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.webapp.workspaceClasses[1].marker.moreResources "true"
@@ -278,20 +278,22 @@ yq w -i "${INSTALLER_CONFIG_PATH}" experimental.webapp.workspaceClasses[1].marke
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["default"].name "default"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["default"].resources.requests.cpu "100m"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["default"].resources.requests.memory "128Mi"
-yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["default"].prebuildPVC.size "30Gi"
+yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["default"].resources.limits.storage "10Gi"
+yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["default"].prebuildPVC.size "10Gi"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["default"].prebuildPVC.storageClass "rook-ceph-block"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["default"].prebuildPVC.snapshotClass "csi-rbdplugin-snapclass"
-yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["default"].pvc.size "30Gi"
+yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["default"].pvc.size "10Gi"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["default"].pvc.storageClass "rook-ceph-block"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["default"].pvc.snapshotClass "csi-rbdplugin-snapclass"
 
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["small"].name "small"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["small"].resources.requests.cpu "100m"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["small"].resources.requests.memory "128Mi"
-yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["small"].prebuildPVC.size "20Gi"
+yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["small"].resources.limits.storage "5Gi"
+yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["small"].prebuildPVC.size "5Gi"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["small"].prebuildPVC.storageClass "rook-ceph-block"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["small"].prebuildPVC.snapshotClass "csi-rbdplugin-snapclass"
-yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["small"].pvc.size "20Gi"
+yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["small"].pvc.size "5Gi"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["small"].pvc.storageClass "rook-ceph-block"
 yq w -i "${INSTALLER_CONFIG_PATH}" experimental.workspace.classes["small"].pvc.snapshotClass "csi-rbdplugin-snapclass"
 

test/pkg/integration/disk-client.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+// Licensed under the GNU Affero General Public License (AGPL).
+// See License.AGPL.txt in the project root for license information.
+
+package integration
+
+import (
+	"fmt"
+	"strings"
+
+	agent "github.com/gitpod-io/gitpod/test/pkg/agent/workspace/api"
+)
+
+type DiskClient struct {
+	rpcClient *RpcClient
+}
+
+func Disk(rsa *RpcClient) DiskClient {
+	return DiskClient{rsa}
+}
+
+var NoSpaceErrorMsg = "No space left on device"
+
+func (d DiskClient) Fallocate(testFilePath string, spaceToAllocate string) error {
+	var resp agent.ExecResponse
+	err := d.rpcClient.Call("WorkspaceAgent.Exec", &agent.ExecRequest{
+		Dir:     "/workspace",
+		Command: "fallocate",
+		Args:    []string{"-l", spaceToAllocate, testFilePath},
+	}, &resp)
+
+	if err != nil {
+		return fmt.Errorf("error: %w", err)
+	}
+	if resp.ExitCode != 0 {
+		return fmt.Errorf("returned returned rc: %d err: %v", resp.ExitCode, resp.Stderr)
+	}
+	if strings.Contains(resp.Stdout, NoSpaceErrorMsg) {
+		return fmt.Errorf(resp.Stdout)
+	}
+
+	return nil
+}