Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add method to delete namespaced PVC in spawner base class #475

Merged
merged 5 commits into from
May 12, 2021
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 51 additions & 1 deletion kubespawner/spawner.py
Original file line number Diff line number Diff line change
Expand Up @@ -2329,7 +2329,7 @@ async def _ensure_not_exists(self, kind, name):
delete = getattr(self.api, "delete_namespaced_{}".format(kind))
read = getattr(self.api, "read_namespaced_{}".format(kind))

# first, attempt to delete the resouce
# first, attempt to delete the resource
try:
self.log.info(f"Deleting {kind}/{name}")
await gen.with_timeout(
Expand Down Expand Up @@ -2577,6 +2577,37 @@ async def _make_delete_pod_request(
else:
raise

async def _make_delete_pvc_request(self, pvc_name, request_timeout):
"""
Make an HTTP request to delete the given PVC

Designed to be used with exponential_backoff, so returns
True / False on success / failure
"""
self.log.info("Deleting pvc %s", pvc_name)
try:
await gen.with_timeout(
timedelta(seconds=request_timeout),
self.asynchronize(
self.api.delete_namespaced_persistent_volume_claim,
name=pvc_name,
namespace=self.namespace,
),
)
return True
except gen.TimeoutError:
return False
except ApiException as e:
if e.status == 404:
self.log.warning(
"No pvc %s to delete. Assuming already deleted.",
pvc_name,
)
# If there isn't a PVC to delete, that's ok too!
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've got a question about this and the warning a couple of lines above. In our deployment we're currently using kubespawner 0.15 and jupyterhub 1.3. I'm looking to upgrade to z2jh 1.2.0, jupyterhub 1.5.0 and kubespawner 1.1.2. We use a single PVC that is backed by cloud object storage so all of the singleuser-server pods share the same PVC. We leave the default of storage_pvc_ensure=False which from looking at the code means kubespawner won't actually create a PVC named self.pvc_name in _start. So shouldn't delete_forever also be checking storage_pvc_ensure? Maybe not if it's assumed that someone could pre-create the PVC using the same pvc_name_template, and in that case we just try to delete the PVC and if it doesn't exist we get a 404 and log the warning.

I guess in our case the point is I don't want to see the warnings when users get culled (which happens after 5 days of inactivity for us) so to avoid that we just need to set delete_pvc=False.

Copy link
Member

@consideRatio consideRatio Nov 8, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I extracted this to #543 to avoid loosing track of this comment! Please follow up there rather than here so we don't disperse the discussion about this in two places.

return True
else:
raise

async def stop(self, now=False):
delete_options = client.V1DeleteOptions()

Expand Down Expand Up @@ -2779,3 +2810,22 @@ async def _ensure_namespace(self):
# It's fine if it already exists
self.log.exception("Failed to create namespace %s", self.namespace)
raise

async def delete_forever(self):
"""Called when a user is deleted.

This can do things like request removal of resources such as persistent storage.
Only called on stopped spawners, and is likely the last action ever taken for the user.

This will only be called once on the user's default Spawner.
Supported by JupyterHub 1.4.0+.
minrk marked this conversation as resolved.
Show resolved Hide resolved
"""
await exponential_backoff(
partial(
self._make_delete_pvc_request,
self.pvc_name,
self.k8s_api_request_timeout,
),
f'Could not delete pvc {self.pvc_name}',
timeout=self.k8s_api_request_retry_timeout,
)
50 changes: 50 additions & 0 deletions tests/test_spawner.py
Original file line number Diff line number Diff line change
Expand Up @@ -627,3 +627,53 @@ async def test_url_changed(kube_ns, kube_client, config, hub_pod, hub):
assert spawner.db.commit.call_count == previous_commit_count

await spawner.stop()


@pytest.mark.asyncio
async def test_delete_pvc(kube_ns, kube_client, hub, config):
config.KubeSpawner.storage_pvc_ensure = True
config.KubeSpawner.storage_capacity = '1M'

spawner = KubeSpawner(
hub=hub,
user=MockUser(name="mockuser"),
config=config,
_mock=True,
)
spawner.api = kube_client

# start the spawner
await spawner.start()

# verify the pod exists
pod_name = "jupyter-%s" % spawner.user.name
pods = kube_client.list_namespaced_pod(kube_ns).items
pod_names = [p.metadata.name for p in pods]
assert pod_name in pod_names

# verify PVC is created
pvc_name = spawner.pvc_name
pvc_list = kube_client.list_namespaced_persistent_volume_claim(kube_ns).items
pvc_names = [s.metadata.name for s in pvc_list]
assert pvc_name in pvc_names

# stop the pod
await spawner.stop()

# verify pod is gone
pods = kube_client.list_namespaced_pod(kube_ns).items
pod_names = [p.metadata.name for p in pods]
assert "jupyter-%s" % spawner.user.name not in pod_names

# delete the PVC
await spawner.delete_forever()

# verify PVC is deleted, it may take a little while
for i in range(5):
pvc_list = kube_client.list_namespaced_persistent_volume_claim(kube_ns).items
pvc_names = [s.metadata.name for s in pvc_list]
if pvc_name in pvc_names:
time.sleep(1)
else:
break
assert pvc_name not in pvc_names