From a2d050486650c4556579340c1bba5440f6a189ed Mon Sep 17 00:00:00 2001 From: Zanie Adkins Date: Tue, 23 May 2023 11:25:50 -0500 Subject: [PATCH 1/5] Guard against changing the profile path from `prefect config set` (#9696) --- src/prefect/cli/config.py | 7 +++++++ tests/cli/test_config.py | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/prefect/cli/config.py b/src/prefect/cli/config.py index 27a1759d60c6..505b86257175 100644 --- a/src/prefect/cli/config.py +++ b/src/prefect/cli/config.py @@ -38,6 +38,13 @@ def set_(settings: List[str]): if setting not in prefect.settings.SETTING_VARIABLES: exit_with_error(f"Unknown setting name {setting!r}.") + # Guard against changing settings that tweak config locations + if setting in {"PREFECT_HOME", "PREFECT_PROFILES_PATH"}: + exit_with_error( + f"Setting {setting!r} cannot be changed with this command. " + "Use an environment variable instead." + ) + parsed_settings[setting] = value try: diff --git a/tests/cli/test_config.py b/tests/cli/test_config.py index ad407c480bd2..0ac5f57fbda0 100644 --- a/tests/cli/test_config.py +++ b/tests/cli/test_config.py @@ -74,6 +74,19 @@ def test_set_with_unknown_setting(): ) +@pytest.mark.parametrize("setting", ["PREFECT_HOME", "PREFECT_PROFILES_PATH"]) +def test_set_with_disallowed_setting(setting): + save_profiles(ProfilesCollection([Profile(name="foo", settings={})], active=None)) + + invoke_and_assert( + ["--profile", "foo", "config", "set", f"{setting}=BAR"], + expected_output=f""" + Setting {setting!r} cannot be changed with this command. Use an environment variable instead. + """, + expected_code=1, + ) + + def test_set_with_invalid_value_type(): save_profiles(ProfilesCollection([Profile(name="foo", settings={})], active=None)) From d2c81760d9f41545fe807a9495cbf614e55101a7 Mon Sep 17 00:00:00 2001 From: seanpwlms <66645429+seanpwlms@users.noreply.github.com> Date: Tue, 23 May 2023 11:52:29 -0700 Subject: [PATCH 2/5] change Prefect server capitalization (#9697) Co-authored-by: Jeff Hale --- docs/api-ref/server/index.md | 2 +- docs/concepts/variables.md | 4 ++-- docs/host/index.md | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/api-ref/server/index.md b/docs/api-ref/server/index.md index 0d7a9f67733b..057878ba2d2d 100644 --- a/docs/api-ref/server/index.md +++ b/docs/api-ref/server/index.md @@ -7,6 +7,6 @@ tags: # Server API -The Prefect Server API is used by the server to work with workflow metadata and enforce orchestration logic. This API is primarily used by Prefect developers. +The Prefect server API is used by the server to work with workflow metadata and enforce orchestration logic. This API is primarily used by Prefect developers. Select links in the left navigation menu to explore. \ No newline at end of file diff --git a/docs/concepts/variables.md b/docs/concepts/variables.md index 360de1f74ac3..0f4c70f7af68 100644 --- a/docs/concepts/variables.md +++ b/docs/concepts/variables.md @@ -7,7 +7,7 @@ tags: # Variables -Variables enable you to store and reuse non-sensitive bits of data, such as configuration information. Variables are named, mutable string values, much like environment variables. Variables are scoped to a Prefect Server instance or a single workspace in Prefect Cloud. +Variables enable you to store and reuse non-sensitive bits of data, such as configuration information. Variables are named, mutable string values, much like environment variables. Variables are scoped to a Prefect server instance or a single workspace in Prefect Cloud. Variables can be created or modified at any time, but are intended for values with infrequent writes and frequent reads. Variable values may be cached for quicker retrieval. @@ -32,7 +32,7 @@ Optionally, you can add tags to the variable. ### Via the Prefect UI -You can see all the variables in your Prefect Server instance or Prefect Cloud workspace on the **Variables** page of the Prefect UI. Both the name and value of all variables are visible to anyone with access to the server or workspace. +You can see all the variables in your Prefect server instance or Prefect Cloud workspace on the **Variables** page of the Prefect UI. Both the name and value of all variables are visible to anyone with access to the server or workspace. To create a new variable, select the **+** button next to the header of the **Variables** page. Enter the name and value of the variable. diff --git a/docs/host/index.md b/docs/host/index.md index f5c2cc96c04d..747b9fc9d007 100644 --- a/docs/host/index.md +++ b/docs/host/index.md @@ -12,11 +12,11 @@ tags: - SQLite --- -# Hosting Prefect Server +# Hosting a Prefect server After you install Prefect you have a Python SDK client that can communicate with [Prefect Cloud](https://app.prefect.cloud), the platform hosted by Prefect. You also have an [API server](/api-ref/) backed by a database and a UI. -In this section you'll learn how to host your own Prefect Server. +In this section you'll learn how to host your own Prefect server. ![Prefect Server UI](/img/ui/flow-run-page-server.png) @@ -37,7 +37,7 @@ Shut down the Prefect server with ctrl + c in the term ### Differences between Prefect Server and Cloud -Prefect Server and Cloud share a base of features. Prefect Cloud also includes the following features that you can read about in the [Cloud](/cloud/) section of the docs. +The self-hosted Prefect server and Prefect Cloud share a base of features. Prefect Cloud also includes the following features that you can read about in the [Cloud](/cloud/) section of the docs. - [User accounts](#user-accounts) — personal accounts for working in Prefect Cloud. - [Workspaces](/cloud/workspaces/) — isolated environments to organize your flows, deployments, and flow runs. @@ -51,7 +51,7 @@ Prefect Server and Cloud share a base of features. Prefect Cloud also includes t - Collaborators — invite others to work in your [workspace](/cloud/workspaces/#workspace-collaborators) or [organization](/cloud/organizations/#organization-members). -### Configuring Prefect Server +### Configuring a Prefect Server Go to your terminal session and run this command to set the API URL to point to a Prefect server instance: @@ -63,7 +63,7 @@ $ prefect config set PREFECT_API_URL="http://127.0.0.1:4200/api" !!! tip "`PREFECT_API_URL` required when running Prefect inside a container" - You must set the API Server address to use Prefect within a container, such a a Docker container. + You must set the API server address to use Prefect within a container, such as a Docker container. You can save the API server address in a [Prefect profile](/concepts/settings/). Whenever that profile is active, the API endpoint will be be at that address. @@ -229,7 +229,7 @@ See the [contributing docs](/contributing/overview/#adding-database-migrations) ## Notifications -When you use [Prefect Cloud](/cloud/) you gain access to a hosted platform with Workspace & User controls, Events, and Automations. Prefect Cloud has an option for automation notifications. The more limited Notifications option is provided for the self-hosted Prefect Server. +When you use [Prefect Cloud](/cloud/) you gain access to a hosted platform with Workspace & User controls, Events, and Automations. Prefect Cloud has an option for automation notifications. The more limited Notifications option is provided for the self-hosted Prefect server. Notifications enable you to set up alerts that are sent when a flow enters any state you specify. When your flow and task runs changes [state](/concepts/states/), Prefect notes the state change and checks whether the new state matches any notification policies. If it does, a new notification is queued. @@ -247,7 +247,7 @@ Prefect supports sending notifications via: ### Configure notifications -To configure a notification in Prefect Server, go to the **Notifications** page and select **Create Notification** or the **+** button. +To configure a notification in a Prefect server, go to the **Notifications** page and select **Create Notification** or the **+** button. ![Creating a notification in the Prefect UI](/img/ui/create-slack-notification.png) From 68f2f0b228bd6d9c536057a9d09147467bcbf4fd Mon Sep 17 00:00:00 2001 From: Chris White Date: Tue, 23 May 2023 18:38:36 -0700 Subject: [PATCH 3/5] Handle deleted deployments gracefully (#9464) --- src/prefect/agent.py | 1 + src/prefect/client/orchestration.py | 8 +++++++- src/prefect/workers/base.py | 10 ++++++++-- tests/client/test_orion_client.py | 2 +- tests/workers/test_base_worker.py | 26 ++++++++++++++++++++++++++ 5 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/prefect/agent.py b/src/prefect/agent.py index f501891affe8..36721bf59abc 100644 --- a/src/prefect/agent.py +++ b/src/prefect/agent.py @@ -378,6 +378,7 @@ async def _mark_flow_run_as_cancelled( async def get_infrastructure(self, flow_run: FlowRun) -> Infrastructure: deployment = await self.client.read_deployment(flow_run.deployment_id) + flow = await self.client.read_flow(deployment.flow_id) # overrides only apply when configuring known infra blocks diff --git a/src/prefect/client/orchestration.py b/src/prefect/client/orchestration.py index 04646eceb076..662ba009ca1b 100644 --- a/src/prefect/client/orchestration.py +++ b/src/prefect/client/orchestration.py @@ -1497,7 +1497,13 @@ async def read_deployment( Returns: a [Deployment model][prefect.server.schemas.core.Deployment] representation of the deployment """ - response = await self._client.get(f"/deployments/{deployment_id}") + try: + response = await self._client.get(f"/deployments/{deployment_id}") + except httpx.HTTPStatusError as e: + if e.response.status_code == status.HTTP_404_NOT_FOUND: + raise prefect.exceptions.ObjectNotFound(http_exc=e) from e + else: + raise return schemas.responses.DeploymentResponse.parse_obj(response.json()) async def read_deployment_by_name( diff --git a/src/prefect/workers/base.py b/src/prefect/workers/base.py index 698bc80d6d87..bf4a972c5ea5 100644 --- a/src/prefect/workers/base.py +++ b/src/prefect/workers/base.py @@ -565,7 +565,13 @@ async def cancel_run(self, flow_run: "FlowRun"): ) return - configuration = await self._get_configuration(flow_run) + try: + configuration = await self._get_configuration(flow_run) + except ObjectNotFound: + self._logger.warning( + f"Flow run {flow_run.id!r} cannot be cancelled by this worker:" + f" associated deployment {flow_run.deployment_id!r} does not exist." + ) try: await self.kill_infrastructure( @@ -740,7 +746,7 @@ async def _submit_run(self, flow_run: "FlowRun") -> None: try: await self._check_flow_run(flow_run) - except ValueError: + except (ValueError, ObjectNotFound): self._logger.exception( ( "Flow run %s did not pass checks and will not be submitted for" diff --git a/tests/client/test_orion_client.py b/tests/client/test_orion_client.py index 5703ab0e376e..f5711fc34891 100644 --- a/tests/client/test_orion_client.py +++ b/tests/client/test_orion_client.py @@ -722,7 +722,7 @@ def foo(): ) await orion_client.delete_deployment(deployment_id) - with pytest.raises(httpx.HTTPStatusError, match="404"): + with pytest.raises(prefect.exceptions.ObjectNotFound): await orion_client.read_deployment(deployment_id) diff --git a/tests/workers/test_base_worker.py b/tests/workers/test_base_worker.py index 90ac2a6b8bc2..ec8600660bf5 100644 --- a/tests/workers/test_base_worker.py +++ b/tests/workers/test_base_worker.py @@ -1487,6 +1487,32 @@ async def test_worker_cancel_run_updates_state_type( post_flow_run = await orion_client.read_flow_run(flow_run.id) assert post_flow_run.state.type == StateType.CANCELLED + @pytest.mark.parametrize( + "cancelling_constructor", [legacy_named_cancelling_state, Cancelling] + ) + async def test_worker_cancel_run_handles_missing_deployment( + self, + orion_client: PrefectClient, + worker_deployment_wq1, + cancelling_constructor, + work_pool, + ): + flow_run = await orion_client.create_flow_run_from_deployment( + worker_deployment_wq1.id, + state=cancelling_constructor(), + ) + + await orion_client.delete_deployment(worker_deployment_wq1.id) + + async with WorkerTestImpl( + work_pool_name=work_pool.name, prefetch_seconds=10 + ) as worker: + await worker.sync_with_backend() + await worker.check_for_cancelled_flow_runs() + + post_flow_run = await orion_client.read_flow_run(flow_run.id) + assert post_flow_run.state.type == StateType.CANCELLED + @pytest.mark.parametrize( "cancelling_constructor", [legacy_named_cancelling_state, Cancelling] ) From bb4583ac47f7507d92e16f805390f4eb9bc7cbc1 Mon Sep 17 00:00:00 2001 From: jakekaplan <40362401+jakekaplan@users.noreply.github.com> Date: Wed, 24 May 2023 09:32:55 -0500 Subject: [PATCH 4/5] fix variable max value update (#9710) --- src/prefect/server/schemas/actions.py | 2 +- tests/server/api/test_variables.py | 96 +++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/src/prefect/server/schemas/actions.py b/src/prefect/server/schemas/actions.py index bdea3baa65f3..9a323fd5ee12 100644 --- a/src/prefect/server/schemas/actions.py +++ b/src/prefect/server/schemas/actions.py @@ -644,7 +644,7 @@ class VariableUpdate(ActionBaseModel): default=None, description="The value of the variable", example="my-value", - max_length=schemas.core.MAX_VARIABLE_NAME_LENGTH, + max_length=schemas.core.MAX_VARIABLE_VALUE_LENGTH, ) tags: Optional[List[str]] = FieldFrom(schemas.core.Variable) diff --git a/tests/server/api/test_variables.py b/tests/server/api/test_variables.py index 7f84a517f05b..f76f414c4d42 100644 --- a/tests/server/api/test_variables.py +++ b/tests/server/api/test_variables.py @@ -534,6 +534,54 @@ async def test_name_unique( assert res assert res.status_code == 409 + async def test_name_max_length( + self, + client: AsyncClient, + variable, + ): + max_length = 255 + + res = await client.patch( + f"/variables/{variable.id}", json={"name": "v" * max_length} + ) + assert res + assert res.status_code == 204 + + max_length_plus1 = max_length + 1 + + res = await client.patch( + f"/variables/{variable.id}", json={"name": "v" * max_length_plus1} + ) + assert res + assert res.status_code == 422 + assert ( + "ensure this value has at most" in res.json()["exception_detail"][0]["msg"] + ) + + async def test_value_max_length( + self, + client: AsyncClient, + variable, + ): + max_length = 5000 + + res = await client.patch( + f"/variables/{variable.id}", json={"value": "v" * max_length} + ) + assert res + assert res.status_code == 204 + + max_length_plus1 = max_length + 1 + + res = await client.patch( + f"/variables/{variable.id}", json={"value": "v" * max_length_plus1} + ) + assert res + assert res.status_code == 422 + assert ( + "ensure this value has at most" in res.json()["exception_detail"][0]["msg"] + ) + class TestUpdateVariableByName: async def test_update_variable( @@ -584,6 +632,54 @@ async def test_name_unique( ) assert res.status_code == 409 + async def test_name_max_length( + self, + client: AsyncClient, + variable, + ): + max_length = 255 + + res = await client.patch( + f"/variables/name/{variable.name}", json={"name": "v" * max_length} + ) + assert res + assert res.status_code == 204 + + max_length_plus1 = max_length + 1 + + res = await client.patch( + f"/variables/name/{variable.name}", json={"name": "v" * max_length_plus1} + ) + assert res + assert res.status_code == 422 + assert ( + "ensure this value has at most" in res.json()["exception_detail"][0]["msg"] + ) + + async def test_value_max_length( + self, + client: AsyncClient, + variable, + ): + max_length = 5000 + + res = await client.patch( + f"/variables/name/{variable.name}", json={"value": "v" * max_length} + ) + assert res + assert res.status_code == 204 + + max_length_plus1 = max_length + 1 + + res = await client.patch( + f"/variables/name/{variable.name}", json={"value": "v" * max_length_plus1} + ) + assert res + assert res.status_code == 422 + assert ( + "ensure this value has at most" in res.json()["exception_detail"][0]["msg"] + ) + class TestDeleteVariable: async def test_delete_variable( From 91105c475c5688d520b4b4de1e548989d533f581 Mon Sep 17 00:00:00 2001 From: Jeff Hale Date: Wed, 24 May 2023 11:10:17 -0400 Subject: [PATCH 5/5] Expand docstrings for artifacts (#9704) Co-authored-by: Serina Grill <42048900+serinamarie@users.noreply.github.com> --- src/prefect/artifacts.py | 44 +++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/src/prefect/artifacts.py b/src/prefect/artifacts.py index 5260a8042c7a..ef20ae193867 100644 --- a/src/prefect/artifacts.py +++ b/src/prefect/artifacts.py @@ -31,11 +31,13 @@ async def _create_artifact( """ Helper function to create an artifact. - Args: - - type: A string identifying the type of artifact. - - key: A string user-provided identifier. - - description: A user-specified description of the artifact. - - data: A JSON payload that allows for a result to be retrieved. + Arguments: + type: A string identifying the type of artifact. + key: A user-provided string identifier. + The key must only contain lowercase letters, numbers, and dashes. + description: A user-specified description of the artifact. + data: A JSON payload that allows for a result to be retrieved. + client: The PrefectClient Returns: - The table artifact ID. @@ -74,11 +76,17 @@ async def create_link_artifact( """ Create a link artifact. - Args: - - link: The link to create. + Arguments: + link: The link to create. + link_text: The link text. + key: A user-provided string identifier. + Required for the artifact to show in the Artifacts page in the UI. + The key must only contain lowercase letters, numbers, and dashes. + description: A user-specified description of the artifact. + Returns: - - The table artifact ID. + The table artifact ID. """ formatted_link = f"[{link_text}]({link})" if link_text else f"[{link}]({link})" artifact = await _create_artifact( @@ -100,11 +108,15 @@ async def create_markdown_artifact( """ Create a markdown artifact. - Args: - - markdown: The markdown to create. + Arguments: + markdown: The markdown to create. + key: A user-provided string identifier. + Required for the artifact to show in the Artifacts page in the UI. + The key must only contain lowercase letters, numbers, and dashes. + description: A user-specified description of the artifact. Returns: - - The table artifact ID. + The table artifact ID. """ artifact = await _create_artifact( key=key, @@ -125,11 +137,15 @@ async def create_table_artifact( """ Create a table artifact. - Args: - - table: The table to create. + Arguments: + table: The table to create. + key: A user-provided string identifier. + Required for the artifact to show in the Artifacts page in the UI. + The key must only contain lowercase letters, numbers, and dashes. + description: A user-specified description of the artifact. Returns: - - The table artifact ID. + The table artifact ID. """ def _sanitize_nan_values(container):