[Alerting] Log warning when rules are not rescheduled due to Saved Ob…

…ject not found error (#101591) (#101827)

* Adding warning to logs when alerting task runner encounters saved object not found and doesn't reschedule rule

* Adding space id to warning message

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: ymao1 <ying.mao@elastic.co>

x-pack/plugins/alerting/server/task_runner/task_runner.test.ts

@@ -2223,12 +2223,61 @@ describe('Task Runner', () => {
       references: [],
     });
 
+    const logger = taskRunnerFactoryInitializerParams.logger;
     return taskRunner.run().catch((ex) => {
       expect(ex).toMatchInlineSnapshot(`[Error: Saved object [alert/1] not found]`);
+      expect(logger.debug).toHaveBeenCalledWith(
+        `Executing Alert "1" has resulted in Error: Saved object [alert/1] not found`
+      );
+      expect(logger.warn).toHaveBeenCalledTimes(1);
+      expect(logger.warn).nthCalledWith(
+        1,
+        `Unable to execute rule "1" because Saved object [alert/1] not found - this rule will not be rescheduled. To restart rule execution, try disabling and re-enabling this rule.`
+      );
       expect(isUnrecoverableError(ex)).toBeTruthy();
     });
   });
 
+  test('correctly logs warning when Alert Task Runner throws due to failing to fetch the alert in a space', async () => {
+    alertsClient.get.mockImplementation(() => {
+      throw SavedObjectsErrorHelpers.createGenericNotFoundError('alert', '1');
+    });
+
+    const taskRunner = new TaskRunner(
+      alertType,
+      {
+        ...mockedTaskInstance,
+        params: {
+          ...mockedTaskInstance.params,
+          spaceId: 'test space',
+        },
+      },
+      taskRunnerFactoryInitializerParams
+    );
+
+    encryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValue({
+      id: '1',
+      type: 'alert',
+      attributes: {
+        apiKey: Buffer.from('123:abc').toString('base64'),
+      },
+      references: [],
+    });
+
+    const logger = taskRunnerFactoryInitializerParams.logger;
+    return taskRunner.run().catch((ex) => {
+      expect(ex).toMatchInlineSnapshot(`[Error: Saved object [alert/1] not found]`);
+      expect(logger.debug).toHaveBeenCalledWith(
+        `Executing Alert "1" has resulted in Error: Saved object [alert/1] not found`
+      );
+      expect(logger.warn).toHaveBeenCalledTimes(1);
+      expect(logger.warn).nthCalledWith(
+        1,
+        `Unable to execute rule "1" in the "test space" space because Saved object [alert/1] not found - this rule will not be rescheduled. To restart rule execution, try disabling and re-enabling this rule.`
+      );
+    });
+  });
+
   test('start time is logged for new alerts', async () => {
     taskRunnerFactoryInitializerParams.actionsPlugin.isActionTypeEnabled.mockReturnValue(true);
     taskRunnerFactoryInitializerParams.actionsPlugin.isActionExecutable.mockReturnValue(true);

x-pack/plugins/alerting/server/task_runner/task_runner.ts

@@ -587,6 +587,10 @@ export class TaskRunner<
       ),
       schedule: resolveErr<IntervalSchedule | undefined, Error>(schedule, (error) => {
         if (isAlertSavedObjectNotFoundError(error, alertId)) {
+          const spaceMessage = spaceId ? `in the "${spaceId}" space ` : '';
+          this.logger.warn(
+            `Unable to execute rule "${alertId}" ${spaceMessage}because ${error.message} - this rule will not be rescheduled. To restart rule execution, try disabling and re-enabling this rule.`
+          );
           throwUnrecoverableError(error);
         }
         return { interval: taskSchedule?.interval ?? FALLBACK_RETRY_INTERVAL };