Fix #2095, Automatic suppression of flooding events

cmake/sample_defs/cpu1_platform_cfg.h

@@ -1337,6 +1337,29 @@
 */
 #define CFE_PLATFORM_EVS_MAX_EVENT_FILTERS 8
 
+/**
+**  \cfeevscfg Maximum number of event before squelching
+**
+**  \par Description:
+**       Maximum number of events that may be emitted per app per second.
+**       Setting this to 0 will cause events to be unrestricted.
+**
+**  \par Limits
+**       This number must be less than or equal to INT_MAX
+*/
+#define CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST 16
+
+/**
+**  \cfeevscfg Sustained number of event messages per second per app before squelching
+**
+**  \par Description:
+**       Sustained number of events that may be emitted per app per second.
+**
+**  \par Limits
+**       This number must be less than or equal to #CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST
+*/
+#define CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC 8
+
 /**
 **  \cfeevscfg Default Event Log Filename
 **

docs/src/mnem_maps/cfe_evs_tlm_mnem_map

@@ -33,7 +33,7 @@ EVS_APP=$sc_$cpu_EVS_APP[CFE_PLATFORM_ES_MAX_APPLICATIONS] \
 EVS_APPID=$sc_$cpu_EVS_APP[CFE_PLATFORM_ES_MAX_APPLICATIONS].APPID \
 EVS_APPMSGSENTC=$sc_$cpu_EVS_APP[CFE_PLATFORM_ES_MAX_APPLICATIONS].APPMSGSENTC \
 EVS_APPENASTAT=$sc_$cpu_EVS_APP[CFE_PLATFORM_ES_MAX_APPLICATIONS].APPENASTAT \
-EVS_SPARE2ALIGN3=$sc_$cpu_EVS_APP[CFE_PLATFORM_ES_MAX_APPLICATIONS].SPARE2ALIGN3 \
+EVS_SQUELCHEDC=$sc_$cpu_EVS_APP[CFE_PLATFORM_ES_MAX_APPLICATIONS].SQUELCHEDC \
 EVS_APPNAME=$sc_$cpu_EVS_APPNAME[OS_MAX_API_NAME] \
 EVS_EVENTID=$sc_$cpu_EVS_EVENTID \
 EVS_EVENTTYPE=$sc_$cpu_EVS_EVENTTYPE \

modules/evs/eds/cfe_evs.xml

@@ -179,6 +179,11 @@
               \cfetlmmnemonic  \EVS_APPENASTAT
             </LongDescription>
           </Entry>
+          <Entry name="AppMessageSquelchedCounter" type="BASE_TYPES/uint8" shortDescription="Application event squelch count">
+            <LongDescription>
+              \cfetlmmnemonic  \EVS_SQUELCHEDC
+            </LongDescription>
+          </Entry>
         </EntryList>
       </ContainerDataType>
 

modules/evs/fsw/inc/cfe_evs_events.h

@@ -466,6 +466,19 @@
  *  Invalid length for the command code in message ID #CFE_EVS_CMD_MID received on the EVS message pipe.
  */
 #define CFE_EVS_LEN_ERR_EID 43
+
+/**
+ * \brief EVS Events Squelched Error Event ID
+ *
+ *  \par Type: ERROR
+ *
+ *  \par Cause:
+ *
+ *  Events generated in app at a rate in excess of
+ *  #CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST in one moment or
+ *  #CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC sustained
+ */
+#define CFE_EVS_SQUELCHED_ERR_EID 43
 /**\}*/
 
 #endif /* CFE_EVS_EVENTS_H */

modules/evs/fsw/inc/cfe_evs_msg.h

@@ -1157,14 +1157,14 @@ typedef CFE_EVS_AppNameEventIDMaskCmd_t CFE_EVS_SetFilterCmd_t;
 /**********************************/
 typedef struct CFE_EVS_AppTlmData
 {
-    CFE_ES_AppId_t AppID;         /**< \cfetlmmnemonic \EVS_APPID
-                                       \brief Numerical application identifier */
-    uint16 AppMessageSentCounter; /**< \cfetlmmnemonic \EVS_APPMSGSENTC
-                                       \brief Application message sent counter */
-    uint8 AppEnableStatus;        /**< \cfetlmmnemonic \EVS_APPENASTAT
-                                       \brief Application event service enable status */
-    uint8 Padding;                /**< \cfetlmmnemonic \EVS_SPARE2ALIGN3
-                                       \brief Padding for 32 bit boundary */
+    CFE_ES_AppId_t AppID;             /**< \cfetlmmnemonic \EVS_APPID
+                                           \brief Numerical application identifier */
+    uint16 AppMessageSentCounter;     /**< \cfetlmmnemonic \EVS_APPMSGSENTC
+                                           \brief Application message sent counter */
+    uint8 AppEnableStatus;            /**< \cfetlmmnemonic \EVS_APPENASTAT
+                                           \brief Application event service enable status */
+    uint8 AppMessageSquelchedCounter; /**< \cfetlmmnemonic \EVS_SQUELCHEDC
+                                       \brief Number of events squelched */
 
 } CFE_EVS_AppTlmData_t;
 

modules/evs/fsw/src/cfe_evs.c

@@ -71,6 +71,7 @@ CFE_Status_t CFE_EVS_Register(const void *Filters, uint16 NumEventFilters, uint1
             /* Initialize application event data */
             AppDataPtr->ActiveFlag           = true;
             AppDataPtr->EventTypesActiveFlag = CFE_PLATFORM_EVS_DEFAULT_TYPE_FLAG;
+            AppDataPtr->SquelchTokens        = CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST;
 
             /* Set limit for number of provided filters */
             if (NumEventFilters < CFE_PLATFORM_EVS_MAX_EVENT_FILTERS)
@@ -141,7 +142,8 @@ CFE_Status_t CFE_EVS_SendEvent(uint16 EventID, uint16 EventType, const char *Spe
             /* Handler for events from apps not registered with EVS */
             Status = EVS_NotRegistered(AppDataPtr, AppID);
         }
-        else if (EVS_IsFiltered(AppDataPtr, EventID, EventType) == false)
+        else if (EVS_IsFiltered(AppDataPtr, EventID, EventType) == false &&
+                 EVS_CheckAndIncrementSquelchTokens(AppDataPtr) == true)
         {
             /* Get current spacecraft time */
             Time = CFE_TIME_GetTime();
@@ -186,7 +188,8 @@ CFE_Status_t CFE_EVS_SendEventWithAppID(uint16 EventID, uint16 EventType, CFE_ES
         /* Handler for events from apps not registered with EVS */
         Status = EVS_NotRegistered(AppDataPtr, AppID);
     }
-    else if (EVS_IsFiltered(AppDataPtr, EventID, EventType) == false)
+    else if (EVS_IsFiltered(AppDataPtr, EventID, EventType) == false &&
+             EVS_CheckAndIncrementSquelchTokens(AppDataPtr) == true)
     {
         /* Get current spacecraft time */
         Time = CFE_TIME_GetTime();
@@ -229,7 +232,8 @@ CFE_Status_t CFE_EVS_SendTimedEvent(CFE_TIME_SysTime_t Time, uint16 EventID, uin
             /* Handler for events from apps not registered with EVS */
             Status = EVS_NotRegistered(AppDataPtr, AppID);
         }
-        else if (EVS_IsFiltered(AppDataPtr, EventID, EventType) == false)
+        else if (EVS_IsFiltered(AppDataPtr, EventID, EventType) == false &&
+                 EVS_CheckAndIncrementSquelchTokens(AppDataPtr) == true)
         {
             /* Send the event packets */
             va_start(Ptr, Spec);

modules/evs/fsw/src/cfe_evs_task.c

@@ -653,9 +653,11 @@ int32 CFE_EVS_ReportHousekeepingCmd(const CFE_MSG_CommandHeader_t *data)
     {
         if (EVS_AppDataIsUsed(AppDataPtr))
         {
-            AppTlmDataPtr->AppID                 = EVS_AppDataGetID(AppDataPtr);
-            AppTlmDataPtr->AppEnableStatus       = AppDataPtr->ActiveFlag;
-            AppTlmDataPtr->AppMessageSentCounter = AppDataPtr->EventCount;
+            AppTlmDataPtr->AppID                      = EVS_AppDataGetID(AppDataPtr);
+            AppTlmDataPtr->AppEnableStatus            = AppDataPtr->ActiveFlag;
+            AppTlmDataPtr->AppMessageSentCounter      = AppDataPtr->EventCount;
+            AppTlmDataPtr->AppMessageSquelchedCounter = AppDataPtr->SquelchedCount;
+
             ++j;
             ++AppTlmDataPtr;
         }
@@ -665,9 +667,10 @@ int32 CFE_EVS_ReportHousekeepingCmd(const CFE_MSG_CommandHeader_t *data)
     /* Clear unused portion of event state data in telemetry packet */
     for (i = j; i < CFE_MISSION_ES_MAX_APPLICATIONS; i++)
     {
-        AppTlmDataPtr->AppID                 = CFE_ES_APPID_UNDEFINED;
-        AppTlmDataPtr->AppEnableStatus       = false;
-        AppTlmDataPtr->AppMessageSentCounter = 0;
+        AppTlmDataPtr->AppID                      = CFE_ES_APPID_UNDEFINED;
+        AppTlmDataPtr->AppEnableStatus            = false;
+        AppTlmDataPtr->AppMessageSentCounter      = 0;
+        AppTlmDataPtr->AppMessageSquelchedCounter = 0;
     }
 
     CFE_SB_TimeStampMsg(CFE_MSG_PTR(CFE_EVS_Global.EVS_TlmPkt.TelemetryHeader));
@@ -1614,6 +1617,7 @@ int32 CFE_EVS_WriteAppDataFileCmd(const CFE_EVS_WriteAppDataFileCmd_t *data)
                     AppDataFile.ActiveFlag           = AppDataPtr->ActiveFlag;
                     AppDataFile.EventCount           = AppDataPtr->EventCount;
                     AppDataFile.EventTypesActiveFlag = AppDataPtr->EventTypesActiveFlag;
+                    AppDataFile.SquelchedCount       = AppDataPtr->SquelchedCount;
 
                     /* Copy application filter data to application file data record */
                     memcpy(AppDataFile.Filters, AppDataPtr->BinFilters,

modules/evs/fsw/src/cfe_evs_task.h

@@ -44,6 +44,7 @@
 #include "cfe_platform_cfg.h"
 #include "cfe_mission_cfg.h"
 #include "osconfig.h"
+#include "cfe_time.h"
 #include "cfe_evs_api_typedefs.h"
 #include "cfe_evs_log_typedef.h"
 #include "cfe_sb_api_typedefs.h"
@@ -86,18 +87,23 @@ typedef struct
 
     EVS_BinFilter_t BinFilters[CFE_PLATFORM_EVS_MAX_EVENT_FILTERS]; /* Array of binary filters */
 
-    uint8  ActiveFlag;           /* Application event service active flag */
-    uint8  EventTypesActiveFlag; /* Application event types active flag */
-    uint16 EventCount;           /* Application event counter */
+    uint8     ActiveFlag;                /* Application event service active flag */
+    uint8     EventTypesActiveFlag;      /* Application event types active flag */
+    uint16    EventCount;                /* Application event counter */
+    OS_time_t LastSquelchCreditableTime; /* Time of last squelch token return */
+    int32     SquelchTokens;             /* Application event squelch token counter */
+    uint8     SquelchedCount;            /* Application events squelched counter */
 
 } EVS_AppData_t;
 
 typedef struct
 {
-    char            AppName[OS_MAX_API_NAME];                    /* Application name */
-    uint8           ActiveFlag;                                  /* Application event service active flag */
-    uint8           EventTypesActiveFlag;                        /* Application event types active flag */
-    uint16          EventCount;                                  /* Application event counter */
+    char            AppName[OS_MAX_API_NAME]; /* Application name */
+    uint8           ActiveFlag;               /* Application event service active flag */
+    uint8           EventTypesActiveFlag;     /* Application event types active flag */
+    uint16          EventCount;               /* Application event counter */
+    uint8           SquelchedCount;           /* Application events squelched counter */
+    uint8           Spare[3];
     EVS_BinFilter_t Filters[CFE_PLATFORM_EVS_MAX_EVENT_FILTERS]; /* Application event filters */
 
 } CFE_EVS_AppDataFile_t;

modules/evs/fsw/src/cfe_evs_utils.c

@@ -28,6 +28,7 @@
 
 /* Include Files */
 #include "cfe_evs_module_all.h" /* All EVS internal definitions and API */
+#include "cfe_evs_utils.h"
 
 #include <stdio.h>
 #include <string.h>
@@ -288,6 +289,105 @@ bool EVS_IsFiltered(EVS_AppData_t *AppDataPtr, uint16 EventID, uint16 EventType)
     return (Filtered);
 }
 
+/*----------------------------------------------------------------
+ *
+ * Function: EVS_CheckAndIncrementSquelchTokens
+ *
+ * Application-scope internal function
+ * See description in header file for argument/return detail
+ *
+ *-----------------------------------------------------------------*/
+bool EVS_CheckAndIncrementSquelchTokens(EVS_AppData_t *AppDataPtr)
+{
+    bool      NotSquelched = true;
+    OS_time_t CurrentTime;
+    int64     DeltaTimeMs;
+    int64     CreditCount;
+    char      AppName[OS_MAX_API_NAME];
+
+    /* Set maximum token credits to burst size */
+    const int32 UPPER_THRESHOLD = CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST;
+    /*
+     * Set lower threshold to stop decrementing
+     * Make this -CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST to add some hysteresis
+     * Events will resume (CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST /
+     * CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC + 1 /
+     * CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC) seconds after flooding stops if
+     * saturated
+     */
+    const int32 LOWER_THRESHOLD = -CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST;
+
+    if (CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST != 0)
+    {
+        /*
+         * We use a timer here since configurations are not guaranteed to send EVS HK wakeups at 1Hz
+         * Use a non-settable timer to prevent this from breaking w/ time changes
+         */
+        CFE_PSP_GetTime(&CurrentTime);
+        DeltaTimeMs = OS_TimeGetTotalMilliseconds(OS_TimeSubtract(CurrentTime, AppDataPtr->LastSquelchCreditableTime));
+
+        /* Calculate how many tokens to credit in elapsed time since last creditable event */
+        CreditCount = DeltaTimeMs * CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC / 1000;
+
+        if (CreditCount > 0)
+        {
+            /* Update last squelch returned time if we credited any tokens*/
+            AppDataPtr->LastSquelchCreditableTime = CurrentTime;
+
+            /*
+             * Add Credits, to a maximum of UPPER_THRESHOLD
+             * Shouldn't rollover, as calculations are done in int64 space due to
+             * promotion rules then bounded before demotion
+             */
+            if (AppDataPtr->SquelchTokens > UPPER_THRESHOLD)
+            {
+                /* Do nothing, squelch tokens deliberately set high by external means for unit testing */
+            }
+            else if (AppDataPtr->SquelchTokens + CreditCount > UPPER_THRESHOLD)
+            {
+                AppDataPtr->SquelchTokens = UPPER_THRESHOLD;
+            }
+            else
+            {
+                AppDataPtr->SquelchTokens += (int32)CreditCount;
+            }
+        }
+
+        /*
+         * Send event messages if we're spamming (need a minimum credit
+         * count of 1 to maintain neutral or more to transition from
+         * negative to positive, else it's positive to negative and we send event).
+         */
+        if (AppDataPtr->SquelchTokens == 0 && CreditCount == 0)
+        {
+            CFE_ES_GetAppName(AppName, EVS_AppDataGetID(AppDataPtr), sizeof(AppName));
+            EVS_SendEvent(CFE_EVS_SQUELCHED_ERR_EID, CFE_EVS_EventType_ERROR, "Events squelched, AppName = %s",
+                          AppName);
+        }
+
+        /* The above is in a separate block from below in case LOWER_THRESHOLD=0 for no hysteresis */
+
+        if (AppDataPtr->SquelchTokens <= LOWER_THRESHOLD)
+        {
+            NotSquelched = false;
+            /* Stop decrementing if lower-threshold reached */
+            AppDataPtr->SquelchedCount++;
+        }
+        else if (AppDataPtr->SquelchTokens <= 0)
+        {
+            NotSquelched = false;
+            AppDataPtr->SquelchTokens--;
+            AppDataPtr->SquelchedCount++;
+        }
+        else /* AppDataPtr->SquelchTokens > 0, no squelching */
+        {
+            /* NotSquelched already true */
+            AppDataPtr->SquelchTokens--;
+        }
+    }
+    return NotSquelched;
+}
+
 /*----------------------------------------------------------------
  *
  * Function: EVS_FindEventID
@@ -555,6 +655,8 @@ int32 EVS_SendEvent(uint16 EventID, uint16 EventType, const char *Spec, ...)
     AppDataPtr = EVS_GetAppDataByID(CFE_EVS_Global.EVS_AppID);
 
     /* Unlikely, but possible that an EVS event filter was added by command */
+    /* Note that we do not squelch events coming from EVS to prevent event recursion,
+     * and EVS is assumed to be "well-behaved" */
     if (EVS_AppDataIsMatch(AppDataPtr, CFE_EVS_Global.EVS_AppID) &&
         EVS_IsFiltered(AppDataPtr, EventID, EventType) == false)
     {

modules/evs/fsw/src/cfe_evs_utils.h

@@ -190,6 +190,18 @@ int32 EVS_NotRegistered(EVS_AppData_t *AppDataPtr, CFE_ES_AppId_t CallerID);
  */
 bool EVS_IsFiltered(EVS_AppData_t *AppDataPtr, uint16 EventID, uint16 EventType);
 
+/*---------------------------------------------------------------------------------------*/
+/**
+ * @brief Check if event is squelched
+ *
+ * This routine returns false if the squelch token counter has become negative.
+ * Otherwise a value of true is returned. In addition, it updates the squelch
+ * token counter based on time, and emits an event message if squelched.
+ *
+ * If #CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST == 0, this function returns true and is otherwise a no-op
+ */
+bool EVS_CheckAndIncrementSquelchTokens(EVS_AppData_t *AppDataPtr);
+
 /*---------------------------------------------------------------------------------------*/
 /**
  * @brief Find the filter record corresponding to the given event ID

modules/evs/fsw/src/cfe_evs_verify.h

@@ -54,6 +54,14 @@
 #error CFE_PLATFORM_EVS_PORT_DEFAULT cannot be greater than 0x0F!
 #endif
 
+#if CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST > INT32_MAX
+#error CFE_PLATFORM_EVS_MAX_APP_EVENTS_PER_SEC cannot be greater than INT32_MAX
+#endif
+
+#if CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC > CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST
+#error CFE_PLATFORM_EVS_APP_EVENTS_PER_SEC must be <= CFE_PLATFORM_EVS_MAX_APP_EVENT_BURST
+#endif
+
 /*
 ** Validate task stack size...
 */