Merge branch 'main' into fix_offline_enrichment

contentctl/actions/initialize.py

@@ -2,8 +2,6 @@
 import shutil
 import os
 import pathlib
-
-from pydantic import RootModel 
 from contentctl.objects.config import test
 from contentctl.output.yml_writer import YmlWriter
 
@@ -17,26 +15,44 @@ def execute(self, config: test) -> None:
 
         YmlWriter.writeYmlFile(str(config.path/'contentctl.yml'), config.model_dump()) 
 
+
         #Create the following empty directories:
-        for emptyDir in ['lookups', 'baselines', 'docs', 'reporting', 'investigations']:
+        for emptyDir in ['lookups', 'baselines', 'data_sources', 'docs', 'reporting', 'investigations', 
+                            'detections/application', 'detections/cloud', 'detections/endpoint', 
+                            'detections/network', 'detections/web', 'macros', 'stories']:
             #Throw an error if this directory already exists
-            (config.path/emptyDir).mkdir(exist_ok=False)
+            (config.path/emptyDir).mkdir(exist_ok=False, parents=True)
+
+        # If this is not a bare config, then populate
+        # a small amount of content into the directories
+        if not config.bare:
+            #copy the contents of all template directories
+            for templateDir, targetDir in [
+                ('../templates/detections/', 'detections'),
+                ('../templates/data_sources/', 'data_sources'),
+                ('../templates/macros/', 'macros'),
+                ('../templates/stories/', 'stories'),
+            ]:
+                source_directory = pathlib.Path(os.path.dirname(__file__))/templateDir
+                target_directory = config.path/targetDir
+
+                # Do not throw an exception if the directory exists. In fact, it was 
+                # created above when the structure of the app was created.
+                shutil.copytree(source_directory, target_directory, dirs_exist_ok=True)
 
-
-        #copy the contents of all template directories
+        # The contents of app_template must ALWAYS be copied because it contains
+        # several special files.
+        # For now, we also copy the deployments because the ability to create custom
+        # deployment files is limited with built-in functionality.
         for templateDir, targetDir in [
             ('../templates/app_template/', 'app_template'),
-            ('../templates/deployments/', 'deployments'),
-            ('../templates/detections/', 'detections'),
-            ('../templates/data_sources/', 'data_sources'),
-            ('../templates/macros/','macros'),
-            ('../templates/stories/', 'stories'),
+            ('../templates/deployments/', 'deployments')
         ]:
             source_directory = pathlib.Path(os.path.dirname(__file__))/templateDir
             target_directory = config.path/targetDir
             #Throw an exception if the target exists
             shutil.copytree(source_directory, target_directory, dirs_exist_ok=False)
-        
+
         # Create a README.md file.  Note that this is the README.md for the repository, not the
         # one which will actually be packaged into the app. That is located in the app_template folder.
         shutil.copyfile(pathlib.Path(os.path.dirname(__file__))/'../templates/README.md','README.md')

contentctl/api.py

@@ -126,7 +126,7 @@ def update_config(config:Union[test,test_servers], **key_value_updates:dict[str,
 def content_to_dict(director:DirectorOutputDto)->dict[str,list[dict[str,Any]]]:
     output_dict:dict[str,list[dict[str,Any]]] = {}
     for contentType in ['detections','stories','baselines','investigations',
-                        'playbooks','macros','lookups','deployments','ssa_detections']:
+                        'playbooks','macros','lookups','deployments',]:
 
         output_dict[contentType] = []
         t:list[SecurityContentObject] = getattr(director,contentType)

contentctl/input/director.py

@@ -18,7 +18,6 @@
 from contentctl.objects.deployment import Deployment
 from contentctl.objects.macro import Macro
 from contentctl.objects.lookup import Lookup
-from contentctl.objects.ssa_detection import SSADetection
 from contentctl.objects.atomic import AtomicTest
 from contentctl.objects.security_content_object import SecurityContentObject
 from contentctl.objects.data_source import DataSource
@@ -33,10 +32,7 @@
 from contentctl.objects.enums import DetectionStatus
 from contentctl.helper.utils import Utils
 
-from contentctl.objects.enums import SecurityContentType
 
-from contentctl.objects.enums import DetectionStatus 
-from contentctl.helper.utils import Utils
 
 
 @dataclass
@@ -60,10 +56,7 @@ class DirectorOutputDto:
 
     def addContentToDictMappings(self, content: SecurityContentObject):
         content_name = content.name
-        if isinstance(content, SSADetection):
-            # Since SSA detections may have the same name as ESCU detection,
-            # for this function we prepend 'SSA ' to the name.
-            content_name = f"SSA {content_name}"
+
 
         if content_name in self.name_to_content_map:
             raise ValueError(
@@ -149,7 +142,7 @@ def createSecurityContent(self, contentType: SecurityContentType) -> None:
                 os.path.join(self.input_dto.path, str(contentType.name))
             )
             security_content_files = [
-                f for f in files if not f.name.startswith("ssa___")
+                f for f in files 
             ]
         else:
             raise (Exception(f"Cannot createSecurityContent for unknown product."))

contentctl/objects/abstract_security_content_objects/detection_abstract.py

@@ -83,15 +83,13 @@ def validate_presence_of_filter_macro(cls, value:str, info:ValidationInfo)->str:
 
 
         Args:
-            value (Union[str, dict[str,Any]]): The search. It can either be a string (and should be
-                SPL or a dict, in which case it is Sigma-formatted.
+            value (str): The SPL search. It must be an SPL-formatted string.
             info (ValidationInfo): The validation info can contain a number of different objects.
                 Today it only contains the director.
 
         Returns:
-            Union[str, dict[str,Any]]: The search, either in sigma or SPL format.
-        """        
-
+            str: The search, as an SPL formatted string.
+        """
 
         # Otherwise, the search is SPL.
 

contentctl/objects/config.py

@@ -171,7 +171,13 @@ def serialize_path(path: DirectoryPath)->str:
         return str(path)
 
 class init(Config_Base):
-    pass
+    model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True)
+    bare: bool = Field(default=False, description="contentctl normally provides some some example content "
+                       "(macros, stories, data_sources, and/or analytic stories).  This option disables "
+                       "initialization with that additional contnet.  Note that even if --bare is used, it "
+                       "init will still create the directory structure of the app, "
+                       "include the app_template directory with default content, and content in "
+                       "the deployment/ directory (since it is not yet easily customizable).")
 
 
 # TODO (#266): disable the use_enum_values configuration
@@ -233,9 +239,6 @@ def getPackageFilePath(self, include_version:bool=False)->pathlib.Path:
             return self.getBuildDir() / f"{self.app.appid}-{self.app.version}.tar.gz"
         else:
             return self.getBuildDir() / f"{self.app.appid}-latest.tar.gz"
-
-    def getSSAPath(self)->pathlib.Path:
-        return self.getBuildDir() / "ssa"
 
     def getAPIPath(self)->pathlib.Path:
         return self.getBuildDir() / "api"

contentctl/objects/enums.py

@@ -54,7 +54,6 @@ class SecurityContentType(enum.Enum):
     deployments = 7
     investigations = 8
     unit_tests = 9
-    ssa_detections = 10
     data_sources = 11
 
 # Bringing these changes back in line will take some time after
@@ -69,7 +68,6 @@ class SecurityContentType(enum.Enum):
 
 class SecurityContentProduct(enum.Enum):
     SPLUNK_APP = 1
-    SSA = 2
     API = 3
     CUSTOM = 4