From 5d5f00a4ff3b78319fbfe35f96be65b9d05d2725 Mon Sep 17 00:00:00 2001
From: ljstella <lstella@splunk.com>
Date: Thu, 22 Aug 2024 14:31:57 -0500
Subject: [PATCH 01/15] Removal of more bits of SSA

---
 contentctl/api.py                           |   2 +-
 contentctl/input/director.py                |  11 +-
 contentctl/objects/enums.py                 |   1 -
 contentctl/objects/ssa_detection.py         | 156 --------------------
 contentctl/objects/ssa_detection_tags.py    | 138 -----------------
 contentctl/output/new_content_yml_output.py |  13 +-
 6 files changed, 7 insertions(+), 314 deletions(-)
 delete mode 100644 contentctl/objects/ssa_detection.py
 delete mode 100644 contentctl/objects/ssa_detection_tags.py

diff --git a/contentctl/api.py b/contentctl/api.py
index 5de988ec..8c996549 100644
--- a/contentctl/api.py
+++ b/contentctl/api.py
@@ -126,7 +126,7 @@ def update_config(config:Union[test,test_servers], **key_value_updates:dict[str,
 def content_to_dict(director:DirectorOutputDto)->dict[str,list[dict[str,Any]]]:
     output_dict:dict[str,list[dict[str,Any]]] = {}
     for contentType in ['detections','stories','baselines','investigations',
-                        'playbooks','macros','lookups','deployments','ssa_detections']:
+                        'playbooks','macros','lookups','deployments',]:
         
         output_dict[contentType] = []
         t:list[SecurityContentObject] = getattr(director,contentType)
diff --git a/contentctl/input/director.py b/contentctl/input/director.py
index 0e27add6..a75090eb 100644
--- a/contentctl/input/director.py
+++ b/contentctl/input/director.py
@@ -18,7 +18,6 @@
 from contentctl.objects.deployment import Deployment
 from contentctl.objects.macro import Macro
 from contentctl.objects.lookup import Lookup
-from contentctl.objects.ssa_detection import SSADetection
 from contentctl.objects.atomic import AtomicTest
 from contentctl.objects.security_content_object import SecurityContentObject
 from contentctl.objects.data_source import DataSource
@@ -33,10 +32,7 @@
 from contentctl.objects.enums import DetectionStatus
 from contentctl.helper.utils import Utils
 
-from contentctl.objects.enums import SecurityContentType
 
-from contentctl.objects.enums import DetectionStatus 
-from contentctl.helper.utils import Utils
 
 
 @dataclass
@@ -60,10 +56,7 @@ class DirectorOutputDto:
 
     def addContentToDictMappings(self, content: SecurityContentObject):
         content_name = content.name
-        if isinstance(content, SSADetection):
-            # Since SSA detections may have the same name as ESCU detection,
-            # for this function we prepend 'SSA ' to the name.
-            content_name = f"SSA {content_name}"
+        
         
         if content_name in self.name_to_content_map:
             raise ValueError(
@@ -149,7 +142,7 @@ def createSecurityContent(self, contentType: SecurityContentType) -> None:
                 os.path.join(self.input_dto.path, str(contentType.name))
             )
             security_content_files = [
-                f for f in files if not f.name.startswith("ssa___")
+                f for f in files 
             ]
         else:
             raise (Exception(f"Cannot createSecurityContent for unknown product."))
diff --git a/contentctl/objects/enums.py b/contentctl/objects/enums.py
index fa294302..4e9f1146 100644
--- a/contentctl/objects/enums.py
+++ b/contentctl/objects/enums.py
@@ -54,7 +54,6 @@ class SecurityContentType(enum.Enum):
     deployments = 7
     investigations = 8
     unit_tests = 9
-    ssa_detections = 10
     data_sources = 11
 
 # Bringing these changes back in line will take some time after
diff --git a/contentctl/objects/ssa_detection.py b/contentctl/objects/ssa_detection.py
deleted file mode 100644
index 036f0b77..00000000
--- a/contentctl/objects/ssa_detection.py
+++ /dev/null
@@ -1,156 +0,0 @@
-from __future__ import annotations
-import uuid
-import string
-import requests
-import time
-from pydantic import BaseModel, validator, root_validator
-from dataclasses import dataclass
-from datetime import datetime
-from typing import Union
-import re
-
-from contentctl.objects.abstract_security_content_objects.detection_abstract import Detection_Abstract
-from contentctl.objects.enums import AnalyticsType
-from contentctl.objects.enums import DataModel
-from contentctl.objects.enums import DetectionStatus
-from contentctl.objects.deployment import Deployment
-from contentctl.objects.ssa_detection_tags import SSADetectionTags
-from contentctl.objects.unit_test_ssa import UnitTestSSA
-from contentctl.objects.unit_test_old import UnitTestOld
-from contentctl.objects.macro import Macro
-from contentctl.objects.lookup import Lookup
-from contentctl.objects.baseline import Baseline
-from contentctl.objects.playbook import Playbook
-from contentctl.helper.link_validator import LinkValidator
-from contentctl.objects.enums import SecurityContentType
-
-class SSADetection(BaseModel):
-    # detection spec
-    name: str
-    id: str
-    version: int
-    date: str
-    author: str
-    type: AnalyticsType = ...
-    status: DetectionStatus = ...
-    detection_type: str = None
-    description: str
-    data_source: list[str]
-    search: Union[str, dict]
-    how_to_implement: str
-    known_false_positives: str
-    references: list
-    tags: SSADetectionTags
-    tests: list[UnitTestSSA] = None
-
-    # enrichments
-    annotations: dict = None
-    risk: list = None
-    mappings: dict = None
-    file_path: str = None
-    source: str = None
-    test: Union[UnitTestSSA, dict, UnitTestOld] = None
-    runtime: str = None
-    internalVersion: int = None
-
-    # @validator('name')v
-    # def name_max_length(cls, v, values):
-    #     if len(v) > 67:
-    #         raise ValueError('name is longer then 67 chars: ' + v)
-    #     return v
-
-    class Config:
-        use_enum_values = True
-
-    '''
-    @validator("name")
-    def name_invalid_chars(cls, v):
-        invalidChars = set(string.punctuation.replace("-", ""))
-        if any(char in invalidChars for char in v):
-            raise ValueError("invalid chars used in name: " + v)
-        return v
-
-    @validator("id")
-    def id_check(cls, v, values):
-        try:
-            uuid.UUID(str(v))
-        except:
-            raise ValueError("uuid is not valid: " + values["name"])
-        return v
-
-    @validator("date")
-    def date_valid(cls, v, values):
-        try:
-            datetime.strptime(v, "%Y-%m-%d")
-        except:
-            raise ValueError("date is not in format YYYY-MM-DD: " + values["name"])
-        return v
-
-    # @validator("type")
-    # def type_valid(cls, v, values):
-    #     if v.lower() not in [el.name.lower() for el in AnalyticsType]:
-    #         raise ValueError("not valid analytics type: " + values["name"])
-    #     return v
-
-    @validator("description", "how_to_implement")
-    def encode_error(cls, v, values, field):
-        try:
-            v.encode("ascii")
-        except UnicodeEncodeError:
-            raise ValueError("encoding error in " + field.name + ": " + values["name"])
-        return v
-
-    # @root_validator
-    # def search_validation(cls, values):
-    #     if 'ssa_' not in values['file_path']:
-    #         if not '_filter' in values['search']:
-    #             raise ValueError('filter macro missing in: ' + values["name"])
-    #         if any(x in values['search'] for x in ['eventtype=', 'sourcetype=', ' source=', 'index=']):
-    #             if not 'index=_internal' in values['search']:
-    #                 raise ValueError('Use source macro instead of eventtype, sourcetype, source or index in detection: ' + values["name"])
-    #     return values
-
-    @root_validator
-    def name_max_length(cls, values):
-        # Check max length only for ESCU searches, SSA does not have that constraint
-        if "ssa_" not in values["file_path"]:
-            if len(values["name"]) > 67:
-                raise ValueError("name is longer then 67 chars: " + values["name"])
-        return values
-
-
-    @root_validator
-    def new_line_check(cls, values):
-        # Check if there is a new line in description and how to implement that is not escaped
-        pattern = r'(?<!\\)\n' 
-        if re.search(pattern, values["description"]):
-            match_obj = re.search(pattern,values["description"])
-            words = values["description"][:match_obj.span()[0]].split()[-10:]
-            newline_context = ' '.join(words)
-            raise ValueError(f"Field named 'description' contains new line that is not escaped using backslash. Add backslash at the end of the line after the words: '{newline_context}' in '{values['name']}'")
-        if re.search(pattern, values["how_to_implement"]):
-            match_obj = re.search(pattern,values["how_to_implement"])
-            words = values["how_to_implement"][:match_obj.span()[0]].split()[-10:]
-            newline_context = ' '.join(words)
-            raise ValueError(f"Field named 'how_to_implement' contains new line that is not escaped using backslash. Add backslash at the end of the line after the words: '{newline_context}' in '{values['name']}'")
-        return values
-
-    # @validator('references')
-    # def references_check(cls, v, values):
-    #     return LinkValidator.SecurityContentObject_validate_references(v, values)
-
-
-    @validator("search")
-    def search_validate(cls, v, values):
-        # write search validator
-        return v
-
-    @validator("tests")
-    def tests_validate(cls, v, values):
-        if (values.get("status","") in [DetectionStatus.production.value, DetectionStatus.validation.value]) and not v:
-            raise ValueError(
-                "At least one test is required for a production or validation detection: " + values["name"]
-            )
-        return v
-
-    '''
\ No newline at end of file
diff --git a/contentctl/objects/ssa_detection_tags.py b/contentctl/objects/ssa_detection_tags.py
deleted file mode 100644
index a2be2b20..00000000
--- a/contentctl/objects/ssa_detection_tags.py
+++ /dev/null
@@ -1,138 +0,0 @@
-from __future__ import annotations
-import re
-from typing import List
-from pydantic import BaseModel, validator, ValidationError, model_validator, Field
-
-from contentctl.objects.mitre_attack_enrichment import MitreAttackEnrichment
-from contentctl.objects.constants import *
-from contentctl.objects.enums import SecurityContentProductName
-
-class SSADetectionTags(BaseModel):
-    # detection spec
-    #name: str
-    analytic_story: list
-    asset_type: str
-    automated_detection_testing: str = None
-    cis20: list = None
-    confidence: int
-    impact: int
-    kill_chain_phases: list = None
-    message: str
-    mitre_attack_id: list = None
-    nist: list = None
-    observable: list
-    product: List[SecurityContentProductName] = Field(...,min_length=1)
-    required_fields: list
-    risk_score: int
-    security_domain: str
-    risk_severity: str = None
-    cve: list = None
-    supported_tas: list = None
-    atomic_guid: list = None
-    drilldown_search: str = None
-    manual_test: str = None
-
-
-    # enrichment
-    mitre_attack_enrichments: list[MitreAttackEnrichment] = []
-    confidence_id: int = None
-    impact_id: int = None
-    context_ids: list = None
-    risk_level_id: int = None
-    risk_level: str = None
-    observable_str: str = None
-    evidence_str: str = None
-    analytics_story_str: str = None
-    kill_chain_phases_id:dict = None
-    kill_chain_phases_str:str = None
-    research_site_url: str = None
-    event_schema: str = None
-    mappings: list = None
-    annotations: dict = None
-
-
-    @validator('cis20')
-    def tags_cis20(cls, v, values):
-        pattern = r'^CIS ([\d|1\d|20)$' #DO NOT match leading zeroes and ensure no extra characters before or after the string
-        for value in v:
-            if not re.match(pattern, value):
-                raise ValueError(f"CIS control '{value}' is not a valid Control ('CIS 1' -> 'CIS 20'):  {values['name']}")
-        return v
-    
-    @validator('nist')
-    def tags_nist(cls, v, values):
-        # Sourced Courtest of NIST: https://www.nist.gov/system/files/documents/cyberframework/cybersecurity-framework-021214.pdf (Page 19)
-        IDENTIFY = [f'ID.{category}' for category in ["AM", "BE", "GV", "RA", "RM"]      ]
-        PROTECT  = [f'PR.{category}' for category in ["AC", "AT", "DS", "IP", "MA", "PT"]]
-        DETECT   = [f'DE.{category}' for category in ["AE", "CM", "DP"]                  ]
-        RESPOND  = [f'RS.{category}' for category in ["RP", "CO", "AN", "MI", "IM"]      ]
-        RECOVER  = [f'RC.{category}' for category in ["RP", "IM", "CO"]                  ]
-        ALL_NIST_CATEGORIES = IDENTIFY + PROTECT + DETECT + RESPOND + RECOVER
-
-        
-        for value in v:
-            if not value in ALL_NIST_CATEGORIES:
-                raise ValueError(f"NIST Category '{value}' is not a valid category")
-        return v
-
-    @validator('confidence')
-    def tags_confidence(cls, v, values):
-        v = int(v)
-        if not (v > 0 and v <= 100):
-             raise ValueError('confidence score is out of range 1-100.' )
-        else:
-            return v
-
-
-    @validator('impact')
-    def tags_impact(cls, v, values):
-        if not (v > 0 and v <= 100):
-             raise ValueError('impact score is out of range 1-100.')
-        else:
-            return v
-
-    @validator('kill_chain_phases')
-    def tags_kill_chain_phases(cls, v, values):
-        valid_kill_chain_phases = SES_KILL_CHAIN_MAPPINGS.keys()
-        for value in v:
-            if value not in valid_kill_chain_phases:
-                raise ValueError('kill chain phase not valid. Valid options are ' + str(valid_kill_chain_phases))
-        return v
-
-    @validator('mitre_attack_id')
-    def tags_mitre_attack_id(cls, v, values):
-        pattern = 'T[0-9]{4}'
-        for value in v:
-            if not re.match(pattern, value):
-                raise ValueError('Mitre Attack ID are not following the pattern Txxxx:' )
-        return v
-
-
-
-    @validator('risk_score')
-    def tags_calculate_risk_score(cls, v, values):
-        calculated_risk_score = round(values['impact'] * values['confidence'] / 100)
-        if calculated_risk_score != int(v):
-            raise ValueError(f"Risk Score must be calculated as round(confidence * impact / 100)"
-                             f"\n  Expected risk_score={calculated_risk_score}, found risk_score={int(v)}: {values['name']}")
-        return v
-
-
-    @model_validator(mode="after")
-    def tags_observable(self):
-        valid_roles = SES_OBSERVABLE_ROLE_MAPPING.keys()
-        valid_types = SES_OBSERVABLE_TYPE_MAPPING.keys()
-        
-        for value in self.observable:
-            if value['type'] in valid_types:
-                if 'Splunk Behavioral Analytics' in self.product:
-                    continue
-
-                if 'role' not in value:
-                    raise ValueError('Observable role is missing')
-                for role in value['role']:
-                    if role not in valid_roles:
-                        raise ValueError(f'Observable role ' + role + ' not valid. Valid options are {str(valid_roles)}')
-            else:
-                raise ValueError(f'Observable type ' + value['type'] + ' not valid. Valid options are {str(valid_types)}')
-        return self
\ No newline at end of file
diff --git a/contentctl/output/new_content_yml_output.py b/contentctl/output/new_content_yml_output.py
index df55dd1c..38730b37 100644
--- a/contentctl/output/new_content_yml_output.py
+++ b/contentctl/output/new_content_yml_output.py
@@ -39,11 +39,8 @@ def convertNameToFileName(self, name: str, product: list):
             .replace('.','_') \
             .replace('/','_') \
             .lower()
-        if 'Splunk Behavioral Analytics' in product:
-            
-            file_name = 'ssa___' + file_name + '.yml'
-        else:
-            file_name = file_name + '.yml'
+        
+        file_name = file_name + '.yml'
         return file_name
 
 
@@ -54,8 +51,6 @@ def convertNameToTestFileName(self, name: str, product: list):
             .replace('.','_') \
             .replace('/','_') \
             .lower()
-        if 'Splunk Behavioral Analytics' in product:          
-            file_name = 'ssa___' + file_name + '.test.yml'
-        else:
-            file_name = file_name + '.test.yml'
+        
+        file_name = file_name + '.test.yml'
         return file_name
\ No newline at end of file

From 1f9f10ba9e166da04f4999cc254cf1a20b24ed2c Mon Sep 17 00:00:00 2001
From: 0xC0FFEEEE <119874251+0xC0FFEEEE@users.noreply.github.com>
Date: Mon, 26 Aug 2024 00:13:02 +0100
Subject: [PATCH 02/15] Add bare init option

---
 contentctl/actions/initialize.py | 47 ++++++++++++++++++--------------
 contentctl/objects/config.py     |  3 +-
 2 files changed, 29 insertions(+), 21 deletions(-)

diff --git a/contentctl/actions/initialize.py b/contentctl/actions/initialize.py
index 73a6152b..603fefa3 100644
--- a/contentctl/actions/initialize.py
+++ b/contentctl/actions/initialize.py
@@ -17,29 +17,36 @@ def execute(self, config: test) -> None:
         
         YmlWriter.writeYmlFile(str(config.path/'contentctl.yml'), config.model_dump()) 
 
-        #Create the following empty directories:
-        for emptyDir in ['lookups', 'baselines', 'docs', 'reporting', 'investigations']:
+        if config.bare:
+            #Create the following empty directories:
+            for emptyDir in ['lookups', 'baselines', 'docs', 'reporting', 'investigations', 'app_template', 
+                             'deployments', 'detections', 'data_sources', 'macros', 'stories']:
             #Throw an error if this directory already exists
-            (config.path/emptyDir).mkdir(exist_ok=False)
+                (config.path/emptyDir).mkdir(exist_ok=False)
         
-
-        #copy the contents of all template directories
-        for templateDir, targetDir in [
-            ('../templates/app_template/', 'app_template'),
-            ('../templates/deployments/', 'deployments'),
-            ('../templates/detections/', 'detections'),
-            ('../templates/data_sources/', 'data_sources'),
-            ('../templates/macros/','macros'),
-            ('../templates/stories/', 'stories'),
-        ]:
-            source_directory = pathlib.Path(os.path.dirname(__file__))/templateDir
-            target_directory = config.path/targetDir
-            #Throw an exception if the target exists
-            shutil.copytree(source_directory, target_directory, dirs_exist_ok=False)
+        else:
+            #Create the following empty directories:
+            for emptyDir in ['lookups', 'baselines', 'docs', 'reporting', 'investigations']:
+            #Throw an error if this directory already exists
+                (config.path/emptyDir).mkdir(exist_ok=False)
+
+            #copy the contents of all template directories
+            for templateDir, targetDir in [
+                ('../templates/app_template/', 'app_template'),
+                ('../templates/deployments/', 'deployments'),
+                ('../templates/detections/', 'detections'),
+                ('../templates/data_sources/', 'data_sources'),
+                ('../templates/macros/','macros'),
+                ('../templates/stories/', 'stories'),
+            ]:
+                source_directory = pathlib.Path(os.path.dirname(__file__))/templateDir
+                target_directory = config.path/targetDir
+                #Throw an exception if the target exists
+                shutil.copytree(source_directory, target_directory, dirs_exist_ok=False)
         
-        # Create a README.md file.  Note that this is the README.md for the repository, not the
-        # one which will actually be packaged into the app. That is located in the app_template folder.
-        shutil.copyfile(pathlib.Path(os.path.dirname(__file__))/'../templates/README.md','README.md')
+            # Create a README.md file.  Note that this is the README.md for the repository, not the
+            # one which will actually be packaged into the app. That is located in the app_template folder.
+            shutil.copyfile(pathlib.Path(os.path.dirname(__file__))/'../templates/README.md','README.md')
 
 
         print(f"The app '{config.app.title}' has been initialized. "
diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py
index 2937923c..611276a2 100644
--- a/contentctl/objects/config.py
+++ b/contentctl/objects/config.py
@@ -164,7 +164,8 @@ def serialize_path(path: DirectoryPath)->str:
         return str(path)
 
 class init(Config_Base):
-    pass
+    model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True)
+    bare: bool = Field(default=False, description="Initialize with empty directory structure")
 
 
 class validate(Config_Base):

From 178d67efcbdbf1ef917b4b6793cbc9f1f124ca5d Mon Sep 17 00:00:00 2001
From: 0xC0FFEEEE <119874251+0xC0FFEEEE@users.noreply.github.com>
Date: Mon, 26 Aug 2024 09:32:31 +0100
Subject: [PATCH 03/15] Create detections subdirs

---
 contentctl/actions/initialize.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/contentctl/actions/initialize.py b/contentctl/actions/initialize.py
index 603fefa3..6653b876 100644
--- a/contentctl/actions/initialize.py
+++ b/contentctl/actions/initialize.py
@@ -20,9 +20,10 @@ def execute(self, config: test) -> None:
         if config.bare:
             #Create the following empty directories:
             for emptyDir in ['lookups', 'baselines', 'docs', 'reporting', 'investigations', 'app_template', 
-                             'deployments', 'detections', 'data_sources', 'macros', 'stories']:
+                             'deployments', 'detections/application', 'detections/cloud', 'detections/endpoint', 
+                             'detections/network', 'detections/web', 'data_sources', 'macros', 'stories']:
             #Throw an error if this directory already exists
-                (config.path/emptyDir).mkdir(exist_ok=False)
+                (config.path/emptyDir).mkdir(exist_ok=False, parents=True)
         
         else:
             #Create the following empty directories:

From 36d5b21b376931949e2e74d8ed956c50a2ccea5f Mon Sep 17 00:00:00 2001
From: 0xC0FFEEEE <119874251+0xC0FFEEEE@users.noreply.github.com>
Date: Mon, 26 Aug 2024 09:34:22 +0100
Subject: [PATCH 04/15] Missing indents

---
 contentctl/actions/initialize.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/contentctl/actions/initialize.py b/contentctl/actions/initialize.py
index 6653b876..83c21552 100644
--- a/contentctl/actions/initialize.py
+++ b/contentctl/actions/initialize.py
@@ -22,13 +22,13 @@ def execute(self, config: test) -> None:
             for emptyDir in ['lookups', 'baselines', 'docs', 'reporting', 'investigations', 'app_template', 
                              'deployments', 'detections/application', 'detections/cloud', 'detections/endpoint', 
                              'detections/network', 'detections/web', 'data_sources', 'macros', 'stories']:
-            #Throw an error if this directory already exists
+                #Throw an error if this directory already exists
                 (config.path/emptyDir).mkdir(exist_ok=False, parents=True)
         
         else:
             #Create the following empty directories:
             for emptyDir in ['lookups', 'baselines', 'docs', 'reporting', 'investigations']:
-            #Throw an error if this directory already exists
+                #Throw an error if this directory already exists
                 (config.path/emptyDir).mkdir(exist_ok=False)
 
             #copy the contents of all template directories

From 821475d675a2b2c0a45726d3ab75b62ddaf1acfb Mon Sep 17 00:00:00 2001
From: 0xC0FFEEEE <119874251+0xC0FFEEEE@users.noreply.github.com>
Date: Mon, 26 Aug 2024 19:32:41 +0100
Subject: [PATCH 05/15] Copy app_template directory contents

---
 contentctl/actions/initialize.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/contentctl/actions/initialize.py b/contentctl/actions/initialize.py
index 83c21552..0fac4e4c 100644
--- a/contentctl/actions/initialize.py
+++ b/contentctl/actions/initialize.py
@@ -19,11 +19,16 @@ def execute(self, config: test) -> None:
 
         if config.bare:
             #Create the following empty directories:
-            for emptyDir in ['lookups', 'baselines', 'docs', 'reporting', 'investigations', 'app_template', 
-                             'deployments', 'detections/application', 'detections/cloud', 'detections/endpoint', 
+            for emptyDir in ['lookups', 'baselines', 'docs', 'reporting', 'investigations', 'deployments', 
+                             'detections/application', 'detections/cloud', 'detections/endpoint', 
                              'detections/network', 'detections/web', 'data_sources', 'macros', 'stories']:
                 #Throw an error if this directory already exists
                 (config.path/emptyDir).mkdir(exist_ok=False, parents=True)
+            
+            # Copy the contents of the app_template directory
+            source_directory = pathlib.Path(os.path.dirname(__file__))/'../templates/app_template/'
+            target_directory = config.path/'app_template'
+            shutil.copytree(source_directory, target_directory, dirs_exist_ok=False)
         
         else:
             #Create the following empty directories:

From a5f37694da651b2692ad339088a7e120eeb0bb1c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 6 Sep 2024 05:25:29 +0000
Subject: [PATCH 06/15] Update bottle requirement from ^0.12.25 to
 >=0.12.25,<0.14.0

Updates the requirements on [bottle](https://github.com/bottlepy/bottle) to permit the latest version.
- [Release notes](https://github.com/bottlepy/bottle/releases)
- [Changelog](https://github.com/bottlepy/bottle/blob/master/docs/changelog.rst)
- [Commits](https://github.com/bottlepy/bottle/compare/0.12.25...0.13.0)

---
updated-dependencies:
- dependency-name: bottle
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 29f27374..5b45be0a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -22,7 +22,7 @@ questionary = "^2.0.1"
 docker = "^7.1.0"
 splunk-sdk = "^2.0.2"
 semantic-version = "^2.10.0"
-bottle = "^0.12.25"
+bottle = ">=0.12.25,<0.14.0"
 tqdm = "^4.66.5"
 pygit2 = "^1.15.1"
 tyro = "^0.8.3"

From 473fd9dd2deabab72071811a9c71a337a1bdb6e7 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 6 Sep 2024 10:35:17 -0700
Subject: [PATCH 07/15] fix some whitespace issues when formatting a string
 field for a conf file

---
 contentctl/output/conf_writer.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/contentctl/output/conf_writer.py b/contentctl/output/conf_writer.py
index 2c8e82f7..b103a291 100644
--- a/contentctl/output/conf_writer.py
+++ b/contentctl/output/conf_writer.py
@@ -34,7 +34,10 @@ def escapeNewlines(obj:Any):
         # Failing to do so will result in an improperly formatted conf files that
         # cannot be parsed
         if isinstance(obj,str):
-            return obj.replace(f"\n"," \\\n")
+            # Remove leading and trailing characters. Conf parsers may erroneously 
+            # Parse fields if they have leading or trailing newlines/whitespace and we 
+            # probably don't want that anyway as it doesn't look good in output
+            return obj.strip().replace(f"\n"," \\\n")
         else:
             return obj
 

From 148c12794b62f3d811b73e9106b53bc390670d2a Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 6 Sep 2024 10:45:05 -0700
Subject: [PATCH 08/15] Update comment/docstring on function

---
 .../detection_abstract.py                              | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py
index 075fb7a2..34904a9f 100644
--- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py
+++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py
@@ -74,6 +74,8 @@ class Detection_Abstract(SecurityContentObject):
 
     data_source_objects: list[DataSource] = []
 
+
+
     @field_validator("search", mode="before")
     @classmethod
     def validate_presence_of_filter_macro(cls, value:str, info:ValidationInfo)->str:
@@ -83,15 +85,13 @@ def validate_presence_of_filter_macro(cls, value:str, info:ValidationInfo)->str:
 
 
         Args:
-            value (Union[str, dict[str,Any]]): The search. It can either be a string (and should be
-                SPL or a dict, in which case it is Sigma-formatted.
+            value (str): The SPL search. It must be an SPL-formatted string.
             info (ValidationInfo): The validation info can contain a number of different objects.
                 Today it only contains the director.
 
         Returns:
-            Union[str, dict[str,Any]]: The search, either in sigma or SPL format.
-        """        
-        
+            str: The search, as an SPL formatted string.
+        """
         
         # Otherwise, the search is SPL.
 

From f8995a2d643f529e41768b5c20a013900405e995 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 10 Sep 2024 10:06:46 -0700
Subject: [PATCH 09/15] Update pyproject.toml

Bump to 4.3.4 in prep for release
---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 29f27374..0c0b8583 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "contentctl"
-version = "4.3.3"
+version = "4.3.4"
 description = "Splunk Content Control Tool"
 authors = ["STRT <research@splunk.com>"]
 license = "Apache 2.0"

From 225840012deaee4576027e78c8100729577ae673 Mon Sep 17 00:00:00 2001
From: ljstella <lstella@splunk.com>
Date: Thu, 12 Sep 2024 11:59:11 -0500
Subject: [PATCH 10/15] Remove unit_test_ssa and unit_test_old

---
 contentctl/objects/unit_test_old.py | 10 ----------
 contentctl/objects/unit_test_ssa.py | 31 -----------------------------
 2 files changed, 41 deletions(-)
 delete mode 100644 contentctl/objects/unit_test_old.py
 delete mode 100644 contentctl/objects/unit_test_ssa.py

diff --git a/contentctl/objects/unit_test_old.py b/contentctl/objects/unit_test_old.py
deleted file mode 100644
index 3858e01a..00000000
--- a/contentctl/objects/unit_test_old.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from __future__ import annotations
-from pydantic import BaseModel
-
-
-from contentctl.objects.unit_test_ssa import UnitTestSSA
-
-
-class UnitTestOld(BaseModel):
-    name: str
-    tests: list[UnitTestSSA]
\ No newline at end of file
diff --git a/contentctl/objects/unit_test_ssa.py b/contentctl/objects/unit_test_ssa.py
deleted file mode 100644
index 150b9efe..00000000
--- a/contentctl/objects/unit_test_ssa.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from __future__ import annotations
-from typing import Optional
-from pydantic import BaseModel, Field
-from pydantic import Field
-
-
-class UnitTestAttackDataSSA(BaseModel):
-    file_name:Optional[str] = None
-    data: str = Field(...)
-    # TODO - should source and sourcetype should be mapped to a list
-    # of supported source and sourcetypes in a given environment?
-    source: str = Field(...)
-
-    sourcetype: Optional[str] = None
-
-
-class UnitTestSSA(BaseModel):
-    """
-    A unit test for a detection
-    """
-    name: str
-
-    # The attack data to be ingested for the unit test
-    attack_data: list[UnitTestAttackDataSSA] = Field(...)
-
-
-
-
-
-    
-

From 2170881c2d5449cd6f8f0d2a14ff2fac33492077 Mon Sep 17 00:00:00 2001
From: ljstella <lstella@splunk.com>
Date: Thu, 12 Sep 2024 12:07:11 -0500
Subject: [PATCH 11/15] Removal from APP enum

---
 contentctl/objects/enums.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/contentctl/objects/enums.py b/contentctl/objects/enums.py
index 4e9f1146..240ba905 100644
--- a/contentctl/objects/enums.py
+++ b/contentctl/objects/enums.py
@@ -68,7 +68,6 @@ class SecurityContentType(enum.Enum):
 
 class SecurityContentProduct(enum.Enum):
     SPLUNK_APP = 1
-    SSA = 2
     API = 3
     CUSTOM = 4
 

From 2a2eeae84451ece3be27fb033cdb994956bc0ee2 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 13 Sep 2024 13:54:39 -0700
Subject: [PATCH 12/15] Remove extra line in python class

---
 .../abstract_security_content_objects/detection_abstract.py      | 1 -
 1 file changed, 1 deletion(-)

diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py
index 34904a9f..6e3a990e 100644
--- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py
+++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py
@@ -75,7 +75,6 @@ class Detection_Abstract(SecurityContentObject):
     data_source_objects: list[DataSource] = []
 
 
-
     @field_validator("search", mode="before")
     @classmethod
     def validate_presence_of_filter_macro(cls, value:str, info:ValidationInfo)->str:

From b60eeb4b2971023fc06dedecc650c97a3bc928f2 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 13 Sep 2024 13:55:53 -0700
Subject: [PATCH 13/15] remove extra line

---
 .../abstract_security_content_objects/detection_abstract.py      | 1 -
 1 file changed, 1 deletion(-)

diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py
index 6e3a990e..02d2756f 100644
--- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py
+++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py
@@ -74,7 +74,6 @@ class Detection_Abstract(SecurityContentObject):
 
     data_source_objects: list[DataSource] = []
 
-
     @field_validator("search", mode="before")
     @classmethod
     def validate_presence_of_filter_macro(cls, value:str, info:ValidationInfo)->str:

From f348bc9e1941c903ed0e7582c492dbc6351f93a9 Mon Sep 17 00:00:00 2001
From: ljstella <lstella@splunk.com>
Date: Fri, 13 Sep 2024 16:45:31 -0500
Subject: [PATCH 14/15] Missed one

---
 contentctl/objects/config.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py
index 93c03660..d07e2459 100644
--- a/contentctl/objects/config.py
+++ b/contentctl/objects/config.py
@@ -233,9 +233,6 @@ def getPackageFilePath(self, include_version:bool=False)->pathlib.Path:
             return self.getBuildDir() / f"{self.app.appid}-{self.app.version}.tar.gz"
         else:
             return self.getBuildDir() / f"{self.app.appid}-latest.tar.gz"
-    
-    def getSSAPath(self)->pathlib.Path:
-        return self.getBuildDir() / "ssa"
 
     def getAPIPath(self)->pathlib.Path:
         return self.getBuildDir() / "api"

From cde4459093e391499acefb89a348de34f1c525e1 Mon Sep 17 00:00:00 2001
From: pyth0n1c <emcginnis@splunk.com>
Date: Fri, 13 Sep 2024 15:43:00 -0700
Subject: [PATCH 15/15] Improve a bit of documentation, add comments, and
 update code structure

---
 contentctl/actions/initialize.py | 59 +++++++++++++++++---------------
 contentctl/objects/config.py     |  7 +++-
 2 files changed, 37 insertions(+), 29 deletions(-)

diff --git a/contentctl/actions/initialize.py b/contentctl/actions/initialize.py
index 0fac4e4c..9a57cd49 100644
--- a/contentctl/actions/initialize.py
+++ b/contentctl/actions/initialize.py
@@ -2,8 +2,6 @@
 import shutil
 import os
 import pathlib
-
-from pydantic import RootModel 
 from contentctl.objects.config import test
 from contentctl.output.yml_writer import YmlWriter
 
@@ -17,42 +15,47 @@ def execute(self, config: test) -> None:
         
         YmlWriter.writeYmlFile(str(config.path/'contentctl.yml'), config.model_dump()) 
 
-        if config.bare:
-            #Create the following empty directories:
-            for emptyDir in ['lookups', 'baselines', 'docs', 'reporting', 'investigations', 'deployments', 
-                             'detections/application', 'detections/cloud', 'detections/endpoint', 
-                             'detections/network', 'detections/web', 'data_sources', 'macros', 'stories']:
-                #Throw an error if this directory already exists
-                (config.path/emptyDir).mkdir(exist_ok=False, parents=True)
-            
-            # Copy the contents of the app_template directory
-            source_directory = pathlib.Path(os.path.dirname(__file__))/'../templates/app_template/'
-            target_directory = config.path/'app_template'
-            shutil.copytree(source_directory, target_directory, dirs_exist_ok=False)
         
-        else:
-            #Create the following empty directories:
-            for emptyDir in ['lookups', 'baselines', 'docs', 'reporting', 'investigations']:
-                #Throw an error if this directory already exists
-                (config.path/emptyDir).mkdir(exist_ok=False)
-
+        #Create the following empty directories:
+        for emptyDir in ['lookups', 'baselines', 'data_sources', 'docs', 'reporting', 'investigations', 
+                            'detections/application', 'detections/cloud', 'detections/endpoint', 
+                            'detections/network', 'detections/web', 'macros', 'stories']:
+            #Throw an error if this directory already exists
+            (config.path/emptyDir).mkdir(exist_ok=False, parents=True)
+
+        # If this is not a bare config, then populate
+        # a small amount of content into the directories
+        if not config.bare:
             #copy the contents of all template directories
             for templateDir, targetDir in [
-                ('../templates/app_template/', 'app_template'),
-                ('../templates/deployments/', 'deployments'),
                 ('../templates/detections/', 'detections'),
                 ('../templates/data_sources/', 'data_sources'),
-                ('../templates/macros/','macros'),
+                ('../templates/macros/', 'macros'),
                 ('../templates/stories/', 'stories'),
             ]:
                 source_directory = pathlib.Path(os.path.dirname(__file__))/templateDir
                 target_directory = config.path/targetDir
-                #Throw an exception if the target exists
-                shutil.copytree(source_directory, target_directory, dirs_exist_ok=False)
+                
+                # Do not throw an exception if the directory exists. In fact, it was 
+                # created above when the structure of the app was created.
+                shutil.copytree(source_directory, target_directory, dirs_exist_ok=True)
         
-            # Create a README.md file.  Note that this is the README.md for the repository, not the
-            # one which will actually be packaged into the app. That is located in the app_template folder.
-            shutil.copyfile(pathlib.Path(os.path.dirname(__file__))/'../templates/README.md','README.md')
+        # The contents of app_template must ALWAYS be copied because it contains
+        # several special files.
+        # For now, we also copy the deployments because the ability to create custom
+        # deployment files is limited with built-in functionality.
+        for templateDir, targetDir in [
+            ('../templates/app_template/', 'app_template'),
+            ('../templates/deployments/', 'deployments')
+        ]:
+            source_directory = pathlib.Path(os.path.dirname(__file__))/templateDir
+            target_directory = config.path/targetDir
+            #Throw an exception if the target exists
+            shutil.copytree(source_directory, target_directory, dirs_exist_ok=False)
+
+        # Create a README.md file.  Note that this is the README.md for the repository, not the
+        # one which will actually be packaged into the app. That is located in the app_template folder.
+        shutil.copyfile(pathlib.Path(os.path.dirname(__file__))/'../templates/README.md','README.md')
 
 
         print(f"The app '{config.app.title}' has been initialized. "
diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py
index 2470295e..cedb4ae7 100644
--- a/contentctl/objects/config.py
+++ b/contentctl/objects/config.py
@@ -172,7 +172,12 @@ def serialize_path(path: DirectoryPath)->str:
 
 class init(Config_Base):
     model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True)
-    bare: bool = Field(default=False, description="Initialize with empty directory structure")
+    bare: bool = Field(default=False, description="contentctl normally provides some some example content "
+                       "(macros, stories, data_sources, and/or analytic stories).  This option disables "
+                       "initialization with that additional contnet.  Note that even if --bare is used, it "
+                       "init will still create the directory structure of the app, "
+                       "include the app_template directory with default content, and content in "
+                       "the deployment/ directory (since it is not yet easily customizable).")
 
 
 # TODO (#266): disable the use_enum_values configuration