Added support for the native docker images in the builds (demisto#23366)

* Added native:latest to the docker_native_image_config.json * Split Run unit test and lint Job into 3 jobs (static names) * Added to nightly * Added parameter to the job reference * Configured poetry to look at the relevant SDK branch * Editing a .py file for triggering lint * Editing a .py file for triggering lint - on a valid file * Fixed the job's name * Added docker image - 'all' flag to the gitlab build * Moved to new naming * Solved conflicts * poetry lock * poetry lock * Added ignored_content_items * poetry lock * Added Rasterize integration to ignored_content_items * Upgraded poetry and re-lock * Upgraded the content-build docker image in the config.yml * Commented restore cache in config.yml * Commented the call to restore cache in config.yml * Added curl commands to prepare env step * Reverted the change in the build docker image * Solved conflicts * Poetry re-lock with poetry version 1.2.0 * Poetry re-lock with poetry version 1.2.0 * Changed the Traps ID * Reverted poetry version change in the config.yml * Added native:dev failures to ignored_content_items * re-lock with poetry configured to the sdk merge commit * Added FindEmailCampaign pylint error to ignore list * Added RegexExtractAll pylint error to ignore list * Reverted the change in OpenLDAP integration * Updated the poetry.lock and pyproject.toml files * Added a change to AD integration to test the lint jobs * Reverted the change in AD integration * Update .circleci/config.yml Co-authored-by: eli sharf <57587340+esharf@users.noreply.github.com> Co-authored-by: yucohen <yucohen@paloaltonetworks.com> Co-authored-by: GuyAfik <guyafik11@gmail.com> Co-authored-by: eli sharf <57587340+esharf@users.noreply.github.com>
BestPig · Jan 24, 2023 · c47f07d · c47f07d
1 parent 8c96994
commit c47f07d
Show file tree

Hide file tree

Showing 3 changed files with 115 additions and 6 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -83,7 +83,6 @@ references:
       when: always
       command: |
         poetry --version
-
         # Check if CircleCI's config file and poetry files files are up to date
         # if poetry isn't up-to-date, checkout from origin/master.
         ./Tests/scripts/is_file_up_to_date.sh .circleci/config.yml $CIRCLE_BRANCH
@@ -174,7 +173,10 @@ references:
 
   run_unit_testing_and_lint: &run_unit_testing_and_lint
     run:
-      name: Run Unit Testing and Lint
+      parameters:
+        dockerimageflag:
+          type: string
+      name: Run Unit Testing And Lint - Docker Image:<< parameters.dockerimageflag >>
       when: always
       no_output_timeout: 5h
       command: |
@@ -192,7 +194,7 @@ references:
         echo "vulture py2 version: $(python2 -m vulture --version 2>&1)"
         echo "vulture py3 version: $(python3 -m vulture --version 2>&1)"
         mkdir ./unit-tests
-        demisto-sdk lint -p 8 -g -vvv --test-xml ./unit-tests --log-path ./artifacts --failure-report ./artifacts --coverage-report $ARTIFACTS_FOLDER/coverage_report
+        demisto-sdk lint -p 8 -g -vvv --test-xml ./unit-tests --log-path ./artifacts --failure-report ./artifacts --coverage-report $ARTIFACTS_FOLDER/coverage_report --docker-image << parameters.dockerimageflag >>
 
   generate_coverage_reports: &generate_coverage_reports
     run:
@@ -282,6 +284,10 @@ references:
         context: nightly_env
         requires:
           - Setup Environment
+        matrix:
+          parameters:
+            dockerimageflag: [ "native:ga", "native:maintenance", "native:dev", "from-yml" ]
+        name: Run Unit Testing And Lint - Docker Image:<< matrix.dockerimageflag >>
     - Run Validations:
         requires:
           - Setup Environment
@@ -308,6 +314,9 @@ jobs:
     <<: *container_config
     resource_class: medium
     <<: *environment
+    parameters:
+      dockerimageflag:
+        type: string
     steps:
       - *attach_workspace
       - *remote_docker
@@ -373,6 +382,10 @@ workflows:
       - Run Unit Testing And Lint:
           requires:
             - Setup Environment
+          matrix:
+            parameters:
+              dockerimageflag: [ "native:ga", "native:maintenance", "native:dev", "from-yml" ]
+          name: Run Unit Testing And Lint - Docker Image:<< matrix.dockerimageflag >>
       - Run Validations:
           requires:
             - Setup Environment

diff --git a/.gitlab/ci/global.yml b/.gitlab/ci/global.yml
@@ -252,7 +252,7 @@
         mkdir ./unit-tests
         if [ -n "$SHOULD_LINT_ALL" ]; then
           echo -e  "----------\nLinting all because:\n${SHOULD_LINT_ALL}\n----------"
-          demisto-sdk lint -vvv -p 10 -a --test-xml ./unit-tests --log-path $ARTIFACTS_FOLDER --failure-report $ARTIFACTS_FOLDER --coverage-report $ARTIFACTS_FOLDER/coverage_report -dt 120 --time-measurements-dir $ARTIFACTS_FOLDER
+          demisto-sdk lint -vvv -p 10 -a --test-xml ./unit-tests --log-path $ARTIFACTS_FOLDER --failure-report $ARTIFACTS_FOLDER --coverage-report $ARTIFACTS_FOLDER/coverage_report -dt 120 --time-measurements-dir $ARTIFACTS_FOLDER --docker-image all
         else
           if [[ -n $BUCKET_UPLOAD ]]; then
             demisto-sdk lint -vvv -p 8 -g --no-mypy --prev-ver $LAST_UPLOAD_COMMIT -v --test-xml ./unit-tests --log-path $ARTIFACTS_FOLDER --failure-report $ARTIFACTS_FOLDER --coverage-report $ARTIFACTS_FOLDER/coverage_report -cdam

diff --git a/Tests/docker_native_image_config.json b/Tests/docker_native_image_config.json
@@ -30,15 +30,111 @@
             "tesseract"
          ],
          "docker_ref": "demisto/py3-native:8.1.0.40949"
+      },
+      "native:dev":{
+         "supported_docker_images":[
+            "python3",
+            "python3-deb",
+            "python3-ubi",
+            "py3-tools",
+            "py3-tools-ubi",
+            "crypto",
+            "readpdf",
+            "parse-emails",
+            "docxpy",
+            "sklearn",
+            "pandas",
+            "ippysocks-py3",
+            "oauthlib",
+            "unzip",
+            "py3ews",
+            "taxii2",
+            "pan-os-python",
+            "slackv3",
+            "google-api-py3",
+            "boto3py3",
+            "pyjwt3",
+            "joe-security",
+            "slack",
+            "office-utils",
+            "chromium",
+            "tesseract"
+         ]
       }
    },
    "ignored_content_items":[
       {
          "id":"UnzipFile",
-         "reason":"Failed unit-test: test_unrar_no_password",
+         "reason":"Issue: CIAC-5246 - Failed unit-test: test_unrar_no_password.",
+         "ignored_native_images":[
+            "native:8.1",
+            "native:dev"
+         ]
+      },
+      {
+         "id":"DockerHardeningCheck",
+         "reason":"Issue: CIAC-5248 - Failed unit-test: get_default_gateway (ip command does not exist).",
+         "ignored_native_images":[
+            "native:8.1",
+            "native:dev"
+         ]
+      },
+      {
+         "id":"FetchIndicatorsFromFile",
+         "reason":"issue: CIAC-5243 - ElementTree object (xml) in python3.9 has getiterator attribute while in python 3.10 this attribute does not exist - causing unit tests failures.",
+         "ignored_native_images":[
+            "native:8.1",
+            "native:dev"
+         ]
+      },
+      {
+         "id":"Traps",
+         "reason":"Integration is Deprecated.",
+         "ignored_native_images": [
+            "native:8.1",
+            "native:dev"
+         ]
+      },
+      {
+         "id":"Rasterize",
+         "reason":"Issue: CRTX-72338.",
+         "ignored_native_images":[
+            "native:8.1"
+         ]
+      },
+      {
+         "id":"Intezer v2",
+         "reason":"Issue: CIAC-5241.",
+         "ignored_native_images":[
+            "native:dev"
+         ]
+      },
+      {
+         "id":"PDFUnlocker",
+         "reason":"Issue: CIAC-5247.",
+         "ignored_native_images":[
+            "native:dev"
+         ]
+      },
+      {
+         "id":"FindEmailCampaign",
+         "reason":"Issue: CIAC-5249 - pylint error.",
+         "ignored_native_images":[
+            "native:8.1",
+            "native:dev"
+         ]
+      },
+      {
+         "id":"RegexExtractAll",
+         "reason":"Issue: CIAC-5259 - pylint error.",
          "ignored_native_images":[
             "native:8.1"
          ]
       }
-   ]
+   ],
+   "flags_versions_mapping": {
+      "native:dev" : "native:dev",
+      "native:ga" : "native:8.1",
+      "native:maintenance" : ""
+   }
 }