[CI] Add --debug flag to the linter.go invoke tasks to better deb…

…ug golangci-lint witchcraft (#29802)

.gitlab/lint/linux.yml

@@ -13,7 +13,7 @@
     - inv -e rtloader.make --install-prefix=$CI_PROJECT_DIR/dev --python-runtimes "3"
     - inv -e rtloader.install
     - inv -e install-tools
-    - inv -e linter.go --cpus $KUBERNETES_CPU_REQUEST $FLAVORS $EXTRA_OPTS
+    - inv -e linter.go --cpus $KUBERNETES_CPU_REQUEST --debug $FLAVORS $EXTRA_OPTS
 
 .linux_x64:
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES

.gitlab/lint/macos.yml

@@ -9,7 +9,7 @@ include:
   script:
     - !reference [.retrieve_linux_go_deps]
     - !reference [.retrieve_linux_go_tools_deps]
-    - inv -e linter.go --cpus 12 --timeout 60
+    - inv -e linter.go --cpus 12 --debug --timeout 60
 
 lint_macos_gitlab_amd64:
   extends: .lint_macos_gitlab

tasks/libs/common/check_tools_version.py

@@ -3,7 +3,10 @@
 import json
 import sys
 
-from invoke import Context
+from invoke import Context, Exit
+
+from tasks.libs.common.color import Color, color_message
+from tasks.libs.common.utils import gitlab_section
 
 # VPATH as in version path
 GO_VPATH = ".go-version"
@@ -45,27 +48,41 @@ def custom_golangci_v(v: str) -> str:
     return f"{v}-custom-gcl"
 
 
-def current_golangci_lint_v(ctx: Context) -> str:
+def current_golangci_lint_v(ctx: Context, debug: bool = False) -> str:
     """
     Returns the current user golangci-lint version by running golangci-lint --version
     """
-    cmd = "golangci-lint --version"
-    return ctx.run(cmd, hide=True).stdout.split(' ')[3]
+    debug_flag = "--debug" if debug else ""
+    cmd = f"golangci-lint version {debug_flag}"
+    version_output = ctx.run(cmd, hide=True).stdout
+    return version_output if debug else version_output.split(' ')[3]
 
 
-def check_tools_version(ctx: Context, tools_list: list[str]) -> bool:
+def check_tools_version(ctx: Context, tools_list: list[str], debug: bool = False) -> bool:
     """
     Check that each installed tool in tools_list is the version expected for the repo.
     """
-    is_expected_versions = True
+    should_exit = False
     tools_versions = {
-        'go': {'current_v': current_go_v(ctx), 'expected_v': expected_go_repo_v()},
+        'go': {
+            'current_v': current_go_v(ctx),
+            'expected_v': expected_go_repo_v(),
+            'debug': '' if not debug else current_go_v(ctx),
+            'exit_on_error': False,
+            'error_msg': "Warning: If you have linter errors it might be due to version mismatches.",
+        },
         'golangci-lint': {
             'current_v': current_golangci_lint_v(ctx),
             'expected_v': custom_golangci_v(expected_golangci_lint_repo_v(ctx)),
+            'debug': '' if not debug else current_golangci_lint_v(ctx, debug=debug),
+            'exit_on_error': True,
+            'error_msg': "Error: The golanci-lint version you are using is not the correct one. Please run inv -e setup to install the correct version.",
         },
     }
     for tool in tools_list:
+        if debug:
+            with gitlab_section(f"{tool} debug info", collapsed=True):
+                print(tools_versions[tool]['debug'])
         if tool not in tools_versions:
             print(
                 f"Warning: Couldn't check '{tool}' expected version. Supported tools: {list(tools_versions.keys())}",
@@ -74,9 +91,14 @@ def check_tools_version(ctx: Context, tools_list: list[str]) -> bool:
         else:
             current_v, expected_v = tools_versions[tool]['current_v'], tools_versions[tool]['expected_v']
             if current_v != expected_v:
-                is_expected_versions = False
                 print(
-                    f"Warning: Expecting {tool} '{expected_v}' but you have {tool} '{current_v}'. Please fix this as you might encounter issues using the tooling.",
+                    color_message(
+                        f"Expecting {tool} '{expected_v}' but you have {tool} '{current_v}'. Please run inv -e install-tools to fix this as you might encounter issues using the tooling.",
+                        Color.RED,
+                    ),
                     file=sys.stderr,
                 )
-    return is_expected_versions
+                should_exit = should_exit or tools_versions[tool]['exit_on_error']
+    if should_exit:
+        raise Exit(code=1)
+    return True

tasks/linter.py

@@ -147,6 +147,7 @@ def go(
     only_modified_packages=False,
     verbose=False,
     run_on=None,  # noqa: U100, F841. Used by the run_on_devcontainer decorator
+    debug=False,
 ):
     """
     Run go linters on the given module and targets.
@@ -160,22 +161,13 @@ def go(
 
     --timeout is the number of minutes after which the linter should time out.
     --headless-mode allows you to output the result in a single json file.
+    --debug prints the go version and the golangci-lint debug information to help debugging lint discrepancies between versions.
 
     Example invokation:
         inv linter.go --targets=./pkg/collector/check,./pkg/aggregator
         inv linter.go --module=.
     """
-    if not check_tools_version(ctx, ['golangci-lint']):
-        print(
-            color_message(
-                "Error: The golanci-lint version you are using is not the correct one. Please run inv -e install-tools to install the correct version.",
-                "red",
-            )
-        )
-        raise Exit(code=1)
-
-    if not check_tools_version(ctx, ['go']):
-        print("Warning: If you have linter errors it might be due to version mismatches.", file=sys.stderr)
+    check_tools_version(ctx, ['golangci-lint', 'go'], debug=debug)
 
     modules, flavor = process_input_args(
         ctx,

tasks/winbuildscripts/lint.ps1

@@ -21,7 +21,7 @@ if($err -ne 0){
 
 & inv -e install-tools
 
-& inv -e linter.go
+& inv -e linter.go --debug
 $err = $LASTEXITCODE
 Write-Host Go linter result is $err
 if($err -ne 0){