Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Revert "Add XUnitLogChecker to log libraries dumps" #94807

Merged
merged 1 commit into from
Nov 15, 2023
Merged

Revert "Add XUnitLogChecker to log libraries dumps" #94807

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions Directory.Build.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@
<TestExclusionListTasksAssemblyPath>$([MSBuild]::NormalizePath('$(TestExclusionListTasksDir)', 'TestExclusionListTasks.dll'))</TestExclusionListTasksAssemblyPath>
<CoreCLRToolPath>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(RuntimeConfiguration)'))</CoreCLRToolPath>
<ILAsmToolPath Condition="'$(DotNetBuildFromSource)' == 'true' or '$(BuildArchitecture)' == 's390x' or '$(BuildArchitecture)' == 'ppc64le'">$(CoreCLRToolPath)</ILAsmToolPath>

<WasmtimeDir Condition="'$(WasmtimeDir)' == '' and '$(WASMTIME_PATH)' != '' and Exists($(WASMTIME_PATH))">$(WASMTIME_PATH)</WasmtimeDir>
<WasmtimeDir Condition="'$(WasmtimeDir)' == ''">$([MSBuild]::NormalizeDirectory($(ArtifactsObjDir), 'wasmtime'))</WasmtimeDir>
<InstallWasmtimeForTests Condition="'$(InstallWasmtimeForTests)' == '' and !Exists($(WasmtimeDir))">true</InstallWasmtimeForTests>
Expand Down Expand Up @@ -331,11 +332,6 @@

<!-- this property is used by the SDK to pull in mono-based runtime packs -->
<UseMonoRuntime Condition="'$(UseMonoRuntime)' == '' and '$(RuntimeFlavor)' == 'Mono'">true</UseMonoRuntime>

<!-- For enabling the use of XUnitLogChecker in coreclr and libraries test runs. -->
<IsXUnitLogCheckerSupported Condition="'$(IsXUnitLogCheckerSupported)' == ''">false</IsXUnitLogCheckerSupported>
<IsXUnitLogCheckerSupported Condition="'$(RuntimeFlavor)' == 'CoreCLR' and '$(TestNativeAot)' != 'true' and '$(TestRunNamePrefixSuffix)' != 'NativeAOT_Release' and '$(TargetOS)' != 'browser' and '$(TargetOS)' != 'wasi' and '$(TargetOS)' != 'ios' and '$(TargetOS)' != 'iossimulator' and '$(TargetOS)' != 'tvos' and '$(TargetOS)' != 'tvossimulator' and '$(TargetOS)' != 'maccatalyst' and '$(TargetOS)' != 'android'">true</IsXUnitLogCheckerSupported>
<XUnitLogCheckerLibrariesOutDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'XUnitLogChecker'))</XUnitLogCheckerLibrariesOutDir>
</PropertyGroup>

<!-- Packaging -->
Expand Down
1 change: 0 additions & 1 deletion eng/pipelines/libraries/prepare-for-bin-publish.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ steps:
ref/**
runtime/**
testhost/**
XUnitLogChecker/**

- task: CopyFiles@2
displayName: Prepare bin folders to publish
Expand Down
35 changes: 0 additions & 35 deletions eng/testing/RunnerTemplate.cmd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,41 +92,6 @@ if %_exit_code%==1 (
)
)
)

if NOT "%__IsXUnitLogCheckerSupported%"=="1" (
echo XUnitLogChecker not supported for this test case. Skipping.
GOTO SKIP_XUNITLOGCHECKER
)

echo ----- start =============== XUnitLogChecker Output =====================================================

set DOTNET_EXE=%RUNTIME_PATH%\dotnet.exe
set XUNITLOGCHECKER_DLL=%HELIX_CORRELATION_PAYLOAD%\XUnitLogChecker.dll
set XUNITLOGCHECKER_COMMAND=%DOTNET_EXE% --roll-forward Major %XUNITLOGCHECKER_DLL% --dumps-path %HELIX_DUMP_FOLDER%
set XUNITLOGCHECKER_EXIT_CODE=1

if NOT EXIST %DOTNET_EXE% (
echo dotnet.exe does not exist in the expected location: %DOTNET_EXE%
GOTO XUNITLOGCHECKER_END
) else if NOT EXIST %XUNITLOGCHECKER_DLL% (
echo XUnitLogChecker.dll does not exist in the expected location: %XUNITLOGCHECKER_DLL%
GOTO XUNITLOGCHECKER_END
)

echo %XUNITLOGCHECKER_COMMAND%
%XUNITLOGCHECKER_COMMAND%
set XUNITLOGCHECKER_EXIT_CODE=%ERRORLEVEL%

:XUNITLOGCHECKER_END

if %XUNITLOGCHECKER_EXIT_CODE% NEQ 0 (
set _exit_code=%XUNITLOGCHECKER_EXIT_CODE%
)

echo ----- end =============== XUnitLogChecker Output - exit code %XUNITLOGCHECKER_EXIT_CODE% ===============

:SKIP_XUNITLOGCHECKER

exit /b %_exit_code%
:: ========================= END Test Execution =================================

Expand Down
178 changes: 100 additions & 78 deletions eng/testing/RunnerTemplate.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,60 +54,80 @@ exitcode_list[131]="SIGQUIT Ctrl-\ occurred. Core dumped."
exitcode_list[132]="SIGILL Illegal Instruction. Core dumped. Likely codegen issue."
exitcode_list[133]="SIGTRAP Breakpoint hit. Core dumped."
exitcode_list[134]="SIGABRT Abort. Managed or native assert, or runtime check such as heap corruption, caused call to abort(). Core dumped."
exitcode_list[135]="IGBUS Unaligned memory access. Core dumped."
exitcode_list[135]="IGBUS Unaligned memory access. Core dumped."
exitcode_list[136]="SIGFPE Bad floating point arguments. Core dumped."
exitcode_list[137]="SIGKILL Killed eg by kill"
exitcode_list[139]="SIGSEGV Illegal memory access. Deref invalid pointer, overrunning buffer, stack overflow etc. Core dumped."
exitcode_list[143]="SIGTERM Terminated. Usually before SIGKILL."
exitcode_list[159]="SIGSYS Bad System Call."

function move_core_file_to_temp_location {
function print_info_from_core_file_using_lldb {
local core_file_name=$1
local executable_name=$2
local plugin_path_name="$RUNTIME_PATH/shared/Microsoft.NETCore.App/9.9.9/libsosplugin.so"

# Append the dmp extension to ensure XUnitLogChecker finds it
local new_location=$HELIX_DUMP_FOLDER/$core_file_name.dmp
# check for existence of lldb on the path
hash lldb 2>/dev/null || { echo >&2 "lldb was not found. Unable to print core file."; return; }

echo "Copying dump file '$core_file_name' to '$new_location'"
cp $core_file_name $new_location
# pe, clrstack, and dumpasync are defined in libsosplugin.so
if [ ! -f $plugin_path_name ]; then
echo $plugin_path_name cannot be found.
return
fi

# Delete the old one
rm $core_file_name
echo ----- start =============== lldb Output =====================================================
echo Printing managed exceptions, managed call stacks, and async state machines.
lldb -O "settings set target.exec-search-paths $RUNTIME_PATH" -o "plugin load $plugin_path_name" -o "clrthreads -managedexception" -o "pe -nested" -o "clrstack -all -a -f" -o "dumpasync -fields -stacks -roots" -o "quit" --core $core_file_name $executable_name
echo ----- end =============== lldb Output =======================================================
}

xunitlogchecker_exit_code=0
function invoke_xunitlogchecker {
local dump_folder=$1

total_dumps=$(find $dump_folder -name "*.dmp" | wc -l)

if [[ $total_dumps > 0 ]]; then
echo "Total dumps found in $dump_folder: $total_dumps"
xunitlogchecker_file_name="$HELIX_CORRELATION_PAYLOAD/XUnitLogChecker.dll"
dotnet_file_name="$RUNTIME_PATH/dotnet"

if [[ ! -f $dotnet_file_name ]]; then
echo "'$dotnet_file_name' was not found. Unable to run XUnitLogChecker."
xunitlogchecker_exit_code=1
elif [[ ! -f $xunitlogchecker_file_name ]]; then
echo "'$xunitlogchecker_file_name' was not found. Unable to print dump file contents."
xunitlogchecker_exit_code=2
elif [[ ! -d $dump_folder ]]; then
echo "The dump directory '$dump_folder' does not exist."
else
echo "Executing XUnitLogChecker in $dump_folder..."
cmd="$dotnet_file_name --roll-forward Major $xunitlogchecker_file_name --dumps-path $dump_folder"
echo "$cmd"
$cmd
xunitlogchecker_exit_code=$?
fi
else
echo "No dumps found in $dump_folder."
function print_info_from_core_file_using_gdb {
local core_file_name=$1
local executable_name=$2

# Check for the existence of GDB on the path
hash gdb 2>/dev/null || { echo >&2 "GDB was not found. Unable to print core file."; return; }

echo ----- start =============== GDB Output =====================================================
# Open the dump in GDB and print the stack from each thread. We can add more
# commands here if desired.
echo printing native stack.
gdb --batch -ex "thread apply all bt full" -ex "quit" $executable_name $core_file_name
echo ----- end =============== GDB Output =======================================================
}

function print_info_from_core_file {
local core_file_name=$1
local executable_name=$RUNTIME_PATH/$2

if ! [ -e $executable_name ]; then
echo "Unable to find executable $executable_name"
return
elif ! [ -e $core_file_name ]; then
echo "Unable to find core file $core_file_name"
return
fi
echo "Printing info from core file $core_file_name"
print_info_from_core_file_using_gdb $core_file_name $executable_name
print_info_from_core_file_using_lldb $core_file_name $executable_name
}

function copy_core_file_to_temp_location {
local core_file_name=$1

local storage_location="/tmp/coredumps"

# Create the directory (this shouldn't fail even if it already exists).
mkdir -p $storage_location

local new_location=$storage_location/core.$RANDOM

echo "Copying core file $core_file_name to $new_location in case you need it."
cp $core_file_name $new_location
}

# ========================= BEGIN Core File Setup ============================
system_name="$(uname -s)"
if [[ $system_name == "Darwin" ]]; then
if [[ "$(uname -s)" == "Darwin" ]]; then
# On OS X, we will enable core dump generation only if there are no core
# files already in /cores/ at this point. This is being done to prevent
# inadvertently flooding the CI machines with dumps.
Expand All @@ -120,11 +140,21 @@ if [[ $system_name == "Darwin" ]]; then
# https://github.com/dotnet/core-eng/issues/15597
ulimit -c 0
fi
fi

export DOTNET_DbgEnableMiniDump=1
export DOTNET_EnableCrashReport=1
export DOTNET_DbgMiniDumpName=$HELIX_DUMP_FOLDER/coredump.%d.dmp
elif [[ "$(uname -s)" == "Linux" ]]; then
# On Linux, we'll enable core file generation unconditionally, and if a dump
# is generated, we will print some useful information from it and delete the
# dump immediately.

if [ -e /proc/self/coredump_filter ]; then
# Include memory in private and shared file-backed mappings in the dump.
# This ensures that we can see disassembly from our shared libraries when
# inspecting the contents of the dump. See 'man core' for details.
echo -n 0x3F > /proc/self/coredump_filter
fi

ulimit -c unlimited
fi
# ========================= END Core File Setup ==============================

# ========================= BEGIN support for SuperPMI collection ==============================
Expand All @@ -141,7 +171,7 @@ if [ ! -z $spmi_enable_collection ]; then
fi
mkdir -p $spmi_collect_dir
export spmi_file_extension=so
if [[ $system_name == "Darwin" ]]; then
if [[ "$(uname -s)" == "Darwin" ]]; then
export spmi_file_extension=dylib
fi
export SuperPMIShimLogPath=$spmi_collect_dir
Expand Down Expand Up @@ -190,59 +220,51 @@ if [[ $test_exitcode -ne 0 ]]; then
echo ulimit -c value: $(ulimit -c)
fi

if [[ $system_name == "Linux" && $test_exitcode -ne 0 ]]; then
if [[ "$(uname -s)" == "Linux" && $test_exitcode -ne 0 ]]; then
if [ -n "$HELIX_WORKITEM_PAYLOAD" ]; then

# For abrupt failures, in Helix, dump some of the kernel log, in case there is a hint
if [[ $test_exitcode -ne 1 ]]; then
dmesg | tail -50
fi

have_sleep=$(which sleep)
if [ -x "$have_sleep" ]; then
echo Waiting a few seconds for any dump to be written..
sleep 10s
fi
fi

echo cat /proc/sys/kernel/core_pattern: $(cat /proc/sys/kernel/core_pattern)
echo cat /proc/sys/kernel/core_uses_pid: $(cat /proc/sys/kernel/core_uses_pid)
echo cat /proc/sys/kernel/coredump_filter: $(cat /proc/sys/kernel/coredump_filter)

echo Looking around for any Linux dump..

# Depending on distro/configuration, the core files may either be named "core"
# or "core.<PID>" by default. We read /proc/sys/kernel/core_uses_pid to
# determine which it is.
core_name_uses_pid=0
if [[ -e /proc/sys/kernel/core_uses_pid && "1" == $(cat /proc/sys/kernel/core_uses_pid) ]]; then
core_name_uses_pid=1
fi

# The osx dumps are too large to egress the machine
echo Looking around for any Linux dumps...

if [[ "$core_name_uses_pid" == "1" ]]; then
# We don't know what the PID of the process was, so let's look at all core
# files whose name matches core.NUMBER
echo "Looking for files matching core.* ..."
for f in $(find . -name "core.*"); do
[[ $f =~ core.[0-9]+ ]] && move_core_file_to_temp_location "$f"
echo Looking for files matching core.* ...
for f in core.*; do
[[ $f =~ core.[0-9]+ ]] && print_info_from_core_file "$f" "dotnet" && copy_core_file_to_temp_location "$f" && rm "$f"
done
elif [ -f core ]; then
echo found a dump named core in $EXECUTION_DIR !
print_info_from_core_file "core" "dotnet"
copy_core_file_to_temp_location "core"
rm "core"
else
echo ... found no dump in $PWD
fi

if [ -f core ]; then
move_core_file_to_temp_location "core"
fi
fi

if [ -n "$HELIX_WORKITEM_PAYLOAD" ]; then
# For abrupt failures, in Helix, dump some of the kernel log, in case there is a hint
if [[ $test_exitcode -ne 1 ]]; then
dmesg | tail -50
fi

fi

if [[ -z "$__IsXUnitLogCheckerSupported" ]]; then
echo "The '__IsXUnitLogCheckerSupported' env var is not set."
elif [[ "$__IsXUnitLogCheckerSupported" != "1" ]]; then
echo "XUnitLogChecker not supported for this test case. Skipping."
else
echo ----- start =============== XUnitLogChecker Output =====================================================

invoke_xunitlogchecker "$HELIX_DUMP_FOLDER"

if [[ $xunitlogchecker_exit_code -ne 0 ]]; then
test_exitcode=$xunitlogchecker_exit_code
fi
echo ----- end =============== XUnitLogChecker Output - exit code $xunitlogchecker_exit_code ===========================
fi

popd >/dev/null
# ======================== END Core File Inspection ==========================
# The helix work item should not exit with non-zero if tests ran and produced results
Expand Down
Loading