[mono] Throw a MarshalDirectiveException when marshalling generic ins…

…tances as return types from pinvoke callbacks. (dotnet#65743)

Fixes dotnet#63962.

Co-authored-by: vargaz <vargaz@users.noreply.github.com>

src/mono/mono/metadata/marshal-ilgen.c

@@ -6321,7 +6321,7 @@ emit_managed_wrapper_ilgen (MonoMethodBuilder *mb, MonoMethodSignature *invoke_s
 	/* ret = method (...) */
 	mono_mb_emit_managed_call (mb, method, NULL);
 
-	if (MONO_TYPE_ISSTRUCT (sig->ret)) {
+	if (MONO_TYPE_ISSTRUCT (sig->ret) && sig->ret->type != MONO_TYPE_GENERICINST) {
 		MonoClass *klass = mono_class_from_mono_type_internal (sig->ret);
 		mono_class_init_internal (klass);
 		if (!(mono_class_is_explicit_layout (klass) || m_class_is_blittable (klass))) {
@@ -6365,6 +6365,10 @@ emit_managed_wrapper_ilgen (MonoMethodBuilder *mb, MonoMethodSignature *invoke_s
 		case MONO_TYPE_SZARRAY:
 			mono_emit_marshal (m, 0, sig->ret, mspecs [0], 0, NULL, MARSHAL_ACTION_MANAGED_CONV_RESULT);
 			break;
+		case MONO_TYPE_GENERICINST: {
+			mono_mb_emit_byte (mb, CEE_POP);
+			break;
+		}
 		default:
 			g_warning ("return type 0x%02x unknown", sig->ret->type);
 			g_assert_not_reached ();

src/mono/mono/metadata/marshal.c

@@ -4003,11 +4003,18 @@ mono_marshal_get_managed_wrapper (MonoMethod *method, MonoClass *delegate_klass,
 			mono_error_set_invalid_program (error, "method %s with UnmanagedCallersOnlyAttribute has types not usable when marshalling is disabled", mono_method_full_name (method, TRUE));
 			return NULL;
 		}
-		mspecs = g_new0 (MonoMarshalSpec*, invoke_sig->param_count + 1);
 	} else {
 		invoke = mono_get_delegate_invoke_internal (delegate_klass);
 		invoke_sig = mono_method_signature_internal (invoke);
-		mspecs = g_new0 (MonoMarshalSpec*, invoke_sig->param_count + 1);
+	}
+
+	if (invoke_sig->ret->type == MONO_TYPE_GENERICINST) {
+		mono_error_set_generic_error (error, "System.Runtime.InteropServices", "MarshalDirectiveException", "%s", "Cannot marshal 'return value': Non-blittable generic types cannot be marshaled.");
+		return NULL;
+	}
+
+	mspecs = g_new0 (MonoMarshalSpec*, invoke_sig->param_count + 1);
+	if (invoke) {
 		mono_method_get_marshal_info (invoke, mspecs);
 		marshalling_enabled = runtime_marshalling_enabled(m_class_get_image (delegate_klass));
 	}

src/mono/mono/tests/libtest.c

@@ -1155,6 +1155,13 @@ mono_test_marshal_icall_delegate (IcallDelegate del)
 	return strcmp (res, "ABC") == 0 ? 0 : 1;
 }
 
+typedef char* (STDCALL *NullableReturnDelegate) (void);
+LIBTEST_API void STDCALL
+mono_test_marshal_nullable_ret_delegate (NullableReturnDelegate del)
+{
+	del ();
+}
+
 LIBTEST_API int STDCALL
 mono_test_marshal_stringbuilder (char *s, int n)
 {

src/mono/mono/tests/pinvoke3.cs

@@ -197,6 +197,9 @@ public static int delegate_test_string_builder_marshalling (StringBuilder s)
 	[DllImport ("libtest", EntryPoint="mono_test_marshal_icall_delegate")]
 	public static extern int mono_test_marshal_icall_delegate (IcallDelegate del);
 
+	[DllImport ("libtest", EntryPoint="mono_test_marshal_nullable_ret_delegate")]
+	public static extern int mono_test_marshal_nullable_ret_delegate (NullableReturnDelegate del);
+
 	public delegate string IcallDelegate (IntPtr p);
 
 	public delegate int TestDelegate (int a, ref SimpleStruct ss, int b);
@@ -225,6 +228,8 @@ public static int delegate_test_string_builder_marshalling (StringBuilder s)
 
 	public delegate int VirtualDelegate (int i);
 
+	public delegate Nullable<int> NullableReturnDelegate ();
+
 	public static int Main () {
 		return TestDriver.RunTests (typeof (Tests));
 	}
@@ -1268,4 +1273,23 @@ public static int test_0_icall_delegate () {
 
 		return mono_test_marshal_icall_delegate ((IcallDelegate)Delegate.CreateDelegate (typeof (IcallDelegate), m));
 	}
+
+	private static Nullable<int> nullable_ret_cb () {
+		return 0;
+	}
+
+	public static int test_0_generic_return () {
+		try {
+			Marshal.GetFunctionPointerForDelegate<NullableReturnDelegate> (nullable_ret_cb);
+			return 1;
+		} catch (MarshalDirectiveException) {
+		}
+		try {
+			mono_test_marshal_nullable_ret_delegate (nullable_ret_cb);
+			return 2;
+		} catch (MarshalDirectiveException) {
+		}
+
+		return 0;
+	}
 }