Started to add stencil support to custom passes

com.unity.render-pipelines.high-definition/Editor/RenderPipeline/CustomPass/DrawRenderersCustomPassDrawer.cs

@@ -47,6 +47,16 @@ private class Styles
             public static GUIContent depthWrite = new GUIContent("Write Depth", "Choose to write depth to the screen.");
             public static GUIContent depthCompareFunction = new GUIContent("Depth Test", "Choose a new test setting for the depth.");
 
+            //Stencil Settings
+            public static GUIContent overrideStencil = new GUIContent("Override Stencil", "Override stencil state of the objects rendered.");
+            public static GUIContent stencilReferenceValue = new GUIContent("Reference", ".");
+            public static GUIContent stencilWriteMask = new GUIContent("Write Mask", ".");
+            public static GUIContent stencilReadMask = new GUIContent("Read Mask", ".");
+            public static GUIContent stencilCompareFunction = new GUIContent("Compare Function", ".");
+            public static GUIContent stencilPassOperation = new GUIContent("Pass", ".");
+            public static GUIContent stencilFailOperation = new GUIContent("Fail", ".");
+            public static GUIContent stencilDepthFailOperation = new GUIContent("Depth Fail", ".");
+
             //Camera Settings
             public static GUIContent overrideCamera = new GUIContent("Camera", "Override camera projections.");
             public static GUIContent cameraFOV = new GUIContent("Field Of View", "Field Of View to render this pass in.");
@@ -85,6 +95,15 @@ private class Styles
         SerializedProperty m_DepthCompareFunction;
         SerializedProperty m_DepthWrite;
 
+        // Override stencil state
+        SerializedProperty m_OverrideStencilState;
+        SerializedProperty m_StencilReferenceValue;
+        SerializedProperty m_StencilWriteMask;
+        SerializedProperty m_StencilReadMask;
+        SerializedProperty m_StencilPassOperation;
+        SerializedProperty m_StencilFailOperation;
+        SerializedProperty m_StencilDepthFailOperation;
+
         ReorderableList m_ShaderPassesList;
 
         CustomPassVolume m_Volume;
@@ -268,6 +287,7 @@ void DoMaterialOverride(ref Rect rect)
                 rect.y += Styles.defaultLineSpace;
             }
 
+            // Depth properties
             EditorGUI.BeginProperty(rect, Styles.overrideDepth, m_OverrideDepthState);
             {
                 if (customDepthIsNone)
@@ -291,6 +311,52 @@ void DoMaterialOverride(ref Rect rect)
                 EditorGUI.PropertyField(rect, m_DepthWrite, Styles.depthWrite);
                 EditorGUI.indentLevel--;
             }
+
+            // Stencil properties
+            EditorGUI.BeginProperty(rect, Styles.overrideStencil, m_OverrideStencilState);
+            {
+                if (customDepthIsNone)
+                {
+                    using (new EditorGUI.DisabledScope(true))
+                        EditorGUI.Toggle(rect, Styles.overrideStencil, false);
+                }
+                else
+                {
+                    EditorGUI.PropertyField(rect, m_OverrideStencilState, Styles.overrideStencil);
+                }
+            }
+            EditorGUI.EndProperty();
+
+            if (m_OverrideStencilState.boolValue && !customDepthIsNone)
+            {
+                EditorGUI.indentLevel++;
+                rect.y += Styles.defaultLineSpace;
+                EditorGUI.PropertyField(rect, m_StencilReferenceValue, Styles.stencilReferenceValue);
+                rect.y += Styles.defaultLineSpace;
+                EditorGUI.PropertyField(rect, m_StencilReadMask, Styles.stencilReadMask);
+                rect.y += Styles.defaultLineSpace;
+                EditorGUI.PropertyField(rect, m_StencilReadMask, Styles.stencilReadMask);
+                rect.y += Styles.defaultLineSpace;
+                EditorGUI.PropertyField(rect, m_StencilReadMask, Styles.stencilReadMask);
+                rect.y += Styles.defaultLineSpace;
+                EditorGUI.PropertyField(rect, m_StencilReadMask, Styles.stencilReadMask);
+                rect.y += Styles.defaultLineSpace;
+                EditorGUI.PropertyField(rect, m_StencilReadMask, Styles.stencilReadMask);
+                rect.y += Styles.defaultLineSpace;
+                EditorGUI.PropertyField(rect, m_StencilReadMask, Styles.stencilReadMask);
+                rect.y += Styles.defaultLineSpace;
+                EditorGUI.PropertyField(rect, m_StencilReadMask, Styles.stencilReadMask);
+
+                // SerializedProperty m_StencilWriteMask;
+                // SerializedProperty m_StencilReadMask;
+                // SerializedProperty m_StencilPassOperation;
+                // SerializedProperty m_StencilFailOperation;
+                // SerializedProperty m_StencilDepthFailOperation;
+
+
+                EditorGUI.indentLevel--;
+
+            }
         }
 
         void DoShaderPassesList(ref Rect rect)
@@ -349,6 +415,7 @@ protected override float GetPassHeight(SerializedProperty customPass)
                 if (showMaterialOverride)
                     height += Styles.defaultLineSpace * m_MaterialLines;
                 height += Styles.defaultLineSpace * (m_OverrideDepthState.boolValue && !customDepthIsNone ? 3 : 1);
+                height += Styles.defaultLineSpace * (m_OverrideStencilState.boolValue && !customDepthIsNone ? 6 : 1);
                 var mat = m_OverrideMaterial.objectReferenceValue as Material;
 
 #if SHOW_PASS_NAMES

com.unity.render-pipelines.high-definition/Editor/RenderPipeline/CustomPass/FullScreenCustomPassDrawer.cs

@@ -25,6 +25,7 @@ private class Styles
             public static GUIContent fetchColorBuffer = new GUIContent("Fetch Color Buffer", "Tick this if your effect sample/fetch the camera color buffer");
 
             public readonly static string writeAndFetchColorBufferWarning = "Fetching and Writing to the camera color buffer at the same time is not supported on most platforms.";
+            public readonly static string stencilWriteOverReservedBits = "The Stencil Write Mask of your material overwrites the bits reserved by HDRP. To avoid rendering errors, set the Write Mask to " + (int)(UserStencilUsage.UserBit0 | UserStencilUsage.UserBit1);
         }
 
         // Fullscreen pass
@@ -72,16 +73,56 @@ protected override void DoPassGUI(SerializedProperty customPass, Rect rect)
                             m_MaterialPassName.stringValue = mat.GetPassName(index);
                     }
                     EditorGUI.EndProperty();
+                    rect.y += Styles.defaultLineSpace;
+
+                    if (DoesWriteMaskContainsReservedBits(mat))
+                    {
+                        Rect helpBoxRect = rect;
+                        helpBoxRect.height = Styles.helpBoxHeight;
+                        EditorGUI.HelpBox(helpBoxRect, Styles.stencilWriteOverReservedBits, MessageType.Warning);
+                        rect.y += Styles.helpBoxHeight;
+                    }
+                    // TODO: get the material stencil property and show a warning if writing to HDRP bits
                 }
             }
         }
 
+        bool DoesWriteMaskContainsReservedBits(Material material)
+        {
+            int writeMask = GetStencilWriteMask(material);
+            return ((writeMask & (int)~(UserStencilUsage.UserBit0 | UserStencilUsage.UserBit1)) != 0);
+        }
+
+        int GetStencilWriteMask(Material material)
+        {
+            if (material.shader == null)
+                return 0;
+
+            var serializedShader = new SerializedObject(material.shader);
+            var parsed = serializedShader.FindProperty("m_ParsedForm");
+            var subShaders = parsed.FindPropertyRelative("m_SubShaders");
+            var subShader = subShaders.GetArrayElementAtIndex(0);
+            var passes = subShader.FindPropertyRelative("m_Passes");
+            var pass = passes.GetArrayElementAtIndex(0);
+            var state = pass.FindPropertyRelative("m_State");
+            var writeMask = state.FindPropertyRelative("stencilWriteMask");
+            var writeMaskFloatValue = writeMask.FindPropertyRelative("val");
+            var writeMaskPropertyName = writeMask.FindPropertyRelative("name");
+
+            if (material.HasProperty(writeMaskPropertyName.stringValue))
+                return (int)material.GetFloat(writeMaskPropertyName.stringValue);
+            else
+                return (int)writeMaskFloatValue.floatValue;
+        }
+
         protected override float GetPassHeight(SerializedProperty customPass)
         {
             int lineCount = (m_FullScreenPassMaterial.objectReferenceValue is Material ? 3 : 2);
             int height = (int)(Styles.defaultLineSpace * lineCount);
 
             height += (m_FetchColorBuffer.boolValue && targetColorBuffer == CustomPass.TargetBuffer.Camera) ? (int)Styles.helpBoxHeight : 0;
+            if (m_FullScreenPassMaterial.objectReferenceValue is Material mat)
+                height += (DoesWriteMaskContainsReservedBits(mat)) ? (int)Styles.helpBoxHeight : 0;
 
             return height;
         }

com.unity.render-pipelines.high-definition/Runtime/RenderPipeline/HDStencilUsage.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections;
 using System.Collections.Generic;
 using UnityEngine;
@@ -52,6 +53,7 @@ internal enum StencilUsage
     ///         Comp Always
     ///         Pass Replace
     /// </summary>
+    [Flags]
     public enum UserStencilUsage
     {
         /// <summary>User stencil bit 0.</summary>

com.unity.render-pipelines.high-definition/Runtime/RenderPipeline/RenderPass/DrawRenderersCustomPass.cs

@@ -66,6 +66,46 @@ public enum ShaderPass
         /// </summary>
         public bool depthWrite = true;
 
+        /// <summary>
+        /// Override the stencil state of the objects.
+        /// </summary>
+        public bool overrideStencil = false;
+
+        /// <summary>
+        /// Stencil reference value.
+        /// </summary>
+        public int stencilReferenceValue;
+
+        /// <summary>
+        /// Write mask of the stencil.
+        /// </summary>
+        public int stencilWriteMask;
+
+        /// <summary>
+        /// Read mask of the stencil
+        /// </summary>
+        public int stencilReadMask;
+
+        /// <summary>
+        /// Comparison operation between the stencil buffer and the reference value.
+        /// </summary>
+        public CompareFunction stencilCompareFunction;
+
+        /// <summary>
+        /// Operation to execute if the stencil test passes.
+        /// </summary>
+        public StencilOp stencilPassOperation;
+
+        /// <summary>
+        /// Operation to execute if the stencil test fails.
+        /// </summary>
+        public StencilOp stencilFailOperation;
+
+        /// <summary>
+        /// Operation to execute if the depth test fails.
+        /// </summary>
+        public StencilOp stencilDepthFailOperation;
+
         /// <summary>
         /// Set the shader pass to use when the override material is null
         /// </summary>