Search Unity

  1. Megacity Metro Demo now available. Download now.
    Dismiss Notice
  2. Unity support for visionOS is now available. Learn more in our blog post.
    Dismiss Notice

Use a custom vertex shader with the standard shader?

Discussion in 'Shaders' started by Sebastianp-ta, Jan 31, 2016.

  1. Sebastianp-ta

    Sebastianp-ta

    Joined:
    Jul 22, 2015
    Posts:
    12
    Hello everyone!

    We are currently working on a small project that does a lot of geometry transformation in the vertex shader and we are struggling on getting things to look nice. The transformation part works great but we only have flat textures with no normals and no shadows.
    Is there a way to extend the standard shader with a custom vertex shader? (I hope there is an easy way as i am no shader expert and i did not write the vertex shader that is in use right now :) )

    Code (CSharp):
    1. Shader "Unlit/HDeformation"
    2. {
    3.     Properties
    4.     {
    5.         _MainTex ("Texture", 2D) = "white" {}
    6.         p0 ("p0", Vector) = (0.0, 0.0, 0.0, 0.0)
    7.         p1 ("p1", Vector) = (0.0, 0.0, 0.33, 0.0)
    8.         p2 ("p2", Vector) = (0.0, 0.0, 0.66, 0.0)
    9.         p3 ("p3", Vector) = (0.0, 0.0, 1.0, 0.0)
    10.  
    11.     }
    12.  
    13.     SubShader
    14.     {
    15.         Tags { "RenderType"="Opaque" "PerformanceChecks"="False" }
    16.         LOD 300
    17.  
    18.         Pass
    19.         {
    20.             CGPROGRAM
    21.             #pragma vertex vert
    22.             #pragma fragment frag
    23.             // make fog work
    24.             #pragma multi_compile_fog
    25.            
    26.             #include "UnityCG.cginc"
    27.  
    28.             struct appdata
    29.             {
    30.                 float4 vertex : POSITION;
    31.                 float2 uv : TEXCOORD0;
    32.             };
    33.  
    34.             struct v2f
    35.             {
    36.                 float2 uv : TEXCOORD0;
    37.                 UNITY_FOG_COORDS(1)
    38.                 float4 vertex : SV_POSITION;
    39.             };
    40.  
    41.             sampler2D _MainTex;
    42.             float4 _MainTex_ST;
    43.            
    44.  
    45.             float3 p0;
    46.             float3 p1;
    47.             float3 p2;  
    48.             float3 p3;
    49.  
    50.             v2f vert (appdata v)
    51.             {
    52.                 v2f o;
    53.                 float t = v.vertex.z + 0.5;
    54.  
    55.                 float3 p01 = p0.xyz * (1.0 - t) + p1.xyz * t;
    56.                 float3 p12 = p1.xyz * (1.0 - t) + p2.xyz * t;
    57.                 float3 p23 = p2.xyz * (1.0 - t) + p3.xyz * t;
    58.                 float3 p012 = p01 * (1.0 - t) + p12 * t;
    59.                 float3 p123 = p12 * (1.0 - t) + p23 * t;
    60.                 float3 p0123 = p012 * (1.0 - t) + p123 * t;
    61.  
    62.                 float3 np01 = p0.xyz * (1.0 - (t+0.01)) + p1.xyz * (t+0.01);
    63.                 float3 np12 = p1.xyz * (1.0 - (t+0.01)) + p2.xyz * (t+0.01);
    64.                 float3 np23 = p2.xyz * (1.0 - (t+0.01)) + p3.xyz * (t+0.01);
    65.                 float3 np012 = p01 * (1.0 - (t+0.01)) + p12 * (t+0.01);
    66.                 float3 np123 = p12 * (1.0 - (t+0.01)) + p23 * (t+0.01);
    67.                 float3 np0123 = p012 * (1.0 - (t+0.01)) + p123 * (t+0.01);
    68.  
    69.                 float3 dir = normalize(np0123 - p0123);
    70.                 float3 binorm = normalize(cross(dir, float3(0,1,0)));
    71.                 float3 norm = normalize(cross(binorm, dir));
    72.                 float4x4 curve = float4x4(float4(norm, 0), float4(binorm, 0), float4(dir, 0), float4(0,0,0,0));
    73.  
    74.                 float4 temp = v.vertex + float4(0, 0, 0.5 - t, 0);
    75.                 temp = mul(transpose(curve), temp);
    76.                 temp += float4(p0123,1);
    77.                 temp = mul(UNITY_MATRIX_V, temp);
    78.                
    79.                 o.vertex = mul(UNITY_MATRIX_P, temp);
    80.                 o.uv = TRANSFORM_TEX(v.uv, _MainTex);
    81.                 UNITY_TRANSFER_FOG(o,o.vertex);
    82.                 return o;
    83.             }
    84.  
    85.        
    86.             fixed4 frag (v2f i) : SV_Target
    87.             {
    88.                 // sample the texture
    89.                 fixed4 col = tex2D(_MainTex, i.uv);
    90.                 // apply fog
    91.                 UNITY_APPLY_FOG(i.fogCoord, col);              
    92.                 return col;
    93.             }
    94.            
    95.             ENDCG
    96.         }
    97.     }
    98. }
     
  2. smd863

    smd863

    Joined:
    Jan 26, 2014
    Posts:
    292
    Easiest thing to do is to use a surface shader with the "vertex" pragma to define your own vertex function. Check out the surface shader examples.

    It should look pretty similar to what you have now, but instead of your "frag" function you use a "surface" function that fills in the surface data structure which will be passed into Unity's internal shading functions. You can customize the behaviour with a few different "pragma" statements to select the features and shading function you want to use.
     
    Sebastianp-ta likes this.
  3. Sebastianp-ta

    Sebastianp-ta

    Joined:
    Jul 22, 2015
    Posts:
    12
    Thank you for the quick reply, i will try that out and report back on how it went :)
     
  4. Sebastianp-ta

    Sebastianp-ta

    Joined:
    Jul 22, 2015
    Posts:
    12
    Ok, it was almost straightforward, a few smaller pitfalls here and there (forward declaration of variables) but i got it to work. Well, at least it compiles and does something...

    It should warp a mesh along a bezier spline defined by the points p0 to p3. The original shader does exactly that, the surface shader, with the same calculations, does this (left: surface shader, right: original):


    Maybe there is something obviously wrong that i am missing? It looks like something gets flipped... The normals look like they are messed up and somehow it seems to be in a wrong coordinate system...

    The code i use:

    Code (CSharp):
    1. Shader "Custom/HDeformationSurface" {
    2.     Properties {
    3.         _Color ("Color", Color) = (1,1,1,1)
    4.         _MainTex ("Albedo (RGB)", 2D) = "white" {}
    5.         _Metallic ("Metallic", 2D) = "white" {}
    6.         p0 ("p0", Vector) = (0.0, 0.0, 0.0, 0.0)
    7.         p1 ("p1", Vector) = (0.0, 0.0, 0.33, 0.0)
    8.         p2 ("p2", Vector) = (0.0, 0.0, 0.66, 0.0)
    9.         p3 ("p3", Vector) = (0.0, 0.0, 1.0, 0.0)
    10.     }
    11.     SubShader {
    12.         Tags { "RenderType"="Opaque" }
    13.         LOD 200
    14.        
    15.         CGPROGRAM
    16.         // Physically based Standard lighting model, and enable shadows on all light types
    17.         #pragma surface surf Standard fullforwardshadows vertex:vert
    18.  
    19.         // Use shader model 3.0 target, to get nicer looking lighting
    20.         #pragma target 3.0
    21.  
    22.         sampler2D _MainTex;
    23.  
    24.         struct Input {
    25.             float2 uv_MainTex;
    26.         };
    27.  
    28.         sampler2D _Metallic;
    29.         fixed4 _Color;
    30.  
    31.         float3 p0;
    32.         float3 p1;
    33.         float3 p2;  
    34.         float3 p3;
    35.  
    36.         void surf (Input IN, inout SurfaceOutputStandard o) {
    37.             // Albedo comes from a texture tinted by color
    38.             fixed4 c = tex2D (_MainTex, IN.uv_MainTex) * _Color;
    39.             o.Albedo = c.rgb;
    40.             // Metallic and smoothness come from slider variables
    41.             o.Metallic = tex2D (_Metallic, IN.uv_MainTex).rgb;
    42.             o.Smoothness = tex2D (_Metallic, IN.uv_MainTex).aaa;
    43.             o.Alpha = c.a;
    44.         }
    45.  
    46.         void vert (inout appdata_full v)
    47.             {
    48.                 float t = v.vertex.z + 0.5;
    49.  
    50.                 float3 p01 = p0.xyz * (1.0 - t) + p1.xyz * t;
    51.                 float3 p12 = p1.xyz * (1.0 - t) + p2.xyz * t;
    52.                 float3 p23 = p2.xyz * (1.0 - t) + p3.xyz * t;
    53.                 float3 p012 = p01 * (1.0 - t) + p12 * t;
    54.                 float3 p123 = p12 * (1.0 - t) + p23 * t;
    55.                 float3 p0123 = p012 * (1.0 - t) + p123 * t;
    56.  
    57.                 float3 np01 = p0.xyz * (1.0 - (t+0.01)) + p1.xyz * (t+0.01);
    58.                 float3 np12 = p1.xyz * (1.0 - (t+0.01)) + p2.xyz * (t+0.01);
    59.                 float3 np23 = p2.xyz * (1.0 - (t+0.01)) + p3.xyz * (t+0.01);
    60.                 float3 np012 = p01 * (1.0 - (t+0.01)) + p12 * (t+0.01);
    61.                 float3 np123 = p12 * (1.0 - (t+0.01)) + p23 * (t+0.01);
    62.                 float3 np0123 = p012 * (1.0 - (t+0.01)) + p123 * (t+0.01);
    63.  
    64.                 float3 dir = normalize(np0123 - p0123);
    65.                 float3 binorm = normalize(cross(dir, float3(0,1,0)));
    66.                 float3 norm = normalize(cross(binorm, dir));
    67.                 float4x4 curve = float4x4(float4(norm, 0), float4(binorm, 0), float4(dir, 0), float4(0,0,0,0));
    68.  
    69.                 float4 temp = v.vertex + float4(0, 0, 0.5 - t, 0);
    70.                 temp = mul(transpose(curve), temp);
    71.                 temp += float4(p0123,1);
    72.                 temp = mul(UNITY_MATRIX_V, temp);
    73.                
    74.                 v.vertex = mul(UNITY_MATRIX_P, temp);
    75.  
    76.             }
    77.  
    78.         ENDCG
    79.     }
    80.     FallBack "Diffuse"
    81. }
    82.  
     
  5. bgolus

    bgolus

    Joined:
    Dec 7, 2012
    Posts:
    12,329
    The vert function for a surface shader is expecting the v.vertex to still be in object space. You're transforming into world space with your spline math and then applying the view and projection matrices to it, and the surface shader is then applying UNITY_MATRIX_MVP to that.

    ...
    float4 temp = v.vertex + float4(0, 0, 0.5 - t, 0);
    temp = mul(transpose(curve), temp);
    temp += float4(p0123,1);
    v.vertex = mul(_World2Object, temp);
    }
     
    Sebastianp-ta likes this.
  6. Sebastianp-ta

    Sebastianp-ta

    Joined:
    Jul 22, 2015
    Posts:
    12
    Thank you very much! Its working now... I was suspecting something like that but i could not find it in the documentation, did i miss it somewhere or is it something you just have to know?

    Anyway if you guys would be so kind to help me some more, i now have the problem that there are no shadows on the objects modified with the shader. I used the #pragma statement fullforwardshadows and was expecting it to do some standard shadowmapping, unfortunately i dont see any shadows... :/

    Edit: Testing more stuff i noticed that i dont get any depth values for my distorted geometry... Is there a way to write to the depth buffer without loosing the rest of the standard shading?
     
    Last edited: Feb 3, 2016
  7. bgolus

    bgolus

    Joined:
    Dec 7, 2012
    Posts:
    12,329
    The answer to both of those questions are kind of hidden in the documentation:
    http://docs.unity3d.com/Manual/SL-SurfaceShaders.html

    For shadows if you're modifying the vertices or using alpha test or anything that might change the shadow shape from the original mesh in the shader, you need to use "addshadow". It's just above "fullforwardshadows" in the documentation.

    For the vert surface shader they're a little less specific, but the documentation states "This function is invoked at start of generated vertex shader, and can modify or compute per-vertex data." and the surface shader example shows just directly adding the normal to the vertex position without any additional transforms.

    Other random thoughts: You're bending the geometry without also changing the normals. This is going to make lighting very funny. Instead of mul(transpose(curve), temp) use mul(temp, curve), the results will be the same but it'll be faster. Also, I might be wrong but you could also just use a float3x3 instead of a float4x4 for the matrix.
     
    Sebastianp-ta likes this.
  8. Sebastianp-ta

    Sebastianp-ta

    Joined:
    Jul 22, 2015
    Posts:
    12
    Hey, thanks a lot! I think i have everything up and running now!

    The normal conversion i am doing is just a guess but it looks ok :)

    Depth was very easy to get, just added a UNITY_TRANSFER_DEPTH(v); at the end and it works...
    I now have shadows, depth of field and nicely transformed elements! For those interested, this is the full shader code now:

    Code (CSharp):
    1. Shader "Custom/HDeformationSurface" {
    2.     Properties {
    3.         _Color ("Color", Color) = (1,1,1,1)
    4.         _MainTex ("Albedo (RGB)", 2D) = "white" {}
    5.         _Metallic ("Metallic", 2D) = "white" {}
    6.         _Normal ("Normal", 2D) = "bump" {}
    7.         p0 ("p0", Vector) = (0.0, 0.0, 0.0, 0.0)
    8.         p1 ("p1", Vector) = (0.0, 0.0, 0.33, 0.0)
    9.         p2 ("p2", Vector) = (0.0, 0.0, 0.66, 0.0)
    10.         p3 ("p3", Vector) = (0.0, 0.0, 1.0, 0.0)
    11.     }
    12.     SubShader {
    13.         Tags { "RenderType"="Opaque" }
    14.         LOD 200
    15.        
    16.         CGPROGRAM
    17.         // Physically based Standard lighting model, and enable shadows on all light types
    18.         #pragma surface surf Standard addshadow vertex:vert
    19.  
    20.         // Use shader model 3.0 target, to get nicer looking lighting
    21.         #pragma target 3.0
    22.  
    23.         sampler2D _MainTex;
    24.  
    25.         struct Input {
    26.             float2 uv_MainTex;
    27.         };
    28.  
    29.         sampler2D _Metallic;
    30.         sampler2D _Normal;
    31.         fixed4 _Color;
    32.  
    33.         float3 p0;
    34.         float3 p1;
    35.         float3 p2;  
    36.         float3 p3;
    37.  
    38.         void surf (Input IN, inout SurfaceOutputStandard o) {
    39.             // Albedo comes from a texture tinted by color
    40.             fixed4 c = tex2D (_MainTex, IN.uv_MainTex) * _Color;
    41.             o.Albedo = c.rgb;
    42.             // Metallic and smoothness come from slider variables
    43.             o.Metallic = tex2D (_Metallic, IN.uv_MainTex).rgb;
    44.             o.Normal = UnpackNormal ( tex2D (_Normal, IN.uv_MainTex) );
    45.             o.Smoothness = tex2D (_Metallic, IN.uv_MainTex).aaa;
    46.             o.Alpha = c.a;
    47.         }
    48.  
    49.         void vert (inout appdata_full v)
    50.             {
    51.                 float t = v.vertex.z + 0.5;
    52.  
    53.                 float3 p01 = p0.xyz * (1.0 - t) + p1.xyz * t;
    54.                 float3 p12 = p1.xyz * (1.0 - t) + p2.xyz * t;
    55.                 float3 p23 = p2.xyz * (1.0 - t) + p3.xyz * t;
    56.                 float3 p012 = p01 * (1.0 - t) + p12 * t;
    57.                 float3 p123 = p12 * (1.0 - t) + p23 * t;
    58.                 float3 p0123 = p012 * (1.0 - t) + p123 * t;
    59.  
    60.                 float3 np01 = p0.xyz * (1.0 - (t+0.01)) + p1.xyz * (t+0.01);
    61.                 float3 np12 = p1.xyz * (1.0 - (t+0.01)) + p2.xyz * (t+0.01);
    62.                 float3 np23 = p2.xyz * (1.0 - (t+0.01)) + p3.xyz * (t+0.01);
    63.                 float3 np012 = p01 * (1.0 - (t+0.01)) + p12 * (t+0.01);
    64.                 float3 np123 = p12 * (1.0 - (t+0.01)) + p23 * (t+0.01);
    65.                 float3 np0123 = p012 * (1.0 - (t+0.01)) + p123 * (t+0.01);
    66.  
    67.                 float3 dir = normalize(np0123 - p0123);
    68.                 float3 binorm = normalize(cross(dir, float3(0,1,0)));
    69.                 float3 norm = normalize(cross(binorm, dir));
    70.                 float4x4 curve = float4x4(float4(norm, 0), float4(binorm, 0), float4(dir, 0), float4(0,0,0,0));
    71.  
    72.                 float4 temp = v.vertex + float4(0, 0, 0.5 - t, 0);
    73.                 temp = mul(temp, curve);
    74.                 temp += float4(p0123,1);
    75.                 float4 tempNormal = mul(v.normal, curve);
    76.                
    77.                 v.vertex = mul(_World2Object, temp);
    78.                 v.normal = tempNormal;
    79.                 UNITY_TRANSFER_DEPTH(v);
    80.  
    81.             }
    82.  
    83.         ENDCG
    84.     }
    85.     FallBack "Diffuse"
    86. }
    87.  
    And this is what it produces:



    Thank you guys again for your help!
     
    daterre likes this.
  9. daterre

    daterre

    Joined:
    Jul 30, 2012
    Posts:
    41