DX11 Tessellation with vertex output, combination not working

Hi,

Im currently working on creating more advanced shaders (in my opinion), but I am not a graphics programmer by trade.
The release of Unity 4, with DX11 was timed perfectly for my project, so I am very grateful for that and trying to use it to the greatest extent possible. But I am unable to combine my vertex/surface shader with the tessellation.

The tessellation works if I remove everything from my void surf() below "//mask" and my ", out Input o" from my void vert().
The problem is that I need this for my second procedural uv channel.
The shader also works without the "tessellate:tessEdge tessphong:_Smoothness" pragmas.

Anyone an idea what I am doing wrong? The warning I am getting is:

Shader warning in 'Example/OceanShaderShort': Program 'tessvert_surf', 'vert': no matching 1 parameter function (compiling for d3d11) at line 9

and some " implicit truncation of vector type" warnings which I don't get when I turn off the tessellation.

Code (csharp):

1. Shader "Example/OceanShaderShort"
2. {
3.     Properties
4.     {
5.         _Diffuse("_Diffuse", 2D) = "white" {}
6.         _StartBV ("Vert Blending start", float) = 5
7.         _EndBV ("Vert Blending end", float) = 1
8.         _StartBM ("Mask Blending start", float) = 6
9.         _EndBM ("Mask Blending end", float) = 1
10.         _StartBW ("Wave Blending start", float) = 7
11.         _EndBW ("Wave Blending end", float) = 1
12.         _TerrainScale("_TerrainScale", float) = 1
13.  
14.         _BaseTerrain("_UVmask", 2D) = "black" {}
15.        
16.         _TerrainTypeWater("_Terrain Type Water", 2D) = "black" {}
17.         _TerrainTypeWater_N("_Terrain Type Water_N", 2D) = "black" {}
18.         _TerrainTypeWater_S("Specular", Range(0,1)) = 0.1
19.         _TerrainTypeWater_SB("Specular Start", Range(0,1)) = 0.1
20.        
21.         _EdgeLength ("Edge length", Range(2,50)) = 5
22.         _Smoothness ("Phong Strengh", Range(0,1)) = 0.5
23.     }
24.    
25.     SubShader
26.     {
27.         Tags
28.         {
29.             "Queue"="Transparent+1"
30.             "IgnoreProjector"="False"
31.             "RenderType"="Opaque"
32.         }
33.        
34.         Blend SrcAlpha OneMinusSrcAlpha
35.         Cull Back
36.         ZWrite Off
37.         ZTest LEqual
38.         ColorMask RGBA
39.         Fog{
40.     }
41.  
42.         CGPROGRAM
43.         #pragma exclude_renderers gles
44.         #pragma surface surf BlinnPhongEditor vertex:vert tessellate:tessEdge tessphong:_Smoothness
45.         #pragma target 5.0
46.         #include "Tessellation.cginc"
47.         #include "UnityCG.cginc"
48.    
49.             sampler2D _Diffuse;
50.             sampler2D _BaseTerrain;
51.            
52.             sampler2D _TerrainTypeWater;
53.             sampler2D _TerrainTypeWater_N;
54.                                    
55.             float _TerrainScale;
56.            
57.             float _TerrainTypeWater_S;
58.             float _TerrainTypeWater_SB;
59.            
60.             struct EditorSurfaceOutput {
61.                 half3 Albedo;
62.                 half3 Normal;
63.                 half3 Emission;
64.                 half3 Gloss;
65.                 half Specular;
66.                 half Alpha;
67.             };
68.            
69.             struct appdata {
70.                 float4 vertex : POSITION;
71.                 float3 normal : NORMAL;
72.                 float2 texcoord : TEXCOORD0;
73.                 float2 texcoord1 : TEXCOORD1;
74.                 float4 tangent : TANGENT;
75.             };
76.            
77.             float _Smoothness;
78.             float _EdgeLength;
79.  
80.             float4 tessEdge (appdata v0, appdata v1, appdata v2)
81.             {
82.                 return UnityEdgeLengthBasedTessCull (v0.vertex, v1.vertex, v2.vertex, _EdgeLength, 0.0);
83.             }
84.                        
85.             struct Input {
86.                 float4 meshUV;
87.                 float3 blendVal;
88.             };
89.  
90.             float _StartBV;
91.             float _EndBV;
92.             float _StartBM;
93.             float _EndBM;
94.             float _StartBW;
95.             float _EndBW;
96.            
97.             void vert (inout appdata v, out Input o)
98.             {
99.                 UNITY_INITIALIZE_OUTPUT(Input,o);
100.                                
101.                 float Pi = 3.1415926535;
102.                 float4x4 rotationMatrix = float4x4(cos(0.5*Pi), -sin(0.5*Pi), 0.0, 0.0,
103.                                             sin(0.5*Pi), cos(0.5*Pi), 0.0, 0.0,
104.                                             0.0, 0.0, 1.0, 0.0,
105.                                             0.0, 0.0, 0.0, 1.0);
106.  
107.                
108.                 float3 vertexR = mul(rotationMatrix,v.vertex.xyz) ;
109.                 float radius = distance(vertexR.xyz, mul(_Object2World, float3(0,0,0)));
110.                 float2 polarUV = float2(atan2(vertexR.y,vertexR.x),acos(vertexR.z/radius)*2);
111.                 polarUV = polarUV*float2(0.1,0.1);
112.                
113.                 o.meshUV.xy = (v.texcoord.xy);
114.                 o.meshUV.zw = (polarUV);
115.                
116.                 v.texcoord1.xy = polarUV;
117.                
118.                 float distRangeV = clamp((_StartBV-(distance(mul (_Object2World, v.vertex).xyz, _WorldSpaceCameraPos.xyz)))*(1/(_StartBV-_EndBV)),0,1);
119.                 float distRangeM = clamp((_StartBM-(distance(mul (_Object2World, v.vertex).xyz, _WorldSpaceCameraPos.xyz)))*(1/(_StartBM-_EndBM)),0,1);
120.                 float distRangeW = clamp((_StartBW-(distance(mul (_Object2World, v.vertex).xyz, _WorldSpaceCameraPos.xyz)))*(1/(_StartBW-_EndBW)),0,1);
121.                
122.                 o.blendVal.x = distRangeV;
123.                 o.blendVal.y = distRangeM;
124.                 o.blendVal.z = distRangeW;
125.             }
126.            
127.             inline half4 LightingBlinnPhongEditor_PrePass (EditorSurfaceOutput s, half4 light)
128.             {
129.                 half3 spec = light.a * s.Gloss;
130.                 half4 c;
131.                 c.rgb = (s.Albedo * light.rgb + light.rgb * spec);
132.                 c.a = s.Alpha;
133.                                
134.                 return c;
135.             }
136.  
137.             inline half4 LightingBlinnPhongEditor (EditorSurfaceOutput s, half3 lightDir, half3 viewDir, half atten)
138.             {
139.                 half3 h = normalize (lightDir + viewDir);
140.                
141.                 half diff = max (0, dot ( lightDir, s.Normal ));
142.                
143.                 float nh = max (0, dot (s.Normal, h));
144.                 float spec = pow (nh, s.Specular*128.0);
145.                
146.                 half4 res;
147.                 res.rgb = _LightColor0.rgb * diff;
148.                 res.w = spec * Luminance (_LightColor0.rgb);
149.                 res *= atten * 2.0;
150.                 res *= s.Alpha;
151.  
152.                 return LightingBlinnPhongEditor_PrePass( s, res );
153.             }
154.                
155.        
156.             void surf (Input IN, inout EditorSurfaceOutput o)
157.             {
158.                 o.Normal = float3(0.0,0.0,1.0);
159.                 o.Alpha = 1;
160.                 o.Albedo = 0.0;
161.                 o.Emission = 0.0;
162.                 o.Gloss = 0;
163.                 o.Specular = 0;
164.                
165.                 //mask
166.                 float4 BlendP0 = tex2D(_BaseTerrain,(IN.meshUV).xy);
167.                
168.                 //start diffuse
169.                 float4 WaterA  = tex2D(_TerrainTypeWater,((IN.meshUV.xy)*_TerrainScale));
170.                 float4 WaterB  = tex2D(_TerrainTypeWater,((IN.meshUV.zw)*_TerrainScale));
171.                 float4 LerpPWat = lerp(WaterA,WaterB,BlendP0.w);
172.  
173.                 //Normals
174.                 float4 TerNwatA = tex2D(_TerrainTypeWater_N,((IN.meshUV.xy)*_TerrainScale));
175.                 float3 UnpackNormalwatA = float3(UnpackNormal(TerNwatA));
176.                 float4 TerNwatB = tex2D(_TerrainTypeWater_N,((IN.meshUV.zw)*_TerrainScale));
177.                 float3 UnpackNormalwatB = float3(UnpackNormal(TerNwatB));
178.                 float3 LerpPwatN = lerp(UnpackNormalwatA,UnpackNormalwatB,BlendP0.w);
179.                                        
180.                 o.Albedo = LerpPWat.xyz;
181.                 o.Normal = normalize(LerpPwatN);
182.                 o.Alpha = IN.blendVal.x*lerp(BlendP0.y,BlendP0.x,IN.blendVal);
183.                 o.Gloss = lerp(_TerrainTypeWater_SB,_TerrainTypeWater_S,IN.blendVal.x);
184.             }
185.         ENDCG
186.     }
187.     Fallback "Diffuse"
188. }

 

I remember running into the problem of not being able to combine a custom vertex shader function with the built in surface shader style tessellation, the last time I experimented with tessellation.

I worked around it by modifying the output of a surface shader with tessellation. But I ran out of time and didn't do any further experiments.

Perhaps it makes sense to move the code that's currently in the custom vertex shader function, to a custom domain shader function. (That way, it'll be applied to all the vertices created by the tessellator, right?)

 

Thanks for your time , I just checked your water shader which is quite neat!

However I am a relative "newb" with shaders... could you expand on how you modified the output of your surface shader with tessellation?

Also you are correct that the vertex shader should also be applied on the new vertices, '
but I have never built domain shaders, nor can I find much examples for them for unity. :neutral:

 

I'm also a real amateur when it comes to tessellation. It's been a while, so I can't remember any of the details, but it basically came down to the following.
I compiled a really basic surface shader with tessellation, with #pragma debug to get an expanded version.

Code (csharp):

1. Shader "Custom/Testellate" {
2.     Properties {
3.         _MainTex ("Base (RGB)", 2D) = "white" {}
4.         _Offset ("Offset", Float) = 10
5.         _Tessellation ("Tessellation", Float) = 1
6.     }
7.     SubShader {
8.         Tags { "RenderType"="Opaque" }
9.        
10.         CGPROGRAM
11.         #pragma surface surf BlinnPhong vertex:displace tessellate:fixedTessellation nolightmap
12.         #pragma target 5.0
13.         #pragma debug
14.        
15.         sampler2D _MainTex;
16.         float _Offset;
17.         float _Tessellation;
18.  
19.        
20.        
21.         float fixedTessellation(){
22.             return _Tessellation;        
23.         }
24.        
25.         struct appdata {
26.             float4 vertex : POSITION;
27. //            float4 tangent : TANGENT;
28.             float3 normal : NORMAL;
29.             float4 texcoord : TEXCOORD0;
30. //            float4 texcoord1 : TEXCOORD1;
31.         };
32.  
33.         struct Input {
34.             float2 uv_MainTex;
35.         };
36.        
37.         void displace(inout appdata v){
38. //            UNITY_INITIALIZE_OUTPUT(Input,o);// Input Initialization required for DirectX11
39.        
40.             // Calculate mesh coordinates,
41.             // Can't use TRANSFORM_TEX
42.             // since _HeightTex_ST does not seem to be available yet at this point in the shader
43.             // And setting  _HeightTex_ST gives problems when it is used later.
44.             // Does not actually change the texture coordinates.
45.             float4 customUVTransform = float4(1,1,0,0);
46.             float2 uv = v.texcoord.xy * customUVTransform.xy + customUVTransform.zw;
47.        
48.             float localTex = tex2Dlod(_MainTex, float4(uv,0,0)).r;
49.            
50.             v.vertex.y += localTex.r * _Offset;
51. //            v.normal = calcGridNormal(_HeightTex, uv, _HeightTex_TexelSize.xy, float3(1, _Offset, 1), localTex.r);// Store normal for lighting calculations
52.         }
53.  
54.  
55.         void surf (Input IN, inout SurfaceOutput o) {
56.             half4 c = tex2D (_MainTex, IN.uv_MainTex);
57.             o.Albedo = c.rgb;
58.         }
59.         ENDCG
60.     }
61. }

(There are neater examples in the Unity docs)
With the help of the MSDN docs information about Hull shaders and Domain Shaders, I pieced together a partially working set of 'regular' shaders (vertex, hull, domain and fragment)
The following is a shader that I found in my git repo. It seems to be the first compiling shader file with tessellation created in the way I described. It doesn't seem to handle perspective correctly at all, but at least the basics work . Those problems are fixed in the next version I have but that version also contains all code for water rendering (I would like to say I could strip all that out for you, to get a version that purely shows the tessellation code, but I don't think I have the time for that)
--[EDIT] This won't render correctly. I recommend using the code I pasted a few posts down instead. --

Code (csharp):

1. Shader "Custom/BasicTessellation" {
2.     Properties {
3.         _MainTex ("Base (RGB)", 2D) = "white" {}
4.         _Offset ("Offset", Float) = 10
5.         _Tess ("Tessellation", Float) = 2
6.     }
7.     SubShader {
8.         Tags { "RenderType"="Opaque" }
9.         Pass {
10.        
11.             CGPROGRAM
12.             #pragma vertex vert
13.             #pragma fragment frag
14.             #pragma hull tessBase
15.             #pragma domain basicDomain
16.             #pragma target 5.0
17.             #include "UnityCG.cginc"
18.             #define INTERNAL_DATA
19.    
20.             sampler2D _MainTex;
21.            
22.             float4 _MainTex_ST;
23.             float _Offset;
24.             float _Tess;
25.            
26.             struct v2f{
27.                 float4 pos : SV_POSITION;
28.                 float2 texcoord : TEXCOORD0;
29.             };
30.            
31.             #ifdef UNITY_CAN_COMPILE_TESSELLATION
32.            
33.             struct inputControlPoint{
34.                 float4 position : WORLDPOS;
35.                 float4 texcoord : TEXCOORD0;
36.                 float4 tangent : TANGENT;
37.                 float3 normal : NORMAL;
38.             };
39.            
40.             struct outputControlPoint{
41.                 float3 position : BEZIERPOS;            
42.             };
43.            
44.             struct outputPatchConstant{
45.                 float edges[3]        : SV_TessFactor;
46.                 float inside        : SV_InsideTessFactor;
47.                
48.                 float3 vTangent[4]    : TANGENT;
49.                 float2 vUV[4]         : TEXCOORD;
50.                 float3 vTanUCorner[4] : TANUCORNER;
51.                 float3 vTanVCorner[4] : TANVCORNER;
52.                 float4 vCWts          : TANWEIGHTS;
53.             };
54.            
55.            
56.             outputPatchConstant patchConstantThing(InputPatch<inputControlPoint, 3> v){
57.                 outputPatchConstant o;
58.                
59.                 o.edges[0] = _Tess;
60.                 o.edges[1] = _Tess;
61.                 o.edges[2] = _Tess;
62.                 o.inside = _Tess;
63.                
64.                 return o;
65.             }
66.            
67.             // tessellation hull shader
68.             [domain("tri")]
69.             [partitioning("fractional_odd")]
70.             [outputtopology("triangle_cw")]
71.             [patchconstantfunc("patchConstantThing")]
72.             [outputcontrolpoints(3)]
73.             inputControlPoint tessBase (InputPatch<inputControlPoint,3> v, uint id : SV_OutputControlPointID) {
74.                 return v[id];
75.             }
76.            
77.             #endif // UNITY_CAN_COMPILE_TESSELLATION
78.            
79.             v2f vert (appdata_tan v){
80.                 v2f o;        
81.                
82.                 o.texcoord = TRANSFORM_TEX (v.texcoord, _MainTex);
83.                
84.                 float localTex = tex2Dlod(_MainTex, float4(o.texcoord,0,0)).r;
85.                 v.vertex.y += localTex.r * _Offset;
86.                
87.                 o.pos = mul(UNITY_MATRIX_MVP,v.vertex);
88.                
89.                 return o;
90.             }
91.            
92.             #ifdef UNITY_CAN_COMPILE_TESSELLATION
93.            
94.             // tessellation domain shader
95.             [domain("tri")]
96.             v2f basicDomain (outputPatchConstant tessFactors, const OutputPatch<inputControlPoint,3> vi, float3 bary : SV_DomainLocation) {
97.                 appdata_tan v;
98.                 v.vertex = vi[0].position*bary.x + vi[1].position*bary.y + vi[2].position*bary.z;
99.                 v.tangent = vi[0].tangent*bary.x + vi[1].tangent*bary.y + vi[2].tangent*bary.z;
100.                 v.normal = vi[0].normal*bary.x + vi[1].normal*bary.y + vi[2].normal*bary.z;
101.                 v.texcoord = vi[0].texcoord*bary.x + vi[1].texcoord*bary.y + vi[2].texcoord*bary.z;
102.                 v2f o = vert( v);
103. //                v2f o = vert_surf (v);
104.                 return o;
105.             }
106.            
107.             #endif // UNITY_CAN_COMPILE_TESSELLATION
108.            
109.            
110.    
111.             float4 frag(in v2f IN):COLOR{
112.                 return tex2D (_MainTex, IN.texcoord);
113.             }
114.            
115.             ENDCG
116.         }
117.     }
118. }
119.  

It might not be a useful shader on its own ^^, but this shader does show a basic working framework with the almost all shader stages present (there's no geometry shader in this one).

Yeah, back in January I couldn't find much helpful information either. But if you are better at reading than me, the documentation on MSDN should be a real help. And even if you're not, those articles contain enough information to allow you to piece it together. Most of the time you can ignore all the CPU code (setting rendering state, stuff like that), since Unity handles that stuff for us. So that means, ignore the 'how to create(...)' articles, and focus on the 'how to design (...)' articles.

Does that help?

 

Thank you very much! , I will look through these and if I can still not find a solution, I think I have give up my "fancy wave system" idea ...

I would be very interested in that water shader though (or a stripped version), maybe we could trade it for something XD? I do procedural art ,
as well as modeling.

I am afraid the "Custom/BasicTessellation" shader does not really work, it does compile, but makes the object invisible and scales it to a vertical line.
I will try to get it working though, but so far I haven't got it yet.

 

Hmmm, It's probably because in that version I still used the vertex shader function in the domain shader. So the position is multiplied with the MVP matrix too many times.

Try this:

Code (csharp):

1. Shader "Custom/BasicTessellation" {
2.     Properties {
3.         _MainTex ("Base (RGB)", 2D) = "white" {}
4.         _Offset ("Offset", Float) = 10
5.         _Tess ("Tessellation", Float) = 2
6.     }
7.     SubShader {
8.         Tags { "RenderType"="Opaque" }
9.         Pass {
10.        
11.             CGPROGRAM
12.             #pragma vertex vert
13.             #pragma fragment frag
14.             #pragma hull tessBase
15.             #pragma domain basicDomain
16.             #pragma target 5.0
17.             #include "UnityCG.cginc"
18.             #define INTERNAL_DATA
19.    
20.             sampler2D _MainTex;
21.            
22.             float4 _MainTex_ST;
23.             float _Offset;
24.             float _Tess;
25.            
26.             struct v2f{
27.                 float4 pos : SV_POSITION;
28.                 float2 texcoord : TEXCOORD0;
29.             };
30.            
31.             #ifdef UNITY_CAN_COMPILE_TESSELLATION
32.            
33.             struct inputControlPoint{
34.                 float4 position : WORLDPOS;
35.                 float4 texcoord : TEXCOORD0;
36.                 float4 tangent : TANGENT;
37.                 float3 normal : NORMAL;
38.             };
39.            
40.             struct outputControlPoint{
41.                 float3 position : BEZIERPOS;            
42.             };
43.            
44.             struct outputPatchConstant{
45.                 float edges[3]        : SV_TessFactor;
46.                 float inside        : SV_InsideTessFactor;
47.                
48.                 float3 vTangent[4]    : TANGENT;
49.                 float2 vUV[4]         : TEXCOORD;
50.                 float3 vTanUCorner[4] : TANUCORNER;
51.                 float3 vTanVCorner[4] : TANVCORNER;
52.                 float4 vCWts          : TANWEIGHTS;
53.             };
54.            
55.            
56.             outputPatchConstant patchConstantThing(InputPatch<inputControlPoint, 3> v){
57.                 outputPatchConstant o;
58.                
59.                 o.edges[0] = _Tess;
60.                 o.edges[1] = _Tess;
61.                 o.edges[2] = _Tess;
62.                 o.inside = _Tess;
63.                
64.                 return o;
65.             }
66.            
67.             // tessellation hull shader
68.             [domain("tri")]
69.             [partitioning("fractional_odd")]
70.             [outputtopology("triangle_cw")]
71.             [patchconstantfunc("patchConstantThing")]
72.             [outputcontrolpoints(3)]
73.             inputControlPoint tessBase (InputPatch<inputControlPoint,3> v, uint id : SV_OutputControlPointID) {
74.                 return v[id];
75.             }
76.            
77.             #endif // UNITY_CAN_COMPILE_TESSELLATION
78.            
79.             v2f vert (appdata_tan v){
80.                 v2f o;
81.                
82.                 o.texcoord = v.texcoord;
83.                 o.pos = v.vertex;
84.                
85.                 return o;
86.             }
87.            
88.             v2f displace (appdata_tan v){
89.                 v2f o;        
90.                
91.                 o.texcoord = TRANSFORM_TEX (v.texcoord, _MainTex);
92.                
93.                 float localTex = tex2Dlod(_MainTex, float4(o.texcoord,0,0)).r;
94.                 v.vertex.y += localTex.r * _Offset;
95.                
96.                 o.pos = mul(UNITY_MATRIX_MVP,v.vertex);
97.                
98.                 return o;
99.             }
100.            
101.             #ifdef UNITY_CAN_COMPILE_TESSELLATION
102.            
103.             // tessellation domain shader
104.             [domain("tri")]
105.             v2f basicDomain (outputPatchConstant tessFactors, const OutputPatch<inputControlPoint,3> vi, float3 bary : SV_DomainLocation) {
106.                 appdata_tan v;
107.                 v.vertex = vi[0].position*bary.x + vi[1].position*bary.y + vi[2].position*bary.z;
108.                 v.tangent = vi[0].tangent*bary.x + vi[1].tangent*bary.y + vi[2].tangent*bary.z;
109.                 v.normal = vi[0].normal*bary.x + vi[1].normal*bary.y + vi[2].normal*bary.z;
110.                 v.texcoord = vi[0].texcoord*bary.x + vi[1].texcoord*bary.y + vi[2].texcoord*bary.z;
111.                 v2f o = displace( v);
112. //                v2f o = vert_surf (v);
113.                 return o;
114.             }
115.            
116.             #endif // UNITY_CAN_COMPILE_TESSELLATION
117.            
118.            
119.    
120.             float4 frag(in v2f IN):COLOR{
121.                 return tex2D (_MainTex, IN.texcoord);
122.             }
123.            
124.             ENDCG
125.         }
126.     }
127. }
128.  
129.  

 

I'm currently working on making the whole water simulation usable as tool, and I want to make it available on the asset store. So that would also include the water shaders. But I'm afraid it wouldn't exactly be cheap, considering a student's budget (I saw you're attending NHTV here in The Netherlands ). So, hopefully the latest version of the shader I just posted is more useful.

 

Im afraid the new shader gives the same problem ,
it squeezes the model to a point/line depending on the offset.

I already saw you were Dutch, pretty cool I guess .
But I understand how you feel about your project.
I am working on a project of my own as well, which I feel similarly about

 

Are you sure Unity is set to DX11 mode? Do the vertex and fragment shaders show up as shader model 5?
*Resets the shader to test default settings*

Ahhh, did I mention that it requires a heightmap? XD
I'm using a terrain heightmap, but for testing purposes you should be able to use any texture. It only cares about the red channel.

 

I am pretty sure I run in DX11, as I already use Shader model 5 shaders for other stuff and I also did apply several textures, but they all seem to give the same result for me somehow :/

I'm currently using Unity 4.0.1.f2

 

Interesting...
And the surface shader version does work?
This is what it looks like for me:

​

Does it work if you remove the displacement code?

Code (csharp):

1. float localTex = tex2Dlod(_MainTex, float4(o.texcoord,0,0)).r;
2. v.vertex.y += localTex.r * _Offset;

 

hmm well over here it looks like this (I renamed the shader):

Also the sampling from the texture works fine (I guess) as with different textures the length of the line and starting height differ. Turning that off, is the same as setting the offset to 0.

I also really appreciate the help, I think I can send you my first asset store pack (which I just sent for approval) with object space normal map shaders, if you want those of course .

Maybe you could send me a unity package with the scene shown in the image above? I really don't see what I am doing wrong here

Edit: I just upgraded to Unity 4.1, but that also did not change anything. (even though there were shader updates in this version)

Edit, I found a different Fragment based Tessellation that does work for me :

Code (csharp):

1. Shader "Tessellation/Simple Tess PN Disp" {
2. Properties {
3.     _MainTex ("Main Texture", 2D) = "white" {}
4.     _DispTex ("Disp Texture", 2D) = "gray" {}
5.     _TessEdge ("Edge Tess", Range(1,16)) = 2
6.     _Displacement ("Displacement", Range(0, 0.3)) = 0.1
7. }
8. SubShader {
9. Pass {
10.     Tags {"LightMode" = "Vertex"}
11.  
12. CGPROGRAM
13. #pragma target 5.0
14.  
15. //#pragma vertex VS_RenderScene
16. //#pragma fragment PS_RenderSceneTextured
17.  
18. #pragma vertex VS_RenderSceneWithTessellation
19. #pragma fragment PS_RenderSceneTextured
20. #pragma hull HS_PNTriangles
21. #pragma domain DS_PNTriangles
22.  
23. #include "UnityCG.cginc"
24.  
25. float _TessEdge;
26. float _Displacement;
27.  
28.  
29. //=================================================================================================================================
30. // Buffers, Textures and Samplers
31. //=================================================================================================================================
32.  
33. Texture2D _MainTex;
34. SamplerState    sampler_MainTex;
35. Texture2D _DispTex;
36. SamplerState    sampler_DispTex;
37.  
38. //=================================================================================================================================
39. // Shader structures
40. //=================================================================================================================================
41.  
42.  
43. struct VS_RenderSceneInput
44. {
45.     float3 vertex : POSITION;
46.     float3 normal : NORMAL;
47.     float2 texcoord : TEXCOORD0;
48. };
49.  
50. struct HS_Input
51. {
52.     float4 f4Position   : POS;
53.     float3 f3Normal     : NORMAL;
54.     float2 f2TexCoord   : TEXCOORD;
55. };
56.  
57. struct HS_ConstantOutput
58. {
59.     // Tess factor for the FF HW block
60.     float fTessFactor[3]    : SV_TessFactor;
61.     float fInsideTessFactor : SV_InsideTessFactor;
62.    
63.     // Geometry cubic generated control points
64.     float3 f3B210    : POS3;
65.     float3 f3B120    : POS4;
66.     float3 f3B021    : POS5;
67.     float3 f3B012    : POS6;
68.     float3 f3B102    : POS7;
69.     float3 f3B201    : POS8;
70.     float3 f3B111    : CENTER;
71.    
72.     // Normal quadratic generated control points
73.     float3 f3N110    : NORMAL3;      
74.     float3 f3N011    : NORMAL4;
75.     float3 f3N101    : NORMAL5;
76. };
77.  
78. struct HS_ControlPointOutput
79. {
80.     float3    f3Position    : POS;
81.     float3    f3Normal      : NORMAL;
82.     float2    f2TexCoord    : TEXCOORD;
83. };
84.  
85. struct DS_Output
86. {
87.     float4 f4Position   : SV_Position;
88.     float2 f2TexCoord   : TEXCOORD0;
89.     float4 f4Diffuse    : COLOR0;
90. };
91.  
92. struct PS_RenderSceneInput
93. {
94.     float4 f4Position   : SV_Position;
95.     float2 f2TexCoord   : TEXCOORD0;
96.     float4 f4Diffuse    : COLOR0;
97. };
98.  
99. struct PS_RenderOutput
100. {
101.     float4 f4Color      : SV_Target0;
102. };
103.  
104. PS_RenderSceneInput VS_RenderScene( VS_RenderSceneInput I )
105. {
106.     PS_RenderSceneInput O;
107.    
108.     O.f4Position = mul (UNITY_MATRIX_MVP, float4(I.vertex, 1.0f));
109.     float3 viewN = mul ((float3x3)UNITY_MATRIX_IT_MV, I.normal);
110.    
111.     // Calc diffuse color    
112.     O.f4Diffuse.rgb = unity_LightColor[0].rgb * max(0, dot(viewN, unity_LightPosition[0].xyz)) + UNITY_LIGHTMODEL_AMBIENT.rgb;
113.     O.f4Diffuse.a = 1.0f;
114.    
115.     // Pass through texture coords
116.     O.f2TexCoord = I.texcoord;
117.    
118.     return O;    
119. }
120.  
121.  
122. HS_Input VS_RenderSceneWithTessellation( VS_RenderSceneInput I )
123. {
124.     HS_Input O;
125.    
126.     // To view space
127.     O.f4Position = mul(UNITY_MATRIX_MV, float4(I.vertex,1.0f));
128.     O.f3Normal = mul ((float3x3)UNITY_MATRIX_IT_MV, I.normal);
129.        
130.     O.f2TexCoord = I.texcoord;
131.    
132.     return O;
133. }
134.  
135.  
136. //=================================================================================================================================
137. // This hull shader passes the tessellation factors through to the HW tessellator,
138. // and the 10 (geometry), 6 (normal) control points of the PN-triangular patch to the domain shader
139. //=================================================================================================================================
140. HS_ConstantOutput HS_PNTrianglesConstant( InputPatch<HS_Input, 3> I )
141. {
142.     HS_ConstantOutput O = (HS_ConstantOutput)0;
143.    
144.     // Simply output the tessellation factors from constant space
145.     // for use by the FF tessellation unit
146.     O.fTessFactor[0] = O.fTessFactor[1] = O.fTessFactor[2] = _TessEdge;
147.     O.fInsideTessFactor = _TessEdge;
148.  
149.     // Assign Positions
150.     float3 f3B003 = I[0].f4Position.xyz;
151.     float3 f3B030 = I[1].f4Position.xyz;
152.     float3 f3B300 = I[2].f4Position.xyz;
153.     // And Normals
154.     float3 f3N002 = I[0].f3Normal;
155.     float3 f3N020 = I[1].f3Normal;
156.     float3 f3N200 = I[2].f3Normal;
157.        
158.     // Compute the cubic geometry control points
159.     // Edge control points
160.     O.f3B210 = ( ( 2.0f * f3B003 ) + f3B030 - ( dot( ( f3B030 - f3B003 ), f3N002 ) * f3N002 ) ) / 3.0f;
161.     O.f3B120 = ( ( 2.0f * f3B030 ) + f3B003 - ( dot( ( f3B003 - f3B030 ), f3N020 ) * f3N020 ) ) / 3.0f;
162.     O.f3B021 = ( ( 2.0f * f3B030 ) + f3B300 - ( dot( ( f3B300 - f3B030 ), f3N020 ) * f3N020 ) ) / 3.0f;
163.     O.f3B012 = ( ( 2.0f * f3B300 ) + f3B030 - ( dot( ( f3B030 - f3B300 ), f3N200 ) * f3N200 ) ) / 3.0f;
164.     O.f3B102 = ( ( 2.0f * f3B300 ) + f3B003 - ( dot( ( f3B003 - f3B300 ), f3N200 ) * f3N200 ) ) / 3.0f;
165.     O.f3B201 = ( ( 2.0f * f3B003 ) + f3B300 - ( dot( ( f3B300 - f3B003 ), f3N002 ) * f3N002 ) ) / 3.0f;
166.     // Center control point
167.     float3 f3E = ( O.f3B210 + O.f3B120 + O.f3B021 + O.f3B012 + O.f3B102 + O.f3B201 ) / 6.0f;
168.     float3 f3V = ( f3B003 + f3B030 + f3B300 ) / 3.0f;
169.     O.f3B111 = f3E + ( ( f3E - f3V ) / 2.0f );
170.    
171.     // Compute the quadratic normal control points
172.     float fV12 = 2.0f * dot( f3B030 - f3B003, f3N002 + f3N020 ) / dot( f3B030 - f3B003, f3B030 - f3B003 );
173.     O.f3N110 = normalize( f3N002 + f3N020 - fV12 * ( f3B030 - f3B003 ) );
174.     float fV23 = 2.0f * dot( f3B300 - f3B030, f3N020 + f3N200 ) / dot( f3B300 - f3B030, f3B300 - f3B030 );
175.     O.f3N011 = normalize( f3N020 + f3N200 - fV23 * ( f3B300 - f3B030 ) );
176.     float fV31 = 2.0f * dot( f3B003 - f3B300, f3N200 + f3N002 ) / dot( f3B003 - f3B300, f3B003 - f3B300 );
177.     O.f3N101 = normalize( f3N200 + f3N002 - fV31 * ( f3B003 - f3B300 ) );
178.            
179.     return O;
180. }
181.  
182. [domain("tri")]
183. [partitioning("fractional_odd")]
184. [outputtopology("triangle_cw")]
185. [patchconstantfunc("HS_PNTrianglesConstant")]
186. [outputcontrolpoints(3)]
187. HS_ControlPointOutput HS_PNTriangles( InputPatch<HS_Input, 3> I, uint uCPID : SV_OutputControlPointID )
188. {
189.     HS_ControlPointOutput O = (HS_ControlPointOutput)0;
190.  
191.     // Just pass through inputs = fast pass through mode triggered
192.     O.f3Position = I[uCPID].f4Position.xyz;
193.     O.f3Normal = I[uCPID].f3Normal;
194.     O.f2TexCoord = I[uCPID].f2TexCoord;
195.    
196.     return O;
197. }
198.  
199.  
200. //=================================================================================================================================
201. // This domain shader applies contol point weighting to the barycentric coords produced by the FF tessellator
202. //=================================================================================================================================
203. [domain("tri")]
204. DS_Output DS_PNTriangles( HS_ConstantOutput HSConstantData, const OutputPatch<HS_ControlPointOutput, 3> I, float3 f3BarycentricCoords : SV_DomainLocation )
205. {
206.     DS_Output O = (DS_Output)0;
207.  
208.     // The barycentric coordinates
209.     float fU = f3BarycentricCoords.x;
210.     float fV = f3BarycentricCoords.y;
211.     float fW = f3BarycentricCoords.z;
212.  
213.     // Precompute squares and squares * 3
214.     float fUU = fU * fU;
215.     float fVV = fV * fV;
216.     float fWW = fW * fW;
217.     float fUU3 = fUU * 3.0f;
218.     float fVV3 = fVV * 3.0f;
219.     float fWW3 = fWW * 3.0f;
220.    
221.     // Compute position from cubic control points and barycentric coords
222.     float3 f3Position = I[0].f3Position * fWW * fW +
223.                         I[1].f3Position * fUU * fU +
224.                         I[2].f3Position * fVV * fV +
225.                         HSConstantData.f3B210 * fWW3 * fU +
226.                         HSConstantData.f3B120 * fW * fUU3 +
227.                         HSConstantData.f3B201 * fWW3 * fV +
228.                         HSConstantData.f3B021 * fUU3 * fV +
229.                         HSConstantData.f3B102 * fW * fVV3 +
230.                         HSConstantData.f3B012 * fU * fVV3 +
231.                         HSConstantData.f3B111 * 6.0f * fW * fU * fV;
232.    
233.     // Compute normal from quadratic control points and barycentric coords
234.     float3 f3Normal =   I[0].f3Normal * fWW +
235.                         I[1].f3Normal * fUU +
236.                         I[2].f3Normal * fVV +
237.                         HSConstantData.f3N110 * fW * fU +
238.                         HSConstantData.f3N011 * fU * fV +
239.                         HSConstantData.f3N101 * fW * fV;
240.  
241.     // Normalize the interpolated normal    
242.     f3Normal = normalize( f3Normal );
243.  
244.     // Linearly interpolate the texture coords
245.     O.f2TexCoord = I[0].f2TexCoord * fW + I[1].f2TexCoord * fU + I[2].f2TexCoord * fV;
246.    
247.     // Displacement map!
248.     float disp = _DispTex.SampleLevel (sampler_DispTex, O.f2TexCoord, 0).r * _Displacement;
249.     f3Position += f3Normal * disp;
250.  
251.     // Calc diffuse color    
252.     O.f4Diffuse.rgb = unity_LightColor[0].rgb * max( 0, dot( f3Normal, unity_LightPosition[0].xyz ) ) + UNITY_LIGHTMODEL_AMBIENT.rgb;
253.     O.f4Diffuse.a = 1.0f;
254.  
255.     // Transform position with projection matrix
256.     O.f4Position = mul (UNITY_MATRIX_P, float4(f3Position.xyz,1.0));
257.        
258.     return O;
259. }
260.  
261.  
262. PS_RenderOutput PS_RenderSceneTextured( PS_RenderSceneInput I )
263. {
264.     PS_RenderOutput O;
265.    
266.     O.f4Color = _MainTex.Sample( sampler_MainTex, I.f2TexCoord ).r * I.f4Diffuse;
267.    
268.     return O;
269. }
270.  
271. ENDCG
272.  
273. }
274. }
275.  
276. Fallback "VertexLit"
277. }

I also realized that this shader (as well as yours) are not camera distance based, which would be preferable. I will try to look into that as well.

 

Now I'm really curious about what's going on... I don't remember seeing you need Unity Pro for Tessellation, or anything like that.
If I ever want to use DX11 tessellation for dynamic level of detail in terrain and water meshes, I need to know why it doesn't work on your machine. So I've created a test scene to figure out where it's exploding.

This is what it looks like on my machine (OH wow, the attached image actually worked immediately this time )

​

Left: Example script from documentation.
Middle: Mutilated example script I used to figure out tessellation
Right: Reconstructed Vertex, Hull, Domain, Fragment shader version

Here's the scene: View attachment $TessellationExperiments.unitypackage

If you could upload a screenshot of what it looks like for you, that might help me figure out what's going wrong. The package includes a basic debug information script that hopefully gives some insight into the differences between your setup and mine. (I didn't create it specifically for this scene though; we probably don't need to know if your pc supports ARGBFloat RenderTextures )

[Edit] I saw your edit too late ^^. Well, good news

 

Thank you very much I just opened you scene,
the interesting part: 2 out of 3 work:

These work:
Tessellation Docs Sample
Surface Shader Tessellation

Still give the same problem:
Vertex Hull Domain Fragment Tessellation

My setup does not support: RenderTextures, ARGBFloat and Depth Textures (pro features I guess)
The rest is all supported. There is a possibility that it is because I use an Nvidia card, where you have an ATI card,
but your water demo-build did work on my machine. Or maybe a missing DLL, I really don't know XD.



I also threw together a distance based tesselation thing:


This was done by editing the following function in the shader in my last post.
Note that the current implementation creates holes in the geometry
and the conversion matrix for the distance is incorrect.
(If you scale or move the object, it won't work anymore)
and the blend distances are hardcoded for now.

Do you think it is even possible to get this seamless,
or will this require the use of a surface shader by default?

Code (csharp):

1. HS_ConstantOutput HS_PNTrianglesConstant( InputPatch<HS_Input, 3> I )
2. {
3.     HS_ConstantOutput O = (HS_ConstantOutput)0;
4.    
5.     // Simply output the tessellation factors from constant space
6.     // for use by the FF tessellation unit
7.  
8.    
9.     // Assign Positions
10.     float3 f3B003 = I[0].f4Position.xyz;
11.     float3 f3B030 = I[1].f4Position.xyz;
12.     float3 f3B300 = I[2].f4Position.xyz;
13.     // And Normals
14.     float3 f3N002 = I[0].f3Normal;
15.     float3 f3N020 = I[1].f3Normal;
16.     float3 f3N200 = I[2].f3Normal;
17.    
18.     float _StartBV = 3;
19.     float _EndBV = 0;
20.     float3 midPos = mul(_Object2World,(f3B003+f3B030+f3B300)/3);
21.     float3 camPos = mul(_World2Object,_WorldSpaceCameraPos.xyz);
22.     //float camDist = clamp((_StartBV-(distance(mul (_Object2World, midPos).xyz, _WorldSpaceCameraPos.xyz)))*(1/(_StartBV-_EndBV)),0,1);
23.     //float camDist = clamp((_StartBV-(distance(mul(_Object2World, midPos).xyz, _WorldSpaceCameraPos.xyz)))*(1/(_StartBV-_EndBV)),0,1);
24.    
25.    
26.     float camDist = clamp(_StartBV-(distance(midPos.xyz, camPos.xyz))*(1/(_StartBV-_EndBV)),0,1);
27.     //float camDist = distance(mul(_Object2World, midPos).xyz, _WorldSpaceCameraPos.xyz);
28.    
29.     O.fTessFactor[0] = O.fTessFactor[1] = O.fTessFactor[2] = max(camDist*_TessEdge,1);
30.     O.fInsideTessFactor = max(camDist*_TessEdge,1);
31.        
32.     // Compute the cubic geometry control points
33.     // Edge control points
34.     O.f3B210 = ( ( 2.0f * f3B003 ) + f3B030 - ( dot( ( f3B030 - f3B003 ), f3N002 ) * f3N002 ) ) / 3.0f;
35.     O.f3B120 = ( ( 2.0f * f3B030 ) + f3B003 - ( dot( ( f3B003 - f3B030 ), f3N020 ) * f3N020 ) ) / 3.0f;
36.     O.f3B021 = ( ( 2.0f * f3B030 ) + f3B300 - ( dot( ( f3B300 - f3B030 ), f3N020 ) * f3N020 ) ) / 3.0f;
37.     O.f3B012 = ( ( 2.0f * f3B300 ) + f3B030 - ( dot( ( f3B030 - f3B300 ), f3N200 ) * f3N200 ) ) / 3.0f;
38.     O.f3B102 = ( ( 2.0f * f3B300 ) + f3B003 - ( dot( ( f3B003 - f3B300 ), f3N200 ) * f3N200 ) ) / 3.0f;
39.     O.f3B201 = ( ( 2.0f * f3B003 ) + f3B300 - ( dot( ( f3B300 - f3B003 ), f3N002 ) * f3N002 ) ) / 3.0f;
40.     // Center control point
41.     float3 f3E = ( O.f3B210 + O.f3B120 + O.f3B021 + O.f3B012 + O.f3B102 + O.f3B201 ) / 6.0f;
42.     float3 f3V = ( f3B003 + f3B030 + f3B300 ) / 3.0f;
43.     O.f3B111 = f3E + ( ( f3E - f3V ) / 2.0f );
44.    
45.     // Compute the quadratic normal control points
46.     float fV12 = 2.0f * dot( f3B030 - f3B003, f3N002 + f3N020 ) / dot( f3B030 - f3B003, f3B030 - f3B003 );
47.     O.f3N110 = normalize( f3N002 + f3N020 - fV12 * ( f3B030 - f3B003 ) );
48.     float fV23 = 2.0f * dot( f3B300 - f3B030, f3N020 + f3N200 ) / dot( f3B300 - f3B030, f3B300 - f3B030 );
49.     O.f3N011 = normalize( f3N020 + f3N200 - fV23 * ( f3B300 - f3B030 ) );
50.     float fV31 = 2.0f * dot( f3B003 - f3B300, f3N200 + f3N002 ) / dot( f3B003 - f3B300, f3B003 - f3B300 );
51.     O.f3N101 = normalize( f3N200 + f3N002 - fV31 * ( f3B003 - f3B300 ) );
52.            
53.     return O;
54. }

Also my offer still counts, if you want my object space normal shaders (which don't plan to be free for everyone )

 

My water simulation doesn't use tessellation yet, (which is also why it'll run slow if you run it at resolutions like 2048 * 2048, on a graphics card like mine).
I think you might be right when you say it could be because of the difference between AMD and Nvidia cards; I probably forgot to include something that's required on Nvidia cards.

The surface shader version is first compiled to regular shaders first, so anything you could do with a surface shader should be possible to do manually. But it requires more work and knowledge of the underlying shader stages. Unfortunately, I don't yet know how to make sure both versions of edges are subdivided by the same amount, so I can't tell you how to fix those gaps.

Have you taken a look at how the distance-based tessellation surface shader example from the docs compiles?

Thanks for the offer, by the way! I've never needed object space normal mapped objects before, but perhaps I'd learn something from those shaders.

 

i'm interested to know if you figured out the scaling issue? i'm also trying to do a few experiments and I'm really interested to know how this turned out. did you guys ever get displacement/height implemented into it as well?

 

I have the same result as @Acey195, despite the fact that my graphics card DOES support DOES support depth, ARGBFloat and render textures. As far as I know this should not be an NVidia vs ATi thing, but may be a Unity free vs. Unity pro thing (I know that is the case for RenderTextures at least). I'm running an NVidia GeForce GTX 970:



The Vertex-Hull-Domain-Fragment shader is rendering what looks like a single pixel, and when it is selected the wireframe is just one line:



I'm going to play with it a bit myself, but @RC-1290, could you save me some time if you already know what's going on? Thanks a million for your code.

 

Since you suspect it might be a Unity Free versus Unity Pro problem, have you tried it in Unity 5?

Actually, I just tested this on my new laptop, which uses a GeForce GTX 860M, and I get the same buggy results in Unity 5, as you describe, so I do think it's a vendor specific problem.

[Edit] same problem in Unity 4 Pro, with Nvidia hardware. It might be worth looking at the alternative shader found above.

 

Hi everyone,

I know this thread is a bit old so sorry to bring it back to life however I'm struggling with this now and was wondering if anyone has got a good way of implimenting tessellation in a vert frag shader ??

Thank you all loads