Outline Shader help

Hey, everyone, I am very new to shaders and just started learning last week.

Right now I am having troubles with creating an outline shader. This is what I have so far but it isn't working it basically creates a huge black blob around the model lol.

Code (CSharp):

1. Shader "Custom/OutlineShader"
2. {
3.     // Variables
4.     Properties
5.     {
6.         _MainTex("Main Texture", 2D) = "white" {}
7.         _AmbientOcclusion("Ambient Occlusion", 2D) = "white" {}
8.  
9.         _OutlineWidth("Outline Width", float) = 2
10.         _OutlineColor("Outline Color", color) = (1,1,1,1)
11.  
12.         _ZValue("Z Value", float) = -0.4
13.     }
14.  
15.     SubShader
16.         {
17.                 Tags{ "Queue" = "Transparent" "IgnoreProjector" = "True" "RenderType" = "Transparent"}
18.             Pass
19.             {
20.  
21.                 ZWrite Off
22.  
23.                 Blend SrcAlpha OneMinusSrcAlpha
24.  
25.                 CGPROGRAM
26.                 #pragma vertex vertexFunction
27.                 #pragma fragment fragmentFunction
28.  
29.                 #include "UnityCG.cginc"
30.  
31.                 struct appdata {
32.                     fixed4 vertex : POSITION;
33.                     fixed3 normal : NORMAL;
34.                 };
35.  
36.                 struct v2f {
37.                     float4 position : SV_POSITION;
38.                 };
39.  
40.                 float _OutlineWidth;
41.                 float4 _OutlineColor;
42.                 half _ZValue;
43.  
44.                 v2f vertexFunction(appdata IN)
45.                 {
46.                     v2f OUT;
47.  
48.                     OUT.position = UnityObjectToClipPos(IN.vertex);
49.                     float3 normal = mul((float3x3)UNITY_MATRIX_IT_MV, IN.normal);
50.                     normal.z = _ZValue;
51.  
52.                     OUT.position = OUT.position + float4(normalize(normal), 0) * _OutlineWidth / 2;
53.                     OUT.position = mul(UNITY_MATRIX_P, OUT.position);
54.                     return OUT;
55.                 }
56.  
57.                 fixed4 fragmentFunction(v2f IN) : COLOR
58.                 {
59.                     return _OutlineColor;
60.                 }
61.                 ENDCG
62.             }
63.  
64.             Pass
65.             {
66.                 CGPROGRAM
67.                 #pragma vertex vertFunction
68.                 #pragma fragment frag
69.  
70.                 struct appdata {
71.                     fixed4 vertex : POSITION;
72.                     fixed2 uv : TEXCOORD0;
73.                 };
74.  
75.                 struct v2f {
76.                     fixed4 position : SV_POSITION;
77.                     fixed2 texcoord : TEXCOORD0;
78.                 };
79.  
80.                 sampler2D _MainTex;
81.                 sampler2D _AmbientOcclusion;
82.  
83.                 v2f vertFunction(appdata IN)
84.                 {
85.                     v2f OUT;
86.  
87.                     OUT.position = UnityObjectToClipPos(IN.vertex);
88.                     OUT.texcoord = IN.uv;
89.  
90.                     return OUT;
91.                 }
92.  
93.                 fixed4 frag(v2f IN) : SV_Target
94.                 {
95.                     fixed4 mainTextureColor = tex2D(_MainTex, IN.texcoord);
96.                    
97.                     fixed4 endColor = mainTextureColor;
98.                     return endColor;
99.                 }
100.                 ENDCG
101.             }
102.            
103.         }
104.     }

 

Here is the "fixed" code. However, doing outlines this way, by pushing vertex-positions along their normals, only works with certain geometry. It works well with a sphere, but breaks apart with a cube for example.

Spoiler: Code

Code (CSharp):

1.     Shader "Custom/OutlineShader"
2.     {
3.         // Variables
4.         Properties
5.         {
6.             _MainTex("Main Texture", 2D) = "white" {}
7.             _AmbientOcclusion("Ambient Occlusion", 2D) = "white" {}
8.    
9.             _OutlineWidth("Outline Width", float) = 2
10.             _OutlineColor("Outline Color", color) = (1,1,1,1)
11.         }
12.    
13.         SubShader
14.             {
15.                     Tags{ "Queue" = "Transparent" "IgnoreProjector" = "True" "RenderType" = "Transparent"}
16.                 Pass
17.                 {
18.    
19.                     ZWrite Off
20.    
21.                     Blend SrcAlpha OneMinusSrcAlpha
22.    
23.                     CGPROGRAM
24.                     #pragma vertex vertexFunction
25.                     #pragma fragment fragmentFunction
26.    
27.                     #include "UnityCG.cginc"
28.    
29.                     struct appdata {
30.                         fixed4 vertex : POSITION;
31.                         fixed3 normal : NORMAL;
32.                     };
33.    
34.                     struct v2f {
35.                         float4 position : SV_POSITION;
36.                     };
37.    
38.                     float _OutlineWidth;
39.                     float4 _OutlineColor;
40.    
41.                     v2f vertexFunction(appdata IN)
42.                     {
43.                         v2f OUT;
44.    
45.                         OUT.position = UnityObjectToClipPos(IN.vertex + IN.normal * _OutlineWidth);
46.                         return OUT;
47.                     }
48.    
49.                     fixed4 fragmentFunction(v2f IN) : COLOR
50.                     {
51.                         return _OutlineColor;
52.                     }
53.                     ENDCG
54.                 }
55.    
56.                 Pass
57.                 {
58.                     CGPROGRAM
59.                     #pragma vertex vertFunction
60.                     #pragma fragment frag
61.    
62.                     struct appdata {
63.                         fixed4 vertex : POSITION;
64.                         fixed2 uv : TEXCOORD0;
65.                     };
66.    
67.                     struct v2f {
68.                         fixed4 position : SV_POSITION;
69.                         fixed2 texcoord : TEXCOORD0;
70.                     };
71.    
72.                     sampler2D _MainTex;
73.                     sampler2D _AmbientOcclusion;
74.    
75.                     v2f vertFunction(appdata IN)
76.                     {
77.                         v2f OUT;
78.    
79.                         OUT.position = UnityObjectToClipPos(IN.vertex);
80.                         OUT.texcoord = IN.uv;
81.    
82.                         return OUT;
83.                     }
84.    
85.                     fixed4 frag(v2f IN) : SV_Target
86.                     {
87.                         fixed4 mainTextureColor = tex2D(_MainTex, IN.texcoord);
88.                      
89.                         fixed4 endColor = mainTextureColor;
90.                         return endColor;
91.                     }
92.                     ENDCG
93.                 }
94.              
95.             }
96.         }
97.  

Perhaps the following link is of interest:
https://forum.unity3d.com/threads/free-open-source-outline-effect.314362/

 

Thanks! Is there anywhere you could recommend I can go to learn more about shaders?

 

The following link has been posted a few times, so I guess it's useful:
http://www.alanzucconi.com/2015/06/10/a-gentle-introduction-to-shaders-in-unity3d/

 

Thanks!

 

Alan Zucconi's intro tutorials are great for getting a good overview of Unity shaders and the basics of the shader rendering pipeline.

For the specific case of doing outlines in a shader that requires an understanding of vertex normals and projection transform matrices, which I honestly don't know a lot of great tutorials on. Most of the ones I've read are either too low level ("just copy these lines of code, and magic happens!"), or too high level ("now we express the result of the transform matrix A upon vector B, in the given Eigenbasis" *a bunch of inscrutable math notation*).

Unfortunately matrices are one of those things you just kind of have to crunch through the basics of if you're going to get into some kinds of shader programming. You could do something like Khan Academy if you want to really understand them, or try Catlike Coding's overview of the basics, but even these might be too in-depth starting out.

The basics of transform matrices when it comes to rendering is matrices are used for moving from one space to another. The easiest way I can think to understand this is parenting objects in Unity and using Local space transform gizmos. The transform component for a gameobject always shows it's position relative to it's parent. In rendering you effectively have 4 "spaces" you're dealing with. Local model (aka object or mesh) space, world space, view (aka camera) space, and clip (aka projection or perspective) space. The various matrices Unity uses in its shaders and the functions it provides are used to move positions and vectors (directions) from one space to another.

UNITY_MATRIX_M or unity_ObjectToWorld are used to move the model space vertex positions into world space.
UNITY_MATRIX_V or unity_WorldToCamera are used to move the world space positions into view space or camera space Note: these are slightly different and not interchangable! The first is explicitly for rendering and ignores the camera scale, the later is for matching more what you would get from a gameObject transform.
UNITY_MATRIX_P or unity_CameraProjection are used to move from view or camera space to projection space. Again, these are not interchangeable, with the first one used for rendering and the second used to match the values you get from script.

Unity also provides combination matrices, or use functions that apply these matrices in an optimized way. For example the old UNITY_MATRIX_MVP was used to go straight from model space vertex positions directly to projection space. They replaced that with the UnityObjectToClipPos() function, which actually works by applying unity_ObjectToWorld then UNITY_MATRIX_VP for complicated reasons.

Your original shader was using UnityObjectToClipPos() to calculate the final clip space position, then trying to add to that a view space direction. These don't match, so you were getting gobbly gook. You could have done it by getting the view space vertex position, adding the view space normal direction, then transforming that into clip space using mul(UNITY_MATRIX_P, viewPos), or you could do what Unity's outline shader does which is calculate the projection space normal direction and add that to the projection space vertex position.

 

Can you please expand upon the two techniques you mentioned here. Unity's editor selected gameobject shader is perfect for my needs, but i can't seem to find it or create a version that works in Diffuse rendering mode that doesn't suffer from the normal's pushing out too far problem. (couldn't get stencil buffer or GrabPass to work)

The google search history on outline shaders goes back half a decade plus, yet there's no good shader based solution out there (that i can understand or find)

Sadly for me the camera based render/command buffer effects out there don't play well with VR headset rendering for various reasons that i can't seem to correct so i'm unable to use them.

 

The selection outline effect in the scene view is done by rendering the selected object into a separate render texture, blurring it in a few shader passes, plus a bunch more steps. It doesn't use the mesh push at all.
https://forum.unity3d.com/threads/selection-outline.429292/#post-2776318

There are both free and paid assets on the store that do something similar, like this one:
https://forum.unity3d.com/threads/free-open-source-outline-effect.314362/

However these all use command buffers and / or extra cameras(!) they won't play nice with VR. Technically command buffers and VR work fine, but using render textures and render targets can be a bit of a headache.

What I was referring to was Unity's Standard Assets Toon shader.

Which Unity's toon shader outline suffers from as well. The only way to solve that is by modifying the content!

Toony Colors Pro 2 has some utilities to help with that.
https://www.assetstore.unity3d.com/en/#!/content/8105

See the "Robot Kyle" demo on this page, but note that the technique he uses to "fix" the problem on hard edged geometry prevents normal mapping from working!
http://jeanmoreno.com/unity/toonycolorspro/

The way I did outlines for both Wayward Sky and Dino Frontier is by just rendering the object again with a shader that does a sobel filter on the depth texture. It can't do an outline outside of the object's edge, only inside, and it doesn't show through walls, but it's a lot easier to implement and works on any solid geometry with out needing any special setup.

 

Is it possible to access the coordinate world space position of the outlining in c sharp?