"Smudge" post process

I'm trying to replicate the smudging effect as seen here:

The current effect I've got works well against the camera's far plane provided the way I interpolate depth. However the closer two objects are the less of an impact the effect has. The only method that immediately comes to mind is to split the 32 bit float alpha value to contain both the depth and the time that the smudge will fade into the background.

Here's the current code:
PostProcessEffect.cs -

Code (CSharp):

1. using UnityEngine;
2. using System.Collections;
3.  
4. [RequireComponent(typeof(Camera))]
5. public class PostProcessEffect : MonoBehaviour
6. {  
7.     private Shader shader = null;
8.     private Material material = null;
9.    
10.     private Matrix4x4 viewProj = Matrix4x4.identity;
11.    
12.     private Camera camera;
13.    
14.     private RenderTexture lastFrame;
15.    
16.     public void Awake()
17.     {
18.         Graphics.Blit(Texture2D.blackTexture, lastFrame);
19.        
20.         shader = Shader.Find("PostProcessEffect");
21.        
22.         if(!shader.isSupported)
23.         {
24.             enabled = false;
25.            
26.             return;
27.         }
28.        
29.         material = new Material(shader);
30.         material.hideFlags = HideFlags.DontSave;
31.        
32.         camera = GetComponent<Camera>();
33.        
34.         camera.depthTextureMode = DepthTextureMode.DepthNormals;
35.        
36.         lastFrame = RenderTexture.GetTemporary(Screen.width, Screen.height);
37.        
38.         viewProj = camera.worldToCameraMatrix.inverse * camera.projectionMatrix;
39.     }
40.    
41.     public void OnDestroy()
42.     {
43.         RenderTexture.ReleaseTemporary(lastFrame);
44.     }
45.    
46.     public void OnRenderImage(RenderTexture source, RenderTexture destination)
47.     {
48.         Matrix4x4 VP = camera.worldToCameraMatrix.inverse * camera.projectionMatrix;
49.        
50.         material.SetMatrix("_UNITY_VP_INV", VP.inverse);
51.         material.SetMatrix("_UNITY_VP_prev", viewProj);
52.        
53.         viewProj = VP;
54.        
55.         material.SetTexture("_LastFrame", lastFrame);
56.        
57.         Graphics.Blit(source, destination, material);
58.         Graphics.Blit(destination, lastFrame);
59.     }
60. }

PostProcessEffect.shader -

Code (Shader):

1. Shader "PostProcessEffect"
2. {
3.     Properties
4.     {
5.         _MainTex("Base (RGB)", 2D) = "white" {}
6.     }
7.  
8.     SubShader
9.     {      
10.         ZTest Always Cull Off ZWrite Off
11.         Fog { Mode off }
12.  
13.         Pass
14.         {
15.             CGPROGRAM
16.  
17.             #include "UnityCG.cginc"
18.  
19.             #pragma target 3.0
20.             #pragma vertex vert_img
21.             #pragma fragment frag
22.  
23.             uniform sampler2D _MainTex;
24.             uniform float4 _MainTex_TexelSize;
25.  
26.             uniform sampler2D _CameraDepthNormalsTexture;
27.  
28.             uniform sampler2D _LastFrame;
29.  
30.             uniform float4x4 _UNITY_VP_INV;
31.             uniform float4x4 _UNITY_VP_prev;
32.  
33.             float4 frag(v2f_img i) : SV_Target
34.             {
35.                 float3 currentColor = tex2D(_MainTex, i.uv).rgb;
36.                 float3 previousColor = float3(0,0,0);
37.  
38.                 float3 norm;
39.                 float depth[3];
40.  
41.                 DecodeDepthNormal(tex2D(_CameraDepthNormalsTexture, float2(i.uv.x, 1-i.uv.y)), depth[0], norm);
42.  
43.                 // Depth values, 1=near, 0=far
44.                 depth[0] = 1.0 - depth[0];            
45.                 depth[1] = tex2D(_LastFrame, i.uv).a;
46.  
47.                 // depth[2] = { min, max }, overwrite with new depths if greater, or if distance is less than epsilon
48.                 if(depth[0] > depth[1] || depth[1] - depth[0] < 0.01)
49.                 {
50.                     float tmp = depth[0];
51.                     depth[0] = depth[1];
52.                     depth[1] = tmp;
53.  
54.                     previousColor = currentColor;
55.                 }
56.                 else
57.                 {
58.                     // Inverse transform from view space to world space, then from world space back to the previous frame's view space
59.                     float4 currentPos = float4(i.uv.x * 2.0 - 1.0, i.uv.y * 2.0 - 1.0, 1.0 - depth[0], 1.0);
60.                     float4 worldPos = mul(currentPos, _UNITY_VP_INV); worldPos /= worldPos.w;
61.                     float4 previousPos = mul(worldPos, _UNITY_VP_prev); previousPos /= previousPos.w;
62.  
63.                     // Blur
64.  
65.                     float2 velocity = (previousPos.xy - currentPos.xy) * 0.1;
66.  
67.                     previousColor += tex2D(_LastFrame, i.uv + velocity).rgb;
68.                     previousColor += tex2D(_LastFrame, i.uv + velocity * 0.75).rgb;
69.                     previousColor += tex2D(_LastFrame, i.uv + velocity * 0.5).rgb;
70.                     previousColor += tex2D(_LastFrame, i.uv + velocity * 0.25).rgb;                
71.                     previousColor += tex2D(_LastFrame, i.uv).rgb;
72.  
73.                     previousColor *= 0.2;
74.                 }
75.  
76.                 float delta = unity_DeltaTime.x * exp(abs(depth[0] - depth[1])) * 3.0;
77.  
78.                 return float4(lerp(previousColor.rgb, currentColor.rgb, delta), lerp(depth[1], depth[0], delta));
79.             }
80.  
81.             ENDCG  
82.         }
83.     }
84. }

And video that shows how it functions against the far plane vs closer objects:

How do you figure to rid of these artifacts?