Generating rays from cameras with custom projection matrices

Hi there, I'm trying to generate a ray from a camera that has a custom projection matrix applied to it. I'm currently trying using the following code but I'm at the limit of my matrix math abilities so I'm having a hard time debugging whats going wrong.

Camera

Code (csharp):

1.  
2. Camera.main.ResetProjectionMatrix ();
3. Matrix4x4mat = Camera.main.projectionMatrix;
4.  
5. Matrix4x4shearMatrix = Matrix4x4.identity;
6. shearMatrix.SetRow (1, newVector4 (0,1,2,0));
7. mat *= shearMatrix;
8. Camera.main.projectionMatrix = mat;
9.  

Raycast Code

Code (csharp):

1.  
2. Vector3viewportPoint = Camera.main.ScreenToViewportPoint(Input.mousePosition);
3. RaycastHithit;
4. Rayray = Camera.main.ScreenPointToRay(Camera.main.projectionMatrix.inverse * viewportPoint);
5.  
6. if (Physics.Raycast(ray, outhit))
7. {
8. ...
9. }
10.  

I've seen similar questions asked but no answers, does anybody know what I could try next?

 

I ended up getting a genius friend to help and we came up with this

Code (csharp):

1.  
2.     usingUnityEngine;
3.     usingSystem.Collections;
4.  
5.     public classShearCameraMatrix : MonoBehaviour
6.     {
7.         public float shear = 1f;
8.         public float sheerOffsetFactor = 60f;
9.  
10.         Camera_camera;
11.  
12.         Matrix4x4 OriginalProj;
13.         Matrix4x4 ShearMat;
14.  
15.         void Start ()
16.         {
17.             SetSkew ();
18.             _camera = GetComponent<Camera> ();
19.  
20.         }
21.  
22.         void SetSkew ()
23.         {
24.             Camera.main.ResetProjectionMatrix ();
25.             Matrix4x4 mat = Camera.main.projectionMatrix;
26.             OriginalProj = mat;
27.             Matrix4x4 shearMatrix = Matrix4x4.identity;
28.             shearMatrix.SetRow (1, newVector4 (0,1,shear,shear * sheerOffsetFactor));
29.             ShearMat = shearMatrix;
30.             mat *= shearMatrix;
31.  
32.             Camera.main.projectionMatrix = mat;
33.         }
34.  
35.  
36.         Vector3 ScreenToWorld(Vector3screenPos)
37.         {
38.         Vector4 ScreenPosH = screenPos;
39.         ScreenPosH.w = 1.0f;
40.         Vector4 WorldPos = _camera.worldToCameraMatrix.inverse * ShearMat.inverse * OriginalProj.inverse * ScreenPosH;
41.         return WorldPos;
42.         }
43.  
44.  
45.         public RayScreenPointToRay(Vector3screenPos)
46.         {
47.  
48.             Vector3 rasterizedScreen = screenPos;
49.             rasterizedScreen.x /= _camera.pixelWidth;
50.             rasterizedScreen.y /= _camera.pixelHeight;
51.             rasterizedScreen.z = 0.0f;
52.             rasterizedScreen *= 2.0f;
53.             rasterizedScreen -= Vector3.one;
54.  
55.             Vector3screen1 = rasterizedScreen;
56.             screen1.z = 0.0f;
57.             Vector3screen2 = rasterizedScreen;
58.             screen2.z = -1;
59.             Vector3 world1 = ScreenToWorld(screen1);
60.             Vector3 world2 = ScreenToWorld(screen2);
61.             Ray ray = newRay (world2, world1 - world2);
62.  
63.             return ray;
64.         }
65.     }
66.