Fix rendering of ray-casted primitives in orthographic projection mode

.polyscope.ini

@@ -0,0 +1,7 @@
+{
+    "uiScale": 1.0,
+    "windowHeight": 1352,
+    "windowPosX": 1283,
+    "windowPosY": 35,
+    "windowWidth": 2554
+}

src/render/opengl/shaders/common.cpp

@@ -179,6 +179,45 @@ vec3 fragmentViewPosition(vec4 viewport, vec2 depthRange, mat4 invProjMat, vec4
   return eyePos.xyz / eyePos.w;
 }
 
+// Detect if the projection matrix is orthographic.
+// In an orthographic projection matrix, projMat[3][3] == 1.0 and projMat[2][3] == 0.0.
+// In a perspective projection matrix, projMat[3][3] == 0.0 and projMat[2][3] == -1.0.
+bool isOrthoProjection(mat4 projMat) {
+  return abs(projMat[3][3] - 1.0) < 0.001;
+}
+
+// Build ray start and ray direction for fragment-based raycasting.
+// For perspective projection: rays originate from the camera (origin) and point towards the fragment view position.
+// For orthographic projection: rays are parallel, all pointing in -Z direction in view space,
+//                              starting from the fragment's XY position.
+void buildRayForFragment(vec4 viewport, vec2 depthRange, mat4 projMat, mat4 invProjMat, vec4 fragCoord,
+                         out vec3 rayStart, out vec3 rayDir) {
+
+  if (isOrthoProjection(projMat)) {
+    // Orthographic: parallel rays pointing in -Z direction
+    // Convert fragment screen position to NDC
+    vec2 ndcXY = ((2.0 * fragCoord.xy) - (2.0 * viewport.xy)) / (viewport.zw) - 1.0;
+
+    // Unproject to view space at near plane (NDC z = -1) and far plane (NDC z = 1)
+    // For orthographic, we just need the XY in view space
+    vec4 ndcNear = vec4(ndcXY, -1.0, 1.0);
+    vec4 viewNear = invProjMat * ndcNear;
+    viewNear /= viewNear.w;
+
+    vec4 ndcFar = vec4(ndcXY, 1.0, 1.0);
+    vec4 viewFar = invProjMat * ndcFar;
+    viewFar /= viewFar.w;
+
+    rayStart = viewNear.xyz;
+    rayDir = normalize(viewFar.xyz - viewNear.xyz);
+  } else {
+    // Perspective: rays from origin through fragment position
+    vec3 viewPos = fragmentViewPosition(viewport, depthRange, invProjMat, fragCoord);
+    rayStart = vec3(0.0, 0.0, 0.0);
+    rayDir = viewPos;
+  }
+}
+
 float fragDepthFromView(mat4 projMat, vec2 depthRange, vec3 viewPoint) {
   vec4 clipPos = projMat * vec4(viewPoint, 1.); // only actually need one element of this result, could save work
   float z_ndc = clipPos.z / clipPos.w;
@@ -206,6 +245,10 @@ bool raySphereIntersection(vec3 rayStart, vec3 rayDir, vec3 sphereCenter, float
     }
 }
 
+)"
+// Split the raw string literal to avoid compiler string length limits
+R"(
+
 bool rayPlaneIntersection(vec3 rayStart, vec3 rayDir, vec3 planePos, vec3 planeDir, out float tHit, out vec3 pHit, out vec3 nHit) {
 
   float num = dot(planePos - rayStart, planeDir);

src/render/opengl/shaders/cylinder_shaders.cpp

@@ -164,6 +164,7 @@ R"(
 
         float LARGE_FLOAT();
         vec3 fragmentViewPosition(vec4 viewport, vec2 depthRange, mat4 invProjMat, vec4 fragCoord);
+        void buildRayForFragment(vec4 viewport, vec2 depthRange, mat4 projMat, mat4 invProjMat, vec4 fragCoord, out vec3 rayStart, out vec3 rayDir);
         bool rayTaperedCylinderIntersection(vec3 rayStart, vec3 rayDir, vec3 cylTail, vec3 cylTip, float cylRadTail, float cylRadTip, out float tHit, out vec3 pHit, out vec3 nHit);
         float fragDepthFromView(mat4 projMat, vec2 depthRange, vec3 viewPoint);
 
@@ -173,7 +174,8 @@ R"(
         {
            // Build a ray corresponding to this fragment
            vec2 depthRange = vec2(gl_DepthRange.near, gl_DepthRange.far);
-           vec3 viewRay = fragmentViewPosition(u_viewport, depthRange, u_invProjMatrix, gl_FragCoord);
+           vec3 rayStart, rayDir;
+           buildRayForFragment(u_viewport, depthRange, u_projMatrix, u_invProjMatrix, gl_FragCoord, rayStart, rayDir);
 
 
            float tipRadius = u_radius;
@@ -184,7 +186,7 @@ R"(
            float tHit;
            vec3 pHit;
            vec3 nHit;
-           rayTaperedCylinderIntersection(vec3(0., 0., 0), viewRay, tailView, tipView, tailRadius, tipRadius, tHit, pHit, nHit);
+           rayTaperedCylinderIntersection(rayStart, rayDir, tailView, tipView, tailRadius, tipRadius, tHit, pHit, nHit);
            if(tHit >= LARGE_FLOAT()) {
               discard;
            }

src/render/opengl/shaders/sphere_shaders.cpp

@@ -73,6 +73,7 @@ R"(
         ${ GEOM_DECLARATIONS }$
 
         void buildTangentBasis(vec3 unitNormal, out vec3 basisX, out vec3 basisY);
+        bool isOrthoProjection(mat4 projMat);
 
         void main() {
 
@@ -82,7 +83,14 @@ R"(
             // Construct the 4 corners of a billboard quad, facing the camera
             // Quad is shifted pointRadius toward the camera, otherwise it doesn't actually necessarily
             // cover the full sphere due to perspective.
-            vec3 dirToCam = normalize(-gl_in[0].gl_Position.xyz);
+            vec3 dirToCam;
+            if (isOrthoProjection(u_projMatrix)) {
+                // Orthographic: camera direction is constant +Z in view space
+                dirToCam = vec3(0.0, 0.0, 1.0);
+            } else {
+                // Perspective: camera is at origin, direction varies per point
+                dirToCam = normalize(-gl_in[0].gl_Position.xyz);
+            }
             vec3 basisX;
             vec3 basisY;
             buildTangentBasis(dirToCam, basisX, basisY);
@@ -142,6 +150,7 @@ R"(
         float LARGE_FLOAT();
         vec3 lightSurfaceMat(vec3 normal, vec3 color, sampler2D t_mat_r, sampler2D t_mat_g, sampler2D t_mat_b, sampler2D t_mat_k);
         vec3 fragmentViewPosition(vec4 viewport, vec2 depthRange, mat4 invProjMat, vec4 fragCoord);
+        void buildRayForFragment(vec4 viewport, vec2 depthRange, mat4 projMat, mat4 invProjMat, vec4 fragCoord, out vec3 rayStart, out vec3 rayDir);
         bool raySphereIntersection(vec3 rayStart, vec3 rayDir, vec3 sphereCenter, float sphereRad, out float tHit, out vec3 pHit, out vec3 nHit);
         float fragDepthFromView(mat4 projMat, vec2 depthRange, vec3 viewPoint);
 
@@ -151,7 +160,8 @@ R"(
         {
            // Build a ray corresponding to this fragment
            vec2 depthRange = vec2(gl_DepthRange.near, gl_DepthRange.far);
-           vec3 viewRay = fragmentViewPosition(u_viewport, depthRange, u_invProjMatrix, gl_FragCoord);
+           vec3 rayStart, rayDir;
+           buildRayForFragment(u_viewport, depthRange, u_projMatrix, u_invProjMatrix, gl_FragCoord, rayStart, rayDir);
 
            float pointRadius = u_pointRadius;
            ${ SPHERE_SET_POINT_RADIUS_FRAG }$
@@ -160,7 +170,7 @@ R"(
            float tHit;
            vec3 pHit;
            vec3 nHit;
-           bool hit = raySphereIntersection(vec3(0., 0., 0), viewRay, sphereCenterView, pointRadius, tHit, pHit, nHit);
+           bool hit = raySphereIntersection(rayStart, rayDir, sphereCenterView, pointRadius, tHit, pHit, nHit);
            if(tHit >= LARGE_FLOAT()) {
               discard;
            }
@@ -258,14 +268,22 @@ R"(
         ${ GEOM_DECLARATIONS }$
 
         void buildTangentBasis(vec3 unitNormal, out vec3 basisX, out vec3 basisY);
+        bool isOrthoProjection(mat4 projMat);
 
         void main() {
 
             float pointRadius = u_pointRadius;
             ${ SPHERE_SET_POINT_RADIUS_GEOM }$
 
             // Construct the 4 corners of a billboard quad, facing the camera
-            vec3 dirToCam = normalize(-gl_in[0].gl_Position.xyz);
+            vec3 dirToCam;
+            if (isOrthoProjection(u_projMatrix)) {
+                // Orthographic: camera direction is constant +Z in view space
+                dirToCam = vec3(0.0, 0.0, 1.0);
+            } else {
+                // Perspective: camera is at origin, direction varies per point
+                dirToCam = normalize(-gl_in[0].gl_Position.xyz);
+            }
             vec3 basisX;
             vec3 basisY;
             buildTangentBasis(dirToCam, basisX, basisY);

src/render/opengl/shaders/vector_shaders.cpp

@@ -218,6 +218,7 @@ R"(
 
         float LARGE_FLOAT();
         vec3 fragmentViewPosition(vec4 viewport, vec2 depthRange, mat4 invProjMat, vec4 fragCoord);
+        void buildRayForFragment(vec4 viewport, vec2 depthRange, mat4 projMat, mat4 invProjMat, vec4 fragCoord, out vec3 rayStart, out vec3 rayDir);
         bool rayCylinderIntersection(vec3 rayStart, vec3 rayDir, vec3 cylTail, vec3 cylTip, float cylRad, out float tHit, out vec3 pHit, out vec3 nHit);
         bool rayConeIntersection(vec3 rayStart, vec3 rayDir, vec3 coneBase, vec3 coneTip, float coneRad, out float tHit, out vec3 pHit, out vec3 nHit);
         float fragDepthFromView(mat4 projMat, vec2 depthRange, vec3 viewPoint);
@@ -228,7 +229,8 @@ R"(
         {
            // Build a ray corresponding to this fragment
            vec2 depthRange = vec2(gl_DepthRange.near, gl_DepthRange.far);
-           vec3 viewRay = fragmentViewPosition(u_viewport, depthRange, u_invProjMatrix, gl_FragCoord);
+           vec3 rayStart, rayDir;
+           buildRayForFragment(u_viewport, depthRange, u_projMatrix, u_invProjMatrix, gl_FragCoord, rayStart, rayDir);
 
            // geometric shape of hte vector
            float tipLengthFrac = 0.2;
@@ -241,13 +243,13 @@ R"(
            vec3 nHit =  vec3(777,777,777);
            vec3 cylEnd = tailView + (1. - tipLengthFrac) * (tipView - tailView);
            float cylRad = tipWidthFrac * adjRadius;
-           rayCylinderIntersection(vec3(0., 0., 0), viewRay, tailView, cylEnd, cylRad, tHit, pHit, nHit);
+           rayCylinderIntersection(rayStart, rayDir, tailView, cylEnd, cylRad, tHit, pHit, nHit);
 
            // Raycast to cone
            float tHitCone;
            vec3 pHitCone;
            vec3 nHitCone;
-           bool coneHit = rayConeIntersection(vec3(0., 0., 0), viewRay, cylEnd, tipView, adjRadius, tHitCone, pHitCone, nHitCone);
+           bool coneHit = rayConeIntersection(rayStart, rayDir, cylEnd, tipView, adjRadius, tHitCone, pHitCone, nHitCone);
            if(tHitCone < tHit) {
              tHit = tHitCone;
              pHit = pHitCone;