Posted by misha on
URL: https://forum.jogamp.org/wrong-depth-values-when-rendering-in-GL-SELECT-mode-tp4025804.html

I am using `GL_SELECT` to perform picking. Everything works fine, objects are correctly picked. However there is a problem with z coordinates of picked objects. For some reason they all are equal to `-2147483648`. Thus when several objects are picked I can not determine which one is nearer.

I have following code:

    private static final int PICK_BUFFER_SIZE = 512;

    private int[] pickBuffer = new int[PICK_BUFFER_SIZE];

    private IntBuffer pickGlBuffer = java.nio.IntBuffer.wrap(pickBuffer);

    public void init(GLAutoDrawable drawable) {
        glu = new GLU();
        GL2 g2 = drawable.getGL().getGL2();

        { // options
            g2.glEnable(GL.GL_DEPTH_TEST);
            g2.glDepthFunc(GL.GL_LEQUAL);
            g2.glShadeModel(GLLightingFunc.GL_SMOOTH);
            g2.glHint(GL2ES1.GL_PERSPECTIVE_CORRECTION_HINT, GL.GL_NICEST);
            g2.glClearColor(0f, 0f, 0f, 1f);
        }

    }

    public void display(final GLAutoDrawable drwabale) {
        GL2 g2 = drwabale.getGL().getGL2();
        g2.glClear(GL.GL_COLOR_BUFFER_BIT | GL.GL_DEPTH_BUFFER_BIT);
        Arrays.fill(pickBuffer, 0); // just for debug
        g2.glSelectBuffer(PICK_BUFFER_SIZE, pickGlBuffer);

        { // selection
            g2.glRenderMode(GL2.GL_SELECT);
            g2.glInitNames();
            g2.glPushName(-1);

            setProjections(g2, GL2.GL_SELECT);

            doDraw(g2);
        }

        int hits = g2.glRenderMode(GL2.GL_RENDER);
        processPicking(hits, pickBuffer);

        { // actual drawing
            setProjections(g2, GL2.GL_RENDER);

            doDraw(g2);
        }
    }

    private void setProjections(final GL2 g2, final int renderMode) {
        g2.glMatrixMode(GLMatrixFunc.GL_PROJECTION);
        g2.glLoadIdentity();
        if (renderMode == GL2.GL_SELECT) {
            glu.gluPickMatrix((double) currentMouseX, (double) (viewPort[3] - currentMouseY), 3, 3, viewPort, 0);
        }
        double width = glCanvas.getWidth(), height = glCanvas.getHeight();
        glu.gluPerspective(45, width / height, 1, 1000);

        {
            double yawRad = getYawAngle() / 180 * Math.PI;
            double pitchRad = getPitchAngle() / 180 * Math.PI;
            double vd = getViewerDistance();
            double vx = Math.sin(pitchRad) * Math.sin(yawRad) * vd;
            double vy = Math.cos(pitchRad) * vd;
            double vz = Math.sin(pitchRad) * Math.cos(yawRad) * vd;
            glu.gluLookAt(
                    vx, vy, vz,
                    0, 0, 0,
                    0, 1, 0);
        }

        g2.glMatrixMode(GLMatrixFunc.GL_MODELVIEW);
        g2.glLoadIdentity();
    }

    private void doDraw(final GL2 g2) {
        if (isAxesPainted()) {
            drawAxes(g2);
        }

        drawPlanes(g2);
    }

    private void drawPlanes(final GL2 g2) {
        g2.glColor3d(1, 1, 1);
        g2.glPolygonMode(GL.GL_FRONT_AND_BACK, GL2GL3.GL_LINE);
        { // xy plane
            g2.glLoadName(XY_PLANE_NAME);
            g2.glBegin(GL2.GL_QUADS);
                g2.glTexCoord2d(0, 0);
                g2.glVertex3d(-xBound / 2, -yBound / 2, xyPosition);
                g2.glTexCoord2d(1, 0);
                g2.glVertex3d(xBound / 2, -yBound / 2, xyPosition);
                g2.glTexCoord2d(1, 1);
                g2.glVertex3d(xBound / 2, yBound / 2, xyPosition);
                g2.glTexCoord2d(0, 1);
                g2.glVertex3d(-xBound / 2, yBound / 2, xyPosition);
            g2.glEnd();
        }
        { // xz plane
            g2.glLoadName(XZ_PLANE_NAME);
            g2.glBegin(GL2.GL_QUADS);
                g2.glTexCoord2d(0, 0);
                g2.glVertex3d(-xBound / 2, xzPosition, -zBound / 2);
                g2.glTexCoord2d(1, 0);
                g2.glVertex3d(xBound / 2, xzPosition, -zBound / 2);
                g2.glTexCoord2d(1, 1);
                g2.glVertex3d(xBound / 2, xzPosition, zBound / 2);
                g2.glTexCoord2d(0, 1);
                g2.glVertex3d(-xBound / 2, xzPosition, yBound / 2);
            g2.glEnd();
        }
        { // yz plane
            g2.glLoadName(YZ_PLANE_NAME);
            g2.glBegin(GL2.GL_QUADS);
                g2.glTexCoord2d(0, 0);
                g2.glVertex3d(yzPosition, yBound / 2, zBound / 2);
                g2.glTexCoord2d(1, 0);
                g2.glVertex3d(yzPosition, yBound / 2, -zBound / 2);
                g2.glTexCoord2d(1, 1);
                g2.glVertex3d(yzPosition, -yBound / 2, -zBound / 2);
                g2.glTexCoord2d(0, 1);
                g2.glVertex3d(yzPosition, -yBound / 2, zBound / 2);
            g2.glEnd();
        }
    }