wrong depth values when rendering in GL_SELECT mode

I am using `GL_SELECT` to perform picking. Everything works fine, objects are correctly picked. However there is a problem with z coordinates of picked objects. For some reason they all are equal to `-2147483648`. Thus when several objects are picked I can not determine which one is nearer.

I have following code:

    private static final int PICK_BUFFER_SIZE = 512;

    private int[] pickBuffer = new int[PICK_BUFFER_SIZE];

    private IntBuffer pickGlBuffer = java.nio.IntBuffer.wrap(pickBuffer);

    public void init(GLAutoDrawable drawable) {
        glu = new GLU();
        GL2 g2 = drawable.getGL().getGL2();

        { // options
            g2.glEnable(GL.GL_DEPTH_TEST);
            g2.glDepthFunc(GL.GL_LEQUAL);
            g2.glShadeModel(GLLightingFunc.GL_SMOOTH);
            g2.glHint(GL2ES1.GL_PERSPECTIVE_CORRECTION_HINT, GL.GL_NICEST);
            g2.glClearColor(0f, 0f, 0f, 1f);
        }

    }

    public void display(final GLAutoDrawable drwabale) {
        GL2 g2 = drwabale.getGL().getGL2();
        g2.glClear(GL.GL_COLOR_BUFFER_BIT | GL.GL_DEPTH_BUFFER_BIT);
        Arrays.fill(pickBuffer, 0); // just for debug
        g2.glSelectBuffer(PICK_BUFFER_SIZE, pickGlBuffer);

        { // selection
            g2.glRenderMode(GL2.GL_SELECT);
            g2.glInitNames();
            g2.glPushName(-1);

            setProjections(g2, GL2.GL_SELECT);

            doDraw(g2);
        }

        int hits = g2.glRenderMode(GL2.GL_RENDER);
        processPicking(hits, pickBuffer);

        { // actual drawing
            setProjections(g2, GL2.GL_RENDER);

            doDraw(g2);
        }
    }

    private void setProjections(final GL2 g2, final int renderMode) {
        g2.glMatrixMode(GLMatrixFunc.GL_PROJECTION);
        g2.glLoadIdentity();
        if (renderMode == GL2.GL_SELECT) {
            glu.gluPickMatrix((double) currentMouseX, (double) (viewPort[3] - currentMouseY), 3, 3, viewPort, 0);
        }
        double width = glCanvas.getWidth(), height = glCanvas.getHeight();
        glu.gluPerspective(45, width / height, 1, 1000);

        {
            double yawRad = getYawAngle() / 180 * Math.PI;
            double pitchRad = getPitchAngle() / 180 * Math.PI;
            double vd = getViewerDistance();
            double vx = Math.sin(pitchRad) * Math.sin(yawRad) * vd;
            double vy = Math.cos(pitchRad) * vd;
            double vz = Math.sin(pitchRad) * Math.cos(yawRad) * vd;
            glu.gluLookAt(
                    vx, vy, vz,
                    0, 0, 0,
                    0, 1, 0);
        }

        g2.glMatrixMode(GLMatrixFunc.GL_MODELVIEW);
        g2.glLoadIdentity();
    }

    private void doDraw(final GL2 g2) {
        if (isAxesPainted()) {
            drawAxes(g2);
        }

        drawPlanes(g2);
    }

    private void drawPlanes(final GL2 g2) {
        g2.glColor3d(1, 1, 1);
        g2.glPolygonMode(GL.GL_FRONT_AND_BACK, GL2GL3.GL_LINE);
        { // xy plane
            g2.glLoadName(XY_PLANE_NAME);
            g2.glBegin(GL2.GL_QUADS);
                g2.glTexCoord2d(0, 0);
                g2.glVertex3d(-xBound / 2, -yBound / 2, xyPosition);
                g2.glTexCoord2d(1, 0);
                g2.glVertex3d(xBound / 2, -yBound / 2, xyPosition);
                g2.glTexCoord2d(1, 1);
                g2.glVertex3d(xBound / 2, yBound / 2, xyPosition);
                g2.glTexCoord2d(0, 1);
                g2.glVertex3d(-xBound / 2, yBound / 2, xyPosition);
            g2.glEnd();
        }
        { // xz plane
            g2.glLoadName(XZ_PLANE_NAME);
            g2.glBegin(GL2.GL_QUADS);
                g2.glTexCoord2d(0, 0);
                g2.glVertex3d(-xBound / 2, xzPosition, -zBound / 2);
                g2.glTexCoord2d(1, 0);
                g2.glVertex3d(xBound / 2, xzPosition, -zBound / 2);
                g2.glTexCoord2d(1, 1);
                g2.glVertex3d(xBound / 2, xzPosition, zBound / 2);
                g2.glTexCoord2d(0, 1);
                g2.glVertex3d(-xBound / 2, xzPosition, yBound / 2);
            g2.glEnd();
        }
        { // yz plane
            g2.glLoadName(YZ_PLANE_NAME);
            g2.glBegin(GL2.GL_QUADS);
                g2.glTexCoord2d(0, 0);
                g2.glVertex3d(yzPosition, yBound / 2, zBound / 2);
                g2.glTexCoord2d(1, 0);
                g2.glVertex3d(yzPosition, yBound / 2, -zBound / 2);
                g2.glTexCoord2d(1, 1);
                g2.glVertex3d(yzPosition, -yBound / 2, -zBound / 2);
                g2.glTexCoord2d(0, 1);
                g2.glVertex3d(yzPosition, -yBound / 2, zBound / 2);
            g2.glEnd();
        }
    }

Hi

Rather use this example:
https://github.com/sgothel/jogl-demos/blob/master/src/demos/misc/Picking.java

I assume you used a C/C++ source code to write your own one and you forgot a few subtle things about Java. I don't see your method processPicking(). Anyway, OpenGL picking is a deprecated feature, you should implement your own software picking.

Edit.: You probably did this mistake I fixed some months ago:
https://github.com/sgothel/jogl-demos/commit/d1830d4ccd8c2db30896e987638228c45e251564

I have seen the first example and mistake is not with pick processing or signed nature of java ints. For some reason z coordinates returned to select buffer are all the same. Thus I can not check what object is nearer.

Problem is reproducible on win xp 32 bit. I tried running this code on win 7 64 bit and there coordinates are correct, so I can properly sort objects. I don't know where problem is: in jogl, in opengl drivers, etc. And I don't know how further investigate this problem :(.

If anyone has any clue I would be very grateful!

misha wrote

I have seen the first example and mistake is not with pick processing or
signed nature of java ints. For some reason z coordinates returned to
select buffer are all the same. Thus I can not check what object is nearer.

Problem is reproducible on win xp 32 bit. I tried running this code on win
7 64 bit and there coordinates are correct, so I can properly sort objects.
I don't know where problem is: in jogl, in opengl drivers, etc. And I don't
know how further investigate this problem :(.

If anyone has any clue I would be very grateful!

You use some basic OpenGL methods in this example (not an high level helper class) and JOGL is mostly a straight Java binding for the OpenGL API. You say that your bug is only reproducible under Windows XP 32 bits, it is probably a driver bug. Try to update your driver but anyway, OpenGL picking should NOT be used anymore:
http://www.opengl.org/wiki/Common_Mistakes

A modern OpenGL program should not use the selection buffer or feedback mode. These are not 3D graphics rendering features yet they have been added to GL since version 1.0. Selection and feedback runs in software (CPU side). On some implementations, when used along with VBOs, it has been reported that performance is lousy.

A modern OpenGL program should do color picking (render each object with some unique color and glReadPixels to find out what object your mouse was on) or do the picking with some 3rd party mathematics library.

Color picking does not work on some machines simulating some colors of their palette, that's why the last option is the best.