I'm using LibGDX to render 2d fog-of-war type functionality. This involves drawing a dark rectangle over the entire map with transparent holes in it where you can see the map below. I'm attempting to use OpenGl stencil buffer to create the circular masks, but I can't seem to get the logic correct.
The code below correctly draws the dark rectangle (the fog) but the circular masks are not being stenciled. i.e. the entire map is dark.
batch.end();
Gdx.gl.glClear(GL_STENCIL_BUFFER_BIT);
Gdx.gl.glEnable(GL20.GL_STENCIL_TEST);
Gdx.gl.glColorMask(false, false, false, false);
//always write the clipped holes to the stencil
Gdx.gl.glStencilFunc(GL20.GL_ALWAYS, 0x1, 0xffffffff);
Gdx.gl.glStencilMask(0xFF);
Gdx.gl.glStencilOp(GL_REPLACE, GL_REPLACE, GL_REPLACE);
shapeRenderer.begin(ShapeRenderer.ShapeType.Filled);
shapeRenderer.setProjectionMatrix(gameUi.camera.combined);
//test circle
shapeRenderer.circle(0, 0, 1000, 100);
shapeRenderer.end();
Gdx.gl.glEnable(GL20.GL_BLEND);
Gdx.gl.glBlendFunc(GL20.GL_SRC_ALPHA, GL20.GL_ONE_MINUS_SRC_ALPHA);
Gdx.gl.glColorMask(true, true, true, true);
Gdx.gl.glEnable(GL20.GL_STENCIL_TEST);
//only draw the fog of war where stencil is 0
Gdx.gl.glStencilFunc(GL_EQUAL, 0x0, 0xffffffff);
Gdx.gl.glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
//draw fog of war
shapeRenderer.begin(ShapeRenderer.ShapeType.Filled);
shapeRenderer.setProjectionMatrix(gameUi.camera.combined);
shapeRenderer.setColor(radarFog);
int dimension = gameUi.mapConfig.getDimension();
shapeRenderer.rect(-dimension,-dimension,dimension*3,dimension*3);
shapeRenderer.end();
Gdx.gl.glDisable(GL20.GL_STENCIL_TEST);
batch.begin();