1

I am currently doing straightforward direct-to-screen (no multiple passes or postprocessing) rendering in WebGL. I would like to determine the average brightness/luminance of the entire rendered image (i.e. a single number), in a way which is efficient enough to do every frame.

What I'm looking to accomplish is to implement “exposure” adjustment (as a video camera or the human eye would) in the scene, so as to view both indoor and outdoor scenes with realistic lighting and no transitions — the brightness of the current frame will be negative feedback to the brightness of the next frame.

I am currently calculating a very rough approximation on the CPU side by sending a few rays through my scene data to find the brightness at those points; this works, but has too few samples to be stable (brightness varies noticeably with view angle as the rays cross light sources). I would prefer to offload the work to the GPU if at all possible, as my application is typically CPU-bound.

3
  • Is it by chance possible to use the information passed to WebGL or do you want an image-based solution? Does the solution need to be exact or is an approximation sufficient? Commented Apr 15, 2012 at 18:25
  • @StefanHanke See edit. Briefly: Approximations fine. Light is easily extracted from the world data but I don't want to add more CPU tasks.
    – Kevin Reid
    Commented Apr 15, 2012 at 21:07
  • 1
    My thoughts were: Using MRT to render to two color attachments with different resolution. Does not work, because WebGL supports only 1 color attachment, and I don't know whether rendering using multiple resolutions is supported even in newer OpenGL versions. Commented Apr 16, 2012 at 4:18

2 Answers 2

3

I just thought of a horrible kludge, namely to render to texture and generateMipmaps on it, then read the smallest level. I hope there's a better way.

What's wrong with that? This way is almost entirely done on the GPU, can be worked nicely into an existing render pipeline, and should give reasonable results. I don't know of any reason to recommend against it.

2
  • What's wrong with it? Well, for example, it does a whole lot of resampling that's utterly irrelevant. But I did claim to be CPU-bound. Guess I'll have to try it.
    – Kevin Reid
    Commented Apr 15, 2012 at 23:45
  • I've now tried implementing it. The basic concept seems to work fine — however, after generateMipmap, the texture immediately becomes black (even if the filters are not set to mipmap). I've made sure the framebuffer is not currently bound when I do the generateMipmap, or the later drawing to another framebuffer in order to readPixels. Removing the generateMipmap produces the results that would be expected. Any suggestions?
    – Kevin Reid
    Commented Apr 25, 2012 at 21:30
3

I know this question is 8 years old but hey....

First off, WebGL1, generateMipmap only works for power of 2 images.

I'd suggest either (1) generating a simple shaders like this

function createShader(texWidth, texHeight) {
  return `
  precision mediump float;
  uniform sampler2D tex;

  void main() {
    vec2 size = vec2(${texWidth}, ${texHeight});
    float totalBrightness = 0.0;
    float minBrightness = 1.0;
    float maxBrightness = 0.0;
    for (int y = 0; y < ${texHeight}; ++y) {
      for (int x = 0; x < ${texWidth}; ++x) {
        vec4 color = texture2D(tex, (vec2(x, y) + 0.5) / size);
        vec3 adjusted = color.rgb * vec3(0.2126, 0.7152, 0.0722);
        float brightness = adjusted.r + adjusted.g + adjusted.b;
        totalBrightness += brightness;
        minBrightness = min(brightness, minBrightness);
        maxBrightness = max(brightness, maxBrightness);
      }
    }
    float averageBrightness = totalBrightness / (size.x * size.y);
    gl_FragColor = vec4(averageBrightness, minBrightness, maxBrightness, 0);
  }
  `;
}

const startElem = document.querySelector('button');
startElem.addEventListener('click', main, {once: true});

function createShader(texWidth, texHeight) {
  return `
  precision mediump float;
  uniform sampler2D tex;

  void main() {
    vec2 size = vec2(${texWidth}, ${texHeight});
    float totalBrightness = 0.0;
    float minBrightness = 1.0;
    float maxBrightness = 0.0;
    for (int y = 0; y < ${texHeight}; ++y) {
      for (int x = 0; x < ${texWidth}; ++x) {
        vec4 color = texture2D(tex, (vec2(x, y) + 0.5) / size);
        vec3 adjusted = color.rgb * vec3(0.2126, 0.7152, 0.0722);
        float brightness = adjusted.r + adjusted.g + adjusted.b;
        totalBrightness += brightness;
        minBrightness = min(brightness, minBrightness);
        maxBrightness = max(brightness, maxBrightness);
      }
    }
    float averageBrightness = totalBrightness / (size.x * size.y);
    gl_FragColor = vec4(averageBrightness, minBrightness, maxBrightness, 0);
  }
  `;
}

const prgs = {}
function getAverageProgram(gl, width, height) {
  const id = `${width}x${height}`;
  const prg = prgs[id];
  if (prg) {
    return prg;
  }
  const vs = `
  attribute vec4 position;
  void main() {
    gl_Position = position;
  }
  `;
  const fs = createShader(width, height);
  // compile shaders, link program, look up uniforms
  const newPrg = twgl.createProgramInfo(gl, [vs, fs]);
  prgs[id] = newPrg;
  return newPrg;
}


function main() {
  const gl = document.querySelector('canvas').getContext('webgl');
  
  let updateTexture = false;
  const video = document.createElement('video');
  video.crossOrigin = 'anonymous';
  video.loop = true;
  video.src = 'https://webglsamples.org/color-adjust/sample-video.mp4';
  if (video.requestVideoFrameCallback) {
    function update() {
      draw();
      video.requestVideoFrameCallback(update);
    };
    video.requestVideoFrameCallback(update);
  } else {
    function update() {
      if (video.currentTime > 0) {
        draw();
      }
      requestAnimationFrame(update);
    }
    requestAnimationFrame(update);
  }
  video.volume = 0;
  video.play();
  
  // create a 1x1 pixel RGBA/UNSIGNED_BYTE framebuffer
  const fbi = twgl.createFramebufferInfo(gl, [
    { internalForamt: gl.RGBA },
  ], 1, 1);
  
  const tVS = `
  attribute vec4 position;
  attribute vec2 texcoord;
  varying vec2 v_texcoord;
  void main() {
    gl_Position = position;
    v_texcoord = texcoord;
  }
  `;
  const tFS = `
  precision mediump float;
  uniform sampler2D tex;
  varying vec2 v_texcoord;
  void main() {
    gl_FragColor = texture2D(tex, v_texcoord);
  }
  `;
  // compile shaders, link program, look up uniforms
  const textureProgInfo = twgl.createProgramInfo(gl, [tVS, tFS]);
  
  const avgMinMaxVS = `
  attribute float id;
  varying float v_id;
  uniform sampler2D avgMinMaxTex;
  void main() {
    vec4 avgMinMax = texture2D(avgMinMaxTex, vec2(0.5));
    float v = id < 1.0
       ? avgMinMax.x
       : id < 2.0
          ? avgMinMax.y
          : avgMinMax.z;
    gl_Position = vec4(1. - (id + 1.0) / 10., v * 2. - 1., 0, 1);
    gl_PointSize = 10.0;
    v_id = id;
  }
  `;
  const avgMinMaxFS = `
  precision mediump float;
  varying float v_id;
  void main() {
    gl_FragColor = vec4(1., v_id / 2., 1. - v_id / 2., 1);
  }
  `;
  // compile shaders, link program, look up uniforms
  const avgMinMaxPrgInfo = twgl.createProgramInfo(gl, [avgMinMaxVS, avgMinMaxFS]);
  
  const planeBufferInfo = twgl.primitives.createXYQuadBufferInfo(gl);
  const idBufferInfo = twgl.createBufferInfoFromArrays(gl, {
    id: {
      data: [0, 1, 2],
      numComponents: 1,
    },
  });
  
  const videoTex = gl.createTexture();
  gl.bindTexture(gl.TEXTURE_2D, videoTex);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
  
  gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);

  function draw() {
    // copy video to texture
    gl.bindTexture(gl.TEXTURE_2D, videoTex);
    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video);
    
    // --- [ compute average, min, max to single pixel ] ---
    
    const averagePrgInfo = getAverageProgram(gl, video.videoWidth, video.videoHeight);
    gl.useProgram(averagePrgInfo.program);
    
    // calls gl.bindFramebuffer and gl.viewport
    twgl.bindFramebufferInfo(gl, fbi);
    
    // calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
    twgl.setBuffersAndAttributes(gl, averagePrgInfo, planeBufferInfo);
    
    // calls gl.drawArrays or gl.drawElements
    twgl.drawBufferInfo(gl, planeBufferInfo);
    
    // --- [ draw video to texture ] ---
    
    // calls gl.bindFramebuffer and gl.viewport
    twgl.bindFramebufferInfo(gl, null);
    
    gl.useProgram(textureProgInfo.program);
    
    // calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
    twgl.setBuffersAndAttributes(gl, textureProgInfo, planeBufferInfo);
    
    // calls gl.drawArrays or gl.drawElements
    twgl.drawBufferInfo(gl, planeBufferInfo);
    
    // -- [ draw 3 points showing avg, min, max] ---
    
    gl.useProgram(avgMinMaxPrgInfo.program);

    gl.bindTexture(gl.TEXTURE_2D, fbi.attachments[0]);
    
    // calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
    twgl.setBuffersAndAttributes(gl, avgMinMaxPrgInfo, idBufferInfo);    

    // calls gl.drawArrays or gl.drawElements
    twgl.drawBufferInfo(gl, idBufferInfo, gl.POINTS);
  }
}
body {
  background: #444;
}
canvas {
  border: 1px solid black;
  display: block;
}
<canvas></canvas>
<button type="button">start</button>
<span style="color: #FF0">■ max brightness</span>
<span style="color: #F80">■ min brightness, </span>
<span style="color: #F0F">■ average brightness, </span>
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>

the only problem with this solution is that it can't be parallelized by the GPU AFAIK so (2) I might test doing something similar to generating mipmaps where I say make a shader that does 16x16 pixels and target it to generate a smaller texture and repeat until I get to 1x1. I'd have to test to see if that's actually faster and what size cell 2x2, 4x4, 16x16 etc is best.

Finally, if possible, like the example above, if I don't actually need the result on the CPU then just pass that 1x1 texture as input to some other shader. The example just draws 3 points but of course you could feed those values into the shader that's drawing the video to do some image processing like crank up the exposure if the brightness is low try to auto level the image based on the min and max brightness etc...

Note that in WebGL2 you wouldn't have to generate a different shader per size as WebGL2, or rather GLSL ES 3.0 you can have loops that are not based on constant values.

const startElem = document.querySelector('button');
startElem.addEventListener('click', main, {once: true});

function main() {
  const gl = document.querySelector('canvas').getContext('webgl2');
  if (!gl) {
    return alert('need WebGL2')
  }
  
  let updateTexture = false;
  const video = document.createElement('video');
  video.crossOrigin = 'anonymous';
  video.loop = true;
  video.src = 'https://webglsamples.org/color-adjust/sample-video.mp4';
  if (video.requestVideoFrameCallback) {
    function update() {
      draw();
      video.requestVideoFrameCallback(update);
    };
    video.requestVideoFrameCallback(update);
  } else {
    function update() {
      if (video.currentTime > 0) {
        draw();
      }
      requestAnimationFrame(update);
    }
    requestAnimationFrame(update);
  }
  video.volume = 0;
  video.play();
  
  // create a 1x1 pixel RGBA/UNSIGNED_BYTE framebuffer
  const fbi = twgl.createFramebufferInfo(gl, [
    { internalForamt: gl.RGBA },
  ], 1, 1);
  
  const avgVS = `#version 300 es
  in vec4 position;
  void main() {
    gl_Position = position;
  }
  `;
  
  const avgFS = `#version 300 es
  precision highp float;
  uniform sampler2D tex;
  out vec4 result;
  void main() {
    ivec2 size = textureSize(tex, 0);
    float totalBrightness = 0.0;
    float minBrightness = 1.0;
    float maxBrightness = 0.0;
    for (int y = 0; y < size.y; ++y) {
      for (int x = 0; x < size.x; ++x) {
        vec4 color = texelFetch(tex, ivec2(x, y), 0);
        vec3 adjusted = color.rgb * vec3(0.2126, 0.7152, 0.0722);
        float brightness = adjusted.r + adjusted.g + adjusted.b;
        totalBrightness += brightness;
        minBrightness = min(brightness, minBrightness);
        maxBrightness = max(brightness, maxBrightness);
      }
    }
    float averageBrightness = totalBrightness / float(size.x * size.y);
    result = vec4(averageBrightness, minBrightness, maxBrightness, 0);
  }
  `;  
  
  // compile shaders, link program, look up uniforms
  const averagePrgInfo = twgl.createProgramInfo(gl, [avgVS, avgFS]);


  const tVS = `#version 300 es
  in vec4 position;
  in vec2 texcoord;
  out vec2 v_texcoord;
  void main() {
    gl_Position = position;
    v_texcoord = texcoord;
  }
  `;
  const tFS = `#version 300 es
  precision mediump float;
  uniform sampler2D tex;
  in vec2 v_texcoord;
  out vec4 fragColor;
  void main() {
    fragColor = texture(tex, v_texcoord);
  }
  `;
  // compile shaders, link program, look up uniforms
  const textureProgInfo = twgl.createProgramInfo(gl, [tVS, tFS]);
  
  const avgMinMaxVS = `#version 300 es
  out float v_id;
  uniform sampler2D avgMinMaxTex;
  void main() {
    vec4 avgMinMax = texelFetch(avgMinMaxTex, ivec2(0), 0);
    float v = gl_VertexID == 0
       ? avgMinMax.x
       : gl_VertexID == 1
          ? avgMinMax.y
          : avgMinMax.z;
    gl_Position = vec4(1. - (float(gl_VertexID) + 1.0) / 10., v * 2. - 1., 0, 1);
    gl_PointSize = 10.0;
    v_id = float(gl_VertexID);
  }
  `;
  const avgMinMaxFS = `#version 300 es
  precision mediump float;
  in float v_id;
  out vec4 fragColor;
  void main() {
    fragColor = vec4(1., v_id / 2., 1. - v_id / 2., 1);
  }
  `;
  // compile shaders, link program, look up uniforms
  const avgMinMaxPrgInfo = twgl.createProgramInfo(gl, [avgMinMaxVS, avgMinMaxFS]);
  
  // creates buffers with positions and texcoords for a -1 to +1 quad
  const planeBufferInfo = twgl.primitives.createXYQuadBufferInfo(gl);
  
  const videoTex = gl.createTexture();
  gl.bindTexture(gl.TEXTURE_2D, videoTex);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
  
  gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);

  function draw() {
    // copy video to texture
    gl.bindTexture(gl.TEXTURE_2D, videoTex);
    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video);
    
    // --- [ compute average, min, max to single pixel ] ---
    
    gl.useProgram(averagePrgInfo.program);
    
    // calls gl.bindFramebuffer and gl.viewport
    twgl.bindFramebufferInfo(gl, fbi);
    
    // calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
    twgl.setBuffersAndAttributes(gl, averagePrgInfo, planeBufferInfo);
    
    // calls gl.drawArrays or gl.drawElements
    twgl.drawBufferInfo(gl, planeBufferInfo);
    
    // --- [ draw video to texture ] ---
    
    // calls gl.bindFramebuffer and gl.viewport
    twgl.bindFramebufferInfo(gl, null);
    
    gl.useProgram(textureProgInfo.program);
    
    // calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
    twgl.setBuffersAndAttributes(gl, textureProgInfo, planeBufferInfo);
    
    // calls gl.drawArrays or gl.drawElements
    twgl.drawBufferInfo(gl, planeBufferInfo);
    
    // -- [ draw 3 points showing avg, min, max] ---
    
    gl.useProgram(avgMinMaxPrgInfo.program);

    gl.bindTexture(gl.TEXTURE_2D, fbi.attachments[0]);

    // draw 3 points
    gl.drawArrays(gl.POINTS, 0, 3);
  }
}
body {
  background: #444;
}
canvas {
  border: 1px solid black;
  display: block;
}
<canvas></canvas>
<button type="button">start</button>
<span style="color: #FF0">■ max brightness</span>
<span style="color: #F80">■ min brightness, </span>
<span style="color: #F0F">■ average brightness, </span>
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Not the answer you're looking for? Browse other questions tagged or ask your own question.