I’m a contributor to MDN, where there’s an issue that was raised about a code demo in the Animating textures in WebGL article.
In order to have a self-contained MRE in this question, I’ve copied the demo code from that article into a runnable snippet here (below).
The demo creates a video
element, sets the autoplay
attribute on it, and then layers that video over a 3D rotating cube as a texture.
My question is: How can we make that demo code work as expected on iOS?
Expected behavior
The expected behavior is for the browser to display a 3D rotating cube, with the video from https://mdn.github.io/dom-examples/webgl-examples/tutorial/sample8/Firefox.mp4 layered over the rotating cube as a texture — like this:
…and in all current browsers on all platforms except for iOS, that is the actual behavior.
Actual behavior on iOS
On iOS, in Safari (mobile) — and I think in any other browsers running on iOS (since they all use WebKit as their engine) — a 3D rotating cube is displayed as expected; however, the video is not layered over the rotating cube as expected but instead the browser just displays a plain blue rotating cube, with no texture at all layered over it — like this:
Troubleshooting steps taken, and error messages seen
I don’t have iOS browser debugging tools myself, but an issue comment from an MDN user says the browser logs this:
setupVideo webgl-demo.js:141 — Unhandled Promise Rejection: NotAllowedError: The request is not allowed by the user agent or the platform in the current context, possibly because the user denied permission.
Because the line numbers don’t match up directly, I’m not sure what’s a line 141 in the code as the iOS browser sees it — but https://github.com/mdn/dom-examples/blob/master/webgl-examples/tutorial/sample8/webgl-demo.js#L154 is my best guess — the video.play()
call.
So the same issue comment which quotes the above message also references Browser denying javascript play() as a related SO question.
…and that SO question cites a similar “not allowed by the user agent or the platform in the current context, possibly because the user denied permission” error message.
Runnable code snippet that doesn’t work as expected on iOS
var cubeRotation = 0.0;
var copyVideo = false;
main();
function main() {
const canvas = document.querySelector("#glcanvas");
const gl = canvas.getContext("webgl");
if (!gl) {
alert(
"Unable to initialize WebGL. Your browser or machine may not support it."
);
return;
}
const vsSource = `
attribute vec4 aVertexPosition;
attribute vec3 aVertexNormal;
attribute vec2 aTextureCoord;
uniform mat4 uNormalMatrix;
uniform mat4 uModelViewMatrix;
uniform mat4 uProjectionMatrix;
varying highp vec2 vTextureCoord;
varying highp vec3 vLighting;
void main(void) {
gl_Position = uProjectionMatrix * uModelViewMatrix * aVertexPosition;
vTextureCoord = aTextureCoord;
// Apply lighting effect
highp vec3 ambientLight = vec3(0.3, 0.3, 0.3);
highp vec3 directionalLightColor = vec3(1, 1, 1);
highp vec3 directionalVector = normalize(vec3(0.85, 0.8, 0.75));
highp vec4 transformedNormal = uNormalMatrix * vec4(aVertexNormal, 1.0);
highp float directional = max(dot(transformedNormal.xyz, directionalVector), 0.0);
vLighting = ambientLight + (directionalLightColor * directional);
}
`;
const fsSource = `
varying highp vec2 vTextureCoord;
varying highp vec3 vLighting;
uniform sampler2D uSampler;
void main(void) {
highp vec4 texelColor = texture2D(uSampler, vTextureCoord);
gl_FragColor = vec4(texelColor.rgb * vLighting, texelColor.a);
}
`;
const shaderProgram = initShaderProgram(gl, vsSource, fsSource);
const programInfo = {
program: shaderProgram,
attribLocations: {
vertexPosition: gl.getAttribLocation(shaderProgram, "aVertexPosition"),
vertexNormal: gl.getAttribLocation(shaderProgram, "aVertexNormal"),
textureCoord: gl.getAttribLocation(shaderProgram, "aTextureCoord"),
},
uniformLocations: {
projectionMatrix: gl.getUniformLocation(
shaderProgram,
"uProjectionMatrix"
),
modelViewMatrix: gl.getUniformLocation(shaderProgram, "uModelViewMatrix"),
normalMatrix: gl.getUniformLocation(shaderProgram, "uNormalMatrix"),
uSampler: gl.getUniformLocation(shaderProgram, "uSampler"),
},
};
const buffers = initBuffers(gl);
const texture = initTexture(gl);
const video = setupVideo("https://mdn.github.io/dom-examples/webgl-examples/tutorial/sample8/Firefox.mp4");
var then = 0;
function render(now) {
now *= 0.001; // convert to seconds
const deltaTime = now - then;
then = now;
if (copyVideo) {
updateTexture(gl, texture, video);
}
drawScene(gl, programInfo, buffers, texture, deltaTime);
requestAnimationFrame(render);
}
requestAnimationFrame(render);
}
function setupVideo(url) {
const video = document.createElement("video");
var playing = false;
var timeupdate = false;
video.muted = true;
video.autoplay = true;
video.loop = true;
video.crossOrigin = "anonymous";
video.addEventListener(
"playing",
function () {
playing = true;
checkReady();
},
true
);
video.addEventListener(
"timeupdate",
function () {
timeupdate = true;
checkReady();
},
true
);
video.src = url;
video.play();
function checkReady() {
if (playing && timeupdate) {
copyVideo = true;
}
}
return video;
}
function initBuffers(gl) {
const positionBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
const positions = [
-1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0,
-1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0, -1.0,
-1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, -1.0,
-1.0, -1.0, -1.0, 1.0, -1.0, -1.0, 1.0, -1.0, 1.0, -1.0, -1.0, 1.0,
1.0, -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0,
-1.0, -1.0, -1.0, -1.0, -1.0, 1.0, -1.0, 1.0, 1.0, -1.0, 1.0, -1.0,
];
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(positions), gl.STATIC_DRAW);
const normalBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, normalBuffer);
const vertexNormals = [
0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0,
0.0, 0.0, -1.0, 0.0, 0.0, -1.0, 0.0, 0.0, -1.0, 0.0, 0.0, -1.0,
0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0,
0.0, -1.0, 0.0, 0.0, -1.0, 0.0, 0.0, -1.0, 0.0, 0.0, -1.0, 0.0,
1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0,
-1.0, 0.0, 0.0, -1.0, 0.0, 0.0, -1.0, 0.0, 0.0, -1.0, 0.0, 0.0,
];
gl.bufferData(
gl.ARRAY_BUFFER,
new Float32Array(vertexNormals),
gl.STATIC_DRAW
);
const textureCoordBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, textureCoordBuffer);
const textureCoordinates = [
0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0,
0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0,
0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0,
0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0,
0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0,
0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0,
];
gl.bufferData(
gl.ARRAY_BUFFER,
new Float32Array(textureCoordinates),
gl.STATIC_DRAW
);
const indexBuffer = gl.createBuffer();
gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, indexBuffer);
const indices = [
0,
1,
2,
0,
2,
3, // front
4,
5,
6,
4,
6,
7, // back
8,
9,
10,
8,
10,
11, // top
12,
13,
14,
12,
14,
15, // bottom
16,
17,
18,
16,
18,
19, // right
20,
21,
22,
20,
22,
23, // left
];
gl.bufferData(
gl.ELEMENT_ARRAY_BUFFER,
new Uint16Array(indices),
gl.STATIC_DRAW
);
return {
position: positionBuffer,
normal: normalBuffer,
textureCoord: textureCoordBuffer,
indices: indexBuffer,
};
}
function initTexture(gl, url) {
const texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
const level = 0;
const internalFormat = gl.RGBA;
const width = 1;
const height = 1;
const border = 0;
const srcFormat = gl.RGBA;
const srcType = gl.UNSIGNED_BYTE;
const pixel = new Uint8Array([0, 0, 255, 255]); // opaque blue
gl.texImage2D(
gl.TEXTURE_2D,
level,
internalFormat,
width,
height,
border,
srcFormat,
srcType,
pixel
);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
return texture;
}
function updateTexture(gl, texture, video) {
const level = 0;
const internalFormat = gl.RGBA;
const srcFormat = gl.RGBA;
const srcType = gl.UNSIGNED_BYTE;
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(
gl.TEXTURE_2D,
level,
internalFormat,
srcFormat,
srcType,
video
);
}
function isPowerOf2(value) {
return (value & (value - 1)) == 0;
}
function drawScene(gl, programInfo, buffers, texture, deltaTime) {
gl.clearColor(0.0, 0.0, 0.0, 1.0); // Clear to black, fully opaque
gl.clearDepth(1.0); // Clear everything
gl.enable(gl.DEPTH_TEST); // Enable depth testing
gl.depthFunc(gl.LEQUAL); // Near things obscure far things
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
const fieldOfView = (45 * Math.PI) / 180; // in radians
const aspect = gl.canvas.clientWidth / gl.canvas.clientHeight;
const zNear = 0.1;
const zFar = 100.0;
const projectionMatrix = mat4.create();
mat4.perspective(projectionMatrix, fieldOfView, aspect, zNear, zFar);
const modelViewMatrix = mat4.create();
mat4.translate(
modelViewMatrix, // destination matrix
modelViewMatrix, // matrix to translate
[-0.0, 0.0, -6.0]
); // amount to translate
mat4.rotate(
modelViewMatrix, // destination matrix
modelViewMatrix, // matrix to rotate
cubeRotation, // amount to rotate in radians
[0, 0, 1]
); // axis to rotate around (Z)
mat4.rotate(
modelViewMatrix, // destination matrix
modelViewMatrix, // matrix to rotate
cubeRotation * 0.7, // amount to rotate in radians
[0, 1, 0]
); // axis to rotate around (X)
const normalMatrix = mat4.create();
mat4.invert(normalMatrix, modelViewMatrix);
mat4.transpose(normalMatrix, normalMatrix);
{
const numComponents = 3;
const type = gl.FLOAT;
const normalize = false;
const stride = 0;
const offset = 0;
gl.bindBuffer(gl.ARRAY_BUFFER, buffers.position);
gl.vertexAttribPointer(
programInfo.attribLocations.vertexPosition,
numComponents,
type,
normalize,
stride,
offset
);
gl.enableVertexAttribArray(programInfo.attribLocations.vertexPosition);
}
{
const numComponents = 2;
const type = gl.FLOAT;
const normalize = false;
const stride = 0;
const offset = 0;
gl.bindBuffer(gl.ARRAY_BUFFER, buffers.textureCoord);
gl.vertexAttribPointer(
programInfo.attribLocations.textureCoord,
numComponents,
type,
normalize,
stride,
offset
);
gl.enableVertexAttribArray(programInfo.attribLocations.textureCoord);
}
{
const numComponents = 3;
const type = gl.FLOAT;
const normalize = false;
const stride = 0;
const offset = 0;
gl.bindBuffer(gl.ARRAY_BUFFER, buffers.normal);
gl.vertexAttribPointer(
programInfo.attribLocations.vertexNormal,
numComponents,
type,
normalize,
stride,
offset
);
gl.enableVertexAttribArray(programInfo.attribLocations.vertexNormal);
}
gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, buffers.indices);
gl.useProgram(programInfo.program);
gl.uniformMatrix4fv(
programInfo.uniformLocations.projectionMatrix,
false,
projectionMatrix
);
gl.uniformMatrix4fv(
programInfo.uniformLocations.modelViewMatrix,
false,
modelViewMatrix
);
gl.uniformMatrix4fv(
programInfo.uniformLocations.normalMatrix,
false,
normalMatrix
);
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.uniform1i(programInfo.uniformLocations.uSampler, 0);
{
const vertexCount = 36;
const type = gl.UNSIGNED_SHORT;
const offset = 0;
gl.drawElements(gl.TRIANGLES, vertexCount, type, offset);
}
cubeRotation += deltaTime;
}
function initShaderProgram(gl, vsSource, fsSource) {
const vertexShader = loadShader(gl, gl.VERTEX_SHADER, vsSource);
const fragmentShader = loadShader(gl, gl.FRAGMENT_SHADER, fsSource);
const shaderProgram = gl.createProgram();
gl.attachShader(shaderProgram, vertexShader);
gl.attachShader(shaderProgram, fragmentShader);
gl.linkProgram(shaderProgram);
if (!gl.getProgramParameter(shaderProgram, gl.LINK_STATUS)) {
alert(
"Unable to initialize the shader program: " +
gl.getProgramInfoLog(shaderProgram)
);
return null;
}
return shaderProgram;
}
function loadShader(gl, type, source) {
const shader = gl.createShader(type);
gl.shaderSource(shader, source);
gl.compileShader(shader);
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
alert(
"An error occurred compiling the shaders: " + gl.getShaderInfoLog(shader)
);
gl.deleteShader(shader);
return null;
}
return shader;
}
canvas { border: 2px solid black; background-color: black; }
video { display: none; }
<!doctype html><meta charset="utf-8"><title>WebGL Demo</title>
<script src="https://cdnjs.cloudflare.com/ajax/libs/gl-matrix/2.8.1/gl-matrix-min.js"></script>
<canvas id="glcanvas" width="640" height="480"></canvas>
iframe
on different origin, maybe theallow="autoplay"
attribute on thatiframe
would help?iframe
— also doesn’t show the video texture as expected on iOS. So it seems like the framing must not be the root problem.