/*! @license
* Shaka Player
* Copyright 2016 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
goog.provide('shaka.ui.VRWebgl');
goog.require('goog.asserts');
goog.require('shaka.log');
goog.require('shaka.Player');
goog.require('shaka.ui.Matrix4x4');
goog.require('shaka.ui.MatrixQuaternion');
goog.require('shaka.ui.VRUtils');
goog.require('shaka.util.EventManager');
goog.require('shaka.util.IReleasable');
goog.require('shaka.util.Timer');
/**
* @implements {shaka.util.IReleasable}
*/
shaka.ui.VRWebgl = class {
/**
* @param {!HTMLMediaElement} video
* @param {!shaka.Player} player
* @param {!HTMLCanvasElement} canvas
* @param {WebGLRenderingContext} gl
* @param {string} projectionMode
*/
constructor(video, player, canvas, gl, projectionMode) {
/** @private {HTMLVideoElement} */
this.video_ = /** @type {!HTMLVideoElement} */ (video);
/** @private {shaka.Player} */
this.player_ = player;
/** @private {HTMLCanvasElement} */
this.canvas_ = canvas;
/** @private {WebGLRenderingContext} */
this.gl_ = gl;
/** @private {shaka.util.EventManager} */
this.eventManager_ = new shaka.util.EventManager();
/** @private {!Float32Array} */
this.originalQuaternion_ = shaka.ui.MatrixQuaternion.create();
/** @private {!Float32Array} */
this.currentQuaternion_ = shaka.ui.MatrixQuaternion.create();
/** @private {?WebGLProgram} */
this.shaderProgram_ = null;
/** @private {?WebGLBuffer} */
this.verticesBuffer_ = null;
/** @private {?WebGLBuffer} */
this.verticesTextureCoordBuffer_ = null;
/** @private {?WebGLBuffer} */
this.verticesIndexBuffer_ = null;
/** @private {!Float32Array} */
this.viewMatrix_ = shaka.ui.Matrix4x4.create();
/** @private {!Float32Array} */
this.projectionMatrix_ = shaka.ui.Matrix4x4.create();
/** @private {!Float32Array} */
this.viewProjectionMatrix_ = shaka.ui.Matrix4x4.create();
/** @private {!Float32Array} */
this.identityMatrix_ = shaka.ui.Matrix4x4.create();
/** @private {?Float32Array} */
this.diff_ = null;
/** @private {boolean} */
this.stereoscopicMode_ = false;
/** @private {?shaka.util.Timer} */
this.activeTimer_ = null;
/** @private {?shaka.util.Timer} */
this.resetTimer_ = null;
/** @private {number} */
this.previousCanvasWidth_ = 0;
/** @private {number} */
this.previousCanvasHeight_ = 0;
/**
* @private {?{vertices: !Array.<number>, textureCoords: !Array.<number>,
* indices: !Array.<number>}}
*/
this.geometry_ = null;
/** @private {?number} */
this.vertexPositionAttribute_ = null;
/** @private {?number} */
this.textureCoordAttribute_ = null;
/** @private {?WebGLTexture} */
this.texture_ = null;
/** @private {number} */
this.positionY_ = 0;
/** @private {number} */
this.fieldOfView_ = 75;
/** @private {number} */
this.cont_ = 0;
/** @private {string} */
this.projectionMode_ = projectionMode;
this.init_();
}
/**
* @override
*/
release() {
if (this.eventManager_) {
this.eventManager_.release();
this.eventManager_ = null;
}
if (this.activeTimer_) {
this.activeTimer_.stop();
this.activeTimer_ = null;
}
if (this.resetTimer_) {
this.resetTimer_.stop();
this.resetTimer_ = null;
}
}
/**
* @return {string}
*/
getProjectionMode() {
return this.projectionMode_;
}
/**
* @param {!Float32Array} quat
* @return {{pitch: number, yaw: number, roll: number}} as radians
* @private
*/
toEulerAngles_(quat) {
const angles = {
pitch: 0,
yaw: 0,
roll: 0,
};
const x = quat[0];
const y = quat[1];
const z = quat[2];
const w = quat[3];
const x2 = x * x;
const y2 = y * y;
const z2 = z * z;
const w2 = w * w;
const unit = x2 + y2 + z2 + w2;
const test = x * w - y * z;
if (test > 0.499995 * unit) {
// singularity at the north pole
angles.pitch = Math.PI / 2;
angles.yaw = 2 * Math.atan2(y, x);
angles.roll = 0;
} else if (test < -0.499995 * unit) {
// singularity at the south pole
angles.pitch = -Math.PI / 2;
angles.yaw = 2 * Math.atan2(y, x);
angles.roll = 0;
} else {
angles.pitch = Math.asin(2 * (x * z - w * y));
angles.yaw = Math.atan2(2 * (x * w + y * z), 1 - 2 * (z2 + w2));
angles.roll = Math.atan2(2 * (x * y + z * w), 1 - 2 * (y2 + z2));
}
return angles;
}
/**
* Toogle stereoscopic mode
*/
toggleStereoscopicMode() {
this.stereoscopicMode_ = !this.stereoscopicMode_;
if (!this.stereoscopicMode_) {
this.gl_.viewport(0, 0, this.canvas_.width, this.canvas_.height);
}
this.renderGL_(false);
}
/**
* Returns true if stereoscopic mode is enabled.
*
* @return {boolean}
*/
isStereoscopicModeEnabled() {
return this.stereoscopicMode_;
}
/**
* @private
*/
init_() {
this.initMatrices_();
this.initGL_();
this.initGLShaders_();
this.initGLBuffers_();
this.initGLTexture_();
this.eventManager_.listenOnce(this.video_, 'loadeddata', () => {
let frameRate;
this.eventManager_.listen(this.video_, 'canplaythrough', () => {
this.renderGL_();
});
this.eventManager_.listen(this.video_, 'playing', () => {
if (this.activeTimer_) {
this.activeTimer_.stop();
}
if (!frameRate) {
const variants = this.player_.getVariantTracks();
for (const variant of variants) {
const variantFrameRate = variant.frameRate;
if (variantFrameRate &&
(!frameRate || frameRate < variantFrameRate)) {
frameRate = variantFrameRate;
}
}
}
if (!frameRate) {
frameRate = 60;
}
this.renderGL_();
this.activeTimer_ = new shaka.util.Timer(() => {
this.renderGL_();
}).tickNow().tickEvery(1 / frameRate);
});
this.eventManager_.listen(this.video_, 'pause', () => {
this.activeTimer_.stop();
this.activeTimer_ = null;
this.renderGL_();
});
this.eventManager_.listen(this.video_, 'seeked', () => {
this.renderGL_();
});
this.eventManager_.listen(document, 'visibilitychange', () => {
this.renderGL_();
});
});
}
/**
* @private
*/
initMatrices_() {
shaka.ui.Matrix4x4.lookAt(
this.viewMatrix_, [0, 0, 0], [1, 0, 0], [0, 1, 0]);
shaka.ui.Matrix4x4.getRotation(
this.originalQuaternion_, this.viewMatrix_);
shaka.ui.Matrix4x4.scale(
this.identityMatrix_, this.identityMatrix_, [4.0, 4.0, 4.0]);
}
/**
* @private
*/
initGL_() {
this.updateViewPort_();
this.gl_.viewport(
0, 0, this.gl_.drawingBufferWidth, this.gl_.drawingBufferHeight);
this.gl_.clearColor(0.0, 0.0, 0.0, 1.0);
this.gl_.enable(this.gl_.CULL_FACE);
this.gl_.cullFace(this.gl_.FRONT);
// Clear the context with the newly set color. This is
// the function call that actually does the drawing.
this.gl_.clear(this.gl_.COLOR_BUFFER_BIT);
}
/**
* @private
*/
initGLShaders_() {
const vertexShader = this.getGLShader_(this.gl_.VERTEX_SHADER);
const fragmentShader = this.getGLShader_(this.gl_.FRAGMENT_SHADER);
// Create program
this.shaderProgram_ = this.gl_.createProgram();
this.gl_.attachShader(this.shaderProgram_, vertexShader);
this.gl_.attachShader(this.shaderProgram_, fragmentShader);
this.gl_.linkProgram(this.shaderProgram_);
// If creating the shader program failed, alert
if (!this.gl_.getProgramParameter(
this.shaderProgram_, this.gl_.LINK_STATUS)) {
shaka.log.error('Unable to initialize the shader program: ',
this.gl_.getProgramInfoLog(this.shaderProgram_));
}
// Bind data
if (this.projectionMode_ == 'cubemap') {
this.vertexPositionAttribute_ = this.gl_.getAttribLocation(
this.shaderProgram_, 'aVertexPosition');
this.textureCoordAttribute_ = this.gl_.getAttribLocation(
this.shaderProgram_, 'aTextureCoord');
} else {
this.vertexPositionAttribute_ = this.gl_.getAttribLocation(
this.shaderProgram_, 'a_vPosition');
this.gl_.enableVertexAttribArray(this.vertexPositionAttribute_);
this.textureCoordAttribute_ = this.gl_.getAttribLocation(
this.shaderProgram_, 'a_TexCoordinate');
this.gl_.enableVertexAttribArray(this.textureCoordAttribute_);
}
}
/**
* Read and generate WebGL shader
*
* @param {number} glType Type of shader requested.
* @return {?WebGLShader}
* @private
*/
getGLShader_(glType) {
let source;
switch (glType) {
case this.gl_.VERTEX_SHADER:
if (this.projectionMode_ == 'cubemap') {
source = shaka.ui.VRUtils.VERTEX_CUBE_SHADER;
} else {
source = shaka.ui.VRUtils.VERTEX_SPHERE_SHADER;
}
break;
case this.gl_.FRAGMENT_SHADER:
if (this.projectionMode_ == 'cubemap') {
source = shaka.ui.VRUtils.FRAGMENT_CUBE_SHADER;
} else {
source = shaka.ui.VRUtils.FRAGMENT_SPHERE_SHADER;
}
break;
default:
return null;
}
const shader = this.gl_.createShader(glType);
this.gl_.shaderSource(shader, source);
this.gl_.compileShader(shader);
if (!this.gl_.getShaderParameter(shader, this.gl_.COMPILE_STATUS)) {
shaka.log.warning('Error in ' + glType + ' shader: ' +
this.gl_.getShaderInfoLog(shader));
}
goog.asserts.assert(shader, 'Should have a shader!');
return shader;
}
/**
* @private
*/
initGLBuffers_() {
if (this.projectionMode_ == 'cubemap') {
this.geometry_ = shaka.ui.VRUtils.generateCube();
} else {
this.geometry_ = shaka.ui.VRUtils.generateSphere(100);
}
this.verticesBuffer_ = this.gl_.createBuffer();
this.gl_.bindBuffer(this.gl_.ARRAY_BUFFER, this.verticesBuffer_);
this.gl_.bufferData(this.gl_.ARRAY_BUFFER,
new Float32Array(this.geometry_.vertices), this.gl_.STATIC_DRAW);
this.verticesTextureCoordBuffer_ = this.gl_.createBuffer();
this.gl_.bindBuffer(
this.gl_.ARRAY_BUFFER, this.verticesTextureCoordBuffer_);
this.gl_.bufferData(this.gl_.ARRAY_BUFFER,
new Float32Array(this.geometry_.textureCoords), this.gl_.STATIC_DRAW);
this.verticesIndexBuffer_ = this.gl_.createBuffer();
this.gl_.bindBuffer(
this.gl_.ELEMENT_ARRAY_BUFFER, this.verticesIndexBuffer_);
this.gl_.bufferData(this.gl_.ELEMENT_ARRAY_BUFFER,
new Uint16Array(this.geometry_.indices), this.gl_.STATIC_DRAW);
}
/**
* @private
*/
initGLTexture_() {
this.texture_ = this.gl_.createTexture();
this.gl_.bindTexture(this.gl_.TEXTURE_2D, this.texture_);
this.gl_.texParameteri(this.gl_.TEXTURE_2D,
this.gl_.TEXTURE_WRAP_S, this.gl_.CLAMP_TO_EDGE);
this.gl_.texParameteri(this.gl_.TEXTURE_2D,
this.gl_.TEXTURE_WRAP_T, this.gl_.CLAMP_TO_EDGE);
this.gl_.texParameteri(this.gl_.TEXTURE_2D,
this.gl_.TEXTURE_MIN_FILTER, this.gl_.NEAREST);
this.gl_.texParameteri(this.gl_.TEXTURE_2D,
this.gl_.TEXTURE_MAG_FILTER, this.gl_.NEAREST);
}
/**
* @param {boolean=} textureUpdate
* @private
*/
renderGL_(textureUpdate = true) {
const loadMode = this.player_.getLoadMode();
const isMSE = loadMode == shaka.Player.LoadMode.MEDIA_SOURCE;
if (!this.video_ || this.video_.readyState < 2 ||
(!isMSE && this.video_.playbackRate == 0)) {
return;
}
shaka.ui.Matrix4x4.perspective(this.projectionMatrix_,
this.fieldOfView_ * Math.PI / 180, 5 / 3.2, 0.1, 100.0);
if (this.projectionMode_ == 'cubemap') {
shaka.ui.Matrix4x4.perspective(this.projectionMatrix_,
this.fieldOfView_ * Math.PI / 180, 5 / 2, 0.1, 100.0);
} else {
shaka.ui.Matrix4x4.perspective(this.projectionMatrix_,
this.fieldOfView_ * Math.PI / 180, 5 / 3.2, 0.1, 100.0);
}
this.gl_.useProgram(this.shaderProgram_);
this.gl_.clear(this.gl_.COLOR_BUFFER_BIT);
this.updateViewPort_();
if (textureUpdate) {
this.gl_.activeTexture(this.gl_.TEXTURE0);
this.gl_.bindTexture(this.gl_.TEXTURE_2D, this.texture_);
this.gl_.pixelStorei(this.gl_.UNPACK_PREMULTIPLY_ALPHA_WEBGL, 0);
this.gl_.texImage2D(this.gl_.TEXTURE_2D, 0, this.gl_.RGBA,
this.gl_.RGBA, this.gl_.UNSIGNED_BYTE, this.video_);
}
// Update matrix
if (this.projectionMode_ == 'equirectangular') {
shaka.ui.Matrix4x4.multiply(this.viewProjectionMatrix_,
this.viewMatrix_, this.identityMatrix_);
shaka.ui.Matrix4x4.multiply(this.viewProjectionMatrix_,
this.projectionMatrix_, this.viewProjectionMatrix_);
}
// Plumbing
// Vertices
this.gl_.bindBuffer(this.gl_.ARRAY_BUFFER, this.verticesBuffer_);
goog.asserts.assert(this.vertexPositionAttribute_ != null,
'Should have a texture attribute!');
this.gl_.vertexAttribPointer(
this.vertexPositionAttribute_, 3, this.gl_.FLOAT, false, 0, 0);
this.gl_.enableVertexAttribArray(this.vertexPositionAttribute_);
// UVs
this.gl_.bindBuffer(
this.gl_.ARRAY_BUFFER, this.verticesTextureCoordBuffer_);
goog.asserts.assert(this.textureCoordAttribute_ != null,
'Should have a texture attribute!');
this.gl_.vertexAttribPointer(
this.textureCoordAttribute_, 2, this.gl_.FLOAT, false, 0, 0);
this.gl_.enableVertexAttribArray(this.textureCoordAttribute_);
this.gl_.bindBuffer(
this.gl_.ELEMENT_ARRAY_BUFFER, this.verticesIndexBuffer_);
this.setMatrixUniforms_();
this.gl_.uniform1i(
this.gl_.getUniformLocation(this.shaderProgram_, 'uSampler'), 0);
if (this.stereoscopicMode_) {
this.gl_.viewport(0, 0, this.canvas_.width / 2, this.canvas_.height);
}
// Draw
this.gl_.drawElements(this.gl_.TRIANGLES,
this.geometry_.indices.length, this.gl_.UNSIGNED_SHORT, 0);
if (this.stereoscopicMode_) {
this.gl_.viewport(this.canvas_.width / 2, 0,
this.canvas_.width / 2, this.canvas_.height);
this.gl_.drawElements(this.gl_.TRIANGLES,
this.geometry_.indices.length, this.gl_.UNSIGNED_SHORT, 0);
}
}
/**
* @private
*/
setMatrixUniforms_() {
if (this.projectionMode_ == 'cubemap') {
this.gl_.uniformMatrix4fv(
this.gl_.getUniformLocation(this.shaderProgram_, 'uProjectionMatrix'),
false, this.projectionMatrix_);
this.gl_.uniformMatrix4fv(
this.gl_.getUniformLocation(this.shaderProgram_, 'uModelViewMatrix'),
false, this.viewProjectionMatrix_);
} else {
this.gl_.uniformMatrix4fv(
this.gl_.getUniformLocation(this.shaderProgram_, 'u_VPMatrix'),
false, this.viewProjectionMatrix_);
}
}
/**
* @private
*/
updateViewPort_() {
let currentWidth = this.video_.videoWidth;
if (!currentWidth) {
currentWidth = this.canvas_.scrollWidth;
}
let currentHeight = this.video_.videoHeight;
if (!currentHeight) {
currentHeight = this.canvas_.scrollHeight;
}
if (this.previousCanvasWidth_ !== currentWidth ||
this.previousCanvasHeight_ !== currentHeight) {
this.canvas_.width = currentWidth;
this.canvas_.height = currentHeight;
this.previousCanvasWidth_ = currentWidth;
this.previousCanvasHeight_ = currentHeight;
const ratio = currentWidth / currentHeight;
this.projectionMatrix_ = shaka.ui.Matrix4x4.frustum(
this.projectionMatrix_, -ratio, ratio, -1, 1, 0, 1);
this.gl_.viewport(0, 0, currentWidth, currentHeight);
}
}
/**
* Rotate the view matrix global
*
* @param {!number} yaw Yaw.
* @param {!number} pitch Pitch.
* @param {!number} roll Roll.
*/
rotateViewGlobal(yaw, pitch, roll) {
const pitchBoundary = 90.0 * Math.PI / 180;
let matrix;
if (this.projectionMode_ == 'cubemap') {
matrix = this.viewProjectionMatrix_;
} else {
matrix = this.viewMatrix_;
}
// Rotate global axis
shaka.ui.Matrix4x4.rotateY(matrix, matrix, yaw);
// Variable to limit the pitch movement
this.positionY_ += pitch;
if (this.positionY_ < pitchBoundary &&
this.positionY_ > -pitchBoundary) {
const out = shaka.ui.Matrix4x4.create();
shaka.ui.Matrix4x4.rotateX(out, shaka.ui.Matrix4x4.create(), -1 * pitch);
// Rotate local axis
shaka.ui.Matrix4x4.multiply(matrix, out, matrix);
} else {
// Doing this we restart the value to the previous position,
// to not mantain a value over 90º or under -90º.
this.positionY_ -= pitch;
}
const out2 = shaka.ui.Matrix4x4.create();
shaka.ui.Matrix4x4.rotateZ(out2, shaka.ui.Matrix4x4.create(), roll);
// Rotate local axis
shaka.ui.Matrix4x4.multiply(matrix, out2, matrix);
this.renderGL_(false);
}
/**
* @param {number} amount
*/
zoom(amount) {
const zoomMin = 20;
const zoomMax = 100;
amount /= 50;
if (this.fieldOfView_ >= zoomMin && this.fieldOfView_ <= zoomMax) {
this.fieldOfView_ += amount;
}
if (this.fieldOfView_ < zoomMin) {
this.fieldOfView_ = zoomMin;
} else if (this.fieldOfView_ > zoomMax) {
this.fieldOfView_ = zoomMax;
}
this.renderGL_(false);
}
/**
* @return {number}
*/
getFieldOfView() {
return this.fieldOfView_;
}
/**
* @param {number} fieldOfView
*/
setFieldOfView(fieldOfView) {
this.fieldOfView_ = fieldOfView;
this.renderGL_(false);
}
/**
* @return {number}
*/
getNorth() {
shaka.ui.Matrix4x4.getRotation(this.currentQuaternion_, this.viewMatrix_);
const angles = this.toEulerAngles_(this.currentQuaternion_);
const normalizedDir = {
x: Math.cos(angles.yaw) * Math.cos(angles.pitch),
y: Math.sin(angles.yaw) * Math.cos(angles.pitch),
z: Math.sin(angles.pitch),
};
const northYaw = Math.acos(normalizedDir.x);
return ((northYaw * 180) / Math.PI);
}
/**
* @param {boolean=} firstTime
*/
reset(firstTime = true) {
const steps = 20;
if (firstTime) {
shaka.ui.Matrix4x4.getRotation(
this.currentQuaternion_, this.viewMatrix_);
this.cont_ = 0;
this.diff_ = shaka.ui.MatrixQuaternion.create();
this.diff_[0] =
(this.currentQuaternion_[0] - this.originalQuaternion_[0]) / steps;
this.diff_[1] =
(this.currentQuaternion_[1] - this.originalQuaternion_[1]) / steps;
this.diff_[2] =
(this.currentQuaternion_[2] - this.originalQuaternion_[2]) / steps;
this.diff_[3] =
(this.currentQuaternion_[3] - this.originalQuaternion_[3]) / steps;
}
this.currentQuaternion_[0] -= this.diff_[0];
this.currentQuaternion_[1] -= this.diff_[1];
this.currentQuaternion_[2] -= this.diff_[2];
this.currentQuaternion_[3] -= this.diff_[3];
// Set the view to the original matrix
const out = shaka.ui.Matrix4x4.create();
shaka.ui.MatrixQuaternion.normalize(
this.currentQuaternion_, this.currentQuaternion_);
shaka.ui.Matrix4x4.fromQuat(out, this.currentQuaternion_);
this.viewMatrix_ = out;
if (this.resetTimer_) {
this.resetTimer_.stop();
this.resetTimer_ = null;
}
if (this.cont_ < steps) {
this.resetTimer_ = new shaka.util.Timer(() => {
this.reset(false);
this.positionY_ = 0;
this.cont_++;
this.renderGL_(false);
}).tickAfter(shaka.ui.VRWebgl.ANIMATION_DURATION_ / steps);
} else {
shaka.ui.Matrix4x4.fromQuat(out, this.originalQuaternion_);
this.viewMatrix_ = out;
}
}
};
/**
* @constant {number}
*/
shaka.ui.VRWebgl.ANIMATION_DURATION_ = 0.5;