Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
[nemo-qtmultimedia-plugins] Add QML filters support. Contributes to J…
…B#48607

Original MR provides the changes:

Fix Qt Multimedia declarative video backend plugin not applying attached video filters:

    Add overrides for QDeclarativeVideoBackend::appendFilter(), clearFilters() to keep track of attached filters locally
    Add TextureVideoBuffer class
    Use TextureVideoBuffer to implement rendering video frame to offscreen framebuffer object
    Use glReadPixels() to get current frame's pixel data
    Call video filters chain from GStreamerVideoTexture::updateTexture()

With this fix video filters are read-only,
and can be used only to analyze incoming video stream in real time.
Ideally filters should be able to modify every frame's pixels,
but rendering modified video frame back to texture is not implemented now.

[nemo-qtmultimedia-plugins] Fix wrong orientation for front camera.
When front camera is active and phone is on 90 position, video is rotated on 270
JB#48607

[nemo-qtmultimedia-plugins] Update headers. JB#48607

[nemo-qtmultimedia-plugins] undelete wronly deleted m_orientation initialization. JB#48607

[nemo-qtmultimedia-plugins] undelete wronly deleted copyright header. JB#48607

[nemo-qtmultimedia-plugins] Restore Jolla copyright. JB#48607

[nemo-qtmultimedia-plugins] Update copyright headers. JB#48607

[nemo-qtmultimedia-plugins] Process filter list without locking. JB#48607
  • Loading branch information
dzakurin authored and rainemak committed Jan 23, 2020
1 parent 684e34c commit 262494e
Show file tree
Hide file tree
Showing 6 changed files with 875 additions and 195 deletions.
2 changes: 1 addition & 1 deletion rpm/nemo-qtmultimedia-plugins-gstvideotexturebackend.spec
@@ -1,6 +1,6 @@
Name: nemo-qtmultimedia-plugins-gstvideotexturebackend
Summary: QtMultimedia QML VideoOutput backend for GStreamer NemoVideoTexture interface
Version: 0.0.0
Version: 0.0.14
Release: 1
Group: System/Libraries
License: BSD
Expand Down
325 changes: 325 additions & 0 deletions src/videotexturebackend/texturevideobuffer.cpp
@@ -0,0 +1,325 @@
/*
* Copyright (c) 2014 - 2019 Jolla Ltd.
* Copyright (c) 2020 Open Mobile Platform LLC.
*
* You may use this file under the terms of the BSD license as follows:
*
* "Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Nemo Mobile nor the names of its contributors
* may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE."
*/

#include <QOpenGLContext>
#include <QOpenGLFunctions>
#include <QOpenGLFramebufferObject>
#include <QOpenGLShaderProgram>

#include "texturevideobuffer.h"

namespace NemoVideoBackend {

static const char *c_vertexShaderCode =
"attribute highp vec4 vertexCoordsArray; \n" \
"attribute highp vec2 textureCoordArray; \n" \
"uniform highp mat4 texMatrix; \n" \
"varying highp vec2 textureCoords; \n" \
"void main(void) \n" \
"{ \n" \
" gl_Position = vertexCoordsArray; \n" \
" textureCoords = (texMatrix * vec4(textureCoordArray, 0.0, 1.0)).xy; \n" \
"}\n";

static const char *c_fragmentShaderCode =
"#extension GL_OES_EGL_image_external : require \n" \
"varying highp vec2 textureCoords; \n" \
"uniform samplerExternalOES frameTexture; \n" \
"void main() \n" \
"{ \n" \
" gl_FragColor = texture2D(frameTexture, textureCoords); \n" \
"}\n";

TextureVideoBuffer::TextureVideoBuffer():
QAbstractVideoBuffer(QAbstractVideoBuffer::GLTextureHandle)
{
}

TextureVideoBuffer::~TextureVideoBuffer()
{
realDeleteGLResources();
}

void TextureVideoBuffer::release()
{
// Quote from: [qtmultimedia.git/src/multimedia/video/qabstractvideobuffer.cpp]
// QVideoFrame calls QAbstractVideoBuffer::release when the buffer is not used
// any more and can be destroyed or returned to the buffer pool.
// The default implementation deletes the buffer instance.
//
// Qt sources do:
// delete this;
// We do not want to shoot ourselves. Do nothing
}

QAbstractVideoBuffer::MapMode TextureVideoBuffer::mapMode() const
{
return m_mapMode;
}

uchar *TextureVideoBuffer::map(MapMode mode, int *numBytes, int *bytesPerLine)
{
QMutexLocker locker(&m_mutex);
return realMap(mode, numBytes, bytesPerLine);
}

uchar *TextureVideoBuffer::realMap(MapMode mode, int *numBytes, int *bytesPerLine)
{
if (m_mapMode == NotMapped && mode == ReadOnly) {
realUpdateFrame();
m_mapMode = mode;
// call toImage() only if image was not created yet by call
// TextureVideoBuffer::toImage(), for example
if (m_image.isNull())
m_image = m_fbo->toImage();

if (numBytes)
*numBytes = m_image.byteCount();

if (bytesPerLine)
*bytesPerLine = m_image.bytesPerLine();

return m_image.bits();
}
return nullptr;
}

void TextureVideoBuffer::unmap()
{
QMutexLocker locker(&m_mutex);
realUnmap();
}

void TextureVideoBuffer::realUnmap()
{
m_image = QImage();
m_mapMode = NotMapped;
}

QVariant TextureVideoBuffer::handle() const
{
return m_textureId;
}

void TextureVideoBuffer::setTextureSize(const QSize &size)
{
QMutexLocker locker(&m_mutex);
return realSetTextureSize(size);
}


void TextureVideoBuffer::realSetTextureSize(const QSize &size)
{
m_size = size;
}

void TextureVideoBuffer::setTextureId(GLuint textureId)
{
QMutexLocker locker(&m_mutex);
realSetTextureId(textureId);
}

void TextureVideoBuffer::realSetTextureId(GLuint textureId)
{
m_textureId = textureId;
m_textureUpdated = false;
}

/**
* @brief TextureVideoBuffer::toImage
* Better to call this after updateFrame() was called
* @return internal image, if rendered. Null image otherwise
*/
QImage TextureVideoBuffer::toImage() const
{
if (m_textureUpdated) {
m_image = m_fbo->toImage();
}
return m_image;
}


void TextureVideoBuffer::updateFrame()
{
QMutexLocker locker(&m_mutex);
realUpdateFrame();
}

void TextureVideoBuffer::realUpdateFrame()
{
if (!m_textureUpdated) {
// update the video texture (called from the render thread)
realRenderFrameToFbo();
m_textureUpdated = true;
}
}

void TextureVideoBuffer::createGLResources()
{
QMutexLocker locker(&m_mutex);
realCreateGLResources();
}

void TextureVideoBuffer::realCreateGLResources()
{
// check opengl context: Returns the last context which called
// makeCurrent in the current thread, or 0, if no context is current.
QOpenGLContext *context = QOpenGLContext::currentContext();
if (!context) {
qWarning() << Q_FUNC_INFO << " There is no current OpenGL context!";
qWarning() << Q_FUNC_INFO << " This should be called from QML render thread!";
return;
}

// Delete FBO if the texture size changed (to recreate it later on)
if (m_fbo && (m_fbo->size() != m_size)) {
m_fbo.reset(nullptr);
}

// create framebuffer object if not exists
if (!m_fbo) {
m_fbo.reset(new QOpenGLFramebufferObject(m_size));
QObject::connect(context, &QOpenGLContext::aboutToBeDestroyed,
this, &TextureVideoBuffer::deleteGLResources,
Qt::UniqueConnection);
}

// init shader programs only once
if (m_program) {
return;
}
m_program.reset(new QOpenGLShaderProgram());

QOpenGLShader *vertexShader = new QOpenGLShader(QOpenGLShader::Vertex, m_program.get());
vertexShader->compileSourceCode(c_vertexShaderCode);
m_program->addShader(vertexShader);

QOpenGLShader *fragmentShader = new QOpenGLShader(QOpenGLShader::Fragment, m_program.get());
fragmentShader->compileSourceCode(c_fragmentShaderCode);
m_program->addShader(fragmentShader);

m_program->bindAttributeLocation("vertexCoordsArray", 0);
m_program->bindAttributeLocation("textureCoordArray", 1);
m_program->link();
}

void TextureVideoBuffer::deleteGLResources()
{
QMutexLocker locker(&m_mutex);
realDeleteGLResources();
}

void TextureVideoBuffer::realDeleteGLResources()
{
// This should be called in owning GStreamerVideoTexture::releaseTexture()
// which is called in slot conected to QQuickWindow::afterRendering(),
// so rendering should be complete and it is safe to delete resources
if (m_mapMode != NotMapped) {
realUnmap();
}
// delete framefuffer object
m_fbo.reset(nullptr);
// delete shader program
m_program.reset(nullptr);
}

void TextureVideoBuffer::renderFrameToFbo()
{
QMutexLocker locker(&m_mutex);
realRenderFrameToFbo();
}

void TextureVideoBuffer::realRenderFrameToFbo()
{
if (!m_size.isValid()) {
return;
}

realCreateGLResources();

glBindTexture(GL_TEXTURE_EXTERNAL_OES, m_textureId);

// save current render states
GLboolean stencilTestEnabled;
GLboolean depthTestEnabled;
GLboolean scissorTestEnabled;
GLboolean blendEnabled;
glGetBooleanv(GL_STENCIL_TEST, &stencilTestEnabled);
glGetBooleanv(GL_DEPTH_TEST, &depthTestEnabled);
glGetBooleanv(GL_SCISSOR_TEST, &scissorTestEnabled);
glGetBooleanv(GL_BLEND, &blendEnabled);

if (stencilTestEnabled) glDisable(GL_STENCIL_TEST);
if (depthTestEnabled) glDisable(GL_DEPTH_TEST);
if (scissorTestEnabled) glDisable(GL_SCISSOR_TEST);
if (blendEnabled) glDisable(GL_BLEND);

m_fbo->bind();

glViewport(0, 0, m_size.width(), m_size.height());

m_program->bind();
m_program->enableAttributeArray(0);
m_program->enableAttributeArray(1);
m_program->setUniformValue("frameTexture", GLuint(0));
m_program->setUniformValue("texMatrix", QMatrix4x4());

static const GLfloat g_vertex_data[] = {
-1.0f, 1.0f, 1.0f, 1.0f,
1.0f, -1.0f, -1.0f, -1.0f
};
static const GLfloat g_texture_data[] = {
0.0f, 0.0f, 1.0f, 0.0f,
1.0f, 1.0f, 0.0f, 1.0f
};

glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 0, g_vertex_data);
glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 0, g_texture_data);

// draw primitive
glDrawArrays(GL_TRIANGLE_FAN, 0, 4);

m_program->disableAttributeArray(0);
m_program->disableAttributeArray(1);

glBindTexture(GL_TEXTURE_EXTERNAL_OES, 0);
// Switch rendering back to the default, windowing
// system provided framebuffer.
m_fbo->release();

// restore render states
if (stencilTestEnabled) glEnable(GL_STENCIL_TEST);
if (depthTestEnabled) glEnable(GL_DEPTH_TEST);
if (scissorTestEnabled) glEnable(GL_SCISSOR_TEST);
if (blendEnabled) glEnable(GL_BLEND);
}
} //namespace NemoVideoBackend

0 comments on commit 262494e

Please sign in to comment.