<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[206257] trunk/Source/WebCore</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://trac.webkit.org/projects/webkit/changeset/206257">206257</a></dd>
<dt>Author</dt> <dd>commit-queue@webkit.org</dd>
<dt>Date</dt> <dd>2016-09-22 05:43:55 -0700 (Thu, 22 Sep 2016)</dd>
</dl>
<h3>Log Message</h3>
<pre>[GStreamer] Support a direct GPU-to-GPU copy of video textures to WebGL
https://bugs.webkit.org/show_bug.cgi?id=159928
Patch by Olivier Blin <olivier.blin@softathome.com> on 2016-09-22
Reviewed by Carlos Garcia Campos.
Copy of GStreamer video frames to WebGL textures was not accelerated.
WebGLRenderingContextBase::texImage2D(HTMLVideoElement) went
through a slow software paint() wrapping the video frame into
cairo surface (ImageGStreamer), downloading it to draw to the
image cache context, copying it again in software, and uploading
it back to an OpenGL texture.
This patch implements copyVideoTextureToPlatformTexture() for
the GStreamer media player backend, to do GPU-to-GPU copy, by
extracting code from nativeImageForCurrentTime().
Doing this also fixes bug #159621: red and blue colors were
swapped in video rendered through WebGL with GSTREAMER_GL enabled.
* platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp:
(WebCore::MediaPlayerPrivateGStreamerBase::prepareContextForCairoPaint):
(WebCore::MediaPlayerPrivateGStreamerBase::paintToCairoSurface):
(WebCore::MediaPlayerPrivateGStreamerBase::copyVideoTextureToPlatformTexture):
(WebCore::MediaPlayerPrivateGStreamerBase::nativeImageForCurrentTime):
* platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.h:</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#trunkSourceWebCoreChangeLog">trunk/Source/WebCore/ChangeLog</a></li>
<li><a href="#trunkSourceWebCoreplatformgraphicsgstreamerMediaPlayerPrivateGStreamerBasecpp">trunk/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp</a></li>
<li><a href="#trunkSourceWebCoreplatformgraphicsgstreamerMediaPlayerPrivateGStreamerBaseh">trunk/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.h</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="trunkSourceWebCoreChangeLog"></a>
<div class="modfile"><h4>Modified: trunk/Source/WebCore/ChangeLog (206256 => 206257)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Source/WebCore/ChangeLog 2016-09-22 12:14:39 UTC (rev 206256)
+++ trunk/Source/WebCore/ChangeLog 2016-09-22 12:43:55 UTC (rev 206257)
</span><span class="lines">@@ -1,3 +1,32 @@
</span><ins>+2016-09-22 Olivier Blin <olivier.blin@softathome.com>
+
+ [GStreamer] Support a direct GPU-to-GPU copy of video textures to WebGL
+ https://bugs.webkit.org/show_bug.cgi?id=159928
+
+ Reviewed by Carlos Garcia Campos.
+
+ Copy of GStreamer video frames to WebGL textures was not accelerated.
+
+ WebGLRenderingContextBase::texImage2D(HTMLVideoElement) went
+ through a slow software paint() wrapping the video frame into
+ cairo surface (ImageGStreamer), downloading it to draw to the
+ image cache context, copying it again in software, and uploading
+ it back to an OpenGL texture.
+
+ This patch implements copyVideoTextureToPlatformTexture() for
+ the GStreamer media player backend, to do GPU-to-GPU copy, by
+ extracting code from nativeImageForCurrentTime().
+
+ Doing this also fixes bug #159621: red and blue colors were
+ swapped in video rendered through WebGL with GSTREAMER_GL enabled.
+
+ * platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp:
+ (WebCore::MediaPlayerPrivateGStreamerBase::prepareContextForCairoPaint):
+ (WebCore::MediaPlayerPrivateGStreamerBase::paintToCairoSurface):
+ (WebCore::MediaPlayerPrivateGStreamerBase::copyVideoTextureToPlatformTexture):
+ (WebCore::MediaPlayerPrivateGStreamerBase::nativeImageForCurrentTime):
+ * platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.h:
+
</ins><span class="cx"> 2016-09-22 Carlos Garcia Campos <cgarcia@igalia.com>
</span><span class="cx">
</span><span class="cx"> [GTK] Rename DataObjectGtk as SelectionData
</span></span></pre></div>
<a id="trunkSourceWebCoreplatformgraphicsgstreamerMediaPlayerPrivateGStreamerBasecpp"></a>
<div class="modfile"><h4>Modified: trunk/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp (206256 => 206257)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp 2016-09-22 12:14:39 UTC (rev 206256)
+++ trunk/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp 2016-09-22 12:43:55 UTC (rev 206257)
</span><span class="lines">@@ -697,38 +697,36 @@
</span><span class="cx"> #endif
</span><span class="cx">
</span><span class="cx"> #if USE(GSTREAMER_GL)
</span><del>-NativeImagePtr MediaPlayerPrivateGStreamerBase::nativeImageForCurrentTime()
</del><ins>+// This should be called with the sample mutex locked.
+GLContext* MediaPlayerPrivateGStreamerBase::prepareContextForCairoPaint(GstVideoInfo& videoInfo, IntSize& size, IntSize& rotatedSize)
</ins><span class="cx"> {
</span><del>-#if USE(CAIRO) && ENABLE(ACCELERATED_2D_CANVAS)
- if (m_usingFallbackVideoSink)
</del><ins>+ if (!getSampleVideoInfo(m_sample.get(), videoInfo))
</ins><span class="cx"> return nullptr;
</span><span class="cx">
</span><del>- WTF::GMutexLocker<GMutex> lock(m_sampleMutex);
</del><ins>+ GLContext* context = PlatformDisplay::sharedDisplayForCompositing().sharingGLContext();
+ context->makeContextCurrent();
</ins><span class="cx">
</span><del>- GstVideoInfo videoInfo;
- if (!getSampleVideoInfo(m_sample.get(), videoInfo))
- return nullptr;
</del><ins>+ // Thread-awareness is a huge performance hit on non-Intel drivers.
+ cairo_gl_device_set_thread_aware(context->cairoDevice(), FALSE);
</ins><span class="cx">
</span><ins>+ size = IntSize(GST_VIDEO_INFO_WIDTH(&videoInfo), GST_VIDEO_INFO_HEIGHT(&videoInfo));
+ rotatedSize = m_videoSourceOrientation.usesWidthAsHeight() ? size.transposedSize() : size;
+
+ return context;
+}
+
+// This should be called with the sample mutex locked.
+bool MediaPlayerPrivateGStreamerBase::paintToCairoSurface(cairo_surface_t* outputSurface, cairo_device_t* device, GstVideoInfo& videoInfo, const IntSize& size, const IntSize& rotatedSize)
+{
</ins><span class="cx"> GstBuffer* buffer = gst_sample_get_buffer(m_sample.get());
</span><span class="cx"> GstVideoFrame videoFrame;
</span><span class="cx"> if (!gst_video_frame_map(&videoFrame, &videoInfo, buffer, static_cast<GstMapFlags>(GST_MAP_READ | GST_MAP_GL)))
</span><del>- return nullptr;
</del><ins>+ return false;
</ins><span class="cx">
</span><del>- GLContext* context = PlatformDisplay::sharedDisplayForCompositing().sharingGLContext();
- context->makeContextCurrent();
- cairo_device_t* device = context->cairoDevice();
-
- // Thread-awareness is a huge performance hit on non-Intel drivers.
- cairo_gl_device_set_thread_aware(device, FALSE);
-
</del><span class="cx"> unsigned textureID = *reinterpret_cast<unsigned*>(videoFrame.data[0]);
</span><del>- IntSize size = IntSize(GST_VIDEO_INFO_WIDTH(&videoInfo), GST_VIDEO_INFO_HEIGHT(&videoInfo));
</del><span class="cx"> RefPtr<cairo_surface_t> surface = adoptRef(cairo_gl_surface_create_for_texture(device, CAIRO_CONTENT_COLOR_ALPHA, textureID, size.width(), size.height()));
</span><ins>+ RefPtr<cairo_t> cr = adoptRef(cairo_create(outputSurface));
</ins><span class="cx">
</span><del>- IntSize rotatedSize = m_videoSourceOrientation.usesWidthAsHeight() ? size.transposedSize() : size;
- RefPtr<cairo_surface_t> rotatedSurface = adoptRef(cairo_gl_surface_create(device, CAIRO_CONTENT_COLOR_ALPHA, rotatedSize.width(), rotatedSize.height()));
- RefPtr<cairo_t> cr = adoptRef(cairo_create(rotatedSurface.get()));
-
</del><span class="cx"> switch (m_videoSourceOrientation) {
</span><span class="cx"> case DefaultImageOrientation:
</span><span class="cx"> break;
</span><span class="lines">@@ -757,6 +755,65 @@
</span><span class="cx">
</span><span class="cx"> gst_video_frame_unmap(&videoFrame);
</span><span class="cx">
</span><ins>+ return true;
+}
+
+bool MediaPlayerPrivateGStreamerBase::copyVideoTextureToPlatformTexture(GraphicsContext3D* context, Platform3DObject outputTexture, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type, bool premultiplyAlpha, bool flipY)
+{
+#if !USE(CAIRO)
+ return false;
+#endif
+
+ if (m_usingFallbackVideoSink)
+ return false;
+
+ if (flipY || premultiplyAlpha)
+ return false;
+
+ GstVideoInfo videoInfo;
+ IntSize size, rotatedSize;
+ WTF::GMutexLocker<GMutex> lock(m_sampleMutex);
+ GLContext* glContext = prepareContextForCairoPaint(videoInfo, size, rotatedSize);
+ if (!glContext)
+ return false;
+
+ // Allocate uninitialized memory for the output texture.
+ context->bindTexture(outputTarget, outputTexture);
+ context->texImage2DDirect(outputTarget, level, internalFormat, rotatedSize.width(), rotatedSize.height(), 0, format, type, nullptr);
+
+ // cairo_gl_surface_create_for_texture sets these parameters to GL_NEAREST, so we need to backup them.
+ GC3Dint minFilter, magFilter;
+ context->getTexParameteriv(outputTarget, GL_TEXTURE_MIN_FILTER, &minFilter);
+ context->getTexParameteriv(outputTarget, GL_TEXTURE_MAG_FILTER, &magFilter);
+
+ RefPtr<cairo_surface_t> outputSurface = adoptRef(cairo_gl_surface_create_for_texture(glContext->cairoDevice(), CAIRO_CONTENT_COLOR_ALPHA, outputTexture, rotatedSize.width(), rotatedSize.height()));
+ if (!paintToCairoSurface(outputSurface.get(), glContext->cairoDevice(), videoInfo, size, rotatedSize))
+ return false;
+
+ context->bindTexture(outputTarget, outputTexture);
+ context->texParameteri(outputTarget, GraphicsContext3D::TEXTURE_MIN_FILTER, minFilter);
+ context->texParameteri(outputTarget, GraphicsContext3D::TEXTURE_MAG_FILTER, magFilter);
+
+ return true;
+}
+
+NativeImagePtr MediaPlayerPrivateGStreamerBase::nativeImageForCurrentTime()
+{
+#if USE(CAIRO) && ENABLE(ACCELERATED_2D_CANVAS)
+ if (m_usingFallbackVideoSink)
+ return nullptr;
+
+ GstVideoInfo videoInfo;
+ IntSize size, rotatedSize;
+ WTF::GMutexLocker<GMutex> lock(m_sampleMutex);
+ GLContext* context = prepareContextForCairoPaint(videoInfo, size, rotatedSize);
+ if (!context)
+ return nullptr;
+
+ RefPtr<cairo_surface_t> rotatedSurface = adoptRef(cairo_gl_surface_create(context->cairoDevice(), CAIRO_CONTENT_COLOR_ALPHA, rotatedSize.width(), rotatedSize.height()));
+ if (!paintToCairoSurface(rotatedSurface.get(), context->cairoDevice(), videoInfo, size, rotatedSize))
+ return nullptr;
+
</ins><span class="cx"> return rotatedSurface;
</span><span class="cx"> #else
</span><span class="cx"> return nullptr;
</span></span></pre></div>
<a id="trunkSourceWebCoreplatformgraphicsgstreamerMediaPlayerPrivateGStreamerBaseh"></a>
<div class="modfile"><h4>Modified: trunk/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.h (206256 => 206257)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.h 2016-09-22 12:14:39 UTC (rev 206256)
+++ trunk/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.h 2016-09-22 12:43:55 UTC (rev 206257)
</span><span class="lines">@@ -47,6 +47,7 @@
</span><span class="cx"> namespace WebCore {
</span><span class="cx">
</span><span class="cx"> class BitmapTextureGL;
</span><ins>+class GLContext;
</ins><span class="cx"> class GraphicsContext;
</span><span class="cx"> class GraphicsContext3D;
</span><span class="cx"> class IntSize;
</span><span class="lines">@@ -116,6 +117,7 @@
</span><span class="cx"> #endif
</span><span class="cx">
</span><span class="cx"> #if USE(GSTREAMER_GL)
</span><ins>+ bool copyVideoTextureToPlatformTexture(GraphicsContext3D*, Platform3DObject, GC3Denum, GC3Dint, GC3Denum, GC3Denum, GC3Denum, bool, bool) override;
</ins><span class="cx"> NativeImagePtr nativeImageForCurrentTime() override;
</span><span class="cx"> #endif
</span><span class="cx">
</span><span class="lines">@@ -130,6 +132,8 @@
</span><span class="cx"> static GstFlowReturn newPrerollCallback(GstElement*, MediaPlayerPrivateGStreamerBase*);
</span><span class="cx"> GstElement* createGLAppSink();
</span><span class="cx"> GstElement* createVideoSinkGL();
</span><ins>+ GLContext* prepareContextForCairoPaint(GstVideoInfo&, IntSize&, IntSize&);
+ bool paintToCairoSurface(cairo_surface_t*, cairo_device_t*, GstVideoInfo&, const IntSize&, const IntSize&);
</ins><span class="cx"> #endif
</span><span class="cx">
</span><span class="cx"> void setStreamVolumeElement(GstStreamVolume*);
</span></span></pre>
</div>
</div>
</body>
</html>