-
Notifications
You must be signed in to change notification settings - Fork 1
/
GstVideoComponent.java
452 lines (409 loc) · 15.8 KB
/
GstVideoComponent.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
/*
* Copyright (c) 2019 Neil C Smith
* Copyright (c) 2007 Wayne Meissner
*
* This code is free software: you can redistribute it and/or modify it under
* the terms of the GNU Lesser General Public License version 3 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
* version 3 for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* version 3 along with this work. If not, see <http://www.gnu.org/licenses/>.
*/
package org.freedesktop.gstreamer.swing;
import java.awt.Color;
import java.awt.Component;
import java.awt.Dimension;
import java.awt.Graphics;
import java.awt.Graphics2D;
import java.awt.GraphicsConfiguration;
import java.awt.RenderingHints;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.awt.event.ComponentAdapter;
import java.awt.event.ComponentEvent;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferInt;
import java.awt.image.VolatileImage;
import java.beans.PropertyChangeEvent;
import java.beans.PropertyChangeListener;
import java.nio.IntBuffer;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import javax.swing.SwingUtilities;
import javax.swing.Timer;
import org.freedesktop.gstreamer.Element;
import org.freedesktop.gstreamer.Structure;
import org.freedesktop.gstreamer.elements.AppSink;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import org.freedesktop.gstreamer.Buffer;
import org.freedesktop.gstreamer.Caps;
import org.freedesktop.gstreamer.FlowReturn;
import org.freedesktop.gstreamer.Sample;
/**
* A Swing component for displaying video from a GStreamer pipeline.
*/
public class GstVideoComponent extends javax.swing.JComponent {
private final Lock bufferLock = new ReentrantLock();
private final AppSink videosink;
private final boolean useVolatile;
private BufferedImage currentImage = null;
private RenderComponent renderComponent = new RenderComponent();
private boolean keepAspect = true;
private Timer resourceTimer;
private VolatileImage volatileImage;
private boolean frameRendered = false;
private volatile boolean updatePending = false;
/**
* Create a GstVideoComponent. A new AppSink element will be created that
* can be accessed using {@link #getElement()} and added to a pipeline.
*/
public GstVideoComponent() {
this(new AppSink("GstVideoComponent"));
}
/**
* Create a GstVideoComponent wrapping the provided AppSink element.
*/
public GstVideoComponent(AppSink appsink) {
this.videosink = appsink;
videosink.set("emit-signals", true);
AppSinkListener listener = new AppSinkListener();
videosink.connect((AppSink.NEW_SAMPLE) listener);
videosink.connect((AppSink.NEW_PREROLL) listener);
StringBuilder caps = new StringBuilder("video/x-raw,pixel-aspect-ratio=1/1,");
// JNA creates ByteBuffer using native byte order, set masks according to that.
if (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) {
caps.append("format=BGRx");
} else {
caps.append("format=xRGB");
}
videosink.setCaps(new Caps(caps.toString()));
useVolatile = true;
// Kick off a timer to free up the volatile image if there have been no recent updates
// (e.g. the player is paused)
//
resourceTimer = new Timer(250, resourceReaper);
//
// Don't use a layout manager - the output component will positioned within this
// component according to the aspect ratio and scaling mode
//
setLayout(null);
add(renderComponent);
//
// Listen for the child changing its preferred size to the size of the
// video stream.
//
renderComponent.addPropertyChangeListener("preferredSize", new PropertyChangeListener() {
public void propertyChange(PropertyChangeEvent evt) {
setPreferredSize(renderComponent.getPreferredSize());
scaleVideoOutput();
}
});
//
// Scale the video output in response to this component being resized
//
addComponentListener(new ComponentAdapter() {
@Override
public void componentResized(ComponentEvent arg0) {
scaleVideoOutput();
}
});
renderComponent.setBounds(getBounds());
setOpaque(true);
setBackground(Color.BLACK);
}
/**
* Scales the video output component according to its aspect ratio
*/
private void scaleVideoOutput() {
final Component child = renderComponent;
final Dimension childSize = child.getPreferredSize();
final int width = getWidth(), height = getHeight();
// Figure out the aspect ratio
double aspect = keepAspect ? (double) childSize.width / (double) childSize.height : 1.0f;
//
// Now scale and position the videoChild component to be in the correct position
// to keep the aspect ratio correct.
//
int scaledHeight = (int) ((double) width / aspect);
if (!keepAspect) {
//
// Just make the child match the parent
//
child.setBounds(0, 0, width, height);
} else if (scaledHeight < height) {
//
// Output window is taller than the image is when scaled, so move the
// video component to sit vertically in the centre of the VideoComponent.
//
final int y = (height - scaledHeight) / 2;
child.setBounds(0, y, width, scaledHeight);
} else {
final int scaledWidth = (int) ((double) height * aspect);
final int x = (width - scaledWidth) / 2;
child.setBounds(x, 0, scaledWidth, height);
}
}
private ActionListener resourceReaper = new ActionListener() {
public void actionPerformed(ActionEvent arg0) {
if (!frameRendered) {
if (volatileImage != null) {
volatileImage.flush();
volatileImage = null;
}
// Stop the timer so we don't wakeup needlessly
resourceTimer.stop();
}
frameRendered = false;
}
};
/**
* Get the wrapped AppSink element.
*
* @return sink element
*/
public Element getElement() {
return videosink;
}
/**
* Set whether to respect the aspect ratio of the video when scaling.
* Defaults to true.
*
* @param keepAspect respect aspect ratio
*/
public void setKeepAspect(boolean keepAspect) {
this.keepAspect = keepAspect;
}
@Override
public boolean isLightweight() {
return true;
}
@Override
protected void paintComponent(Graphics g) {
if (isOpaque()) {
Graphics2D g2d = (Graphics2D) g.create();
g2d.setColor(getBackground());
g2d.fillRect(0, 0, getWidth(), getHeight());
g2d.dispose();
}
}
private class RenderComponent extends javax.swing.JComponent {
private static final long serialVersionUID = -4736605073704494268L;
@Override
protected void paintComponent(Graphics g) {
int width = getWidth(), height = getHeight();
Graphics2D g2d = (Graphics2D) g.create();
g2d.setRenderingHint(RenderingHints.KEY_INTERPOLATION,
RenderingHints.VALUE_INTERPOLATION_BILINEAR);
if (currentImage != null) {
GraphicsConfiguration gc = getGraphicsConfiguration();
render(g2d, 0, 0, width, height);
} else {
g2d.setColor(getBackground());
g2d.fillRect(0, 0, width, height);
}
g2d.dispose();
}
@Override
public boolean isOpaque() {
return GstVideoComponent.this.isOpaque();
}
@Override
public boolean isLightweight() {
return true;
}
}
private void renderVolatileImage(BufferedImage bufferedImage) {
do {
int w = bufferedImage.getWidth(), h = bufferedImage.getHeight();
GraphicsConfiguration gc = getGraphicsConfiguration();
if (volatileImage == null || volatileImage.getWidth() != w
|| volatileImage.getHeight() != h
|| volatileImage.validate(gc) == VolatileImage.IMAGE_INCOMPATIBLE) {
if (volatileImage != null) {
volatileImage.flush();
}
volatileImage = gc.createCompatibleVolatileImage(w, h);
volatileImage.setAccelerationPriority(1.0f);
}
//
// Now paint the BufferedImage into the accelerated image
//
Graphics2D g = volatileImage.createGraphics();
g.drawImage(bufferedImage, 0, 0, null);
g.dispose();
} while (volatileImage.contentsLost());
}
/**
* Renders to a volatile image, and then paints that to the screen. This
* helps with scaling performance on accelerated surfaces (e.g. OpenGL)
*
* @param g the graphics to paint the image to
* @param x the left coordinate to start painting at.
* @param y the top coordinate to start painting at.
* @param w the width of the paint area
* @param h the height of the paint area
*/
private void volatileRender(Graphics g, int x, int y, int w, int h) {
do {
if (updatePending || volatileImage == null
|| volatileImage.validate(getGraphicsConfiguration()) != VolatileImage.IMAGE_OK) {
bufferLock.lock();
try {
updatePending = false;
renderVolatileImage(currentImage);
} finally {
bufferLock.unlock();
}
}
g.drawImage(volatileImage, x, y, w, h, null);
} while (volatileImage.contentsLost());
}
/**
* Renders directly to the given <tt>Graphics</tt>. This is only really
* useful on MacOS where swing graphics are unaccelerated so using a
* volatile just incurs an extra memcpy().
*
* @param g the graphics to paint the image to
* @param x the left coordinate to start painting at.
* @param y the top coordinate to start painting at.
* @param w the width of the paint area
* @param h the height of the paint area
*/
private void heapRender(Graphics g, int x, int y, int w, int h) {
bufferLock.lock();
try {
updatePending = false;
g.drawImage(currentImage, x, y, w, h, null);
} finally {
bufferLock.unlock();
}
}
/**
* Renders the current frame to the given <tt>Graphics</tt>.
*
* @param g the graphics to paint the image to
* @param x the left coordinate to start painting at.
* @param y the top coordinate to start painting at.
* @param w the width of the paint area
* @param h the height of the paint area
*/
private void render(Graphics g, int x, int y, int w, int h) {
if (useVolatile) {
volatileRender(g, x, y, w, h);
} else {
heapRender(g, x, y, w, h);
}
//
// Restart the resource reaper timer if neccessary
//
if (!frameRendered) {
frameRendered = true;
if (!resourceTimer.isRunning()) {
resourceTimer.restart();
}
}
}
private int imgWidth = 0, imgHeight = 0;
private final void update(final int width, final int height) {
SwingUtilities.invokeLater(new Runnable() {
public void run() {
//
// If the image changed size, resize the component to fit
//
if (width != imgWidth || height != imgHeight) {
renderComponent.setPreferredSize(new Dimension(width, height));
imgWidth = width;
imgHeight = height;
}
if (renderComponent.isVisible()) {
renderComponent.paintImmediately(0, 0,
renderComponent.getWidth(), renderComponent.getHeight());
}
}
});
}
private BufferedImage getBufferedImage(int width, int height) {
if (currentImage != null && currentImage.getWidth() == width
&& currentImage.getHeight() == height) {
return currentImage;
}
if (currentImage != null) {
currentImage.flush();
}
currentImage = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
currentImage.setAccelerationPriority(0.0f);
return currentImage;
}
private class AppSinkListener implements AppSink.NEW_SAMPLE, AppSink.NEW_PREROLL {
@Override
public FlowReturn newSample(AppSink elem) {
Sample sample = elem.pullSample();
Structure capsStruct = sample.getCaps().getStructure(0);
int w = capsStruct.getInteger("width");
int h = capsStruct.getInteger("height");
Buffer buffer = sample.getBuffer();
ByteBuffer bb = buffer.map(false);
if (bb != null) {
rgbFrame(false, w, h, bb.asIntBuffer());
buffer.unmap();
}
sample.dispose();
return FlowReturn.OK;
}
@Override
public FlowReturn newPreroll(AppSink elem) {
Sample sample = elem.pullPreroll();
Structure capsStruct = sample.getCaps().getStructure(0);
int w = capsStruct.getInteger("width");
int h = capsStruct.getInteger("height");
Buffer buffer = sample.getBuffer();
ByteBuffer bb = buffer.map(false);
if (bb != null) {
rgbFrame(true, w, h, bb.asIntBuffer());
buffer.unmap();
}
sample.dispose();
return FlowReturn.OK;
}
private void rgbFrame(boolean isPrerollFrame, int width, int height, IntBuffer rgb) {
// If the EDT is still copying data from the buffer, just drop this frame
//
if (!bufferLock.tryLock()) {
return;
}
//
// If there is already a swing update pending, also drop this frame.
//
if (updatePending && !isPrerollFrame) {
bufferLock.unlock();
return;
}
try {
final BufferedImage renderImage = getBufferedImage(width, height);
int[] pixels = ((DataBufferInt) renderImage.getRaster().getDataBuffer()).getData();
rgb.get(pixels, 0, width * height);
updatePending = true;
} finally {
bufferLock.unlock();
}
// int scaledWidth = currentImage.getWidth();
// if (keepAspect) {
// // Scale width according to pixel aspect ratio.
// Caps videoCaps = videoPad.getNegotiatedCaps();
// Structure capsStruct = videoCaps.getStructure(0);
// if (capsStruct.hasField("pixel-aspect-ratio")) {
// Fraction pixelAspectRatio = capsStruct.getFraction("pixel-aspect-ratio");
// scaledWidth = scaledWidth * pixelAspectRatio.getNumerator() / pixelAspectRatio.getDenominator();
// }
// }
// Tell swing to use the new buffer
update(currentImage.getWidth(), currentImage.getHeight());
}
}
}