-
-
Notifications
You must be signed in to change notification settings - Fork 10.3k
/
ScreenEncoder.java
257 lines (221 loc) · 10.1 KB
/
ScreenEncoder.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
package com.genymobile.scrcpy;
import com.genymobile.scrcpy.wrappers.SurfaceControl;
import android.graphics.Rect;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaCodecList;
import android.media.MediaFormat;
import android.os.Build;
import android.os.IBinder;
import android.view.Surface;
import java.io.FileDescriptor;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
public class ScreenEncoder implements Device.RotationListener {
private static final int DEFAULT_I_FRAME_INTERVAL = 10; // seconds
private static final int REPEAT_FRAME_DELAY_US = 100_000; // repeat after 100ms
private static final String KEY_MAX_FPS_TO_ENCODER = "max-fps-to-encoder";
private static final int NO_PTS = -1;
private final AtomicBoolean rotationChanged = new AtomicBoolean();
private final ByteBuffer headerBuffer = ByteBuffer.allocate(12);
private String encoderName;
private List<CodecOption> codecOptions;
private int bitRate;
private int maxFps;
private boolean sendFrameMeta;
private long ptsOrigin;
public ScreenEncoder(boolean sendFrameMeta, int bitRate, int maxFps, List<CodecOption> codecOptions, String encoderName) {
this.sendFrameMeta = sendFrameMeta;
this.bitRate = bitRate;
this.maxFps = maxFps;
this.codecOptions = codecOptions;
this.encoderName = encoderName;
}
@Override
public void onRotationChanged(int rotation) {
rotationChanged.set(true);
}
public boolean consumeRotationChange() {
return rotationChanged.getAndSet(false);
}
public void streamScreen(Device device, FileDescriptor fd) throws IOException {
Workarounds.prepareMainLooper();
try {
internalStreamScreen(device, fd);
} catch (NullPointerException e) {
// Retry with workarounds enabled:
// <https://github.com/Genymobile/scrcpy/issues/365>
// <https://github.com/Genymobile/scrcpy/issues/940>
Ln.d("Applying workarounds to avoid NullPointerException");
Workarounds.fillAppInfo();
internalStreamScreen(device, fd);
}
}
private void internalStreamScreen(Device device, FileDescriptor fd) throws IOException {
MediaFormat format = createFormat(bitRate, maxFps, codecOptions);
device.setRotationListener(this);
boolean alive;
try {
do {
MediaCodec codec = createCodec(encoderName);
IBinder display = createDisplay();
ScreenInfo screenInfo = device.getScreenInfo();
Rect contentRect = screenInfo.getContentRect();
// include the locked video orientation
Rect videoRect = screenInfo.getVideoSize().toRect();
// does not include the locked video orientation
Rect unlockedVideoRect = screenInfo.getUnlockedVideoSize().toRect();
int videoRotation = screenInfo.getVideoRotation();
int layerStack = device.getLayerStack();
setSize(format, videoRect.width(), videoRect.height());
configure(codec, format);
Surface surface = codec.createInputSurface();
setDisplaySurface(display, surface, videoRotation, contentRect, unlockedVideoRect, layerStack);
codec.start();
try {
alive = encode(codec, fd);
// do not call stop() on exception, it would trigger an IllegalStateException
codec.stop();
} finally {
destroyDisplay(display);
codec.release();
surface.release();
}
} while (alive);
} finally {
device.setRotationListener(null);
}
}
private boolean encode(MediaCodec codec, FileDescriptor fd) throws IOException {
boolean eof = false;
MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
while (!consumeRotationChange() && !eof) {
int outputBufferId = codec.dequeueOutputBuffer(bufferInfo, -1);
eof = (bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
try {
if (consumeRotationChange()) {
// must restart encoding with new size
break;
}
if (outputBufferId >= 0) {
ByteBuffer codecBuffer = codec.getOutputBuffer(outputBufferId);
if (sendFrameMeta) {
writeFrameMeta(fd, bufferInfo, codecBuffer.remaining());
}
IO.writeFully(fd, codecBuffer);
}
} finally {
if (outputBufferId >= 0) {
codec.releaseOutputBuffer(outputBufferId, false);
}
}
}
return !eof;
}
private void writeFrameMeta(FileDescriptor fd, MediaCodec.BufferInfo bufferInfo, int packetSize) throws IOException {
headerBuffer.clear();
long pts;
if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
pts = NO_PTS; // non-media data packet
} else {
if (ptsOrigin == 0) {
ptsOrigin = bufferInfo.presentationTimeUs;
}
pts = bufferInfo.presentationTimeUs - ptsOrigin;
}
headerBuffer.putLong(pts);
headerBuffer.putInt(packetSize);
headerBuffer.flip();
IO.writeFully(fd, headerBuffer);
}
private static MediaCodecInfo[] listEncoders() {
List<MediaCodecInfo> result = new ArrayList<>();
MediaCodecList list = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
for (MediaCodecInfo codecInfo : list.getCodecInfos()) {
if (codecInfo.isEncoder() && Arrays.asList(codecInfo.getSupportedTypes()).contains(MediaFormat.MIMETYPE_VIDEO_AVC)) {
result.add(codecInfo);
}
}
return result.toArray(new MediaCodecInfo[result.size()]);
}
private static MediaCodec createCodec(String encoderName) throws IOException {
if (encoderName != null) {
Ln.d("Creating encoder by name: '" + encoderName + "'");
try {
return MediaCodec.createByCodecName(encoderName);
} catch (IllegalArgumentException e) {
MediaCodecInfo[] encoders = listEncoders();
throw new InvalidEncoderException(encoderName, encoders);
}
}
MediaCodec codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_VIDEO_AVC);
Ln.d("Using encoder: '" + codec.getName() + "'");
return codec;
}
private static void setCodecOption(MediaFormat format, CodecOption codecOption) {
String key = codecOption.getKey();
Object value = codecOption.getValue();
if (value instanceof Integer) {
format.setInteger(key, (Integer) value);
} else if (value instanceof Long) {
format.setLong(key, (Long) value);
} else if (value instanceof Float) {
format.setFloat(key, (Float) value);
} else if (value instanceof String) {
format.setString(key, (String) value);
}
Ln.d("Codec option set: " + key + " (" + value.getClass().getSimpleName() + ") = " + value);
}
private static MediaFormat createFormat(int bitRate, int maxFps, List<CodecOption> codecOptions) {
MediaFormat format = new MediaFormat();
format.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_VIDEO_AVC);
format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);
// must be present to configure the encoder, but does not impact the actual frame rate, which is variable
format.setInteger(MediaFormat.KEY_FRAME_RATE, 60);
format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, DEFAULT_I_FRAME_INTERVAL);
// display the very first frame, and recover from bad quality when no new frames
format.setLong(MediaFormat.KEY_REPEAT_PREVIOUS_FRAME_AFTER, REPEAT_FRAME_DELAY_US); // µs
if (maxFps > 0) {
// The key existed privately before Android 10:
// <https://android.googlesource.com/platform/frameworks/base/+/625f0aad9f7a259b6881006ad8710adce57d1384%5E%21/>
// <https://github.com/Genymobile/scrcpy/issues/488#issuecomment-567321437>
format.setFloat(KEY_MAX_FPS_TO_ENCODER, maxFps);
}
if (codecOptions != null) {
for (CodecOption option : codecOptions) {
setCodecOption(format, option);
}
}
return format;
}
private static IBinder createDisplay() {
// Since Android 12, secure displays could not be created with shell permissions anymore
boolean secure = Build.VERSION.SDK_INT <= Build.VERSION_CODES.R;
return SurfaceControl.createDisplay("scrcpy", secure);
}
private static void configure(MediaCodec codec, MediaFormat format) {
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
}
private static void setSize(MediaFormat format, int width, int height) {
format.setInteger(MediaFormat.KEY_WIDTH, width);
format.setInteger(MediaFormat.KEY_HEIGHT, height);
}
private static void setDisplaySurface(IBinder display, Surface surface, int orientation, Rect deviceRect, Rect displayRect, int layerStack) {
SurfaceControl.openTransaction();
try {
SurfaceControl.setDisplaySurface(display, surface);
SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect);
SurfaceControl.setDisplayLayerStack(display, layerStack);
} finally {
SurfaceControl.closeTransaction();
}
}
private static void destroyDisplay(IBinder display) {
SurfaceControl.destroyDisplay(display);
}
}