Skip to content
This repository has been archived by the owner on Mar 14, 2018. It is now read-only.

Commit

Permalink
revised: Android support: now using raw capture, which cuts down the …
Browse files Browse the repository at this point in the history 
…capture elapsed time to about 40% for the whole screen and even down to less than 10% for parts of the screen (the faster, the nearer the capture region is to the top of the screen ) compared to "capture -p" (returning png format image).
  • Loading branch information
@RaiMan
RaiMan committed Jul 15, 2016
1 parent f0cef76 commit 0169775
Show file tree
Hide file tree
Showing 4 changed files with 100 additions and 81 deletions.
6 changes: 3 additions & 3 deletions API/src/main/java/org/sikuli/android/ADBClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ private static void init() {
if (jadb == null) {
try {
new AdbServerLauncher().launch();
Debug.info("ADBClient: ADBServer started");
Debug.log(3, "ADBClient: ADBServer started");
getConnection(false);
if (jadb != null) {
shouldStopServer = true;
Expand Down Expand Up @@ -66,7 +66,7 @@ private static void init() {
}
}
if (device != null) {
Debug.info("ADBClient: init: attached device: serial(%s) state(%s)", serial, state);
Debug.log(3, "ADBClient: init: attached device: serial(%s) state(%s)", serial, state);
}
}

Expand All @@ -75,7 +75,7 @@ private static void getConnection(boolean quiet) {
try {
jadb = new JadbConnection();
jadb.getHostVersion();
Debug.info("ADBClient: ADBServer connection established");
Debug.log(3, "ADBClient: ADBServer connection established");
} catch (Exception e) {
if (!quiet) {
Debug.error("ADBClient: ADBServer connection not possible: %s", e.getMessage());
Expand Down
109 changes: 65 additions & 44 deletions API/src/main/java/org/sikuli/android/ADBDevice.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@

package org.sikuli.android;

import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.imgproc.Imgproc;
import org.sikuli.basics.Debug;
import org.sikuli.script.RunTime;
import org.sikuli.script.ScreenImage;
Expand All @@ -15,10 +18,10 @@
import javax.imageio.ImageIO;
import java.awt.*;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferByte;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

Expand All @@ -31,8 +34,8 @@ private static void log(int level, String message, Object... args) {
}

private JadbDevice device = null;
private int w = 0;
private int h = 0;
private int devW = -1;
private int devH = -1;
private ADBRobot robot = null;
private ADBScreen screen = null;

Expand Down Expand Up @@ -70,72 +73,90 @@ public String getDeviceSerial() {
}

public Rectangle getBounds() {
Dimension dim = getDisplayDimension();
w = (int) dim.getWidth();
h = (int) dim.getHeight();
return new Rectangle(0, 0, w, h);
if (devW < 0) {
Dimension dim = getDisplayDimension();
devW = (int) dim.getWidth();
devH = (int) dim.getHeight();
}
return new Rectangle(0, 0, devW, devH);
}

public ScreenImage captureScreen(Rectangle screenRect) {
public ScreenImage captureScreen() {
BufferedImage bimg = captureDeviceScreen();
BufferedImage subBimg = bimg.getSubimage(screenRect.x, screenRect.y, screenRect.width, screenRect.height);
return new ScreenImage(screenRect, subBimg);
return new ScreenImage(getBounds(), bimg);
}

private BufferedImage captureDeviceScreen() {
BufferedImage bImg = null;
Debug timer = Debug.startTimer();
try {
InputStream stdout = device.executeShell("screencap", "-p");
bImg = ImageIO.read(stdout);
log(lvl, "capture: %d", timer.end());
} catch (IOException | JadbException e) {
log(-1, "capture: %s", e);
}
return bImg;
public ScreenImage captureScreen(Rectangle rect) {
BufferedImage bimg = captureDeviceScreen(rect.x, rect.y, rect.width, rect.height);
return new ScreenImage(rect, bimg);
}

public BufferedImage captureDeviceScreen() {
return captureDeviceScreen(0, 0, devW, devH);
}

public byte[] captureDeviceScreenRaw() {
return captureDeviceScreenRaw(0, 0, w, h);
public BufferedImage captureDeviceScreen(int y, int _h) {
return captureDeviceScreen(0, y, devW, _h);
}

public byte[] captureDeviceScreenRaw(int y, int _h) {
return captureDeviceScreenRaw(0, y, w, _h);
public BufferedImage captureDeviceScreen(int x, int y, int w, int h) {
Mat matImage = captureDeviceScreenMat(x, y, w, h);
BufferedImage bImage = null;
if (matImage != null) {
bImage = new BufferedImage(matImage.width(), matImage.height(), BufferedImage.TYPE_3BYTE_BGR);
byte[] bImageData = ((DataBufferByte) bImage.getRaster().getDataBuffer()).getData();
matImage.get(0, 0, bImageData);
}
return bImage;
}

public byte[] captureDeviceScreenRaw(int x, int y, int _w, int _h) {
public Mat captureDeviceScreenMat(int x, int y, int w, int h) {
byte[] imagePrefix = new byte[12];
byte[] image = new byte[0];
if (x + _w > w) {
_w = w - x;
int actW = w;
if (x + w > devW) {
actW = devW - x;
}
if (y + _h > h) {
_h = h - y;
int actH = h;
if (y + h > devH) {
actH = devH - y;
}
Debug timer = Debug.startTimer();
try {
InputStream stdout = device.executeShell("screencap");
stdout.read(imagePrefix);
if (imagePrefix[8] != 0x01) {
log(-1, "captureDeviceScreenRaw: image type not RGBA");
return image;
log(-1, "captureDeviceScreenMat: image type not RGBA");
return null;
}
if (byte2int(imagePrefix, 0, 4) != devW || byte2int(imagePrefix, 4, 4) != devH) {
log(-1, "captureDeviceScreenMat: width or height differ from device values");
return null;
}
int imgW = byte2int(imagePrefix, 0, 4);
int imgH = byte2int(imagePrefix, 4, 4);
byte[] imageRow = new byte[imgW * 4];
image = new byte[_w * _h * 4];
stdout.skip(y * imgW * 4);
for (int count = 0; count < _h; count++) {
stdout.skip(x * 4);
stdout.read(image, count * _w * 4, _w * 4);
stdout.skip((imgW - x -_w) * 4);
image = new byte[actW * actH * 4];
int lenRow = devW * 4;
byte[] row = new byte[lenRow];
for (int count = 0; count < y; count++) {
stdout.read(row);
}
for (int count = 0; count < actH; count++) {
if (x > 0) {
stdout.read(row);
System.arraycopy(row, x * 4, image, count * actW * 4, actW * 4);
} else {
stdout.read(image, count * actW * 4, actW * 4);
}
}
long duration = timer.end();
log(lvl, "raw-capture:[%d,%d %dx%d] %d", x, y, _w, _h, duration);
log(lvl, "captureDeviceScreenMat:[%d,%d %dx%d] %d", x, y, actW, actH, duration);
} catch (IOException | JadbException e) {
log(-1, "raw-capture: [%d,%d %dx%d] %s", x, y, _w, _h, e);
log(-1, "captureDeviceScreenMat: [%d,%d %dx%d] %s", x, y, actW, actH, e);
}
return image;
Mat matOrg = new Mat(actH, actW, CvType.CV_8UC4);
matOrg.put(0, 0, image);
Mat matImage = new Mat();
Imgproc.cvtColor(matOrg, matImage, Imgproc.COLOR_RGBA2BGR, 3);
return matImage;
}

private int byte2int(byte[] bytes, int start, int len) {
Expand Down
13 changes: 5 additions & 8 deletions API/src/main/java/org/sikuli/android/ADBScreen.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@
*/
public class ADBScreen extends Region implements EventObserver, IScreen {

static {
RunTime.loadLibrary("VisionProxy");
}

private static String me = "ADBScreen: ";
private static void log(int level, String message, Object... args) {
Debug.logx(level, me + message, args);
Expand Down Expand Up @@ -204,17 +208,10 @@ public ScreenImage capture(Region reg) {
return capture(reg.x, reg.y, reg.w, reg.h);
}

// @Override
@Override
public ScreenImage capture(Rectangle rect) {
return capture(rect.x, rect.y, rect.width, rect.height);
}
public byte[] rawcapture(Rectangle rect) {
ScreenImage sImg = null;
Debug timer = Debug.startTimer();
byte[] imageBytes = device.captureDeviceScreenRaw(rect.x, rect.y, rect.width, rect.height);
long duration = timer.end();
return imageBytes;
}

public void showTarget(Location loc) {
showTarget(loc, Settings.SlowMotionDelay);
Expand Down
53 changes: 27 additions & 26 deletions API/src/main/java/org/sikuli/android/ADBTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,18 @@

package org.sikuli.android;

import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.highgui.Highgui;
import org.opencv.imgproc.Imgproc;
import org.sikuli.basics.Debug;
import org.sikuli.script.*;
import org.sikuli.script.Image;

import java.awt.*;
import java.awt.image.BufferedImage;
import java.awt.image.DataBufferByte;
import java.io.File;

/**
* Created by RaiMan on 12.07.16.
Expand All @@ -29,17 +36,18 @@ private static void logp(String message, Object... args) {
System.out.println(String.format(message, args));
}

private static RunTime rt = null;

public static void main(String[] args) throws FindFailed {

boolean runTests = false;
boolean runTests = true;

ADBScreen aScr = startTest();

if (aScr.isValid()) {
if (runTests) {

basicTest(aScr);
captureRawTest(aScr);

System.exit(0);
}
Expand All @@ -49,16 +57,20 @@ public static void main(String[] args) throws FindFailed {

// ********* playground
Rectangle rect = null;
byte[] image = null;
for (int i = 0; i < 1920; i += 100) {
rect = new Rectangle(0, i, 1200, 100);
image = aScr.rawcapture(rect);
}
ScreenImage image = null;
int x = 248;
int y = 188;
int h = 128;
int w = 128;
rect = new Rectangle(x, y, w, h);
image = aScr.capture(rect);
logp("%s", image);
}

private static ADBScreen startTest() {
ADBScreen adbs = new ADBScreen();
Debug.on(3);
rt = RunTime.get();
ADBScreen adbs = new ADBScreen();
log(lvl, "%s", adbs);
if (adbs.isValid()) {
adbs.wakeUp(2);
Expand All @@ -71,23 +83,12 @@ private static ADBScreen startTest() {


private static void basicTest(ADBScreen adbs) throws FindFailed {
Debug.info("**************** running basic test");
adbs.swipeLeft();
adbs.swipeRight();
adbs.wait(1f);
ScreenImage sIMg = adbs.userCapture("Android");
sIMg.save(RunTime.get().fSikulixStore.getAbsolutePath(), "android");
adbs.tap(new Image(sIMg));
}

private static void captureRawTest(ADBScreen adbs) {
Debug.info("**************** running test raw capture");
byte[] image = null;
image = adbs.getDevice().captureDeviceScreenRaw();
image = adbs.getDevice().captureDeviceScreenRaw(0, 250);
image = adbs.getDevice().captureDeviceScreenRaw((int) (adbs.h / 2 - 125), 250);
image = adbs.getDevice().captureDeviceScreenRaw(adbs.h - 250 - 1, 250);
image = adbs.getDevice().captureDeviceScreenRaw(500, 1000, 250, 250);
image = adbs.getDevice().captureDeviceScreenRaw(adbs.getCenter().x, adbs.getCenter().y, 1, 1);
log(lvl, "**************** running basic test");
adbs.swipeLeft();
adbs.swipeRight();
adbs.wait(1f);
ScreenImage sIMg = adbs.userCapture("Android");
sIMg.save(RunTime.get().fSikulixStore.getAbsolutePath(), "android");
adbs.tap(new Image(sIMg));
}
}

0 comments on commit 0169775

Please sign in to comment.