Merge branch 'dev' of https://projects.cispa.saarland/nataniel.borges…

…/droidmate into dev

.gitmodules

@@ -1,6 +1,6 @@
-[submodule "project/pcComponents/explorationModel"]
-	path = project/pcComponents/explorationModel
-	url = https://github.com/Hotzkow/ExplorationModel
+#[submodule "project/pcComponents/explorationModel"]
+#	path = project/pcComponents/explorationModel
+#	url = https://github.com/Hotzkow/ExplorationModel
 [submodule "project/deviceCommonLib/deviceDaemonLib"]
 	path = project/deviceCommonLib/deviceDaemonLib
 	url = https://github.com/Hotzkow/PlatformInterfaceLib

droidmate_usage_examples/src/main/kotlin/org/droidmate/example/ExampleModelFeature.kt

@@ -12,7 +12,7 @@ class ExampleModelFeature: ModelFeature(){
 	override val coroutineContext: CoroutineContext = CoroutineName("ExampleModelFeature")
 
 
-	override suspend fun onNewAction(traceId: UUID, interactions: List<Interaction>, prevState: State, newState: State) {
+	override suspend fun onNewAction(traceId: UUID, interactions: List<Interaction<*>>, prevState: State<*>, newState: State<*>) {
 		super.onNewAction(traceId, interactions, prevState, newState)
 		val firstAction = interactions.first()
 

project/deviceComponents/deviceControlDaemon/src/androidTest/java/org/droidmate/uiautomator2daemon/exploration/ActionExecution.kt

@@ -288,16 +288,17 @@ suspend fun fetchDeviceData(env: UiAutomationEnvironment, afterAction: Boolean =
 	debugOut("compute img pixels",debugFetch)
 	val imgPixels =	getOrStoreImgPixels(img,env)
 
+	var xml: String = "TODO parse widget list on Pc if we need the XML or introduce a debug property to enable parsing" +
+			", because (currently) we would have to traverse the tree a second time"
+	if(debugEnabled) xml = UiHierarchy.getXml(env)
+
 	env.lastResponse = DeviceResponse.create( isSuccessful = isSuccessful, uiHierarchy = uiHierarchy,
-			uiDump =
-			"TODO parse widget list on Pc if we need the XML or introduce a debug property to enable parsing" +
-					", because (currently) we would have to traverse the tree a second time"
-//									xmlDump
-			, launchedActivity = env.launchedMainActivity,
-			capturedScreen = img != null,
-			screenshot = imgPixels,
-			appWindows = windows.mapNotNull { if(it.isExtracted()) it.w else null },
-			isHomeScreen = windows.isHomeScreen()
+		uiDump = xml,
+		launchedActivity = env.launchedMainActivity,
+		capturedScreen = img != null,
+		screenshot = imgPixels,
+		appWindows = windows.mapNotNull { if(it.isExtracted()) it.w else null },
+		isHomeScreen = windows.isHomeScreen()
 	)
 
 	env.lastResponse

project/deviceComponents/deviceControlDaemon/src/androidTest/java/org/droidmate/uiautomator2daemon/uiautomatorExtensions/UiHierarchy.kt

@@ -6,7 +6,6 @@ import android.app.UiAutomation
 import android.graphics.Bitmap
 import android.graphics.Rect
 import android.support.test.uiautomator.NodeProcessor
-import android.support.test.uiautomator.UiDevice
 import android.support.test.uiautomator.apply
 import android.support.test.uiautomator.processTopDown
 import android.util.Log
@@ -56,24 +55,38 @@ object UiHierarchy : UiParser() {
 	}
 
 
-	suspend fun getXml(device: UiDevice):String = debugT(" fetching gui Dump ", {
+	suspend fun getXml(env: UiAutomationEnvironment):String = debugT(" fetching gui Dump ", {
+		val device = env.device
 		StringWriter().use { out ->
 			device.waitForIdle()
 
-			val serializer = Xml.newSerializer()
-			serializer.setFeature("http://xmlpull.org/v1/doc/modelFeatures.html#indent-output", true)
-			serializer.setOutput(out)//, "UTF-8")
-
-			serializer.startDocument("UTF-8", true)
-			serializer.startTag("", "hierarchy")
-			serializer.attribute("", "rotation", Integer.toString(device.displayRotation))
+			val ser = Xml.newSerializer()
+			ser.setOutput(out)//, "UTF-8")
+			ser.startDocument("UTF-8", true)
+			ser.setFeature("http://xmlpull.org/v1/doc/features.html#indent-output", true);
+
+			ser.startTag("", "hierarchy")
+			ser.attribute("", "rotation", Integer.toString(device.displayRotation))
+
+			ser.startTag("","windows")
+			env.lastWindows.forEach {
+				ser.startTag("","window")
+				ser.addAttribute("windowId",it.w.windowId)
+				ser.addAttribute("windowLayer",it.layer)
+				ser.addAttribute("isExtracted",it.isExtracted())
+				ser.addAttribute("isKeyboard",it.isKeyboard)
+				ser.addAttribute("windowType",it.windowType)
+				ser.addAttribute("boundaries",it.bounds)
+				ser.endTag("","window")
+			}
+			ser.endTag("","windows")
 
-			device.apply(nodeDumper(serializer, device.displayWidth, device.displayHeight)
-			) { serializer.endTag("", "node") }
+			device.apply(nodeDumper(ser, device.displayWidth, device.displayHeight)
+			) { ser.endTag("","node") } // need to set the anding tag in the post processor to have a proper 'tree' representation
 
-			serializer.endTag("", "hierarchy")
-			serializer.endDocument()
-			serializer.flush()
+			ser.endTag("", "hierarchy")
+			ser.endDocument()
+			ser.flush()
 			out.toString()
 		}
 	}, inMillis = true)

project/deviceComponents/deviceControlDaemon/src/androidTest/java/org/droidmate/uiautomator2daemon/uiautomatorExtensions/UiParser.kt

@@ -71,7 +71,7 @@ abstract class UiParser {
 			nodeRect.bottom = nodeRect.top+t.height()
 		}
 
-		props.add("origBounds (l,t,r,b)= $nodeRect")
+		props.add("defBounds (l,t,r,b)= $nodeRect")
 		props.add("boundsInParent= $t")
 		props.add("actionList = ${this.actionList}")
 		if(api>=24)	props.add("drawingOrder = ${this.drawingOrder}")
@@ -126,7 +126,7 @@ abstract class UiParser {
 				className = safeCharSeqToString(className),
 				packageName = safeCharSeqToString(packageName),
 				enabled = isEnabled,
-				isInputField = isEditable,
+				isInputField = isEditable || actionList.contains(AccessibilityNodeInfo.AccessibilityAction.ACTION_SET_TEXT),
 				isPassword = isPassword,
 				clickable = isClickable,
 				longClickable = isLongClickable,
@@ -177,35 +177,47 @@ abstract class UiParser {
 	*/
 
 	protected val nodeDumper:(serializer: XmlSerializer, width: Int, height: Int)-> NodeProcessor =
-			{ serializer: XmlSerializer, _: Int, _: Int ->
+			{ ser: XmlSerializer, _: Int, _: Int ->
 				{ node: AccessibilityNodeInfo, index: Int, _ ->
 					val nodeRect = Rect()
 					node.getBoundsInScreen(nodeRect)  // determine the 'overall' boundaries these may be outside of the app window or even outside of the screen
 
-					serializer.startTag("", "node")
-					if (!nafExcludedClass(node))
-						serializer.attribute("", "NAF", java.lang.Boolean.toString(true))
-					serializer.attribute("", "index", Integer.toString(index))
-					serializer.attribute("", "text", safeCharSeqToString(node.text))
-					serializer.attribute("", "resource-id", safeCharSeqToString(node.viewIdResourceName))
-					serializer.attribute("", "class", safeCharSeqToString(node.className))
-					serializer.attribute("", "package", safeCharSeqToString(node.packageName))
-					serializer.attribute("", "content-desc", safeCharSeqToString(node.contentDescription))
-					serializer.attribute("", "checkable", java.lang.Boolean.toString(node.isCheckable))
-					serializer.attribute("", "checked", java.lang.Boolean.toString(node.isChecked))
-					serializer.attribute("", "clickable", java.lang.Boolean.toString(node.isClickable))
-					serializer.attribute("", "enabled", java.lang.Boolean.toString(node.isEnabled))
-					serializer.attribute("", "focusable", java.lang.Boolean.toString(node.isFocusable))
-					serializer.attribute("", "focused", java.lang.Boolean.toString(node.isFocused))
-					serializer.attribute("", "scrollable", java.lang.Boolean.toString(node.isScrollable))
-					serializer.attribute("", "long-clickable", java.lang.Boolean.toString(node.isLongClickable))
-					serializer.attribute("", "password", java.lang.Boolean.toString(node.isPassword))
-					serializer.attribute("", "selected", java.lang.Boolean.toString(node.isSelected))
-					serializer.attribute("", "definedAsVisible-to-user", java.lang.Boolean.toString(node.isVisibleToUser))
-					serializer.attribute("", "bounds", nodeRect.toShortString())
+					ser.startTag("", "node")
+					ser.addAttribute("index",index)
+					ser.addAttribute("drawingOrder", if(api>=24) node.drawingOrder else "NA")
+					ser.addAttribute("windowId", node.windowId)
+
+					ser.addAttribute("text", node.text)
+					ser.addAttribute("resource-id", node.viewIdResourceName)
+					ser.addAttribute("class", node.className)
+					ser.addAttribute("package", node.packageName)
+					ser.addAttribute("content-desc", node.contentDescription)
+
+					ser.startTag("","action-types")
+					ser.addAttribute("actionList", node.actionList)
+					ser.addAttribute("inputType", node.inputType)
+					ser.addAttribute("checkable", node.isCheckable)
+					ser.addAttribute("checked", node.isChecked)
+					ser.addAttribute("clickable", node.isClickable)
+					ser.addAttribute("enabled", node.isEnabled)
+					ser.addAttribute("focusable", node.isFocusable)
+					ser.addAttribute("focused", node.isFocused)
+					ser.addAttribute("scrollable", node.isScrollable)
+					ser.addAttribute("long-clickable", node.isLongClickable)
+					ser.addAttribute("isInputField", node.isEditable) // could be useful for custom widget classes to identify input fields
+					ser.addAttribute("password", node.isPassword)
+					ser.addAttribute("selected", node.isSelected)
+					ser.endTag("","action-types")
+
+					ser.startTag("","boundaries")
+					ser.addAttribute("definedAsVisible-to-user", node.isVisibleToUser)
+					ser.addAttribute("defBounds", nodeRect.toShortString())
+					val pBounds = Rect().apply { node.getBoundsInParent(this) }
+					ser.addAttribute("boundsInParent", pBounds.toShortString())
+					ser.addAttribute("liveRegion", node.liveRegion)
+					ser.endTag("","boundaries")
 
 					/** custom attributes, usually not definedAsVisible in the device-UiDump */
-					serializer.attribute("", "isInputField", java.lang.Boolean.toString(node.isEditable)) // could be usefull for custom widget classes to identify input fields
 					/** experimental */
 //		serializer.attribute("", "canOpenPopup", java.lang.Boolean.toString(node.canOpenPopup()))
 //		serializer.attribute("", "isDismissable", java.lang.Boolean.toString(node.isDismissable))
@@ -217,62 +229,23 @@ abstract class UiParser {
 				}
 			}
 
-	private fun safeCharSeqToString(cs: CharSequence?): String {
-		return if (cs == null)	""
-		else
-			stripInvalidXMLChars(cs).replace(";", "<semicolon>").replace(Regex("\\r\\n|\\r|\\n"), "<newline>").trim()
-	}
-
-	private fun stripInvalidXMLChars(cs: CharSequence): String {
-		val ret = StringBuffer()
-		var ch: Char
-		/* http://www.w3.org/TR/xml11/#charsets
-				[#x1-#x8], [#xB-#xC], [#xE-#x1F], [#x7F-#x84], [#x86-#x9F], [#xFDD0-#xFDDF],
-				[#x1FFFE-#x1FFFF], [#x2FFFE-#x2FFFF], [#x3FFFE-#x3FFFF],
-				[#x4FFFE-#x4FFFF], [#x5FFFE-#x5FFFF], [#x6FFFE-#x6FFFF],
-				[#x7FFFE-#x7FFFF], [#x8FFFE-#x8FFFF], [#x9FFFE-#x9FFFF],
-				[#xAFFFE-#xAFFFF], [#xBFFFE-#xBFFFF], [#xCFFFE-#xCFFFF],
-				[#xDFFFE-#xDFFFF], [#xEFFFE-#xEFFFF], [#xFFFFE-#xFFFFF],
-				[#x10FFFE-#x10FFFF].
-				 */
-		for (i in 0 until cs.length) {
-			ch = cs[i]
-
-			if (ch.toInt() in 0x1..0x8 || ch.toInt() in 0xB..0xC || ch.toInt() in 0xE..0x1F ||
-					ch.toInt() in 0x7F..0x84 || ch.toInt() in 0x86..0x9f ||
-					ch.toInt() in 0xFDD0..0xFDDF || ch.toInt() in 0x1FFFE..0x1FFFF ||
-					ch.toInt() in 0x2FFFE..0x2FFFF || ch.toInt() in 0x3FFFE..0x3FFFF ||
-					ch.toInt() in 0x4FFFE..0x4FFFF || ch.toInt() in 0x5FFFE..0x5FFFF ||
-					ch.toInt() in 0x6FFFE..0x6FFFF || ch.toInt() in 0x7FFFE..0x7FFFF ||
-					ch.toInt() in 0x8FFFE..0x8FFFF || ch.toInt() in 0x9FFFE..0x9FFFF ||
-					ch.toInt() in 0xAFFFE..0xAFFFF || ch.toInt() in 0xBFFFE..0xBFFFF ||
-					ch.toInt() in 0xCFFFE..0xCFFFF || ch.toInt() in 0xDFFFE..0xDFFFF ||
-					ch.toInt() in 0xEFFFE..0xEFFFF || ch.toInt() in 0xFFFFE..0xFFFFF ||
-					ch.toInt() in 0x10FFFE..0x10FFFF)
-				ret.append(".")
-			else
-				ret.append(ch)
-		}
-		return ret.toString()
-	}
-
-	@Suppress("PrivatePropertyName")
-	private val NAF_EXCLUDED_CLASSES = arrayOf(android.widget.GridView::class.java.name, android.widget.GridLayout::class.java.name, android.widget.ListView::class.java.name, android.widget.TableLayout::class.java.name)
-	/**
-	 * The list of classes to exclude my not be complete. We're attempting to
-	 * only reduce noise from standard layout classes that may be falsely
-	 * configured to accept clicks and are also enabled.
-	 *
-	 * @param node
-	 * @return true if node is excluded.
-	 */
-	private fun nafExcludedClass(node: AccessibilityNodeInfo): Boolean {
-		val className = safeCharSeqToString(node.className)
-		for (excludedClassName in NAF_EXCLUDED_CLASSES) {
-			if (className.endsWith(excludedClassName))
-				return true
-		}
-		return false
-	}
+//	@Suppress("PrivatePropertyName")
+//	private val NAF_EXCLUDED_CLASSES = arrayOf(android.widget.GridView::class.java.name, android.widget.GridLayout::class.java.name, android.widget.ListView::class.java.name, android.widget.TableLayout::class.java.name)
+//	/**
+//	 * The list of classes to exclude my not be complete. We're attempting to
+//	 * only reduce noise from standard layout classes that may be falsely
+//	 * configured to accept clicks and are also enabled.
+//	 *
+//	 * @param node
+//	 * @return true if node is excluded.
+//	 */
+//	private fun nafExcludedClass(node: AccessibilityNodeInfo): Boolean {
+//		val className = safeCharSeqToString(node.className)
+//		for (excludedClassName in NAF_EXCLUDED_CLASSES) {
+//			if (className.endsWith(excludedClassName))
+//				return true
+//		}
+//		return false
+//	}
 
 }

project/deviceComponents/deviceControlDaemon/src/androidTest/java/org/droidmate/uiautomator2daemon/uiautomatorExtensions/utils.kt

@@ -9,6 +9,7 @@ import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.Job
 import org.droidmate.deviceInterface.exploration.Rectangle
+import org.xmlpull.v1.XmlSerializer
 import java.nio.ByteBuffer
 import java.util.*
 import kotlin.math.abs
@@ -120,4 +121,58 @@ var debugEnabled = true
 fun debugOut(msg: String, enabled: Boolean = true){
 	@Suppress("ConstantConditionIf")
 	if(debugEnabled && enabled) Log.i("droidmate/uiad/DEBUG", msg)
-}
+}
+
+fun XmlSerializer.addAttribute(name: String, value: Any?){
+	val valueString = when (value){
+		null -> "null"
+		is Int -> Integer.toString(value)
+		is Boolean -> java.lang.Boolean.toString(value)
+		else -> safeCharSeqToString(value.toString().replace("<","&lt").replace(">","&gt"))
+	}
+	try {
+		attribute("", name, valueString)
+	} catch (e: Throwable) {
+		throw java.lang.RuntimeException("'$name':'$value' contains illegal characters")
+	}
+}
+
+fun safeCharSeqToString(cs: CharSequence?): String {
+	return if (cs == null)	""
+	else
+		stripInvalidXMLChars(cs).trim()
+}
+
+private fun stripInvalidXMLChars(cs: CharSequence): String {
+	val ret = StringBuffer()
+	var ch: Char
+	/* http://www.w3.org/TR/xml11/#charsets
+			[#x1-#x8], [#xB-#xC], [#xE-#x1F], [#x7F-#x84], [#x86-#x9F], [#xFDD0-#xFDDF],
+			[#x1FFFE-#x1FFFF], [#x2FFFE-#x2FFFF], [#x3FFFE-#x3FFFF],
+			[#x4FFFE-#x4FFFF], [#x5FFFE-#x5FFFF], [#x6FFFE-#x6FFFF],
+			[#x7FFFE-#x7FFFF], [#x8FFFE-#x8FFFF], [#x9FFFE-#x9FFFF],
+			[#xAFFFE-#xAFFFF], [#xBFFFE-#xBFFFF], [#xCFFFE-#xCFFFF],
+			[#xDFFFE-#xDFFFF], [#xEFFFE-#xEFFFF], [#xFFFFE-#xFFFFF],
+			[#x10FFFE-#x10FFFF].
+			 */
+	for (i in 0 until cs.length) {
+		ch = cs[i]
+
+		if (ch.toInt() in 0x1..0x8 || ch.toInt() in 0xB..0xC || ch.toInt() in 0xE..0x1F ||
+			ch.toInt() == 0x14 ||
+			ch.toInt() in 0x7F..0x84 || ch.toInt() in 0x86..0x9f ||
+			ch.toInt() in 0xFDD0..0xFDDF || ch.toInt() in 0x1FFFE..0x1FFFF ||
+			ch.toInt() in 0x2FFFE..0x2FFFF || ch.toInt() in 0x3FFFE..0x3FFFF ||
+			ch.toInt() in 0x4FFFE..0x4FFFF || ch.toInt() in 0x5FFFE..0x5FFFF ||
+			ch.toInt() in 0x6FFFE..0x6FFFF || ch.toInt() in 0x7FFFE..0x7FFFF ||
+			ch.toInt() in 0x8FFFE..0x8FFFF || ch.toInt() in 0x9FFFE..0x9FFFF ||
+			ch.toInt() in 0xAFFFE..0xAFFFF || ch.toInt() in 0xBFFFE..0xBFFFF ||
+			ch.toInt() in 0xCFFFE..0xCFFFF || ch.toInt() in 0xDFFFE..0xDFFFF ||
+			ch.toInt() in 0xEFFFE..0xEFFFF || ch.toInt() in 0xFFFFE..0xFFFFF ||
+			ch.toInt() in 0x10FFFE..0x10FFFF)
+			ret.append(".")
+		else
+			ret.append(ch)
+	}
+	return ret.toString()
+}