adb view

.gitignore

@@ -44,8 +44,9 @@ MediaDeviceSalts-journal
 sources/
 /sources
 /sources/
-/.intellijPlatform/
 
 ### no keys
 composeApp/src/commonMain/resources/keys/*.key
 /composeApp/src/commonMain/resources/keys/*.key
+.intellijPlatform/
+.intellijPlatform

.intellijPlatform/self-update.lock

@@ -1 +1 @@
-2025-06-12
+2025-10-10

.run/mjdev-desktop [desktopRun].run.xml

@@ -0,0 +1,28 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="mjdev-desktop [desktopRun]" type="GradleRunConfiguration" factoryName="Gradle">
+    <ExternalSystemSettings>
+      <option name="executionName" />
+      <option name="externalProjectPath" value="$PROJECT_DIR$" />
+      <option name="externalSystemIdString" value="GRADLE" />
+      <option name="scriptParameters" value="" />
+      <option name="taskDescriptions">
+        <list />
+      </option>
+      <option name="taskNames">
+        <list>
+          <option value="desktopRun" />
+        </list>
+      </option>
+      <option name="vmOptions" />
+    </ExternalSystemSettings>
+    <ExternalSystemDebugServerProcess>true</ExternalSystemDebugServerProcess>
+    <ExternalSystemReattachDebugProcess>true</ExternalSystemReattachDebugProcess>
+    <EXTENSION ID="com.android.tools.idea.testartifacts.testsuite.GradleRunConfigurationExtension">
+      <com.android.tools.idea.testartifacts.testsuite.SHOW_TEST_RESULT_IN_ANDROID_TEST_SUITE_VIEW>false</com.android.tools.idea.testartifacts.testsuite.SHOW_TEST_RESULT_IN_ANDROID_TEST_SUITE_VIEW>
+      <android.execution.deploysToLocalDevice>false</android.execution.deploysToLocalDevice>
+    </EXTENSION>
+    <DebugAllEnabled>false</DebugAllEnabled>
+    <RunAsTest>false</RunAsTest>
+    <method v="2" />
+  </configuration>
+</component>

composeApp/composeApp.gradle.kts

@@ -185,7 +185,7 @@ kotlin {
                 implementation(libs.flock.aigentic.vertexai)
                 implementation(libs.flock.aigentic.http)
                 implementation(libs.flock.aigentic.openapi)
-//                ksp("community.flock.aigentic:ksp-processor:0.1.0")
+                // ksp("community.flock.aigentic:ksp-processor:0.1.0")
                 // ktor
                 implementation(libs.ktor.client.core)
                 implementation(libs.ktor.client.cio)
@@ -210,7 +210,9 @@ kotlin {
                 implementation("io.ktor:ktor-server-auth:3.2.1")
                 implementation("io.ktor:ktor-server-compression:3.2.1")
                 implementation("ch.qos.logback:logback-classic:1.3.11")
-//                implementation("com.squareup.okhttp3:okhttp-ws:4.12.0")
+                // ai
+                implementation("ai.koog:koog-agents:0.3.0")
+                // implementation("com.squareup.okhttp3:okhttp-ws:4.12.0")
                 // notification
                 // implementation("io.github.shadmanadman:knotif:0.56.0")
                 // sensors
@@ -234,6 +236,10 @@ kotlin {
 //                kapt(libs.popkorn.compiler)
                 // tor
                 // implementation("io.matthewnelson.kmp-tor:runtime:2.1.0")
+                // web
+                implementation("io.github.shadmanadman:kwebview:1.31.0")
+                // adb
+                implementation("dev.mobile:dadb:1.2.10")
             }
         }
 
@@ -256,6 +262,8 @@ kotlin {
                 implementation(libs.accompanist.permissions)
                 // sensors
                 //implementation("io.github.shadmanadman:KSensor:0.59.0")
+                // adb
+                // implementation("dev.mobile:dadb:1.2.10")
             }
         }
 
@@ -278,6 +286,8 @@ kotlin {
                 implementation(libs.ychat)
                 // desktop files
                 implementation(libs.ini4j)
+                // adb
+                // implementation("dev.mobile:dadb:1.2.10")
             }
         }
 

composeApp/src/androidMain/kotlin/org/mjdev/desktop/context/DesktopContext.kt

@@ -133,7 +133,7 @@ class DesktopContext(
         IAiManager::class -> AiManager(
             context = this,
             // todo user can configure
-            pluginAI = AiPluginGemini(this@DesktopContext),
+            pluginAI = AiPluginOpenAi(this@DesktopContext),
             pluginTTS = TTSPluginAndroid(this@DesktopContext),
             pluginSTT = STTPluginEmpty(this@DesktopContext)
         )

composeApp/src/androidMain/kotlin/org/mjdev/desktop/helpers/DecoderAndroid.kt

@@ -0,0 +1,16 @@
+package org.mjdev.desktop.helpers
+
+import android.graphics.BitmapFactory
+import androidx.compose.ui.graphics.asImageBitmap
+import org.mjdev.desktop.data.DecodedImage
+
+actual fun decodeImage(
+    bytes: ByteArray
+): DecodedImage? = try {
+    val bmp = BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
+    val imageBitmap = bmp.asImageBitmap()
+    DecodedImage(imageBitmap, bmp.width, bmp.height)
+} catch (e: Exception) {
+    e.printStackTrace()
+    null
+}

composeApp/src/commonMain/kotlin/org/mjdev/desktop/components/adb/AdbScreenMirror.kt

@@ -0,0 +1,146 @@
+package org.mjdev.desktop.components.adb
+
+import androidx.compose.runtime.mutableStateOf
+import androidx.compose.foundation.Image
+import androidx.compose.foundation.gestures.detectTapGestures
+import androidx.compose.foundation.layout.Box
+import androidx.compose.foundation.layout.fillMaxSize
+import androidx.compose.material3.CircularProgressIndicator
+import androidx.compose.material3.Text
+import androidx.compose.runtime.*
+import androidx.compose.ui.Alignment
+import androidx.compose.ui.graphics.ImageBitmap
+import androidx.compose.ui.input.pointer.pointerInput
+import dadb.Dadb
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.graphics.toComposeImageBitmap
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.launch
+import org.jetbrains.skia.Image as SkiaImage
+import org.jetbrains.compose.ui.tooling.preview.Preview
+import org.mjdev.desktop.context.DesktopContextScope.Companion.withDesktopContext
+
+@Preview
+@Composable
+fun AdbScreenMirror(
+    modifier: Modifier = Modifier,
+    visibilityState: MutableState<Boolean> = mutableStateOf(false),
+    scope : CoroutineScope = rememberCoroutineScope()
+)  = withDesktopContext {
+    val refreshInterval = 100L
+    val deviceState = remember {
+        DeviceState(scope, visibilityState.value, refreshInterval)
+    }
+    DisposableEffect(visibilityState.value) {
+        deviceState.start()
+        onDispose {
+            deviceState.stop()
+        }
+    }
+    Box(
+        modifier = modifier,
+        contentAlignment = Alignment.Center
+    ) {
+        when {
+            deviceState.isConnecting -> CircularProgressIndicator()
+            deviceState.error != null -> Text("Error: ${deviceState.error}")
+            deviceState.imageBitmap != null -> {
+                val currentImage = deviceState.imageBitmap!!
+                Image(
+                    bitmap = currentImage,
+                    contentDescription = "Device screen",
+                    modifier = Modifier
+                        .fillMaxSize()
+                        .pointerInput(deviceState.dadbInstance) {
+                            detectTapGestures { offset ->
+                                deviceState.tapOnDevice(
+                                    offset.x.toInt(),
+                                    offset.y.toInt(),
+                                    size.width,
+                                    size.height
+                                )
+                            }
+                        }
+                )
+            }
+
+            else -> Text("Waiting for device...")
+        }
+    }
+}
+
+class DeviceState(
+    private val scope:CoroutineScope,
+    private val visibility: Boolean,
+    private val refreshInterval: Long,
+) {
+    var imageBitmap by mutableStateOf<ImageBitmap?>(null)
+    var error by mutableStateOf<String?>(null)
+    var isConnecting by mutableStateOf(true)
+    var dadbInstance by mutableStateOf<Dadb?>(null)
+    var screenWidth by mutableIntStateOf(0)
+    var screenHeight by mutableIntStateOf(0)
+    private var job: Job? = null
+
+    fun start() {
+        job = scope.launch {
+            try {
+                val dadb = Dadb.discover()
+                if (dadb == null) {
+                    error = "No device found"
+                    isConnecting = false
+                    return@launch
+                }
+                dadbInstance = dadb
+                isConnecting = false
+                while (job?.isActive == true) {
+                    try {
+                        val stream = dadb.open("exec:screencap -p")
+                        val bytes = stream.source.readByteArray()
+                        stream.close()
+                        if (bytes.isNotEmpty()) {
+                            val skiaImage = SkiaImage.makeFromEncoded(bytes)
+                            imageBitmap = skiaImage.toComposeImageBitmap()
+                            screenWidth = skiaImage.width
+                            screenHeight = skiaImage.height
+                        }
+                    } catch (e: Exception) {
+                        error = e.message
+                    }
+                    delay(refreshInterval)
+                }
+            } catch (e: Exception) {
+                error = e.message
+                isConnecting = false
+            }
+        }
+    }
+
+    fun stop() {
+        job?.cancel()
+        dadbInstance?.close()
+    }
+
+    fun tapOnDevice(
+        offsetX: Int,
+        offsetY: Int,
+        sizeWidth: Int,
+        sizeHeight: Int
+    ) {
+        dadbInstance?.let { dadb ->
+            scope.launch {
+                try {
+                    val scaleX = screenWidth.toFloat() / sizeWidth
+                    val scaleY = screenHeight.toFloat() / sizeHeight
+                    val deviceX = (offsetX * scaleX).toInt()
+                    val deviceY = (offsetY * scaleY).toInt()
+                    dadb.shell("input tap $deviceX $deviceY")
+                } catch (e: Exception) {
+                    e.printStackTrace()
+                }
+            }
+        }
+    }
+}

composeApp/src/commonMain/kotlin/org/mjdev/desktop/components/desktop/Desktop.kt

@@ -1,10 +1,11 @@
 package org.mjdev.desktop.components.desktop
 
 import androidx.compose.foundation.layout.Box
-import androidx.compose.foundation.layout.BoxWithConstraints
 import androidx.compose.foundation.layout.PaddingValues
+import androidx.compose.foundation.layout.fillMaxHeight
 import androidx.compose.foundation.layout.fillMaxSize
 import androidx.compose.foundation.layout.padding
+import androidx.compose.foundation.layout.wrapContentWidth
 import androidx.compose.runtime.Composable
 import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
@@ -23,6 +24,7 @@ import org.mjdev.desktop.extensions.Modifier.onLeftClick
 import org.mjdev.desktop.extensions.Modifier.onMousePress
 import org.mjdev.desktop.extensions.Modifier.onRightClick
 import org.jetbrains.compose.ui.tooling.preview.Preview
+import org.mjdev.desktop.components.adb.AdbScreenMirror
 
 @Suppress("FunctionName", "UNUSED_PARAMETER")
 @Composable
@@ -69,8 +71,9 @@ fun Desktop(
                 }
             }
         )
-        BoxWithConstraints(
-            modifier = Modifier.padding(16.dp)
+        Box(
+            modifier = Modifier.padding(16.dp),
+            contentAlignment = Alignment.CenterEnd
         ) {
             Box(
                 modifier = Modifier.fillMaxSize()
@@ -86,6 +89,12 @@ fun Desktop(
 //            ComposeWebView(
 //                modifier = Modifier.align(Alignment.Center).size(640.dp, 480.dp)
 //            )
+                AdbScreenMirror(
+                    modifier = Modifier
+                        .align(Alignment.TopCenter)
+                        .fillMaxHeight()
+                        .wrapContentWidth()
+                )
             }
         }
     }

composeApp/src/commonMain/kotlin/org/mjdev/desktop/data/DecodedImage.kt

@@ -0,0 +1,9 @@
+package org.mjdev.desktop.data
+
+import androidx.compose.ui.graphics.ImageBitmap
+
+data class DecodedImage(
+    val bitmap: ImageBitmap,
+    val width: Int,
+    val height: Int
+)

composeApp/src/commonMain/kotlin/org/mjdev/desktop/helpers/DecodedImage.kt

@@ -0,0 +1,5 @@
+package org.mjdev.desktop.helpers
+
+import org.mjdev.desktop.data.DecodedImage
+
+expect fun decodeImage(bytes: ByteArray): DecodedImage?

composeApp/src/commonMain/kotlin/org/mjdev/desktop/helpers/generic/JsonHelper.kt

@@ -7,7 +7,6 @@ object JsonHelper {
     val gson: Gson by lazy {
         GsonBuilder()
             .setPrettyPrinting()
-            .setLenient()
             .serializeNulls()
             .serializeSpecialFloatingPointValues()
             .create()

composeApp/src/commonMain/kotlin/org/mjdev/desktop/managers/ai/plugins/AiPluginGemini.kt

@@ -17,7 +17,6 @@ import community.flock.aigentic.gemini.dsl.geminiModel
 import community.flock.aigentic.gemini.model.GeminiModelIdentifier
 import community.flock.aigentic.gemini.model.GeminiModelIdentifier.Gemini1_5Pro
 import org.mjdev.desktop.context.IDesktopContext
-import org.mjdev.desktop.log.Log
 import org.mjdev.desktop.managers.ai.plugins.base.AIPlugin
 
 // todo tools = actions

composeApp/src/commonMain/kotlin/org/mjdev/desktop/managers/ai/plugins/AiPluginOpenAi.kt

@@ -1,46 +1,37 @@
 package org.mjdev.desktop.managers.ai.plugins
 
+import ai.koog.agents.core.agent.AIAgent
+import ai.koog.prompt.executor.clients.openai.OpenAIModels
+import ai.koog.prompt.executor.llms.all.simpleOpenAIExecutor
+import ai.koog.prompt.llm.LLModel
 import org.mjdev.desktop.context.IDesktopContext
 import org.mjdev.desktop.managers.ai.plugins.base.AIPlugin
-import community.flock.aigentic.core.agent.tool.Result.Stuck
-import community.flock.aigentic.core.agent.tool.Result.Fatal
-import community.flock.aigentic.core.agent.tool.Result.Finished
-import community.flock.aigentic.core.agent.start
-import community.flock.aigentic.core.dsl.agent
-import community.flock.aigentic.openai.dsl.openAIModel
-import community.flock.aigentic.openai.model.OpenAIModelIdentifier
-import community.flock.aigentic.openai.model.OpenAIModelIdentifier.GPT4OMini
-import org.mjdev.desktop.log.Log
 
 // todo tools = actions
 // Open AI keys: https://platform.openai.com/api-keys
 class AiPluginOpenAi(
     private val context: IDesktopContext,
     private val apiKey: String = context.keysManager.loadKey("open-ai"),
-    private val model: OpenAIModelIdentifier = GPT4OMini,
+    private val model: LLModel = OpenAIModels.Chat.GPT4o,
 ) : AIPlugin {
+    val agent by lazy {
+        runCatching {
+            AIAgent(
+                executor = simpleOpenAIExecutor(apiKey),
+                systemPrompt = "You are a helpful assistant. Answer user questions concisely.",
+                llmModel = model
+            )
+        }
+    }
 
+    @Suppress("USELESS_CAST")
     override suspend fun ask(
         question: String
-    ): String = agent {
-//        Log.d("Using api key : $apiKey")
-        openAIModel {
-            apiKey(apiKey)
-            modelIdentifier(model)
-        }
-        task("Provide information") {
-            addInstruction("Respond to user queries with relevant information.")
-        }
-        context {
-            addText(question)
-        }
-        finishResponse()
-    }.start().result.let { result ->
-        when (result) {
-            is Finished<*> -> result.response.toString()
-            is Stuck -> result.reason
-            is Fatal -> result.message
+    ): String = runCatching {
+        agent.getOrThrow().let { agent ->
+            (agent as AIAgent<String, String>).run("Hello! How can you help me?")
         }
+    }.getOrElse { e ->
+        "Error at: ${e.stackTraceToString()}"
     }
-
 }

composeApp/src/desktopMain/kotlin/org/mjdev/desktop/context/DeskptopContext.kt

@@ -31,6 +31,7 @@ import org.mjdev.desktop.log.Log
 import org.mjdev.desktop.managers.ai.AiManager
 import org.mjdev.desktop.managers.ai.IAiManager
 import org.mjdev.desktop.managers.ai.plugins.AiPluginGemini
+import org.mjdev.desktop.managers.ai.plugins.AiPluginOpenAi
 import org.mjdev.desktop.managers.ai.stt.STTPluginEmpty
 import org.mjdev.desktop.managers.ai.tts.TTSPluginSwift
 import org.mjdev.desktop.managers.apps.IAppsManager
@@ -246,7 +247,7 @@ class DesktopContext(
         IAiManager::class -> AiManager(
             context = this,
             // todo user can configure
-            pluginAI = AiPluginGemini(this@DesktopContext),
+            pluginAI = AiPluginOpenAi(this@DesktopContext),
             pluginTTS = TTSPluginSwift(this@DesktopContext),
             pluginSTT = STTPluginEmpty(this@DesktopContext)
         )

composeApp/src/desktopMain/kotlin/org/mjdev/desktop/helpers/DecodedImage.desktop.kt

@@ -0,0 +1,16 @@
+package org.mjdev.desktop.helpers
+
+import androidx.compose.ui.graphics.toComposeImageBitmap
+import org.jetbrains.skia.Image
+import org.mjdev.desktop.data.DecodedImage
+
+actual fun decodeImage(
+    bytes: ByteArray
+): DecodedImage? = try {
+    val skiaImage = Image.makeFromEncoded(bytes)
+    val bitmap = skiaImage.toComposeImageBitmap()
+    DecodedImage(bitmap, skiaImage.width, skiaImage.height)
+} catch (e: Exception) {
+    e.printStackTrace()
+    null
+}

gradle/libs.versions.toml

@@ -19,7 +19,7 @@ android-min-sdk = "26"
 vlc-version = "3.0.21-2"
 vlc-dir = "composeApp/src/commonMain/resources/vlc/"
 ### lib versions ###################################################################################
-gradle = "8.11.0"
+gradle = "8.13.0"
 #gradle-kotlinter = "4.3.0"
 androidx-activity-compose = "1.10.1"
 androidx-lifecycle = "2.9.1"
@@ -31,16 +31,16 @@ core-splash-screen = "1.0.1"
 generative-ai = "0.9.0-1.1.0"
 gettext-lib = "1.11.0"
 charts = "1.1.0"
-human-readable = "1.11.0"
+human-readable = "1.12.0"
 #i18n4k-core = "0.9.0"
 ini4j = "0.5.4"
 jetlime = "3.0.1"
 kotlin = "2.2.0"
 kotlinx-coroutines = "1.10.2"
-kotlinx-date-time = "0.6.2"
+kotlinx-date-time = "0.7.1"
 #logback = "1.5.7"
-okhttp3 = "4.12.0"
-okio = "3.13.0"
+okhttp3 = "5.1.0"
+okio = "3.15.0"
 qrose = "1.0.1"
 tts = "3.0.0"
 ychat = "1.4.1"
@@ -51,7 +51,7 @@ kover = "0.9.1"
 qodana = "2025.1.1"
 intelliJPlatform = "2.6.0"
 changelog = "2.2.1"
-gradle-ktlint = "12.3.0"
+gradle-ktlint = "13.0.0"
 #foojay-resolver-convention="0.9.0"
 manes-versions = "0.52.0"
 coreKtx = "1.16.0"
@@ -65,9 +65,9 @@ vlc-setup = "0.1.0"
 vlcj = "4.11.0"
 jna = "5.17.0"
 image-comparison = "4.4.0"
-haze = "1.6.4"
-kotlinx-serialization-json = "1.8.1"
-ktor = "3.1.0"
+haze = "1.6.8"
+kotlinx-serialization-json = "1.9.0"
+ktor = "3.2.2"
 flock = "0.1.0"
 markwon = "4.6.2"
 html2text = "-SNAPSHOT"