mmanningau/alice
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Check in part way through the engine update to MediaPipe. Have the system working and now troubleshooting the LLM models and sizes and how they handle the JSON and tool calling loops...Check in to allow dev on the laptop as well from here.

Browse Source
This commit is contained in:
mmanningau
2026-03-04 18:58:03 +11:00
parent e703df9ec1
commit 8ba9cb7a62
10 changed files with 317 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
app/build.gradle.kts
View File

@@ -89,6 +89,9 @@ dependencies {
// Llama.cpp Kotlin Multiplatform Wrapper
implementation("com.llamatik:library:0.8.1")
// Google AI Edge - MediaPipe LLM Inference API
implementation("com.google.mediapipe:tasks-genai:0.10.27")
// Extended Material Icons (for Download, CheckCircle, etc.)
implementation("androidx.compose.material:material-icons-extended")
}

13
app/src/main/AndroidManifest.xml
View File

@@ -4,6 +4,8 @@
<uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.BIND_ACCESSIBILITY_SERVICE" />
<application
android:name="AliceApp"
android:usesCleartextTraffic="true"
@@ -26,6 +28,17 @@
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
</activity>
<service
android:name=".AliceAccessibilityService"
android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE"
android:exported="false">
<intent-filter>
<action android:name="android.accessibilityservice.AccessibilityService" />
</intent-filter>
<meta-data
android:name="android.accessibilityservice"
android:resource="@xml/accessibility_service_config" />
</service>
</application>
</manifest>

21
app/src/main/java/net/mmanningau/alice/AliceAccessibilityService.kt Normal file
View File

@@ -0,0 +1,21 @@
package net.mmanningau.alice
import android.accessibilityservice.AccessibilityService
import android.view.accessibility.AccessibilityEvent
import android.util.Log
class AliceAccessibilityService : AccessibilityService() {
override fun onServiceConnected() {
super.onServiceConnected()
Log.d("AliceAccessibility", "Service Connected and Ready!")
}
override fun onAccessibilityEvent(event: AccessibilityEvent?) {
// We will build the screen-reading logic here later!
}
override fun onInterrupt() {
Log.e("AliceAccessibility", "Service Interrupted")
}
}

43
app/src/main/java/net/mmanningau/alice/LlmManager.kt
View File

@@ -1,5 +1,7 @@
package net.mmanningau.alice
import android.content.Context
import android.util.Log
import dev.langchain4j.data.message.AiMessage
import dev.langchain4j.data.message.ChatMessage
import dev.langchain4j.data.message.SystemMessage
@@ -11,13 +13,17 @@ import java.time.Duration
import java.text.SimpleDateFormat
import java.util.Date
import java.util.Locale
import android.content.Context
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
object LlmManager {
private var chatModel: ChatLanguageModel? = null
var currentMode: String = "Remote"
private set
// Hardware telemetry for the UI
private val _hardwareBackend = MutableStateFlow("Standby")
val hardwareBackend: StateFlow<String> = _hardwareBackend
// Database tracking
private var chatDao: ChatDao? = null
@@ -29,7 +35,7 @@ object LlmManager {
// Initialization now makes the dao optional so the UI can safely call it!
fun initialize(
context: Context,dao: ChatDao?, mode: String, url: String, modelName: String, apiKey: String, systemPrompt: String
context: Context, dao: ChatDao?, mode: String, url: String, modelName: String, apiKey: String, systemPrompt: String
) {
// Only update the DAO if one was passed in (like on app boot)
if (dao != null) {
@@ -52,9 +58,36 @@ object LlmManager {
.logResponses(true)
.build()
} else if (mode == "Local") {
// NEW: Grab the absolute path from the registry and boot the middleman!
// Grab the absolute path from the registry
val fullPath = ModelRegistry.getModelPath(context, modelName)
chatModel = LlamaCppAdapter(fullPath)
// NEW: The Switchboard! Route the path to the correct engine based on file type
when {
fullPath.endsWith(".task") || fullPath.endsWith(".litertlm") -> {
Log.d("AliceEngine", "Routing to MediaPipe Engine (Formula 1 Mode)")
// Reset to standby when a new model is selected
_hardwareBackend.value = "Standby"
chatModel = MediaPipeAdapter(context, fullPath) { systemEvent ->
// Intercept the hardware broadcast
if (systemEvent.startsWith("HARDWARE_STATE:")) {
_hardwareBackend.value = systemEvent.removePrefix("HARDWARE_STATE:").trim()
} else {
Log.w("AliceSystem", systemEvent)
}
}
}
fullPath.endsWith(".gguf") -> {
Log.d("AliceEngine", "Routing to Llama.cpp Engine (Flexible Mode)")
// Llama.cpp manages its own Vulkan backend, so we just label it
_hardwareBackend.value = "Vulkan"
chatModel = LlamaCppAdapter(fullPath)
}
else -> {
Log.e("AliceEngine", "Unsupported model file extension: $fullPath")
chatModel = null
}
}
}
// Database Startup Logic
@@ -105,7 +138,7 @@ object LlmManager {
}
fun chat(userText: String): String {
if (currentMode == "Local" && chatModel == null) return "System: Llamatik Ollama On-Device engine is selected but not yet installed."
if (currentMode == "Local" && chatModel == null) return "System: Local engine is selected but not properly initialized or unsupported file format."
val currentModel = chatModel ?: return "Error: LLM engine not initialized."
// If the history size is 1, it means only the System prompt exists. This is the first message!

31
app/src/main/java/net/mmanningau/alice/MainActivity.kt
View File

@@ -50,6 +50,8 @@ class MainActivity : ComponentActivity() {
fun MainChatScreen() {
val drawerState = rememberDrawerState(initialValue = DrawerValue.Closed)
val scope = rememberCoroutineScope()
// Observe the live hardware state
val hardwareBackend by LlmManager.hardwareBackend.collectAsState()
var currentScreen by remember { mutableStateOf("Chat") }
var inputText by remember { mutableStateOf("") }
@@ -135,7 +137,29 @@ fun MainChatScreen() {
Scaffold(
topBar = {
TopAppBar(
title = { Text("Alice Agent") },
title = {
Row(verticalAlignment = Alignment.CenterVertically) {
Text("Alice Agent")
// The Live Hardware Telemetry Badge
if (hardwareBackend != "Standby") {
Spacer(modifier = Modifier.width(8.dp))
Surface(
color = if (hardwareBackend == "GPU") MaterialTheme.colorScheme.primary
else MaterialTheme.colorScheme.error,
shape = RoundedCornerShape(4.dp)
) {
Text(
text = hardwareBackend,
style = MaterialTheme.typography.labelSmall,
color = if (hardwareBackend == "GPU") MaterialTheme.colorScheme.onPrimary
else MaterialTheme.colorScheme.onError,
modifier = Modifier.padding(horizontal = 6.dp, vertical = 2.dp)
)
}
}
}
},
navigationIcon = {
IconButton(onClick = { scope.launch { drawerState.open() } }) {
Icon(Icons.Default.Menu, contentDescription = "Menu")
@@ -519,7 +543,10 @@ fun ModelManagerScreen(onBackClicked: () -> Unit) {
// Launch the background download
scope.launch {
ModelDownloader.downloadModel(context, model.downloadUrl, model.fileName)
// Grab the Hugging Face token from the API Key settings field
val hfToken = prefs.getString("apiKey", "") ?: ""
ModelDownloader.downloadModel(context, model.downloadUrl, model.fileName, hfToken)
.collect { progress ->
downloadProgress[model.id] = progress
}

175
app/src/main/java/net/mmanningau/alice/MediaPipeAdapter.kt Normal file
View File

@@ -0,0 +1,175 @@
package net.mmanningau.alice
import android.content.Context
import android.util.Log
import com.google.mediapipe.tasks.genai.llminference.LlmInference
import dev.langchain4j.agent.tool.ToolExecutionRequest
import dev.langchain4j.agent.tool.ToolSpecification
import dev.langchain4j.data.message.AiMessage
import dev.langchain4j.data.message.ChatMessage
import dev.langchain4j.data.message.SystemMessage
import dev.langchain4j.data.message.ToolExecutionResultMessage
import dev.langchain4j.data.message.UserMessage
import dev.langchain4j.model.chat.ChatLanguageModel
import dev.langchain4j.model.output.Response
import org.json.JSONObject
import java.io.File
import java.util.UUID
class MediaPipeAdapter(
private val context: Context,
private val modelPath: String,
private val onSystemEvent: (String) -> Unit // Flexible routing for UI notifications & Accessibility
) : ChatLanguageModel {
private var engine: LlmInference? = null
private fun getOrInitEngine(): LlmInference {
if (engine == null) {
val modelFile = File(modelPath)
if (!modelFile.exists()) {
throw IllegalStateException("Task file not found: $modelPath. Please download it.")
}
try {
// THE PUSH: Aggressively demand the Adreno GPU
val gpuOptions = LlmInference.LlmInferenceOptions.builder()
.setModelPath(modelPath)
.setMaxTokens(4096)
.setPreferredBackend(LlmInference.Backend.GPU)
.build()
engine = LlmInference.createFromOptions(context, gpuOptions)
Log.d("AliceEngine", "Formula 1 Mode: GPU Initialized successfully.")
// Broadcast the successful hardware lock!
onSystemEvent("HARDWARE_STATE: GPU")
} catch (e: Exception) {
// THE FALLBACK: If GPU fails, notify the UI and drop to CPU
Log.e("AliceEngine", "GPU Initialization failed: ${e.message}")
onSystemEvent("Hardware Notice: GPU not supported for this model. Falling back to CPU. Generation will be slower and consume more battery.")
val cpuOptions = LlmInference.LlmInferenceOptions.builder()
.setModelPath(modelPath)
.setMaxTokens(4096)
.setPreferredBackend(LlmInference.Backend.CPU)
.build()
engine = LlmInference.createFromOptions(context, cpuOptions)
// Broadcast the fallback
onSystemEvent("HARDWARE_STATE: CPU")
}
}
return engine!!
}
override fun generate(messages: List<ChatMessage>): Response<AiMessage> {
return generate(messages, emptyList())
}
override fun generate(
messages: List<ChatMessage>,
toolSpecifications: List<ToolSpecification>
): Response<AiMessage> {
val activeEngine = getOrInitEngine()
val promptBuilder = java.lang.StringBuilder()
// 1. The Strict Negative-Constraint Schema
val toolsPrompt = java.lang.StringBuilder()
if (toolSpecifications.isNotEmpty()) {
toolsPrompt.append("\n\n# AVAILABLE TOOLS\n")
for (tool in toolSpecifications) {
toolsPrompt.append("- ${tool.name()}: ${tool.description()} | Params: ${tool.parameters()?.toString() ?: "{}"}\n")
}
toolsPrompt.append("\nCRITICAL RULES:\n")
toolsPrompt.append("1. NEVER guess or fabricate data (like battery levels, IP addresses, or network latency). You MUST use a tool to fetch real data.\n")
toolsPrompt.append("2. Do NOT invent your own syntax. You must use ONLY the exact JSON format below.\n")
toolsPrompt.append("3. To execute a tool, reply with ONLY this JSON object and absolutely no other text:\n")
toolsPrompt.append("{\"name\": \"<tool_name>\", \"arguments\": {<args>}}\n")
}
// 2. Format Chat History using GEMMA 3 TAGS (Merging System into User)
var isFirstUserMessage = true
for (message in messages) {
when (message) {
is SystemMessage -> {
// IGNORE: We do not append a 'system' tag because Gemma 3 doesn't support it.
// We already built the toolsPrompt string in Step 1, so we just hold it.
}
is UserMessage -> {
if (isFirstUserMessage) {
// Merge the draconian tools prompt and the user's first message into one block
promptBuilder.append("<start_of_turn>user\n${toolsPrompt.toString()}\n\n${message.text()}<end_of_turn>\n")
isFirstUserMessage = false
} else {
promptBuilder.append("<start_of_turn>user\n${message.text()}<end_of_turn>\n")
}
}
is ToolExecutionResultMessage -> {
promptBuilder.append("<start_of_turn>user\n[SYSTEM DATA VIA TOOL '${message.toolName()}']: ${message.text()}\nUse this real data to answer the previous question.<end_of_turn>\n")
}
is AiMessage -> {
if (message.hasToolExecutionRequests()) {
val request = message.toolExecutionRequests()[0]
promptBuilder.append("<start_of_turn>model\n{\"name\": \"${request.name()}\", \"arguments\": ${request.arguments()}}<end_of_turn>\n")
} else {
val cleanText = message.text()?.replace(Regex("Calling tool:.*?\\.\\.\\."), "")?.trim() ?: ""
if (cleanText.isNotBlank()) {
promptBuilder.append("<start_of_turn>model\n$cleanText<end_of_turn>\n")
}
}
}
}
}
promptBuilder.append("<start_of_turn>model\n")
// 3. Execution on MediaPipe
val rawResponse = activeEngine.generateResponse(promptBuilder.toString())
Log.d("AliceEngine", "RAW_RESPONSE_LENGTH: ${rawResponse.length}")
Log.d("AliceEngine", "RAW_RESPONSE: $rawResponse")
Log.d("AliceEngine", "Engine state after gen - messages count: ${messages.size}")
val responseText = rawResponse.replace("<end_of_turn>", "").trim()
// Strip the markdown code blocks if Gemma adds them
val cleanText = responseText.replace(Regex("```(?:json)?"), "").replace("```", "").trim()
// 4. The Bulletproof Regex JSON Parser
if (toolSpecifications.isNotEmpty()) {
// Hunt directly for the tool name, bypassing strict JSON validation
val nameRegex = Regex("\"name\"\\s*:\\s*\"([^\"]+)\"")
val match = nameRegex.find(cleanText)
if (match != null) {
val toolName = match.groupValues[1]
// Safely attempt to grab arguments. If they are hallucinated garbage, default to {}
var argumentsJson = "{}"
// Old regex - pre Claude ..... val argRegex = Regex("\"arguments\"\\s*:\\s*(\\{.*?\\})")
val argRegex = Regex("\"arguments\"\\s*:\\s*(\\{.*?\\})", RegexOption.DOT_MATCHES_ALL)
val argMatch = argRegex.find(cleanText)
if (argMatch != null) {
val foundArgs = argMatch.groupValues[1]
try {
// Test if the args are valid JSON
JSONObject(foundArgs)
argumentsJson = foundArgs
} catch (e: Exception) {
// It was garbage (like the infinite 7777s). Keep the "{}" default.
Log.w("AliceEngine", "Discarded malformed arguments: $foundArgs")
}
}
val request = ToolExecutionRequest.builder()
.id(UUID.randomUUID().toString())
.name(toolName)
.arguments(argumentsJson)
.build()
return Response.from(AiMessage.from("Calling tool: $toolName...", listOf(request)))
}
}
return Response.from(AiMessage(cleanText))
}
}

8
app/src/main/java/net/mmanningau/alice/ModelDownloader.kt
View File

@@ -12,7 +12,7 @@ import java.io.File
object ModelDownloader {
fun downloadModel(context: Context, url: String, fileName: String): Flow<Int> = flow {
fun downloadModel(context: Context, url: String, fileName: String, hfToken: String = ""): Flow<Int> = flow {
val downloadManager = context.getSystemService(Context.DOWNLOAD_SERVICE) as DownloadManager
// Ensure the directory exists
@@ -22,10 +22,14 @@ object ModelDownloader {
.setTitle(fileName)
.setDescription("Downloading AI Model for Alice...")
.setNotificationVisibility(DownloadManager.Request.VISIBILITY_VISIBLE_NOTIFY_COMPLETED)
// Save it directly into our app's specific Models folder
.setDestinationUri(Uri.fromFile(File(modelsDir, fileName)))
.setAllowedOverMetered(true) // Allow cellular downloads
// THE FIX: Inject the Hugging Face Authorization header so we bypass the gate
if (hfToken.isNotBlank()) {
request.addRequestHeader("Authorization", "Bearer $hfToken")
}
val downloadId = downloadManager.enqueue(request)
var finishDownload = false
var progress = 0

24
app/src/main/java/net/mmanningau/alice/ModelRegistry.kt
View File

@@ -46,6 +46,30 @@ object ModelRegistry {
fileName = "qwen2.5-3b-instruct-q4_k_m.gguf",
downloadUrl = "https://huggingface.co/Qwen/Qwen2.5-3B-Instruct-GGUF/resolve/main/qwen2.5-3b-instruct-q4_k_m.gguf",
sizeMb = 2020
),
LocalModel(
id = "gemma3-1b",
name = "Gemma 3 (1B)",
sizeMb = 555, // Update these sizes based on the exact HuggingFace .task file
description = "Google's highly optimized mobile intelligence. Best balance of speed and reasoning.",
fileName = "gemma-3-1b-it.task",
downloadUrl = "https://huggingface.co/litert-community/Gemma3-1B-IT/resolve/main/gemma3-1b-it-int4.task" // Update with exact raw URL
),
LocalModel(
id = "gemma3n-e2b",
name = "Gemma 3n (E2B)",
sizeMb = 3600,
description = "Elastic architecture. Activates fewer parameters for battery efficiency while maintaining high logic.",
fileName = "gemma-3n-e2b-it.task",
downloadUrl = "https://huggingface.co/google/gemma-3n-E2B-it-litert-lm/resolve/main/gemma-3n-E2B-it-int4.litertlm"
),
LocalModel(
id = "Gemma3-1B-IT_multi-prefill-seq_q8_ekv4096",
name = "Gemma 3 (1B) Prefill",
sizeMb = 3390,
description = "A highly optimised and fine tuned model for agentic tasks and function calling.",
fileName = "Gemma3-1B-IT_multi-prefill-seq_q8_ekv4096.task",
downloadUrl = "https://huggingface.co/litert-community/Gemma3-1B-IT/resolve/main/Gemma3-1B-IT_multi-prefill-seq_q8_ekv4096.task"
)
)

1
app/src/main/res/values/strings.xml
View File

@@ -1,3 +1,4 @@
<resources>
<string name="app_name">Alice</string>
<string name="accessibility_service_description">Alice Screen Reader Service</string>
</resources>

7
app/src/main/res/xml/accessibility_service_config.xml Normal file
View File

@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
android:accessibilityEventTypes="typeWindowStateChanged|typeWindowContentChanged"
android:accessibilityFeedbackType="feedbackGeneric"
android:accessibilityFlags="flagDefault"
android:canRetrieveWindowContent="true"
android:description="@string/accessibility_service_description" />