added multimedia support and STT. STT and photos are have bugs but are functional. file upload is conceptualized but will require additional backend work. video is on hold for now.

mobile/app/lib/components/util/backend.dart

@@ -0,0 +1,231 @@
+import 'dart:convert';
+import 'dart:io';
+import 'dart:typed_data';
+import 'package:flutter/widgets.dart';
+import 'package:app/components/conversation_thread/conversation.dart';
+import 'package:camera/camera.dart';
+import 'package:http/http.dart' as http;
+
+import 'package:app/components/conversation_thread/chat.dart';
+
+import 'package:app/components/util/speech_to_text.dart';
+
+import 'model.dart';
+
+// Uses Singleton design pattern to ensure that any server references across the app are using the same instance.
+class Backend {
+  //     SERVER VARIABLES
+  static final Backend singleBackend = Backend._internal();
+  Backend._internal();
+  // TODO: One-time process to establish server connection is required for easy setup/maintenance.
+  static const url = "http://10.0.2.2:11434/api/"; // For emulated device in testing
+  //static final url = "http://192.168.1.68:11434/api"; // For local access to home server
+  static String endpoint = "";
+  static final Map<String, String> headers = {'Content-Type': 'application/json'};
+  static final bool UI_TESTING = true;
+
+
+
+  //     AI VARIABLES
+  static final SpeechToText stt = SpeechToText();
+  static late bool _speechEnabled;
+
+
+
+  //     PRIVACY-NECESSARY VARIABLES
+  // TODO: Reorder based on most recent usage.
+  static List<Conversation> conversations = [];
+  static List<Chat> loadedChats = [];
+  static late Conversation loadedConversation;
+  static late List<CameraDescription> cameras;
+  static late CameraController controller;
+
+
+  //     RUNTIME & STATUS VARIABLES
+  static late int conversationID;
+  static late int maxConversationID;
+  static late Model model;
+  static String prompt = "";
+  static String response = "";
+  static bool initiated = false;
+
+  // TODO: Verify all different components of STT, TTS, LLM, etc have been initialized.
+  // TODO: Handshake w/ server to validate identity
+  factory Backend() {
+
+
+    return singleBackend;
+  }
+
+  void init() async {
+    // Prevent multiple initiations
+    if(!initiated) {
+      initiated = true;
+      // Speech-To-Text engine activation.
+      _speechEnabled = stt.isSpeechEnabled();
+
+      model = Model("llama3"); // Primarily used llama3, testing gemma2
+      maxConversationID = conversations.length;
+
+      cameras = await availableCameras();
+      controller = CameraController(cameras[0], ResolutionPreset.max);
+      controller.initialize(); // TODO: Add error catching on camera
+    }
+
+    // TODO: Dynamically get available models and models that can be pulled.
+    // TODO: Dynamically get the user's default model
+    // TODO: Dynamically obtain conversations.
+    // TODO: Obtain all user permissions at once.
+  }
+
+
+
+  //     PRIMARY INTERFACES
+  @Deprecated("Use STT interface directly and use respondWhenReady().")
+  static Future<String> sttGetResponseWhenReady() async {
+    // TODO: In case on-device STT is unavailable, use server-side STT service.
+    if (!_speechEnabled) {
+      return "Unable to process Speech-To-Text on-device.";
+    }
+
+    prompt = stt.getTextWhenReady() as String;
+    conversations[conversationID].add(Chat(prompt, true, 0));
+    if(!UI_TESTING) {
+      httpSendRequest();
+    }
+    conversations[conversationID].add(Chat(response, false, 0));
+
+    return response;
+  }
+
+  static void respondWhenReady(String p) async {
+    prompt = p;
+    conversations[conversationID].add(Chat(prompt, true, 0));
+    if(!UI_TESTING) {
+      httpSendRequest();
+    }
+  }
+
+  static ImageProvider convertFromBase64(img) {
+    Uint8List imageBytes = base64Decode(img);
+
+    return Image.memory(imageBytes).image;
+  }
+
+  // Converts image to base64 encoded string for data transfer.
+  static String convertToBase64(f)  {
+    List<int> imageBytes = f.readAsBytesSync();
+    String img = base64Encode(imageBytes);
+
+    // TODO: Add special handling for images included in prompts.
+    conversations[conversationID].add(Chat.image(img, true, 1));
+    return img;
+  }
+
+
+  //     CONVERSATION MANAGEMENT
+  @Deprecated("")
+  static List<Conversation> getConversationsList() { return conversations; }
+
+  static Conversation getLoadedConversation() { return loadedConversation; }
+
+  // Load conversation should be called prior to any other server requests.
+  static void loadConversation(id) {
+    conversationID = id;
+
+    loadedConversation = conversations[id];
+    loadedChats = loadedConversation.chats;
+  }
+
+  // TODO: Better conversation ID management is needed. Preferably randomized IDs to not indicate the number of conversations.
+  static int createConversation() {
+    int id = conversations.length;
+    conversations.add(Conversation(id, "Conversation $id", "Another AI conversation"));
+    return id;
+  }
+
+  static void deleteConversation(id) {
+    loadedChats = conversations[conversationID].chats; // TODO: Temporarily saving the chats for "undo" popup.
+    conversations.removeAt(id);
+  }
+
+  @Deprecated('Just directly get loadedChats list. Unless multiple chats can be opened simultaneously?')
+  static List<Chat> getChatsByConversationID(id) { return conversations[id].chats; }
+
+
+
+  //     SERVER UTILS
+  static List<Map<String, String>> convertMessages() {
+    // TODO: Integrate with getConversationByID()
+    List<Map<String, String>> messageList = [];
+    for (Chat c in loadedChats) {
+      messageList.add({
+        "role": c.role ? "user" : "assistant",
+        "content": c.content,
+      });
+    }
+
+    return messageList;
+  }
+
+  // Formats the prompt in JSON.
+  static String constructPrompt() {
+    // TODO: Allow for additional flags, ie. continuous conversation, images, etc.
+    // TODO: Allow for toggleable states between stream
+    // TODO: Allow for different models to be dynamically selected.
+    Map<String, dynamic> data;
+
+    if(conversations[conversationID].conversationContext) {
+      loadedChats = [...loadedChats, Chat(prompt, true, 0)];
+      endpoint = "chat";
+
+      List<Map<String, String>> messageList = convertMessages();
+
+      data = {
+        "model": model.name,
+        "messages": messageList,
+        "stream": loadedConversation.stream,
+      };
+    }
+    else {
+      endpoint = "generate";
+      data = {
+        "model": model.name,
+        "prompt": prompt,
+        "stream": loadedConversation.stream,
+      };
+    }
+
+    // Encode the JSON payload
+    return json.encode(data);
+  }
+
+  // Send the HTTP request to the server.
+  static void httpSendRequest() async {
+    // TODO: Add AES256 encryption to the prompt/messages thread.
+    // TODO: Add SHA256 checksum to be sent and verified to prevent/notify in case of data loss.
+    String json = constructPrompt();
+
+    // Send the HTTP POST request
+    final serverResponse = await http.post(
+      Uri.parse('$url$endpoint'),
+      headers: headers,
+      body: json,
+    );
+
+    if (serverResponse.statusCode == 200) {
+      Map<String, dynamic> re = jsonDecode(serverResponse.body);
+      if(conversations[conversationID].conversationContext) {
+        response = re["message"]["content"];
+        // TODO: Modify in case of different response types.
+        loadedChats = [...loadedChats, Chat(response, false, 0)];
+      } else {
+        response = re["response"];
+      }
+    }
+    conversations[conversationID].add(Chat(response, false, 0));
+
+    prompt = "";
+    response = "";
+  }
+}

mobile/app/lib/components/util/model.dart

@@ -0,0 +1,5 @@
+class Model {
+  late String name;
+
+  Model(this.name);
+}

mobile/app/lib/components/util/speech_to_text.dart

@@ -0,0 +1,59 @@
+import 'package:speech_to_text/speech_recognition_result.dart';
+import 'package:speech_to_text/speech_to_text.dart' as stt;
+import 'package:speech_to_text/speech_to_text.dart';
+
+/// Handles standard Speech-To-Text using on-device processing.
+class SpeechToText {
+  final stt.SpeechToText speechToText = stt.SpeechToText();
+  // TODO: User customizable STT??
+  final SpeechListenOptions options = stt.SpeechListenOptions(
+    partialResults: true,
+    onDevice: true,
+  );
+
+  bool speechEnabled = false;
+  String text = "";
+
+  // Check if speech recognition is enabled.
+  bool isSpeechEnabled() => speechEnabled;
+
+  Future<String> getTextWhenReady() async {
+    await speechToText.stop();
+    return text;
+  }
+
+  // Retrieve the text.
+  @Deprecated("TODO: Deprecated message for speechToText.getText")
+  String getText() => text;
+
+  // Reset either after saving the text or prior to using STT again.
+  void reset() => text = "";
+
+  /// Start a speech recognition session
+  void startListening() async {
+    await speechToText.listen(
+      onResult: onSpeechResult,
+      pauseFor: const Duration(seconds: 3),
+
+      // TODO: I think this is a bug with the speech_to_text library? The docs correctly identify the named parameter of SpeechListenOptions, but it's not recognized.
+      // SpeechListenOptions: options,
+    );
+  }
+
+  /// Manually stop the active speech recognition session
+  /// Note that there are also timeouts that each platform enforces
+  /// and the SpeechToText plugin supports setting timeouts on the
+  /// listen method.
+  void stopListening() async => await speechToText.stop();
+
+  SpeechToText() {
+    initSpeech();
+  }
+
+  /// This has to happen only once per app
+  void initSpeech() async => speechEnabled = await speechToText.initialize();
+
+  /// This is the callback that the SpeechToText plugin calls when
+  /// the platform returns recognized words.
+  void onSpeechResult(SpeechRecognitionResult result) => (text = result.recognizedWords);
+}

mobile/app/lib/components/util/usage_queue.dart

@@ -0,0 +1,317 @@
+import 'dart:math';
+
+class UsageQueue<T> implements List {
+  @override
+  var first;
+
+  @override
+  var last;
+
+  @override
+  late int length;
+
+  @override
+  List operator +(List other) {
+    // TODO: implement +
+    throw UnimplementedError();
+  }
+
+  @override
+  operator [](int index) {
+    // TODO: implement []
+    throw UnimplementedError();
+  }
+
+  @override
+  void operator []=(int index, value) {
+    // TODO: implement []=
+  }
+
+  @override
+  void add(value) {
+    // TODO: implement add
+  }
+
+  @override
+  void addAll(Iterable iterable) {
+    // TODO: implement addAll
+  }
+
+  @override
+  bool any(bool Function(dynamic element) test) {
+    // TODO: implement any
+    throw UnimplementedError();
+  }
+
+  @override
+  Map<int, dynamic> asMap() {
+    // TODO: implement asMap
+    throw UnimplementedError();
+  }
+
+  @override
+  List<R> cast<R>() {
+    // TODO: implement cast
+    throw UnimplementedError();
+  }
+
+  @override
+  void clear() {
+    // TODO: implement clear
+  }
+
+  @override
+  bool contains(Object? element) {
+    // TODO: implement contains
+    throw UnimplementedError();
+  }
+
+  @override
+  elementAt(int index) {
+    // TODO: implement elementAt
+    throw UnimplementedError();
+  }
+
+  @override
+  bool every(bool Function(dynamic element) test) {
+    // TODO: implement every
+    throw UnimplementedError();
+  }
+
+  @override
+  Iterable<T> expand<T>(Iterable<T> Function(dynamic element) toElements) {
+    // TODO: implement expand
+    throw UnimplementedError();
+  }
+
+  @override
+  void fillRange(int start, int end, [fillValue]) {
+    // TODO: implement fillRange
+  }
+
+  @override
+  firstWhere(bool Function(dynamic element) test, {Function()? orElse}) {
+    // TODO: implement firstWhere
+    throw UnimplementedError();
+  }
+
+  @override
+  T fold<T>(T initialValue, T Function(T previousValue, dynamic element) combine) {
+    // TODO: implement fold
+    throw UnimplementedError();
+  }
+
+  @override
+  Iterable followedBy(Iterable other) {
+    // TODO: implement followedBy
+    throw UnimplementedError();
+  }
+
+  @override
+  void forEach(void Function(dynamic element) action) {
+    // TODO: implement forEach
+  }
+
+  @override
+  Iterable getRange(int start, int end) {
+    // TODO: implement getRange
+    throw UnimplementedError();
+  }
+
+  @override
+  int indexOf(element, [int start = 0]) {
+    // TODO: implement indexOf
+    throw UnimplementedError();
+  }
+
+  @override
+  int indexWhere(bool Function(dynamic element) test, [int start = 0]) {
+    // TODO: implement indexWhere
+    throw UnimplementedError();
+  }
+
+  @override
+  void insert(int index, element) {
+    // TODO: implement insert
+  }
+
+  @override
+  void insertAll(int index, Iterable iterable) {
+    // TODO: implement insertAll
+  }
+
+  @override
+  // TODO: implement isEmpty
+  bool get isEmpty => throw UnimplementedError();
+
+  @override
+  // TODO: implement isNotEmpty
+  bool get isNotEmpty => throw UnimplementedError();
+
+  @override
+  // TODO: implement iterator
+  Iterator get iterator => throw UnimplementedError();
+
+  @override
+  String join([String separator = ""]) {
+    // TODO: implement join
+    throw UnimplementedError();
+  }
+
+  @override
+  int lastIndexOf(element, [int? start]) {
+    // TODO: implement lastIndexOf
+    throw UnimplementedError();
+  }
+
+  @override
+  int lastIndexWhere(bool Function(dynamic element) test, [int? start]) {
+    // TODO: implement lastIndexWhere
+    throw UnimplementedError();
+  }
+
+  @override
+  lastWhere(bool Function(dynamic element) test, {Function()? orElse}) {
+    // TODO: implement lastWhere
+    throw UnimplementedError();
+  }
+
+  @override
+  Iterable<T> map<T>(T Function(dynamic e) toElement) {
+    // TODO: implement map
+    throw UnimplementedError();
+  }
+
+  @override
+  reduce(Function(dynamic value, dynamic element) combine) {
+    // TODO: implement reduce
+    throw UnimplementedError();
+  }
+
+  @override
+  bool remove(Object? value) {
+    // TODO: implement remove
+    throw UnimplementedError();
+  }
+
+  @override
+  removeAt(int index) {
+    // TODO: implement removeAt
+    throw UnimplementedError();
+  }
+
+  @override
+  removeLast() {
+    // TODO: implement removeLast
+    throw UnimplementedError();
+  }
+
+  @override
+  void removeRange(int start, int end) {
+    // TODO: implement removeRange
+  }
+
+  @override
+  void removeWhere(bool Function(dynamic element) test) {
+    // TODO: implement removeWhere
+  }
+
+  @override
+  void replaceRange(int start, int end, Iterable replacements) {
+    // TODO: implement replaceRange
+  }
+
+  @override
+  void retainWhere(bool Function(dynamic element) test) {
+    // TODO: implement retainWhere
+  }
+
+  @override
+  // TODO: implement reversed
+  Iterable get reversed => throw UnimplementedError();
+
+  @override
+  void setAll(int index, Iterable iterable) {
+    // TODO: implement setAll
+  }
+
+  @override
+  void setRange(int start, int end, Iterable iterable, [int skipCount = 0]) {
+    // TODO: implement setRange
+  }
+
+  @override
+  void shuffle([Random? random]) {
+    // TODO: implement shuffle
+  }
+
+  @override
+  // TODO: implement single
+  get single => throw UnimplementedError();
+
+  @override
+  singleWhere(bool Function(dynamic element) test, {Function()? orElse}) {
+    // TODO: implement singleWhere
+    throw UnimplementedError();
+  }
+
+  @override
+  Iterable skip(int count) {
+    // TODO: implement skip
+    throw UnimplementedError();
+  }
+
+  @override
+  Iterable skipWhile(bool Function(dynamic value) test) {
+    // TODO: implement skipWhile
+    throw UnimplementedError();
+  }
+
+  @override
+  void sort([int Function(dynamic a, dynamic b)? compare]) {
+    // TODO: implement sort
+  }
+
+  @override
+  List sublist(int start, [int? end]) {
+    // TODO: implement sublist
+    throw UnimplementedError();
+  }
+
+  @override
+  Iterable take(int count) {
+    // TODO: implement take
+    throw UnimplementedError();
+  }
+
+  @override
+  Iterable takeWhile(bool Function(dynamic value) test) {
+    // TODO: implement takeWhile
+    throw UnimplementedError();
+  }
+
+  @override
+  List toList({bool growable = true}) {
+    // TODO: implement toList
+    throw UnimplementedError();
+  }
+
+  @override
+  Set toSet() {
+    // TODO: implement toSet
+    throw UnimplementedError();
+  }
+
+  @override
+  Iterable where(bool Function(dynamic element) test) {
+    // TODO: implement where
+    throw UnimplementedError();
+  }
+
+  @override
+  Iterable<T> whereType<T>() {
+    // TODO: implement whereType
+    throw UnimplementedError();
+  }
+
+}