added multimedia support and STT. STT and photos are have bugs but are functional. file upload is conceptualized but will require additional backend work. video is on hold for now.

mobile/app/README.md

@@ -14,3 +14,5 @@ A few resources to get you started if this is your first Flutter project:
 For help getting started with Flutter development, view the
 [online documentation](https://docs.flutter.dev/), which offers tutorials,
 samples, guidance on mobile development, and a full API reference.
+
+# TODO

mobile/app/android/app/build.gradle

@@ -24,7 +24,7 @@ if (flutterVersionName == null) {
 }
 
 android {
-    namespace = "com.example.app"
+    namespace = "quaxlyqueen.dev.oneai"
     compileSdk = flutter.compileSdkVersion
     ndkVersion = flutter.ndkVersion
 
@@ -35,7 +35,7 @@ android {
 
     defaultConfig {
         // TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html).
-    	applicationId = "com.example.app"
+    	applicationId = "quaxlyqueen.dev.oneai"
         // You can update the following values to match your application needs.
         // For more information, see: https://docs.flutter.dev/deployment/android#reviewing-the-gradle-build-configuration.
         minSdk = flutter.minSdkVersion

mobile/app/ios/Podfile.lock

@@ -5,6 +5,8 @@ PODS:
     - HaishinKit (= 1.3.0)
   - ApiVideoLiveStream (1.2.2):
     - HaishinKit (= 1.3.0)
+  - camera_avfoundation (0.0.1):
+    - Flutter
   - DKImagePickerController/Core (4.3.9):
     - DKImagePickerController/ImageDataManager
     - DKImagePickerController/Resource
@@ -44,6 +46,8 @@ PODS:
     - Flutter
   - HaishinKit (1.3.0):
     - Logboard (~> 2.3.0)
+  - image_picker_ios (0.0.1):
+    - Flutter
   - Logboard (2.3.1)
   - native_device_orientation (0.0.1):
     - Flutter
@@ -78,9 +82,11 @@ PODS:
 
 DEPENDENCIES:
   - apivideo_live_stream (from `.symlinks/plugins/apivideo_live_stream/ios`)
+  - camera_avfoundation (from `.symlinks/plugins/camera_avfoundation/ios`)
   - file_picker (from `.symlinks/plugins/file_picker/ios`)
   - Flutter (from `Flutter`)
   - flutter_credit_card (from `.symlinks/plugins/flutter_credit_card/ios`)
+  - image_picker_ios (from `.symlinks/plugins/image_picker_ios/ios`)
   - native_device_orientation (from `.symlinks/plugins/native_device_orientation/ios`)
   - path_provider_foundation (from `.symlinks/plugins/path_provider_foundation/darwin`)
   - pointer_interceptor_ios (from `.symlinks/plugins/pointer_interceptor_ios/ios`)
@@ -106,12 +112,16 @@ SPEC REPOS:
 EXTERNAL SOURCES:
   apivideo_live_stream:
     :path: ".symlinks/plugins/apivideo_live_stream/ios"
+  camera_avfoundation:
+    :path: ".symlinks/plugins/camera_avfoundation/ios"
   file_picker:
     :path: ".symlinks/plugins/file_picker/ios"
   Flutter:
     :path: Flutter
   flutter_credit_card:
     :path: ".symlinks/plugins/flutter_credit_card/ios"
+  image_picker_ios:
+    :path: ".symlinks/plugins/image_picker_ios/ios"
   native_device_orientation:
     :path: ".symlinks/plugins/native_device_orientation/ios"
   path_provider_foundation:
@@ -136,12 +146,14 @@ EXTERNAL SOURCES:
 SPEC CHECKSUMS:
   apivideo_live_stream: 64fa696699cfc531e07f5d5d701c7e3cd5ce5e9d
   ApiVideoLiveStream: 5842571806a2ed26671597dadd370ec38d9b9ba4
+  camera_avfoundation: dd002b0330f4981e1bbcb46ae9b62829237459a4
   DKImagePickerController: 946cec48c7873164274ecc4624d19e3da4c1ef3c
   DKPhotoGallery: b3834fecb755ee09a593d7c9e389d8b5d6deed60
   file_picker: 09aa5ec1ab24135ccd7a1621c46c84134bfd6655
   Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7
   flutter_credit_card: 48fb35065294868bc6a3e7f18a42f28929b84f48
   HaishinKit: 013229683accc4abe46630b04629da3ff90428b7
+  image_picker_ios: c560581cceedb403a6ff17f2f816d7fea1421fc1
   Logboard: 3d98bb85de6a36b255ab637e8178eb5671c5c3a6
   native_device_orientation: 3b4cfc9565a7b879cc4fde282b3e27745e852d0d
   path_provider_foundation: 2b6b4c569c0fb62ec74538f866245ac84301af46

mobile/app/ios/Runner/Info.plist

@@ -2,8 +2,14 @@
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
 <dict>
-    	<key>NSMicrophoneUsageDescription</key>
+	<key>NSCameraUsageDescription</key>
-    	<string>Required for on-device speech recognition.</string>
+	<string>This app needs access to camera when used</string>
+	<key>NSPhotoLibraryUsageDescription</key>
+	<string>TODO: Better description</string>
+	<key>LSApplicationCategoryType</key>
+	<string>Productivity</string>
+	<key>NSMicrophoneUsageDescription</key>
+	<string>Required for on-device speech recognition.</string>
 	<key>NSSpeechRecognitionUsageDescription</key>
 	<string>On-device speech-to-text.</string>
 	<key>CADisableMinimumFrameDurationOnPhone</key>

mobile/app/lib/components/conversation_thread/chat.dart

@@ -1,9 +1,32 @@
 class Chat {
   final String content;
   final bool role;
+  int chatType = 0;
 
   // Most basic chat, text from server or from user.
-  Chat(this.content, this.role);
+  Chat(this.content, this.role, this.chatType);
 
-  // TODO: Overload constructors to allow for multimedia.
+  // A chat with an included image.
+  Chat.image(this.content, this.role, this.chatType);
+
+  // A chat with an included file.
+  Chat.file(this.content, this.role, this.chatType);
+}
+
+// TODO: Add proper filetype support. File should either be parsed on-device, or uploaded to the server.
+// TODO: Implement dynamic pathing based upon filetypes.
+enum Filetype {
+  pdf,
+  doc,
+  docx,
+  pages,
+  note,
+  todo,
+  cal,
+  calendar,
+  event,
+  contact,
+  mp3,
+  mp4,
+  wav,
 }

mobile/app/lib/components/conversation_thread/chat_bubble_widget.dart

@@ -1,3 +1,6 @@
+import 'dart:convert';
+
+import 'package:app/components/util/backend.dart';
 import 'package:app/theme.dart';
 import 'package:flutter/material.dart';
 import 'chat.dart';
@@ -29,10 +32,20 @@ class ChatBubbleWidget extends StatelessWidget{
             ),
             color: c.role ? AppTheme.darkest : AppTheme.dark,
           ),
-          child: Text(
+          child:
-            c.content,
+             // TODO: Properly display filetypes in the thread.
-            style: AppTheme.bodyMedium,
+             c.chatType == 0 ? Text(
-          ),
+               c.content,
+               style: AppTheme.bodyMedium,
+             )
+             : c.chatType == 1 ? Image(image: Backend.convertFromBase64(c.content))
+             : c.chatType == 2 ? const Icon(
+                 Icons.file_copy,
+             ) : const Text("null"),
+          // Text(
+          //   c.content,
+          //   style: AppTheme.bodyMedium,
+          // ),
         ),
       ],
     );

mobile/app/lib/components/conversation_thread/conversation.dart

@@ -1,4 +1,4 @@
-import 'package:app/util/backend.dart';
+import 'package:app/components/util/backend.dart';
 import 'package:flutter/material.dart';
 import 'chat.dart';
 
@@ -6,10 +6,12 @@ class Conversation with ChangeNotifier {
   int conversationID;
   String conversationLabel;
   String conversationSubLabel;
-  bool history = true;
+  // TODO: Fix history, very, verry strange behavior when disabled.
-  // History is disabled by default. This means chats are auto-deleted once the user exits that conversation.
+  bool history = true; // History is disabled by default. This means chats are auto-deleted once the user exits that conversation.\
   bool conversationContext = true; // The prior prompts and responses are used as additional context for the next prompt and response.
   late bool stream = false; // Update the UI with a stream of characters as received from the server, or as one text.
+  var lastAccessTime = DateTime.timestamp(); // Used to sort conversations based on most recent usage.
+
   List<Chat> chats = [];
 
   Conversation(this.conversationID, this.conversationLabel, this.conversationSubLabel);
@@ -23,11 +25,13 @@ class Conversation with ChangeNotifier {
   void toggleStream() => stream = !stream;
 
   void add(Chat message) {
+    lastAccessTime = DateTime.timestamp();
     chats.add(message);
     notifyListeners();
   }
 
   void loadChats() {
+    lastAccessTime = DateTime.timestamp();
     Backend.loadConversation;
     chats = Backend.loadedChats;
   }
@@ -41,10 +45,12 @@ class Conversation with ChangeNotifier {
   }
 
   List<Chat> getChats() {
+    lastAccessTime = DateTime.timestamp();
     return chats;
   }
 
   void setLabels(String cLabel, String cSubLabel) {
+    lastAccessTime = DateTime.timestamp();
     conversationLabel = cLabel;
     conversationSubLabel = cSubLabel;
   }

mobile/app/lib/components/conversation_thread/conversation_thread_model.dart

@@ -1,7 +1,7 @@
 import 'package:app/components/conversation_thread/chat_bubble_model.dart';
 import 'package:app/components/conversation_thread/chat.dart';
 import 'package:app/components/conversation_thread/conversation.dart';
-import 'package:app/util/backend.dart';
+import 'package:app/components/util/backend.dart';
 
 import 'package:app/components/prompt_box/prompt_box_widget.dart';
 import 'package:flutterflow_ui/flutterflow_ui.dart';

mobile/app/lib/components/conversation_thread/conversation_thread_widget.dart

@@ -1,5 +1,5 @@
 import 'package:app/components/conversation_thread/conversation.dart';
-import 'package:app/util/backend.dart';
+import 'package:app/components/util/backend.dart';
 import 'package:provider/provider.dart';
 
 import '/components/conversation_thread/chat_bubble_widget.dart';
@@ -50,6 +50,7 @@ class _ConversationThreadWidgetState extends State<ConversationThreadWidget> {
 
   @override
   Widget build(BuildContext context) {
+    // TODO: Add a scrollbar
       return Container(
         width: double.infinity,
         height: double.infinity,

mobile/app/lib/components/conversation_widget.dart

@@ -9,8 +9,8 @@ import 'package:flutter/services.dart';
 
 import 'package:app/theme.dart';
 
-import 'conversation_model.dart';
+import '../components/conversation_model.dart';
-export 'conversation_model.dart';
+export '../components/conversation_model.dart';
 
 class ConversationWidget extends StatefulWidget {
   final Conversation conversation;
@@ -64,8 +64,8 @@ class _ConversationWidgetState extends State<ConversationWidget> {
               size: 30,
             ),
             onPressed: () async {
-              print("something pressed in conversation_widget.dart");
               // TODO: Smooth out the animation to prevent the conversations list from loading half-way up the screen.
+              // Note: that only applies if the input was already selected.
               SystemChannels.textInput.invokeMethod('TextInput.hide');
               Navigator.pop(context);
             },

mobile/app/lib/components/conversations_list_widget.dart

@@ -1,15 +1,15 @@
 import 'package:app/components/conversation_thread/conversation.dart';
 import 'package:app/theme.dart';
 
-import 'package:app/pages/conversation_widget.dart';
+import 'package:app/components/conversation_widget.dart';
-import 'package:app/util/backend.dart';
+import 'package:app/components/util/backend.dart';
 
 import 'package:flutterflow_ui/flutterflow_ui.dart';
 import 'package:flutter/material.dart';
 
 import 'package:app/components/options_dialog/options_dialog_widget.dart';
-import 'conversations_list_model.dart';
+import '../components/conversations_list_model.dart';
-export 'conversations_list_model.dart';
+export '../components/conversations_list_model.dart';
 
 class ConversationsListWidget extends StatefulWidget {
   const ConversationsListWidget({super.key});
@@ -213,9 +213,6 @@ class _ConversationsListWidgetState extends State<ConversationsListWidget> {
                                         },
                                       ).then((value) =>
                                           safeSetState(() {})),
-
-                                    // TODO: Need to implement.
-                                    print("opening options")
                                   },
                                 ),
                               );
@@ -235,7 +232,7 @@ class _ConversationsListWidgetState extends State<ConversationsListWidget> {
                       Navigator.push(
                         context,
                         MaterialPageRoute(
-                          builder: (context) => ConversationWidget(conversation: Backend.getLoadedConversation(),),
+                          builder: (context) => ConversationWidget(conversation: Backend.loadedConversation,),
                         ),
                       ), // Handle the button press
                     },

mobile/app/lib/components/no_chats_model.dart

@@ -1,13 +0,0 @@
-import 'package:flutterflow_ui/flutterflow_ui.dart';
-import 'no_chats_widget.dart' show NoChatsWidget;
-import 'package:flutter/material.dart';
-import 'package:google_fonts/google_fonts.dart';
-import 'package:provider/provider.dart';
-
-class NoChatsModel extends FlutterFlowModel<NoChatsWidget> {
-  @override
-  void initState(BuildContext context) {}
-
-  @override
-  void dispose() {}
-}

mobile/app/lib/components/no_chats_widget.dart

@@ -5,9 +5,6 @@ import 'package:provider/provider.dart';
 
 import 'package:app/theme.dart';
 
-import 'no_chats_model.dart';
-export 'no_chats_model.dart';
-
 class NoChatsWidget extends StatefulWidget {
   const NoChatsWidget({
     super.key,
@@ -26,24 +23,19 @@ class NoChatsWidget extends StatefulWidget {
 }
 
 class _NoChatsWidgetState extends State<NoChatsWidget> {
-  late NoChatsModel _model;
 
   @override
   void setState(VoidCallback callback) {
     super.setState(callback);
-    _model.onUpdate();
   }
 
   @override
   void initState() {
     super.initState();
-    _model = createModel(context, () => NoChatsModel());
   }
 
   @override
   void dispose() {
-    _model.maybeDispose();
-
     super.dispose();
   }
 

mobile/app/lib/components/prompt_box/prompt_box_widget.dart

@@ -1,10 +1,17 @@
-import 'package:app/util/backend.dart';
+import 'dart:io';
+
+import 'package:app/components/util/backend.dart';
+import 'package:camera/camera.dart';
+import 'package:file_picker/file_picker.dart';
+import 'package:flutter/services.dart';
 import 'package:flutterflow_ui/flutterflow_ui.dart';
 import 'package:easy_debounce/easy_debounce.dart';
 import 'package:flutter/material.dart';
 
 import 'package:app/theme.dart';
+import 'package:image_picker/image_picker.dart';
 
+import '../conversation_thread/chat.dart';
 import 'prompt_box_model.dart';
 export 'prompt_box_model.dart';
 
@@ -17,7 +24,7 @@ class PromptBoxWidget extends StatefulWidget {
 
 class _PromptBoxWidgetState extends State<PromptBoxWidget> {
   late PromptBoxModel _model;
-  late String entry_text;
+  String entryText = "";
 
   @override
   void setState(VoidCallback callback) {
@@ -43,6 +50,7 @@ class _PromptBoxWidgetState extends State<PromptBoxWidget> {
 
   @override
   Widget build(BuildContext context) {
+    const double buttonPadding = 4;
     return SafeArea(
       child: Container(
         width: double.infinity,
@@ -62,50 +70,59 @@ class _PromptBoxWidgetState extends State<PromptBoxWidget> {
               children: [
                 Expanded( // TODO: File context upload
                   child: Padding(
-                    padding: const EdgeInsets.all(14),
+                    padding: const EdgeInsets.all(buttonPadding),
-                    child: FlutterFlowIconButton(
+                    child: IconButton(
-                      borderColor: Colors.transparent,
-                      borderRadius: 30,
-                      borderWidth: 1,
-                      buttonSize: 40,
                       icon: const Icon(
                         Icons.file_upload_outlined,
                         color: AppTheme.darkest,
                         size: 30,
                       ),
-                      onPressed: () {
+                      onPressed: () async {
-                        print('IconButton pressed ...');
+                        final picker = FilePicker.platform;
+                        final result = await picker.pickFiles(allowMultiple: true); // Allow multiple files
+                        if (result != null) {
+                          for (final platformFile in result.files) {
+                            // TODO: Upload file to server
+                            final file = File(platformFile.path!);
+                          }
+                        } else {
+                          print('User canceled file selection');
+                        }
                       },
                     ),
                   ),
                 ),
-                Expanded( // TODO: Photo context upload
+                Expanded( // TODO: Gallery context upload
                   child: Padding(
-                    padding: const EdgeInsets.all(14),
+                    padding: const EdgeInsets.all(buttonPadding),
-                    child: FlutterFlowIconButton(
+                    child: IconButton(
-                      borderColor: Colors.transparent,
-                      borderRadius: 30,
-                      borderWidth: 1,
-                      buttonSize: 40,
                       icon: const Icon(
                         Icons.photo_outlined,
                         color: AppTheme.darkest,
                         size: 30,
                       ),
-                      onPressed: () {
+                      onPressed: () async {
-                        print('IconButton pressed ...');
+                        final picker = ImagePicker();
+                        final image = await picker.pickImage(source: ImageSource.gallery);
+
+                        // TODO: Allow for multiple pictures/photos to be selected.
+                        // final images = await picker.pickMultiImage();
+                        if (image != null) {
+
+                          // TODO: Upload to server and add to conversation.chats
+                          String i = Backend.convertToBase64(File(image.path)).toString();
+                          Backend.loadedConversation.add(Chat.image(i, true, 1));
+                        } else {
+                          print('User canceled image selection');
+                        }
                       },
                     ),
                   ),
                 ),
                 Expanded( // TODO: Video context upload
                   child: Padding(
-                    padding: const EdgeInsets.all(14),
+                    padding: const EdgeInsets.all(buttonPadding),
-                    child: FlutterFlowIconButton(
+                    child: IconButton(
-                      borderColor: Colors.transparent,
-                      borderRadius: 30,
-                      borderWidth: 1,
-                      buttonSize: 40,
                       icon: const Icon(
                         Icons.videocam_outlined,
                         color: AppTheme.darkest,
@@ -119,19 +136,48 @@ class _PromptBoxWidgetState extends State<PromptBoxWidget> {
                 ),
                 Expanded( // TODO: Camera context upload
                   child: Padding(
-                    padding: const EdgeInsets.all(14),
+                    padding: const EdgeInsets.all(buttonPadding),
-                    child: FlutterFlowIconButton(
+                    child: IconButton(
-                      borderColor: Colors.transparent,
-                      borderRadius: 30,
-                      borderWidth: 1,
-                      buttonSize: 40,
                       icon: const Icon(
                         Icons.camera_alt_outlined,
                         color: AppTheme.darkest,
                         size: 30,
                       ),
+                      onPressed: () async {
+                        final picker = ImagePicker();
+                        final image = await picker.pickImage(source: ImageSource.camera);
+
+                        if (image != null) {
+                          // TODO: Upload to server and add to conversation.chats
+                          final i = Backend.convertToBase64(File(image.path));
+                          Backend.loadedConversation.add(Chat.image(i.toString(), true, 1));
+                        } else {
+                          print('User canceled photo capture');
+                        }
+                      },
+                    ),
+                  ),
+                ),
+                Expanded(
+                  child: Padding(
+                    padding: const EdgeInsets.all(buttonPadding),
+                    child: IconButton(
+                      icon: Icon(
+                        Backend.stt.speechToText.isNotListening ? Icons.mic_off_outlined : Icons.mic_none,
+                        color: AppTheme.darkest,
+                        size: 30,
+                      ),
                       onPressed: () {
-                        print('IconButton pressed ...');
+                        // TODO: Audio input is still buggy, it appears to only enter the text field once the button is pressed again to stop.
+                        if(Backend.stt.speechToText.isNotListening) {
+                          print('STT engine starting ...');
+                          Backend.stt.startListening();
+                          updateText();
+                        } else {
+                          print('STT engine stopping');
+                          Backend.stt.stopListening();
+                          updateText();
+                        }
                       },
                     ),
                   ),
@@ -190,10 +236,13 @@ class _PromptBoxWidgetState extends State<PromptBoxWidget> {
                         fillColor: AppTheme.alternate,
                         suffixIcon: GestureDetector(
                           onTap: () {
-                            // TODO: Connect with multimedia
+                            if(_model.textController.text != "") {
-                            entry_text = _model.textController.text;
+                              // TODO: Connect with multimedia
-                            Backend.respondWhenReady(entry_text);
+                              entryText = _model.textController.text;
-                            _model.textController?.clear(); // Clear the text field
+                              Backend.respondWhenReady(entryText);
+                              _model.textController?.clear(); // Clear the text field
+                              SystemChannels.textInput.invokeMethod('TextInput.hide');
+                            }
                           },
                           child: const Icon(
                             Icons.send,
@@ -206,12 +255,14 @@ class _PromptBoxWidgetState extends State<PromptBoxWidget> {
                       textAlign: TextAlign.start,
                       maxLines: 5,
                       minLines: 1,
-                      validator: _model.textControllerValidator.asValidator(context),
                       textInputAction: TextInputAction.send, // "Send" on keyboard
                       onFieldSubmitted: (text) {
-                        entry_text = _model.textController.text;
+                        // Don't take any action unless there's an actual input.
-                        Backend.respondWhenReady(entry_text); // TODO: Connect with multimedia
+                        if(_model.textController.text != "") {
-                        _model.textController?.clear(); // Clear the text field
+                          entryText = _model.textController.text;
+                          Backend.respondWhenReady(entryText); // TODO: Connect with multimedia
+                          _model.textController?.clear(); // Clear the text field
+                        }
                       },
                     ),
                   ),
@@ -223,4 +274,12 @@ class _PromptBoxWidgetState extends State<PromptBoxWidget> {
       ),
     );
   }
+
+  void updateText() {
+    _model.textController.text = Backend.stt.text;
+    entryText = _model.textController.text;
+    print(entryText);
+    print(_model.textController.text);
+    Backend.stt.reset();
+  }
 }

mobile/app/lib/components/support_form_widget.dart

@@ -4,8 +4,8 @@ import 'package:flutter_animate/flutter_animate.dart';
 import 'package:google_fonts/google_fonts.dart';
 import 'package:provider/provider.dart';
 
-import 'support_form_model.dart';
+import '../components/support_form_model.dart';
-export 'support_form_model.dart';
+export '../components/support_form_model.dart';
 
 import 'package:app/theme.dart';
 

mobile/app/lib/components/util/backend.dart

@@ -1,10 +1,14 @@
 import 'dart:convert';
+import 'dart:io';
+import 'dart:typed_data';
+import 'package:flutter/widgets.dart';
 import 'package:app/components/conversation_thread/conversation.dart';
+import 'package:camera/camera.dart';
 import 'package:http/http.dart' as http;
 
 import 'package:app/components/conversation_thread/chat.dart';
 
-import 'package:app/util/speech_to_text.dart';
+import 'package:app/components/util/speech_to_text.dart';
 
 import 'model.dart';
 
@@ -18,6 +22,7 @@ class Backend {
   //static final url = "http://192.168.1.68:11434/api"; // For local access to home server
   static String endpoint = "";
   static final Map<String, String> headers = {'Content-Type': 'application/json'};
+  static final bool UI_TESTING = true;
 
 
 
@@ -27,11 +32,13 @@ class Backend {
 
 
 
-  //     PRIVACY-FOCUSED VARIABLES
+  //     PRIVACY-NECESSARY VARIABLES
   // TODO: Reorder based on most recent usage.
   static List<Conversation> conversations = [];
   static List<Chat> loadedChats = [];
   static late Conversation loadedConversation;
+  static late List<CameraDescription> cameras;
+  static late CameraController controller;
 
 
   //     RUNTIME & STATUS VARIABLES
@@ -45,40 +52,35 @@ class Backend {
   // TODO: Verify all different components of STT, TTS, LLM, etc have been initialized.
   // TODO: Handshake w/ server to validate identity
   factory Backend() {
-    // Speech-To-Text engine activation.
-    _speechEnabled = stt.isSpeechEnabled();
 
-    // loadedChats = conversations[conversationID].chats;
 
     return singleBackend;
   }
 
-  void init() {
+  void init() async {
-    initiated = true;
+    // Prevent multiple initiations
+    if(!initiated) {
+      initiated = true;
+      // Speech-To-Text engine activation.
+      _speechEnabled = stt.isSpeechEnabled();
+
+      model = Model("llama3"); // Primarily used llama3, testing gemma2
+      maxConversationID = conversations.length;
+
+      cameras = await availableCameras();
+      controller = CameraController(cameras[0], ResolutionPreset.max);
+      controller.initialize(); // TODO: Add error catching on camera
+    }
 
     // TODO: Dynamically get available models and models that can be pulled.
     // TODO: Dynamically get the user's default model
-    model = Model("llama3"); // Primarily used llama3, testing gemma2
-
     // TODO: Dynamically obtain conversations.
-    // conversations.add(
+    // TODO: Obtain all user permissions at once.
-    //   Conversation.completeConversation(
-    //     0,
-    //     "Test Conversation",
-    //     "Testing Functionality",
-    //     [
-    //       (Chat("Hello there! You are part of a prototype and are running locally on my Macbook Air. Please keep your responses brief as every prompt freezes my computer.", true)),
-    //       (Chat("Understood! How can I help?", false)),
-    //     ]
-    //   )
-    // );
-
-    maxConversationID = conversations.length;
   }
 
 
 
-  //     PRIMARY INTERFACE
+  //     PRIMARY INTERFACES
   @Deprecated("Use STT interface directly and use respondWhenReady().")
   static Future<String> sttGetResponseWhenReady() async {
     // TODO: In case on-device STT is unavailable, use server-side STT service.
@@ -87,19 +89,38 @@ class Backend {
     }
 
     prompt = stt.getTextWhenReady() as String;
-    conversations[conversationID].add(Chat(prompt, true));
+    conversations[conversationID].add(Chat(prompt, true, 0));
-    httpSendRequest();
+    if(!UI_TESTING) {
-    conversations[conversationID].add(Chat(response, false));
+      httpSendRequest();
+    }
+    conversations[conversationID].add(Chat(response, false, 0));
 
     return response;
   }
 
   static void respondWhenReady(String p) async {
     prompt = p;
-    conversations[conversationID].add(Chat(prompt, true));
+    conversations[conversationID].add(Chat(prompt, true, 0));
-    httpSendRequest();
+    if(!UI_TESTING) {
+      httpSendRequest();
+    }
   }
 
+  static ImageProvider convertFromBase64(img) {
+    Uint8List imageBytes = base64Decode(img);
+
+    return Image.memory(imageBytes).image;
+  }
+
+  // Converts image to base64 encoded string for data transfer.
+  static String convertToBase64(f)  {
+    List<int> imageBytes = f.readAsBytesSync();
+    String img = base64Encode(imageBytes);
+
+    // TODO: Add special handling for images included in prompts.
+    conversations[conversationID].add(Chat.image(img, true, 1));
+    return img;
+  }
 
 
   //     CONVERSATION MANAGEMENT
@@ -155,7 +176,7 @@ class Backend {
     Map<String, dynamic> data;
 
     if(conversations[conversationID].conversationContext) {
-      loadedChats = [...loadedChats, Chat(prompt, true)];
+      loadedChats = [...loadedChats, Chat(prompt, true, 0)];
       endpoint = "chat";
 
       List<Map<String, String>> messageList = convertMessages();
@@ -196,12 +217,13 @@ class Backend {
       Map<String, dynamic> re = jsonDecode(serverResponse.body);
       if(conversations[conversationID].conversationContext) {
         response = re["message"]["content"];
-        loadedChats = [...loadedChats, Chat(response, false)];
+        // TODO: Modify in case of different response types.
+        loadedChats = [...loadedChats, Chat(response, false, 0)];
       } else {
         response = re["response"];
       }
     }
-    conversations[conversationID].add(Chat(response, false));
+    conversations[conversationID].add(Chat(response, false, 0));
 
     prompt = "";
     response = "";

mobile/app/lib/components/util/speech_to_text.dart

@@ -1,9 +1,16 @@
 import 'package:speech_to_text/speech_recognition_result.dart';
 import 'package:speech_to_text/speech_to_text.dart' as stt;
+import 'package:speech_to_text/speech_to_text.dart';
 
 /// Handles standard Speech-To-Text using on-device processing.
 class SpeechToText {
   final stt.SpeechToText speechToText = stt.SpeechToText();
+  // TODO: User customizable STT??
+  final SpeechListenOptions options = stt.SpeechListenOptions(
+    partialResults: true,
+    onDevice: true,
+  );
+
   bool speechEnabled = false;
   String text = "";
 
@@ -16,13 +23,22 @@ class SpeechToText {
   }
 
   // Retrieve the text.
+  @Deprecated("TODO: Deprecated message for speechToText.getText")
   String getText() => text;
 
   // Reset either after saving the text or prior to using STT again.
   void reset() => text = "";
 
   /// Start a speech recognition session
-  void startListening() async => await speechToText.listen(onResult: onSpeechResult);
+  void startListening() async {
+    await speechToText.listen(
+      onResult: onSpeechResult,
+      pauseFor: const Duration(seconds: 3),
+
+      // TODO: I think this is a bug with the speech_to_text library? The docs correctly identify the named parameter of SpeechListenOptions, but it's not recognized.
+      // SpeechListenOptions: options,
+    );
+  }
 
   /// Manually stop the active speech recognition session
   /// Note that there are also timeouts that each platform enforces

mobile/app/lib/components/util/usage_queue.dart

@@ -0,0 +1,317 @@
+import 'dart:math';
+
+class UsageQueue<T> implements List {
+  @override
+  var first;
+
+  @override
+  var last;
+
+  @override
+  late int length;
+
+  @override
+  List operator +(List other) {
+    // TODO: implement +
+    throw UnimplementedError();
+  }
+
+  @override
+  operator [](int index) {
+    // TODO: implement []
+    throw UnimplementedError();
+  }
+
+  @override
+  void operator []=(int index, value) {
+    // TODO: implement []=
+  }
+
+  @override
+  void add(value) {
+    // TODO: implement add
+  }
+
+  @override
+  void addAll(Iterable iterable) {
+    // TODO: implement addAll
+  }
+
+  @override
+  bool any(bool Function(dynamic element) test) {
+    // TODO: implement any
+    throw UnimplementedError();
+  }
+
+  @override
+  Map<int, dynamic> asMap() {
+    // TODO: implement asMap
+    throw UnimplementedError();
+  }
+
+  @override
+  List<R> cast<R>() {
+    // TODO: implement cast
+    throw UnimplementedError();
+  }
+
+  @override
+  void clear() {
+    // TODO: implement clear
+  }
+
+  @override
+  bool contains(Object? element) {
+    // TODO: implement contains
+    throw UnimplementedError();
+  }
+
+  @override
+  elementAt(int index) {
+    // TODO: implement elementAt
+    throw UnimplementedError();
+  }
+
+  @override
+  bool every(bool Function(dynamic element) test) {
+    // TODO: implement every
+    throw UnimplementedError();
+  }
+
+  @override
+  Iterable<T> expand<T>(Iterable<T> Function(dynamic element) toElements) {
+    // TODO: implement expand
+    throw UnimplementedError();
+  }
+
+  @override
+  void fillRange(int start, int end, [fillValue]) {
+    // TODO: implement fillRange
+  }
+
+  @override
+  firstWhere(bool Function(dynamic element) test, {Function()? orElse}) {
+    // TODO: implement firstWhere
+    throw UnimplementedError();
+  }
+
+  @override
+  T fold<T>(T initialValue, T Function(T previousValue, dynamic element) combine) {
+    // TODO: implement fold
+    throw UnimplementedError();
+  }
+
+  @override
+  Iterable followedBy(Iterable other) {
+    // TODO: implement followedBy
+    throw UnimplementedError();
+  }
+
+  @override
+  void forEach(void Function(dynamic element) action) {
+    // TODO: implement forEach
+  }
+
+  @override
+  Iterable getRange(int start, int end) {
+    // TODO: implement getRange
+    throw UnimplementedError();
+  }
+
+  @override
+  int indexOf(element, [int start = 0]) {
+    // TODO: implement indexOf
+    throw UnimplementedError();
+  }
+
+  @override
+  int indexWhere(bool Function(dynamic element) test, [int start = 0]) {
+    // TODO: implement indexWhere
+    throw UnimplementedError();
+  }
+
+  @override
+  void insert(int index, element) {
+    // TODO: implement insert
+  }
+
+  @override
+  void insertAll(int index, Iterable iterable) {
+    // TODO: implement insertAll
+  }
+
+  @override
+  // TODO: implement isEmpty
+  bool get isEmpty => throw UnimplementedError();
+
+  @override
+  // TODO: implement isNotEmpty
+  bool get isNotEmpty => throw UnimplementedError();
+
+  @override
+  // TODO: implement iterator
+  Iterator get iterator => throw UnimplementedError();
+
+  @override
+  String join([String separator = ""]) {
+    // TODO: implement join
+    throw UnimplementedError();
+  }
+
+  @override
+  int lastIndexOf(element, [int? start]) {
+    // TODO: implement lastIndexOf
+    throw UnimplementedError();
+  }
+
+  @override
+  int lastIndexWhere(bool Function(dynamic element) test, [int? start]) {
+    // TODO: implement lastIndexWhere
+    throw UnimplementedError();
+  }
+
+  @override
+  lastWhere(bool Function(dynamic element) test, {Function()? orElse}) {
+    // TODO: implement lastWhere
+    throw UnimplementedError();
+  }
+
+  @override
+  Iterable<T> map<T>(T Function(dynamic e) toElement) {
+    // TODO: implement map
+    throw UnimplementedError();
+  }
+
+  @override
+  reduce(Function(dynamic value, dynamic element) combine) {
+    // TODO: implement reduce
+    throw UnimplementedError();
+  }
+
+  @override
+  bool remove(Object? value) {
+    // TODO: implement remove
+    throw UnimplementedError();
+  }
+
+  @override
+  removeAt(int index) {
+    // TODO: implement removeAt
+    throw UnimplementedError();
+  }
+
+  @override
+  removeLast() {
+    // TODO: implement removeLast
+    throw UnimplementedError();
+  }
+
+  @override
+  void removeRange(int start, int end) {
+    // TODO: implement removeRange
+  }
+
+  @override
+  void removeWhere(bool Function(dynamic element) test) {
+    // TODO: implement removeWhere
+  }
+
+  @override
+  void replaceRange(int start, int end, Iterable replacements) {
+    // TODO: implement replaceRange
+  }
+
+  @override
+  void retainWhere(bool Function(dynamic element) test) {
+    // TODO: implement retainWhere
+  }
+
+  @override
+  // TODO: implement reversed
+  Iterable get reversed => throw UnimplementedError();
+
+  @override
+  void setAll(int index, Iterable iterable) {
+    // TODO: implement setAll
+  }
+
+  @override
+  void setRange(int start, int end, Iterable iterable, [int skipCount = 0]) {
+    // TODO: implement setRange
+  }
+
+  @override
+  void shuffle([Random? random]) {
+    // TODO: implement shuffle
+  }
+
+  @override
+  // TODO: implement single
+  get single => throw UnimplementedError();
+
+  @override
+  singleWhere(bool Function(dynamic element) test, {Function()? orElse}) {
+    // TODO: implement singleWhere
+    throw UnimplementedError();
+  }
+
+  @override
+  Iterable skip(int count) {
+    // TODO: implement skip
+    throw UnimplementedError();
+  }
+
+  @override
+  Iterable skipWhile(bool Function(dynamic value) test) {
+    // TODO: implement skipWhile
+    throw UnimplementedError();
+  }
+
+  @override
+  void sort([int Function(dynamic a, dynamic b)? compare]) {
+    // TODO: implement sort
+  }
+
+  @override
+  List sublist(int start, [int? end]) {
+    // TODO: implement sublist
+    throw UnimplementedError();
+  }
+
+  @override
+  Iterable take(int count) {
+    // TODO: implement take
+    throw UnimplementedError();
+  }
+
+  @override
+  Iterable takeWhile(bool Function(dynamic value) test) {
+    // TODO: implement takeWhile
+    throw UnimplementedError();
+  }
+
+  @override
+  List toList({bool growable = true}) {
+    // TODO: implement toList
+    throw UnimplementedError();
+  }
+
+  @override
+  Set toSet() {
+    // TODO: implement toSet
+    throw UnimplementedError();
+  }
+
+  @override
+  Iterable where(bool Function(dynamic element) test) {
+    // TODO: implement where
+    throw UnimplementedError();
+  }
+
+  @override
+  Iterable<T> whereType<T>() {
+    // TODO: implement whereType
+    throw UnimplementedError();
+  }
+
+}

mobile/app/lib/main.dart

@@ -1,6 +1,6 @@
-import 'package:app/pages/conversation_widget.dart';
+import 'package:app/components/conversation_widget.dart';
-import 'package:app/pages/settings_widget.dart';
+import 'package:app/components/settings_widget.dart';
-import 'package:app/pages/conversations_list_widget.dart';
+import 'package:app/components/conversations_list_widget.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter_riverpod/flutter_riverpod.dart';
 import 'package:provider/provider.dart';
@@ -36,6 +36,7 @@ BottomNavigationBarItem _buildNavigationItem(int index) {
     icon: Icon(iconData),
     label: label,
     backgroundColor: AppTheme.lightest,
+
   );
 }
 
@@ -67,21 +68,14 @@ class MyApp extends ConsumerWidget {
       title: 'One AI',
       home: Scaffold(
         backgroundColor: AppTheme.medium,
-        appBar: AppBar(
-            backgroundColor: AppTheme.secondaryBackground,
-            automaticallyImplyLeading: false,
-            title: const Text(
-              'One AI',
-              style: AppTheme.displayLarge,
-            ),
-            actions: const [],
-            centerTitle: false,
-            elevation: 0,
-          ),
         body: _pages[selectedIndex],
         bottomNavigationBar: BottomNavigationBar(
           backgroundColor: AppTheme.darkest,
           selectedItemColor: AppTheme.lightest,
+          showUnselectedLabels: false,
+          showSelectedLabels: false,
+          iconSize: 30,
+
           // Set the number of items to the length of the pages list
           items: List.generate(_pages.length, (index) => _buildNavigationItem(index)),
           currentIndex: selectedIndex,

mobile/app/linux/flutter/generated_plugin_registrant.cc

@@ -6,9 +6,13 @@
 
 #include "generated_plugin_registrant.h"
 
+#include <file_selector_linux/file_selector_plugin.h>
 #include <url_launcher_linux/url_launcher_plugin.h>
 
 void fl_register_plugins(FlPluginRegistry* registry) {
+  g_autoptr(FlPluginRegistrar) file_selector_linux_registrar =
+      fl_plugin_registry_get_registrar_for_plugin(registry, "FileSelectorPlugin");
+  file_selector_plugin_register_with_registrar(file_selector_linux_registrar);
   g_autoptr(FlPluginRegistrar) url_launcher_linux_registrar =
       fl_plugin_registry_get_registrar_for_plugin(registry, "UrlLauncherPlugin");
   url_launcher_plugin_register_with_registrar(url_launcher_linux_registrar);

mobile/app/linux/flutter/generated_plugins.cmake

@@ -3,6 +3,7 @@
 #
 
 list(APPEND FLUTTER_PLUGIN_LIST
+  file_selector_linux
   url_launcher_linux
 )
 

mobile/app/macos/Flutter/GeneratedPluginRegistrant.swift

@@ -5,6 +5,7 @@
 import FlutterMacOS
 import Foundation
 
+import file_selector_macos
 import path_provider_foundation
 import rive_common
 import shared_preferences_foundation
@@ -14,6 +15,7 @@ import url_launcher_macos
 import video_player_avfoundation
 
 func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) {
+  FileSelectorPlugin.register(with: registry.registrar(forPlugin: "FileSelectorPlugin"))
   PathProviderPlugin.register(with: registry.registrar(forPlugin: "PathProviderPlugin"))
   RivePlugin.register(with: registry.registrar(forPlugin: "RivePlugin"))
   SharedPreferencesPlugin.register(with: registry.registrar(forPlugin: "SharedPreferencesPlugin"))

mobile/app/pubspec.lock

@@ -89,6 +89,46 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "1.2.0"
+  camera:
+    dependency: "direct main"
+    description:
+      name: camera
+      sha256: "2170a943dcb67be2af2c6bcda8775e74b41d4c02d6a4eb10bdc832ee185c4eea"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.11.0+1"
+  camera_android_camerax:
+    dependency: transitive
+    description:
+      name: camera_android_camerax
+      sha256: "7c03940cb8c92eb5b184952674a07cc4a73c6ba2b3568aad70255ad4cb913660"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.6.7+1"
+  camera_avfoundation:
+    dependency: transitive
+    description:
+      name: camera_avfoundation
+      sha256: "95c2f40b4d06cdb0fd2ad893c762d1f2c931a5e370793ec34c939eb4fcbf96bb"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.9.17"
+  camera_platform_interface:
+    dependency: transitive
+    description:
+      name: camera_platform_interface
+      sha256: b3ede1f171532e0d83111fe0980b46d17f1aa9788a07a2fbed07366bbdbb9061
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.8.0"
+  camera_web:
+    dependency: transitive
+    description:
+      name: camera_web
+      sha256: b9235ec0a2ce949daec546f1f3d86f05c3921ed31c7d9ab6b7c03214d152fc2d
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.3.4"
   carousel_slider:
     dependency: transitive
     description:
@@ -234,13 +274,45 @@ packages:
     source: hosted
     version: "7.0.0"
   file_picker:
-    dependency: transitive
+    dependency: "direct main"
     description:
       name: file_picker
       sha256: "824f5b9f389bfc4dddac3dea76cd70c51092d9dff0b2ece7ef4f53db8547d258"
       url: "https://pub.dev"
     source: hosted
     version: "8.0.6"
+  file_selector_linux:
+    dependency: transitive
+    description:
+      name: file_selector_linux
+      sha256: "045d372bf19b02aeb69cacf8b4009555fb5f6f0b7ad8016e5f46dd1387ddd492"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.9.2+1"
+  file_selector_macos:
+    dependency: transitive
+    description:
+      name: file_selector_macos
+      sha256: f42eacb83b318e183b1ae24eead1373ab1334084404c8c16e0354f9a3e55d385
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.9.4"
+  file_selector_platform_interface:
+    dependency: transitive
+    description:
+      name: file_selector_platform_interface
+      sha256: a3994c26f10378a039faa11de174d7b78eb8f79e4dd0af2a451410c1a5c3f66b
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.6.2"
+  file_selector_windows:
+    dependency: transitive
+    description:
+      name: file_selector_windows
+      sha256: d3547240c20cabf205c7c7f01a50ecdbc413755814d6677f3cb366f04abcead0
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.9.3+1"
   fixnum:
     dependency: transitive
     description:
@@ -480,6 +552,70 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "4.0.2"
+  image_picker:
+    dependency: "direct main"
+    description:
+      name: image_picker
+      sha256: "021834d9c0c3de46bf0fe40341fa07168407f694d9b2bb18d532dc1261867f7a"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.1.2"
+  image_picker_android:
+    dependency: transitive
+    description:
+      name: image_picker_android
+      sha256: ff39a10ab4f48f4ac70776d0494a97bf073cd2570892cd46bc8a5cac162c25db
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.8.12+4"
+  image_picker_for_web:
+    dependency: transitive
+    description:
+      name: image_picker_for_web
+      sha256: "5d6eb13048cd47b60dbf1a5495424dea226c5faf3950e20bf8120a58efb5b5f3"
+      url: "https://pub.dev"
+    source: hosted
+    version: "3.0.4"
+  image_picker_ios:
+    dependency: transitive
+    description:
+      name: image_picker_ios
+      sha256: "6703696ad49f5c3c8356d576d7ace84d1faf459afb07accbb0fae780753ff447"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.8.12"
+  image_picker_linux:
+    dependency: transitive
+    description:
+      name: image_picker_linux
+      sha256: "4ed1d9bb36f7cd60aa6e6cd479779cc56a4cb4e4de8f49d487b1aaad831300fa"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.2.1+1"
+  image_picker_macos:
+    dependency: transitive
+    description:
+      name: image_picker_macos
+      sha256: "3f5ad1e8112a9a6111c46d0b57a7be2286a9a07fc6e1976fdf5be2bd31d4ff62"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.2.1+1"
+  image_picker_platform_interface:
+    dependency: transitive
+    description:
+      name: image_picker_platform_interface
+      sha256: "9ec26d410ff46f483c5519c29c02ef0e02e13a543f882b152d4bfd2f06802f80"
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.10.0"
+  image_picker_windows:
+    dependency: transitive
+    description:
+      name: image_picker_windows
+      sha256: "6ad07afc4eb1bc25f3a01084d28520496c4a3bb0cb13685435838167c9dcedeb"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.2.1+1"
   intl:
     dependency: transitive
     description:
@@ -600,6 +736,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "1.14.0"
+  mime:
+    dependency: transitive
+    description:
+      name: mime
+      sha256: "2e123074287cc9fd6c09de8336dae606d1ddb88d9ac47358826db698c176a1f2"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.0.5"
   mime_type:
     dependency: transitive
     description:

mobile/app/pubspec.yaml

@@ -46,6 +46,9 @@ dependencies:
   video_player: ^2.9.1
   flutter_riverpod: ^2.5.1
   speech_to_text: ^6.6.0
+  camera: ^0.11.0+1
+  image_picker: ^1.1.2
+  file_picker: ^8.0.6
 dev_dependencies:
   flutter_test:
     sdk: flutter

mobile/app/windows/flutter/generated_plugin_registrant.cc

@@ -6,10 +6,13 @@
 
 #include "generated_plugin_registrant.h"
 
+#include <file_selector_windows/file_selector_windows.h>
 #include <rive_common/rive_plugin.h>
 #include <url_launcher_windows/url_launcher_windows.h>
 
 void RegisterPlugins(flutter::PluginRegistry* registry) {
+  FileSelectorWindowsRegisterWithRegistrar(
+      registry->GetRegistrarForPlugin("FileSelectorWindows"));
   RivePluginRegisterWithRegistrar(
       registry->GetRegistrarForPlugin("RivePlugin"));
   UrlLauncherWindowsRegisterWithRegistrar(

mobile/app/windows/flutter/generated_plugins.cmake

@@ -3,6 +3,7 @@
 #
 
 list(APPEND FLUTTER_PLUGIN_LIST
+  file_selector_windows
   rive_common
   url_launcher_windows
 )