From cbd040f57b6a3cf8f47186e1c2e554c29acb382f Mon Sep 17 00:00:00 2001 From: Joshua Date: Wed, 10 Jul 2024 19:15:03 -0600 Subject: [PATCH] added multimedia support and STT. STT and photos are have bugs but are functional. file upload is conceptualized but will require additional backend work. video is on hold for now. --- mobile/app/README.md | 2 + mobile/app/android/app/build.gradle | 4 +- mobile/app/ios/Podfile.lock | 12 + mobile/app/ios/Runner/Info.plist | 10 +- .../conversation_model.dart | 0 .../components/conversation_thread/chat.dart | 27 +- .../chat_bubble_widget.dart | 21 +- .../conversation_thread/conversation.dart | 12 +- .../conversation_thread_model.dart | 2 +- .../conversation_thread_widget.dart | 3 +- .../conversation_widget.dart | 6 +- .../conversations_list_model.dart | 0 .../conversations_list_widget.dart | 13 +- .../image_details_model.dart | 0 .../image_details_widget.dart | 0 mobile/app/lib/components/no_chats_model.dart | 13 - .../app/lib/components/no_chats_widget.dart | 8 - .../prompt_box/prompt_box_widget.dart | 141 +++++--- .../{pages => components}/settings_model.dart | 0 .../settings_widget.dart | 0 .../{pages => components}/setup_model.dart | 0 .../{pages => components}/setup_widget.dart | 0 .../support_form_model.dart | 0 .../support_form_widget.dart | 4 +- .../lib/{ => components}/util/backend.dart | 84 +++-- .../app/lib/{ => components}/util/model.dart | 0 .../{ => components}/util/speech_to_text.dart | 18 +- .../app/lib/components/util/usage_queue.dart | 317 ++++++++++++++++++ mobile/app/lib/main.dart | 22 +- .../flutter/generated_plugin_registrant.cc | 4 + .../app/linux/flutter/generated_plugins.cmake | 1 + .../Flutter/GeneratedPluginRegistrant.swift | 2 + mobile/app/pubspec.lock | 146 +++++++- mobile/app/pubspec.yaml | 3 + .../flutter/generated_plugin_registrant.cc | 3 + .../windows/flutter/generated_plugins.cmake | 1 + 36 files changed, 742 insertions(+), 137 deletions(-) rename mobile/app/lib/{pages => components}/conversation_model.dart (100%) rename mobile/app/lib/{pages => components}/conversation_widget.dart (96%) rename mobile/app/lib/{pages => components}/conversations_list_model.dart (100%) rename mobile/app/lib/{pages => components}/conversations_list_widget.dart (97%) rename mobile/app/lib/{pages => components}/image_details_model.dart (100%) rename mobile/app/lib/{pages => components}/image_details_widget.dart (100%) delete mode 100644 mobile/app/lib/components/no_chats_model.dart rename mobile/app/lib/{pages => components}/settings_model.dart (100%) rename mobile/app/lib/{pages => components}/settings_widget.dart (100%) rename mobile/app/lib/{pages => components}/setup_model.dart (100%) rename mobile/app/lib/{pages => components}/setup_widget.dart (100%) rename mobile/app/lib/{pages => components}/support_form_model.dart (100%) rename mobile/app/lib/{pages => components}/support_form_widget.dart (99%) rename mobile/app/lib/{ => components}/util/backend.dart (72%) rename mobile/app/lib/{ => components}/util/model.dart (100%) rename mobile/app/lib/{ => components}/util/speech_to_text.dart (68%) create mode 100644 mobile/app/lib/components/util/usage_queue.dart diff --git a/mobile/app/README.md b/mobile/app/README.md index b73cfb0..d67227f 100644 --- a/mobile/app/README.md +++ b/mobile/app/README.md @@ -14,3 +14,5 @@ A few resources to get you started if this is your first Flutter project: For help getting started with Flutter development, view the [online documentation](https://docs.flutter.dev/), which offers tutorials, samples, guidance on mobile development, and a full API reference. + +# TODO \ No newline at end of file diff --git a/mobile/app/android/app/build.gradle b/mobile/app/android/app/build.gradle index d277963..1ce74cc 100644 --- a/mobile/app/android/app/build.gradle +++ b/mobile/app/android/app/build.gradle @@ -24,7 +24,7 @@ if (flutterVersionName == null) { } android { - namespace = "com.example.app" + namespace = "quaxlyqueen.dev.oneai" compileSdk = flutter.compileSdkVersion ndkVersion = flutter.ndkVersion @@ -35,7 +35,7 @@ android { defaultConfig { // TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html). - applicationId = "com.example.app" + applicationId = "quaxlyqueen.dev.oneai" // You can update the following values to match your application needs. // For more information, see: https://docs.flutter.dev/deployment/android#reviewing-the-gradle-build-configuration. minSdk = flutter.minSdkVersion diff --git a/mobile/app/ios/Podfile.lock b/mobile/app/ios/Podfile.lock index a494a6c..c9f0587 100644 --- a/mobile/app/ios/Podfile.lock +++ b/mobile/app/ios/Podfile.lock @@ -5,6 +5,8 @@ PODS: - HaishinKit (= 1.3.0) - ApiVideoLiveStream (1.2.2): - HaishinKit (= 1.3.0) + - camera_avfoundation (0.0.1): + - Flutter - DKImagePickerController/Core (4.3.9): - DKImagePickerController/ImageDataManager - DKImagePickerController/Resource @@ -44,6 +46,8 @@ PODS: - Flutter - HaishinKit (1.3.0): - Logboard (~> 2.3.0) + - image_picker_ios (0.0.1): + - Flutter - Logboard (2.3.1) - native_device_orientation (0.0.1): - Flutter @@ -78,9 +82,11 @@ PODS: DEPENDENCIES: - apivideo_live_stream (from `.symlinks/plugins/apivideo_live_stream/ios`) + - camera_avfoundation (from `.symlinks/plugins/camera_avfoundation/ios`) - file_picker (from `.symlinks/plugins/file_picker/ios`) - Flutter (from `Flutter`) - flutter_credit_card (from `.symlinks/plugins/flutter_credit_card/ios`) + - image_picker_ios (from `.symlinks/plugins/image_picker_ios/ios`) - native_device_orientation (from `.symlinks/plugins/native_device_orientation/ios`) - path_provider_foundation (from `.symlinks/plugins/path_provider_foundation/darwin`) - pointer_interceptor_ios (from `.symlinks/plugins/pointer_interceptor_ios/ios`) @@ -106,12 +112,16 @@ SPEC REPOS: EXTERNAL SOURCES: apivideo_live_stream: :path: ".symlinks/plugins/apivideo_live_stream/ios" + camera_avfoundation: + :path: ".symlinks/plugins/camera_avfoundation/ios" file_picker: :path: ".symlinks/plugins/file_picker/ios" Flutter: :path: Flutter flutter_credit_card: :path: ".symlinks/plugins/flutter_credit_card/ios" + image_picker_ios: + :path: ".symlinks/plugins/image_picker_ios/ios" native_device_orientation: :path: ".symlinks/plugins/native_device_orientation/ios" path_provider_foundation: @@ -136,12 +146,14 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: apivideo_live_stream: 64fa696699cfc531e07f5d5d701c7e3cd5ce5e9d ApiVideoLiveStream: 5842571806a2ed26671597dadd370ec38d9b9ba4 + camera_avfoundation: dd002b0330f4981e1bbcb46ae9b62829237459a4 DKImagePickerController: 946cec48c7873164274ecc4624d19e3da4c1ef3c DKPhotoGallery: b3834fecb755ee09a593d7c9e389d8b5d6deed60 file_picker: 09aa5ec1ab24135ccd7a1621c46c84134bfd6655 Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7 flutter_credit_card: 48fb35065294868bc6a3e7f18a42f28929b84f48 HaishinKit: 013229683accc4abe46630b04629da3ff90428b7 + image_picker_ios: c560581cceedb403a6ff17f2f816d7fea1421fc1 Logboard: 3d98bb85de6a36b255ab637e8178eb5671c5c3a6 native_device_orientation: 3b4cfc9565a7b879cc4fde282b3e27745e852d0d path_provider_foundation: 2b6b4c569c0fb62ec74538f866245ac84301af46 diff --git a/mobile/app/ios/Runner/Info.plist b/mobile/app/ios/Runner/Info.plist index 624ef01..835f899 100644 --- a/mobile/app/ios/Runner/Info.plist +++ b/mobile/app/ios/Runner/Info.plist @@ -2,8 +2,14 @@ - NSMicrophoneUsageDescription - Required for on-device speech recognition. + NSCameraUsageDescription + This app needs access to camera when used + NSPhotoLibraryUsageDescription + TODO: Better description + LSApplicationCategoryType + Productivity + NSMicrophoneUsageDescription + Required for on-device speech recognition. NSSpeechRecognitionUsageDescription On-device speech-to-text. CADisableMinimumFrameDurationOnPhone diff --git a/mobile/app/lib/pages/conversation_model.dart b/mobile/app/lib/components/conversation_model.dart similarity index 100% rename from mobile/app/lib/pages/conversation_model.dart rename to mobile/app/lib/components/conversation_model.dart diff --git a/mobile/app/lib/components/conversation_thread/chat.dart b/mobile/app/lib/components/conversation_thread/chat.dart index ade2c2b..cde1f9c 100644 --- a/mobile/app/lib/components/conversation_thread/chat.dart +++ b/mobile/app/lib/components/conversation_thread/chat.dart @@ -1,9 +1,32 @@ class Chat { final String content; final bool role; + int chatType = 0; // Most basic chat, text from server or from user. - Chat(this.content, this.role); + Chat(this.content, this.role, this.chatType); - // TODO: Overload constructors to allow for multimedia. + // A chat with an included image. + Chat.image(this.content, this.role, this.chatType); + + // A chat with an included file. + Chat.file(this.content, this.role, this.chatType); +} + +// TODO: Add proper filetype support. File should either be parsed on-device, or uploaded to the server. +// TODO: Implement dynamic pathing based upon filetypes. +enum Filetype { + pdf, + doc, + docx, + pages, + note, + todo, + cal, + calendar, + event, + contact, + mp3, + mp4, + wav, } \ No newline at end of file diff --git a/mobile/app/lib/components/conversation_thread/chat_bubble_widget.dart b/mobile/app/lib/components/conversation_thread/chat_bubble_widget.dart index 2233eb2..4517a55 100644 --- a/mobile/app/lib/components/conversation_thread/chat_bubble_widget.dart +++ b/mobile/app/lib/components/conversation_thread/chat_bubble_widget.dart @@ -1,3 +1,6 @@ +import 'dart:convert'; + +import 'package:app/components/util/backend.dart'; import 'package:app/theme.dart'; import 'package:flutter/material.dart'; import 'chat.dart'; @@ -29,10 +32,20 @@ class ChatBubbleWidget extends StatelessWidget{ ), color: c.role ? AppTheme.darkest : AppTheme.dark, ), - child: Text( - c.content, - style: AppTheme.bodyMedium, - ), + child: + // TODO: Properly display filetypes in the thread. + c.chatType == 0 ? Text( + c.content, + style: AppTheme.bodyMedium, + ) + : c.chatType == 1 ? Image(image: Backend.convertFromBase64(c.content)) + : c.chatType == 2 ? const Icon( + Icons.file_copy, + ) : const Text("null"), + // Text( + // c.content, + // style: AppTheme.bodyMedium, + // ), ), ], ); diff --git a/mobile/app/lib/components/conversation_thread/conversation.dart b/mobile/app/lib/components/conversation_thread/conversation.dart index 7d79cde..9a8c047 100644 --- a/mobile/app/lib/components/conversation_thread/conversation.dart +++ b/mobile/app/lib/components/conversation_thread/conversation.dart @@ -1,4 +1,4 @@ -import 'package:app/util/backend.dart'; +import 'package:app/components/util/backend.dart'; import 'package:flutter/material.dart'; import 'chat.dart'; @@ -6,10 +6,12 @@ class Conversation with ChangeNotifier { int conversationID; String conversationLabel; String conversationSubLabel; - bool history = true; - // History is disabled by default. This means chats are auto-deleted once the user exits that conversation. + // TODO: Fix history, very, verry strange behavior when disabled. + bool history = true; // History is disabled by default. This means chats are auto-deleted once the user exits that conversation.\ bool conversationContext = true; // The prior prompts and responses are used as additional context for the next prompt and response. late bool stream = false; // Update the UI with a stream of characters as received from the server, or as one text. + var lastAccessTime = DateTime.timestamp(); // Used to sort conversations based on most recent usage. + List chats = []; Conversation(this.conversationID, this.conversationLabel, this.conversationSubLabel); @@ -23,11 +25,13 @@ class Conversation with ChangeNotifier { void toggleStream() => stream = !stream; void add(Chat message) { + lastAccessTime = DateTime.timestamp(); chats.add(message); notifyListeners(); } void loadChats() { + lastAccessTime = DateTime.timestamp(); Backend.loadConversation; chats = Backend.loadedChats; } @@ -41,10 +45,12 @@ class Conversation with ChangeNotifier { } List getChats() { + lastAccessTime = DateTime.timestamp(); return chats; } void setLabels(String cLabel, String cSubLabel) { + lastAccessTime = DateTime.timestamp(); conversationLabel = cLabel; conversationSubLabel = cSubLabel; } diff --git a/mobile/app/lib/components/conversation_thread/conversation_thread_model.dart b/mobile/app/lib/components/conversation_thread/conversation_thread_model.dart index c95fa3a..a706d35 100644 --- a/mobile/app/lib/components/conversation_thread/conversation_thread_model.dart +++ b/mobile/app/lib/components/conversation_thread/conversation_thread_model.dart @@ -1,7 +1,7 @@ import 'package:app/components/conversation_thread/chat_bubble_model.dart'; import 'package:app/components/conversation_thread/chat.dart'; import 'package:app/components/conversation_thread/conversation.dart'; -import 'package:app/util/backend.dart'; +import 'package:app/components/util/backend.dart'; import 'package:app/components/prompt_box/prompt_box_widget.dart'; import 'package:flutterflow_ui/flutterflow_ui.dart'; diff --git a/mobile/app/lib/components/conversation_thread/conversation_thread_widget.dart b/mobile/app/lib/components/conversation_thread/conversation_thread_widget.dart index a5cf391..27b3ddb 100644 --- a/mobile/app/lib/components/conversation_thread/conversation_thread_widget.dart +++ b/mobile/app/lib/components/conversation_thread/conversation_thread_widget.dart @@ -1,5 +1,5 @@ import 'package:app/components/conversation_thread/conversation.dart'; -import 'package:app/util/backend.dart'; +import 'package:app/components/util/backend.dart'; import 'package:provider/provider.dart'; import '/components/conversation_thread/chat_bubble_widget.dart'; @@ -50,6 +50,7 @@ class _ConversationThreadWidgetState extends State { @override Widget build(BuildContext context) { + // TODO: Add a scrollbar return Container( width: double.infinity, height: double.infinity, diff --git a/mobile/app/lib/pages/conversation_widget.dart b/mobile/app/lib/components/conversation_widget.dart similarity index 96% rename from mobile/app/lib/pages/conversation_widget.dart rename to mobile/app/lib/components/conversation_widget.dart index 26d2e4d..f6cf3ec 100644 --- a/mobile/app/lib/pages/conversation_widget.dart +++ b/mobile/app/lib/components/conversation_widget.dart @@ -9,8 +9,8 @@ import 'package:flutter/services.dart'; import 'package:app/theme.dart'; -import 'conversation_model.dart'; -export 'conversation_model.dart'; +import '../components/conversation_model.dart'; +export '../components/conversation_model.dart'; class ConversationWidget extends StatefulWidget { final Conversation conversation; @@ -64,8 +64,8 @@ class _ConversationWidgetState extends State { size: 30, ), onPressed: () async { - print("something pressed in conversation_widget.dart"); // TODO: Smooth out the animation to prevent the conversations list from loading half-way up the screen. + // Note: that only applies if the input was already selected. SystemChannels.textInput.invokeMethod('TextInput.hide'); Navigator.pop(context); }, diff --git a/mobile/app/lib/pages/conversations_list_model.dart b/mobile/app/lib/components/conversations_list_model.dart similarity index 100% rename from mobile/app/lib/pages/conversations_list_model.dart rename to mobile/app/lib/components/conversations_list_model.dart diff --git a/mobile/app/lib/pages/conversations_list_widget.dart b/mobile/app/lib/components/conversations_list_widget.dart similarity index 97% rename from mobile/app/lib/pages/conversations_list_widget.dart rename to mobile/app/lib/components/conversations_list_widget.dart index 0a38f42..6018d93 100644 --- a/mobile/app/lib/pages/conversations_list_widget.dart +++ b/mobile/app/lib/components/conversations_list_widget.dart @@ -1,15 +1,15 @@ import 'package:app/components/conversation_thread/conversation.dart'; import 'package:app/theme.dart'; -import 'package:app/pages/conversation_widget.dart'; -import 'package:app/util/backend.dart'; +import 'package:app/components/conversation_widget.dart'; +import 'package:app/components/util/backend.dart'; import 'package:flutterflow_ui/flutterflow_ui.dart'; import 'package:flutter/material.dart'; import 'package:app/components/options_dialog/options_dialog_widget.dart'; -import 'conversations_list_model.dart'; -export 'conversations_list_model.dart'; +import '../components/conversations_list_model.dart'; +export '../components/conversations_list_model.dart'; class ConversationsListWidget extends StatefulWidget { const ConversationsListWidget({super.key}); @@ -213,9 +213,6 @@ class _ConversationsListWidgetState extends State { }, ).then((value) => safeSetState(() {})), - - // TODO: Need to implement. - print("opening options") }, ), ); @@ -235,7 +232,7 @@ class _ConversationsListWidgetState extends State { Navigator.push( context, MaterialPageRoute( - builder: (context) => ConversationWidget(conversation: Backend.getLoadedConversation(),), + builder: (context) => ConversationWidget(conversation: Backend.loadedConversation,), ), ), // Handle the button press }, diff --git a/mobile/app/lib/pages/image_details_model.dart b/mobile/app/lib/components/image_details_model.dart similarity index 100% rename from mobile/app/lib/pages/image_details_model.dart rename to mobile/app/lib/components/image_details_model.dart diff --git a/mobile/app/lib/pages/image_details_widget.dart b/mobile/app/lib/components/image_details_widget.dart similarity index 100% rename from mobile/app/lib/pages/image_details_widget.dart rename to mobile/app/lib/components/image_details_widget.dart diff --git a/mobile/app/lib/components/no_chats_model.dart b/mobile/app/lib/components/no_chats_model.dart deleted file mode 100644 index d50872d..0000000 --- a/mobile/app/lib/components/no_chats_model.dart +++ /dev/null @@ -1,13 +0,0 @@ -import 'package:flutterflow_ui/flutterflow_ui.dart'; -import 'no_chats_widget.dart' show NoChatsWidget; -import 'package:flutter/material.dart'; -import 'package:google_fonts/google_fonts.dart'; -import 'package:provider/provider.dart'; - -class NoChatsModel extends FlutterFlowModel { - @override - void initState(BuildContext context) {} - - @override - void dispose() {} -} diff --git a/mobile/app/lib/components/no_chats_widget.dart b/mobile/app/lib/components/no_chats_widget.dart index 3fca32d..8116461 100644 --- a/mobile/app/lib/components/no_chats_widget.dart +++ b/mobile/app/lib/components/no_chats_widget.dart @@ -5,9 +5,6 @@ import 'package:provider/provider.dart'; import 'package:app/theme.dart'; -import 'no_chats_model.dart'; -export 'no_chats_model.dart'; - class NoChatsWidget extends StatefulWidget { const NoChatsWidget({ super.key, @@ -26,24 +23,19 @@ class NoChatsWidget extends StatefulWidget { } class _NoChatsWidgetState extends State { - late NoChatsModel _model; @override void setState(VoidCallback callback) { super.setState(callback); - _model.onUpdate(); } @override void initState() { super.initState(); - _model = createModel(context, () => NoChatsModel()); } @override void dispose() { - _model.maybeDispose(); - super.dispose(); } diff --git a/mobile/app/lib/components/prompt_box/prompt_box_widget.dart b/mobile/app/lib/components/prompt_box/prompt_box_widget.dart index 0a13904..6704614 100644 --- a/mobile/app/lib/components/prompt_box/prompt_box_widget.dart +++ b/mobile/app/lib/components/prompt_box/prompt_box_widget.dart @@ -1,10 +1,17 @@ -import 'package:app/util/backend.dart'; +import 'dart:io'; + +import 'package:app/components/util/backend.dart'; +import 'package:camera/camera.dart'; +import 'package:file_picker/file_picker.dart'; +import 'package:flutter/services.dart'; import 'package:flutterflow_ui/flutterflow_ui.dart'; import 'package:easy_debounce/easy_debounce.dart'; import 'package:flutter/material.dart'; import 'package:app/theme.dart'; +import 'package:image_picker/image_picker.dart'; +import '../conversation_thread/chat.dart'; import 'prompt_box_model.dart'; export 'prompt_box_model.dart'; @@ -17,7 +24,7 @@ class PromptBoxWidget extends StatefulWidget { class _PromptBoxWidgetState extends State { late PromptBoxModel _model; - late String entry_text; + String entryText = ""; @override void setState(VoidCallback callback) { @@ -43,6 +50,7 @@ class _PromptBoxWidgetState extends State { @override Widget build(BuildContext context) { + const double buttonPadding = 4; return SafeArea( child: Container( width: double.infinity, @@ -62,50 +70,59 @@ class _PromptBoxWidgetState extends State { children: [ Expanded( // TODO: File context upload child: Padding( - padding: const EdgeInsets.all(14), - child: FlutterFlowIconButton( - borderColor: Colors.transparent, - borderRadius: 30, - borderWidth: 1, - buttonSize: 40, + padding: const EdgeInsets.all(buttonPadding), + child: IconButton( icon: const Icon( Icons.file_upload_outlined, color: AppTheme.darkest, size: 30, ), - onPressed: () { - print('IconButton pressed ...'); + onPressed: () async { + final picker = FilePicker.platform; + final result = await picker.pickFiles(allowMultiple: true); // Allow multiple files + if (result != null) { + for (final platformFile in result.files) { + // TODO: Upload file to server + final file = File(platformFile.path!); + } + } else { + print('User canceled file selection'); + } }, ), ), ), - Expanded( // TODO: Photo context upload + Expanded( // TODO: Gallery context upload child: Padding( - padding: const EdgeInsets.all(14), - child: FlutterFlowIconButton( - borderColor: Colors.transparent, - borderRadius: 30, - borderWidth: 1, - buttonSize: 40, + padding: const EdgeInsets.all(buttonPadding), + child: IconButton( icon: const Icon( Icons.photo_outlined, color: AppTheme.darkest, size: 30, ), - onPressed: () { - print('IconButton pressed ...'); + onPressed: () async { + final picker = ImagePicker(); + final image = await picker.pickImage(source: ImageSource.gallery); + + // TODO: Allow for multiple pictures/photos to be selected. + // final images = await picker.pickMultiImage(); + if (image != null) { + + // TODO: Upload to server and add to conversation.chats + String i = Backend.convertToBase64(File(image.path)).toString(); + Backend.loadedConversation.add(Chat.image(i, true, 1)); + } else { + print('User canceled image selection'); + } }, ), ), ), Expanded( // TODO: Video context upload child: Padding( - padding: const EdgeInsets.all(14), - child: FlutterFlowIconButton( - borderColor: Colors.transparent, - borderRadius: 30, - borderWidth: 1, - buttonSize: 40, + padding: const EdgeInsets.all(buttonPadding), + child: IconButton( icon: const Icon( Icons.videocam_outlined, color: AppTheme.darkest, @@ -119,19 +136,48 @@ class _PromptBoxWidgetState extends State { ), Expanded( // TODO: Camera context upload child: Padding( - padding: const EdgeInsets.all(14), - child: FlutterFlowIconButton( - borderColor: Colors.transparent, - borderRadius: 30, - borderWidth: 1, - buttonSize: 40, + padding: const EdgeInsets.all(buttonPadding), + child: IconButton( icon: const Icon( Icons.camera_alt_outlined, color: AppTheme.darkest, size: 30, ), + onPressed: () async { + final picker = ImagePicker(); + final image = await picker.pickImage(source: ImageSource.camera); + + if (image != null) { + // TODO: Upload to server and add to conversation.chats + final i = Backend.convertToBase64(File(image.path)); + Backend.loadedConversation.add(Chat.image(i.toString(), true, 1)); + } else { + print('User canceled photo capture'); + } + }, + ), + ), + ), + Expanded( + child: Padding( + padding: const EdgeInsets.all(buttonPadding), + child: IconButton( + icon: Icon( + Backend.stt.speechToText.isNotListening ? Icons.mic_off_outlined : Icons.mic_none, + color: AppTheme.darkest, + size: 30, + ), onPressed: () { - print('IconButton pressed ...'); + // TODO: Audio input is still buggy, it appears to only enter the text field once the button is pressed again to stop. + if(Backend.stt.speechToText.isNotListening) { + print('STT engine starting ...'); + Backend.stt.startListening(); + updateText(); + } else { + print('STT engine stopping'); + Backend.stt.stopListening(); + updateText(); + } }, ), ), @@ -190,10 +236,13 @@ class _PromptBoxWidgetState extends State { fillColor: AppTheme.alternate, suffixIcon: GestureDetector( onTap: () { - // TODO: Connect with multimedia - entry_text = _model.textController.text; - Backend.respondWhenReady(entry_text); - _model.textController?.clear(); // Clear the text field + if(_model.textController.text != "") { + // TODO: Connect with multimedia + entryText = _model.textController.text; + Backend.respondWhenReady(entryText); + _model.textController?.clear(); // Clear the text field + SystemChannels.textInput.invokeMethod('TextInput.hide'); + } }, child: const Icon( Icons.send, @@ -206,12 +255,14 @@ class _PromptBoxWidgetState extends State { textAlign: TextAlign.start, maxLines: 5, minLines: 1, - validator: _model.textControllerValidator.asValidator(context), textInputAction: TextInputAction.send, // "Send" on keyboard onFieldSubmitted: (text) { - entry_text = _model.textController.text; - Backend.respondWhenReady(entry_text); // TODO: Connect with multimedia - _model.textController?.clear(); // Clear the text field + // Don't take any action unless there's an actual input. + if(_model.textController.text != "") { + entryText = _model.textController.text; + Backend.respondWhenReady(entryText); // TODO: Connect with multimedia + _model.textController?.clear(); // Clear the text field + } }, ), ), @@ -223,4 +274,12 @@ class _PromptBoxWidgetState extends State { ), ); } -} + + void updateText() { + _model.textController.text = Backend.stt.text; + entryText = _model.textController.text; + print(entryText); + print(_model.textController.text); + Backend.stt.reset(); + } +} \ No newline at end of file diff --git a/mobile/app/lib/pages/settings_model.dart b/mobile/app/lib/components/settings_model.dart similarity index 100% rename from mobile/app/lib/pages/settings_model.dart rename to mobile/app/lib/components/settings_model.dart diff --git a/mobile/app/lib/pages/settings_widget.dart b/mobile/app/lib/components/settings_widget.dart similarity index 100% rename from mobile/app/lib/pages/settings_widget.dart rename to mobile/app/lib/components/settings_widget.dart diff --git a/mobile/app/lib/pages/setup_model.dart b/mobile/app/lib/components/setup_model.dart similarity index 100% rename from mobile/app/lib/pages/setup_model.dart rename to mobile/app/lib/components/setup_model.dart diff --git a/mobile/app/lib/pages/setup_widget.dart b/mobile/app/lib/components/setup_widget.dart similarity index 100% rename from mobile/app/lib/pages/setup_widget.dart rename to mobile/app/lib/components/setup_widget.dart diff --git a/mobile/app/lib/pages/support_form_model.dart b/mobile/app/lib/components/support_form_model.dart similarity index 100% rename from mobile/app/lib/pages/support_form_model.dart rename to mobile/app/lib/components/support_form_model.dart diff --git a/mobile/app/lib/pages/support_form_widget.dart b/mobile/app/lib/components/support_form_widget.dart similarity index 99% rename from mobile/app/lib/pages/support_form_widget.dart rename to mobile/app/lib/components/support_form_widget.dart index 5037056..cfe090a 100644 --- a/mobile/app/lib/pages/support_form_widget.dart +++ b/mobile/app/lib/components/support_form_widget.dart @@ -4,8 +4,8 @@ import 'package:flutter_animate/flutter_animate.dart'; import 'package:google_fonts/google_fonts.dart'; import 'package:provider/provider.dart'; -import 'support_form_model.dart'; -export 'support_form_model.dart'; +import '../components/support_form_model.dart'; +export '../components/support_form_model.dart'; import 'package:app/theme.dart'; diff --git a/mobile/app/lib/util/backend.dart b/mobile/app/lib/components/util/backend.dart similarity index 72% rename from mobile/app/lib/util/backend.dart rename to mobile/app/lib/components/util/backend.dart index ccd2f7b..638bc6a 100644 --- a/mobile/app/lib/util/backend.dart +++ b/mobile/app/lib/components/util/backend.dart @@ -1,10 +1,14 @@ import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; +import 'package:flutter/widgets.dart'; import 'package:app/components/conversation_thread/conversation.dart'; +import 'package:camera/camera.dart'; import 'package:http/http.dart' as http; import 'package:app/components/conversation_thread/chat.dart'; -import 'package:app/util/speech_to_text.dart'; +import 'package:app/components/util/speech_to_text.dart'; import 'model.dart'; @@ -18,6 +22,7 @@ class Backend { //static final url = "http://192.168.1.68:11434/api"; // For local access to home server static String endpoint = ""; static final Map headers = {'Content-Type': 'application/json'}; + static final bool UI_TESTING = true; @@ -27,11 +32,13 @@ class Backend { - // PRIVACY-FOCUSED VARIABLES + // PRIVACY-NECESSARY VARIABLES // TODO: Reorder based on most recent usage. static List conversations = []; static List loadedChats = []; static late Conversation loadedConversation; + static late List cameras; + static late CameraController controller; // RUNTIME & STATUS VARIABLES @@ -45,40 +52,35 @@ class Backend { // TODO: Verify all different components of STT, TTS, LLM, etc have been initialized. // TODO: Handshake w/ server to validate identity factory Backend() { - // Speech-To-Text engine activation. - _speechEnabled = stt.isSpeechEnabled(); - // loadedChats = conversations[conversationID].chats; return singleBackend; } - void init() { - initiated = true; + void init() async { + // Prevent multiple initiations + if(!initiated) { + initiated = true; + // Speech-To-Text engine activation. + _speechEnabled = stt.isSpeechEnabled(); + + model = Model("llama3"); // Primarily used llama3, testing gemma2 + maxConversationID = conversations.length; + + cameras = await availableCameras(); + controller = CameraController(cameras[0], ResolutionPreset.max); + controller.initialize(); // TODO: Add error catching on camera + } // TODO: Dynamically get available models and models that can be pulled. // TODO: Dynamically get the user's default model - model = Model("llama3"); // Primarily used llama3, testing gemma2 - // TODO: Dynamically obtain conversations. - // conversations.add( - // Conversation.completeConversation( - // 0, - // "Test Conversation", - // "Testing Functionality", - // [ - // (Chat("Hello there! You are part of a prototype and are running locally on my Macbook Air. Please keep your responses brief as every prompt freezes my computer.", true)), - // (Chat("Understood! How can I help?", false)), - // ] - // ) - // ); - - maxConversationID = conversations.length; + // TODO: Obtain all user permissions at once. } - // PRIMARY INTERFACE + // PRIMARY INTERFACES @Deprecated("Use STT interface directly and use respondWhenReady().") static Future sttGetResponseWhenReady() async { // TODO: In case on-device STT is unavailable, use server-side STT service. @@ -87,19 +89,38 @@ class Backend { } prompt = stt.getTextWhenReady() as String; - conversations[conversationID].add(Chat(prompt, true)); - httpSendRequest(); - conversations[conversationID].add(Chat(response, false)); + conversations[conversationID].add(Chat(prompt, true, 0)); + if(!UI_TESTING) { + httpSendRequest(); + } + conversations[conversationID].add(Chat(response, false, 0)); return response; } static void respondWhenReady(String p) async { prompt = p; - conversations[conversationID].add(Chat(prompt, true)); - httpSendRequest(); + conversations[conversationID].add(Chat(prompt, true, 0)); + if(!UI_TESTING) { + httpSendRequest(); + } } + static ImageProvider convertFromBase64(img) { + Uint8List imageBytes = base64Decode(img); + + return Image.memory(imageBytes).image; + } + + // Converts image to base64 encoded string for data transfer. + static String convertToBase64(f) { + List imageBytes = f.readAsBytesSync(); + String img = base64Encode(imageBytes); + + // TODO: Add special handling for images included in prompts. + conversations[conversationID].add(Chat.image(img, true, 1)); + return img; + } // CONVERSATION MANAGEMENT @@ -155,7 +176,7 @@ class Backend { Map data; if(conversations[conversationID].conversationContext) { - loadedChats = [...loadedChats, Chat(prompt, true)]; + loadedChats = [...loadedChats, Chat(prompt, true, 0)]; endpoint = "chat"; List> messageList = convertMessages(); @@ -196,12 +217,13 @@ class Backend { Map re = jsonDecode(serverResponse.body); if(conversations[conversationID].conversationContext) { response = re["message"]["content"]; - loadedChats = [...loadedChats, Chat(response, false)]; + // TODO: Modify in case of different response types. + loadedChats = [...loadedChats, Chat(response, false, 0)]; } else { response = re["response"]; } } - conversations[conversationID].add(Chat(response, false)); + conversations[conversationID].add(Chat(response, false, 0)); prompt = ""; response = ""; diff --git a/mobile/app/lib/util/model.dart b/mobile/app/lib/components/util/model.dart similarity index 100% rename from mobile/app/lib/util/model.dart rename to mobile/app/lib/components/util/model.dart diff --git a/mobile/app/lib/util/speech_to_text.dart b/mobile/app/lib/components/util/speech_to_text.dart similarity index 68% rename from mobile/app/lib/util/speech_to_text.dart rename to mobile/app/lib/components/util/speech_to_text.dart index 4ec5e2f..0b7e634 100644 --- a/mobile/app/lib/util/speech_to_text.dart +++ b/mobile/app/lib/components/util/speech_to_text.dart @@ -1,9 +1,16 @@ import 'package:speech_to_text/speech_recognition_result.dart'; import 'package:speech_to_text/speech_to_text.dart' as stt; +import 'package:speech_to_text/speech_to_text.dart'; /// Handles standard Speech-To-Text using on-device processing. class SpeechToText { final stt.SpeechToText speechToText = stt.SpeechToText(); + // TODO: User customizable STT?? + final SpeechListenOptions options = stt.SpeechListenOptions( + partialResults: true, + onDevice: true, + ); + bool speechEnabled = false; String text = ""; @@ -16,13 +23,22 @@ class SpeechToText { } // Retrieve the text. + @Deprecated("TODO: Deprecated message for speechToText.getText") String getText() => text; // Reset either after saving the text or prior to using STT again. void reset() => text = ""; /// Start a speech recognition session - void startListening() async => await speechToText.listen(onResult: onSpeechResult); + void startListening() async { + await speechToText.listen( + onResult: onSpeechResult, + pauseFor: const Duration(seconds: 3), + + // TODO: I think this is a bug with the speech_to_text library? The docs correctly identify the named parameter of SpeechListenOptions, but it's not recognized. + // SpeechListenOptions: options, + ); + } /// Manually stop the active speech recognition session /// Note that there are also timeouts that each platform enforces diff --git a/mobile/app/lib/components/util/usage_queue.dart b/mobile/app/lib/components/util/usage_queue.dart new file mode 100644 index 0000000..267c7fc --- /dev/null +++ b/mobile/app/lib/components/util/usage_queue.dart @@ -0,0 +1,317 @@ +import 'dart:math'; + +class UsageQueue implements List { + @override + var first; + + @override + var last; + + @override + late int length; + + @override + List operator +(List other) { + // TODO: implement + + throw UnimplementedError(); + } + + @override + operator [](int index) { + // TODO: implement [] + throw UnimplementedError(); + } + + @override + void operator []=(int index, value) { + // TODO: implement []= + } + + @override + void add(value) { + // TODO: implement add + } + + @override + void addAll(Iterable iterable) { + // TODO: implement addAll + } + + @override + bool any(bool Function(dynamic element) test) { + // TODO: implement any + throw UnimplementedError(); + } + + @override + Map asMap() { + // TODO: implement asMap + throw UnimplementedError(); + } + + @override + List cast() { + // TODO: implement cast + throw UnimplementedError(); + } + + @override + void clear() { + // TODO: implement clear + } + + @override + bool contains(Object? element) { + // TODO: implement contains + throw UnimplementedError(); + } + + @override + elementAt(int index) { + // TODO: implement elementAt + throw UnimplementedError(); + } + + @override + bool every(bool Function(dynamic element) test) { + // TODO: implement every + throw UnimplementedError(); + } + + @override + Iterable expand(Iterable Function(dynamic element) toElements) { + // TODO: implement expand + throw UnimplementedError(); + } + + @override + void fillRange(int start, int end, [fillValue]) { + // TODO: implement fillRange + } + + @override + firstWhere(bool Function(dynamic element) test, {Function()? orElse}) { + // TODO: implement firstWhere + throw UnimplementedError(); + } + + @override + T fold(T initialValue, T Function(T previousValue, dynamic element) combine) { + // TODO: implement fold + throw UnimplementedError(); + } + + @override + Iterable followedBy(Iterable other) { + // TODO: implement followedBy + throw UnimplementedError(); + } + + @override + void forEach(void Function(dynamic element) action) { + // TODO: implement forEach + } + + @override + Iterable getRange(int start, int end) { + // TODO: implement getRange + throw UnimplementedError(); + } + + @override + int indexOf(element, [int start = 0]) { + // TODO: implement indexOf + throw UnimplementedError(); + } + + @override + int indexWhere(bool Function(dynamic element) test, [int start = 0]) { + // TODO: implement indexWhere + throw UnimplementedError(); + } + + @override + void insert(int index, element) { + // TODO: implement insert + } + + @override + void insertAll(int index, Iterable iterable) { + // TODO: implement insertAll + } + + @override + // TODO: implement isEmpty + bool get isEmpty => throw UnimplementedError(); + + @override + // TODO: implement isNotEmpty + bool get isNotEmpty => throw UnimplementedError(); + + @override + // TODO: implement iterator + Iterator get iterator => throw UnimplementedError(); + + @override + String join([String separator = ""]) { + // TODO: implement join + throw UnimplementedError(); + } + + @override + int lastIndexOf(element, [int? start]) { + // TODO: implement lastIndexOf + throw UnimplementedError(); + } + + @override + int lastIndexWhere(bool Function(dynamic element) test, [int? start]) { + // TODO: implement lastIndexWhere + throw UnimplementedError(); + } + + @override + lastWhere(bool Function(dynamic element) test, {Function()? orElse}) { + // TODO: implement lastWhere + throw UnimplementedError(); + } + + @override + Iterable map(T Function(dynamic e) toElement) { + // TODO: implement map + throw UnimplementedError(); + } + + @override + reduce(Function(dynamic value, dynamic element) combine) { + // TODO: implement reduce + throw UnimplementedError(); + } + + @override + bool remove(Object? value) { + // TODO: implement remove + throw UnimplementedError(); + } + + @override + removeAt(int index) { + // TODO: implement removeAt + throw UnimplementedError(); + } + + @override + removeLast() { + // TODO: implement removeLast + throw UnimplementedError(); + } + + @override + void removeRange(int start, int end) { + // TODO: implement removeRange + } + + @override + void removeWhere(bool Function(dynamic element) test) { + // TODO: implement removeWhere + } + + @override + void replaceRange(int start, int end, Iterable replacements) { + // TODO: implement replaceRange + } + + @override + void retainWhere(bool Function(dynamic element) test) { + // TODO: implement retainWhere + } + + @override + // TODO: implement reversed + Iterable get reversed => throw UnimplementedError(); + + @override + void setAll(int index, Iterable iterable) { + // TODO: implement setAll + } + + @override + void setRange(int start, int end, Iterable iterable, [int skipCount = 0]) { + // TODO: implement setRange + } + + @override + void shuffle([Random? random]) { + // TODO: implement shuffle + } + + @override + // TODO: implement single + get single => throw UnimplementedError(); + + @override + singleWhere(bool Function(dynamic element) test, {Function()? orElse}) { + // TODO: implement singleWhere + throw UnimplementedError(); + } + + @override + Iterable skip(int count) { + // TODO: implement skip + throw UnimplementedError(); + } + + @override + Iterable skipWhile(bool Function(dynamic value) test) { + // TODO: implement skipWhile + throw UnimplementedError(); + } + + @override + void sort([int Function(dynamic a, dynamic b)? compare]) { + // TODO: implement sort + } + + @override + List sublist(int start, [int? end]) { + // TODO: implement sublist + throw UnimplementedError(); + } + + @override + Iterable take(int count) { + // TODO: implement take + throw UnimplementedError(); + } + + @override + Iterable takeWhile(bool Function(dynamic value) test) { + // TODO: implement takeWhile + throw UnimplementedError(); + } + + @override + List toList({bool growable = true}) { + // TODO: implement toList + throw UnimplementedError(); + } + + @override + Set toSet() { + // TODO: implement toSet + throw UnimplementedError(); + } + + @override + Iterable where(bool Function(dynamic element) test) { + // TODO: implement where + throw UnimplementedError(); + } + + @override + Iterable whereType() { + // TODO: implement whereType + throw UnimplementedError(); + } + +} \ No newline at end of file diff --git a/mobile/app/lib/main.dart b/mobile/app/lib/main.dart index 697b98a..c1ac660 100644 --- a/mobile/app/lib/main.dart +++ b/mobile/app/lib/main.dart @@ -1,6 +1,6 @@ -import 'package:app/pages/conversation_widget.dart'; -import 'package:app/pages/settings_widget.dart'; -import 'package:app/pages/conversations_list_widget.dart'; +import 'package:app/components/conversation_widget.dart'; +import 'package:app/components/settings_widget.dart'; +import 'package:app/components/conversations_list_widget.dart'; import 'package:flutter/material.dart'; import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:provider/provider.dart'; @@ -36,6 +36,7 @@ BottomNavigationBarItem _buildNavigationItem(int index) { icon: Icon(iconData), label: label, backgroundColor: AppTheme.lightest, + ); } @@ -67,21 +68,14 @@ class MyApp extends ConsumerWidget { title: 'One AI', home: Scaffold( backgroundColor: AppTheme.medium, - appBar: AppBar( - backgroundColor: AppTheme.secondaryBackground, - automaticallyImplyLeading: false, - title: const Text( - 'One AI', - style: AppTheme.displayLarge, - ), - actions: const [], - centerTitle: false, - elevation: 0, - ), body: _pages[selectedIndex], bottomNavigationBar: BottomNavigationBar( backgroundColor: AppTheme.darkest, selectedItemColor: AppTheme.lightest, + showUnselectedLabels: false, + showSelectedLabels: false, + iconSize: 30, + // Set the number of items to the length of the pages list items: List.generate(_pages.length, (index) => _buildNavigationItem(index)), currentIndex: selectedIndex, diff --git a/mobile/app/linux/flutter/generated_plugin_registrant.cc b/mobile/app/linux/flutter/generated_plugin_registrant.cc index f6f23bf..7299b5c 100644 --- a/mobile/app/linux/flutter/generated_plugin_registrant.cc +++ b/mobile/app/linux/flutter/generated_plugin_registrant.cc @@ -6,9 +6,13 @@ #include "generated_plugin_registrant.h" +#include #include void fl_register_plugins(FlPluginRegistry* registry) { + g_autoptr(FlPluginRegistrar) file_selector_linux_registrar = + fl_plugin_registry_get_registrar_for_plugin(registry, "FileSelectorPlugin"); + file_selector_plugin_register_with_registrar(file_selector_linux_registrar); g_autoptr(FlPluginRegistrar) url_launcher_linux_registrar = fl_plugin_registry_get_registrar_for_plugin(registry, "UrlLauncherPlugin"); url_launcher_plugin_register_with_registrar(url_launcher_linux_registrar); diff --git a/mobile/app/linux/flutter/generated_plugins.cmake b/mobile/app/linux/flutter/generated_plugins.cmake index f16b4c3..786ff5c 100644 --- a/mobile/app/linux/flutter/generated_plugins.cmake +++ b/mobile/app/linux/flutter/generated_plugins.cmake @@ -3,6 +3,7 @@ # list(APPEND FLUTTER_PLUGIN_LIST + file_selector_linux url_launcher_linux ) diff --git a/mobile/app/macos/Flutter/GeneratedPluginRegistrant.swift b/mobile/app/macos/Flutter/GeneratedPluginRegistrant.swift index 7bc02f3..54bfa47 100644 --- a/mobile/app/macos/Flutter/GeneratedPluginRegistrant.swift +++ b/mobile/app/macos/Flutter/GeneratedPluginRegistrant.swift @@ -5,6 +5,7 @@ import FlutterMacOS import Foundation +import file_selector_macos import path_provider_foundation import rive_common import shared_preferences_foundation @@ -14,6 +15,7 @@ import url_launcher_macos import video_player_avfoundation func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) { + FileSelectorPlugin.register(with: registry.registrar(forPlugin: "FileSelectorPlugin")) PathProviderPlugin.register(with: registry.registrar(forPlugin: "PathProviderPlugin")) RivePlugin.register(with: registry.registrar(forPlugin: "RivePlugin")) SharedPreferencesPlugin.register(with: registry.registrar(forPlugin: "SharedPreferencesPlugin")) diff --git a/mobile/app/pubspec.lock b/mobile/app/pubspec.lock index 7721e71..b09a339 100644 --- a/mobile/app/pubspec.lock +++ b/mobile/app/pubspec.lock @@ -89,6 +89,46 @@ packages: url: "https://pub.dev" source: hosted version: "1.2.0" + camera: + dependency: "direct main" + description: + name: camera + sha256: "2170a943dcb67be2af2c6bcda8775e74b41d4c02d6a4eb10bdc832ee185c4eea" + url: "https://pub.dev" + source: hosted + version: "0.11.0+1" + camera_android_camerax: + dependency: transitive + description: + name: camera_android_camerax + sha256: "7c03940cb8c92eb5b184952674a07cc4a73c6ba2b3568aad70255ad4cb913660" + url: "https://pub.dev" + source: hosted + version: "0.6.7+1" + camera_avfoundation: + dependency: transitive + description: + name: camera_avfoundation + sha256: "95c2f40b4d06cdb0fd2ad893c762d1f2c931a5e370793ec34c939eb4fcbf96bb" + url: "https://pub.dev" + source: hosted + version: "0.9.17" + camera_platform_interface: + dependency: transitive + description: + name: camera_platform_interface + sha256: b3ede1f171532e0d83111fe0980b46d17f1aa9788a07a2fbed07366bbdbb9061 + url: "https://pub.dev" + source: hosted + version: "2.8.0" + camera_web: + dependency: transitive + description: + name: camera_web + sha256: b9235ec0a2ce949daec546f1f3d86f05c3921ed31c7d9ab6b7c03214d152fc2d + url: "https://pub.dev" + source: hosted + version: "0.3.4" carousel_slider: dependency: transitive description: @@ -234,13 +274,45 @@ packages: source: hosted version: "7.0.0" file_picker: - dependency: transitive + dependency: "direct main" description: name: file_picker sha256: "824f5b9f389bfc4dddac3dea76cd70c51092d9dff0b2ece7ef4f53db8547d258" url: "https://pub.dev" source: hosted version: "8.0.6" + file_selector_linux: + dependency: transitive + description: + name: file_selector_linux + sha256: "045d372bf19b02aeb69cacf8b4009555fb5f6f0b7ad8016e5f46dd1387ddd492" + url: "https://pub.dev" + source: hosted + version: "0.9.2+1" + file_selector_macos: + dependency: transitive + description: + name: file_selector_macos + sha256: f42eacb83b318e183b1ae24eead1373ab1334084404c8c16e0354f9a3e55d385 + url: "https://pub.dev" + source: hosted + version: "0.9.4" + file_selector_platform_interface: + dependency: transitive + description: + name: file_selector_platform_interface + sha256: a3994c26f10378a039faa11de174d7b78eb8f79e4dd0af2a451410c1a5c3f66b + url: "https://pub.dev" + source: hosted + version: "2.6.2" + file_selector_windows: + dependency: transitive + description: + name: file_selector_windows + sha256: d3547240c20cabf205c7c7f01a50ecdbc413755814d6677f3cb366f04abcead0 + url: "https://pub.dev" + source: hosted + version: "0.9.3+1" fixnum: dependency: transitive description: @@ -480,6 +552,70 @@ packages: url: "https://pub.dev" source: hosted version: "4.0.2" + image_picker: + dependency: "direct main" + description: + name: image_picker + sha256: "021834d9c0c3de46bf0fe40341fa07168407f694d9b2bb18d532dc1261867f7a" + url: "https://pub.dev" + source: hosted + version: "1.1.2" + image_picker_android: + dependency: transitive + description: + name: image_picker_android + sha256: ff39a10ab4f48f4ac70776d0494a97bf073cd2570892cd46bc8a5cac162c25db + url: "https://pub.dev" + source: hosted + version: "0.8.12+4" + image_picker_for_web: + dependency: transitive + description: + name: image_picker_for_web + sha256: "5d6eb13048cd47b60dbf1a5495424dea226c5faf3950e20bf8120a58efb5b5f3" + url: "https://pub.dev" + source: hosted + version: "3.0.4" + image_picker_ios: + dependency: transitive + description: + name: image_picker_ios + sha256: "6703696ad49f5c3c8356d576d7ace84d1faf459afb07accbb0fae780753ff447" + url: "https://pub.dev" + source: hosted + version: "0.8.12" + image_picker_linux: + dependency: transitive + description: + name: image_picker_linux + sha256: "4ed1d9bb36f7cd60aa6e6cd479779cc56a4cb4e4de8f49d487b1aaad831300fa" + url: "https://pub.dev" + source: hosted + version: "0.2.1+1" + image_picker_macos: + dependency: transitive + description: + name: image_picker_macos + sha256: "3f5ad1e8112a9a6111c46d0b57a7be2286a9a07fc6e1976fdf5be2bd31d4ff62" + url: "https://pub.dev" + source: hosted + version: "0.2.1+1" + image_picker_platform_interface: + dependency: transitive + description: + name: image_picker_platform_interface + sha256: "9ec26d410ff46f483c5519c29c02ef0e02e13a543f882b152d4bfd2f06802f80" + url: "https://pub.dev" + source: hosted + version: "2.10.0" + image_picker_windows: + dependency: transitive + description: + name: image_picker_windows + sha256: "6ad07afc4eb1bc25f3a01084d28520496c4a3bb0cb13685435838167c9dcedeb" + url: "https://pub.dev" + source: hosted + version: "0.2.1+1" intl: dependency: transitive description: @@ -600,6 +736,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.14.0" + mime: + dependency: transitive + description: + name: mime + sha256: "2e123074287cc9fd6c09de8336dae606d1ddb88d9ac47358826db698c176a1f2" + url: "https://pub.dev" + source: hosted + version: "1.0.5" mime_type: dependency: transitive description: diff --git a/mobile/app/pubspec.yaml b/mobile/app/pubspec.yaml index 1d4867d..85f8a47 100644 --- a/mobile/app/pubspec.yaml +++ b/mobile/app/pubspec.yaml @@ -46,6 +46,9 @@ dependencies: video_player: ^2.9.1 flutter_riverpod: ^2.5.1 speech_to_text: ^6.6.0 + camera: ^0.11.0+1 + image_picker: ^1.1.2 + file_picker: ^8.0.6 dev_dependencies: flutter_test: sdk: flutter diff --git a/mobile/app/windows/flutter/generated_plugin_registrant.cc b/mobile/app/windows/flutter/generated_plugin_registrant.cc index 72ff4b7..6c32c2f 100644 --- a/mobile/app/windows/flutter/generated_plugin_registrant.cc +++ b/mobile/app/windows/flutter/generated_plugin_registrant.cc @@ -6,10 +6,13 @@ #include "generated_plugin_registrant.h" +#include #include #include void RegisterPlugins(flutter::PluginRegistry* registry) { + FileSelectorWindowsRegisterWithRegistrar( + registry->GetRegistrarForPlugin("FileSelectorWindows")); RivePluginRegisterWithRegistrar( registry->GetRegistrarForPlugin("RivePlugin")); UrlLauncherWindowsRegisterWithRegistrar( diff --git a/mobile/app/windows/flutter/generated_plugins.cmake b/mobile/app/windows/flutter/generated_plugins.cmake index e141d51..e9a4302 100644 --- a/mobile/app/windows/flutter/generated_plugins.cmake +++ b/mobile/app/windows/flutter/generated_plugins.cmake @@ -3,6 +3,7 @@ # list(APPEND FLUTTER_PLUGIN_LIST + file_selector_windows rive_common url_launcher_windows )