From 80f20e73a9a8671f938a88b64c2b98fe01b100c5 Mon Sep 17 00:00:00 2001 From: Joshua Date: Sun, 7 Jul 2024 14:15:29 -0600 Subject: [PATCH] integrated code from one-ai-mobile repo. working on GUI integration now. --- .../conversation_bubbles_model.dart | 3 + .../conversation_bubbles_widget.dart | 3 + .../conversation_thread_model.dart | 5 +- .../conversation_thread_widget.dart | 11 +- mobile/app/lib/pages/conversation_widget.dart | 4 +- mobile/app/lib/util/server.dart | 129 ++++++++++++++++++ mobile/app/lib/util/speech_to_text.dart | 43 ++++++ .../Flutter/GeneratedPluginRegistrant.swift | 2 + mobile/app/pubspec.lock | 32 +++++ mobile/app/pubspec.yaml | 1 + 10 files changed, 221 insertions(+), 12 deletions(-) create mode 100644 mobile/app/lib/util/server.dart create mode 100644 mobile/app/lib/util/speech_to_text.dart diff --git a/mobile/app/lib/components/conversation_bubbles/conversation_bubbles_model.dart b/mobile/app/lib/components/conversation_bubbles/conversation_bubbles_model.dart index e69de29..6c42b17 100644 --- a/mobile/app/lib/components/conversation_bubbles/conversation_bubbles_model.dart +++ b/mobile/app/lib/components/conversation_bubbles/conversation_bubbles_model.dart @@ -0,0 +1,3 @@ +class ConversationBubblesModel { + +} \ No newline at end of file diff --git a/mobile/app/lib/components/conversation_bubbles/conversation_bubbles_widget.dart b/mobile/app/lib/components/conversation_bubbles/conversation_bubbles_widget.dart index e69de29..11bd06d 100644 --- a/mobile/app/lib/components/conversation_bubbles/conversation_bubbles_widget.dart +++ b/mobile/app/lib/components/conversation_bubbles/conversation_bubbles_widget.dart @@ -0,0 +1,3 @@ +class ConversationBubblesWidget { + +} \ No newline at end of file diff --git a/mobile/app/lib/components/conversation_thread/conversation_thread_model.dart b/mobile/app/lib/components/conversation_thread/conversation_thread_model.dart index c574eb4..123bd4f 100644 --- a/mobile/app/lib/components/conversation_thread/conversation_thread_model.dart +++ b/mobile/app/lib/components/conversation_thread/conversation_thread_model.dart @@ -1,4 +1,4 @@ -import '/components/conversation_bubbles/conversation_bubbles_widget.dart'; +import '../conversation_bubbles/conversation_bubbles_model.dart'; import '/components/prompt_box/prompt_box_widget.dart'; import 'package:flutterflow_ui/flutterflow_ui.dart'; import 'conversation_thread_widget.dart' show ConversationThreadWidget; @@ -13,7 +13,8 @@ class ConversationThreadModel final formKey = GlobalKey(); // Model for ConversationBubbles component. - //late ConversationBubblesModel conversationBubblesModel; + late ConversationBubblesModel conversationBubblesModel; + // Model for PromptBox component. late PromptBoxModel promptBoxModel; diff --git a/mobile/app/lib/components/conversation_thread/conversation_thread_widget.dart b/mobile/app/lib/components/conversation_thread/conversation_thread_widget.dart index 869317f..aca9e13 100644 --- a/mobile/app/lib/components/conversation_thread/conversation_thread_widget.dart +++ b/mobile/app/lib/components/conversation_thread/conversation_thread_widget.dart @@ -55,12 +55,7 @@ class _ConversationThreadWidgetState extends State { children: [ Expanded( child: ListView( - padding: const EdgeInsets.fromLTRB( - 0, - 12, - 0, - 24, - ), + padding: const EdgeInsets.fromLTRB(0, 12, 0, 24,), reverse: true, scrollDirection: Axis.vertical, children: [ @@ -142,14 +137,14 @@ class _ConversationThreadWidgetState extends State { // ), Align( alignment: const AlignmentDirectional(-1, -1), + // TODO: Make the delete button conditional upon media upload. child: FlutterFlowIconButton( borderColor: AppTheme.error, borderRadius: 20, borderWidth: 2, buttonSize: 40, - fillColor: AppTheme - .primaryBackground, + fillColor: AppTheme.primaryBackground, icon: const Icon( Icons.delete_outline_rounded, color: AppTheme.error, diff --git a/mobile/app/lib/pages/conversation_widget.dart b/mobile/app/lib/pages/conversation_widget.dart index dc5990f..a22bbba 100644 --- a/mobile/app/lib/pages/conversation_widget.dart +++ b/mobile/app/lib/pages/conversation_widget.dart @@ -3,8 +3,6 @@ import '/components/conversation_thread/conversation_thread_widget.dart'; import '/components/model_item/model_item_widget.dart'; import 'package:flutterflow_ui/flutterflow_ui.dart'; import 'package:flutter/material.dart'; -import 'package:google_fonts/google_fonts.dart'; -import 'package:provider/provider.dart'; import 'package:app/theme.dart'; @@ -48,6 +46,7 @@ class _ConversationWidgetState extends State { appBar: AppBar( backgroundColor: AppTheme.primaryBackground, automaticallyImplyLeading: false, + // ---------- Back Button leading: FlutterFlowIconButton( borderColor: Colors.transparent, borderRadius: 30, @@ -81,6 +80,7 @@ class _ConversationWidgetState extends State { actions: [ Padding( padding: const EdgeInsetsDirectional.fromSTEB(0, 8, 16, 8), + // ---------- Options Button child: FlutterFlowIconButton( borderColor: AppTheme.alternate, borderRadius: 12, diff --git a/mobile/app/lib/util/server.dart b/mobile/app/lib/util/server.dart new file mode 100644 index 0000000..e1076a5 --- /dev/null +++ b/mobile/app/lib/util/server.dart @@ -0,0 +1,129 @@ +import 'dart:convert'; +import 'package:http/http.dart' as http; + +import 'package:app/util/speech_to_text.dart'; + +class Server { + // TODO: One-time process to establish server connection is required for easy setup/maintenance. + final url = "http://10.0.2.2:11434"; + final headers = {'Content-Type': 'application/json'}; + + final SpeechToText stt = SpeechToText(); + late bool _speechEnabled; + + String prompt = ""; + String response = ""; + + Server() { + // Speech-To-Text engine activation. + _speechEnabled = stt.isSpeechEnabled(); + + // TODO: Initialize all different components of STT, TTS, LLM, etc. + // TODO: Handshake w/ server to validate identity + } + + // Using native on-device Speech-To-Text capability, get the server response when ready. + Future sttGetResponseWhenReady() async { + // TODO: In case on-device STT is unavailable, use server-side STT service. + if (!_speechEnabled) "Unable to process Speech-To-Text on-device."; + + setPrompt(stt.getTextWhenReady() as String); + constructPrompt(); + httpSendRequest(); + + return response; + } + + // Arbitrary prompt, used for standard text interaction rather than STT. + void setPrompt(String p) => prompt = p; + + // Get the last response or request a response from the server. + String getResponse() { + if(response.isEmpty) { + constructPrompt(); + httpSendRequest(); + } + + return response; + } + + // Get the latest prompt. + String getPrompt() => prompt; + + // Formats the prompt in JSON. + String constructPrompt() { + // Construct the JSON payload + final Map data = { + "model": + "llama3", // TODO: Allow for different models to be dynamically selected. + "prompt": prompt, + "stream": false // TODO: Allow for toggleable states between stream + // TODO: Allow for additional flags, ie. continuous conversation, images, etc. + }; + + // Encode the JSON payload + final String body = json.encode(data); + + return body; + } + + // Send the HTTP request to the server. + void httpSendRequest() async { + String json = constructPrompt(); + + // Send the HTTP POST request + final serverResponse = await http.post( + Uri.parse('$url/api/generate'), + headers: headers, + body: json, + ); + + if (serverResponse.statusCode == 200) { + Map re = jsonDecode(serverResponse.body); + response = re["response"]!; + } else { + response = ('Error: ${serverResponse.statusCode}'); + } + } +} +// +// +// // GUI components +// child: Column( +// mainAxisAlignment: MainAxisAlignment.center, +// children: [ +// Container( +// padding: const EdgeInsets.all(16), +// child: const Text( +// 'Recognized words:', +// style: TextStyle(fontSize: 20.0), +// ), +// ), +// Expanded( +// child: Container( +// padding: const EdgeInsets.all(16), +// child: Text( +// // If listening is active show the recognized words +// _speechToText.isListening +// ? prompt +// // If listening isn't active but could be tell the user +// // how to start it, otherwise indicate that speech +// // recognition is not yet ready or not supported on +// // the target device +// : _speechEnabled +// ? responseFromAI +// : 'Speech disabled', +// ), +// ), +// ), +// ], +// ), +// ), +// floatingActionButton: FloatingActionButton( +// onPressed: +// // If not yet listening for speech start, otherwise stop +// //sendPrompt, +// _speechToText.isNotListening ? _startListening : sendPrompt, +// tooltip: 'Listen', +// child: Icon(_speechToText.isNotListening ? Icons.mic_off : Icons.mic), +// ), diff --git a/mobile/app/lib/util/speech_to_text.dart b/mobile/app/lib/util/speech_to_text.dart new file mode 100644 index 0000000..27f9d81 --- /dev/null +++ b/mobile/app/lib/util/speech_to_text.dart @@ -0,0 +1,43 @@ +import 'package:speech_to_text/speech_recognition_result.dart'; +import 'package:speech_to_text/speech_to_text.dart' as stt; + +/// Handles standard Speech-To-Text using on-device processing. +class SpeechToText { + final stt.SpeechToText _speechToText = stt.SpeechToText(); + bool _speechEnabled = false; + String text = ""; + + // Check if speech recognition is enabled. + bool isSpeechEnabled() => _speechEnabled; + + Future getTextWhenReady() async { + await _speechToText.stop(); + return text; + } + + // Retrieve the text. + String getText() => text; + + // Reset either after saving the text or prior to using STT again. + void reset() => text = ""; + + /// Start a speech recognition session + void _startListening() async => await _speechToText.listen(onResult: _onSpeechResult); + + /// Manually stop the active speech recognition session + /// Note that there are also timeouts that each platform enforces + /// and the SpeechToText plugin supports setting timeouts on the + /// listen method. + void _stopListening() async => await _speechToText.stop(); + + SpeechToText() { + _initSpeech(); + } + + /// This has to happen only once per app + void _initSpeech() async => _speechEnabled = await _speechToText.initialize(); + + /// This is the callback that the SpeechToText plugin calls when + /// the platform returns recognized words. + void _onSpeechResult(SpeechRecognitionResult result) => text = result.recognizedWords; +} \ No newline at end of file diff --git a/mobile/app/macos/Flutter/GeneratedPluginRegistrant.swift b/mobile/app/macos/Flutter/GeneratedPluginRegistrant.swift index 513efaa..7bc02f3 100644 --- a/mobile/app/macos/Flutter/GeneratedPluginRegistrant.swift +++ b/mobile/app/macos/Flutter/GeneratedPluginRegistrant.swift @@ -8,6 +8,7 @@ import Foundation import path_provider_foundation import rive_common import shared_preferences_foundation +import speech_to_text_macos import sqflite import url_launcher_macos import video_player_avfoundation @@ -16,6 +17,7 @@ func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) { PathProviderPlugin.register(with: registry.registrar(forPlugin: "PathProviderPlugin")) RivePlugin.register(with: registry.registrar(forPlugin: "RivePlugin")) SharedPreferencesPlugin.register(with: registry.registrar(forPlugin: "SharedPreferencesPlugin")) + SpeechToTextMacosPlugin.register(with: registry.registrar(forPlugin: "SpeechToTextMacosPlugin")) SqflitePlugin.register(with: registry.registrar(forPlugin: "SqflitePlugin")) UrlLauncherPlugin.register(with: registry.registrar(forPlugin: "UrlLauncherPlugin")) FVPVideoPlayerPlugin.register(with: registry.registrar(forPlugin: "FVPVideoPlayerPlugin")) diff --git a/mobile/app/pubspec.lock b/mobile/app/pubspec.lock index 8adbd99..7721e71 100644 --- a/mobile/app/pubspec.lock +++ b/mobile/app/pubspec.lock @@ -696,6 +696,14 @@ packages: url: "https://pub.dev" source: hosted version: "2.2.1" + pedantic: + dependency: transitive + description: + name: pedantic + sha256: "67fc27ed9639506c856c840ccce7594d0bdcd91bc8d53d6e52359449a1d50602" + url: "https://pub.dev" + source: hosted + version: "1.11.1" percent_indicator: dependency: transitive description: @@ -917,6 +925,30 @@ packages: url: "https://pub.dev" source: hosted version: "1.10.0" + speech_to_text: + dependency: "direct main" + description: + name: speech_to_text + sha256: "57fef1d41bdebe298e84842c89bb4ac91f31cdbec7830c8cb1fc6b91d03abd42" + url: "https://pub.dev" + source: hosted + version: "6.6.0" + speech_to_text_macos: + dependency: transitive + description: + name: speech_to_text_macos + sha256: e685750f7542fcaa087a5396ee471e727ec648bf681f4da83c84d086322173f6 + url: "https://pub.dev" + source: hosted + version: "1.1.0" + speech_to_text_platform_interface: + dependency: transitive + description: + name: speech_to_text_platform_interface + sha256: a0df1a907091ea09880077dc25aae02af9f79811264e6e97ddb08639b7f771c2 + url: "https://pub.dev" + source: hosted + version: "2.2.0" sprintf: dependency: transitive description: diff --git a/mobile/app/pubspec.yaml b/mobile/app/pubspec.yaml index adfa4e0..1d4867d 100644 --- a/mobile/app/pubspec.yaml +++ b/mobile/app/pubspec.yaml @@ -45,6 +45,7 @@ dependencies: easy_debounce: any video_player: ^2.9.1 flutter_riverpod: ^2.5.1 + speech_to_text: ^6.6.0 dev_dependencies: flutter_test: sdk: flutter