added multimedia support and STT. STT and photos are have bugs but are functional. file upload is conceptualized but will require additional backend work. video is on hold for now.

This commit is contained in:
Joshua
2024-07-10 19:15:03 -06:00
parent e5b69aa2b6
commit cbd040f57b
36 changed files with 742 additions and 137 deletions
+2
View File
@@ -14,3 +14,5 @@ A few resources to get you started if this is your first Flutter project:
For help getting started with Flutter development, view the
[online documentation](https://docs.flutter.dev/), which offers tutorials,
samples, guidance on mobile development, and a full API reference.
# TODO
+2 -2
View File
@@ -24,7 +24,7 @@ if (flutterVersionName == null) {
}
android {
namespace = "com.example.app"
namespace = "quaxlyqueen.dev.oneai"
compileSdk = flutter.compileSdkVersion
ndkVersion = flutter.ndkVersion
@@ -35,7 +35,7 @@ android {
defaultConfig {
// TODO: Specify your own unique Application ID (https://developer.android.com/studio/build/application-id.html).
applicationId = "com.example.app"
applicationId = "quaxlyqueen.dev.oneai"
// You can update the following values to match your application needs.
// For more information, see: https://docs.flutter.dev/deployment/android#reviewing-the-gradle-build-configuration.
minSdk = flutter.minSdkVersion
+12
View File
@@ -5,6 +5,8 @@ PODS:
- HaishinKit (= 1.3.0)
- ApiVideoLiveStream (1.2.2):
- HaishinKit (= 1.3.0)
- camera_avfoundation (0.0.1):
- Flutter
- DKImagePickerController/Core (4.3.9):
- DKImagePickerController/ImageDataManager
- DKImagePickerController/Resource
@@ -44,6 +46,8 @@ PODS:
- Flutter
- HaishinKit (1.3.0):
- Logboard (~> 2.3.0)
- image_picker_ios (0.0.1):
- Flutter
- Logboard (2.3.1)
- native_device_orientation (0.0.1):
- Flutter
@@ -78,9 +82,11 @@ PODS:
DEPENDENCIES:
- apivideo_live_stream (from `.symlinks/plugins/apivideo_live_stream/ios`)
- camera_avfoundation (from `.symlinks/plugins/camera_avfoundation/ios`)
- file_picker (from `.symlinks/plugins/file_picker/ios`)
- Flutter (from `Flutter`)
- flutter_credit_card (from `.symlinks/plugins/flutter_credit_card/ios`)
- image_picker_ios (from `.symlinks/plugins/image_picker_ios/ios`)
- native_device_orientation (from `.symlinks/plugins/native_device_orientation/ios`)
- path_provider_foundation (from `.symlinks/plugins/path_provider_foundation/darwin`)
- pointer_interceptor_ios (from `.symlinks/plugins/pointer_interceptor_ios/ios`)
@@ -106,12 +112,16 @@ SPEC REPOS:
EXTERNAL SOURCES:
apivideo_live_stream:
:path: ".symlinks/plugins/apivideo_live_stream/ios"
camera_avfoundation:
:path: ".symlinks/plugins/camera_avfoundation/ios"
file_picker:
:path: ".symlinks/plugins/file_picker/ios"
Flutter:
:path: Flutter
flutter_credit_card:
:path: ".symlinks/plugins/flutter_credit_card/ios"
image_picker_ios:
:path: ".symlinks/plugins/image_picker_ios/ios"
native_device_orientation:
:path: ".symlinks/plugins/native_device_orientation/ios"
path_provider_foundation:
@@ -136,12 +146,14 @@ EXTERNAL SOURCES:
SPEC CHECKSUMS:
apivideo_live_stream: 64fa696699cfc531e07f5d5d701c7e3cd5ce5e9d
ApiVideoLiveStream: 5842571806a2ed26671597dadd370ec38d9b9ba4
camera_avfoundation: dd002b0330f4981e1bbcb46ae9b62829237459a4
DKImagePickerController: 946cec48c7873164274ecc4624d19e3da4c1ef3c
DKPhotoGallery: b3834fecb755ee09a593d7c9e389d8b5d6deed60
file_picker: 09aa5ec1ab24135ccd7a1621c46c84134bfd6655
Flutter: e0871f40cf51350855a761d2e70bf5af5b9b5de7
flutter_credit_card: 48fb35065294868bc6a3e7f18a42f28929b84f48
HaishinKit: 013229683accc4abe46630b04629da3ff90428b7
image_picker_ios: c560581cceedb403a6ff17f2f816d7fea1421fc1
Logboard: 3d98bb85de6a36b255ab637e8178eb5671c5c3a6
native_device_orientation: 3b4cfc9565a7b879cc4fde282b3e27745e852d0d
path_provider_foundation: 2b6b4c569c0fb62ec74538f866245ac84301af46
+8 -2
View File
@@ -2,8 +2,14 @@
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>NSMicrophoneUsageDescription</key>
<string>Required for on-device speech recognition.</string>
<key>NSCameraUsageDescription</key>
<string>This app needs access to camera when used</string>
<key>NSPhotoLibraryUsageDescription</key>
<string>TODO: Better description</string>
<key>LSApplicationCategoryType</key>
<string>Productivity</string>
<key>NSMicrophoneUsageDescription</key>
<string>Required for on-device speech recognition.</string>
<key>NSSpeechRecognitionUsageDescription</key>
<string>On-device speech-to-text.</string>
<key>CADisableMinimumFrameDurationOnPhone</key>
@@ -1,9 +1,32 @@
class Chat {
final String content;
final bool role;
int chatType = 0;
// Most basic chat, text from server or from user.
Chat(this.content, this.role);
Chat(this.content, this.role, this.chatType);
// TODO: Overload constructors to allow for multimedia.
// A chat with an included image.
Chat.image(this.content, this.role, this.chatType);
// A chat with an included file.
Chat.file(this.content, this.role, this.chatType);
}
// TODO: Add proper filetype support. File should either be parsed on-device, or uploaded to the server.
// TODO: Implement dynamic pathing based upon filetypes.
enum Filetype {
pdf,
doc,
docx,
pages,
note,
todo,
cal,
calendar,
event,
contact,
mp3,
mp4,
wav,
}
@@ -1,3 +1,6 @@
import 'dart:convert';
import 'package:app/components/util/backend.dart';
import 'package:app/theme.dart';
import 'package:flutter/material.dart';
import 'chat.dart';
@@ -29,10 +32,20 @@ class ChatBubbleWidget extends StatelessWidget{
),
color: c.role ? AppTheme.darkest : AppTheme.dark,
),
child: Text(
c.content,
style: AppTheme.bodyMedium,
),
child:
// TODO: Properly display filetypes in the thread.
c.chatType == 0 ? Text(
c.content,
style: AppTheme.bodyMedium,
)
: c.chatType == 1 ? Image(image: Backend.convertFromBase64(c.content))
: c.chatType == 2 ? const Icon(
Icons.file_copy,
) : const Text("null"),
// Text(
// c.content,
// style: AppTheme.bodyMedium,
// ),
),
],
);
@@ -1,4 +1,4 @@
import 'package:app/util/backend.dart';
import 'package:app/components/util/backend.dart';
import 'package:flutter/material.dart';
import 'chat.dart';
@@ -6,10 +6,12 @@ class Conversation with ChangeNotifier {
int conversationID;
String conversationLabel;
String conversationSubLabel;
bool history = true;
// History is disabled by default. This means chats are auto-deleted once the user exits that conversation.
// TODO: Fix history, very, verry strange behavior when disabled.
bool history = true; // History is disabled by default. This means chats are auto-deleted once the user exits that conversation.\
bool conversationContext = true; // The prior prompts and responses are used as additional context for the next prompt and response.
late bool stream = false; // Update the UI with a stream of characters as received from the server, or as one text.
var lastAccessTime = DateTime.timestamp(); // Used to sort conversations based on most recent usage.
List<Chat> chats = [];
Conversation(this.conversationID, this.conversationLabel, this.conversationSubLabel);
@@ -23,11 +25,13 @@ class Conversation with ChangeNotifier {
void toggleStream() => stream = !stream;
void add(Chat message) {
lastAccessTime = DateTime.timestamp();
chats.add(message);
notifyListeners();
}
void loadChats() {
lastAccessTime = DateTime.timestamp();
Backend.loadConversation;
chats = Backend.loadedChats;
}
@@ -41,10 +45,12 @@ class Conversation with ChangeNotifier {
}
List<Chat> getChats() {
lastAccessTime = DateTime.timestamp();
return chats;
}
void setLabels(String cLabel, String cSubLabel) {
lastAccessTime = DateTime.timestamp();
conversationLabel = cLabel;
conversationSubLabel = cSubLabel;
}
@@ -1,7 +1,7 @@
import 'package:app/components/conversation_thread/chat_bubble_model.dart';
import 'package:app/components/conversation_thread/chat.dart';
import 'package:app/components/conversation_thread/conversation.dart';
import 'package:app/util/backend.dart';
import 'package:app/components/util/backend.dart';
import 'package:app/components/prompt_box/prompt_box_widget.dart';
import 'package:flutterflow_ui/flutterflow_ui.dart';
@@ -1,5 +1,5 @@
import 'package:app/components/conversation_thread/conversation.dart';
import 'package:app/util/backend.dart';
import 'package:app/components/util/backend.dart';
import 'package:provider/provider.dart';
import '/components/conversation_thread/chat_bubble_widget.dart';
@@ -50,6 +50,7 @@ class _ConversationThreadWidgetState extends State<ConversationThreadWidget> {
@override
Widget build(BuildContext context) {
// TODO: Add a scrollbar
return Container(
width: double.infinity,
height: double.infinity,
@@ -9,8 +9,8 @@ import 'package:flutter/services.dart';
import 'package:app/theme.dart';
import 'conversation_model.dart';
export 'conversation_model.dart';
import '../components/conversation_model.dart';
export '../components/conversation_model.dart';
class ConversationWidget extends StatefulWidget {
final Conversation conversation;
@@ -64,8 +64,8 @@ class _ConversationWidgetState extends State<ConversationWidget> {
size: 30,
),
onPressed: () async {
print("something pressed in conversation_widget.dart");
// TODO: Smooth out the animation to prevent the conversations list from loading half-way up the screen.
// Note: that only applies if the input was already selected.
SystemChannels.textInput.invokeMethod('TextInput.hide');
Navigator.pop(context);
},
@@ -1,15 +1,15 @@
import 'package:app/components/conversation_thread/conversation.dart';
import 'package:app/theme.dart';
import 'package:app/pages/conversation_widget.dart';
import 'package:app/util/backend.dart';
import 'package:app/components/conversation_widget.dart';
import 'package:app/components/util/backend.dart';
import 'package:flutterflow_ui/flutterflow_ui.dart';
import 'package:flutter/material.dart';
import 'package:app/components/options_dialog/options_dialog_widget.dart';
import 'conversations_list_model.dart';
export 'conversations_list_model.dart';
import '../components/conversations_list_model.dart';
export '../components/conversations_list_model.dart';
class ConversationsListWidget extends StatefulWidget {
const ConversationsListWidget({super.key});
@@ -213,9 +213,6 @@ class _ConversationsListWidgetState extends State<ConversationsListWidget> {
},
).then((value) =>
safeSetState(() {})),
// TODO: Need to implement.
print("opening options")
},
),
);
@@ -235,7 +232,7 @@ class _ConversationsListWidgetState extends State<ConversationsListWidget> {
Navigator.push(
context,
MaterialPageRoute(
builder: (context) => ConversationWidget(conversation: Backend.getLoadedConversation(),),
builder: (context) => ConversationWidget(conversation: Backend.loadedConversation,),
),
), // Handle the button press
},
@@ -1,13 +0,0 @@
import 'package:flutterflow_ui/flutterflow_ui.dart';
import 'no_chats_widget.dart' show NoChatsWidget;
import 'package:flutter/material.dart';
import 'package:google_fonts/google_fonts.dart';
import 'package:provider/provider.dart';
class NoChatsModel extends FlutterFlowModel<NoChatsWidget> {
@override
void initState(BuildContext context) {}
@override
void dispose() {}
}
@@ -5,9 +5,6 @@ import 'package:provider/provider.dart';
import 'package:app/theme.dart';
import 'no_chats_model.dart';
export 'no_chats_model.dart';
class NoChatsWidget extends StatefulWidget {
const NoChatsWidget({
super.key,
@@ -26,24 +23,19 @@ class NoChatsWidget extends StatefulWidget {
}
class _NoChatsWidgetState extends State<NoChatsWidget> {
late NoChatsModel _model;
@override
void setState(VoidCallback callback) {
super.setState(callback);
_model.onUpdate();
}
@override
void initState() {
super.initState();
_model = createModel(context, () => NoChatsModel());
}
@override
void dispose() {
_model.maybeDispose();
super.dispose();
}
@@ -1,10 +1,17 @@
import 'package:app/util/backend.dart';
import 'dart:io';
import 'package:app/components/util/backend.dart';
import 'package:camera/camera.dart';
import 'package:file_picker/file_picker.dart';
import 'package:flutter/services.dart';
import 'package:flutterflow_ui/flutterflow_ui.dart';
import 'package:easy_debounce/easy_debounce.dart';
import 'package:flutter/material.dart';
import 'package:app/theme.dart';
import 'package:image_picker/image_picker.dart';
import '../conversation_thread/chat.dart';
import 'prompt_box_model.dart';
export 'prompt_box_model.dart';
@@ -17,7 +24,7 @@ class PromptBoxWidget extends StatefulWidget {
class _PromptBoxWidgetState extends State<PromptBoxWidget> {
late PromptBoxModel _model;
late String entry_text;
String entryText = "";
@override
void setState(VoidCallback callback) {
@@ -43,6 +50,7 @@ class _PromptBoxWidgetState extends State<PromptBoxWidget> {
@override
Widget build(BuildContext context) {
const double buttonPadding = 4;
return SafeArea(
child: Container(
width: double.infinity,
@@ -62,50 +70,59 @@ class _PromptBoxWidgetState extends State<PromptBoxWidget> {
children: [
Expanded( // TODO: File context upload
child: Padding(
padding: const EdgeInsets.all(14),
child: FlutterFlowIconButton(
borderColor: Colors.transparent,
borderRadius: 30,
borderWidth: 1,
buttonSize: 40,
padding: const EdgeInsets.all(buttonPadding),
child: IconButton(
icon: const Icon(
Icons.file_upload_outlined,
color: AppTheme.darkest,
size: 30,
),
onPressed: () {
print('IconButton pressed ...');
onPressed: () async {
final picker = FilePicker.platform;
final result = await picker.pickFiles(allowMultiple: true); // Allow multiple files
if (result != null) {
for (final platformFile in result.files) {
// TODO: Upload file to server
final file = File(platformFile.path!);
}
} else {
print('User canceled file selection');
}
},
),
),
),
Expanded( // TODO: Photo context upload
Expanded( // TODO: Gallery context upload
child: Padding(
padding: const EdgeInsets.all(14),
child: FlutterFlowIconButton(
borderColor: Colors.transparent,
borderRadius: 30,
borderWidth: 1,
buttonSize: 40,
padding: const EdgeInsets.all(buttonPadding),
child: IconButton(
icon: const Icon(
Icons.photo_outlined,
color: AppTheme.darkest,
size: 30,
),
onPressed: () {
print('IconButton pressed ...');
onPressed: () async {
final picker = ImagePicker();
final image = await picker.pickImage(source: ImageSource.gallery);
// TODO: Allow for multiple pictures/photos to be selected.
// final images = await picker.pickMultiImage();
if (image != null) {
// TODO: Upload to server and add to conversation.chats
String i = Backend.convertToBase64(File(image.path)).toString();
Backend.loadedConversation.add(Chat.image(i, true, 1));
} else {
print('User canceled image selection');
}
},
),
),
),
Expanded( // TODO: Video context upload
child: Padding(
padding: const EdgeInsets.all(14),
child: FlutterFlowIconButton(
borderColor: Colors.transparent,
borderRadius: 30,
borderWidth: 1,
buttonSize: 40,
padding: const EdgeInsets.all(buttonPadding),
child: IconButton(
icon: const Icon(
Icons.videocam_outlined,
color: AppTheme.darkest,
@@ -119,19 +136,48 @@ class _PromptBoxWidgetState extends State<PromptBoxWidget> {
),
Expanded( // TODO: Camera context upload
child: Padding(
padding: const EdgeInsets.all(14),
child: FlutterFlowIconButton(
borderColor: Colors.transparent,
borderRadius: 30,
borderWidth: 1,
buttonSize: 40,
padding: const EdgeInsets.all(buttonPadding),
child: IconButton(
icon: const Icon(
Icons.camera_alt_outlined,
color: AppTheme.darkest,
size: 30,
),
onPressed: () async {
final picker = ImagePicker();
final image = await picker.pickImage(source: ImageSource.camera);
if (image != null) {
// TODO: Upload to server and add to conversation.chats
final i = Backend.convertToBase64(File(image.path));
Backend.loadedConversation.add(Chat.image(i.toString(), true, 1));
} else {
print('User canceled photo capture');
}
},
),
),
),
Expanded(
child: Padding(
padding: const EdgeInsets.all(buttonPadding),
child: IconButton(
icon: Icon(
Backend.stt.speechToText.isNotListening ? Icons.mic_off_outlined : Icons.mic_none,
color: AppTheme.darkest,
size: 30,
),
onPressed: () {
print('IconButton pressed ...');
// TODO: Audio input is still buggy, it appears to only enter the text field once the button is pressed again to stop.
if(Backend.stt.speechToText.isNotListening) {
print('STT engine starting ...');
Backend.stt.startListening();
updateText();
} else {
print('STT engine stopping');
Backend.stt.stopListening();
updateText();
}
},
),
),
@@ -190,10 +236,13 @@ class _PromptBoxWidgetState extends State<PromptBoxWidget> {
fillColor: AppTheme.alternate,
suffixIcon: GestureDetector(
onTap: () {
// TODO: Connect with multimedia
entry_text = _model.textController.text;
Backend.respondWhenReady(entry_text);
_model.textController?.clear(); // Clear the text field
if(_model.textController.text != "") {
// TODO: Connect with multimedia
entryText = _model.textController.text;
Backend.respondWhenReady(entryText);
_model.textController?.clear(); // Clear the text field
SystemChannels.textInput.invokeMethod('TextInput.hide');
}
},
child: const Icon(
Icons.send,
@@ -206,12 +255,14 @@ class _PromptBoxWidgetState extends State<PromptBoxWidget> {
textAlign: TextAlign.start,
maxLines: 5,
minLines: 1,
validator: _model.textControllerValidator.asValidator(context),
textInputAction: TextInputAction.send, // "Send" on keyboard
onFieldSubmitted: (text) {
entry_text = _model.textController.text;
Backend.respondWhenReady(entry_text); // TODO: Connect with multimedia
_model.textController?.clear(); // Clear the text field
// Don't take any action unless there's an actual input.
if(_model.textController.text != "") {
entryText = _model.textController.text;
Backend.respondWhenReady(entryText); // TODO: Connect with multimedia
_model.textController?.clear(); // Clear the text field
}
},
),
),
@@ -223,4 +274,12 @@ class _PromptBoxWidgetState extends State<PromptBoxWidget> {
),
);
}
}
void updateText() {
_model.textController.text = Backend.stt.text;
entryText = _model.textController.text;
print(entryText);
print(_model.textController.text);
Backend.stt.reset();
}
}
@@ -4,8 +4,8 @@ import 'package:flutter_animate/flutter_animate.dart';
import 'package:google_fonts/google_fonts.dart';
import 'package:provider/provider.dart';
import 'support_form_model.dart';
export 'support_form_model.dart';
import '../components/support_form_model.dart';
export '../components/support_form_model.dart';
import 'package:app/theme.dart';
@@ -1,10 +1,14 @@
import 'dart:convert';
import 'dart:io';
import 'dart:typed_data';
import 'package:flutter/widgets.dart';
import 'package:app/components/conversation_thread/conversation.dart';
import 'package:camera/camera.dart';
import 'package:http/http.dart' as http;
import 'package:app/components/conversation_thread/chat.dart';
import 'package:app/util/speech_to_text.dart';
import 'package:app/components/util/speech_to_text.dart';
import 'model.dart';
@@ -18,6 +22,7 @@ class Backend {
//static final url = "http://192.168.1.68:11434/api"; // For local access to home server
static String endpoint = "";
static final Map<String, String> headers = {'Content-Type': 'application/json'};
static final bool UI_TESTING = true;
@@ -27,11 +32,13 @@ class Backend {
// PRIVACY-FOCUSED VARIABLES
// PRIVACY-NECESSARY VARIABLES
// TODO: Reorder based on most recent usage.
static List<Conversation> conversations = [];
static List<Chat> loadedChats = [];
static late Conversation loadedConversation;
static late List<CameraDescription> cameras;
static late CameraController controller;
// RUNTIME & STATUS VARIABLES
@@ -45,40 +52,35 @@ class Backend {
// TODO: Verify all different components of STT, TTS, LLM, etc have been initialized.
// TODO: Handshake w/ server to validate identity
factory Backend() {
// Speech-To-Text engine activation.
_speechEnabled = stt.isSpeechEnabled();
// loadedChats = conversations[conversationID].chats;
return singleBackend;
}
void init() {
initiated = true;
void init() async {
// Prevent multiple initiations
if(!initiated) {
initiated = true;
// Speech-To-Text engine activation.
_speechEnabled = stt.isSpeechEnabled();
model = Model("llama3"); // Primarily used llama3, testing gemma2
maxConversationID = conversations.length;
cameras = await availableCameras();
controller = CameraController(cameras[0], ResolutionPreset.max);
controller.initialize(); // TODO: Add error catching on camera
}
// TODO: Dynamically get available models and models that can be pulled.
// TODO: Dynamically get the user's default model
model = Model("llama3"); // Primarily used llama3, testing gemma2
// TODO: Dynamically obtain conversations.
// conversations.add(
// Conversation.completeConversation(
// 0,
// "Test Conversation",
// "Testing Functionality",
// [
// (Chat("Hello there! You are part of a prototype and are running locally on my Macbook Air. Please keep your responses brief as every prompt freezes my computer.", true)),
// (Chat("Understood! How can I help?", false)),
// ]
// )
// );
maxConversationID = conversations.length;
// TODO: Obtain all user permissions at once.
}
// PRIMARY INTERFACE
// PRIMARY INTERFACES
@Deprecated("Use STT interface directly and use respondWhenReady().")
static Future<String> sttGetResponseWhenReady() async {
// TODO: In case on-device STT is unavailable, use server-side STT service.
@@ -87,19 +89,38 @@ class Backend {
}
prompt = stt.getTextWhenReady() as String;
conversations[conversationID].add(Chat(prompt, true));
httpSendRequest();
conversations[conversationID].add(Chat(response, false));
conversations[conversationID].add(Chat(prompt, true, 0));
if(!UI_TESTING) {
httpSendRequest();
}
conversations[conversationID].add(Chat(response, false, 0));
return response;
}
static void respondWhenReady(String p) async {
prompt = p;
conversations[conversationID].add(Chat(prompt, true));
httpSendRequest();
conversations[conversationID].add(Chat(prompt, true, 0));
if(!UI_TESTING) {
httpSendRequest();
}
}
static ImageProvider convertFromBase64(img) {
Uint8List imageBytes = base64Decode(img);
return Image.memory(imageBytes).image;
}
// Converts image to base64 encoded string for data transfer.
static String convertToBase64(f) {
List<int> imageBytes = f.readAsBytesSync();
String img = base64Encode(imageBytes);
// TODO: Add special handling for images included in prompts.
conversations[conversationID].add(Chat.image(img, true, 1));
return img;
}
// CONVERSATION MANAGEMENT
@@ -155,7 +176,7 @@ class Backend {
Map<String, dynamic> data;
if(conversations[conversationID].conversationContext) {
loadedChats = [...loadedChats, Chat(prompt, true)];
loadedChats = [...loadedChats, Chat(prompt, true, 0)];
endpoint = "chat";
List<Map<String, String>> messageList = convertMessages();
@@ -196,12 +217,13 @@ class Backend {
Map<String, dynamic> re = jsonDecode(serverResponse.body);
if(conversations[conversationID].conversationContext) {
response = re["message"]["content"];
loadedChats = [...loadedChats, Chat(response, false)];
// TODO: Modify in case of different response types.
loadedChats = [...loadedChats, Chat(response, false, 0)];
} else {
response = re["response"];
}
}
conversations[conversationID].add(Chat(response, false));
conversations[conversationID].add(Chat(response, false, 0));
prompt = "";
response = "";
@@ -1,9 +1,16 @@
import 'package:speech_to_text/speech_recognition_result.dart';
import 'package:speech_to_text/speech_to_text.dart' as stt;
import 'package:speech_to_text/speech_to_text.dart';
/// Handles standard Speech-To-Text using on-device processing.
class SpeechToText {
final stt.SpeechToText speechToText = stt.SpeechToText();
// TODO: User customizable STT??
final SpeechListenOptions options = stt.SpeechListenOptions(
partialResults: true,
onDevice: true,
);
bool speechEnabled = false;
String text = "";
@@ -16,13 +23,22 @@ class SpeechToText {
}
// Retrieve the text.
@Deprecated("TODO: Deprecated message for speechToText.getText")
String getText() => text;
// Reset either after saving the text or prior to using STT again.
void reset() => text = "";
/// Start a speech recognition session
void startListening() async => await speechToText.listen(onResult: onSpeechResult);
void startListening() async {
await speechToText.listen(
onResult: onSpeechResult,
pauseFor: const Duration(seconds: 3),
// TODO: I think this is a bug with the speech_to_text library? The docs correctly identify the named parameter of SpeechListenOptions, but it's not recognized.
// SpeechListenOptions: options,
);
}
/// Manually stop the active speech recognition session
/// Note that there are also timeouts that each platform enforces
@@ -0,0 +1,317 @@
import 'dart:math';
class UsageQueue<T> implements List {
@override
var first;
@override
var last;
@override
late int length;
@override
List operator +(List other) {
// TODO: implement +
throw UnimplementedError();
}
@override
operator [](int index) {
// TODO: implement []
throw UnimplementedError();
}
@override
void operator []=(int index, value) {
// TODO: implement []=
}
@override
void add(value) {
// TODO: implement add
}
@override
void addAll(Iterable iterable) {
// TODO: implement addAll
}
@override
bool any(bool Function(dynamic element) test) {
// TODO: implement any
throw UnimplementedError();
}
@override
Map<int, dynamic> asMap() {
// TODO: implement asMap
throw UnimplementedError();
}
@override
List<R> cast<R>() {
// TODO: implement cast
throw UnimplementedError();
}
@override
void clear() {
// TODO: implement clear
}
@override
bool contains(Object? element) {
// TODO: implement contains
throw UnimplementedError();
}
@override
elementAt(int index) {
// TODO: implement elementAt
throw UnimplementedError();
}
@override
bool every(bool Function(dynamic element) test) {
// TODO: implement every
throw UnimplementedError();
}
@override
Iterable<T> expand<T>(Iterable<T> Function(dynamic element) toElements) {
// TODO: implement expand
throw UnimplementedError();
}
@override
void fillRange(int start, int end, [fillValue]) {
// TODO: implement fillRange
}
@override
firstWhere(bool Function(dynamic element) test, {Function()? orElse}) {
// TODO: implement firstWhere
throw UnimplementedError();
}
@override
T fold<T>(T initialValue, T Function(T previousValue, dynamic element) combine) {
// TODO: implement fold
throw UnimplementedError();
}
@override
Iterable followedBy(Iterable other) {
// TODO: implement followedBy
throw UnimplementedError();
}
@override
void forEach(void Function(dynamic element) action) {
// TODO: implement forEach
}
@override
Iterable getRange(int start, int end) {
// TODO: implement getRange
throw UnimplementedError();
}
@override
int indexOf(element, [int start = 0]) {
// TODO: implement indexOf
throw UnimplementedError();
}
@override
int indexWhere(bool Function(dynamic element) test, [int start = 0]) {
// TODO: implement indexWhere
throw UnimplementedError();
}
@override
void insert(int index, element) {
// TODO: implement insert
}
@override
void insertAll(int index, Iterable iterable) {
// TODO: implement insertAll
}
@override
// TODO: implement isEmpty
bool get isEmpty => throw UnimplementedError();
@override
// TODO: implement isNotEmpty
bool get isNotEmpty => throw UnimplementedError();
@override
// TODO: implement iterator
Iterator get iterator => throw UnimplementedError();
@override
String join([String separator = ""]) {
// TODO: implement join
throw UnimplementedError();
}
@override
int lastIndexOf(element, [int? start]) {
// TODO: implement lastIndexOf
throw UnimplementedError();
}
@override
int lastIndexWhere(bool Function(dynamic element) test, [int? start]) {
// TODO: implement lastIndexWhere
throw UnimplementedError();
}
@override
lastWhere(bool Function(dynamic element) test, {Function()? orElse}) {
// TODO: implement lastWhere
throw UnimplementedError();
}
@override
Iterable<T> map<T>(T Function(dynamic e) toElement) {
// TODO: implement map
throw UnimplementedError();
}
@override
reduce(Function(dynamic value, dynamic element) combine) {
// TODO: implement reduce
throw UnimplementedError();
}
@override
bool remove(Object? value) {
// TODO: implement remove
throw UnimplementedError();
}
@override
removeAt(int index) {
// TODO: implement removeAt
throw UnimplementedError();
}
@override
removeLast() {
// TODO: implement removeLast
throw UnimplementedError();
}
@override
void removeRange(int start, int end) {
// TODO: implement removeRange
}
@override
void removeWhere(bool Function(dynamic element) test) {
// TODO: implement removeWhere
}
@override
void replaceRange(int start, int end, Iterable replacements) {
// TODO: implement replaceRange
}
@override
void retainWhere(bool Function(dynamic element) test) {
// TODO: implement retainWhere
}
@override
// TODO: implement reversed
Iterable get reversed => throw UnimplementedError();
@override
void setAll(int index, Iterable iterable) {
// TODO: implement setAll
}
@override
void setRange(int start, int end, Iterable iterable, [int skipCount = 0]) {
// TODO: implement setRange
}
@override
void shuffle([Random? random]) {
// TODO: implement shuffle
}
@override
// TODO: implement single
get single => throw UnimplementedError();
@override
singleWhere(bool Function(dynamic element) test, {Function()? orElse}) {
// TODO: implement singleWhere
throw UnimplementedError();
}
@override
Iterable skip(int count) {
// TODO: implement skip
throw UnimplementedError();
}
@override
Iterable skipWhile(bool Function(dynamic value) test) {
// TODO: implement skipWhile
throw UnimplementedError();
}
@override
void sort([int Function(dynamic a, dynamic b)? compare]) {
// TODO: implement sort
}
@override
List sublist(int start, [int? end]) {
// TODO: implement sublist
throw UnimplementedError();
}
@override
Iterable take(int count) {
// TODO: implement take
throw UnimplementedError();
}
@override
Iterable takeWhile(bool Function(dynamic value) test) {
// TODO: implement takeWhile
throw UnimplementedError();
}
@override
List toList({bool growable = true}) {
// TODO: implement toList
throw UnimplementedError();
}
@override
Set toSet() {
// TODO: implement toSet
throw UnimplementedError();
}
@override
Iterable where(bool Function(dynamic element) test) {
// TODO: implement where
throw UnimplementedError();
}
@override
Iterable<T> whereType<T>() {
// TODO: implement whereType
throw UnimplementedError();
}
}
+8 -14
View File
@@ -1,6 +1,6 @@
import 'package:app/pages/conversation_widget.dart';
import 'package:app/pages/settings_widget.dart';
import 'package:app/pages/conversations_list_widget.dart';
import 'package:app/components/conversation_widget.dart';
import 'package:app/components/settings_widget.dart';
import 'package:app/components/conversations_list_widget.dart';
import 'package:flutter/material.dart';
import 'package:flutter_riverpod/flutter_riverpod.dart';
import 'package:provider/provider.dart';
@@ -36,6 +36,7 @@ BottomNavigationBarItem _buildNavigationItem(int index) {
icon: Icon(iconData),
label: label,
backgroundColor: AppTheme.lightest,
);
}
@@ -67,21 +68,14 @@ class MyApp extends ConsumerWidget {
title: 'One AI',
home: Scaffold(
backgroundColor: AppTheme.medium,
appBar: AppBar(
backgroundColor: AppTheme.secondaryBackground,
automaticallyImplyLeading: false,
title: const Text(
'One AI',
style: AppTheme.displayLarge,
),
actions: const [],
centerTitle: false,
elevation: 0,
),
body: _pages[selectedIndex],
bottomNavigationBar: BottomNavigationBar(
backgroundColor: AppTheme.darkest,
selectedItemColor: AppTheme.lightest,
showUnselectedLabels: false,
showSelectedLabels: false,
iconSize: 30,
// Set the number of items to the length of the pages list
items: List.generate(_pages.length, (index) => _buildNavigationItem(index)),
currentIndex: selectedIndex,
@@ -6,9 +6,13 @@
#include "generated_plugin_registrant.h"
#include <file_selector_linux/file_selector_plugin.h>
#include <url_launcher_linux/url_launcher_plugin.h>
void fl_register_plugins(FlPluginRegistry* registry) {
g_autoptr(FlPluginRegistrar) file_selector_linux_registrar =
fl_plugin_registry_get_registrar_for_plugin(registry, "FileSelectorPlugin");
file_selector_plugin_register_with_registrar(file_selector_linux_registrar);
g_autoptr(FlPluginRegistrar) url_launcher_linux_registrar =
fl_plugin_registry_get_registrar_for_plugin(registry, "UrlLauncherPlugin");
url_launcher_plugin_register_with_registrar(url_launcher_linux_registrar);
@@ -3,6 +3,7 @@
#
list(APPEND FLUTTER_PLUGIN_LIST
file_selector_linux
url_launcher_linux
)
@@ -5,6 +5,7 @@
import FlutterMacOS
import Foundation
import file_selector_macos
import path_provider_foundation
import rive_common
import shared_preferences_foundation
@@ -14,6 +15,7 @@ import url_launcher_macos
import video_player_avfoundation
func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) {
FileSelectorPlugin.register(with: registry.registrar(forPlugin: "FileSelectorPlugin"))
PathProviderPlugin.register(with: registry.registrar(forPlugin: "PathProviderPlugin"))
RivePlugin.register(with: registry.registrar(forPlugin: "RivePlugin"))
SharedPreferencesPlugin.register(with: registry.registrar(forPlugin: "SharedPreferencesPlugin"))
+145 -1
View File
@@ -89,6 +89,46 @@ packages:
url: "https://pub.dev"
source: hosted
version: "1.2.0"
camera:
dependency: "direct main"
description:
name: camera
sha256: "2170a943dcb67be2af2c6bcda8775e74b41d4c02d6a4eb10bdc832ee185c4eea"
url: "https://pub.dev"
source: hosted
version: "0.11.0+1"
camera_android_camerax:
dependency: transitive
description:
name: camera_android_camerax
sha256: "7c03940cb8c92eb5b184952674a07cc4a73c6ba2b3568aad70255ad4cb913660"
url: "https://pub.dev"
source: hosted
version: "0.6.7+1"
camera_avfoundation:
dependency: transitive
description:
name: camera_avfoundation
sha256: "95c2f40b4d06cdb0fd2ad893c762d1f2c931a5e370793ec34c939eb4fcbf96bb"
url: "https://pub.dev"
source: hosted
version: "0.9.17"
camera_platform_interface:
dependency: transitive
description:
name: camera_platform_interface
sha256: b3ede1f171532e0d83111fe0980b46d17f1aa9788a07a2fbed07366bbdbb9061
url: "https://pub.dev"
source: hosted
version: "2.8.0"
camera_web:
dependency: transitive
description:
name: camera_web
sha256: b9235ec0a2ce949daec546f1f3d86f05c3921ed31c7d9ab6b7c03214d152fc2d
url: "https://pub.dev"
source: hosted
version: "0.3.4"
carousel_slider:
dependency: transitive
description:
@@ -234,13 +274,45 @@ packages:
source: hosted
version: "7.0.0"
file_picker:
dependency: transitive
dependency: "direct main"
description:
name: file_picker
sha256: "824f5b9f389bfc4dddac3dea76cd70c51092d9dff0b2ece7ef4f53db8547d258"
url: "https://pub.dev"
source: hosted
version: "8.0.6"
file_selector_linux:
dependency: transitive
description:
name: file_selector_linux
sha256: "045d372bf19b02aeb69cacf8b4009555fb5f6f0b7ad8016e5f46dd1387ddd492"
url: "https://pub.dev"
source: hosted
version: "0.9.2+1"
file_selector_macos:
dependency: transitive
description:
name: file_selector_macos
sha256: f42eacb83b318e183b1ae24eead1373ab1334084404c8c16e0354f9a3e55d385
url: "https://pub.dev"
source: hosted
version: "0.9.4"
file_selector_platform_interface:
dependency: transitive
description:
name: file_selector_platform_interface
sha256: a3994c26f10378a039faa11de174d7b78eb8f79e4dd0af2a451410c1a5c3f66b
url: "https://pub.dev"
source: hosted
version: "2.6.2"
file_selector_windows:
dependency: transitive
description:
name: file_selector_windows
sha256: d3547240c20cabf205c7c7f01a50ecdbc413755814d6677f3cb366f04abcead0
url: "https://pub.dev"
source: hosted
version: "0.9.3+1"
fixnum:
dependency: transitive
description:
@@ -480,6 +552,70 @@ packages:
url: "https://pub.dev"
source: hosted
version: "4.0.2"
image_picker:
dependency: "direct main"
description:
name: image_picker
sha256: "021834d9c0c3de46bf0fe40341fa07168407f694d9b2bb18d532dc1261867f7a"
url: "https://pub.dev"
source: hosted
version: "1.1.2"
image_picker_android:
dependency: transitive
description:
name: image_picker_android
sha256: ff39a10ab4f48f4ac70776d0494a97bf073cd2570892cd46bc8a5cac162c25db
url: "https://pub.dev"
source: hosted
version: "0.8.12+4"
image_picker_for_web:
dependency: transitive
description:
name: image_picker_for_web
sha256: "5d6eb13048cd47b60dbf1a5495424dea226c5faf3950e20bf8120a58efb5b5f3"
url: "https://pub.dev"
source: hosted
version: "3.0.4"
image_picker_ios:
dependency: transitive
description:
name: image_picker_ios
sha256: "6703696ad49f5c3c8356d576d7ace84d1faf459afb07accbb0fae780753ff447"
url: "https://pub.dev"
source: hosted
version: "0.8.12"
image_picker_linux:
dependency: transitive
description:
name: image_picker_linux
sha256: "4ed1d9bb36f7cd60aa6e6cd479779cc56a4cb4e4de8f49d487b1aaad831300fa"
url: "https://pub.dev"
source: hosted
version: "0.2.1+1"
image_picker_macos:
dependency: transitive
description:
name: image_picker_macos
sha256: "3f5ad1e8112a9a6111c46d0b57a7be2286a9a07fc6e1976fdf5be2bd31d4ff62"
url: "https://pub.dev"
source: hosted
version: "0.2.1+1"
image_picker_platform_interface:
dependency: transitive
description:
name: image_picker_platform_interface
sha256: "9ec26d410ff46f483c5519c29c02ef0e02e13a543f882b152d4bfd2f06802f80"
url: "https://pub.dev"
source: hosted
version: "2.10.0"
image_picker_windows:
dependency: transitive
description:
name: image_picker_windows
sha256: "6ad07afc4eb1bc25f3a01084d28520496c4a3bb0cb13685435838167c9dcedeb"
url: "https://pub.dev"
source: hosted
version: "0.2.1+1"
intl:
dependency: transitive
description:
@@ -600,6 +736,14 @@ packages:
url: "https://pub.dev"
source: hosted
version: "1.14.0"
mime:
dependency: transitive
description:
name: mime
sha256: "2e123074287cc9fd6c09de8336dae606d1ddb88d9ac47358826db698c176a1f2"
url: "https://pub.dev"
source: hosted
version: "1.0.5"
mime_type:
dependency: transitive
description:
+3
View File
@@ -46,6 +46,9 @@ dependencies:
video_player: ^2.9.1
flutter_riverpod: ^2.5.1
speech_to_text: ^6.6.0
camera: ^0.11.0+1
image_picker: ^1.1.2
file_picker: ^8.0.6
dev_dependencies:
flutter_test:
sdk: flutter
@@ -6,10 +6,13 @@
#include "generated_plugin_registrant.h"
#include <file_selector_windows/file_selector_windows.h>
#include <rive_common/rive_plugin.h>
#include <url_launcher_windows/url_launcher_windows.h>
void RegisterPlugins(flutter::PluginRegistry* registry) {
FileSelectorWindowsRegisterWithRegistrar(
registry->GetRegistrarForPlugin("FileSelectorWindows"));
RivePluginRegisterWithRegistrar(
registry->GetRegistrarForPlugin("RivePlugin"));
UrlLauncherWindowsRegisterWithRegistrar(
@@ -3,6 +3,7 @@
#
list(APPEND FLUTTER_PLUGIN_LIST
file_selector_windows
rive_common
url_launcher_windows
)