From c24171aec9dcaedad4343bbb99eea1c560f0d32d Mon Sep 17 00:00:00 2001 From: Ashton Date: Sat, 8 Jun 2024 16:53:29 -0600 Subject: [PATCH] init commit & proof of concept --- .gitignore | 1 + README.md | 7 ++ requirements.txt | 267 +++++++++++++++++++++++++++++++++++++++++++++++ stt.py | 72 +++++++++++++ 4 files changed, 347 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 requirements.txt create mode 100644 stt.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0a764a4 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +env diff --git a/README.md b/README.md new file mode 100644 index 0000000..5214483 --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +Dependencies: +pip install scipy pydub torch torchaudio wavio silero sounddevice; + +Create output.txt file, TODO: Implement error handling. +touch output.txt; + +Currently, you have 5 seconds to record a prompt. It will automatically place "in 50 words or less" at the start of your prompt for faster testing. The Text To Speech (TTS) engine is not great quality, at least at first. Just getting the basic structure together, more to come... diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..fd5ad5d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,267 @@ +aiodns==3.0.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +antlr4-python3-runtime==4.9.3 +anyio==3.7.1 +appdirs==1.4.4 +argcomplete==3.3.0 +argon2-cffi==21.3.0 +argon2-cffi-bindings==21.2.0 +arrow==1.2.3 +asttokens==2.4.1 +async-lru==2.0.4 +attrs==23.2.0 +Babel==2.15.0 +backcall==0.1.0 +Beaker==1.12.1 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blivet==3.10.0 +blivet-gui==2.5.0 +blosc2==2.5.1 +boto3==1.34.99 +botocore==1.34.99 +Bottleneck==1.3.7 +Brlapi==0.8.5 +Brotli==1.1.0 +cachetools==5.3.2 +certifi==2023.5.7 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +click-plugins==1.1.1 +colorama==0.4.6 +comm==0.2.1 +contourpy==1.2.0 +cryptography==41.0.7 +cssselect==1.1.0 +cupshelpers==1.0 +cycler==0.11.0 +dasbus==1.7 +dbus-python==1.3.2 +decorator==5.1.1 +defusedxml==0.7.1 +distro==1.9.0 +entrypoints==0.4 +et-xmlfile==1.1.0 +executing==2.0.1 +fastjsonschema==2.18.0 +file-magic==0.4.0 +filelock==3.14.0 +fonttools==4.50.0 +fqdn==1.5.1 +fros==1.1 +frozenlist==1.4.1 +fs==2.4.16 +fsspec==2024.3.1 +gcsfs==2023.6.0+1.g7cc53d9 +google-api-core==2.11.1 +google-auth==2.29.0 +google-auth-oauthlib==0.8.0 +google-cloud-core==2.3.3 +google-cloud-storage==2.14.0 +google-crc32c==1.5.0 +google-resumable-media==2.7.0 +googleapis-common-protos==1.63.0 +gpg==1.23.2 +greenlet==3.0.3 +grpcio==1.48.4 +grpcio-status==1.48.4 +h11==0.14.0 +html5lib==1.1 +httpcore==1.0.2 +httpx==0.26.0 +humanize==3.13.1 +idna==3.7 +inkex==1.3.1 +ipykernel==6.29.3 +ipython==8.23.0 +ipython-genutils==0.2.0 +iso639==0.1.4 +isoduration==20.11.0 +jdcal==1.4.1 +jedi==0.19.1 +Jinja2==3.1.4 +jmespath==1.0.1 +json5==0.9.24 +jsonpointer==2.3 +jsonschema==4.19.1 +jsonschema-specifications==2023.11.2 +jupyter-events==0.9.0 +jupyter-lsp==2.2.2 +jupyter_client==7.4.9 +jupyter_core==5.1.0 +jupyter_server==2.12.5 +jupyter_server_terminals==0.4.2 +jupyterlab==4.1.0 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.25.2 +kiwisolver==1.4.5 +langtable==0.0.66 +libcomps==0.1.20 +louis==3.28.0 +lxml==5.1.0 +Mako==1.2.3 +MarkupSafe==2.1.3 +matplotlib==3.8.4 +matplotlib-inline==0.1.6 +mistune==2.0.4 +mpmath==1.3.0 +msgpack==1.0.6 +multidict==6.0.4 +munkres==1.1.2 +nbclassic==0.5.6 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +ndindex==1.7 +nest-asyncio==1.6.0 +netifaces==0.11.0 +networkx==3.3 +nftables==0.1 +notebook==7.0.7 +notebook_shim==0.2.3 +numexpr==2.8.5 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==8.9.2.26 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.40 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +odfpy==1.4.1 +olefile==0.47 +omegaconf==2.3.0 +openpyxl==3.1.2 +packaging==23.2 +pandas==2.2.1 +pandocfilters==1.5.1 +parso==0.8.3 +Paste==3.7.1 +pexpect==4.9.0 +pickleshare==0.7.5 +pid==2.2.3 +pillow==10.3.0 +platformdirs==3.11.0 +ply==3.11 +pooch==1.5.2 +productmd==1.38 +progressbar2==3.53.2 +prometheus-client==0.19.0 +prompt-toolkit==3.0.41 +protobuf==3.19.6 +psutil==5.9.8 +psycopg2==2.9.9 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pwquality==1.4.5 +py-cpuinfo==9.0.0 +pyarrow==15.0.2 +pyasn1==0.5.1 +pyasn1-modules==0.3.0 +pycairo==1.25.1 +pycares==4.3.0 +pycparser==2.20 +pycrypto==2.6.1 +pycups==2.0.4 +pycurl==7.45.2 +pydub==0.25.1 +pyenchant==3.2.2 +pygit2==1.14.0 +Pygments==2.17.2 +PyGObject==3.48.2 +pykickstart==3.52 +PyMySQL==1.1.0 +PyOpenGL==3.1.7 +PyOpenGL-accelerate==3.1.7 +pyOpenSSL==23.2.0 +pyparsing==3.1.2 +pyparted==3.13.0 +PyQt5==5.15.10 +PyQt5-sip==12.13.0 +pyqtgraph==0.13.3 +pyrsistent==0.20.0 +pyserial==3.5 +PySocks==1.7.1 +python-augeas==1.1.0 +python-dateutil==2.8.2 +python-json-logger==2.0.4 +python-meh==0.51 +python-pam==2.0.2 +python-snappy==0.6.1 +python-utils==3.7.0 +pytz==2024.1 +pyudev==0.24.1 +pyxdg==0.27 +PyYAML==6.0.1 +pyzmq==25.1.1 +QtPy==2.4.1 +referencing==0.31.1 +regex==2024.4.16 +requests==2.31.0 +requests-file==2.0.0 +requests-ftp==0.3.1 +requests-oauthlib==1.3.1 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rpds-py==0.18.0 +rpm==4.19.1.1 +rpmautospec==0.6.3 +rpmautospec-core==0.1.4 +rsa==4.9 +s3transfer==0.10.1 +SciPy==1.11.3 +scour==0.38.2 +selinux @ file:///builddir/build/BUILD/libselinux-3.6/src +Send2Trash==1.8.2 +sepolicy @ file:///builddir/build/BUILD/selinux-3.6/python/sepolicy +setools==4.5.1 +setuptools==69.0.3 +shtab==1.6.1 +silero==0.4.1 +simpleaudio==1.0.4 +simpleline==1.9.0 +six==1.16.0 +sniffio==1.3.0 +sos==4.7.1 +sounddevice==0.4.6 +soupsieve==2.5 +SQLAlchemy==2.0.30 +stack-data==0.6.3 +sympy==1.12 +systemd-python==235 +tables==3.9.2 +tabulate==0.9.0 +Tempita==0.5.2 +termcolor==2.3.0 +terminado==0.18.0 +thrift==0.15.0 +tinycss2==1.2.1 +tldr==3.2.0 +torch==2.3.0 +torchaudio==2.3.0 +tornado==6.3.3 +traitlets==5.14.1 +typing_extensions==4.9.0 +uri-template==1.2.0 +urllib3==1.26.18 +wavio==0.0.9 +wcwidth==0.2.13 +webcolors==1.13 +webencodings==0.5.1 +websocket-client==1.3.3 +xarray==2023.8.0 +xkbregistry==0.3 +xlrd==2.0.1 +XlsxWriter==3.1.9 +xlwt==1.3.0 +yarl==1.9.2 +zstandard==0.22.0 diff --git a/stt.py b/stt.py new file mode 100644 index 0000000..323ad20 --- /dev/null +++ b/stt.py @@ -0,0 +1,72 @@ +# import required libraries +from glob import glob +from scipy.io.wavfile import write +from pydub import AudioSegment +from pydub.playback import play + +import os +import torch +import zipfile +import torchaudio +import sounddevice as sd +import wave +import wavio as wv + +########### Record audio +def record_audio(audio, duration=5, freq=192000): + # Start recorder with the given values of duration and sample frequency + recording = sd.rec(int(duration * freq), samplerate=freq, channels=2) + + print('Please say your prompt. You have ' + str(duration) + ' seconds.') + # Record audio for the given number of seconds + sd.wait() + + # This will convert the NumPy array to an audio file with the given sampling frequency + write(audio, freq, recording) + +def speech_to_text(audio): + device = torch.device('cuda') # cuda also works, but our models are fast enough for CPU + model, decoder, utils = torch.hub.load(repo_or_dir='snakers4/silero-models', model='silero_stt', language='en', device=device) + (read_batch, split_into_batches, read_audio, prepare_model_input) = utils # see function signature for details + + test_files = glob(audio) + batches = split_into_batches(test_files, batch_size=100) + input = prepare_model_input(read_batch(batches[0]), device=device) + + text = "" + output = model(input) + for example in output: + text += decoder(example.cpu()) + + return text + +def text_to_speech(): + language = 'en' + model_id = 'v3_en' + sample_rate = 8000 + speaker = 'en_1' + device = torch.device('cuda') + + model, example_text = torch.hub.load(repo_or_dir='snakers4/silero-models', model='silero_tts', language=language, speaker=model_id) + model.to(device) # cuda or cpu + + text = open("output.txt").read() + audio = model.apply_tts(text, speaker=speaker, sample_rate=sample_rate) + torchaudio.save("output.mp3", audio.unsqueeze(0), sample_rate=8000) + + song = AudioSegment.from_mp3("output.mp3") + play(song) + +def main(): + audio = "input.wav" + record_audio(audio) + prompt = "in 50 words or less, " + speech_to_text(audio) + print(prompt) + + command = "ollama run llama3 " + prompt + " > output.txt" + os.system(command) + text_to_speech() + + +if __name__ == "__main__": + main()