Updates
This commit is contained in:
124
main.py
124
main.py
@@ -7,7 +7,9 @@ import sys
|
|||||||
import os
|
import os
|
||||||
import yaml
|
import yaml
|
||||||
import threading
|
import threading
|
||||||
|
import logging
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).parent))
|
sys.path.insert(0, str(Path(__file__).parent))
|
||||||
|
|
||||||
@@ -34,38 +36,57 @@ _NOISE_PHRASES = (
|
|||||||
|
|
||||||
|
|
||||||
class MeetingAssistant:
|
class MeetingAssistant:
|
||||||
def __init__(self, config_path="config.yaml"):
|
def __init__(self):
|
||||||
|
"""Initialize all components"""
|
||||||
print("🚀 Starting Meeting Assistant...")
|
print("🚀 Starting Meeting Assistant...")
|
||||||
|
|
||||||
config_file = Path(__file__).parent / config_path
|
|
||||||
with open(config_file, "r") as f:
|
|
||||||
self.config = yaml.safe_load(f)
|
|
||||||
|
|
||||||
self.logger = setup_logging()
|
|
||||||
print(f"💻 Platform: {get_platform()}")
|
print(f"💻 Platform: {get_platform()}")
|
||||||
|
|
||||||
self.context_manager = ContextManager(self.config)
|
# Load config
|
||||||
|
config_path = Path(__file__).parent / "config.yaml"
|
||||||
|
with open(config_path, 'r') as f:
|
||||||
|
self.config = yaml.safe_load(f)
|
||||||
|
|
||||||
|
# Setup logging
|
||||||
|
self.logger = setup_logging()
|
||||||
|
|
||||||
|
# Initialize components
|
||||||
self.ai_engine = AIEngine(self.config)
|
self.ai_engine = AIEngine(self.config)
|
||||||
|
self.context_manager = ContextManager(self.config)
|
||||||
self.overlay = InvisibleOverlay(self.config)
|
self.overlay = InvisibleOverlay(self.config)
|
||||||
|
|
||||||
|
# Audio listener with callback
|
||||||
self.audio_listener = AudioListener(self.config, self.on_audio_transcript)
|
self.audio_listener = AudioListener(self.config, self.on_audio_transcript)
|
||||||
|
|
||||||
self._last_question = "" # Simple debounce
|
# State management
|
||||||
|
self.current_answer = None
|
||||||
|
self.answering = False
|
||||||
|
self.answer_lock = threading.Lock()
|
||||||
|
self._last_question = ""
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
print("✅ Meeting Assistant Ready!")
|
||||||
# Audio callback — runs in the audio listener thread
|
print("==================================================")
|
||||||
# ------------------------------------------------------------------
|
print("🎤 Listening for questions via microphone")
|
||||||
|
print(" Answers appear in the overlay (top-right)")
|
||||||
|
print(" Type a question here + Enter to test AI")
|
||||||
|
print(" Ctrl+C to stop")
|
||||||
|
print("==================================================")
|
||||||
|
|
||||||
def on_audio_transcript(self, text, timestamp):
|
def on_audio_transcript(self, text, timestamp):
|
||||||
|
"""Audio callback — runs in the audio listener thread"""
|
||||||
|
if not text or len(text.strip()) < 2:
|
||||||
|
return
|
||||||
|
|
||||||
self.context_manager.add_audio_context(text, timestamp)
|
self.context_manager.add_audio_context(text, timestamp)
|
||||||
|
|
||||||
if self._is_question(text):
|
if self._is_question(text):
|
||||||
print(f"❓ Question: {text}")
|
print(f"\n❓ Question detected: {text}")
|
||||||
self._generate_answer(text)
|
self._generate_answer(text)
|
||||||
else:
|
else:
|
||||||
print(f" Context: {text}")
|
print(f" Context (not a question): {text}")
|
||||||
|
|
||||||
def _is_question(self, text):
|
def _is_question(self, text):
|
||||||
if len(text.split()) < 3:
|
"""Enhanced question detection for faster responses"""
|
||||||
|
if len(text.split()) < 2: # Reduced from 3 for shorter questions
|
||||||
return False
|
return False
|
||||||
|
|
||||||
text_lower = text.lower().strip()
|
text_lower = text.lower().strip()
|
||||||
@@ -78,21 +99,78 @@ class MeetingAssistant:
|
|||||||
if text.rstrip().endswith("?"):
|
if text.rstrip().endswith("?"):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
# Check for math expressions (2+2, two plus two)
|
||||||
|
import re
|
||||||
|
math_patterns = [
|
||||||
|
r'\d+\s*[\+\-\*\/]\s*\d+', # 2+2, 5-3
|
||||||
|
r'\d+\s*plus\s*\d+', # 2 plus 2
|
||||||
|
r'\d+\s*minus\s*\d+', # 5 minus 3
|
||||||
|
r'\d+\s*times\s*\d+', # 4 times 5
|
||||||
|
r'\d+\s*divided by\s*\d+', # 10 divided by 2
|
||||||
|
]
|
||||||
|
for pattern in math_patterns:
|
||||||
|
if re.search(pattern, text_lower):
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Check for short implicit questions (2-5 words without filler)
|
||||||
|
words = text_lower.split()
|
||||||
|
if 2 <= len(words) <= 5:
|
||||||
|
filler = {'um', 'uh', 'like', 'so', 'well', 'actually', 'basically'}
|
||||||
|
content_words = [w for w in words if w not in filler]
|
||||||
|
if len(content_words) >= 2:
|
||||||
|
# "two plus two", "capital france", "python example"
|
||||||
|
return True
|
||||||
|
|
||||||
# Starts with or contains a question word
|
# Starts with or contains a question word
|
||||||
return any(text_lower.startswith(w) or f" {w} " in text_lower for w in _QUESTION_WORDS)
|
if any(text_lower.startswith(w) or f" {w} " in text_lower for w in _QUESTION_WORDS):
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Check for command starters
|
||||||
|
command_starters = ['tell me', 'explain', 'describe', 'show me', 'give me', 'find', 'search']
|
||||||
|
for starter in command_starters:
|
||||||
|
if text_lower.startswith(starter):
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
def _generate_answer(self, question):
|
def _generate_answer(self, question):
|
||||||
|
"""Generate answer with interruption support"""
|
||||||
# Debounce: skip exact repeat within same run
|
# Debounce: skip exact repeat within same run
|
||||||
if question.lower() == self._last_question.lower():
|
if question.lower() == self._last_question.lower():
|
||||||
return
|
return
|
||||||
self._last_question = question
|
self._last_question = question
|
||||||
|
|
||||||
|
# Check if we should interrupt current answer
|
||||||
|
with self.answer_lock:
|
||||||
|
if self.answering:
|
||||||
|
print("🔄 Interrupting previous answer...")
|
||||||
|
if hasattr(self.ai_engine, 'interrupt'):
|
||||||
|
self.ai_engine.interrupt()
|
||||||
|
|
||||||
|
# Start answer generation in background thread
|
||||||
|
threading.Thread(target=self._answer_worker, args=(question,), daemon=True).start()
|
||||||
|
|
||||||
|
def _answer_worker(self, question):
|
||||||
|
"""Worker thread for answer generation"""
|
||||||
|
with self.answer_lock:
|
||||||
|
self.answering = True
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Get context and generate answer
|
||||||
context = self.context_manager.get_context()
|
context = self.context_manager.get_context()
|
||||||
answer = self.ai_engine.answer_question(question, context)
|
answer = self.ai_engine.answer_question(question, context)
|
||||||
|
|
||||||
|
# Show answer in overlay
|
||||||
self.overlay.show_answer(answer, question)
|
self.overlay.show_answer(answer, question)
|
||||||
self.logger.info(f"Q: {question}")
|
self.logger.info(f"Q: {question}")
|
||||||
self.logger.info(f"A: {answer}")
|
self.logger.info(f"A: {answer}")
|
||||||
print(f"💡 Answer: {answer}\n")
|
print(f"\n💡 Answer: {answer}\n")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"❌ Error generating answer: {e}")
|
||||||
|
finally:
|
||||||
|
with self.answer_lock:
|
||||||
|
self.answering = False
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
# Manual terminal input for testing
|
# Manual terminal input for testing
|
||||||
@@ -114,14 +192,7 @@ class MeetingAssistant:
|
|||||||
# ------------------------------------------------------------------
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
print("\n✅ Meeting Assistant Ready!")
|
print("\n🎤 Starting audio listener...")
|
||||||
print("=" * 50)
|
|
||||||
print("🎤 Listening for questions via microphone")
|
|
||||||
print(" Answers appear in the overlay (top-right)")
|
|
||||||
print(" Type a question here + Enter to test AI")
|
|
||||||
print(" Ctrl+C to stop")
|
|
||||||
print("=" * 50)
|
|
||||||
|
|
||||||
self.overlay.start()
|
self.overlay.start()
|
||||||
self.audio_listener.start()
|
self.audio_listener.start()
|
||||||
|
|
||||||
@@ -137,9 +208,12 @@ class MeetingAssistant:
|
|||||||
|
|
||||||
def shutdown(self):
|
def shutdown(self):
|
||||||
print("\n🛑 Shutting down...")
|
print("\n🛑 Shutting down...")
|
||||||
|
if hasattr(self, 'audio_listener'):
|
||||||
self.audio_listener.stop()
|
self.audio_listener.stop()
|
||||||
|
if hasattr(self, 'overlay'):
|
||||||
self.overlay.stop()
|
self.overlay.stop()
|
||||||
print("👋 Goodbye!")
|
print("👋 Goodbye!")
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
|||||||
184
src/ai_engine.py
184
src/ai_engine.py
@@ -4,14 +4,17 @@ from pathlib import Path
|
|||||||
from collections import deque
|
from collections import deque
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from rapidfuzz import fuzz
|
from rapidfuzz import fuzz
|
||||||
|
import threading
|
||||||
|
|
||||||
|
|
||||||
class AIEngine:
|
class AIEngine:
|
||||||
def __init__(self, config):
|
def __init__(self, config):
|
||||||
self.config = config
|
self.config = config
|
||||||
self.model = None
|
self.model = None
|
||||||
|
|
||||||
self.memory = deque(maxlen=8)
|
self.memory = deque(maxlen=8)
|
||||||
|
self.fast_mode = True
|
||||||
|
self.current_generation = None
|
||||||
|
self.interrupt_event = threading.Event()
|
||||||
self.load_model()
|
self.load_model()
|
||||||
|
|
||||||
def load_model(self):
|
def load_model(self):
|
||||||
@@ -28,29 +31,46 @@ class AIEngine:
|
|||||||
self.model = Llama(
|
self.model = Llama(
|
||||||
model_path=str(model_path),
|
model_path=str(model_path),
|
||||||
|
|
||||||
# PERFORMANCE
|
# PERFORMANCE - MAXIMUM SPEED
|
||||||
n_gpu_layers=-1,
|
n_gpu_layers=-1,
|
||||||
n_ctx=4096,
|
n_ctx=2048, # Reduced from 4096 for speed
|
||||||
n_batch=1024,
|
n_batch=512, # Reduced for faster prompt processing
|
||||||
n_threads=max(os.cpu_count() - 1, 1),
|
n_threads=max(os.cpu_count() - 1, 1),
|
||||||
|
|
||||||
# SPEED
|
# SPEED OPTIMIZATIONS
|
||||||
offload_kqv=True,
|
offload_kqv=True,
|
||||||
flash_attn=True,
|
flash_attn=True,
|
||||||
use_mmap=True,
|
use_mmap=True,
|
||||||
use_mlock=False,
|
use_mlock=False,
|
||||||
|
|
||||||
|
# FASTER INFERENCE
|
||||||
|
n_parts=-1,
|
||||||
|
seed=42, # Deterministic for speed
|
||||||
|
f16_kv=True, # Use half-precision for KV cache
|
||||||
|
|
||||||
# STABILITY
|
# STABILITY
|
||||||
verbose=False
|
verbose=False
|
||||||
)
|
)
|
||||||
|
|
||||||
print("✅ AI ready")
|
print("✅ AI ready (fast mode enabled)")
|
||||||
|
|
||||||
|
def answer_question(self, question, context, is_partial=False):
|
||||||
|
"""Answer a question with interruption support"""
|
||||||
|
|
||||||
|
# Check for interrupt
|
||||||
|
if self.interrupt_event.is_set():
|
||||||
|
self.interrupt_event.clear()
|
||||||
|
return None
|
||||||
|
|
||||||
def answer_question(self, question, context):
|
|
||||||
question = self._normalize_question(question)
|
question = self._normalize_question(question)
|
||||||
|
|
||||||
|
# For partial questions, answer faster with fewer tokens
|
||||||
|
if is_partial:
|
||||||
|
response = self._generate_fast(question, context)
|
||||||
|
else:
|
||||||
response = self._generate(question, context)
|
response = self._generate(question, context)
|
||||||
|
|
||||||
|
if response: # Only store if not interrupted
|
||||||
self.memory.append({
|
self.memory.append({
|
||||||
"question": question,
|
"question": question,
|
||||||
"response": response,
|
"response": response,
|
||||||
@@ -59,6 +79,11 @@ class AIEngine:
|
|||||||
|
|
||||||
return response
|
return response
|
||||||
|
|
||||||
|
def interrupt(self):
|
||||||
|
"""Interrupt current generation"""
|
||||||
|
self.interrupt_event.set()
|
||||||
|
print("🛑 Generation interrupted")
|
||||||
|
|
||||||
def _normalize_question(self, question):
|
def _normalize_question(self, question):
|
||||||
question = question.strip()
|
question = question.strip()
|
||||||
|
|
||||||
@@ -70,12 +95,10 @@ class AIEngine:
|
|||||||
}
|
}
|
||||||
|
|
||||||
words = question.split()
|
words = question.split()
|
||||||
|
|
||||||
normalized = []
|
normalized = []
|
||||||
|
|
||||||
for word in words:
|
for word in words:
|
||||||
lowered = word.lower()
|
lowered = word.lower()
|
||||||
|
|
||||||
if lowered in fixes:
|
if lowered in fixes:
|
||||||
normalized.append(fixes[lowered])
|
normalized.append(fixes[lowered])
|
||||||
else:
|
else:
|
||||||
@@ -91,74 +114,106 @@ class AIEngine:
|
|||||||
|
|
||||||
return question
|
return question
|
||||||
|
|
||||||
def _generate(self, question, context):
|
def _generate_fast(self, question, context):
|
||||||
recent_audio = context.get("audio", "")[-1500:]
|
"""Ultra-fast generation for partial/interruptible responses"""
|
||||||
recent_screen = context.get("screen", "")[-700:]
|
|
||||||
|
|
||||||
memory_context = ""
|
# Check for interrupt before generation
|
||||||
|
if self.interrupt_event.is_set():
|
||||||
|
self.interrupt_event.clear()
|
||||||
|
return None
|
||||||
|
|
||||||
if self.memory:
|
# Simplified prompt for speed
|
||||||
recent = list(self.memory)[-2:]
|
system_prompt = "You are a fast assistant. Answer in 1-2 short sentences max."
|
||||||
|
|
||||||
for item in recent:
|
user_prompt = f"Q: {question}\nA:"
|
||||||
memory_context += (
|
|
||||||
f"Q: {item['question']}\n"
|
|
||||||
f"A: {item['response']}\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
system_prompt = """
|
|
||||||
You are a realtime meeting copilot.
|
|
||||||
|
|
||||||
Rules:
|
|
||||||
- Answer immediately.
|
|
||||||
- Never ask for clarification unless impossible.
|
|
||||||
- Never invent missing words.
|
|
||||||
- Never complete partial speech.
|
|
||||||
- Keep answers under 80 words.
|
|
||||||
- Prioritize only explicit meaning.
|
|
||||||
- Ignore minor transcription mistakes.
|
|
||||||
- If speech is incomplete, answer conservatively.
|
|
||||||
- Sound precise and direct.
|
|
||||||
"""
|
|
||||||
|
|
||||||
user_prompt = f"""
|
|
||||||
Recent conversation:
|
|
||||||
{memory_context}
|
|
||||||
|
|
||||||
Meeting transcript:
|
|
||||||
{recent_audio}
|
|
||||||
|
|
||||||
Screen:
|
|
||||||
{recent_screen}
|
|
||||||
|
|
||||||
User question:
|
|
||||||
{question}
|
|
||||||
"""
|
|
||||||
|
|
||||||
prompt = (
|
prompt = (
|
||||||
f"<|im_start|>system\n"
|
f"<|im_start|>system\n{system_prompt}<|im_end|>\n"
|
||||||
f"{system_prompt}"
|
f"<|im_start|>user\n{user_prompt}<|im_end|>\n"
|
||||||
f"<|im_end|>\n"
|
|
||||||
f"<|im_start|>user\n"
|
|
||||||
f"{user_prompt}"
|
|
||||||
f"<|im_end|>\n"
|
|
||||||
f"<|im_start|>assistant\n"
|
f"<|im_start|>assistant\n"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
output = self.model(
|
output = self.model(
|
||||||
prompt,
|
prompt,
|
||||||
|
max_tokens=30, # Very short for fast responses
|
||||||
max_tokens=120,
|
temperature=0.1,
|
||||||
temperature=0.2,
|
top_p=0.9,
|
||||||
top_p=0.85,
|
repeat_penalty=1.0,
|
||||||
repeat_penalty=1.05,
|
stop=["<|im_end|>", "\n", ".", "!", "?"],
|
||||||
stop=["<|im_end|>"]
|
echo=False
|
||||||
)
|
)
|
||||||
|
|
||||||
text = output["choices"][0]["text"]
|
# Check for interrupt during generation
|
||||||
|
if self.interrupt_event.is_set():
|
||||||
|
self.interrupt_event.clear()
|
||||||
|
return None
|
||||||
|
|
||||||
|
text = output["choices"][0]["text"]
|
||||||
return self._clean(text)
|
return self._clean(text)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"⚠️ Fast generation error: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _generate(self, question, context):
|
||||||
|
"""Standard generation for complete questions"""
|
||||||
|
|
||||||
|
# Check for interrupt before generation
|
||||||
|
if self.interrupt_event.is_set():
|
||||||
|
self.interrupt_event.clear()
|
||||||
|
return None
|
||||||
|
|
||||||
|
recent_audio = context.get("audio", "")[-1500:] if context else ""
|
||||||
|
recent_screen = context.get("screen", "")[-700:] if context else ""
|
||||||
|
|
||||||
|
memory_context = ""
|
||||||
|
if self.memory:
|
||||||
|
recent = list(self.memory)[-2:]
|
||||||
|
for item in recent:
|
||||||
|
memory_context += f"Q: {item['question']}\nA: {item['response']}\n"
|
||||||
|
|
||||||
|
# Ultra-concise system prompt for speed
|
||||||
|
system_prompt = """
|
||||||
|
You are a realtime assistant. Answer immediately and concisely.
|
||||||
|
Keep answers under 40 words. Be direct. No explanations unless asked.
|
||||||
|
"""
|
||||||
|
|
||||||
|
user_prompt = f"""
|
||||||
|
Q: {question}
|
||||||
|
A:"""
|
||||||
|
|
||||||
|
prompt = (
|
||||||
|
f"<|im_start|>system\n{system_prompt}<|im_end|>\n"
|
||||||
|
f"<|im_start|>user\n{user_prompt}<|im_end|>\n"
|
||||||
|
f"<|im_start|>assistant\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
output = self.model(
|
||||||
|
prompt,
|
||||||
|
max_tokens=60, # Reduced for speed
|
||||||
|
temperature=0.1, # Lower for faster, more deterministic
|
||||||
|
top_p=0.9,
|
||||||
|
repeat_penalty=1.0, # Disabled for speed
|
||||||
|
stop=["<|im_end|>", "\n", "."],
|
||||||
|
echo=False,
|
||||||
|
frequency_penalty=0.0,
|
||||||
|
presence_penalty=0.0
|
||||||
|
)
|
||||||
|
|
||||||
|
# Check for interrupt during generation
|
||||||
|
if self.interrupt_event.is_set():
|
||||||
|
self.interrupt_event.clear()
|
||||||
|
return None
|
||||||
|
|
||||||
|
text = output["choices"][0]["text"]
|
||||||
|
return self._clean(text)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"⚠️ Generation error: {e}")
|
||||||
|
return "I couldn't process that."
|
||||||
|
|
||||||
def _clean(self, text):
|
def _clean(self, text):
|
||||||
text = re.sub(r"<.*?>", "", text)
|
text = re.sub(r"<.*?>", "", text)
|
||||||
text = re.sub(r"\s+", " ", text).strip()
|
text = re.sub(r"\s+", " ", text).strip()
|
||||||
@@ -166,7 +221,12 @@ User question:
|
|||||||
if not text:
|
if not text:
|
||||||
return "No response generated."
|
return "No response generated."
|
||||||
|
|
||||||
|
# Ensure it ends with punctuation
|
||||||
if text[-1] not in ".!?":
|
if text[-1] not in ".!?":
|
||||||
text += "."
|
text += "."
|
||||||
|
|
||||||
|
# Capitalize first letter
|
||||||
|
if text and text[0].islower():
|
||||||
|
text = text[0].upper() + text[1:]
|
||||||
|
|
||||||
return text
|
return text
|
||||||
@@ -19,23 +19,22 @@ from faster_whisper import WhisperModel
|
|||||||
os.environ["HF_HUB_DISABLE_SSL_VERIFY"] = "1"
|
os.environ["HF_HUB_DISABLE_SSL_VERIFY"] = "1"
|
||||||
|
|
||||||
# ============================================================
|
# ============================================================
|
||||||
# 🔧 TUNING
|
# 🔧 TUNING - OPTIMIZED FOR SPEED
|
||||||
# ============================================================
|
# ============================================================
|
||||||
MIN_SPEECH_SECONDS = 1.2 # Need at least 1.2s of speech
|
MIN_SPEECH_SECONDS = 0.8
|
||||||
SILENCE_SECONDS = 0.7 # 0.7s silence to end
|
SILENCE_SECONDS = 0.3
|
||||||
MAX_SPEECH_SECONDS = 8.0 # Max before force
|
MAX_SPEECH_SECONDS = 8.0
|
||||||
SPEECH_THRESHOLD_MULTIPLIER = 1.3
|
SPEECH_THRESHOLD_MULTIPLIER = 1.3
|
||||||
MIN_THRESHOLD_GAP = 400
|
MIN_THRESHOLD_GAP = 400
|
||||||
CONSECUTIVE_SPEECH_TO_START = 8 # Must have 8 consecutive speech frames to start
|
CONSECUTIVE_SPEECH_TO_START = 4
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
# ============================================================
|
||||||
|
|
||||||
|
|
||||||
class AudioListener:
|
class AudioListener:
|
||||||
def __init__(self, config, callback):
|
def __init__(self, config, callback):
|
||||||
self.config = config
|
self.config = config
|
||||||
self.callback = callback
|
self.callback = callback # Expects callback(text, timestamp)
|
||||||
self.running = False
|
self.running = False
|
||||||
|
|
||||||
self.sample_rate = 16000
|
self.sample_rate = 16000
|
||||||
@@ -52,8 +51,8 @@ class AudioListener:
|
|||||||
self.is_speaking = False
|
self.is_speaking = False
|
||||||
self.speech_frames = 0
|
self.speech_frames = 0
|
||||||
self.silence_frames = 0
|
self.silence_frames = 0
|
||||||
self.consecutive_speech = 0 # Consecutive speech frames
|
self.consecutive_speech = 0
|
||||||
self.consecutive_silence = 0 # Consecutive silence frames
|
self.consecutive_silence = 0
|
||||||
self.total_frames_in_utterance = 0
|
self.total_frames_in_utterance = 0
|
||||||
|
|
||||||
self.noise_floor = 0
|
self.noise_floor = 0
|
||||||
@@ -66,11 +65,12 @@ class AudioListener:
|
|||||||
self.speech_start_time = 0
|
self.speech_start_time = 0
|
||||||
|
|
||||||
self.last_transcription_time = 0
|
self.last_transcription_time = 0
|
||||||
self.min_transcription_interval = 0.5
|
self.min_transcription_interval = 0.3
|
||||||
|
|
||||||
self.calibrated = False
|
self.calibrated = False
|
||||||
|
|
||||||
print("🔄 Loading Whisper model...")
|
print("🔄 Loading Whisper model...")
|
||||||
|
try:
|
||||||
self.model = WhisperModel(
|
self.model = WhisperModel(
|
||||||
"models/whisper",
|
"models/whisper",
|
||||||
device="cpu",
|
device="cpu",
|
||||||
@@ -79,21 +79,15 @@ class AudioListener:
|
|||||||
cpu_threads=max(os.cpu_count() - 1, 1)
|
cpu_threads=max(os.cpu_count() - 1, 1)
|
||||||
)
|
)
|
||||||
print("✅ Whisper model ready")
|
print("✅ Whisper model ready")
|
||||||
|
except Exception as e:
|
||||||
self.command_starters = {
|
print(f"⚠️ Could not load Whisper from models/whisper, trying default: {e}")
|
||||||
"what", "why", "how", "when", "where", "who", "which",
|
self.model = WhisperModel(
|
||||||
"can", "could", "would", "will", "do", "does", "did",
|
"base",
|
||||||
"is", "are", "was", "were", "should", "shall", "may",
|
device="cpu",
|
||||||
"have", "has", "had", "am",
|
compute_type="int8",
|
||||||
"tell", "explain", "describe", "compare", "define",
|
cpu_threads=max(os.cpu_count() - 1, 1)
|
||||||
"find", "show", "give", "list", "name", "provide",
|
)
|
||||||
"write", "create", "make", "generate", "build", "code",
|
print("✅ Whisper model ready (using default 'base' model)")
|
||||||
"calculate", "compute", "solve", "convert", "translate",
|
|
||||||
"search", "look", "check", "get", "fetch", "run",
|
|
||||||
"start", "stop", "open", "close", "save", "delete",
|
|
||||||
"draw", "plot", "graph", "print", "display", "output",
|
|
||||||
"summarize", "analyze", "review", "evaluate",
|
|
||||||
}
|
|
||||||
|
|
||||||
def start(self):
|
def start(self):
|
||||||
self.running = True
|
self.running = True
|
||||||
@@ -129,7 +123,7 @@ class AudioListener:
|
|||||||
rms = np.sqrt(np.mean(chunk ** 2))
|
rms = np.sqrt(np.mean(chunk ** 2))
|
||||||
rms_values.append(rms)
|
rms_values.append(rms)
|
||||||
|
|
||||||
self.noise_floor = np.median(rms_values)
|
self.noise_floor = np.median(rms_values) if rms_values else 1500
|
||||||
|
|
||||||
self.speech_threshold = max(
|
self.speech_threshold = max(
|
||||||
self.noise_floor * SPEECH_THRESHOLD_MULTIPLIER,
|
self.noise_floor * SPEECH_THRESHOLD_MULTIPLIER,
|
||||||
@@ -150,7 +144,7 @@ class AudioListener:
|
|||||||
self.silence_threshold = 1700
|
self.silence_threshold = 1700
|
||||||
self.calibrated = True
|
self.calibrated = True
|
||||||
|
|
||||||
print(f"\n📋 Settings:")
|
print(f"\n📋 Settings (FAST MODE):")
|
||||||
print(f" Min speech: {MIN_SPEECH_SECONDS}s | Max: {MAX_SPEECH_SECONDS}s")
|
print(f" Min speech: {MIN_SPEECH_SECONDS}s | Max: {MAX_SPEECH_SECONDS}s")
|
||||||
print(f" Silence to end: {SILENCE_SECONDS}s")
|
print(f" Silence to end: {SILENCE_SECONDS}s")
|
||||||
print(f" Consecutive speech to start: {CONSECUTIVE_SPEECH_TO_START} frames")
|
print(f" Consecutive speech to start: {CONSECUTIVE_SPEECH_TO_START} frames")
|
||||||
@@ -199,29 +193,23 @@ class AudioListener:
|
|||||||
self.audio_queue.put((audio_bytes, rms))
|
self.audio_queue.put((audio_bytes, rms))
|
||||||
|
|
||||||
def _processing_loop(self):
|
def _processing_loop(self):
|
||||||
# Pre-speech buffer - capture audio BEFORE we confirm speech
|
|
||||||
pre_speech_buffer = bytearray()
|
pre_speech_buffer = bytearray()
|
||||||
pre_speech_frames = 0
|
pre_speech_frames = 0
|
||||||
MAX_PRE_SPEECH = 15 # Keep ~0.45s of audio before speech confirmation
|
MAX_PRE_SPEECH = 10
|
||||||
|
|
||||||
while self.running:
|
while self.running:
|
||||||
try:
|
try:
|
||||||
audio_bytes, rms = self.audio_queue.get(timeout=1)
|
audio_bytes, rms = self.audio_queue.get(timeout=0.5)
|
||||||
self.recent_rms.append(rms)
|
self.recent_rms.append(rms)
|
||||||
|
|
||||||
# Always keep a small buffer of recent audio
|
|
||||||
pre_speech_buffer.extend(audio_bytes)
|
pre_speech_buffer.extend(audio_bytes)
|
||||||
pre_speech_frames += 1
|
pre_speech_frames += 1
|
||||||
if pre_speech_frames > MAX_PRE_SPEECH:
|
if pre_speech_frames > MAX_PRE_SPEECH:
|
||||||
# Trim oldest frames
|
|
||||||
excess = pre_speech_frames - MAX_PRE_SPEECH
|
excess = pre_speech_frames - MAX_PRE_SPEECH
|
||||||
bytes_to_trim = excess * self.frame_size
|
bytes_to_trim = excess * self.frame_size
|
||||||
pre_speech_buffer = pre_speech_buffer[bytes_to_trim:]
|
pre_speech_buffer = pre_speech_buffer[bytes_to_trim:]
|
||||||
pre_speech_frames = MAX_PRE_SPEECH
|
pre_speech_frames = MAX_PRE_SPEECH
|
||||||
|
|
||||||
# ========================
|
|
||||||
# FRAME CLASSIFICATION
|
|
||||||
# ========================
|
|
||||||
is_voice_frame = False
|
is_voice_frame = False
|
||||||
if rms >= self.speech_threshold:
|
if rms >= self.speech_threshold:
|
||||||
try:
|
try:
|
||||||
@@ -229,21 +217,15 @@ class AudioListener:
|
|||||||
except:
|
except:
|
||||||
is_voice_frame = True
|
is_voice_frame = True
|
||||||
|
|
||||||
# ========================
|
|
||||||
# STATE: NOT SPEAKING
|
|
||||||
# ========================
|
|
||||||
if not self.is_speaking:
|
if not self.is_speaking:
|
||||||
if is_voice_frame:
|
if is_voice_frame:
|
||||||
self.consecutive_speech += 1
|
self.consecutive_speech += 1
|
||||||
self.consecutive_silence = 0
|
self.consecutive_silence = 0
|
||||||
|
|
||||||
# Need CONSECUTIVE_SPEECH_TO_START frames to confirm speech
|
|
||||||
if self.consecutive_speech >= CONSECUTIVE_SPEECH_TO_START:
|
if self.consecutive_speech >= CONSECUTIVE_SPEECH_TO_START:
|
||||||
# CONFIRMED SPEECH - start capturing
|
|
||||||
self.is_speaking = True
|
self.is_speaking = True
|
||||||
self.speech_start_time = time.time()
|
self.speech_start_time = time.time()
|
||||||
|
|
||||||
# Include pre-speech buffer for context
|
|
||||||
self.current_audio = bytearray(pre_speech_buffer)
|
self.current_audio = bytearray(pre_speech_buffer)
|
||||||
self.speech_frames = pre_speech_frames
|
self.speech_frames = pre_speech_frames
|
||||||
self.silence_frames = 0
|
self.silence_frames = 0
|
||||||
@@ -255,9 +237,6 @@ class AudioListener:
|
|||||||
self.consecutive_speech = 0
|
self.consecutive_speech = 0
|
||||||
self.consecutive_silence += 1
|
self.consecutive_silence += 1
|
||||||
|
|
||||||
# ========================
|
|
||||||
# STATE: SPEAKING
|
|
||||||
# ========================
|
|
||||||
else:
|
else:
|
||||||
self.current_audio.extend(audio_bytes)
|
self.current_audio.extend(audio_bytes)
|
||||||
self.speech_frames += 1
|
self.speech_frames += 1
|
||||||
@@ -266,55 +245,38 @@ class AudioListener:
|
|||||||
if rms > self.peak_rms:
|
if rms > self.peak_rms:
|
||||||
self.peak_rms = rms
|
self.peak_rms = rms
|
||||||
|
|
||||||
# Dynamic silence threshold based on actual speech levels
|
|
||||||
if len(self.speech_rms_values) > 15:
|
if len(self.speech_rms_values) > 15:
|
||||||
speech_median = np.median(self.speech_rms_values)
|
speech_median = np.median(self.speech_rms_values)
|
||||||
# Silence = below 60% of your speech median, or below noise-based threshold
|
dynamic_silence = max(self.silence_threshold, speech_median * 0.6)
|
||||||
dynamic_silence = max(
|
|
||||||
self.silence_threshold,
|
|
||||||
speech_median * 0.6
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
dynamic_silence = self.silence_threshold
|
dynamic_silence = self.silence_threshold
|
||||||
|
|
||||||
# Check if this frame is silence
|
|
||||||
if rms < dynamic_silence and not is_voice_frame:
|
if rms < dynamic_silence and not is_voice_frame:
|
||||||
self.silence_frames += 1
|
self.silence_frames += 1
|
||||||
else:
|
else:
|
||||||
# Reset silence counter if we hear voice
|
|
||||||
if is_voice_frame:
|
if is_voice_frame:
|
||||||
self.silence_frames = 0
|
self.silence_frames = 0
|
||||||
|
|
||||||
# ========================
|
|
||||||
# DEBUG DISPLAY
|
|
||||||
# ========================
|
|
||||||
if self.is_speaking:
|
if self.is_speaking:
|
||||||
bar_len = max(0, min(int((rms - self.noise_floor) / 60), 35))
|
bar_len = max(0, min(int((rms - self.noise_floor) / 60), 35))
|
||||||
bar = "█" * bar_len
|
bar = "█" * bar_len
|
||||||
sil = f" 🔇{self.silence_frames}" if self.silence_frames > 0 else ""
|
sil = f" 🔇{self.silence_frames}" if self.silence_frames > 0 else ""
|
||||||
dyn = f" thr:{dynamic_silence:.0f}" if len(self.speech_rms_values) > 15 else ""
|
print(f"\r🎙️ {rms:5d} |{bar}{sil} ", end="")
|
||||||
print(f"\r🎙️ {rms:5d} |{bar}{sil}{dyn} ", end="")
|
|
||||||
elif self.consecutive_speech > 0:
|
elif self.consecutive_speech > 0:
|
||||||
print(f"\r👂 {rms:5d} | detecting... {self.consecutive_speech}/{CONSECUTIVE_SPEECH_TO_START} ",
|
print(f"\r👂 {rms:5d} | detecting... {self.consecutive_speech}/{CONSECUTIVE_SPEECH_TO_START} ",
|
||||||
end="")
|
end="")
|
||||||
|
|
||||||
# ========================
|
|
||||||
# TRANSCRIPTION TRIGGERS
|
|
||||||
# ========================
|
|
||||||
if self.is_speaking:
|
if self.is_speaking:
|
||||||
speech_duration = time.time() - self.speech_start_time
|
speech_duration = time.time() - self.speech_start_time
|
||||||
silence_duration = (self.silence_frames * self.frame_duration_ms) / 1000
|
silence_duration = (self.silence_frames * self.frame_duration_ms) / 1000
|
||||||
|
|
||||||
# Trigger 1: Sufficient silence after minimum speech
|
if (speech_duration >= MIN_SPEECH_SECONDS and silence_duration >= SILENCE_SECONDS):
|
||||||
if (speech_duration >= MIN_SPEECH_SECONDS and
|
|
||||||
silence_duration >= SILENCE_SECONDS):
|
|
||||||
print(f"\n✅ End ({speech_duration:.1f}s, peak: {self.peak_rms})")
|
print(f"\n✅ End ({speech_duration:.1f}s, peak: {self.peak_rms})")
|
||||||
self._safe_transcribe()
|
self._safe_transcribe()
|
||||||
self._reset_speech_state()
|
self._reset_speech_state()
|
||||||
pre_speech_buffer = bytearray()
|
pre_speech_buffer = bytearray()
|
||||||
pre_speech_frames = 0
|
pre_speech_frames = 0
|
||||||
|
|
||||||
# Trigger 2: Max duration
|
|
||||||
elif speech_duration >= MAX_SPEECH_SECONDS:
|
elif speech_duration >= MAX_SPEECH_SECONDS:
|
||||||
print(f"\n⏰ Max ({speech_duration:.1f}s, peak: {self.peak_rms})")
|
print(f"\n⏰ Max ({speech_duration:.1f}s, peak: {self.peak_rms})")
|
||||||
self._safe_transcribe()
|
self._safe_transcribe()
|
||||||
@@ -353,7 +315,7 @@ class AudioListener:
|
|||||||
self.last_transcription_time = current_time
|
self.last_transcription_time = current_time
|
||||||
|
|
||||||
audio_duration = len(self.current_audio) / (2 * self.sample_rate)
|
audio_duration = len(self.current_audio) / (2 * self.sample_rate)
|
||||||
if audio_duration < 0.5:
|
if audio_duration < 0.3:
|
||||||
print(" (Too short)")
|
print(" (Too short)")
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -366,13 +328,13 @@ class AudioListener:
|
|||||||
segments, info = self.model.transcribe(
|
segments, info = self.model.transcribe(
|
||||||
audio_np,
|
audio_np,
|
||||||
language="en",
|
language="en",
|
||||||
beam_size=5,
|
beam_size=3,
|
||||||
best_of=5,
|
best_of=3,
|
||||||
temperature=[0.0, 0.2, 0.4],
|
temperature=[0.0, 0.2],
|
||||||
condition_on_previous_text=False,
|
condition_on_previous_text=False,
|
||||||
compression_ratio_threshold=1.8,
|
compression_ratio_threshold=1.8,
|
||||||
no_speech_threshold=0.5,
|
no_speech_threshold=0.6,
|
||||||
log_prob_threshold=-0.8,
|
log_prob_threshold=-1.0,
|
||||||
word_timestamps=False,
|
word_timestamps=False,
|
||||||
vad_filter=True,
|
vad_filter=True,
|
||||||
)
|
)
|
||||||
@@ -405,6 +367,8 @@ class AudioListener:
|
|||||||
tag = "🎯 Command" if is_command else "📝 Speech"
|
tag = "🎯 Command" if is_command else "📝 Speech"
|
||||||
print(f"{tag}: {full_text}")
|
print(f"{tag}: {full_text}")
|
||||||
|
|
||||||
|
# Callback with just text and timestamp (original format)
|
||||||
|
if self.callback:
|
||||||
self.callback(full_text, datetime.now())
|
self.callback(full_text, datetime.now())
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -428,6 +392,7 @@ class AudioListener:
|
|||||||
if unique_ratio < 0.3 and len(words) > 8:
|
if unique_ratio < 0.3 and len(words) > 8:
|
||||||
return True
|
return True
|
||||||
mid = len(words) // 2
|
mid = len(words) // 2
|
||||||
|
if mid > 0:
|
||||||
first = " ".join(words[:mid])
|
first = " ".join(words[:mid])
|
||||||
second = " ".join(words[mid:mid * 2])
|
second = " ".join(words[mid:mid * 2])
|
||||||
if first == second and len(first) > 15:
|
if first == second and len(first) > 15:
|
||||||
@@ -435,32 +400,57 @@ class AudioListener:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
def _is_command(self, text):
|
def _is_command(self, text):
|
||||||
|
"""Aggressive command/question detection"""
|
||||||
|
if not text or len(text) < 2:
|
||||||
|
return False
|
||||||
|
|
||||||
lowered = text.lower().strip()
|
lowered = text.lower().strip()
|
||||||
words = lowered.split()
|
|
||||||
if words and words[0] in self.command_starters:
|
# Always command if ends with ?
|
||||||
return True
|
|
||||||
command_phrases = [
|
|
||||||
"what is", "what are", "how do", "how to", "how can",
|
|
||||||
"can you", "could you", "will you", "would you",
|
|
||||||
"tell me", "explain", "describe", "show me",
|
|
||||||
"i want", "i need", "please", "give me",
|
|
||||||
"where is", "when did", "why do", "why is",
|
|
||||||
"difference between", "compare",
|
|
||||||
"write a", "write me", "create a", "create me",
|
|
||||||
"generate", "make a", "build a", "code a",
|
|
||||||
"calculate", "compute", "solve", "find",
|
|
||||||
"list", "name", "provide", "search for",
|
|
||||||
"what does", "what would", "what if",
|
|
||||||
]
|
|
||||||
for phrase in command_phrases:
|
|
||||||
if phrase in lowered:
|
|
||||||
return True
|
|
||||||
if lowered.endswith('?'):
|
if lowered.endswith('?'):
|
||||||
return True
|
return True
|
||||||
if len(words) >= 3:
|
|
||||||
filler_words = {"um", "uh", "like", "you know", "i mean", "okay", "alright", "so", "well", "actually",
|
# Math expressions
|
||||||
"basically"}
|
math_patterns = [
|
||||||
content_words = [w for w in words if w not in filler_words]
|
r'\d+\s*[\+\-\*\/]\s*\d+',
|
||||||
if len(content_words) >= 3:
|
r'\d+\s*plus\s*\d+',
|
||||||
|
r'\d+\s*minus\s*\d+',
|
||||||
|
r'\d+\s*times\s*\d+',
|
||||||
|
r'\d+\s*divided by\s*\d+',
|
||||||
|
]
|
||||||
|
import re
|
||||||
|
for pattern in math_patterns:
|
||||||
|
if re.search(pattern, lowered):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
# Question starters
|
||||||
|
starters = {
|
||||||
|
'what', 'why', 'how', 'when', 'where', 'who', 'which',
|
||||||
|
'can', 'could', 'would', 'will', 'do', 'does', 'did',
|
||||||
|
'is', 'are', 'was', 'were', 'should', 'shall', 'may',
|
||||||
|
'tell', 'explain', 'describe', 'show', 'give'
|
||||||
|
}
|
||||||
|
words = lowered.split()
|
||||||
|
if words and words[0] in starters:
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Question phrases
|
||||||
|
question_phrases = [
|
||||||
|
'what is', 'what are', 'how do', 'how to', 'can you',
|
||||||
|
'tell me', 'explain', 'describe', 'show me'
|
||||||
|
]
|
||||||
|
for phrase in question_phrases:
|
||||||
|
if phrase in lowered:
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Short statements (2-6 words) - treat as commands/questions
|
||||||
|
if 2 <= len(words) <= 6:
|
||||||
|
filler = {'um', 'uh', 'like', 'you', 'know', 'so', 'well'}
|
||||||
|
content_words = [w for w in words if w not in filler]
|
||||||
|
if len(content_words) >= 2:
|
||||||
|
if any(op in lowered for op in ['plus', 'minus', 'times']):
|
||||||
|
return True
|
||||||
|
if len(content_words) <= 4:
|
||||||
|
return True
|
||||||
|
|
||||||
return False
|
return False
|
||||||
Reference in New Issue
Block a user