This commit is contained in:
Charles Wambua
2026-05-25 16:43:05 +03:00
parent b7b7397876
commit 1dd5b757be
3 changed files with 337 additions and 213 deletions

140
main.py
View File

@@ -7,7 +7,9 @@ import sys
import os import os
import yaml import yaml
import threading import threading
import logging
from pathlib import Path from pathlib import Path
from datetime import datetime
sys.path.insert(0, str(Path(__file__).parent)) sys.path.insert(0, str(Path(__file__).parent))
@@ -34,38 +36,57 @@ _NOISE_PHRASES = (
class MeetingAssistant: class MeetingAssistant:
def __init__(self, config_path="config.yaml"): def __init__(self):
"""Initialize all components"""
print("🚀 Starting Meeting Assistant...") print("🚀 Starting Meeting Assistant...")
config_file = Path(__file__).parent / config_path
with open(config_file, "r") as f:
self.config = yaml.safe_load(f)
self.logger = setup_logging()
print(f"💻 Platform: {get_platform()}") print(f"💻 Platform: {get_platform()}")
self.context_manager = ContextManager(self.config) # Load config
config_path = Path(__file__).parent / "config.yaml"
with open(config_path, 'r') as f:
self.config = yaml.safe_load(f)
# Setup logging
self.logger = setup_logging()
# Initialize components
self.ai_engine = AIEngine(self.config) self.ai_engine = AIEngine(self.config)
self.context_manager = ContextManager(self.config)
self.overlay = InvisibleOverlay(self.config) self.overlay = InvisibleOverlay(self.config)
# Audio listener with callback
self.audio_listener = AudioListener(self.config, self.on_audio_transcript) self.audio_listener = AudioListener(self.config, self.on_audio_transcript)
self._last_question = "" # Simple debounce # State management
self.current_answer = None
self.answering = False
self.answer_lock = threading.Lock()
self._last_question = ""
# ------------------------------------------------------------------ print("✅ Meeting Assistant Ready!")
# Audio callback — runs in the audio listener thread print("==================================================")
# ------------------------------------------------------------------ print("🎤 Listening for questions via microphone")
print(" Answers appear in the overlay (top-right)")
print(" Type a question here + Enter to test AI")
print(" Ctrl+C to stop")
print("==================================================")
def on_audio_transcript(self, text, timestamp): def on_audio_transcript(self, text, timestamp):
"""Audio callback — runs in the audio listener thread"""
if not text or len(text.strip()) < 2:
return
self.context_manager.add_audio_context(text, timestamp) self.context_manager.add_audio_context(text, timestamp)
if self._is_question(text): if self._is_question(text):
print(f"❓ Question: {text}") print(f"\n❓ Question detected: {text}")
self._generate_answer(text) self._generate_answer(text)
else: else:
print(f" Context: {text}") print(f" Context (not a question): {text}")
def _is_question(self, text): def _is_question(self, text):
if len(text.split()) < 3: """Enhanced question detection for faster responses"""
if len(text.split()) < 2: # Reduced from 3 for shorter questions
return False return False
text_lower = text.lower().strip() text_lower = text.lower().strip()
@@ -78,21 +99,78 @@ class MeetingAssistant:
if text.rstrip().endswith("?"): if text.rstrip().endswith("?"):
return True return True
# Check for math expressions (2+2, two plus two)
import re
math_patterns = [
r'\d+\s*[\+\-\*\/]\s*\d+', # 2+2, 5-3
r'\d+\s*plus\s*\d+', # 2 plus 2
r'\d+\s*minus\s*\d+', # 5 minus 3
r'\d+\s*times\s*\d+', # 4 times 5
r'\d+\s*divided by\s*\d+', # 10 divided by 2
]
for pattern in math_patterns:
if re.search(pattern, text_lower):
return True
# Check for short implicit questions (2-5 words without filler)
words = text_lower.split()
if 2 <= len(words) <= 5:
filler = {'um', 'uh', 'like', 'so', 'well', 'actually', 'basically'}
content_words = [w for w in words if w not in filler]
if len(content_words) >= 2:
# "two plus two", "capital france", "python example"
return True
# Starts with or contains a question word # Starts with or contains a question word
return any(text_lower.startswith(w) or f" {w} " in text_lower for w in _QUESTION_WORDS) if any(text_lower.startswith(w) or f" {w} " in text_lower for w in _QUESTION_WORDS):
return True
# Check for command starters
command_starters = ['tell me', 'explain', 'describe', 'show me', 'give me', 'find', 'search']
for starter in command_starters:
if text_lower.startswith(starter):
return True
return False
def _generate_answer(self, question): def _generate_answer(self, question):
"""Generate answer with interruption support"""
# Debounce: skip exact repeat within same run # Debounce: skip exact repeat within same run
if question.lower() == self._last_question.lower(): if question.lower() == self._last_question.lower():
return return
self._last_question = question self._last_question = question
context = self.context_manager.get_context() # Check if we should interrupt current answer
answer = self.ai_engine.answer_question(question, context) with self.answer_lock:
self.overlay.show_answer(answer, question) if self.answering:
self.logger.info(f"Q: {question}") print("🔄 Interrupting previous answer...")
self.logger.info(f"A: {answer}") if hasattr(self.ai_engine, 'interrupt'):
print(f"💡 Answer: {answer}\n") self.ai_engine.interrupt()
# Start answer generation in background thread
threading.Thread(target=self._answer_worker, args=(question,), daemon=True).start()
def _answer_worker(self, question):
"""Worker thread for answer generation"""
with self.answer_lock:
self.answering = True
try:
# Get context and generate answer
context = self.context_manager.get_context()
answer = self.ai_engine.answer_question(question, context)
# Show answer in overlay
self.overlay.show_answer(answer, question)
self.logger.info(f"Q: {question}")
self.logger.info(f"A: {answer}")
print(f"\n💡 Answer: {answer}\n")
except Exception as e:
print(f"❌ Error generating answer: {e}")
finally:
with self.answer_lock:
self.answering = False
# ------------------------------------------------------------------ # ------------------------------------------------------------------
# Manual terminal input for testing # Manual terminal input for testing
@@ -114,14 +192,7 @@ class MeetingAssistant:
# ------------------------------------------------------------------ # ------------------------------------------------------------------
def run(self): def run(self):
print("\n✅ Meeting Assistant Ready!") print("\n🎤 Starting audio listener...")
print("=" * 50)
print("🎤 Listening for questions via microphone")
print(" Answers appear in the overlay (top-right)")
print(" Type a question here + Enter to test AI")
print(" Ctrl+C to stop")
print("=" * 50)
self.overlay.start() self.overlay.start()
self.audio_listener.start() self.audio_listener.start()
@@ -137,9 +208,12 @@ class MeetingAssistant:
def shutdown(self): def shutdown(self):
print("\n🛑 Shutting down...") print("\n🛑 Shutting down...")
self.audio_listener.stop() if hasattr(self, 'audio_listener'):
self.overlay.stop() self.audio_listener.stop()
if hasattr(self, 'overlay'):
self.overlay.stop()
print("👋 Goodbye!") print("👋 Goodbye!")
sys.exit(0)
def main(): def main():
@@ -148,4 +222,4 @@ def main():
if __name__ == "__main__": if __name__ == "__main__":
main() main()

View File

@@ -4,23 +4,26 @@ from pathlib import Path
from collections import deque from collections import deque
from datetime import datetime from datetime import datetime
from rapidfuzz import fuzz from rapidfuzz import fuzz
import threading
class AIEngine: class AIEngine:
def __init__(self, config): def __init__(self, config):
self.config = config self.config = config
self.model = None self.model = None
self.memory = deque(maxlen=8) self.memory = deque(maxlen=8)
self.fast_mode = True
self.current_generation = None
self.interrupt_event = threading.Event()
self.load_model() self.load_model()
def load_model(self): def load_model(self):
print("🤖 Loading AI model...") print("🤖 Loading AI model...")
model_path = ( model_path = (
Path(__file__).parent.parent Path(__file__).parent.parent
/ "models" / "models"
/ "Qwen2.5-7B-Instruct-Q4_K_M.gguf" / "Qwen2.5-7B-Instruct-Q4_K_M.gguf"
) )
from llama_cpp import Llama from llama_cpp import Llama
@@ -28,37 +31,59 @@ class AIEngine:
self.model = Llama( self.model = Llama(
model_path=str(model_path), model_path=str(model_path),
# PERFORMANCE # PERFORMANCE - MAXIMUM SPEED
n_gpu_layers=-1, n_gpu_layers=-1,
n_ctx=4096, n_ctx=2048, # Reduced from 4096 for speed
n_batch=1024, n_batch=512, # Reduced for faster prompt processing
n_threads=max(os.cpu_count() - 1, 1), n_threads=max(os.cpu_count() - 1, 1),
# SPEED # SPEED OPTIMIZATIONS
offload_kqv=True, offload_kqv=True,
flash_attn=True, flash_attn=True,
use_mmap=True, use_mmap=True,
use_mlock=False, use_mlock=False,
# FASTER INFERENCE
n_parts=-1,
seed=42, # Deterministic for speed
f16_kv=True, # Use half-precision for KV cache
# STABILITY # STABILITY
verbose=False verbose=False
) )
print("✅ AI ready") print("✅ AI ready (fast mode enabled)")
def answer_question(self, question, context, is_partial=False):
"""Answer a question with interruption support"""
# Check for interrupt
if self.interrupt_event.is_set():
self.interrupt_event.clear()
return None
def answer_question(self, question, context):
question = self._normalize_question(question) question = self._normalize_question(question)
response = self._generate(question, context) # For partial questions, answer faster with fewer tokens
if is_partial:
response = self._generate_fast(question, context)
else:
response = self._generate(question, context)
self.memory.append({ if response: # Only store if not interrupted
"question": question, self.memory.append({
"response": response, "question": question,
"time": datetime.now() "response": response,
}) "time": datetime.now()
})
return response return response
def interrupt(self):
"""Interrupt current generation"""
self.interrupt_event.set()
print("🛑 Generation interrupted")
def _normalize_question(self, question): def _normalize_question(self, question):
question = question.strip() question = question.strip()
@@ -70,12 +95,10 @@ class AIEngine:
} }
words = question.split() words = question.split()
normalized = [] normalized = []
for word in words: for word in words:
lowered = word.lower() lowered = word.lower()
if lowered in fixes: if lowered in fixes:
normalized.append(fixes[lowered]) normalized.append(fixes[lowered])
else: else:
@@ -91,73 +114,105 @@ class AIEngine:
return question return question
def _generate(self, question, context): def _generate_fast(self, question, context):
recent_audio = context.get("audio", "")[-1500:] """Ultra-fast generation for partial/interruptible responses"""
recent_screen = context.get("screen", "")[-700:]
memory_context = "" # Check for interrupt before generation
if self.interrupt_event.is_set():
self.interrupt_event.clear()
return None
if self.memory: # Simplified prompt for speed
recent = list(self.memory)[-2:] system_prompt = "You are a fast assistant. Answer in 1-2 short sentences max."
for item in recent: user_prompt = f"Q: {question}\nA:"
memory_context += (
f"Q: {item['question']}\n"
f"A: {item['response']}\n"
)
system_prompt = """
You are a realtime meeting copilot.
Rules:
- Answer immediately.
- Never ask for clarification unless impossible.
- Never invent missing words.
- Never complete partial speech.
- Keep answers under 80 words.
- Prioritize only explicit meaning.
- Ignore minor transcription mistakes.
- If speech is incomplete, answer conservatively.
- Sound precise and direct.
"""
user_prompt = f"""
Recent conversation:
{memory_context}
Meeting transcript:
{recent_audio}
Screen:
{recent_screen}
User question:
{question}
"""
prompt = ( prompt = (
f"<|im_start|>system\n" f"<|im_start|>system\n{system_prompt}<|im_end|>\n"
f"{system_prompt}" f"<|im_start|>user\n{user_prompt}<|im_end|>\n"
f"<|im_end|>\n"
f"<|im_start|>user\n"
f"{user_prompt}"
f"<|im_end|>\n"
f"<|im_start|>assistant\n" f"<|im_start|>assistant\n"
) )
output = self.model( try:
prompt, output = self.model(
prompt,
max_tokens=30, # Very short for fast responses
temperature=0.1,
top_p=0.9,
repeat_penalty=1.0,
stop=["<|im_end|>", "\n", ".", "!", "?"],
echo=False
)
max_tokens=120, # Check for interrupt during generation
temperature=0.2, if self.interrupt_event.is_set():
top_p=0.85, self.interrupt_event.clear()
repeat_penalty=1.05, return None
stop=["<|im_end|>"]
text = output["choices"][0]["text"]
return self._clean(text)
except Exception as e:
print(f"⚠️ Fast generation error: {e}")
return None
def _generate(self, question, context):
"""Standard generation for complete questions"""
# Check for interrupt before generation
if self.interrupt_event.is_set():
self.interrupt_event.clear()
return None
recent_audio = context.get("audio", "")[-1500:] if context else ""
recent_screen = context.get("screen", "")[-700:] if context else ""
memory_context = ""
if self.memory:
recent = list(self.memory)[-2:]
for item in recent:
memory_context += f"Q: {item['question']}\nA: {item['response']}\n"
# Ultra-concise system prompt for speed
system_prompt = """
You are a realtime assistant. Answer immediately and concisely.
Keep answers under 40 words. Be direct. No explanations unless asked.
"""
user_prompt = f"""
Q: {question}
A:"""
prompt = (
f"<|im_start|>system\n{system_prompt}<|im_end|>\n"
f"<|im_start|>user\n{user_prompt}<|im_end|>\n"
f"<|im_start|>assistant\n"
) )
text = output["choices"][0]["text"] try:
output = self.model(
prompt,
max_tokens=60, # Reduced for speed
temperature=0.1, # Lower for faster, more deterministic
top_p=0.9,
repeat_penalty=1.0, # Disabled for speed
stop=["<|im_end|>", "\n", "."],
echo=False,
frequency_penalty=0.0,
presence_penalty=0.0
)
return self._clean(text) # Check for interrupt during generation
if self.interrupt_event.is_set():
self.interrupt_event.clear()
return None
text = output["choices"][0]["text"]
return self._clean(text)
except Exception as e:
print(f"⚠️ Generation error: {e}")
return "I couldn't process that."
def _clean(self, text): def _clean(self, text):
text = re.sub(r"<.*?>", "", text) text = re.sub(r"<.*?>", "", text)
@@ -166,7 +221,12 @@ User question:
if not text: if not text:
return "No response generated." return "No response generated."
# Ensure it ends with punctuation
if text[-1] not in ".!?": if text[-1] not in ".!?":
text += "." text += "."
# Capitalize first letter
if text and text[0].islower():
text = text[0].upper() + text[1:]
return text return text

View File

@@ -19,23 +19,22 @@ from faster_whisper import WhisperModel
os.environ["HF_HUB_DISABLE_SSL_VERIFY"] = "1" os.environ["HF_HUB_DISABLE_SSL_VERIFY"] = "1"
# ============================================================ # ============================================================
# 🔧 TUNING # 🔧 TUNING - OPTIMIZED FOR SPEED
# ============================================================ # ============================================================
MIN_SPEECH_SECONDS = 1.2 # Need at least 1.2s of speech MIN_SPEECH_SECONDS = 0.8
SILENCE_SECONDS = 0.7 # 0.7s silence to end SILENCE_SECONDS = 0.3
MAX_SPEECH_SECONDS = 8.0 # Max before force MAX_SPEECH_SECONDS = 8.0
SPEECH_THRESHOLD_MULTIPLIER = 1.3 SPEECH_THRESHOLD_MULTIPLIER = 1.3
MIN_THRESHOLD_GAP = 400 MIN_THRESHOLD_GAP = 400
CONSECUTIVE_SPEECH_TO_START = 8 # Must have 8 consecutive speech frames to start CONSECUTIVE_SPEECH_TO_START = 4
# ============================================================ # ============================================================
class AudioListener: class AudioListener:
def __init__(self, config, callback): def __init__(self, config, callback):
self.config = config self.config = config
self.callback = callback self.callback = callback # Expects callback(text, timestamp)
self.running = False self.running = False
self.sample_rate = 16000 self.sample_rate = 16000
@@ -52,8 +51,8 @@ class AudioListener:
self.is_speaking = False self.is_speaking = False
self.speech_frames = 0 self.speech_frames = 0
self.silence_frames = 0 self.silence_frames = 0
self.consecutive_speech = 0 # Consecutive speech frames self.consecutive_speech = 0
self.consecutive_silence = 0 # Consecutive silence frames self.consecutive_silence = 0
self.total_frames_in_utterance = 0 self.total_frames_in_utterance = 0
self.noise_floor = 0 self.noise_floor = 0
@@ -66,34 +65,29 @@ class AudioListener:
self.speech_start_time = 0 self.speech_start_time = 0
self.last_transcription_time = 0 self.last_transcription_time = 0
self.min_transcription_interval = 0.5 self.min_transcription_interval = 0.3
self.calibrated = False self.calibrated = False
print("🔄 Loading Whisper model...") print("🔄 Loading Whisper model...")
self.model = WhisperModel( try:
"models/whisper", self.model = WhisperModel(
device="cpu", "models/whisper",
compute_type="int8", device="cpu",
local_files_only=True, compute_type="int8",
cpu_threads=max(os.cpu_count() - 1, 1) local_files_only=True,
) cpu_threads=max(os.cpu_count() - 1, 1)
print("✅ Whisper model ready") )
print("✅ Whisper model ready")
self.command_starters = { except Exception as e:
"what", "why", "how", "when", "where", "who", "which", print(f"⚠️ Could not load Whisper from models/whisper, trying default: {e}")
"can", "could", "would", "will", "do", "does", "did", self.model = WhisperModel(
"is", "are", "was", "were", "should", "shall", "may", "base",
"have", "has", "had", "am", device="cpu",
"tell", "explain", "describe", "compare", "define", compute_type="int8",
"find", "show", "give", "list", "name", "provide", cpu_threads=max(os.cpu_count() - 1, 1)
"write", "create", "make", "generate", "build", "code", )
"calculate", "compute", "solve", "convert", "translate", print("✅ Whisper model ready (using default 'base' model)")
"search", "look", "check", "get", "fetch", "run",
"start", "stop", "open", "close", "save", "delete",
"draw", "plot", "graph", "print", "display", "output",
"summarize", "analyze", "review", "evaluate",
}
def start(self): def start(self):
self.running = True self.running = True
@@ -129,7 +123,7 @@ class AudioListener:
rms = np.sqrt(np.mean(chunk ** 2)) rms = np.sqrt(np.mean(chunk ** 2))
rms_values.append(rms) rms_values.append(rms)
self.noise_floor = np.median(rms_values) self.noise_floor = np.median(rms_values) if rms_values else 1500
self.speech_threshold = max( self.speech_threshold = max(
self.noise_floor * SPEECH_THRESHOLD_MULTIPLIER, self.noise_floor * SPEECH_THRESHOLD_MULTIPLIER,
@@ -150,7 +144,7 @@ class AudioListener:
self.silence_threshold = 1700 self.silence_threshold = 1700
self.calibrated = True self.calibrated = True
print(f"\n📋 Settings:") print(f"\n📋 Settings (FAST MODE):")
print(f" Min speech: {MIN_SPEECH_SECONDS}s | Max: {MAX_SPEECH_SECONDS}s") print(f" Min speech: {MIN_SPEECH_SECONDS}s | Max: {MAX_SPEECH_SECONDS}s")
print(f" Silence to end: {SILENCE_SECONDS}s") print(f" Silence to end: {SILENCE_SECONDS}s")
print(f" Consecutive speech to start: {CONSECUTIVE_SPEECH_TO_START} frames") print(f" Consecutive speech to start: {CONSECUTIVE_SPEECH_TO_START} frames")
@@ -199,29 +193,23 @@ class AudioListener:
self.audio_queue.put((audio_bytes, rms)) self.audio_queue.put((audio_bytes, rms))
def _processing_loop(self): def _processing_loop(self):
# Pre-speech buffer - capture audio BEFORE we confirm speech
pre_speech_buffer = bytearray() pre_speech_buffer = bytearray()
pre_speech_frames = 0 pre_speech_frames = 0
MAX_PRE_SPEECH = 15 # Keep ~0.45s of audio before speech confirmation MAX_PRE_SPEECH = 10
while self.running: while self.running:
try: try:
audio_bytes, rms = self.audio_queue.get(timeout=1) audio_bytes, rms = self.audio_queue.get(timeout=0.5)
self.recent_rms.append(rms) self.recent_rms.append(rms)
# Always keep a small buffer of recent audio
pre_speech_buffer.extend(audio_bytes) pre_speech_buffer.extend(audio_bytes)
pre_speech_frames += 1 pre_speech_frames += 1
if pre_speech_frames > MAX_PRE_SPEECH: if pre_speech_frames > MAX_PRE_SPEECH:
# Trim oldest frames
excess = pre_speech_frames - MAX_PRE_SPEECH excess = pre_speech_frames - MAX_PRE_SPEECH
bytes_to_trim = excess * self.frame_size bytes_to_trim = excess * self.frame_size
pre_speech_buffer = pre_speech_buffer[bytes_to_trim:] pre_speech_buffer = pre_speech_buffer[bytes_to_trim:]
pre_speech_frames = MAX_PRE_SPEECH pre_speech_frames = MAX_PRE_SPEECH
# ========================
# FRAME CLASSIFICATION
# ========================
is_voice_frame = False is_voice_frame = False
if rms >= self.speech_threshold: if rms >= self.speech_threshold:
try: try:
@@ -229,21 +217,15 @@ class AudioListener:
except: except:
is_voice_frame = True is_voice_frame = True
# ========================
# STATE: NOT SPEAKING
# ========================
if not self.is_speaking: if not self.is_speaking:
if is_voice_frame: if is_voice_frame:
self.consecutive_speech += 1 self.consecutive_speech += 1
self.consecutive_silence = 0 self.consecutive_silence = 0
# Need CONSECUTIVE_SPEECH_TO_START frames to confirm speech
if self.consecutive_speech >= CONSECUTIVE_SPEECH_TO_START: if self.consecutive_speech >= CONSECUTIVE_SPEECH_TO_START:
# CONFIRMED SPEECH - start capturing
self.is_speaking = True self.is_speaking = True
self.speech_start_time = time.time() self.speech_start_time = time.time()
# Include pre-speech buffer for context
self.current_audio = bytearray(pre_speech_buffer) self.current_audio = bytearray(pre_speech_buffer)
self.speech_frames = pre_speech_frames self.speech_frames = pre_speech_frames
self.silence_frames = 0 self.silence_frames = 0
@@ -255,9 +237,6 @@ class AudioListener:
self.consecutive_speech = 0 self.consecutive_speech = 0
self.consecutive_silence += 1 self.consecutive_silence += 1
# ========================
# STATE: SPEAKING
# ========================
else: else:
self.current_audio.extend(audio_bytes) self.current_audio.extend(audio_bytes)
self.speech_frames += 1 self.speech_frames += 1
@@ -266,55 +245,38 @@ class AudioListener:
if rms > self.peak_rms: if rms > self.peak_rms:
self.peak_rms = rms self.peak_rms = rms
# Dynamic silence threshold based on actual speech levels
if len(self.speech_rms_values) > 15: if len(self.speech_rms_values) > 15:
speech_median = np.median(self.speech_rms_values) speech_median = np.median(self.speech_rms_values)
# Silence = below 60% of your speech median, or below noise-based threshold dynamic_silence = max(self.silence_threshold, speech_median * 0.6)
dynamic_silence = max(
self.silence_threshold,
speech_median * 0.6
)
else: else:
dynamic_silence = self.silence_threshold dynamic_silence = self.silence_threshold
# Check if this frame is silence
if rms < dynamic_silence and not is_voice_frame: if rms < dynamic_silence and not is_voice_frame:
self.silence_frames += 1 self.silence_frames += 1
else: else:
# Reset silence counter if we hear voice
if is_voice_frame: if is_voice_frame:
self.silence_frames = 0 self.silence_frames = 0
# ========================
# DEBUG DISPLAY
# ========================
if self.is_speaking: if self.is_speaking:
bar_len = max(0, min(int((rms - self.noise_floor) / 60), 35)) bar_len = max(0, min(int((rms - self.noise_floor) / 60), 35))
bar = "" * bar_len bar = "" * bar_len
sil = f" 🔇{self.silence_frames}" if self.silence_frames > 0 else "" sil = f" 🔇{self.silence_frames}" if self.silence_frames > 0 else ""
dyn = f" thr:{dynamic_silence:.0f}" if len(self.speech_rms_values) > 15 else "" print(f"\r🎙️ {rms:5d} |{bar}{sil} ", end="")
print(f"\r🎙️ {rms:5d} |{bar}{sil}{dyn} ", end="")
elif self.consecutive_speech > 0: elif self.consecutive_speech > 0:
print(f"\r👂 {rms:5d} | detecting... {self.consecutive_speech}/{CONSECUTIVE_SPEECH_TO_START} ", print(f"\r👂 {rms:5d} | detecting... {self.consecutive_speech}/{CONSECUTIVE_SPEECH_TO_START} ",
end="") end="")
# ========================
# TRANSCRIPTION TRIGGERS
# ========================
if self.is_speaking: if self.is_speaking:
speech_duration = time.time() - self.speech_start_time speech_duration = time.time() - self.speech_start_time
silence_duration = (self.silence_frames * self.frame_duration_ms) / 1000 silence_duration = (self.silence_frames * self.frame_duration_ms) / 1000
# Trigger 1: Sufficient silence after minimum speech if (speech_duration >= MIN_SPEECH_SECONDS and silence_duration >= SILENCE_SECONDS):
if (speech_duration >= MIN_SPEECH_SECONDS and
silence_duration >= SILENCE_SECONDS):
print(f"\n✅ End ({speech_duration:.1f}s, peak: {self.peak_rms})") print(f"\n✅ End ({speech_duration:.1f}s, peak: {self.peak_rms})")
self._safe_transcribe() self._safe_transcribe()
self._reset_speech_state() self._reset_speech_state()
pre_speech_buffer = bytearray() pre_speech_buffer = bytearray()
pre_speech_frames = 0 pre_speech_frames = 0
# Trigger 2: Max duration
elif speech_duration >= MAX_SPEECH_SECONDS: elif speech_duration >= MAX_SPEECH_SECONDS:
print(f"\n⏰ Max ({speech_duration:.1f}s, peak: {self.peak_rms})") print(f"\n⏰ Max ({speech_duration:.1f}s, peak: {self.peak_rms})")
self._safe_transcribe() self._safe_transcribe()
@@ -353,7 +315,7 @@ class AudioListener:
self.last_transcription_time = current_time self.last_transcription_time = current_time
audio_duration = len(self.current_audio) / (2 * self.sample_rate) audio_duration = len(self.current_audio) / (2 * self.sample_rate)
if audio_duration < 0.5: if audio_duration < 0.3:
print(" (Too short)") print(" (Too short)")
return return
@@ -366,13 +328,13 @@ class AudioListener:
segments, info = self.model.transcribe( segments, info = self.model.transcribe(
audio_np, audio_np,
language="en", language="en",
beam_size=5, beam_size=3,
best_of=5, best_of=3,
temperature=[0.0, 0.2, 0.4], temperature=[0.0, 0.2],
condition_on_previous_text=False, condition_on_previous_text=False,
compression_ratio_threshold=1.8, compression_ratio_threshold=1.8,
no_speech_threshold=0.5, no_speech_threshold=0.6,
log_prob_threshold=-0.8, log_prob_threshold=-1.0,
word_timestamps=False, word_timestamps=False,
vad_filter=True, vad_filter=True,
) )
@@ -405,7 +367,9 @@ class AudioListener:
tag = "🎯 Command" if is_command else "📝 Speech" tag = "🎯 Command" if is_command else "📝 Speech"
print(f"{tag}: {full_text}") print(f"{tag}: {full_text}")
self.callback(full_text, datetime.now()) # Callback with just text and timestamp (original format)
if self.callback:
self.callback(full_text, datetime.now())
except Exception as e: except Exception as e:
print(f"❌ Transcription error: {e}") print(f"❌ Transcription error: {e}")
@@ -428,39 +392,65 @@ class AudioListener:
if unique_ratio < 0.3 and len(words) > 8: if unique_ratio < 0.3 and len(words) > 8:
return True return True
mid = len(words) // 2 mid = len(words) // 2
first = " ".join(words[:mid]) if mid > 0:
second = " ".join(words[mid:mid * 2]) first = " ".join(words[:mid])
if first == second and len(first) > 15: second = " ".join(words[mid:mid * 2])
return True if first == second and len(first) > 15:
return True
return False return False
def _is_command(self, text): def _is_command(self, text):
"""Aggressive command/question detection"""
if not text or len(text) < 2:
return False
lowered = text.lower().strip() lowered = text.lower().strip()
words = lowered.split()
if words and words[0] in self.command_starters: # Always command if ends with ?
return True
command_phrases = [
"what is", "what are", "how do", "how to", "how can",
"can you", "could you", "will you", "would you",
"tell me", "explain", "describe", "show me",
"i want", "i need", "please", "give me",
"where is", "when did", "why do", "why is",
"difference between", "compare",
"write a", "write me", "create a", "create me",
"generate", "make a", "build a", "code a",
"calculate", "compute", "solve", "find",
"list", "name", "provide", "search for",
"what does", "what would", "what if",
]
for phrase in command_phrases:
if phrase in lowered:
return True
if lowered.endswith('?'): if lowered.endswith('?'):
return True return True
if len(words) >= 3:
filler_words = {"um", "uh", "like", "you know", "i mean", "okay", "alright", "so", "well", "actually", # Math expressions
"basically"} math_patterns = [
content_words = [w for w in words if w not in filler_words] r'\d+\s*[\+\-\*\/]\s*\d+',
if len(content_words) >= 3: r'\d+\s*plus\s*\d+',
r'\d+\s*minus\s*\d+',
r'\d+\s*times\s*\d+',
r'\d+\s*divided by\s*\d+',
]
import re
for pattern in math_patterns:
if re.search(pattern, lowered):
return True return True
# Question starters
starters = {
'what', 'why', 'how', 'when', 'where', 'who', 'which',
'can', 'could', 'would', 'will', 'do', 'does', 'did',
'is', 'are', 'was', 'were', 'should', 'shall', 'may',
'tell', 'explain', 'describe', 'show', 'give'
}
words = lowered.split()
if words and words[0] in starters:
return True
# Question phrases
question_phrases = [
'what is', 'what are', 'how do', 'how to', 'can you',
'tell me', 'explain', 'describe', 'show me'
]
for phrase in question_phrases:
if phrase in lowered:
return True
# Short statements (2-6 words) - treat as commands/questions
if 2 <= len(words) <= 6:
filler = {'um', 'uh', 'like', 'you', 'know', 'so', 'well'}
content_words = [w for w in words if w not in filler]
if len(content_words) >= 2:
if any(op in lowered for op in ['plus', 'minus', 'times']):
return True
if len(content_words) <= 4:
return True
return False return False