Updates
This commit is contained in:
138
main.py
138
main.py
@@ -7,7 +7,9 @@ import sys
|
||||
import os
|
||||
import yaml
|
||||
import threading
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
@@ -34,38 +36,57 @@ _NOISE_PHRASES = (
|
||||
|
||||
|
||||
class MeetingAssistant:
|
||||
def __init__(self, config_path="config.yaml"):
|
||||
def __init__(self):
|
||||
"""Initialize all components"""
|
||||
print("🚀 Starting Meeting Assistant...")
|
||||
|
||||
config_file = Path(__file__).parent / config_path
|
||||
with open(config_file, "r") as f:
|
||||
self.config = yaml.safe_load(f)
|
||||
|
||||
self.logger = setup_logging()
|
||||
print(f"💻 Platform: {get_platform()}")
|
||||
|
||||
self.context_manager = ContextManager(self.config)
|
||||
# Load config
|
||||
config_path = Path(__file__).parent / "config.yaml"
|
||||
with open(config_path, 'r') as f:
|
||||
self.config = yaml.safe_load(f)
|
||||
|
||||
# Setup logging
|
||||
self.logger = setup_logging()
|
||||
|
||||
# Initialize components
|
||||
self.ai_engine = AIEngine(self.config)
|
||||
self.context_manager = ContextManager(self.config)
|
||||
self.overlay = InvisibleOverlay(self.config)
|
||||
|
||||
# Audio listener with callback
|
||||
self.audio_listener = AudioListener(self.config, self.on_audio_transcript)
|
||||
|
||||
self._last_question = "" # Simple debounce
|
||||
# State management
|
||||
self.current_answer = None
|
||||
self.answering = False
|
||||
self.answer_lock = threading.Lock()
|
||||
self._last_question = ""
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Audio callback — runs in the audio listener thread
|
||||
# ------------------------------------------------------------------
|
||||
print("✅ Meeting Assistant Ready!")
|
||||
print("==================================================")
|
||||
print("🎤 Listening for questions via microphone")
|
||||
print(" Answers appear in the overlay (top-right)")
|
||||
print(" Type a question here + Enter to test AI")
|
||||
print(" Ctrl+C to stop")
|
||||
print("==================================================")
|
||||
|
||||
def on_audio_transcript(self, text, timestamp):
|
||||
"""Audio callback — runs in the audio listener thread"""
|
||||
if not text or len(text.strip()) < 2:
|
||||
return
|
||||
|
||||
self.context_manager.add_audio_context(text, timestamp)
|
||||
|
||||
if self._is_question(text):
|
||||
print(f"❓ Question: {text}")
|
||||
print(f"\n❓ Question detected: {text}")
|
||||
self._generate_answer(text)
|
||||
else:
|
||||
print(f" Context: {text}")
|
||||
print(f" Context (not a question): {text}")
|
||||
|
||||
def _is_question(self, text):
|
||||
if len(text.split()) < 3:
|
||||
"""Enhanced question detection for faster responses"""
|
||||
if len(text.split()) < 2: # Reduced from 3 for shorter questions
|
||||
return False
|
||||
|
||||
text_lower = text.lower().strip()
|
||||
@@ -78,21 +99,78 @@ class MeetingAssistant:
|
||||
if text.rstrip().endswith("?"):
|
||||
return True
|
||||
|
||||
# Check for math expressions (2+2, two plus two)
|
||||
import re
|
||||
math_patterns = [
|
||||
r'\d+\s*[\+\-\*\/]\s*\d+', # 2+2, 5-3
|
||||
r'\d+\s*plus\s*\d+', # 2 plus 2
|
||||
r'\d+\s*minus\s*\d+', # 5 minus 3
|
||||
r'\d+\s*times\s*\d+', # 4 times 5
|
||||
r'\d+\s*divided by\s*\d+', # 10 divided by 2
|
||||
]
|
||||
for pattern in math_patterns:
|
||||
if re.search(pattern, text_lower):
|
||||
return True
|
||||
|
||||
# Check for short implicit questions (2-5 words without filler)
|
||||
words = text_lower.split()
|
||||
if 2 <= len(words) <= 5:
|
||||
filler = {'um', 'uh', 'like', 'so', 'well', 'actually', 'basically'}
|
||||
content_words = [w for w in words if w not in filler]
|
||||
if len(content_words) >= 2:
|
||||
# "two plus two", "capital france", "python example"
|
||||
return True
|
||||
|
||||
# Starts with or contains a question word
|
||||
return any(text_lower.startswith(w) or f" {w} " in text_lower for w in _QUESTION_WORDS)
|
||||
if any(text_lower.startswith(w) or f" {w} " in text_lower for w in _QUESTION_WORDS):
|
||||
return True
|
||||
|
||||
# Check for command starters
|
||||
command_starters = ['tell me', 'explain', 'describe', 'show me', 'give me', 'find', 'search']
|
||||
for starter in command_starters:
|
||||
if text_lower.startswith(starter):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _generate_answer(self, question):
|
||||
"""Generate answer with interruption support"""
|
||||
# Debounce: skip exact repeat within same run
|
||||
if question.lower() == self._last_question.lower():
|
||||
return
|
||||
self._last_question = question
|
||||
|
||||
context = self.context_manager.get_context()
|
||||
answer = self.ai_engine.answer_question(question, context)
|
||||
self.overlay.show_answer(answer, question)
|
||||
self.logger.info(f"Q: {question}")
|
||||
self.logger.info(f"A: {answer}")
|
||||
print(f"💡 Answer: {answer}\n")
|
||||
# Check if we should interrupt current answer
|
||||
with self.answer_lock:
|
||||
if self.answering:
|
||||
print("🔄 Interrupting previous answer...")
|
||||
if hasattr(self.ai_engine, 'interrupt'):
|
||||
self.ai_engine.interrupt()
|
||||
|
||||
# Start answer generation in background thread
|
||||
threading.Thread(target=self._answer_worker, args=(question,), daemon=True).start()
|
||||
|
||||
def _answer_worker(self, question):
|
||||
"""Worker thread for answer generation"""
|
||||
with self.answer_lock:
|
||||
self.answering = True
|
||||
|
||||
try:
|
||||
# Get context and generate answer
|
||||
context = self.context_manager.get_context()
|
||||
answer = self.ai_engine.answer_question(question, context)
|
||||
|
||||
# Show answer in overlay
|
||||
self.overlay.show_answer(answer, question)
|
||||
self.logger.info(f"Q: {question}")
|
||||
self.logger.info(f"A: {answer}")
|
||||
print(f"\n💡 Answer: {answer}\n")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error generating answer: {e}")
|
||||
finally:
|
||||
with self.answer_lock:
|
||||
self.answering = False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Manual terminal input for testing
|
||||
@@ -114,14 +192,7 @@ class MeetingAssistant:
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def run(self):
|
||||
print("\n✅ Meeting Assistant Ready!")
|
||||
print("=" * 50)
|
||||
print("🎤 Listening for questions via microphone")
|
||||
print(" Answers appear in the overlay (top-right)")
|
||||
print(" Type a question here + Enter to test AI")
|
||||
print(" Ctrl+C to stop")
|
||||
print("=" * 50)
|
||||
|
||||
print("\n🎤 Starting audio listener...")
|
||||
self.overlay.start()
|
||||
self.audio_listener.start()
|
||||
|
||||
@@ -137,9 +208,12 @@ class MeetingAssistant:
|
||||
|
||||
def shutdown(self):
|
||||
print("\n🛑 Shutting down...")
|
||||
self.audio_listener.stop()
|
||||
self.overlay.stop()
|
||||
if hasattr(self, 'audio_listener'):
|
||||
self.audio_listener.stop()
|
||||
if hasattr(self, 'overlay'):
|
||||
self.overlay.stop()
|
||||
print("👋 Goodbye!")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
206
src/ai_engine.py
206
src/ai_engine.py
@@ -4,23 +4,26 @@ from pathlib import Path
|
||||
from collections import deque
|
||||
from datetime import datetime
|
||||
from rapidfuzz import fuzz
|
||||
import threading
|
||||
|
||||
|
||||
class AIEngine:
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.model = None
|
||||
|
||||
self.memory = deque(maxlen=8)
|
||||
self.fast_mode = True
|
||||
self.current_generation = None
|
||||
self.interrupt_event = threading.Event()
|
||||
self.load_model()
|
||||
|
||||
def load_model(self):
|
||||
print("🤖 Loading AI model...")
|
||||
|
||||
model_path = (
|
||||
Path(__file__).parent.parent
|
||||
/ "models"
|
||||
/ "Qwen2.5-7B-Instruct-Q4_K_M.gguf"
|
||||
Path(__file__).parent.parent
|
||||
/ "models"
|
||||
/ "Qwen2.5-7B-Instruct-Q4_K_M.gguf"
|
||||
)
|
||||
|
||||
from llama_cpp import Llama
|
||||
@@ -28,37 +31,59 @@ class AIEngine:
|
||||
self.model = Llama(
|
||||
model_path=str(model_path),
|
||||
|
||||
# PERFORMANCE
|
||||
# PERFORMANCE - MAXIMUM SPEED
|
||||
n_gpu_layers=-1,
|
||||
n_ctx=4096,
|
||||
n_batch=1024,
|
||||
n_ctx=2048, # Reduced from 4096 for speed
|
||||
n_batch=512, # Reduced for faster prompt processing
|
||||
n_threads=max(os.cpu_count() - 1, 1),
|
||||
|
||||
# SPEED
|
||||
# SPEED OPTIMIZATIONS
|
||||
offload_kqv=True,
|
||||
flash_attn=True,
|
||||
use_mmap=True,
|
||||
use_mlock=False,
|
||||
|
||||
# FASTER INFERENCE
|
||||
n_parts=-1,
|
||||
seed=42, # Deterministic for speed
|
||||
f16_kv=True, # Use half-precision for KV cache
|
||||
|
||||
# STABILITY
|
||||
verbose=False
|
||||
)
|
||||
|
||||
print("✅ AI ready")
|
||||
print("✅ AI ready (fast mode enabled)")
|
||||
|
||||
def answer_question(self, question, context, is_partial=False):
|
||||
"""Answer a question with interruption support"""
|
||||
|
||||
# Check for interrupt
|
||||
if self.interrupt_event.is_set():
|
||||
self.interrupt_event.clear()
|
||||
return None
|
||||
|
||||
def answer_question(self, question, context):
|
||||
question = self._normalize_question(question)
|
||||
|
||||
response = self._generate(question, context)
|
||||
# For partial questions, answer faster with fewer tokens
|
||||
if is_partial:
|
||||
response = self._generate_fast(question, context)
|
||||
else:
|
||||
response = self._generate(question, context)
|
||||
|
||||
self.memory.append({
|
||||
"question": question,
|
||||
"response": response,
|
||||
"time": datetime.now()
|
||||
})
|
||||
if response: # Only store if not interrupted
|
||||
self.memory.append({
|
||||
"question": question,
|
||||
"response": response,
|
||||
"time": datetime.now()
|
||||
})
|
||||
|
||||
return response
|
||||
|
||||
def interrupt(self):
|
||||
"""Interrupt current generation"""
|
||||
self.interrupt_event.set()
|
||||
print("🛑 Generation interrupted")
|
||||
|
||||
def _normalize_question(self, question):
|
||||
question = question.strip()
|
||||
|
||||
@@ -70,12 +95,10 @@ class AIEngine:
|
||||
}
|
||||
|
||||
words = question.split()
|
||||
|
||||
normalized = []
|
||||
|
||||
for word in words:
|
||||
lowered = word.lower()
|
||||
|
||||
if lowered in fixes:
|
||||
normalized.append(fixes[lowered])
|
||||
else:
|
||||
@@ -91,73 +114,105 @@ class AIEngine:
|
||||
|
||||
return question
|
||||
|
||||
def _generate(self, question, context):
|
||||
recent_audio = context.get("audio", "")[-1500:]
|
||||
recent_screen = context.get("screen", "")[-700:]
|
||||
def _generate_fast(self, question, context):
|
||||
"""Ultra-fast generation for partial/interruptible responses"""
|
||||
|
||||
memory_context = ""
|
||||
# Check for interrupt before generation
|
||||
if self.interrupt_event.is_set():
|
||||
self.interrupt_event.clear()
|
||||
return None
|
||||
|
||||
if self.memory:
|
||||
recent = list(self.memory)[-2:]
|
||||
# Simplified prompt for speed
|
||||
system_prompt = "You are a fast assistant. Answer in 1-2 short sentences max."
|
||||
|
||||
for item in recent:
|
||||
memory_context += (
|
||||
f"Q: {item['question']}\n"
|
||||
f"A: {item['response']}\n"
|
||||
)
|
||||
|
||||
system_prompt = """
|
||||
You are a realtime meeting copilot.
|
||||
|
||||
Rules:
|
||||
- Answer immediately.
|
||||
- Never ask for clarification unless impossible.
|
||||
- Never invent missing words.
|
||||
- Never complete partial speech.
|
||||
- Keep answers under 80 words.
|
||||
- Prioritize only explicit meaning.
|
||||
- Ignore minor transcription mistakes.
|
||||
- If speech is incomplete, answer conservatively.
|
||||
- Sound precise and direct.
|
||||
"""
|
||||
|
||||
user_prompt = f"""
|
||||
Recent conversation:
|
||||
{memory_context}
|
||||
|
||||
Meeting transcript:
|
||||
{recent_audio}
|
||||
|
||||
Screen:
|
||||
{recent_screen}
|
||||
|
||||
User question:
|
||||
{question}
|
||||
"""
|
||||
user_prompt = f"Q: {question}\nA:"
|
||||
|
||||
prompt = (
|
||||
f"<|im_start|>system\n"
|
||||
f"{system_prompt}"
|
||||
f"<|im_end|>\n"
|
||||
f"<|im_start|>user\n"
|
||||
f"{user_prompt}"
|
||||
f"<|im_end|>\n"
|
||||
f"<|im_start|>system\n{system_prompt}<|im_end|>\n"
|
||||
f"<|im_start|>user\n{user_prompt}<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
output = self.model(
|
||||
prompt,
|
||||
try:
|
||||
output = self.model(
|
||||
prompt,
|
||||
max_tokens=30, # Very short for fast responses
|
||||
temperature=0.1,
|
||||
top_p=0.9,
|
||||
repeat_penalty=1.0,
|
||||
stop=["<|im_end|>", "\n", ".", "!", "?"],
|
||||
echo=False
|
||||
)
|
||||
|
||||
max_tokens=120,
|
||||
temperature=0.2,
|
||||
top_p=0.85,
|
||||
repeat_penalty=1.05,
|
||||
stop=["<|im_end|>"]
|
||||
# Check for interrupt during generation
|
||||
if self.interrupt_event.is_set():
|
||||
self.interrupt_event.clear()
|
||||
return None
|
||||
|
||||
text = output["choices"][0]["text"]
|
||||
return self._clean(text)
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Fast generation error: {e}")
|
||||
return None
|
||||
|
||||
def _generate(self, question, context):
|
||||
"""Standard generation for complete questions"""
|
||||
|
||||
# Check for interrupt before generation
|
||||
if self.interrupt_event.is_set():
|
||||
self.interrupt_event.clear()
|
||||
return None
|
||||
|
||||
recent_audio = context.get("audio", "")[-1500:] if context else ""
|
||||
recent_screen = context.get("screen", "")[-700:] if context else ""
|
||||
|
||||
memory_context = ""
|
||||
if self.memory:
|
||||
recent = list(self.memory)[-2:]
|
||||
for item in recent:
|
||||
memory_context += f"Q: {item['question']}\nA: {item['response']}\n"
|
||||
|
||||
# Ultra-concise system prompt for speed
|
||||
system_prompt = """
|
||||
You are a realtime assistant. Answer immediately and concisely.
|
||||
Keep answers under 40 words. Be direct. No explanations unless asked.
|
||||
"""
|
||||
|
||||
user_prompt = f"""
|
||||
Q: {question}
|
||||
A:"""
|
||||
|
||||
prompt = (
|
||||
f"<|im_start|>system\n{system_prompt}<|im_end|>\n"
|
||||
f"<|im_start|>user\n{user_prompt}<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
text = output["choices"][0]["text"]
|
||||
try:
|
||||
output = self.model(
|
||||
prompt,
|
||||
max_tokens=60, # Reduced for speed
|
||||
temperature=0.1, # Lower for faster, more deterministic
|
||||
top_p=0.9,
|
||||
repeat_penalty=1.0, # Disabled for speed
|
||||
stop=["<|im_end|>", "\n", "."],
|
||||
echo=False,
|
||||
frequency_penalty=0.0,
|
||||
presence_penalty=0.0
|
||||
)
|
||||
|
||||
return self._clean(text)
|
||||
# Check for interrupt during generation
|
||||
if self.interrupt_event.is_set():
|
||||
self.interrupt_event.clear()
|
||||
return None
|
||||
|
||||
text = output["choices"][0]["text"]
|
||||
return self._clean(text)
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ Generation error: {e}")
|
||||
return "I couldn't process that."
|
||||
|
||||
def _clean(self, text):
|
||||
text = re.sub(r"<.*?>", "", text)
|
||||
@@ -166,7 +221,12 @@ User question:
|
||||
if not text:
|
||||
return "No response generated."
|
||||
|
||||
# Ensure it ends with punctuation
|
||||
if text[-1] not in ".!?":
|
||||
text += "."
|
||||
|
||||
# Capitalize first letter
|
||||
if text and text[0].islower():
|
||||
text = text[0].upper() + text[1:]
|
||||
|
||||
return text
|
||||
@@ -19,23 +19,22 @@ from faster_whisper import WhisperModel
|
||||
os.environ["HF_HUB_DISABLE_SSL_VERIFY"] = "1"
|
||||
|
||||
# ============================================================
|
||||
# 🔧 TUNING
|
||||
# 🔧 TUNING - OPTIMIZED FOR SPEED
|
||||
# ============================================================
|
||||
MIN_SPEECH_SECONDS = 1.2 # Need at least 1.2s of speech
|
||||
SILENCE_SECONDS = 0.7 # 0.7s silence to end
|
||||
MAX_SPEECH_SECONDS = 8.0 # Max before force
|
||||
MIN_SPEECH_SECONDS = 0.8
|
||||
SILENCE_SECONDS = 0.3
|
||||
MAX_SPEECH_SECONDS = 8.0
|
||||
SPEECH_THRESHOLD_MULTIPLIER = 1.3
|
||||
MIN_THRESHOLD_GAP = 400
|
||||
CONSECUTIVE_SPEECH_TO_START = 8 # Must have 8 consecutive speech frames to start
|
||||
CONSECUTIVE_SPEECH_TO_START = 4
|
||||
|
||||
|
||||
# ============================================================
|
||||
|
||||
|
||||
class AudioListener:
|
||||
def __init__(self, config, callback):
|
||||
self.config = config
|
||||
self.callback = callback
|
||||
self.callback = callback # Expects callback(text, timestamp)
|
||||
self.running = False
|
||||
|
||||
self.sample_rate = 16000
|
||||
@@ -52,8 +51,8 @@ class AudioListener:
|
||||
self.is_speaking = False
|
||||
self.speech_frames = 0
|
||||
self.silence_frames = 0
|
||||
self.consecutive_speech = 0 # Consecutive speech frames
|
||||
self.consecutive_silence = 0 # Consecutive silence frames
|
||||
self.consecutive_speech = 0
|
||||
self.consecutive_silence = 0
|
||||
self.total_frames_in_utterance = 0
|
||||
|
||||
self.noise_floor = 0
|
||||
@@ -66,34 +65,29 @@ class AudioListener:
|
||||
self.speech_start_time = 0
|
||||
|
||||
self.last_transcription_time = 0
|
||||
self.min_transcription_interval = 0.5
|
||||
self.min_transcription_interval = 0.3
|
||||
|
||||
self.calibrated = False
|
||||
|
||||
print("🔄 Loading Whisper model...")
|
||||
self.model = WhisperModel(
|
||||
"models/whisper",
|
||||
device="cpu",
|
||||
compute_type="int8",
|
||||
local_files_only=True,
|
||||
cpu_threads=max(os.cpu_count() - 1, 1)
|
||||
)
|
||||
print("✅ Whisper model ready")
|
||||
|
||||
self.command_starters = {
|
||||
"what", "why", "how", "when", "where", "who", "which",
|
||||
"can", "could", "would", "will", "do", "does", "did",
|
||||
"is", "are", "was", "were", "should", "shall", "may",
|
||||
"have", "has", "had", "am",
|
||||
"tell", "explain", "describe", "compare", "define",
|
||||
"find", "show", "give", "list", "name", "provide",
|
||||
"write", "create", "make", "generate", "build", "code",
|
||||
"calculate", "compute", "solve", "convert", "translate",
|
||||
"search", "look", "check", "get", "fetch", "run",
|
||||
"start", "stop", "open", "close", "save", "delete",
|
||||
"draw", "plot", "graph", "print", "display", "output",
|
||||
"summarize", "analyze", "review", "evaluate",
|
||||
}
|
||||
try:
|
||||
self.model = WhisperModel(
|
||||
"models/whisper",
|
||||
device="cpu",
|
||||
compute_type="int8",
|
||||
local_files_only=True,
|
||||
cpu_threads=max(os.cpu_count() - 1, 1)
|
||||
)
|
||||
print("✅ Whisper model ready")
|
||||
except Exception as e:
|
||||
print(f"⚠️ Could not load Whisper from models/whisper, trying default: {e}")
|
||||
self.model = WhisperModel(
|
||||
"base",
|
||||
device="cpu",
|
||||
compute_type="int8",
|
||||
cpu_threads=max(os.cpu_count() - 1, 1)
|
||||
)
|
||||
print("✅ Whisper model ready (using default 'base' model)")
|
||||
|
||||
def start(self):
|
||||
self.running = True
|
||||
@@ -129,7 +123,7 @@ class AudioListener:
|
||||
rms = np.sqrt(np.mean(chunk ** 2))
|
||||
rms_values.append(rms)
|
||||
|
||||
self.noise_floor = np.median(rms_values)
|
||||
self.noise_floor = np.median(rms_values) if rms_values else 1500
|
||||
|
||||
self.speech_threshold = max(
|
||||
self.noise_floor * SPEECH_THRESHOLD_MULTIPLIER,
|
||||
@@ -150,7 +144,7 @@ class AudioListener:
|
||||
self.silence_threshold = 1700
|
||||
self.calibrated = True
|
||||
|
||||
print(f"\n📋 Settings:")
|
||||
print(f"\n📋 Settings (FAST MODE):")
|
||||
print(f" Min speech: {MIN_SPEECH_SECONDS}s | Max: {MAX_SPEECH_SECONDS}s")
|
||||
print(f" Silence to end: {SILENCE_SECONDS}s")
|
||||
print(f" Consecutive speech to start: {CONSECUTIVE_SPEECH_TO_START} frames")
|
||||
@@ -199,29 +193,23 @@ class AudioListener:
|
||||
self.audio_queue.put((audio_bytes, rms))
|
||||
|
||||
def _processing_loop(self):
|
||||
# Pre-speech buffer - capture audio BEFORE we confirm speech
|
||||
pre_speech_buffer = bytearray()
|
||||
pre_speech_frames = 0
|
||||
MAX_PRE_SPEECH = 15 # Keep ~0.45s of audio before speech confirmation
|
||||
MAX_PRE_SPEECH = 10
|
||||
|
||||
while self.running:
|
||||
try:
|
||||
audio_bytes, rms = self.audio_queue.get(timeout=1)
|
||||
audio_bytes, rms = self.audio_queue.get(timeout=0.5)
|
||||
self.recent_rms.append(rms)
|
||||
|
||||
# Always keep a small buffer of recent audio
|
||||
pre_speech_buffer.extend(audio_bytes)
|
||||
pre_speech_frames += 1
|
||||
if pre_speech_frames > MAX_PRE_SPEECH:
|
||||
# Trim oldest frames
|
||||
excess = pre_speech_frames - MAX_PRE_SPEECH
|
||||
bytes_to_trim = excess * self.frame_size
|
||||
pre_speech_buffer = pre_speech_buffer[bytes_to_trim:]
|
||||
pre_speech_frames = MAX_PRE_SPEECH
|
||||
|
||||
# ========================
|
||||
# FRAME CLASSIFICATION
|
||||
# ========================
|
||||
is_voice_frame = False
|
||||
if rms >= self.speech_threshold:
|
||||
try:
|
||||
@@ -229,21 +217,15 @@ class AudioListener:
|
||||
except:
|
||||
is_voice_frame = True
|
||||
|
||||
# ========================
|
||||
# STATE: NOT SPEAKING
|
||||
# ========================
|
||||
if not self.is_speaking:
|
||||
if is_voice_frame:
|
||||
self.consecutive_speech += 1
|
||||
self.consecutive_silence = 0
|
||||
|
||||
# Need CONSECUTIVE_SPEECH_TO_START frames to confirm speech
|
||||
if self.consecutive_speech >= CONSECUTIVE_SPEECH_TO_START:
|
||||
# CONFIRMED SPEECH - start capturing
|
||||
self.is_speaking = True
|
||||
self.speech_start_time = time.time()
|
||||
|
||||
# Include pre-speech buffer for context
|
||||
self.current_audio = bytearray(pre_speech_buffer)
|
||||
self.speech_frames = pre_speech_frames
|
||||
self.silence_frames = 0
|
||||
@@ -255,9 +237,6 @@ class AudioListener:
|
||||
self.consecutive_speech = 0
|
||||
self.consecutive_silence += 1
|
||||
|
||||
# ========================
|
||||
# STATE: SPEAKING
|
||||
# ========================
|
||||
else:
|
||||
self.current_audio.extend(audio_bytes)
|
||||
self.speech_frames += 1
|
||||
@@ -266,55 +245,38 @@ class AudioListener:
|
||||
if rms > self.peak_rms:
|
||||
self.peak_rms = rms
|
||||
|
||||
# Dynamic silence threshold based on actual speech levels
|
||||
if len(self.speech_rms_values) > 15:
|
||||
speech_median = np.median(self.speech_rms_values)
|
||||
# Silence = below 60% of your speech median, or below noise-based threshold
|
||||
dynamic_silence = max(
|
||||
self.silence_threshold,
|
||||
speech_median * 0.6
|
||||
)
|
||||
dynamic_silence = max(self.silence_threshold, speech_median * 0.6)
|
||||
else:
|
||||
dynamic_silence = self.silence_threshold
|
||||
|
||||
# Check if this frame is silence
|
||||
if rms < dynamic_silence and not is_voice_frame:
|
||||
self.silence_frames += 1
|
||||
else:
|
||||
# Reset silence counter if we hear voice
|
||||
if is_voice_frame:
|
||||
self.silence_frames = 0
|
||||
|
||||
# ========================
|
||||
# DEBUG DISPLAY
|
||||
# ========================
|
||||
if self.is_speaking:
|
||||
bar_len = max(0, min(int((rms - self.noise_floor) / 60), 35))
|
||||
bar = "█" * bar_len
|
||||
sil = f" 🔇{self.silence_frames}" if self.silence_frames > 0 else ""
|
||||
dyn = f" thr:{dynamic_silence:.0f}" if len(self.speech_rms_values) > 15 else ""
|
||||
print(f"\r🎙️ {rms:5d} |{bar}{sil}{dyn} ", end="")
|
||||
print(f"\r🎙️ {rms:5d} |{bar}{sil} ", end="")
|
||||
elif self.consecutive_speech > 0:
|
||||
print(f"\r👂 {rms:5d} | detecting... {self.consecutive_speech}/{CONSECUTIVE_SPEECH_TO_START} ",
|
||||
end="")
|
||||
|
||||
# ========================
|
||||
# TRANSCRIPTION TRIGGERS
|
||||
# ========================
|
||||
if self.is_speaking:
|
||||
speech_duration = time.time() - self.speech_start_time
|
||||
silence_duration = (self.silence_frames * self.frame_duration_ms) / 1000
|
||||
|
||||
# Trigger 1: Sufficient silence after minimum speech
|
||||
if (speech_duration >= MIN_SPEECH_SECONDS and
|
||||
silence_duration >= SILENCE_SECONDS):
|
||||
if (speech_duration >= MIN_SPEECH_SECONDS and silence_duration >= SILENCE_SECONDS):
|
||||
print(f"\n✅ End ({speech_duration:.1f}s, peak: {self.peak_rms})")
|
||||
self._safe_transcribe()
|
||||
self._reset_speech_state()
|
||||
pre_speech_buffer = bytearray()
|
||||
pre_speech_frames = 0
|
||||
|
||||
# Trigger 2: Max duration
|
||||
elif speech_duration >= MAX_SPEECH_SECONDS:
|
||||
print(f"\n⏰ Max ({speech_duration:.1f}s, peak: {self.peak_rms})")
|
||||
self._safe_transcribe()
|
||||
@@ -353,7 +315,7 @@ class AudioListener:
|
||||
self.last_transcription_time = current_time
|
||||
|
||||
audio_duration = len(self.current_audio) / (2 * self.sample_rate)
|
||||
if audio_duration < 0.5:
|
||||
if audio_duration < 0.3:
|
||||
print(" (Too short)")
|
||||
return
|
||||
|
||||
@@ -366,13 +328,13 @@ class AudioListener:
|
||||
segments, info = self.model.transcribe(
|
||||
audio_np,
|
||||
language="en",
|
||||
beam_size=5,
|
||||
best_of=5,
|
||||
temperature=[0.0, 0.2, 0.4],
|
||||
beam_size=3,
|
||||
best_of=3,
|
||||
temperature=[0.0, 0.2],
|
||||
condition_on_previous_text=False,
|
||||
compression_ratio_threshold=1.8,
|
||||
no_speech_threshold=0.5,
|
||||
log_prob_threshold=-0.8,
|
||||
no_speech_threshold=0.6,
|
||||
log_prob_threshold=-1.0,
|
||||
word_timestamps=False,
|
||||
vad_filter=True,
|
||||
)
|
||||
@@ -405,7 +367,9 @@ class AudioListener:
|
||||
tag = "🎯 Command" if is_command else "📝 Speech"
|
||||
print(f"{tag}: {full_text}")
|
||||
|
||||
self.callback(full_text, datetime.now())
|
||||
# Callback with just text and timestamp (original format)
|
||||
if self.callback:
|
||||
self.callback(full_text, datetime.now())
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Transcription error: {e}")
|
||||
@@ -428,39 +392,65 @@ class AudioListener:
|
||||
if unique_ratio < 0.3 and len(words) > 8:
|
||||
return True
|
||||
mid = len(words) // 2
|
||||
first = " ".join(words[:mid])
|
||||
second = " ".join(words[mid:mid * 2])
|
||||
if first == second and len(first) > 15:
|
||||
return True
|
||||
if mid > 0:
|
||||
first = " ".join(words[:mid])
|
||||
second = " ".join(words[mid:mid * 2])
|
||||
if first == second and len(first) > 15:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _is_command(self, text):
|
||||
"""Aggressive command/question detection"""
|
||||
if not text or len(text) < 2:
|
||||
return False
|
||||
|
||||
lowered = text.lower().strip()
|
||||
words = lowered.split()
|
||||
if words and words[0] in self.command_starters:
|
||||
return True
|
||||
command_phrases = [
|
||||
"what is", "what are", "how do", "how to", "how can",
|
||||
"can you", "could you", "will you", "would you",
|
||||
"tell me", "explain", "describe", "show me",
|
||||
"i want", "i need", "please", "give me",
|
||||
"where is", "when did", "why do", "why is",
|
||||
"difference between", "compare",
|
||||
"write a", "write me", "create a", "create me",
|
||||
"generate", "make a", "build a", "code a",
|
||||
"calculate", "compute", "solve", "find",
|
||||
"list", "name", "provide", "search for",
|
||||
"what does", "what would", "what if",
|
||||
]
|
||||
for phrase in command_phrases:
|
||||
if phrase in lowered:
|
||||
return True
|
||||
|
||||
# Always command if ends with ?
|
||||
if lowered.endswith('?'):
|
||||
return True
|
||||
if len(words) >= 3:
|
||||
filler_words = {"um", "uh", "like", "you know", "i mean", "okay", "alright", "so", "well", "actually",
|
||||
"basically"}
|
||||
content_words = [w for w in words if w not in filler_words]
|
||||
if len(content_words) >= 3:
|
||||
|
||||
# Math expressions
|
||||
math_patterns = [
|
||||
r'\d+\s*[\+\-\*\/]\s*\d+',
|
||||
r'\d+\s*plus\s*\d+',
|
||||
r'\d+\s*minus\s*\d+',
|
||||
r'\d+\s*times\s*\d+',
|
||||
r'\d+\s*divided by\s*\d+',
|
||||
]
|
||||
import re
|
||||
for pattern in math_patterns:
|
||||
if re.search(pattern, lowered):
|
||||
return True
|
||||
|
||||
# Question starters
|
||||
starters = {
|
||||
'what', 'why', 'how', 'when', 'where', 'who', 'which',
|
||||
'can', 'could', 'would', 'will', 'do', 'does', 'did',
|
||||
'is', 'are', 'was', 'were', 'should', 'shall', 'may',
|
||||
'tell', 'explain', 'describe', 'show', 'give'
|
||||
}
|
||||
words = lowered.split()
|
||||
if words and words[0] in starters:
|
||||
return True
|
||||
|
||||
# Question phrases
|
||||
question_phrases = [
|
||||
'what is', 'what are', 'how do', 'how to', 'can you',
|
||||
'tell me', 'explain', 'describe', 'show me'
|
||||
]
|
||||
for phrase in question_phrases:
|
||||
if phrase in lowered:
|
||||
return True
|
||||
|
||||
# Short statements (2-6 words) - treat as commands/questions
|
||||
if 2 <= len(words) <= 6:
|
||||
filler = {'um', 'uh', 'like', 'you', 'know', 'so', 'well'}
|
||||
content_words = [w for w in words if w not in filler]
|
||||
if len(content_words) >= 2:
|
||||
if any(op in lowered for op in ['plus', 'minus', 'times']):
|
||||
return True
|
||||
if len(content_words) <= 4:
|
||||
return True
|
||||
|
||||
return False
|
||||
Reference in New Issue
Block a user