Alpha/InterviewAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit: meeting assistant

Browse Source
This commit is contained in:
Charles Wambua
2026-05-22 15:16:01 +03:00
commit b7b7397876
18 changed files with 155011 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
.gitignore vendored Normal file
View File

@@ -0,0 +1,104 @@
# Python
__pycache__/
*.py[cod]
*$py.class
*.so
.Python
*.egg-info/
dist/
build/
eggs/
.eggs/
*.egg
# Virtual Environment
.venv/
venv/
env/
ENV/
env.bak/
venv.bak/
# IDE and Editor files
.idea/
.vscode/
*.swp
*.swo
*~
.DS_Store
*.DS_Store
# Logs
logs/
*.log
# Configuration files with sensitive data
config.yaml
config/local.yaml
config/production.yaml
*.local.yaml
*.secret.yaml
# Model files (large binary files)
models/*.gguf
models/*.bin
models/whisper/model.bin
models/whisper/*.gguf
# Cache directories
.cache/
.pytest_cache/
.mypy_cache/
.ruff_cache/
# Environment variables
.env
.env.local
.env.production
# Database files
*.db
*.sqlite
*.sqlite3
# OS generated files
Thumbs.db
ehthumbs.db
Desktop.ini
# Application specific
*.pyc
*.pyo
*.pyd
# Jupyter Notebook
.ipynb_checkpoints/
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
coverage.xml
*.cover
.hypothesis/
# pyenv
.python-version
# pipenv
Pipfile.lock
# poetry
poetry.lock
# pdm
.pdm.toml
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# pyre type checker
.pyre/models/whisper/model.bin

14
config/settings.py Normal file
View File

@@ -0,0 +1,14 @@
class Config:
def __init__(self):
# Audio settings
self.sample_rate = 16000
self.chunk_duration = 2.0
self.vad_threshold = 0.5
# AI settings
self.context_window = 8192
self.max_history = 10
# Recognition settings
self.whisper_model = "base" # 'tiny', 'base', 'small', 'medium'
self.language = "en"

151
main.py Normal file
View File

@@ -0,0 +1,151 @@
#!/usr/bin/env python3
"""
Meeting Assistant - Real-time AI Copilot for Meetings
"""
import sys
import os
import yaml
import threading
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent))
from src.audio_listener import AudioListener
from src.ai_engine import AIEngine
from src.overlay import InvisibleOverlay
from src.context_manager import ContextManager
from src.utils import setup_logging, get_platform
# Question words that indicate the speaker wants an answer
_QUESTION_WORDS = (
"what", "why", "how", "when", "where", "who", "which",
"can you", "could you", "would you", "do you", "did you",
"is there", "are there", "is it", "are you",
"tell me", "explain", "describe", "define",
"your name", "help me",
)
# Phrases to skip even if they contain question words (noise)
_NOISE_PHRASES = (
"timer cannot", "qobject", "qml", "pyside", "pyqt",
"def ", "class ", "import ", "return ", "self.",
)
class MeetingAssistant:
def __init__(self, config_path="config.yaml"):
print("🚀 Starting Meeting Assistant...")
config_file = Path(__file__).parent / config_path
with open(config_file, "r") as f:
self.config = yaml.safe_load(f)
self.logger = setup_logging()
print(f"💻 Platform: {get_platform()}")
self.context_manager = ContextManager(self.config)
self.ai_engine = AIEngine(self.config)
self.overlay = InvisibleOverlay(self.config)
self.audio_listener = AudioListener(self.config, self.on_audio_transcript)
self._last_question = "" # Simple debounce
# ------------------------------------------------------------------
# Audio callback — runs in the audio listener thread
# ------------------------------------------------------------------
def on_audio_transcript(self, text, timestamp):
self.context_manager.add_audio_context(text, timestamp)
if self._is_question(text):
print(f"❓ Question: {text}")
self._generate_answer(text)
else:
print(f" Context: {text}")
def _is_question(self, text):
if len(text.split()) < 3:
return False
text_lower = text.lower().strip()
# Skip obvious noise / code text
if any(p in text_lower for p in _NOISE_PHRASES):
return False
# Ends with "?" → always a question
if text.rstrip().endswith("?"):
return True
# Starts with or contains a question word
return any(text_lower.startswith(w) or f" {w} " in text_lower for w in _QUESTION_WORDS)
def _generate_answer(self, question):
# Debounce: skip exact repeat within same run
if question.lower() == self._last_question.lower():
return
self._last_question = question
context = self.context_manager.get_context()
answer = self.ai_engine.answer_question(question, context)
self.overlay.show_answer(answer, question)
self.logger.info(f"Q: {question}")
self.logger.info(f"A: {answer}")
print(f"💡 Answer: {answer}\n")
# ------------------------------------------------------------------
# Manual terminal input for testing
# ------------------------------------------------------------------
def _terminal_input_loop(self):
print("\n Type a question and press Enter to test the AI directly.")
print(" (Leave blank to skip)\n")
while True:
try:
line = input()
if line.strip():
self._generate_answer(line.strip())
except (EOFError, KeyboardInterrupt):
break
# ------------------------------------------------------------------
# Run / shutdown
# ------------------------------------------------------------------
def run(self):
print("\n✅ Meeting Assistant Ready!")
print("=" * 50)
print("🎤 Listening for questions via microphone")
print(" Answers appear in the overlay (top-right)")
print(" Type a question here + Enter to test AI")
print(" Ctrl+C to stop")
print("=" * 50)
self.overlay.start()
self.audio_listener.start()
# Terminal input runs in a background daemon thread
t = threading.Thread(target=self._terminal_input_loop, daemon=True)
t.start()
try:
if self.overlay.app:
sys.exit(self.overlay.app.exec_())
except KeyboardInterrupt:
self.shutdown()
def shutdown(self):
print("\n🛑 Shutting down...")
self.audio_listener.stop()
self.overlay.stop()
print("👋 Goodbye!")
def main():
assistant = MeetingAssistant()
assistant.run()
if __name__ == "__main__":
main()

42
models/whisper/README.md Normal file
View File

@@ -0,0 +1,42 @@
---
language:
- en
tags:
- audio
- automatic-speech-recognition
license: mit
library_name: ctranslate2
---
# Whisper base.en model for CTranslate2
This repository contains the conversion of [openai/whisper-base.en](https://huggingface.co/openai/whisper-base.en) to the [CTranslate2](https://github.com/OpenNMT/CTranslate2) model format.
This model can be used in CTranslate2 or projects based on CTranslate2 such as [faster-whisper](https://github.com/systran/faster-whisper).
## Example
```python
from faster_whisper import WhisperModel
model = WhisperModel("base.en")
segments, info = model.transcribe("audio.mp3")
for segment in segments:
print("[%.2fs -> %.2fs] %s" % (segment.start, segment.end, segment.text))
```
## Conversion details
The original model was converted with the following command:
```
ct2-transformers-converter --model openai/whisper-base.en --output_dir faster-whisper-base.en \
--copy_files tokenizer.json --quantization float16
```
Note that the model weights are saved in FP16. This type can be changed when the model is loaded using the [`compute_type` option in CTranslate2](https://opennmt.net/CTranslate2/quantization.html).
## More information
**For more information about the original model, see its [model card](https://huggingface.co/openai/whisper-base.en).**

220
models/whisper/config.json Normal file
View File

@@ -0,0 +1,220 @@
{
"alignment_heads": [
[
3,
3
],
[
4,
7
],
[
5,
1
],
[
5,
5
],
[
5,
7
]
],
"lang_ids": [
50259,
50260,
50261,
50262,
50263,
50264,
50265,
50266,
50267,
50268,
50269,
50270,
50271,
50272,
50273,
50274,
50275,
50276,
50277,
50278,
50279,
50280,
50281,
50282,
50283,
50284,
50285,
50286,
50287,
50288,
50289,
50290,
50291,
50292,
50293,
50294,
50295,
50296,
50297,
50298,
50299,
50300,
50301,
50302,
50303,
50304,
50305,
50306,
50307,
50308,
50309,
50310,
50311,
50312,
50313,
50314,
50315,
50316,
50317,
50318,
50319,
50320,
50321,
50322,
50323,
50324,
50325,
50326,
50327,
50328,
50329,
50330,
50331,
50332,
50333,
50334,
50335,
50336,
50337,
50338,
50339,
50340,
50341,
50342,
50343,
50344,
50345,
50346,
50347,
50348,
50349,
50350,
50351,
50352,
50353,
50354,
50355,
50356
],
"suppress_ids": [
1,
2,
7,
8,
9,
10,
14,
25,
26,
27,
28,
29,
31,
58,
59,
60,
61,
62,
63,
90,
91,
92,
93,
357,
366,
438,
532,
685,
705,
796,
930,
1058,
1220,
1267,
1279,
1303,
1343,
1377,
1391,
1635,
1782,
1875,
2162,
2361,
2488,
3467,
4008,
4211,
4600,
4808,
5299,
5855,
6329,
7203,
9609,
9959,
10563,
10786,
11420,
11709,
11907,
13163,
13697,
13700,
14808,
15306,
16410,
16791,
17992,
19203,
19510,
20724,
22305,
22935,
27007,
30109,
30420,
33409,
34949,
40283,
40493,
40549,
47282,
49146,
50257,
50357,
50358,
50359,
50360,
50361
],
"suppress_ids_begin": [
220,
50256
]
}

101342
models/whisper/tokenizer.json Normal file
View File

File diff suppressed because it is too large Load Diff

51864
models/whisper/vocabulary.txt Normal file
View File

File diff suppressed because it is too large Load Diff

35
requirements.txt Normal file
View File

@@ -0,0 +1,35 @@
# Core dependencies
torch
torchvision
# Audio processing
sounddevice
SpeechRecognition
webrtcvad
pyaudio; platform_system == "Windows"
# Screen capture & OCR
mss
Pillow
pytesseract
easyocr
opencv-python
# Local LLM
ctransformers
llama-cpp-python
# GUI
PyQt5
pyobjc-framework-Cocoa; platform_system == "Darwin"
pywin32; platform_system == "Windows"
# Utilities
pyautogui
keyboard
pyyaml
numpy
scipy
faster-whisper
pyaudio
sentence-transformers

14
run.bat Normal file
View File

@@ -0,0 +1,14 @@
@echo off
echo Starting Meeting Assistant...
REM Check Python
python --version >nul 2>&1
if errorlevel 1 (
echo Python not found. Please install Python 3.8+
pause
exit /b 1
)
REM Run the assistant
python main.py
pause

26
run.sh Executable file
View File

@@ -0,0 +1,26 @@
#!/bin/bash
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$SCRIPT_DIR"
echo "🎙️ Starting Meeting Assistant..."
# Activate virtual environment
if [ -d ".venv" ]; then
source .venv/bin/activate
echo "✅ Using virtual environment: .venv"
else
echo "⚠️ No .venv found — using system Python"
echo " Run: python3 -m venv .venv && source .venv/bin/activate && pip install -r requirements.txt"
fi
# Install PortAudio if needed (macOS)
if [[ "$OSTYPE" == "darwin"* ]]; then
if command -v brew &> /dev/null && ! brew list portaudio &> /dev/null 2>&1; then
echo "📦 Installing PortAudio via Homebrew..."
brew install portaudio
fi
fi
python3 main.py

94
setup.py Normal file
View File

@@ -0,0 +1,94 @@
#!/usr/bin/env python3
"""
First-time setup script for Meeting Assistant
Downloads required AI models and sets up environment
"""
import os
import sys
import subprocess
import platform
def install_dependencies():
"""Install Python dependencies"""
print("📦 Installing Python dependencies...")
subprocess.check_call([sys.executable, "-m", "pip", "install", "-r", "requirements.txt"])
def download_models():
"""Download AI models"""
print("🤖 Downloading AI models...")
# Create models directory
os.makedirs("models", exist_ok=True)
# Download TinyLlama (small, fast, works on CPU)
model_urls = {
"tinyllama-1.1b.Q4_K_M.gguf": "https://huggingface.co/TheBloke/TinyLlama-1.1B-GGUF/resolve/main/tinyllama-1.1b.Q4_K_M.gguf"
}
import urllib.request
for model_name, url in model_urls.items():
model_path = os.path.join("models", model_name)
if not os.path.exists(model_path):
print(f"Downloading {model_name}...")
try:
urllib.request.urlretrieve(url, model_path)
print(f"✅ Downloaded {model_name}")
except Exception as e:
print(f"❌ Failed to download {model_name}: {e}")
print("⚠️ Will run in fallback mode without local LLM")
else:
print(f"✅ {model_name} already exists")
def setup_audio():
"""Test audio setup"""
print("🎤 Testing audio input...")
try:
import sounddevice as sd
devices = sd.query_devices()
input_devices = [d for d in devices if d['max_input_channels'] > 0]
if input_devices:
print(f"✅ Found {len(input_devices)} audio input devices")
for device in input_devices:
print(f" - {device['name']}")
else:
print("⚠️ No audio input devices found")
except Exception as e:
print(f"⚠️ Audio setup error: {e}")
def main():
"""Run setup"""
print("🔧 Meeting Assistant - Setup")
print("=" * 50)
# Check Python version
if sys.version_info < (3, 8):
print("❌ Python 3.8 or higher required")
sys.exit(1)
# Platform info
print(f"💻 Platform: {platform.system()}")
# Install dependencies
install_dependencies()
# Download models
download_models()
# Setup audio
setup_audio()
print("\n✅ Setup complete!")
print("Run 'python main.py' to start the assistant")
if __name__ == "__main__":
main()

0
src/__init__.py Normal file
View File

172
src/ai_engine.py Normal file
View File

@@ -0,0 +1,172 @@
import os
import re
from pathlib import Path
from collections import deque
from datetime import datetime
from rapidfuzz import fuzz
class AIEngine:
def __init__(self, config):
self.config = config
self.model = None
self.memory = deque(maxlen=8)
self.load_model()
def load_model(self):
print("🤖 Loading AI model...")
model_path = (
Path(__file__).parent.parent
/ "models"
/ "Qwen2.5-7B-Instruct-Q4_K_M.gguf"
)
from llama_cpp import Llama
self.model = Llama(
model_path=str(model_path),
# PERFORMANCE
n_gpu_layers=-1,
n_ctx=4096,
n_batch=1024,
n_threads=max(os.cpu_count() - 1, 1),
# SPEED
offload_kqv=True,
flash_attn=True,
use_mmap=True,
use_mlock=False,
# STABILITY
verbose=False
)
print("✅ AI ready")
def answer_question(self, question, context):
question = self._normalize_question(question)
response = self._generate(question, context)
self.memory.append({
"question": question,
"response": response,
"time": datetime.now()
})
return response
def _normalize_question(self, question):
question = question.strip()
fixes = {
"jav": "java",
"py": "python",
"js": "javascript",
"api": "API"
}
words = question.split()
normalized = []
for word in words:
lowered = word.lower()
if lowered in fixes:
normalized.append(fixes[lowered])
else:
normalized.append(word)
question = " ".join(normalized)
# Fix cut-off endings
if question.endswith("between"):
if self.memory:
last = self.memory[-1]["question"]
question += f" and {last}"
return question
def _generate(self, question, context):
recent_audio = context.get("audio", "")[-1500:]
recent_screen = context.get("screen", "")[-700:]
memory_context = ""
if self.memory:
recent = list(self.memory)[-2:]
for item in recent:
memory_context += (
f"Q: {item['question']}\n"
f"A: {item['response']}\n"
)
system_prompt = """
You are a realtime meeting copilot.
Rules:
- Answer immediately.
- Never ask for clarification unless impossible.
- Never invent missing words.
- Never complete partial speech.
- Keep answers under 80 words.
- Prioritize only explicit meaning.
- Ignore minor transcription mistakes.
- If speech is incomplete, answer conservatively.
- Sound precise and direct.
"""
user_prompt = f"""
Recent conversation:
{memory_context}
Meeting transcript:
{recent_audio}
Screen:
{recent_screen}
User question:
{question}
"""
prompt = (
f"<|im_start|>system\n"
f"{system_prompt}"
f"<|im_end|>\n"
f"<|im_start|>user\n"
f"{user_prompt}"
f"<|im_end|>\n"
f"<|im_start|>assistant\n"
)
output = self.model(
prompt,
max_tokens=120,
temperature=0.2,
top_p=0.85,
repeat_penalty=1.05,
stop=["<|im_end|>"]
)
text = output["choices"][0]["text"]
return self._clean(text)
def _clean(self, text):
text = re.sub(r"<.*?>", "", text)
text = re.sub(r"\s+", " ", text).strip()
if not text:
return "No response generated."
if text[-1] not in ".!?":
text += "."
return text

466
src/audio_listener.py Normal file
View File

@@ -0,0 +1,466 @@
import os
import queue
import threading
import time
import re
from collections import deque
from datetime import datetime
import numpy as np
import sounddevice as sd
import webrtcvad
import logging
logging.getLogger("faster_whisper").setLevel(logging.WARNING)
logging.getLogger("ctranslate2").setLevel(logging.WARNING)
from faster_whisper import WhisperModel
os.environ["HF_HUB_DISABLE_SSL_VERIFY"] = "1"
# ============================================================
# 🔧 TUNING
# ============================================================
MIN_SPEECH_SECONDS = 1.2 # Need at least 1.2s of speech
SILENCE_SECONDS = 0.7 # 0.7s silence to end
MAX_SPEECH_SECONDS = 8.0 # Max before force
SPEECH_THRESHOLD_MULTIPLIER = 1.3
MIN_THRESHOLD_GAP = 400
CONSECUTIVE_SPEECH_TO_START = 8 # Must have 8 consecutive speech frames to start
# ============================================================
class AudioListener:
def __init__(self, config, callback):
self.config = config
self.callback = callback
self.running = False
self.sample_rate = 16000
self.frame_duration_ms = 30
self.frame_size = int(self.sample_rate * self.frame_duration_ms / 1000)
self.vad = webrtcvad.Vad(2)
self.audio_queue = queue.Queue()
self.current_audio = bytearray()
self.history = deque(maxlen=20)
# Speech state machine
self.is_speaking = False
self.speech_frames = 0
self.silence_frames = 0
self.consecutive_speech = 0 # Consecutive speech frames
self.consecutive_silence = 0 # Consecutive silence frames
self.total_frames_in_utterance = 0
self.noise_floor = 0
self.speech_threshold = 2000
self.silence_threshold = 1500
self.recent_rms = deque(maxlen=30)
self.speech_rms_values = []
self.peak_rms = 0
self.speech_start_time = 0
self.last_transcription_time = 0
self.min_transcription_interval = 0.5
self.calibrated = False
print("🔄 Loading Whisper model...")
self.model = WhisperModel(
"models/whisper",
device="cpu",
compute_type="int8",
local_files_only=True,
cpu_threads=max(os.cpu_count() - 1, 1)
)
print("✅ Whisper model ready")
self.command_starters = {
"what", "why", "how", "when", "where", "who", "which",
"can", "could", "would", "will", "do", "does", "did",
"is", "are", "was", "were", "should", "shall", "may",
"have", "has", "had", "am",
"tell", "explain", "describe", "compare", "define",
"find", "show", "give", "list", "name", "provide",
"write", "create", "make", "generate", "build", "code",
"calculate", "compute", "solve", "convert", "translate",
"search", "look", "check", "get", "fetch", "run",
"start", "stop", "open", "close", "save", "delete",
"draw", "plot", "graph", "print", "display", "output",
"summarize", "analyze", "review", "evaluate",
}
def start(self):
self.running = True
print("\n🎤 Available audio devices:\n")
devices = sd.query_devices()
for i, dev in enumerate(devices):
print(f" [{i}] {dev['name']} (in: {dev['max_input_channels']}, out: {dev['max_output_channels']})")
input_device = self._select_input_device()
device_name = sd.query_devices(input_device)['name']
print(f"\n🎤 Using input device: {device_name}\n")
print("🔧 Calibrating background noise (3 seconds)...")
print(" Please stay COMPLETELY silent...")
try:
calibration_audio = sd.rec(
int(3.0 * self.sample_rate),
samplerate=self.sample_rate,
channels=1,
dtype="int16",
device=input_device
)
sd.wait()
cal_np = np.frombuffer(calibration_audio.tobytes(), dtype=np.int16).astype(np.float32)
chunk_size = self.frame_size
rms_values = []
for i in range(0, len(cal_np) - chunk_size, chunk_size):
chunk = cal_np[i:i + chunk_size]
rms = np.sqrt(np.mean(chunk ** 2))
rms_values.append(rms)
self.noise_floor = np.median(rms_values)
self.speech_threshold = max(
self.noise_floor * SPEECH_THRESHOLD_MULTIPLIER,
self.noise_floor + MIN_THRESHOLD_GAP
)
self.silence_threshold = self.noise_floor * 1.15
print(f"📊 Noise floor: {self.noise_floor:.0f} RMS")
print(f"📊 Speech threshold: {self.speech_threshold:.0f} RMS")
print(f"📊 Silence threshold: {self.silence_threshold:.0f} RMS")
self.calibrated = True
except Exception as e:
print(f"⚠️ Calibration failed: {e}")
self.noise_floor = 1500
self.speech_threshold = 2200
self.silence_threshold = 1700
self.calibrated = True
print(f"\n📋 Settings:")
print(f" Min speech: {MIN_SPEECH_SECONDS}s | Max: {MAX_SPEECH_SECONDS}s")
print(f" Silence to end: {SILENCE_SECONDS}s")
print(f" Consecutive speech to start: {CONSECUTIVE_SPEECH_TO_START} frames")
self.stream = sd.RawInputStream(
samplerate=self.sample_rate,
blocksize=self.frame_size,
dtype="int16",
channels=1,
device=input_device,
callback=self._audio_callback
)
self.stream.start()
threading.Thread(target=self._processing_loop, daemon=True).start()
print(f"\n🎤 Listening... Speak now!")
print(" Press Ctrl+C to stop\n")
def _select_input_device(self):
devices = sd.query_devices()
default_input = sd.default.device[0]
if default_input is not None and default_input < len(devices):
dev = devices[default_input]
if dev['max_input_channels'] > 0:
return default_input
for i, dev in enumerate(devices):
if dev['max_input_channels'] > 0:
return i
return 0
def stop(self):
self.running = False
if hasattr(self, 'stream'):
self.stream.stop()
self.stream.close()
print("🛑 Stopped")
def _audio_callback(self, indata, frames, time_info, status):
if status:
print(f"⚠️ Audio status: {status}")
audio_bytes = bytes(indata)
rms = int(np.sqrt(np.mean(
np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32) ** 2
)))
self.audio_queue.put((audio_bytes, rms))
def _processing_loop(self):
# Pre-speech buffer - capture audio BEFORE we confirm speech
pre_speech_buffer = bytearray()
pre_speech_frames = 0
MAX_PRE_SPEECH = 15 # Keep ~0.45s of audio before speech confirmation
while self.running:
try:
audio_bytes, rms = self.audio_queue.get(timeout=1)
self.recent_rms.append(rms)
# Always keep a small buffer of recent audio
pre_speech_buffer.extend(audio_bytes)
pre_speech_frames += 1
if pre_speech_frames > MAX_PRE_SPEECH:
# Trim oldest frames
excess = pre_speech_frames - MAX_PRE_SPEECH
bytes_to_trim = excess * self.frame_size
pre_speech_buffer = pre_speech_buffer[bytes_to_trim:]
pre_speech_frames = MAX_PRE_SPEECH
# ========================
# FRAME CLASSIFICATION
# ========================
is_voice_frame = False
if rms >= self.speech_threshold:
try:
is_voice_frame = self.vad.is_speech(audio_bytes, self.sample_rate)
except:
is_voice_frame = True
# ========================
# STATE: NOT SPEAKING
# ========================
if not self.is_speaking:
if is_voice_frame:
self.consecutive_speech += 1
self.consecutive_silence = 0
# Need CONSECUTIVE_SPEECH_TO_START frames to confirm speech
if self.consecutive_speech >= CONSECUTIVE_SPEECH_TO_START:
# CONFIRMED SPEECH - start capturing
self.is_speaking = True
self.speech_start_time = time.time()
# Include pre-speech buffer for context
self.current_audio = bytearray(pre_speech_buffer)
self.speech_frames = pre_speech_frames
self.silence_frames = 0
self.speech_rms_values = list(self.recent_rms)[-pre_speech_frames:]
self.peak_rms = max(self.speech_rms_values) if self.speech_rms_values else rms
print(f"\n🔴 SPEAKING (peak: {self.peak_rms})")
else:
self.consecutive_speech = 0
self.consecutive_silence += 1
# ========================
# STATE: SPEAKING
# ========================
else:
self.current_audio.extend(audio_bytes)
self.speech_frames += 1
self.speech_rms_values.append(rms)
if rms > self.peak_rms:
self.peak_rms = rms
# Dynamic silence threshold based on actual speech levels
if len(self.speech_rms_values) > 15:
speech_median = np.median(self.speech_rms_values)
# Silence = below 60% of your speech median, or below noise-based threshold
dynamic_silence = max(
self.silence_threshold,
speech_median * 0.6
)
else:
dynamic_silence = self.silence_threshold
# Check if this frame is silence
if rms < dynamic_silence and not is_voice_frame:
self.silence_frames += 1
else:
# Reset silence counter if we hear voice
if is_voice_frame:
self.silence_frames = 0
# ========================
# DEBUG DISPLAY
# ========================
if self.is_speaking:
bar_len = max(0, min(int((rms - self.noise_floor) / 60), 35))
bar = "█" * bar_len
sil = f" 🔇{self.silence_frames}" if self.silence_frames > 0 else ""
dyn = f" thr:{dynamic_silence:.0f}" if len(self.speech_rms_values) > 15 else ""
print(f"\r🎙️ {rms:5d} |{bar}{sil}{dyn} ", end="")
elif self.consecutive_speech > 0:
print(f"\r👂 {rms:5d} | detecting... {self.consecutive_speech}/{CONSECUTIVE_SPEECH_TO_START} ",
end="")
# ========================
# TRANSCRIPTION TRIGGERS
# ========================
if self.is_speaking:
speech_duration = time.time() - self.speech_start_time
silence_duration = (self.silence_frames * self.frame_duration_ms) / 1000
# Trigger 1: Sufficient silence after minimum speech
if (speech_duration >= MIN_SPEECH_SECONDS and
silence_duration >= SILENCE_SECONDS):
print(f"\n✅ End ({speech_duration:.1f}s, peak: {self.peak_rms})")
self._safe_transcribe()
self._reset_speech_state()
pre_speech_buffer = bytearray()
pre_speech_frames = 0
# Trigger 2: Max duration
elif speech_duration >= MAX_SPEECH_SECONDS:
print(f"\n⏰ Max ({speech_duration:.1f}s, peak: {self.peak_rms})")
self._safe_transcribe()
self._reset_speech_state()
pre_speech_buffer = bytearray()
pre_speech_frames = 0
except queue.Empty:
if self.is_speaking:
elapsed = time.time() - self.speech_start_time
silence_duration = (self.silence_frames * self.frame_duration_ms) / 1000
if elapsed >= MIN_SPEECH_SECONDS and silence_duration >= SILENCE_SECONDS:
print(f"\n✅ Final ({elapsed:.1f}s)")
self._safe_transcribe()
self._reset_speech_state()
continue
except Exception as e:
print(f"\n❌ Error: {e}")
continue
def _reset_speech_state(self):
self.is_speaking = False
self.current_audio = bytearray()
self.speech_frames = 0
self.silence_frames = 0
self.consecutive_speech = 0
self.peak_rms = 0
self.speech_rms_values = []
def _safe_transcribe(self):
current_time = time.time()
if current_time - self.last_transcription_time < self.min_transcription_interval:
return
self.last_transcription_time = current_time
audio_duration = len(self.current_audio) / (2 * self.sample_rate)
if audio_duration < 0.5:
print(" (Too short)")
return
self._transcribe()
def _transcribe(self):
try:
audio_np = np.frombuffer(self.current_audio, dtype=np.int16).astype(np.float32) / 32768.0
segments, info = self.model.transcribe(
audio_np,
language="en",
beam_size=5,
best_of=5,
temperature=[0.0, 0.2, 0.4],
condition_on_previous_text=False,
compression_ratio_threshold=1.8,
no_speech_threshold=0.5,
log_prob_threshold=-0.8,
word_timestamps=False,
vad_filter=True,
)
text_parts = []
for segment in segments:
text = segment.text.strip()
if text:
text_parts.append(text)
full_text = " ".join(text_parts)
full_text = self._clean_text(full_text)
if not full_text:
print(" (No text)")
return
if self._is_repetitive(full_text):
print(f" 🚫 Repetitive")
return
print(f"\n🎤 \"{full_text}\"")
self.history.append({
"text": full_text,
"timestamp": datetime.now()
})
is_command = self._is_command(full_text)
tag = "🎯 Command" if is_command else "📝 Speech"
print(f"{tag}: {full_text}")
self.callback(full_text, datetime.now())
except Exception as e:
print(f"❌ Transcription error: {e}")
def _clean_text(self, text):
text = re.sub(r'\s+', ' ', text).strip()
if not text:
return ""
text = re.sub(r'\b(\w+)( \1\b){2,}', r'\1', text, flags=re.IGNORECASE)
text = ''.join(char for char in text if char.isprintable() or char in ' \t\n\r')
if text:
text = text[0].upper() + text[1:] if len(text) > 1 else text.upper()
return text.strip()
def _is_repetitive(self, text):
words = text.lower().split()
if len(words) < 4:
return False
unique_ratio = len(set(words)) / len(words)
if unique_ratio < 0.3 and len(words) > 8:
return True
mid = len(words) // 2
first = " ".join(words[:mid])
second = " ".join(words[mid:mid * 2])
if first == second and len(first) > 15:
return True
return False
def _is_command(self, text):
lowered = text.lower().strip()
words = lowered.split()
if words and words[0] in self.command_starters:
return True
command_phrases = [
"what is", "what are", "how do", "how to", "how can",
"can you", "could you", "will you", "would you",
"tell me", "explain", "describe", "show me",
"i want", "i need", "please", "give me",
"where is", "when did", "why do", "why is",
"difference between", "compare",
"write a", "write me", "create a", "create me",
"generate", "make a", "build a", "code a",
"calculate", "compute", "solve", "find",
"list", "name", "provide", "search for",
"what does", "what would", "what if",
]
for phrase in command_phrases:
if phrase in lowered:
return True
if lowered.endswith('?'):
return True
if len(words) >= 3:
filler_words = {"um", "uh", "like", "you know", "i mean", "okay", "alright", "so", "well", "actually",
"basically"}
content_words = [w for w in words if w not in filler_words]
if len(content_words) >= 3:
return True
return False

49
src/context_manager.py Normal file
View File

@@ -0,0 +1,49 @@
from collections import deque
import json
class ContextManager:
def __init__(self, config):
self.config = config
self.audio_context = deque(maxlen=10) # Last 10 audio chunks
self.screen_context = deque(maxlen=5) # Last 5 screen captures
self.last_text = ""
def add_audio_context(self, text, timestamp):
"""Add audio transcript to context"""
self.audio_context.append({
'text': text,
'timestamp': timestamp.isoformat(),
'type': 'audio'
})
self.last_text = text
def add_screen_context(self, text, timestamp, region):
"""Add screen text to context"""
self.screen_context.append({
'text': text,
'timestamp': timestamp.isoformat(),
'region': region,
'type': 'screen'
})
self.last_text = text
def get_context(self):
"""Get current context for LLM"""
context = {
'audio': ' '.join([item['text'] for item in self.audio_context]),
'screen': ' '.join([item['text'] for item in self.screen_context]),
'recent_audio': list(self.audio_context)[-3:],
'recent_screen': list(self.screen_context)[-2:]
}
return context
def get_last_text(self):
"""Get last detected text"""
return self.last_text
def clear(self):
"""Clear all context"""
self.audio_context.clear()
self.screen_context.clear()
self.last_text = ""

182
src/overlay.py Normal file
View File

@@ -0,0 +1,182 @@
import sys
import platform
from PyQt5.QtWidgets import (
QApplication,
QMainWindow,
QTextEdit,
QWidget,
QVBoxLayout,
QLabel,
)
from PyQt5.QtCore import Qt, QTimer, QObject, pyqtSignal
from PyQt5.QtGui import QFont, QTextCursor, QColor
class _Bridge(QObject):
"""Signal bridge so audio/AI threads can safely update the Qt UI."""
answer_ready = pyqtSignal(str, str) # answer, question
status_ready = pyqtSignal(str)
hide_now = pyqtSignal()
class InvisibleOverlay:
def __init__(self, config):
self.config = config
self.app = None
self.window = None
self.text_widget = None
self.header_label = None
self.hide_timer = None
self._bridge = None
# ------------------------------------------------------------------
# Startup
# ------------------------------------------------------------------
def start(self):
print("🪟 Creating overlay window...")
self.app = QApplication.instance() or QApplication(sys.argv)
# All Qt objects must be created AFTER QApplication exists
self._bridge = _Bridge()
self._bridge.answer_ready.connect(self._on_answer)
self._bridge.status_ready.connect(self._on_status)
self._bridge.hide_now.connect(self._auto_hide)
self.hide_timer = QTimer()
self.hide_timer.setSingleShot(True)
self.hide_timer.timeout.connect(self._auto_hide)
self._build_window()
print("✅ Overlay window ready")
def _build_window(self):
self.window = QMainWindow()
self.window.setWindowTitle("Meeting Assistant")
self.window.setWindowFlags(
Qt.WindowStaysOnTopHint | Qt.FramelessWindowHint | Qt.Tool
)
self.window.setAttribute(Qt.WA_TranslucentBackground)
central = QWidget()
central.setStyleSheet("background: transparent;")
layout = QVBoxLayout()
layout.setContentsMargins(0, 0, 0, 0)
layout.setSpacing(0)
self.text_widget = QTextEdit()
self.text_widget.setReadOnly(True)
self.text_widget.setStyleSheet("""
QTextEdit {
background-color: rgba(10, 10, 10, 235);
color: #00ff99;
border-radius: 14px;
padding: 16px;
font-size: 13px;
font-family: Menlo, Monaco, "Courier New", monospace;
border: 1px solid rgba(0, 255, 153, 0.25);
line-height: 1.5;
}
QScrollBar:vertical {
background: transparent;
width: 6px;
}
QScrollBar::handle:vertical {
background: rgba(0, 255, 153, 0.3);
border-radius: 3px;
}
""")
layout.addWidget(self.text_widget)
central.setLayout(layout)
self.window.setCentralWidget(central)
# Position: top-right corner
width, height = 620, 250
screen = self.app.primaryScreen().geometry()
x = screen.width() - width - 20
y = 20
self.window.setGeometry(x, y, width, height)
self.text_widget.setPlainText("Meeting Assistant Ready\n\nListening for questions...")
self.window.show()
# Must be called after show() so the native NSWindow is fully realised
self._exclude_from_screen_capture()
def _exclude_from_screen_capture(self):
"""macOS only: set NSWindowSharingNone so Zoom/screen-record can't see this window."""
if platform.system() != "Darwin":
return
try:
import objc
from ctypes import c_void_p
# winId() returns a pointer to the native NSView backing the Qt window
nsview = objc.objc_object(c_void_p=int(self.window.winId()))
nswindow = nsview.window()
# NSWindowSharingNone = 0 (defined in AppKit/NSWindow.h)
# This prevents ANY screen-capture process (Zoom, QuickTime, etc.) from seeing the window
nswindow.setSharingType_(0)
print("🔒 Overlay hidden from screen capture")
except Exception as e:
print(f"⚠️ Screen capture exclusion unavailable: {e}")
# ------------------------------------------------------------------
# Public API — safe to call from any thread
# ------------------------------------------------------------------
def show_answer(self, answer, question=None):
if self._bridge:
self._bridge.answer_ready.emit(answer or "", question or "")
def show_status(self, status):
if self._bridge:
self._bridge.status_ready.emit(status or "")
def toggle_visibility(self):
if self.window:
if self.window.isVisible():
self.window.hide()
else:
self.window.show()
def stop(self):
if self.window:
self.window.close()
if self.app:
self.app.quit()
# ------------------------------------------------------------------
# Private slots — always run on the main Qt thread
# ------------------------------------------------------------------
def _on_answer(self, answer, question):
lines = []
if question:
lines.append(f"Q: {question}")
lines.append("")
lines.append(f"A: {answer}")
self.text_widget.setPlainText("\n".join(lines))
cursor = self.text_widget.textCursor()
cursor.movePosition(QTextCursor.Start)
self.text_widget.setTextCursor(cursor)
self.window.show()
self.window.raise_()
self.hide_timer.start(12000)
def _on_status(self, status):
self.text_widget.setPlainText(status)
self.window.show()
self.hide_timer.start(4000)
def _auto_hide(self):
if self.window:
self.window.hide()
# Keep old name for backward compatibility
def hide_overlay(self):
self._auto_hide()

179
src/screen_scanner.py Normal file
View File

@@ -0,0 +1,179 @@
# import threading
# import time
# from datetime import datetime
# import re
#
#
# class ScreenScanner:
# def __init__(self, config, callback):
# self.config = config
# self.callback = callback
# self.running = False
# self.thread = None
# self.last_text = ""
# self.last_scan_time = 0
#
# def start(self):
# self.running = True
# self.thread = threading.Thread(target=self._scan_loop)
# self.thread.daemon = True
# self.thread.start()
# print("📸 Screen scanner started")
#
# def _is_code_or_ide_text(self, text):
# """Filter out code, IDE text, and non-question content"""
# if not text or len(text) < 15:
# return True
#
# # Patterns that indicate code or IDE text
# code_patterns = [
# r'\.py$', r'\.yaml$', r'\.txt$', r'def\s+\w+', r'class\s+\w+',
# r'import\s+\w+', r'return\s+', r'self\.', r'__init__', r'\.\.\.',
# r'{\s*$', r'}\s*$', r'\[\s*$', r'\]\s*$', r'=\s*\w+',
# r'print\(', r'if\s+.*:', r'else:', r'try:', r'except:',
# r'while\s+', r'for\s+', r'in\s+range', r'logger\.', r'\.error',
# r'\.info\(', r'\.warning', r'QObject::', r'Timer cannot'
# ]
#
# # IDE/editor text patterns
# ide_patterns = [
# r'External Libraries', r'Scratches and Consoles', r'\.venv',
# r'Python \d+\.\d+', r'meeting_assistant', r'src/', r'models/',
# r'logs/', r'config\.yaml', r'requirements\.txt', r'\.\.\.$',
# r'^\s*\.{3,}', r'^\s*→', r'^\s*›', r'^\s*❓'
# ]
#
# # Combine patterns
# all_patterns = code_patterns + ide_patterns
#
# for pattern in all_patterns:
# if re.search(pattern, text, re.IGNORECASE):
# return True
#
# # Check if it looks like a real question
# question_indicators = ['?', 'what', 'how', 'why', 'when', 'where', 'who',
# 'can you', 'could you', 'would you', 'please explain']
# has_question = any(indicator in text.lower() for indicator in question_indicators)
#
# # Only keep real questions with proper length
# if has_question and 15 < len(text) < 200:
# return False
#
# return True
#
# def _is_valid_question(self, text):
# """Check if text is a valid question worth answering"""
# # Avoid repeating the same text
# if text == self.last_text:
# return False
#
# # Minimum length for a real question
# if len(text) < 20 or len(text) > 300:
# return False
#
# # Must have question words or question mark
# question_words = ['what', 'how', 'why', 'when', 'where', 'who', 'which',
# 'can you', 'could you', 'would you', 'do you', 'is it']
#
# text_lower = text.lower()
# has_question_word = any(word in text_lower for word in question_words)
# has_question_mark = '?' in text
#
# if not (has_question_word or has_question_mark):
# return False
#
# return True
#
# def _scan_loop(self):
# """Continuous screen capture and OCR"""
# import easyocr
# import mss
# import numpy as np
# from PIL import Image
#
# # Initialize OCR once
# print("📸 Initializing OCR for screen scanning...")
# try:
# reader = easyocr.Reader(['en'], gpu=False, verbose=False)
# print("✅ OCR ready")
# except Exception as e:
# print(f"⚠️ OCR initialization failed: {e}")
# print("Screen scanning disabled")
# return
#
# with mss.mss() as sct:
# while self.running:
# try:
# # Capture screen
# screenshot = sct.grab(sct.monitors[1])
#
# # Convert to PIL Image
# img = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX")
#
# # Convert to numpy array
# import cv2
# img_np = cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
#
# # Perform OCR - handle different return formats
# results = reader.readtext(img_np, paragraph=True)
#
# # Process results safely
# if results and len(results) > 0:
# for result in results:
# try:
# # Handle different result formats
# if len(result) == 3:
# bbox, text, confidence = result
# elif len(result) == 2:
# bbox, text = result
# confidence = 0.5
# else:
# continue
#
# # Filter and process
# if confidence > 0.4 and not self._is_code_or_ide_text(text):
# if self._is_valid_question(text):
# timestamp = datetime.now()
# print(f"❓ Question detected: {text[:100]}")
# self.callback(text, timestamp, None)
# self.last_text = text
# time.sleep(3) # Prevent rapid-fire
#
# except Exception as e:
# continue
#
# # Scan every 5 seconds
# time.sleep(5)
#
# except Exception as e:
# print(f"Screen scan error: {e}")
# time.sleep(3)
#
# def stop(self):
# self.running = False
# if self.thread:
# self.thread.join(timeout=2)
import threading
import time
from datetime import datetime
class ScreenScanner:
def __init__(self, config, callback):
self.config = config
self.callback = callback
self.running = False
self.thread = None
def start(self):
print("📸 Screen scanner is DISABLED - using audio only mode")
print(" (Screen OCR requires better filtering, will be added later)")
# Don't start scanning to avoid spam
return
def stop(self):
self.running = False
if self.thread:
self.thread.join(timeout=2)

57
src/utils.py Normal file
View File

@@ -0,0 +1,57 @@
import logging
import platform
from pathlib import Path
def setup_logging():
"""Setup logging configuration"""
log_dir = Path("logs")
log_dir.mkdir(exist_ok=True)
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler(log_dir / "meeting_assistant.log"),
logging.StreamHandler()
]
)
return logging.getLogger(__name__)
def get_platform():
"""Detect operating system"""
system = platform.system()
if system == "Darwin":
return "macOS"
elif system == "Windows":
return "Windows"
else:
return "Linux"
def check_dependencies():
"""Check if all dependencies are installed"""
missing = []
try:
import torch
except ImportError:
missing.append("torch")
try:
import sounddevice
except ImportError:
missing.append("sounddevice")
try:
import easyocr
except ImportError:
missing.append("easyocr")
if missing:
print(f"Missing dependencies: {', '.join(missing)}")
print("Run: pip install -r requirements.txt")
return False
return True