From 138580475caf4e0f591640d3506af7b3cae2b684 Mon Sep 17 00:00:00 2001 From: Charles Wambua Date: Fri, 26 Jun 2026 15:38:09 +0300 Subject: [PATCH] Added context match --- .gitignore | 11 ++-- main.py | 52 ++++++++++++++- src/ai_engine.py | 81 +++++++++++++++++++---- src/audio_listener.py | 141 ++++++++++++++++++++++++++++++----------- src/context_manager.py | 29 ++++++++- src/hotkeys.py | 38 ++++++++++- 6 files changed, 294 insertions(+), 58 deletions(-) diff --git a/.gitignore b/.gitignore index c99e48c..c8fdcb1 100644 --- a/.gitignore +++ b/.gitignore @@ -39,11 +39,12 @@ config/production.yaml *.local.yaml *.secret.yaml -# Model files (large binary files) -models/*.gguf -models/*.bin -models/whisper/model.bin -models/whisper/*.gguf +# Model files (large binaries ā€” downloaded by setup.py / faster-whisper, not tracked) +models/ + +# Build artifacts / archives +*.zip +meeting_assistant.zip # Cache directories .cache/ diff --git a/main.py b/main.py index 59c6fab..6415a3c 100644 --- a/main.py +++ b/main.py @@ -102,9 +102,13 @@ class MeetingAssistant: on_disarm=self.audio_listener.disarm, on_capture=self.overlay.request_capture, # marshals onto Qt thread on_toggle=self.overlay.request_toggle, # marshals onto Qt thread + on_summary=self._handle_summary, # spawns its own worker thread + on_listen=self._handle_toggle_listen, # switch auto-listen / push-to-talk ptt_key=audio_cfg.get("ptt_key", "alt_r"), capture_key=screen_cfg.get("capture_key", "ctrl+shift+space"), toggle_key=hotkeys_cfg.get("toggle_overlay", "ctrl+shift+h"), + summary_key=hotkeys_cfg.get("meeting_summary", "ctrl+shift+s"), + listen_key=hotkeys_cfg.get("toggle_listening", "ctrl+shift+m"), ) # Screen-capture handler runs on the Qt main thread (via the overlay bridge) self.overlay.on_capture = self._handle_screen_capture @@ -121,11 +125,15 @@ class MeetingAssistant: mode = audio_cfg.get("capture_mode", "push_to_talk") print("āœ… Meeting Assistant Ready!") print("==================================================") + listen_key = hotkeys_cfg.get("toggle_listening", "ctrl+shift+m") if mode == "push_to_talk": print(f"šŸŽ™ļø Hold [{ptt}] to capture audio, release to answer") else: - print("šŸŽ¤ Listening continuously for questions") + print("šŸŽ¤ Auto-listening (no key needed)") + print(f"šŸ” Press [{listen_key}] to toggle auto-listen ā‡„ push-to-talk") print(f"šŸ“ø Press [{cap}] then drag a box over an on-screen question") + summ = hotkeys_cfg.get("meeting_summary", "ctrl+shift+s") + print(f"šŸ“ Press [{summ}] for a summary of the meeting so far") print(" Answers appear in the overlay (top-right)") print(" Type a question here + Enter to test AI") print(" Ctrl+C to stop") @@ -314,6 +322,48 @@ class MeetingAssistant: except Exception: pass + # ------------------------------------------------------------------ + # End-of-meeting summary + # ------------------------------------------------------------------ + + def _handle_toggle_listen(self): + """Hotkey handler: switch between auto-listen and push-to-talk.""" + on = self.audio_listener.toggle_auto_listen() + if on: + self.overlay.show_status("šŸ”Š Auto-listen ON ā€” listening on its own") + else: + self.overlay.show_status("šŸŽ™ļø Push-to-talk ā€” hold the key to capture") + + def _handle_summary(self): + """Hotkey handler (pynput thread): summarize the meeting in the background.""" + threading.Thread(target=self._summary_worker, daemon=True).start() + + def _summary_worker(self): + transcript = self.context_manager.get_full_transcript() + if not transcript: + self.overlay.show_status("No meeting captured yet ā€” nothing to summarize.") + print(" (No transcript captured yet.)") + return + with self.answer_lock: + if self.answering and hasattr(self.ai_engine, "interrupt"): + self.ai_engine.interrupt() + self.answering = True + try: + self.overlay.show_status("šŸ“ Summarizing the meeting...") + print("\nšŸ“ Summarizing the meeting...") + summary = self.ai_engine.summarize_meeting(transcript) + if not summary: + return + self.overlay.show_answer(summary, "Meeting summary", suggested=False) + self.logger.info("Meeting summary generated.") + self.logger.info(f"Summary: {summary}") + print(f"\nšŸ“‹ Meeting Summary:\n{summary}\n") + except Exception as e: + print(f"āŒ Error summarizing meeting: {e}") + finally: + with self.answer_lock: + self.answering = False + # ------------------------------------------------------------------ # Manual terminal input for testing # ------------------------------------------------------------------ diff --git a/src/ai_engine.py b/src/ai_engine.py index 6fc6661..5cc4d44 100644 --- a/src/ai_engine.py +++ b/src/ai_engine.py @@ -190,13 +190,16 @@ class AIEngine: ) if kind == "factual": return ( - "You are a fast, accurate assistant. Give the direct factual answer in " - "one short sentence. No preamble, no hedging." + "You are a fast, accurate assistant in an ongoing conversation. " + "Use the recent conversation to resolve follow-ups and references " + "(e.g. \"its\", \"that\", \"the second one\"). Give the direct factual " + "answer in one or two short sentences. No preamble, no hedging." ) return ( - f"You are a real-time meeting copilot for {self.user_name}. " - "Answer the latest question accurately and concisely (under ~70 words). " - "Use the meeting transcript only for context. Be direct." + f"You are a real-time meeting copilot for {self.user_name} in an ongoing " + "conversation. Use the recent conversation and meeting transcript to " + "resolve follow-ups and references (\"it\", \"that\", \"those\"). Answer the " + "current question accurately and concisely (under ~70 words). Be direct." ) def _user_prompt(self, question, context): @@ -204,20 +207,28 @@ class AIEngine: if context: transcript = (context.get("audio", "") or "")[-1800:].strip() + # Recent Q&A thread so follow-ups resolve ("state its 4 core principles" + # after "what is Java" ā†’ the model sees the Java exchange). memory_block = "" if self.memory: - for item in list(self.memory)[-2:]: - memory_block += f"Earlier Q: {item['question']}\nEarlier A: {item['response']}\n" + turns = [] + for item in list(self.memory)[-5:]: + turns.append(f"Q: {item['question']}\nA: {item['response']}") + memory_block = "\n".join(turns) parts = [] if transcript: parts.append(f"[Meeting transcript so far]\n{transcript}\n") if memory_block: - parts.append(memory_block) - parts.append(f"[Question]\n{question}") + parts.append( + "[Recent conversation ā€” resolve references like \"it\", \"that\", " + "\"those\", \"the second one\" against these earlier turns]\n" + f"{memory_block}\n" + ) + parts.append(f"[Current question]\n{question}") return "\n".join(parts) - def _stream_chat(self, messages, max_tokens): + def _stream_chat(self, messages, max_tokens, multiline=False): """Run a chat completion, streaming so a new question can interrupt the current one mid-generation. Returns cleaned text, or None if interrupted.""" if self.interrupt_event.is_set(): @@ -247,7 +258,7 @@ class AIEngine: print(f"āš ļø Generation error: {e}") return "I couldn't process that." - return self._clean("".join(chunks)) + return self._clean("".join(chunks), multiline=multiline) def _generate(self, question, context, kind, suggested): messages = [ @@ -309,9 +320,55 @@ class AIEngine: "time": datetime.now()}) return text - def _clean(self, text): + # ------------------------------------------------------------------ + # End-of-meeting summary + # ------------------------------------------------------------------ + + def summarize_meeting(self, transcript): + """Summarize a speaker-labeled meeting transcript into a structured recap. + + Returns the summary text, or None if interrupted / nothing to summarize.""" + if not transcript or not transcript.strip(): + return None + + system = ( + f"You are a meeting note-taker for {self.user_name}. You are given a " + "speaker-labeled transcript of a call. Write a clear, faithful recap. " + "Only use what is in the transcript ā€” never invent decisions, names, or " + "numbers. If a section has nothing, write \"None\"." + ) + user = ( + "Summarize this meeting under these exact headings:\n" + "Overview: 1-2 sentences on what the meeting was about.\n" + "Key points: the main things discussed (bullet list).\n" + "Decisions / agreements: what was agreed (bullet list).\n" + "Proposals: who proposed what ā€” attribute to the speaker label " + "(e.g. \"Person 1 proposed ā€¦\").\n" + "Action items: who needs to do what next (bullet list).\n" + "Open questions: anything left unresolved.\n\n" + f"[Transcript]\n{transcript}" + ) + messages = [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + # Summaries are longer than live answers ā€” allow more room and keep the + # line/section structure (don't collapse to a single run-on line). + return self._stream_chat(messages, max_tokens=max(self.max_tokens, 600), + multiline=True) + + def _clean(self, text, multiline=False): text = re.sub(r"<\|.*?\|>", "", text) text = re.sub(r"<.*?>", "", text) + if multiline: + # Preserve line/section structure for summaries; just tidy spacing + # and drop stray markdown so it reads cleanly in the plain-text overlay. + text = text.replace("**", "") + text = re.sub(r"(?m)^\s*#{1,6}\s*", "", text) # markdown headings + text = re.sub(r"[ \t]+", " ", text) + text = re.sub(r" *\n *", "\n", text) + text = re.sub(r"\n{3,}", "\n\n", text).strip() + return text or "No response generated." text = re.sub(r"\s+", " ", text).strip() if not text: return "No response generated." diff --git a/src/audio_listener.py b/src/audio_listener.py index d3d61af..b2d0d53 100644 --- a/src/audio_listener.py +++ b/src/audio_listener.py @@ -62,6 +62,7 @@ class StreamSegmenter: self.noise_floor = 0.0 self.speech_threshold = 2000.0 self.silence_threshold = 1500.0 + self.calibrated = False # set once live calibration has run # rolling pre-speech buffer so we don't clip the start of words self._pre_buffer = bytearray() @@ -196,6 +197,13 @@ class AudioListener: # Capture mode: "push_to_talk" (only buffer audio while armed) or # "continuous" (always-on VAD segmentation, the original behavior). self.capture_mode = audio_cfg.get("capture_mode", "push_to_talk") + # Auto-listen ON ā†’ continuous VAD, listens on its own (no key). + # Auto-listen OFF ā†’ push-to-talk, capture only while the key is held. + # Togglable at runtime via toggle_auto_listen(); starts from config. + self._auto_listen = self.capture_mode == "continuous" + self._reset_segs = False # frame loop drops in-progress utterances + self._calib_rms = {} # source -> rms samples for live calibration + self._warned_silent_system = False # one-time "loopback is silent" notice self._armed = False self._ptt_buffers = {} # source -> bytearray, while armed self._ptt_lock = threading.Lock() @@ -358,17 +366,14 @@ class AudioListener: ) print(f"\nšŸŽ§ Capturing: {summary}\n") - ptt = self.capture_mode == "push_to_talk" - - # In continuous mode each source gets a calibrated VAD segmenter. - # In push-to-talk mode we buffer the held window instead, so neither - # calibration nor the always-on segmenter is needed. - if not ptt: - for source, device in plan.items(): - seg = StreamSegmenter(source, self.sample_rate, self.frame_size, - self.frame_duration_ms, self.vad, self.utterance_queue) - self._calibrate(seg, device) - self.segmenters[source] = seg + # Build a VAD segmenter per source so auto-listen can run at ANY time + # (the mode is togglable at runtime). They self-calibrate from the live + # audio the first time auto-listen is active, so startup stays fast and + # push-to-talk users don't pay a calibration delay. + for source, device in plan.items(): + seg = StreamSegmenter(source, self.sample_rate, self.frame_size, + self.frame_duration_ms, self.vad, self.utterance_queue) + self.segmenters[source] = seg # open one input stream per source for source, device in plan.items(): @@ -383,14 +388,14 @@ class AudioListener: stream.start() self.streams.append(stream) - if not ptt: - threading.Thread(target=self._frame_loop, daemon=True).start() + # The frame loop runs always but only does work while auto-listen feeds it. + threading.Thread(target=self._frame_loop, daemon=True).start() threading.Thread(target=self._transcribe_loop, daemon=True).start() - if ptt: - print("šŸŽ¤ Ready. Hold the push-to-talk key to capture; release to answer.\n") + if self._auto_listen: + print("šŸŽ¤ Auto-listening (no key needed). Toggle to push-to-talk anytime.\n") else: - print("šŸŽ¤ Listening... Speak now! (Ctrl+C to stop)\n") + print("šŸŽ¤ Ready. Hold the push-to-talk key to capture; release to answer.\n") # ------------------------------------------------------------------ # Push-to-talk: only capture audio while armed @@ -398,7 +403,7 @@ class AudioListener: def arm(self): """Begin buffering audio from every source (push-to-talk key down).""" - if self.capture_mode != "push_to_talk" or self._armed: + if self._auto_listen or self._armed: # key is ignored while auto-listening return with self._ptt_lock: # seed each source with its pre-roll so the leading word survives @@ -411,7 +416,7 @@ class AudioListener: def disarm(self): """Stop buffering and queue what was captured for transcription (key up).""" - if self.capture_mode != "push_to_talk" or not self._armed: + if self._auto_listen or not self._armed: return self._armed = False with self._ptt_lock: @@ -421,10 +426,38 @@ class AudioListener: for source, buf in buffers.items(): audio = bytes(buf) if len(audio) / (2 * self.sample_rate) >= 0.3: # ignore < 0.3s blips + # Flag a silent loopback (meeting audio not routed into BlackHole). + if source == "system" and not self._warned_silent_system: + samples = np.frombuffer(audio, dtype=np.int16).astype(np.float32) + if samples.size and np.sqrt(np.mean(samples ** 2)) < 15: + self._warned_silent_system = True + print(" āš ļø Captured nothing from the other participants ā€” the " + "loopback (BlackHole) is silent.") + print(" Set your meeting app's Speaker to a Multi-Output " + "Device that includes BlackHole (README ā†’ 'Hearing other " + "participants').") self.utterance_queue.put((source, audio)) queued = True print("ā³ Transcribing..." if queued else " (too short ā€” nothing captured)") + def toggle_auto_listen(self): + """Flip between auto-listen (continuous VAD, no key) and push-to-talk. + + Returns the new auto-listen state (True = auto-listening). Safe to call + from the hotkey thread.""" + self._auto_listen = not self._auto_listen + if self._auto_listen: + # entering auto-listen: drop any half-held PTT capture + self._armed = False + with self._ptt_lock: + self._ptt_buffers = {} + print("\nšŸ”Š Auto-listen ON ā€” listening on its own (no key needed).") + else: + # leaving auto-listen: tell the frame loop to drop in-progress speech + self._reset_segs = True + print("\nšŸŽ™ļø Auto-listen OFF ā€” back to push-to-talk (hold the key).") + return self._auto_listen + def _calibrate(self, segmenter, device): dev_name = sd.query_devices(device)["name"] print(f"šŸ”§ Calibrating noise floor for '{dev_name}' (2s)...") @@ -450,22 +483,22 @@ class AudioListener: # underruns are noisy and harmless; skip logging them pass audio_bytes = bytes(indata) - # push-to-talk: only retain audio while the key is held - if self.capture_mode == "push_to_talk": - if self._armed: - with self._ptt_lock: - self._ptt_buffers.setdefault(source, bytearray()).extend(audio_bytes) - else: - # keep a short rolling pre-roll so the first word isn't clipped - pr = self._preroll.get(source) - if pr is None: - pr = self._preroll[source] = deque(maxlen=self._PREROLL_FRAMES) - pr.append(audio_bytes) + # auto-listen: feed the VAD segmenter via the frame queue + if self._auto_listen: + samples = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32) + rms = int(np.sqrt(np.mean(samples ** 2))) if samples.size else 0 + self.frame_queue.put((source, audio_bytes, rms)) return - # continuous mode: feed the VAD segmenter via the frame queue - samples = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32) - rms = int(np.sqrt(np.mean(samples ** 2))) if samples.size else 0 - self.frame_queue.put((source, audio_bytes, rms)) + # push-to-talk: only retain audio while the key is held + if self._armed: + with self._ptt_lock: + self._ptt_buffers.setdefault(source, bytearray()).extend(audio_bytes) + else: + # keep a short rolling pre-roll so the first word isn't clipped + pr = self._preroll.get(source) + if pr is None: + pr = self._preroll[source] = deque(maxlen=self._PREROLL_FRAMES) + pr.append(audio_bytes) return _cb def _select_input_device(self): @@ -490,15 +523,51 @@ class AudioListener: while self.running: try: source, audio_bytes, rms = self.frame_queue.get(timeout=0.5) - seg = self.segmenters.get(source) - if seg: - seg.process_frame(audio_bytes, rms, time.time()) except queue.Empty: + if self._reset_segs: + for seg in self.segmenters.values(): + seg._reset() + self._reset_segs = False now = time.time() for seg in self.segmenters.values(): seg.flush_if_idle(now) + continue except Exception as e: print(f"\nāŒ Frame error: {e}") + continue + + # auto-listen was just turned off ā†’ drop any in-progress utterances + if self._reset_segs: + for seg in self.segmenters.values(): + seg._reset() + self._reset_segs = False + + seg = self.segmenters.get(source) + if not seg: + continue + + # Self-calibrate the noise floor from the first ~1s of live audio the + # first time this source is heard, then start detecting speech. + if not seg.calibrated: + buf = self._calib_rms.setdefault(source, []) + buf.append(rms) + if len(buf) >= 33: # ~1s at 30 ms frames + median_rms = float(np.median(buf)) + seg.calibrate(buf) + seg.calibrated = True + self._calib_rms.pop(source, None) + print(f" šŸ“Š auto-listen calibrated [{source}]: " + f"speech>={seg.speech_threshold:.0f}") + # A silent loopback means the meeting audio isn't routed in. + if source == "system" and median_rms < 15: + print(" āš ļø System/loopback device looks SILENT ā€” the meeting " + "audio isn't reaching it.") + print(" Set your meeting app's Speaker to a Multi-Output " + "Device that includes BlackHole (see README ā†’ 'Hearing " + "other participants').") + continue + + seg.process_frame(audio_bytes, rms, time.time()) def _transcribe_loop(self): diff --git a/src/context_manager.py b/src/context_manager.py index ccacec2..55c4650 100644 --- a/src/context_manager.py +++ b/src/context_manager.py @@ -6,17 +6,22 @@ class ContextManager: self.config = config self.audio_context = deque(maxlen=20) # last 20 utterances (both sources) self.screen_context = deque(maxlen=5) + # full speaker-labeled meeting transcript (for the end-of-call summary) + self.full_transcript = deque(maxlen=2000) self.last_text = "" def add_audio_context(self, text, timestamp, source="microphone", speaker=None): """Add a transcribed utterance, tagged with who spoke.""" - self.audio_context.append({ + speaker = speaker or self._speaker(source) + entry = { "text": text, "timestamp": timestamp.isoformat(), "source": source, - "speaker": speaker or self._speaker(source), + "speaker": speaker, "type": "audio", - }) + } + self.audio_context.append(entry) + self.full_transcript.append(entry) self.last_text = text def add_screen_context(self, text, timestamp, region): @@ -47,7 +52,25 @@ class ContextManager: def get_last_text(self): return self.last_text + def get_full_transcript(self, max_chars=9000): + """Whole speaker-labeled meeting transcript for summarization. + + Trimmed from the front to ``max_chars`` so it fits the model context; + returns "" if nothing has been captured yet.""" + lines = [ + f"{item.get('speaker') or self._speaker(item.get('source'))}: {item['text']}" + for item in self.full_transcript + ] + transcript = "\n".join(lines).strip() + if len(transcript) > max_chars: + transcript = "ā€¦(earlier conversation trimmed)ā€¦\n" + transcript[-max_chars:] + return transcript + + def utterance_count(self): + return len(self.full_transcript) + def clear(self): self.audio_context.clear() self.screen_context.clear() + self.full_transcript.clear() self.last_text = "" diff --git a/src/hotkeys.py b/src/hotkeys.py index 1bc284e..9dddfce 100644 --- a/src/hotkeys.py +++ b/src/hotkeys.py @@ -53,19 +53,27 @@ def _build_maps(): class HotkeyManager: def __init__(self, on_arm=None, on_disarm=None, on_capture=None, ptt_key="alt_r", capture_key="ctrl+shift+space", - on_toggle=None, toggle_key=""): + on_toggle=None, toggle_key="", + on_summary=None, summary_key="", + on_listen=None, listen_key=""): self.on_arm = on_arm self.on_disarm = on_disarm self.on_capture = on_capture self.on_toggle = on_toggle + self.on_summary = on_summary + self.on_listen = on_listen self.ptt_key_name = (ptt_key or "alt_r").strip().lower() self.capture_key_name = (capture_key or "").strip().lower() self.toggle_key_name = (toggle_key or "").strip().lower() + self.summary_key_name = (summary_key or "").strip().lower() + self.listen_key_name = (listen_key or "").strip().lower() self._listener = None self._ptt_key = None self._capture_tokens = set() self._toggle_tokens = set() + self._summary_tokens = set() + self._listen_tokens = set() self._named = {} self._canon = {} @@ -73,6 +81,8 @@ class HotkeyManager: self._armed = False # ptt key currently down self._capture_fired = False self._toggle_fired = False + self._summary_fired = False + self._listen_fired = False # ------------------------------------------------------------------ @@ -117,6 +127,12 @@ class HotkeyManager: self._toggle_tokens = { t.strip() for t in self.toggle_key_name.split("+") if t.strip() } + self._summary_tokens = { + t.strip() for t in self.summary_key_name.split("+") if t.strip() + } + self._listen_tokens = { + t.strip() for t in self.listen_key_name.split("+") if t.strip() + } self._listener = keyboard.Listener( on_press=self._on_press, on_release=self._on_release @@ -129,6 +145,10 @@ class HotkeyManager: print(f"āŒØļø Screen grab: press [{self.capture_key_name}] then drag a box.") if self._toggle_tokens: print(f"āŒØļø Show/hide overlay: press [{self.toggle_key_name}].") + if self._summary_tokens: + print(f"āŒØļø Meeting summary: press [{self.summary_key_name}].") + if self._listen_tokens: + print(f"āŒØļø Auto-listen on/off: press [{self.listen_key_name}].") print(" (macOS: grant Accessibility permission if keys don't respond.)") return True @@ -158,6 +178,16 @@ class HotkeyManager: self._toggle_fired = True self._safe(self.on_toggle) + if (self._summary_tokens and not self._summary_fired + and self._summary_tokens.issubset(self._down)): + self._summary_fired = True + self._safe(self.on_summary) + + if (self._listen_tokens and not self._listen_fired + and self._listen_tokens.issubset(self._down)): + self._listen_fired = True + self._safe(self.on_listen) + def _on_release(self, key): if self._ptt_key is not None and key == self._ptt_key and self._armed: self._armed = False @@ -172,3 +202,9 @@ class HotkeyManager: if self._toggle_tokens and not self._toggle_tokens.issubset(self._down): self._toggle_fired = False + + if self._summary_tokens and not self._summary_tokens.issubset(self._down): + self._summary_fired = False + + if self._listen_tokens and not self._listen_tokens.issubset(self._down): + self._listen_fired = False