Added context match

.gitignore

@@ -39,11 +39,12 @@ config/production.yaml
 *.local.yaml
 *.secret.yaml
 
-# Model files (large binary files)
+# Model files (large binaries — downloaded by setup.py / faster-whisper, not tracked)
-models/*.gguf
+models/
-models/*.bin
+
-models/whisper/model.bin
+# Build artifacts / archives
-models/whisper/*.gguf
+*.zip
+meeting_assistant.zip
 
 # Cache directories
 .cache/

main.py

@@ -102,9 +102,13 @@ class MeetingAssistant:
             on_disarm=self.audio_listener.disarm,
             on_capture=self.overlay.request_capture,   # marshals onto Qt thread
             on_toggle=self.overlay.request_toggle,     # marshals onto Qt thread
+            on_summary=self._handle_summary,           # spawns its own worker thread
+            on_listen=self._handle_toggle_listen,      # switch auto-listen / push-to-talk
             ptt_key=audio_cfg.get("ptt_key", "alt_r"),
             capture_key=screen_cfg.get("capture_key", "ctrl+shift+space"),
             toggle_key=hotkeys_cfg.get("toggle_overlay", "ctrl+shift+h"),
+            summary_key=hotkeys_cfg.get("meeting_summary", "ctrl+shift+s"),
+            listen_key=hotkeys_cfg.get("toggle_listening", "ctrl+shift+m"),
         )
         # Screen-capture handler runs on the Qt main thread (via the overlay bridge)
         self.overlay.on_capture = self._handle_screen_capture
@@ -121,11 +125,15 @@ class MeetingAssistant:
         mode = audio_cfg.get("capture_mode", "push_to_talk")
         print("✅ Meeting Assistant Ready!")
         print("==================================================")
+        listen_key = hotkeys_cfg.get("toggle_listening", "ctrl+shift+m")
         if mode == "push_to_talk":
             print(f"🎙️  Hold [{ptt}] to capture audio, release to answer")
         else:
-            print("🎤 Listening continuously for questions")
+            print("🎤 Auto-listening (no key needed)")
+        print(f"🔁 Press [{listen_key}] to toggle auto-listen ⇄ push-to-talk")
         print(f"📸 Press [{cap}] then drag a box over an on-screen question")
+        summ = hotkeys_cfg.get("meeting_summary", "ctrl+shift+s")
+        print(f"📝 Press [{summ}] for a summary of the meeting so far")
         print("   Answers appear in the overlay (top-right)")
         print("   Type a question here + Enter to test AI")
         print("   Ctrl+C to stop")
@@ -314,6 +322,48 @@ class MeetingAssistant:
             except Exception:
                 pass
 
+    # ------------------------------------------------------------------
+    # End-of-meeting summary
+    # ------------------------------------------------------------------
+
+    def _handle_toggle_listen(self):
+        """Hotkey handler: switch between auto-listen and push-to-talk."""
+        on = self.audio_listener.toggle_auto_listen()
+        if on:
+            self.overlay.show_status("🔊 Auto-listen ON — listening on its own")
+        else:
+            self.overlay.show_status("🎙️ Push-to-talk — hold the key to capture")
+
+    def _handle_summary(self):
+        """Hotkey handler (pynput thread): summarize the meeting in the background."""
+        threading.Thread(target=self._summary_worker, daemon=True).start()
+
+    def _summary_worker(self):
+        transcript = self.context_manager.get_full_transcript()
+        if not transcript:
+            self.overlay.show_status("No meeting captured yet — nothing to summarize.")
+            print("   (No transcript captured yet.)")
+            return
+        with self.answer_lock:
+            if self.answering and hasattr(self.ai_engine, "interrupt"):
+                self.ai_engine.interrupt()
+            self.answering = True
+        try:
+            self.overlay.show_status("📝 Summarizing the meeting...")
+            print("\n📝 Summarizing the meeting...")
+            summary = self.ai_engine.summarize_meeting(transcript)
+            if not summary:
+                return
+            self.overlay.show_answer(summary, "Meeting summary", suggested=False)
+            self.logger.info("Meeting summary generated.")
+            self.logger.info(f"Summary: {summary}")
+            print(f"\n📋 Meeting Summary:\n{summary}\n")
+        except Exception as e:
+            print(f"❌ Error summarizing meeting: {e}")
+        finally:
+            with self.answer_lock:
+                self.answering = False
+
     # ------------------------------------------------------------------
     # Manual terminal input for testing
     # ------------------------------------------------------------------

src/ai_engine.py

@@ -190,13 +190,16 @@ class AIEngine:
             )
         if kind == "factual":
             return (
-                "You are a fast, accurate assistant. Give the direct factual answer in "
+                "You are a fast, accurate assistant in an ongoing conversation. "
-                "one short sentence. No preamble, no hedging."
+                "Use the recent conversation to resolve follow-ups and references "
+                "(e.g. \"its\", \"that\", \"the second one\"). Give the direct factual "
+                "answer in one or two short sentences. No preamble, no hedging."
             )
         return (
-            f"You are a real-time meeting copilot for {self.user_name}. "
+            f"You are a real-time meeting copilot for {self.user_name} in an ongoing "
-            "Answer the latest question accurately and concisely (under ~70 words). "
+            "conversation. Use the recent conversation and meeting transcript to "
-            "Use the meeting transcript only for context. Be direct."
+            "resolve follow-ups and references (\"it\", \"that\", \"those\"). Answer the "
+            "current question accurately and concisely (under ~70 words). Be direct."
         )
 
     def _user_prompt(self, question, context):
@@ -204,20 +207,28 @@ class AIEngine:
         if context:
             transcript = (context.get("audio", "") or "")[-1800:].strip()
 
+        # Recent Q&A thread so follow-ups resolve ("state its 4 core principles"
+        # after "what is Java" → the model sees the Java exchange).
         memory_block = ""
         if self.memory:
-            for item in list(self.memory)[-2:]:
+            turns = []
-                memory_block += f"Earlier Q: {item['question']}\nEarlier A: {item['response']}\n"
+            for item in list(self.memory)[-5:]:
+                turns.append(f"Q: {item['question']}\nA: {item['response']}")
+            memory_block = "\n".join(turns)
 
         parts = []
         if transcript:
             parts.append(f"[Meeting transcript so far]\n{transcript}\n")
         if memory_block:
-            parts.append(memory_block)
+            parts.append(
-        parts.append(f"[Question]\n{question}")
+                "[Recent conversation — resolve references like \"it\", \"that\", "
+                "\"those\", \"the second one\" against these earlier turns]\n"
+                f"{memory_block}\n"
+            )
+        parts.append(f"[Current question]\n{question}")
         return "\n".join(parts)
 
-    def _stream_chat(self, messages, max_tokens):
+    def _stream_chat(self, messages, max_tokens, multiline=False):
         """Run a chat completion, streaming so a new question can interrupt the
         current one mid-generation. Returns cleaned text, or None if interrupted."""
         if self.interrupt_event.is_set():
@@ -247,7 +258,7 @@ class AIEngine:
                 print(f"⚠️ Generation error: {e}")
                 return "I couldn't process that."
 
-        return self._clean("".join(chunks))
+        return self._clean("".join(chunks), multiline=multiline)
 
     def _generate(self, question, context, kind, suggested):
         messages = [
@@ -309,9 +320,55 @@ class AIEngine:
                                 "time": datetime.now()})
         return text
 
-    def _clean(self, text):
+    # ------------------------------------------------------------------
+    # End-of-meeting summary
+    # ------------------------------------------------------------------
+
+    def summarize_meeting(self, transcript):
+        """Summarize a speaker-labeled meeting transcript into a structured recap.
+
+        Returns the summary text, or None if interrupted / nothing to summarize."""
+        if not transcript or not transcript.strip():
+            return None
+
+        system = (
+            f"You are a meeting note-taker for {self.user_name}. You are given a "
+            "speaker-labeled transcript of a call. Write a clear, faithful recap. "
+            "Only use what is in the transcript — never invent decisions, names, or "
+            "numbers. If a section has nothing, write \"None\"."
+        )
+        user = (
+            "Summarize this meeting under these exact headings:\n"
+            "Overview: 1-2 sentences on what the meeting was about.\n"
+            "Key points: the main things discussed (bullet list).\n"
+            "Decisions / agreements: what was agreed (bullet list).\n"
+            "Proposals: who proposed what — attribute to the speaker label "
+            "(e.g. \"Person 1 proposed …\").\n"
+            "Action items: who needs to do what next (bullet list).\n"
+            "Open questions: anything left unresolved.\n\n"
+            f"[Transcript]\n{transcript}"
+        )
+        messages = [
+            {"role": "system", "content": system},
+            {"role": "user", "content": user},
+        ]
+        # Summaries are longer than live answers — allow more room and keep the
+        # line/section structure (don't collapse to a single run-on line).
+        return self._stream_chat(messages, max_tokens=max(self.max_tokens, 600),
+                                 multiline=True)
+
+    def _clean(self, text, multiline=False):
         text = re.sub(r"<\|.*?\|>", "", text)
         text = re.sub(r"<.*?>", "", text)
+        if multiline:
+            # Preserve line/section structure for summaries; just tidy spacing
+            # and drop stray markdown so it reads cleanly in the plain-text overlay.
+            text = text.replace("**", "")
+            text = re.sub(r"(?m)^\s*#{1,6}\s*", "", text)   # markdown headings
+            text = re.sub(r"[ \t]+", " ", text)
+            text = re.sub(r" *\n *", "\n", text)
+            text = re.sub(r"\n{3,}", "\n\n", text).strip()
+            return text or "No response generated."
         text = re.sub(r"\s+", " ", text).strip()
         if not text:
             return "No response generated."

src/audio_listener.py

@@ -62,6 +62,7 @@ class StreamSegmenter:
         self.noise_floor = 0.0
         self.speech_threshold = 2000.0
         self.silence_threshold = 1500.0
+        self.calibrated = False         # set once live calibration has run
 
         # rolling pre-speech buffer so we don't clip the start of words
         self._pre_buffer = bytearray()
@@ -196,6 +197,13 @@ class AudioListener:
         # Capture mode: "push_to_talk" (only buffer audio while armed) or
         # "continuous" (always-on VAD segmentation, the original behavior).
         self.capture_mode = audio_cfg.get("capture_mode", "push_to_talk")
+        # Auto-listen ON  → continuous VAD, listens on its own (no key).
+        # Auto-listen OFF → push-to-talk, capture only while the key is held.
+        # Togglable at runtime via toggle_auto_listen(); starts from config.
+        self._auto_listen = self.capture_mode == "continuous"
+        self._reset_segs = False          # frame loop drops in-progress utterances
+        self._calib_rms = {}              # source -> rms samples for live calibration
+        self._warned_silent_system = False  # one-time "loopback is silent" notice
         self._armed = False
         self._ptt_buffers = {}            # source -> bytearray, while armed
         self._ptt_lock = threading.Lock()
@@ -358,17 +366,14 @@ class AudioListener:
         )
         print(f"\n🎧 Capturing: {summary}\n")
 
-        ptt = self.capture_mode == "push_to_talk"
+        # Build a VAD segmenter per source so auto-listen can run at ANY time
-
+        # (the mode is togglable at runtime). They self-calibrate from the live
-        # In continuous mode each source gets a calibrated VAD segmenter.
+        # audio the first time auto-listen is active, so startup stays fast and
-        # In push-to-talk mode we buffer the held window instead, so neither
+        # push-to-talk users don't pay a calibration delay.
-        # calibration nor the always-on segmenter is needed.
+        for source, device in plan.items():
-        if not ptt:
+            seg = StreamSegmenter(source, self.sample_rate, self.frame_size,
-            for source, device in plan.items():
+                                  self.frame_duration_ms, self.vad, self.utterance_queue)
-                seg = StreamSegmenter(source, self.sample_rate, self.frame_size,
+            self.segmenters[source] = seg
-                                      self.frame_duration_ms, self.vad, self.utterance_queue)
-                self._calibrate(seg, device)
-                self.segmenters[source] = seg
 
         # open one input stream per source
         for source, device in plan.items():
@@ -383,14 +388,14 @@ class AudioListener:
             stream.start()
             self.streams.append(stream)
 
-        if not ptt:
+        # The frame loop runs always but only does work while auto-listen feeds it.
-            threading.Thread(target=self._frame_loop, daemon=True).start()
+        threading.Thread(target=self._frame_loop, daemon=True).start()
         threading.Thread(target=self._transcribe_loop, daemon=True).start()
 
-        if ptt:
+        if self._auto_listen:
-            print("🎤 Ready. Hold the push-to-talk key to capture; release to answer.\n")
+            print("🎤 Auto-listening (no key needed). Toggle to push-to-talk anytime.\n")
         else:
-            print("🎤 Listening... Speak now! (Ctrl+C to stop)\n")
+            print("🎤 Ready. Hold the push-to-talk key to capture; release to answer.\n")
 
     # ------------------------------------------------------------------
     # Push-to-talk: only capture audio while armed
@@ -398,7 +403,7 @@ class AudioListener:
 
     def arm(self):
         """Begin buffering audio from every source (push-to-talk key down)."""
-        if self.capture_mode != "push_to_talk" or self._armed:
+        if self._auto_listen or self._armed:   # key is ignored while auto-listening
             return
         with self._ptt_lock:
             # seed each source with its pre-roll so the leading word survives
@@ -411,7 +416,7 @@ class AudioListener:
 
     def disarm(self):
         """Stop buffering and queue what was captured for transcription (key up)."""
-        if self.capture_mode != "push_to_talk" or not self._armed:
+        if self._auto_listen or not self._armed:
             return
         self._armed = False
         with self._ptt_lock:
@@ -421,10 +426,38 @@ class AudioListener:
         for source, buf in buffers.items():
             audio = bytes(buf)
             if len(audio) / (2 * self.sample_rate) >= 0.3:   # ignore < 0.3s blips
+                # Flag a silent loopback (meeting audio not routed into BlackHole).
+                if source == "system" and not self._warned_silent_system:
+                    samples = np.frombuffer(audio, dtype=np.int16).astype(np.float32)
+                    if samples.size and np.sqrt(np.mean(samples ** 2)) < 15:
+                        self._warned_silent_system = True
+                        print("   ⚠️  Captured nothing from the other participants — the "
+                              "loopback (BlackHole) is silent.")
+                        print("      Set your meeting app's Speaker to a Multi-Output "
+                              "Device that includes BlackHole (README → 'Hearing other "
+                              "participants').")
                 self.utterance_queue.put((source, audio))
                 queued = True
         print("⏳ Transcribing..." if queued else "   (too short — nothing captured)")
 
+    def toggle_auto_listen(self):
+        """Flip between auto-listen (continuous VAD, no key) and push-to-talk.
+
+        Returns the new auto-listen state (True = auto-listening). Safe to call
+        from the hotkey thread."""
+        self._auto_listen = not self._auto_listen
+        if self._auto_listen:
+            # entering auto-listen: drop any half-held PTT capture
+            self._armed = False
+            with self._ptt_lock:
+                self._ptt_buffers = {}
+            print("\n🔊 Auto-listen ON — listening on its own (no key needed).")
+        else:
+            # leaving auto-listen: tell the frame loop to drop in-progress speech
+            self._reset_segs = True
+            print("\n🎙️  Auto-listen OFF — back to push-to-talk (hold the key).")
+        return self._auto_listen
+
     def _calibrate(self, segmenter, device):
         dev_name = sd.query_devices(device)["name"]
         print(f"🔧 Calibrating noise floor for '{dev_name}' (2s)...")
@@ -450,22 +483,22 @@ class AudioListener:
                 # underruns are noisy and harmless; skip logging them
                 pass
             audio_bytes = bytes(indata)
-            # push-to-talk: only retain audio while the key is held
+            # auto-listen: feed the VAD segmenter via the frame queue
-            if self.capture_mode == "push_to_talk":
+            if self._auto_listen:
-                if self._armed:
+                samples = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32)
-                    with self._ptt_lock:
+                rms = int(np.sqrt(np.mean(samples ** 2))) if samples.size else 0
-                        self._ptt_buffers.setdefault(source, bytearray()).extend(audio_bytes)
+                self.frame_queue.put((source, audio_bytes, rms))
-                else:
-                    # keep a short rolling pre-roll so the first word isn't clipped
-                    pr = self._preroll.get(source)
-                    if pr is None:
-                        pr = self._preroll[source] = deque(maxlen=self._PREROLL_FRAMES)
-                    pr.append(audio_bytes)
                 return
-            # continuous mode: feed the VAD segmenter via the frame queue
+            # push-to-talk: only retain audio while the key is held
-            samples = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32)
+            if self._armed:
-            rms = int(np.sqrt(np.mean(samples ** 2))) if samples.size else 0
+                with self._ptt_lock:
-            self.frame_queue.put((source, audio_bytes, rms))
+                    self._ptt_buffers.setdefault(source, bytearray()).extend(audio_bytes)
+            else:
+                # keep a short rolling pre-roll so the first word isn't clipped
+                pr = self._preroll.get(source)
+                if pr is None:
+                    pr = self._preroll[source] = deque(maxlen=self._PREROLL_FRAMES)
+                pr.append(audio_bytes)
         return _cb
 
     def _select_input_device(self):
@@ -490,15 +523,51 @@ class AudioListener:
         while self.running:
             try:
                 source, audio_bytes, rms = self.frame_queue.get(timeout=0.5)
-                seg = self.segmenters.get(source)
-                if seg:
-                    seg.process_frame(audio_bytes, rms, time.time())
             except queue.Empty:
+                if self._reset_segs:
+                    for seg in self.segmenters.values():
+                        seg._reset()
+                    self._reset_segs = False
                 now = time.time()
                 for seg in self.segmenters.values():
                     seg.flush_if_idle(now)
+                continue
             except Exception as e:
                 print(f"\n❌ Frame error: {e}")
+                continue
+
+            # auto-listen was just turned off → drop any in-progress utterances
+            if self._reset_segs:
+                for seg in self.segmenters.values():
+                    seg._reset()
+                self._reset_segs = False
+
+            seg = self.segmenters.get(source)
+            if not seg:
+                continue
+
+            # Self-calibrate the noise floor from the first ~1s of live audio the
+            # first time this source is heard, then start detecting speech.
+            if not seg.calibrated:
+                buf = self._calib_rms.setdefault(source, [])
+                buf.append(rms)
+                if len(buf) >= 33:                      # ~1s at 30 ms frames
+                    median_rms = float(np.median(buf))
+                    seg.calibrate(buf)
+                    seg.calibrated = True
+                    self._calib_rms.pop(source, None)
+                    print(f"   📊 auto-listen calibrated [{source}]: "
+                          f"speech>={seg.speech_threshold:.0f}")
+                    # A silent loopback means the meeting audio isn't routed in.
+                    if source == "system" and median_rms < 15:
+                        print("   ⚠️  System/loopback device looks SILENT — the meeting "
+                              "audio isn't reaching it.")
+                        print("      Set your meeting app's Speaker to a Multi-Output "
+                              "Device that includes BlackHole (see README → 'Hearing "
+                              "other participants').")
+                continue
+
+            seg.process_frame(audio_bytes, rms, time.time())
 
 
     def _transcribe_loop(self):

src/context_manager.py

@@ -6,17 +6,22 @@ class ContextManager:
         self.config = config
         self.audio_context = deque(maxlen=20)   # last 20 utterances (both sources)
         self.screen_context = deque(maxlen=5)
+        # full speaker-labeled meeting transcript (for the end-of-call summary)
+        self.full_transcript = deque(maxlen=2000)
         self.last_text = ""
 
     def add_audio_context(self, text, timestamp, source="microphone", speaker=None):
         """Add a transcribed utterance, tagged with who spoke."""
-        self.audio_context.append({
+        speaker = speaker or self._speaker(source)
+        entry = {
             "text": text,
             "timestamp": timestamp.isoformat(),
             "source": source,
-            "speaker": speaker or self._speaker(source),
+            "speaker": speaker,
             "type": "audio",
-        })
+        }
+        self.audio_context.append(entry)
+        self.full_transcript.append(entry)
         self.last_text = text
 
     def add_screen_context(self, text, timestamp, region):
@@ -47,7 +52,25 @@ class ContextManager:
     def get_last_text(self):
         return self.last_text
 
+    def get_full_transcript(self, max_chars=9000):
+        """Whole speaker-labeled meeting transcript for summarization.
+
+        Trimmed from the front to ``max_chars`` so it fits the model context;
+        returns "" if nothing has been captured yet."""
+        lines = [
+            f"{item.get('speaker') or self._speaker(item.get('source'))}: {item['text']}"
+            for item in self.full_transcript
+        ]
+        transcript = "\n".join(lines).strip()
+        if len(transcript) > max_chars:
+            transcript = "…(earlier conversation trimmed)…\n" + transcript[-max_chars:]
+        return transcript
+
+    def utterance_count(self):
+        return len(self.full_transcript)
+
     def clear(self):
         self.audio_context.clear()
         self.screen_context.clear()
+        self.full_transcript.clear()
         self.last_text = ""

src/hotkeys.py

@@ -53,19 +53,27 @@ def _build_maps():
 class HotkeyManager:
     def __init__(self, on_arm=None, on_disarm=None, on_capture=None,
                  ptt_key="alt_r", capture_key="ctrl+shift+space",
-                 on_toggle=None, toggle_key=""):
+                 on_toggle=None, toggle_key="",
+                 on_summary=None, summary_key="",
+                 on_listen=None, listen_key=""):
         self.on_arm = on_arm
         self.on_disarm = on_disarm
         self.on_capture = on_capture
         self.on_toggle = on_toggle
+        self.on_summary = on_summary
+        self.on_listen = on_listen
         self.ptt_key_name = (ptt_key or "alt_r").strip().lower()
         self.capture_key_name = (capture_key or "").strip().lower()
         self.toggle_key_name = (toggle_key or "").strip().lower()
+        self.summary_key_name = (summary_key or "").strip().lower()
+        self.listen_key_name = (listen_key or "").strip().lower()
 
         self._listener = None
         self._ptt_key = None
         self._capture_tokens = set()
         self._toggle_tokens = set()
+        self._summary_tokens = set()
+        self._listen_tokens = set()
         self._named = {}
         self._canon = {}
 
@@ -73,6 +81,8 @@ class HotkeyManager:
         self._armed = False         # ptt key currently down
         self._capture_fired = False
         self._toggle_fired = False
+        self._summary_fired = False
+        self._listen_fired = False
 
     # ------------------------------------------------------------------
 
@@ -117,6 +127,12 @@ class HotkeyManager:
         self._toggle_tokens = {
             t.strip() for t in self.toggle_key_name.split("+") if t.strip()
         }
+        self._summary_tokens = {
+            t.strip() for t in self.summary_key_name.split("+") if t.strip()
+        }
+        self._listen_tokens = {
+            t.strip() for t in self.listen_key_name.split("+") if t.strip()
+        }
 
         self._listener = keyboard.Listener(
             on_press=self._on_press, on_release=self._on_release
@@ -129,6 +145,10 @@ class HotkeyManager:
             print(f"⌨️  Screen grab: press [{self.capture_key_name}] then drag a box.")
         if self._toggle_tokens:
             print(f"⌨️  Show/hide overlay: press [{self.toggle_key_name}].")
+        if self._summary_tokens:
+            print(f"⌨️  Meeting summary: press [{self.summary_key_name}].")
+        if self._listen_tokens:
+            print(f"⌨️  Auto-listen on/off: press [{self.listen_key_name}].")
         print("   (macOS: grant Accessibility permission if keys don't respond.)")
         return True
 
@@ -158,6 +178,16 @@ class HotkeyManager:
             self._toggle_fired = True
             self._safe(self.on_toggle)
 
+        if (self._summary_tokens and not self._summary_fired
+                and self._summary_tokens.issubset(self._down)):
+            self._summary_fired = True
+            self._safe(self.on_summary)
+
+        if (self._listen_tokens and not self._listen_fired
+                and self._listen_tokens.issubset(self._down)):
+            self._listen_fired = True
+            self._safe(self.on_listen)
+
     def _on_release(self, key):
         if self._ptt_key is not None and key == self._ptt_key and self._armed:
             self._armed = False
@@ -172,3 +202,9 @@ class HotkeyManager:
 
         if self._toggle_tokens and not self._toggle_tokens.issubset(self._down):
             self._toggle_fired = False
+
+        if self._summary_tokens and not self._summary_tokens.issubset(self._down):
+            self._summary_fired = False
+
+        if self._listen_tokens and not self._listen_tokens.issubset(self._down):
+            self._listen_fired = False