Fix (beta): Catch consumer files created during watcher re-creations (#13013)

src/documents/tests/test_management_consumer.py

@@ -684,6 +684,7 @@ class ConsumerThread(Thread):
         subdirs_as_tags: bool = False,
         polling_interval: float = 0,
         stability_delay: float = 0.1,
+        rescan_interval: float | None = None,
     ) -> None:
         super().__init__()
         self.consumption_dir = consumption_dir
@@ -693,6 +694,8 @@ class ConsumerThread(Thread):
         self.polling_interval = polling_interval
         self.stability_delay = stability_delay
         self.cmd = Command()
+        if rescan_interval is not None:
+            self.cmd.rescan_interval_s = rescan_interval
         self.cmd.stop_flag.clear()
         # Non-daemon ensures finally block runs and connections are closed
         self.daemon = False
@@ -1052,3 +1055,200 @@ class TestCommandWatchEdgeCases:
             thread.stop_and_wait(timeout=5.0)
             # Clean up any Tags created by the thread
             Tag.objects.all().delete()
+
+
+class TestRescanExistingFiles:
+    """
+    Unit tests for the rescan safety net.
+
+    Each ``watch()`` recreation silently adopts the current directory contents
+    as its baseline, so a file appearing between one batch and the next
+    watcher's baseline is never reported and would sit in the consume directory
+    forever. ``_rescan_existing_files`` re-injects such files into the
+    stability tracker as a periodic safety net (see GH issue #13011).
+    """
+
+    @pytest.fixture
+    def pdf_only_filter(self) -> ConsumerFilter:
+        return ConsumerFilter(
+            supported_extensions=frozenset({".pdf"}),
+            ignore_patterns=[],
+        )
+
+    def _rescan(
+        self,
+        directory: Path,
+        consumer_filter: ConsumerFilter,
+        tracker: FileStabilityTracker,
+        queued: set[Path],
+        *,
+        recursive: bool = False,
+    ) -> None:
+        Command()._rescan_existing_files(
+            directory=directory,
+            recursive=recursive,
+            consumer_filter=consumer_filter,
+            tracker=tracker,
+            queued=queued,
+        )
+
+    def test_tracks_stranded_file(
+        self,
+        consumption_dir: Path,
+        sample_pdf: Path,
+        pdf_only_filter: ConsumerFilter,
+    ) -> None:
+        """A supported on-disk file the watcher never reported gets tracked."""
+        target = consumption_dir / "stranded.pdf"
+        shutil.copy(sample_pdf, target)
+        tracker = FileStabilityTracker(stability_delay=0.1)
+
+        self._rescan(consumption_dir, pdf_only_filter, tracker, set())
+
+        assert tracker.is_tracking(target) is True
+        assert tracker.pending_count == 1
+
+    def test_skips_already_tracked_file(
+        self,
+        consumption_dir: Path,
+        sample_pdf: Path,
+        pdf_only_filter: ConsumerFilter,
+    ) -> None:
+        """A file already being tracked by the watcher is not double-tracked."""
+        target = consumption_dir / "tracked.pdf"
+        shutil.copy(sample_pdf, target)
+        tracker = FileStabilityTracker(stability_delay=0.1)
+        tracker.track(target, Change.added)
+
+        self._rescan(consumption_dir, pdf_only_filter, tracker, set())
+
+        assert tracker.pending_count == 1
+
+    def test_skips_queued_file(
+        self,
+        consumption_dir: Path,
+        sample_pdf: Path,
+        pdf_only_filter: ConsumerFilter,
+    ) -> None:
+        """A file already queued and awaiting consumption is not re-tracked."""
+        target = consumption_dir / "inflight.pdf"
+        shutil.copy(sample_pdf, target)
+        tracker = FileStabilityTracker(stability_delay=0.1)
+        queued = {target.resolve()}
+
+        self._rescan(consumption_dir, pdf_only_filter, tracker, queued)
+
+        assert tracker.pending_count == 0
+
+    def test_prunes_vanished_queued_paths(
+        self,
+        consumption_dir: Path,
+        pdf_only_filter: ConsumerFilter,
+    ) -> None:
+        """Queued paths no longer on disk are dropped so the name can recur."""
+        gone = (consumption_dir / "gone.pdf").resolve()
+        tracker = FileStabilityTracker(stability_delay=0.1)
+        queued = {gone}
+
+        self._rescan(consumption_dir, pdf_only_filter, tracker, queued)
+
+        assert gone not in queued
+
+    def test_skips_unsupported_extension(
+        self,
+        consumption_dir: Path,
+        pdf_only_filter: ConsumerFilter,
+    ) -> None:
+        """Files filtered out by the consumer filter are not tracked."""
+        (consumption_dir / "notes.xyz").write_bytes(b"content")
+        tracker = FileStabilityTracker(stability_delay=0.1)
+
+        self._rescan(consumption_dir, pdf_only_filter, tracker, set())
+
+        assert tracker.pending_count == 0
+
+    def test_recursive_respects_flag(
+        self,
+        consumption_dir: Path,
+        sample_pdf: Path,
+        pdf_only_filter: ConsumerFilter,
+    ) -> None:
+        """Nested files are only found when recursive scanning is enabled."""
+        subdir = consumption_dir / "nested"
+        subdir.mkdir()
+        target = subdir / "deep.pdf"
+        shutil.copy(sample_pdf, target)
+
+        shallow = FileStabilityTracker(stability_delay=0.1)
+        self._rescan(consumption_dir, pdf_only_filter, shallow, set())
+        assert shallow.pending_count == 0
+
+        deep = FileStabilityTracker(stability_delay=0.1)
+        self._rescan(consumption_dir, pdf_only_filter, deep, set(), recursive=True)
+        assert deep.is_tracking(target) is True
+
+
+class TestProcessExistingFilesQueued:
+    """Tests that startup processing reports which paths it queued."""
+
+    @pytest.mark.usefixtures("mock_supported_extensions")
+    def test_returns_queued_paths(
+        self,
+        consumption_dir: Path,
+        sample_pdf: Path,
+        mock_consume_file_delay: MagicMock,
+        settings: SettingsWrapper,
+    ) -> None:
+        """The set returned seeds the rescan's queued set, avoiding re-queue."""
+        target = consumption_dir / "document.pdf"
+        shutil.copy(sample_pdf, target)
+        settings.CONSUMER_IGNORE_PATTERNS = []
+
+        queued = Command()._process_existing_files(
+            directory=consumption_dir,
+            recursive=False,
+            subdirs_as_tags=False,
+            consumer_filter=ConsumerFilter(ignore_patterns=[]),
+        )
+
+        assert target.resolve() in queued
+
+
+@pytest.mark.management
+@pytest.mark.django_db
+class TestCommandRescanRecovery:
+    """End-to-end test that the rescan recovers files the watcher misses."""
+
+    def test_rescan_consumes_file_the_watcher_never_reports(
+        self,
+        consumption_dir: Path,
+        sample_pdf: Path,
+        mock_consume_file_delay: MagicMock,
+        start_consumer: Callable[..., ConsumerThread],
+    ) -> None:
+        """
+        Isolate the rescan path: a long polling interval guarantees the
+        watcher cannot report the file within the test window, so only the
+        periodic rescan can consume it.
+        """
+        # poll interval far longer than the test window -> watcher stays silent
+        thread = start_consumer(
+            polling_interval=30.0,
+            stability_delay=0.1,
+            rescan_interval=0.5,
+        )
+
+        # created after startup, so _process_existing_files did not see it
+        target = consumption_dir / "stranded.pdf"
+        shutil.copy(sample_pdf, target)
+
+        wait_for_mock_call(mock_consume_file_delay.apply_async, timeout_s=5.0)
+
+        if thread.exception:
+            raise thread.exception
+
+        mock_consume_file_delay.apply_async.assert_called()
+        call_args = mock_consume_file_delay.apply_async.call_args.kwargs["kwargs"][
+            "input_doc"
+        ]
+        assert call_args.original_file.name == "stranded.pdf"