diff --git a/test/test_postprocessors.py b/test/test_postprocessors.py
index 1ca90cd4c7..8d3fac86cb 100644
--- a/test/test_postprocessors.py
+++ b/test/test_postprocessors.py
@@ -567,7 +567,7 @@ def test_round_remove_chapter_Common(self):
         ])
         self.assertEqual(chapters, [
             {'start_time': 0, 'end_time': 2},
-            {'start_time': 3, 'end_time': 5, 'remove': True},
+            {'start_time': 2, 'end_time': 5, 'remove': True},
             {'start_time': 6, 'end_time': 10, 'remove': False},
         ])
 
diff --git a/yt_dlp/postprocessor/modify_chapters.py b/yt_dlp/postprocessor/modify_chapters.py
index 7eec34474c..a6477451ff 100644
--- a/yt_dlp/postprocessor/modify_chapters.py
+++ b/yt_dlp/postprocessor/modify_chapters.py
@@ -341,11 +341,6 @@ def _round_remove_chapters(keyframes, chapters):
                 result.append(c)
                 continue
 
-            start_frame = bisect.bisect_left(keyframes, c['start_time'])
-            if start_frame >= len(keyframes):
-                continue
-
-            c['start_time'] = keyframes[start_frame]
             if c['end_time'] < keyframes[-1]:
                 c['end_time'] = keyframes[bisect.bisect_right(keyframes, c['end_time']) - 1]
             result.append(c)