From c20024f5073aa17b7a80ca314551488a6d67adab Mon Sep 17 00:00:00 2001 From: Zihan Dai Date: Wed, 11 Nov 2026 02:35:47 +1100 Subject: [PATCH] Keep Unpacker.tell() at the start of an incomplete object When the buffer ends inside an object, the C extension decodes the part it has and keeps it for the next feed. tell() then included those bytes, so it pointed into the middle of the next object instead of at its start. Code that saves tell() to resume a stream later restarted mid-object and decoded garbage. The pure Python implementation already returns the start. --- msgpack/_unpacker.pyx | 7 +++++++ test/test_sequnpack.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 3c81808f..b0e84528 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -325,6 +325,8 @@ cdef class Unpacker: cdef object unicode_errors cdef Py_ssize_t max_buffer_size cdef uint64_t stream_offset + # stream_offset at the start of an incomplete object + cdef uint64_t object_offset cdef bint _unpacking def __dealloc__(self): @@ -486,6 +488,8 @@ cdef class Unpacker: raise RuntimeError("Unpacker cannot be called recursively while unpacking is in progress") if self.ctx.execute != NULL and self.ctx.execute != execute: raise ValueError("Cannot switch unpacking methods while an object is incomplete") + if self.ctx.execute == NULL: + self.object_offset = self.stream_offset self._unpacking = True try: @@ -585,6 +589,9 @@ cdef class Unpacker: number of bytes that were read from the input, also the starting position of the next object. """ + if self.ctx.execute != NULL: + # Part of the next object has been decoded, but it still starts here. + return self.object_offset return self.stream_offset def __iter__(self): diff --git a/test/test_sequnpack.py b/test/test_sequnpack.py index 904be42f..15682721 100644 --- a/test/test_sequnpack.py +++ b/test/test_sequnpack.py @@ -169,3 +169,22 @@ def test_unpack_tell(): m2 = next(unpacker) assert m == m2 assert o == unpacker.tell() + + +def test_unpack_tell_incomplete_object(): + # tell() is where the next object starts, even when only part of it has arrived. + first = packb([1, 2, 3]) + second = packb(["x" * 40, "y" * 40]) + packed = first + second + for cut in (len(first) + 1, len(first) + 3, len(first) + 45): + unpacker = Unpacker() + unpacker.feed(packed[:cut]) + assert list(unpacker) == [[1, 2, 3]] + assert unpacker.tell() == len(first) + unpacker.feed(packed[cut:]) + assert list(unpacker) == [["x" * 40, "y" * 40]] + assert unpacker.tell() == len(packed) + + unpacker = Unpacker(io.BytesIO(packed[:cut]), read_size=16) + assert list(unpacker) == [[1, 2, 3]] + assert unpacker.tell() == len(first)