diff --git a/src/stream.rs b/src/stream.rs
index 82f7be3..88c3fed 100644
--- a/src/stream.rs
+++ b/src/stream.rs
@@ -101,13 +101,15 @@ impl ByteStreamReader {
 		}
 	}
 
-	/// Tries to fill the internal read buffer from the underlying stream.
+	/// Tries to fill the internal read buffer from the underlying stream if it is empty.
 	/// Calling this might be necessary to ensure that `.eos()` returns a correct
 	/// result, otherwise the reader might not be aware that the underlying
 	/// stream has nothing left to return.
 	pub async fn fill_buffer(&mut self) {
-		let packet = self.stream.next().await;
-		self.add_stream_next(packet);
+		if self.buf.is_empty() {
+			let packet = self.stream.next().await;
+			self.add_stream_next(packet);
+		}
 	}
 
 	/// Clears the internal read buffer and returns its content