diff --git a/src/spek-audio.cc b/src/spek-audio.cc
index 82d7271..5bb18b6 100644
--- a/src/spek-audio.cc
+++ b/src/spek-audio.cc
@@ -51,7 +51,6 @@ private:
     int channel;
 
     AVPacket *packet;
-    int offset;
     AVFrame *frame;
     int buffer_len;
     float *buffer;
@@ -204,7 +203,6 @@ AudioFileImpl::AudioFileImpl(
     this->packet = av_packet_alloc();
     this->packet->data = nullptr;
     this->packet->size = 0;
-    this->offset = 0;
     this->frame = av_frame_alloc();
     this->buffer_len = 0;
     this->buffer = nullptr;
@@ -221,11 +219,8 @@ AudioFileImpl::~AudioFileImpl()
     if (this->frame) {
         av_frame_free(&this->frame);
     }
-    if (this->packet->data) {
-        this->packet->data -= this->offset;
-        this->packet->size += this->offset;
-        this->offset = 0;
-        av_packet_unref(this->packet);
+    if (this->packet) {
+        av_packet_free(&this->packet);
     }
     if (this->codec_context) {
         avcodec_free_context(&codec_context);
@@ -254,36 +249,58 @@ void AudioFileImpl::start(int channel, int samples)
 int AudioFileImpl::read()
 {
     if (!!this->error) {
+        // Stop on error.
         return -1;
     }
 
+    // This runs a state machine to allow incremental calls to decode and return the next chunk of samples.
+    // FFmpeg docs: https://ffmpeg.org/doxygen/5.1/group__lavc__encdec.html
     for (;;) {
-        while (this->packet->size > 0) {
-            av_frame_unref(this->frame);
-            int ret;
-            ret = avcodec_send_packet(this->codec_context, this->packet);
-            if (ret < 0) {
-                // Error sending a packet for decoding, skip the frame.
+        if (!this->packet) {
+            // Finished decoding.
+            return 0;
+        }
+
+        // Read the next packet.
+        int ret = 0;
+        while ((ret = av_read_frame(this->format_context, this->packet)) >= 0) {
+            if (this->packet->stream_index == this->audio_stream) {
                 break;
             }
-            ret = avcodec_receive_frame(this->codec_context, this->frame);
-            if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) {
-                // Error during decoding, skip the frame.
-                break;
-            }
-            int len = this->packet->size;
-            this->packet->data += len;
-            this->packet->size -= len;
-            this->offset += len;
+            av_packet_unref(this->packet);
+        }
+
+        if (ret < 0) {
+            // End of file or error, empty the packet to flush the decoder.
+            av_packet_unref(this->packet);
+            av_packet_free(&this->packet);
+            this->packet = nullptr;
+        }
+
+        ret = avcodec_send_packet(this->codec_context, this->packet);
+        if (this->packet) {
+            av_packet_unref(this->packet);
+        }
+        if (ret < 0) {
+            // Skip the packet.
+            continue;
+        }
+
+        // The packet can contain multiple frames, read all of them.
+        int total_samples = 0;
+        while ((ret = avcodec_receive_frame(this->codec_context, this->frame)) >= 0) {
             int samples = this->frame->nb_samples;
-            // Occasionally the frame has no samples, move on to the next one.
-            if (samples == 0) continue;
-            // We have data, return it and come back for more later.
-            if (samples > this->buffer_len) {
+            if (samples == 0) {
+                // Occasionally the frame has no samples, move on to the next one.
+                av_frame_unref(this->frame);
+                continue;
+            }
+            // We have the data, normalise and write to the buffer.
+            if ((total_samples + samples) > this->buffer_len) {
                 this->buffer = static_cast<float*>(
-                    av_realloc(this->buffer, samples * sizeof(float))
+                    av_realloc(this->buffer, (total_samples + samples) * sizeof(float))
                 );
-                this->buffer_len = samples;
+                this->buffer_len = total_samples + samples;
             }
 
             AVSampleFormat format = static_cast<AVSampleFormat>(this->frame->format);
@@ -322,27 +339,14 @@ int AudioFileImpl::read()
                     value = 0.0f;
                     break;
                 }
-                this->buffer[sample] = value;
+                this->buffer[total_samples + sample] = value;
             }
-            return samples;
+            total_samples += samples;
+            av_frame_unref(this->frame);
         }
-        if (this->packet->data) {
-            this->packet->data -= this->offset;
-            this->packet->size += this->offset;
-            this->offset = 0;
-            av_packet_unref(this->packet);
-        }
-
-        int res = 0;
-        while ((res = av_read_frame(this->format_context, this->packet)) >= 0) {
-            if (this->packet->stream_index == this->audio_stream) {
-                break;
-            }
-            av_packet_unref(this->packet);
-        }
-        if (res < 0) {
-            // End of file or error.
-            return 0;
+        if (total_samples > 0) {
+            return total_samples;
         }
+        // Error during decoding or EOF, skip the packet.
     }
 }
diff --git a/tests/test-audio.cc b/tests/test-audio.cc
index 25031c7..d2b002e 100644
--- a/tests/test-audio.cc
+++ b/tests/test-audio.cc
@@ -94,7 +94,7 @@ void test_audio()
         {"2ch-44100Hz-q100.m4a",
             {AudioError::OK, "AAC", 159649, 44100, 0, 2, AAC_T, 5120}},
         {"2ch-44100Hz-q5.ogg",
-            {AudioError::OK, "Vorbis", 160000, 44100, 0, 2, 0.1, 4672}},
+            {AudioError::OK, "Vorbis", 160000, 44100, 0, 2, 0.1, 4282}},
         {"2ch-44100Hz.dts",
             {AudioError::OK, "DCA", 1411200, 44100, 0, 2, DCA_T, 5120}},
         {"2ch-44100Hz.ac3",
@@ -102,9 +102,9 @@ void test_audio()
         {"2ch-44100Hz-std.mpc",
             {AudioError::OK, "Musepack", 0, 44100, 0, 2, 0.0, 5760}},
         {"2ch-44100Hz-v1.wma",
-            {AudioError::OK, "Windows Media Audio 1", 128000, 44100, 0, 2, 0.138, 6 * 1024}},
+            {AudioError::OK, "Windows Media Audio 1", 128000, 44100, 0, 2, 0.138, 4 * 1024}},
         {"2ch-44100Hz-v2.wma",
-            {AudioError::OK, "Windows Media Audio 2", 128000, 44100, 0, 2, 0.138, 6 * 1024}},
+            {AudioError::OK, "Windows Media Audio 2", 128000, 44100, 0, 2, 0.138, 4 * 1024}},
     };
 
     Audio audio;