diff --git a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h
--- a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h
+++ b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.h
@@ -146,10 +146,15 @@
   RefPtr<ImageContainer> mImageContainer;
   VideoInfo mInfo;
   int mDecodedFrames;
 #if LIBAVCODEC_VERSION_MAJOR >= 58
   int mDecodedFramesLate;
+  // Tracks when decode time of recent frame and averange decode time of
+  // previous frames is bigger than frame interval,
+  // i.e. we fail to decode in time.
+  // We switch to SW decode when we hit HW_DECODE_LATE_FRAMES treshold.
+  int mMissedDecodeInAverangeTime;
 #endif
   float mAverangeDecodeTime;
 
   class PtsCorrectionContext {
    public:
diff --git a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp
--- a/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp
+++ b/dom/media/platforms/ffmpeg/FFmpegVideoDecoder.cpp
@@ -14,10 +14,13 @@
 #include "VPXDecoder.h"
 #include "mozilla/layers/KnowsCompositor.h"
 #if LIBAVCODEC_VERSION_MAJOR >= 57
 #  include "mozilla/layers/TextureClient.h"
 #endif
+#if LIBAVCODEC_VERSION_MAJOR >= 58
+#  include "mozilla/ProfilerMarkers.h"
+#endif
 #ifdef MOZ_WAYLAND_USE_VAAPI
 #  include "H264.h"
 #  include "mozilla/layers/DMABUFSurfaceImage.h"
 #  include "mozilla/widget/DMABufLibWrapper.h"
 #  include "FFmpegVideoFramePool.h"
@@ -56,13 +59,14 @@
 typedef int VAStatus;
 #  define VA_EXPORT_SURFACE_READ_ONLY 0x0001
 #  define VA_EXPORT_SURFACE_SEPARATE_LAYERS 0x0004
 #  define VA_STATUS_SUCCESS 0x00000000
 #endif
-
 // Use some extra HW frames for potential rendering lags.
 #define EXTRA_HW_FRAMES 6
+// Defines number of delayed frames until we switch back to SW decode.
+#define HW_DECODE_LATE_FRAMES 15
 
 #if LIBAVCODEC_VERSION_MAJOR >= 57 && LIBAVUTIL_VERSION_MAJOR >= 56
 #  define CUSTOMIZED_BUFFER_ALLOCATION 1
 #endif
 
@@ -386,10 +390,11 @@
       mImageContainer(aImageContainer),
       mInfo(aConfig),
       mDecodedFrames(0),
 #if LIBAVCODEC_VERSION_MAJOR >= 58
       mDecodedFramesLate(0),
+      mMissedDecodeInAverangeTime(0),
 #endif
       mAverangeDecodeTime(0),
       mLowLatency(aLowLatency) {
   FFMPEG_LOG("FFmpegVideoDecoder::FFmpegVideoDecoder MIME %s Codec ID %d",
              aConfig.mMimeType.get(), mCodecID);
@@ -781,22 +786,32 @@
   float decodeTime = (TimeStamp::Now() - aDecodeStart).ToMilliseconds();
   mAverangeDecodeTime =
       (mAverangeDecodeTime * (mDecodedFrames - 1) + decodeTime) /
       mDecodedFrames;
   FFMPEG_LOG(
-      "  decode time %.2f ms averange decode time %.2f ms decoded frames %d\n",
+      "Frame decode finished, time %.2f ms averange decode time %.2f ms "
+      "decoded %d frames\n",
       decodeTime, mAverangeDecodeTime, mDecodedFrames);
 #if LIBAVCODEC_VERSION_MAJOR >= 58
-  int frameDuration = mFrame->pkt_duration;
-  if (frameDuration > 0 && frameDuration / 1000.0 < decodeTime) {
-    mDecodedFramesLate++;
-    FFMPEG_LOG(
-        "  slow decode: failed to decode in time, frame duration %.2f ms, "
-        "decode time %.2f\n",
-        frameDuration / 1000.0, decodeTime);
-    FFMPEG_LOG("  all decoded frames / late decoded frames %d/%d\n",
-               mDecodedFrames, mDecodedFramesLate);
+  if (mFrame->pkt_duration > 0) {
+    // Switch frame duration to ms
+    float frameDuration = mFrame->pkt_duration / 1000.0f;
+    if (frameDuration < decodeTime) {
+      PROFILER_MARKER_TEXT("FFmpegVideoDecoder::DoDecode", MEDIA_PLAYBACK, {},
+                           "frame decode takes too long");
+      mDecodedFramesLate++;
+      if (frameDuration < mAverangeDecodeTime) {
+        mMissedDecodeInAverangeTime++;
+      }
+      FFMPEG_LOG(
+          "  slow decode: failed to decode in time, frame duration %.2f ms, "
+          "decode time %.2f\n",
+          frameDuration, decodeTime);
+      FFMPEG_LOG("  frames: all decoded %d late decoded %d over averange %d\n",
+                 mDecodedFrames, mDecodedFramesLate,
+                 mMissedDecodeInAverangeTime);
+    }
   }
 #endif
 }
 
 MediaResult FFmpegVideoDecoder<LIBAV_VER>::DoDecode(
@@ -866,10 +881,18 @@
     decodeStart = TimeStamp::Now();
 
     MediaResult rv;
 #  ifdef MOZ_WAYLAND_USE_VAAPI
     if (IsHardwareAccelerated()) {
+      if (mMissedDecodeInAverangeTime > HW_DECODE_LATE_FRAMES) {
+        PROFILER_MARKER_TEXT("FFmpegVideoDecoder::DoDecode", MEDIA_PLAYBACK, {},
+                             "Fallback to SW decode");
+        FFMPEG_LOG("  HW decoding is slow, switch back to SW decode");
+        return MediaResult(
+            NS_ERROR_DOM_MEDIA_DECODE_ERR,
+            RESULT_DETAIL("HW decoding is slow, switch back to SW decode"));
+      }
       rv = CreateImageVAAPI(mFrame->pkt_pos, GetFramePts(mFrame),
                             mFrame->pkt_duration, aResults);
       // If VA-API playback failed, just quit. Decoder is going to be restarted
       // without VA-API.
       if (NS_FAILED(rv)) {
@@ -1129,11 +1152,11 @@
 }
 
 MediaResult FFmpegVideoDecoder<LIBAV_VER>::CreateImageVAAPI(
     int64_t aOffset, int64_t aPts, int64_t aDuration,
     MediaDataDecoder::DecodedData& aResults) {
-  FFMPEG_LOG("VA-API Got one frame output with pts=%" PRId64 "dts=%" PRId64
+  FFMPEG_LOG("VA-API Got one frame output with pts=%" PRId64 " dts=%" PRId64
              " duration=%" PRId64 " opaque=%" PRId64,
              aPts, mFrame->pkt_dts, aDuration, mCodecContext->reordered_opaque);
 
   VADRMPRIMESurfaceDescriptor vaDesc;
   if (!GetVAAPISurfaceDescriptor(&vaDesc)) {