diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 2144e54a6c89..bd55a0cd7bde 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -894,6 +894,9 @@ static int xts_crypt(struct skcipher_request *req, bool encrypt)
 			kernel_fpu_begin();
 	}
 
+	if (walk.nbytes == 0)
+		kernel_fpu_end();
+
 	if (unlikely(tail > 0 && !err)) {
 		struct scatterlist sg_src[2], sg_dst[2];
 		struct scatterlist *src, *dst;