diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c
index f1bbce4350c4..e8911edf7858 100644
--- a/fs/ext4/extents.c
+++ b/fs/ext4/extents.c
@@ -4932,6 +4932,12 @@ long ext4_fallocate(struct file *file, int mode, loff_t offset, loff_t len)
 		     FALLOC_FL_INSERT_RANGE))
 		return -EOPNOTSUPP;
 
+	inode_lock(inode);
+	ret = ext4_convert_inline_data(inode);
+	inode_unlock(inode);
+	if (ret)
+		return ret;
+
 	if (mode & FALLOC_FL_PUNCH_HOLE)
 		return ext4_punch_hole(inode, offset, len);
 
diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
index 1cac574911a7..d8fee911d4f4 100644
--- a/fs/ext4/inode.c
+++ b/fs/ext4/inode.c
@@ -4322,15 +4322,6 @@ int ext4_punch_hole(struct inode *inode, loff_t offset, loff_t length)
 
 	trace_ext4_punch_hole(inode, offset, length, 0);
 
-	ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
-	if (ext4_has_inline_data(inode)) {
-		down_write(&EXT4_I(inode)->i_mmap_sem);
-		ret = ext4_convert_inline_data(inode);
-		up_write(&EXT4_I(inode)->i_mmap_sem);
-		if (ret)
-			return ret;
-	}
-
 	/*
 	 * Write out all dirty pages to avoid race conditions
 	 * Then release them.