x86_64_start_reservations+0x18/0x30 arch/x86/kernel/head64.c:310 x86_64_start_kernel+0x130/0x190 arch/x86/kernel/head64.c:291 common_startup_64+0x13e/0x148 hrtimer: interrupt took 3712658619 ns BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 173s! BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=-20 stuck for 168s! Showing busy workqueues and worker pools: workqueue events: flags=0x100 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=16 refcnt=17 pending: 4*nsim_dev_hwstats_traffic_work, psi_avgs_work, vmstat_shepherd, 2*psi_avgs_work, 4*ovs_dp_masks_rebalance, switchdev_deferred_process_work, 3*rht_deferred_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=15 refcnt=16 in-flight: 6703:nsim_fib_event_work pending: 6*nsim_dev_hwstats_traffic_work, 4*psi_avgs_work, 4*ovs_dp_masks_rebalance workqueue events_highpri: flags=0x110 pwq 3: cpus=0 node=0 flags=0x0 nice=-20 active=3 refcnt=4 in-flight: 93:snd_vmidi_output_work snd_vmidi_output_work pending: fill_page_cache_func pwq 7: cpus=1 node=0 flags=0x0 nice=-20 active=1 refcnt=2 pending: fill_page_cache_func workqueue events_long: flags=0x100 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=5 refcnt=6 pending: defense_work_handler, 3*br_multicast_gc_work, br_fdb_cleanup pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=9 refcnt=10 pending: 7*defense_work_handler, 2*br_fdb_cleanup workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 in-flight: 3684:call_usermodehelper_exec_work pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3 in-flight: 6693:toggle_allocation_gate pending: flush_memcg_stats_dwork workqueue events_unbound: flags=0x2 pwq 8: cpus=0-1 flags=0x4 nice=0 active=12 refcnt=13 pending: 2*nsim_dev_trap_report_work, 5*cfg80211_wiphy_work, macvlan_process_broadcast, crng_reseed, 2*macvlan_process_broadcast, idle_cull_fn pwq 8: cpus=0-1 flags=0x4 nice=0 active=27 refcnt=28 in-flight: 60:linkwatch_event ,6569:cfg80211_wiphy_work pending: 8*nsim_dev_trap_report_work, 7*cfg80211_wiphy_work, 9*macvlan_process_broadcast, idle_cull_fn workqueue events_power_efficient: flags=0x180 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=7 refcnt=8 pending: hash_ipport4_gc, wg_ratelimiter_gc_entries, check_lifetime, gc_worker, reg_check_chans_work, 2*check_lifetime pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=11 refcnt=12 pending: hash_ipport4_gc, 2*neigh_managed_work, 2*neigh_periodic_work, 2*check_lifetime, do_cache_clean, 3*check_lifetime workqueue kvfree_rcu_reclaim: flags=0xa pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: kfree_rcu_monitor pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: kfree_rcu_monitor workqueue mm_percpu_wq: flags=0x8 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: vmstat_update workqueue writeback: flags=0x4a pwq 8: cpus=0-1 flags=0x4 nice=0 active=2 refcnt=3 pending: wb_update_bandwidth_workfn, wb_workfn pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wb_workfn workqueue kblockd: flags=0x18 pwq 3: cpus=0 node=0 flags=0x0 nice=-20 active=1 refcnt=2 pending: blk_mq_timeout_work pwq 7: cpus=1 node=0 flags=0x0 nice=-20 active=1 refcnt=2 pending: blk_mq_timeout_work workqueue mld: flags=0x40108 pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: mld_ifc_work workqueue ipv6_addrconf: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=11 pending: addrconf_verify_work inactive: 7*addrconf_verify_work workqueue krxrpcd: flags=0x2001a pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=11 pending: rxrpc_peer_keepalive_worker inactive: 7*rxrpc_peer_keepalive_worker workqueue bat_events: flags=0x6000a pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=51 pending: batadv_mcast_mla_update inactive: 4*batadv_mcast_mla_update, batadv_purge_orig, batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_purge_orig, 2*batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_purge_orig, 3*batadv_iv_send_outstanding_bat_ogm_packet, batadv_purge_orig, 12*batadv_iv_send_outstanding_bat_ogm_packet, batadv_mcast_mla_update, 3*batadv_tt_purge, batadv_bla_periodic_work, batadv_dat_purge, 3*batadv_tt_purge, batadv_bla_periodic_work, 2*batadv_dat_purge, 2*batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge, batadv_bla_periodic_work, batadv_dat_purge workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue hci3: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 pending: hci_conn_timeout workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg1: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg0: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-kex-wg2: flags=0x6 pwq 8: cpus=0-1 flags=0x4 nice=0 active=1 refcnt=2 pending: wg_packet_handshake_send_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue hci5: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=4 pending: hci_conn_timeout workqueue wg-crypt-wg0: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg1: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue wg-crypt-wg2: flags=0x128 pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2 pending: wg_packet_encrypt_worker workqueue hci1: flags=0x20012 pwq 9: cpus=0-1 node=0 flags=0x4 nice=-20 active=1 refcnt=5 pending: hci_cmd_timeout inactive: hci_conn_timeout pool 3: cpus=0 node=0 flags=0x0 nice=-20 hung=170s workers=3 idle: 11 7711 pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=174s workers=16 idle: 5941 5942 6500 932 6137 6727 6085 9053 6079 30 5943 6153 5914 24 6778 pool 8: cpus=0-1 flags=0x4 nice=0 hung=175s workers=31 idle: 3455 6567 6005 4564 6262 2054 6350 49 6026 1040 8546 13 6099 6496 6002 12 8621 6057 6069 8336 2079 633 6779 37 3604 8238 6084 Showing backtraces of running workers in stalled CPU-bound worker pools: pool 6: task:kworker/1:15 state:R running task stack:26360 pid:6703 tgid:6703 ppid:2 task_flags:0x4208060 flags:0x00080000 Workqueue: events nsim_fib_event_work Call Trace: context_switch kernel/sched/core.c:5256 [inline] __schedule+0x1139/0x6150 kernel/sched/core.c:6863 preempt_schedule_irq+0x51/0x90 kernel/sched/core.c:7190 irqentry_exit+0x1d8/0x8c0 kernel/entry/common.c:216 asm_sysvec_reschedule_ipi+0x1a/0x20 arch/x86/include/asm/idtentry.h:702 RIP: 0010:arch_stack_walk+0x9a/0x100 arch/x86/kernel/stacktrace.c:24 Code: 6a 09 00 8b 95 70 ff ff ff 85 d2 75 21 eb 2f 4c 89 ef 2e e8 d8 9c 16 0a 84 c0 74 22 48 89 df e8 0c 4a 09 00 8b 85 70 ff ff ff <85> c0 74 10 48 89 df e8 4a 44 09 00 48 89 c6 48 85 c0 75 d1 48 8b RSP: 0018:ffffc90003bcf650 EFLAGS: 00000286 RAX: 0000000000000000 RBX: ffffc90003bcf650 RCX: ffffc90003bcf5a4 RDX: 0000000000000000 RSI: ffffffff8daa6ced RDI: ffff88807a3e2304 RBP: ffffc90003bcf6e0 R08: ffffffff911b927c R09: 00000000195af3d4 R10: 0000000000000002 R11: 0000000000002b91 R12: ffffffff81a96200 R13: ffffc90003bcf710 R14: 0000000000000000 R15: ffff88807a3e1e80 stack_trace_save+0x8e/0xc0 kernel/stacktrace.c:122 kasan_save_stack+0x33/0x60 mm/kasan/common.c:57 kasan_save_track+0x14/0x30 mm/kasan/common.c:78 kasan_save_free_info+0x3b/0x60 mm/kasan/generic.c:584 poison_slab_object mm/kasan/common.c:253 [inline] __kasan_slab_free+0x5f/0x80 mm/kasan/common.c:285 kasan_slab_free include/linux/kasan.h:235 [inline] slab_free_hook mm/slub.c:2540 [inline] slab_free mm/slub.c:6670 [inline] kfree+0x2f8/0x6e0 mm/slub.c:6878 nsim_fib_event_work+0x405/0x65f0 drivers/net/netdevsim/fib.c:1495 process_one_work+0x9ba/0x1b20 kernel/workqueue.c:3257 process_scheduled_works kernel/workqueue.c:3340 [inline] worker_thread+0x6c8/0xf10 kernel/workqueue.c:3421 kthread+0x3c5/0x780 kernel/kthread.c:463 ret_from_fork+0x983/0xb10 arch/x86/kernel/process.c:158 ret_from_fork_asm+0x1a/0x30 arch/x86/entry/entry_64.S:246 ---------------- Code disassembly (best guess), 1 bytes skipped: 0: 09 00 or %eax,(%rax) 2: 8b 95 70 ff ff ff mov -0x90(%rbp),%edx 8: 85 d2 test %edx,%edx a: 75 21 jne 0x2d c: eb 2f jmp 0x3d e: 4c 89 ef mov %r13,%rdi 11: 2e e8 d8 9c 16 0a cs call 0xa169cef 17: 84 c0 test %al,%al 19: 74 22 je 0x3d 1b: 48 89 df mov %rbx,%rdi 1e: e8 0c 4a 09 00 call 0x94a2f 23: 8b 85 70 ff ff ff mov -0x90(%rbp),%eax * 29: 85 c0 test %eax,%eax <-- trapping instruction 2b: 74 10 je 0x3d 2d: 48 89 df mov %rbx,%rdi 30: e8 4a 44 09 00 call 0x9447f 35: 48 89 c6 mov %rax,%rsi 38: 48 85 c0 test %rax,%rax 3b: 75 d1 jne 0xe 3d: 48 rex.W 3e: 8b .byte 0x8b