BUG: workqueue lockup - pool cpus=1 node=0 flags=0x0 nice=0 stuck for 48s!
Showing busy workqueues and worker pools:
workqueue events: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=5 refcnt=6
    in-flight: 2354:nsim_fib_event_work
    pending: nsim_dev_hwstats_traffic_work, psi_avgs_work, vmstat_shepherd, ovs_dp_masks_rebalance
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=17 refcnt=18
    in-flight: 6672:nsim_fib_event_work nsim_fib_event_work ,26:nsim_fib_event_work nsim_fib_event_work ,6675:nsim_fib_event_work nsim_fib_event_work ,24:rht_deferred_worker
    pending: ovs_dp_masks_rebalance, rfkill_global_led_trigger_worker, psi_avgs_work, ovs_dp_masks_rebalance, psi_avgs_work, drm_fb_helper_damage_work, 4*nsim_dev_hwstats_traffic_work
workqueue events_long: flags=0x100
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=5 refcnt=6
    pending: 5*defense_work_handler
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: defense_work_handler
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: flush_memcg_stats_dwork
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=11 refcnt=12
    in-flight: 4527:cfg80211_wiphy_work cfg80211_wiphy_work ,12:cfg80211_wiphy_work cfg80211_wiphy_work ,15:cfg80211_wiphy_work cfg80211_wiphy_work ,4633:cfg80211_wiphy_work cfg80211_wiphy_work
    pending: toggle_allocation_gate, cfg80211_wiphy_work, crng_reseed
workqueue events_unbound: flags=0x2
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=3 refcnt=4
    in-flight: 6918:nsim_dev_trap_report_work
    pending: 2*nsim_dev_trap_report_work
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=6 refcnt=7
    in-flight: 4443:fsnotify_mark_destroy_workfn ,41:linkwatch_event ,6917:nsim_dev_trap_report_work ,1421:macvlan_process_broadcast
    pending: macvlan_process_broadcast, nsim_dev_trap_report_work
workqueue events_power_efficient: flags=0x182
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=3 refcnt=4
    pending: 2*neigh_managed_work, fb_flashcursor
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=7 refcnt=8
    in-flight: 4811:wg_ratelimiter_gc_entries ,42:crda_timeout_work
    pending: 2*neigh_periodic_work, do_cache_clean, gc_worker, reg_check_chans_work
workqueue rcu_gp: flags=0x108
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: srcu_invoke_callbacks
workqueue kvfree_rcu_reclaim: flags=0xa
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=3 MAYDAY
    pending: kfree_rcu_monitor
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=3 refcnt=6 MAYDAY
    in-flight: 6(RESCUER):kfree_rcu_work kfree_rcu_work
    pending: kfree_rcu_monitor
workqueue mm_percpu_wq: flags=0x8
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: vmstat_update
workqueue mld: flags=0x40108
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=104
    pending: mld_ifc_work
    inactive: 6*mld_ifc_work, 4*mld_dad_work, 6*mld_ifc_work, 6*mld_dad_work, mld_ifc_work, 2*mld_dad_work, 4*mld_ifc_work, 3*mld_dad_work, mld_ifc_work, 6*mld_dad_work, mld_ifc_work, mld_dad_work, 3*mld_ifc_work, 5*mld_dad_work, 3*mld_ifc_work, 2*mld_dad_work, mld_ifc_work, mld_dad_work, 2*mld_ifc_work, mld_dad_work, mld_ifc_work, 7*mld_dad_work, mld_ifc_work, 4*mld_dad_work, 2*mld_ifc_work, 14*mld_dad_work, mld_ifc_work, 4*mld_dad_work, 3*mld_ifc_work, mld_dad_work, mld_ifc_work, 4*mld_dad_work
workqueue ipv6_addrconf: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=256
    in-flight: 653:addrconf_dad_work
    inactive: 251*addrconf_dad_work, addrconf_verify_work
workqueue bat_events: flags=0x6000a
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=44 MAYDAY
    in-flight: 4309(RESCUER):batadv_dat_purge
    inactive: batadv_purge_orig, batadv_bla_periodic_work, batadv_dat_purge, batadv_iv_send_outstanding_bat_ogm_packet, batadv_dat_purge, batadv_bla_periodic_work, 3*batadv_iv_send_outstanding_bat_ogm_packet, batadv_bla_periodic_work, batadv_dat_purge, 5*batadv_iv_send_outstanding_bat_ogm_packet, batadv_mcast_mla_update, batadv_iv_send_outstanding_bat_ogm_packet, batadv_mcast_mla_update, 2*batadv_iv_send_outstanding_bat_ogm_packet, 3*batadv_mcast_mla_update, batadv_bla_periodic_work, batadv_dat_purge, 4*batadv_purge_orig, batadv_iv_send_outstanding_bat_ogm_packet, 2*batadv_tt_purge, 2*batadv_iv_send_outstanding_bat_ogm_packet, 3*batadv_tt_purge
workqueue wg-kex-wg0: flags=0x124
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    in-flight: 6764:wg_packet_handshake_receive_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=4 refcnt=5
    in-flight: 6914:wg_packet_encrypt_worker wg_packet_encrypt_worker
    pending: wg_packet_tx_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    in-flight: 11:wg_packet_tx_worker
    pending: wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_tx_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_tx_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg0: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_tx_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    in-flight: 6654:wg_packet_tx_worker
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker, wg_packet_tx_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_encrypt_worker, wg_packet_decrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg1: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=2 refcnt=3
    pending: wg_packet_decrypt_worker, wg_packet_encrypt_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg2: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-crypt-wg2: flags=0x128
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    pending: wg_packet_tx_worker
  pwq 6: cpus=1 node=0 flags=0x0 nice=0 active=3 refcnt=4
    pending: wg_packet_encrypt_worker, wg_packet_tx_worker, wg_packet_decrypt_worker
workqueue wg-kex-wg0: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=1 refcnt=2
    pending: wg_packet_handshake_send_worker
workqueue wg-kex-wg1: flags=0x124
  pwq 2: cpus=0 node=0 flags=0x0 nice=0 active=1 refcnt=2
    in-flight: 6916:wg_packet_handshake_receive_worker
workqueue wg-kex-wg1: flags=0x6
  pwq 8: cpus=0-1 flags=0x6 nice=0 active=2 refcnt=3
    in-flight: 6919:wg_packet_handshake_send_worker
    pending: wg_packet_handshake_send_worker
pool 2: cpus=0 node=0 flags=0x0 nice=0 hung=2s workers=8 idle: 6875 9
pool 6: cpus=1 node=0 flags=0x0 nice=0 hung=48s workers=9 idle: 6870 1857 6910 6912 6868
pool 8: cpus=0-1 flags=0x6 nice=0 hung=1s workers=14 manager: 6920
Showing backtraces of running workers in stalled CPU-bound worker pools:
pool 6:
task:kworker/1:3     state:R  running task     stack:0     pid:6672  tgid:6672  ppid:2      task_flags:0x4208060 flags:0x00000010
Workqueue: events nsim_fib_event_work
Call trace:
 __switch_to+0x418/0x87c arch/arm64/kernel/process.c:741 (T)
 context_switch kernel/sched/core.c:5325 [inline]
 __schedule+0x13b0/0x2864 kernel/sched/core.c:6929
 preempt_schedule_irq+0x80/0x188 kernel/sched/core.c:7256
 raw_irqentry_exit_cond_resched+0x30/0x44 kernel/entry/common.c:168
 irqentry_exit+0xd4/0x108 kernel/entry/common.c:211
 exit_to_kernel_mode+0x10/0x1c arch/arm64/kernel/entry-common.c:58
 __el1_irq arch/arm64/kernel/entry-common.c:500 [inline]
 el1_interrupt+0x4c/0x60 arch/arm64/kernel/entry-common.c:510
 el1h_64_irq_handler+0x18/0x24 arch/arm64/kernel/entry-common.c:515
 el1h_64_irq+0x6c/0x70 arch/arm64/kernel/entry.S:592
 __debug_check_no_obj_freed lib/debugobjects.c:1108 [inline] (P)
 debug_check_no_obj_freed+0x454/0x470 lib/debugobjects.c:1129 (P)
 free_pages_prepare mm/page_alloc.c:1401 [inline]
 __free_frozen_pages+0x4b8/0xcac mm/page_alloc.c:2906
 free_frozen_pages+0x14/0x20 mm/page_alloc.c:2944
 __free_slab+0x108/0x1c0 mm/slub.c:3301
 free_slab+0x24/0x120 mm/slub.c:3324
 discard_slab+0x48/0x98 mm/slub.c:3330
 __slab_free+0x1c4/0x240 mm/slub.c:5970
 do_slab_free mm/slub.c:6582 [inline]
 ___cache_free+0x150/0x170 mm/slub.c:6708
 qlink_free+0x48/0x94 mm/kasan/quarantine.c:163
 qlist_free_all+0x44/0xb8 mm/kasan/quarantine.c:179
 kasan_quarantine_reduce+0x118/0x124 mm/kasan/quarantine.c:286
 __kasan_slab_alloc+0x2c/0x88 mm/kasan/common.c:352
 kasan_slab_alloc include/linux/kasan.h:252 [inline]
 slab_post_alloc_hook mm/slub.c:4978 [inline]
 slab_alloc_node mm/slub.c:5288 [inline]
 __kmalloc_cache_noprof+0x338/0x65c mm/slub.c:5766
 kmalloc_noprof include/linux/slab.h:957 [inline]
 kzalloc_noprof include/linux/slab.h:1094 [inline]
 nsim_fib4_rt_create drivers/net/netdevsim/fib.c:280 [inline]
 nsim_fib4_rt_insert drivers/net/netdevsim/fib.c:426 [inline]
 nsim_fib4_event drivers/net/netdevsim/fib.c:464 [inline]
 nsim_fib_event+0x3c4/0x6fc0 drivers/net/netdevsim/fib.c:884
 nsim_fib_event_work+0x1c8/0x308 drivers/net/netdevsim/fib.c:1493
 process_one_work+0x7e8/0x155c kernel/workqueue.c:3263
 process_scheduled_works kernel/workqueue.c:3346 [inline]
 worker_thread+0x990/0xed8 kernel/workqueue.c:3427
 kthread+0x5fc/0x75c kernel/kthread.c:463
 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:844
pool 6:
task:kworker/1:1     state:R  running task     stack:0     pid:26    tgid:26    ppid:2      task_flags:0x4208060 flags:0x00000010
Workqueue: events nsim_fib_event_work
Call trace:
 __switch_to+0x418/0x87c arch/arm64/kernel/process.c:741 (T)
 context_switch kernel/sched/core.c:5325 [inline]
 __schedule+0x13b0/0x2864 kernel/sched/core.c:6929
 preempt_schedule_irq+0x80/0x188 kernel/sched/core.c:7256
 raw_irqentry_exit_cond_resched+0x30/0x44 kernel/entry/common.c:168
 irqentry_exit+0xd4/0x108 kernel/entry/common.c:211
 exit_to_kernel_mode+0x10/0x1c arch/arm64/kernel/entry-common.c:58
 __el1_irq arch/arm64/kernel/entry-common.c:500 [inline]
 el1_interrupt+0x4c/0x60 arch/arm64/kernel/entry-common.c:510
 el1h_64_irq_handler+0x18/0x24 arch/arm64/kernel/entry-common.c:515
 el1h_64_irq+0x6c/0x70 arch/arm64/kernel/entry.S:592
 __try_to_del_timer_sync+0x0/0x358 kernel/time/timer.c:1489 (P)
 timer_delete_sync+0x24/0x34 kernel/time/timer.c:1675
 schedule_timeout+0x130/0x270 kernel/time/sleep_timeout.c:100
 schedule_timeout_uninterruptible+0x7c/0xc0 kernel/time/sleep_timeout.c:158
 msleep+0x3c/0x68 kernel/time/sleep_timeout.c:318
 nsim_fib6_rt_add drivers/net/netdevsim/fib.c:693 [inline]
 nsim_fib6_rt_insert drivers/net/netdevsim/fib.c:759 [inline]
 nsim_fib6_event drivers/net/netdevsim/fib.c:856 [inline]
 nsim_fib_event+0x6264/0x6fc0 drivers/net/netdevsim/fib.c:889
 nsim_fib_event_work+0x1c8/0x308 drivers/net/netdevsim/fib.c:1493
 process_one_work+0x7e8/0x155c kernel/workqueue.c:3263
 process_scheduled_works kernel/workqueue.c:3346 [inline]
 worker_thread+0x990/0xed8 kernel/workqueue.c:3427
 kthread+0x5fc/0x75c kernel/kthread.c:463
 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:844
pool 6:
task:kworker/1:4     state:R  running task     stack:0     pid:6675  tgid:6675  ppid:2      task_flags:0x4208060 flags:0x00000010
Workqueue: events nsim_fib_event_work
Call trace:
 __switch_to+0x418/0x87c arch/arm64/kernel/process.c:741 (T)
 context_switch kernel/sched/core.c:5325 [inline]
 __schedule+0x13b0/0x2864 kernel/sched/core.c:6929
 preempt_schedule_irq+0x80/0x188 kernel/sched/core.c:7256
 raw_irqentry_exit_cond_resched+0x30/0x44 kernel/entry/common.c:168
 irqentry_exit+0xd4/0x108 kernel/entry/common.c:211
 exit_to_kernel_mode+0x10/0x1c arch/arm64/kernel/entry-common.c:58
 __el1_irq arch/arm64/kernel/entry-common.c:500 [inline]
 el1_interrupt+0x4c/0x60 arch/arm64/kernel/entry-common.c:510
 el1h_64_irq_handler+0x18/0x24 arch/arm64/kernel/entry-common.c:515
 el1h_64_irq+0x6c/0x70 arch/arm64/kernel/entry.S:592
 __daif_local_irq_restore arch/arm64/include/asm/irqflags.h:175 [inline] (P)
 arch_local_irq_restore arch/arm64/include/asm/irqflags.h:195 [inline] (P)
 rht_unlock+0xfc/0x154 include/linux/rhashtable.h:355 (P)
 __rhashtable_insert_fast include/linux/rhashtable.h:736 [inline]
 rhashtable_insert_fast include/linux/rhashtable.h:834 [inline]
 nsim_fib6_rt_add drivers/net/netdevsim/fib.c:686 [inline]
 nsim_fib6_rt_insert drivers/net/netdevsim/fib.c:759 [inline]
 nsim_fib6_event drivers/net/netdevsim/fib.c:856 [inline]
 nsim_fib_event+0x56ac/0x6fc0 drivers/net/netdevsim/fib.c:889
 nsim_fib_event_work+0x1c8/0x308 drivers/net/netdevsim/fib.c:1493
 process_one_work+0x7e8/0x155c kernel/workqueue.c:3263
 process_scheduled_works kernel/workqueue.c:3346 [inline]
 worker_thread+0x990/0xed8 kernel/workqueue.c:3427
 kthread+0x5fc/0x75c kernel/kthread.c:463
 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:844
pool 6:
task:kworker/1:0     state:R  running task     stack:0     pid:24    tgid:24    ppid:2      task_flags:0x4208060 flags:0x00000010
Workqueue: events rht_deferred_worker
Call trace:
 __switch_to+0x418/0x87c arch/arm64/kernel/process.c:741 (T)
 context_switch kernel/sched/core.c:5325 [inline]
 __schedule+0x13b0/0x2864 kernel/sched/core.c:6929
 preempt_schedule_irq+0x80/0x188 kernel/sched/core.c:7256
 raw_irqentry_exit_cond_resched+0x30/0x44 kernel/entry/common.c:168
 irqentry_exit+0xd4/0x108 kernel/entry/common.c:211
 exit_to_kernel_mode+0x10/0x1c arch/arm64/kernel/entry-common.c:58
 __el1_irq arch/arm64/kernel/entry-common.c:500 [inline]
 el1_interrupt+0x4c/0x60 arch/arm64/kernel/entry-common.c:510
 el1h_64_irq_handler+0x18/0x24 arch/arm64/kernel/entry-common.c:515
 el1h_64_irq+0x6c/0x70 arch/arm64/kernel/entry.S:592
 debug_lockdep_rcu_enabled+0x0/0x38 kernel/locking/lockdep.c:5329 (P)
 rhashtable_rehash_table lib/rhashtable.c:335 [inline]
 rht_deferred_worker+0x135c/0x1c08 lib/rhashtable.c:436
 process_one_work+0x7e8/0x155c kernel/workqueue.c:3263
 process_scheduled_works kernel/workqueue.c:3346 [inline]
 worker_thread+0x958/0xed8 kernel/workqueue.c:3427
 kthread+0x5fc/0x75c kernel/kthread.c:463
 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:844