panic: uma_zalloc_debug: called within spinlock or critical section

panic: uma_zalloc_debug: called within spinlock or critical section
Status: fixed on 2020/11/28 12:59
Reported-by: syzbot+6743fa07b9b7528dc561@syzkaller.appspotmail.com
Fix commit: e07e3fa3c95c kern: cpuset: drop the lock to allocate domainsets
First crash: 541d, last: 537d

Sample crash report:
login: panic: uma_zalloc_debug: called within spinlock or critical section
cpuid = 1
time = 1606150379
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x47/frame 0xfffffe0025568830
vpanic() at vpanic+0x1c7/frame 0xfffffe0025568890
panic() at panic+0x43/frame 0xfffffe00255688f0
uma_zalloc_arg() at uma_zalloc_arg+0x2df/frame 0xfffffe0025568950
cpuset_modify_domain() at cpuset_modify_domain+0x1d7/frame 0xfffffe00255689e0
kern_cpuset_setdomain() at kern_cpuset_setdomain+0x6e3/frame 0xfffffe0025568a80
sys_cpuset_setdomain() at sys_cpuset_setdomain+0x35/frame 0xfffffe0025568ab0
amd64_syscall() at amd64_syscall+0x247/frame 0xfffffe0025568bf0
fast_syscall_common() at fast_syscall_common+0xf8/frame 0xfffffe0025568bf0
--- syscall (0, FreeBSD ELF64, nosys), rip = 0x2b3d8a, rsp = 0x7fffffffe9f8, rbp = 0x7fffffffea10 ---
KDB: enter: panic
[ thread pid 769 tid 100100 ]
Stopped at      kdb_enter+0x67: movq    $0,0x1488216(%rip)
db> 
db> set $lines = 0
db> set $maxwidth = 0
db> show registers
cs                        0x20
ds                        0x3b
es                        0x3b
fs                        0x13
gs                        0x1b
ss                        0x28
rax                       0x12
rcx                       0x80
rdx         0xffffffff818bcba0
rbx                          0
rsp         0xfffffe0025568810
rbp         0xfffffe0025568830
rsi                        0x1
rdi                          0
r8                           0
r9                  0xffffffff
r10                          0
r11         0xfffffe0023bfa810
r12         0xffffffff820671c0  ddb_dbbe
r13                          0
r14         0xffffffff819643fa
r15         0xffffffff819643fa
rip         0xffffffff810d33e7  kdb_enter+0x67
rflags                    0x82
kdb_enter+0x67: movq    $0,0x1488216(%rip)
db> show proc
Process 769 (syz-executor3815473) at 0xfffff8001cf99000:
 state: NORMAL
 uid: 0  gids: 0, 0, 5
 parent: pid 767 at 0xfffff80004b7b528
 ABI: FreeBSD ELF64
 arguments: ./syz-executor381547389
 reaper: 0xfffff80004291528 reapsubtree: 1
 sigparent: 20
 vmspace: 0xfffffe0023b68000
   (map 0xfffffe0023b68000)
   (map.pmap 0xfffffe0023b680c0)
   (pmap 0xfffffe0023b68120)
 threads: 1
100100                   Run     CPU 1                       syz-executor3815473
db> ps
  pid  ppid  pgrp   uid  state   wmesg   wchan               cmd
  769   767   765     0  R       CPU 1                       syz-executor3815473
  767   765   765     0  S       nanslp  0xffffffff8252ed90  syz-executor3815473
  765   763   765     0  Ss      pause   0xfffff80004c160b0  csh
  763   682   763     0  Ss      select  0xfffff8002511dac0  sshd
  742     1   742     0  Ss+     ttyin   0xfffff80004783cb0  getty
  741     1   741     0  Ss+     ttyin   0xfffff80004a688b0  getty
  740     1   740     0  Ss+     ttyin   0xfffff80004a68cb0  getty
  739     1   739     0  Ss+     ttyin   0xfffff80004a600b0  getty
  738     1   738     0  Ss+     ttyin   0xfffff80004a604b0  getty
  737     1   737     0  Ss+     ttyin   0xfffff80004a608b0  getty
  736     1   736     0  Ss+     ttyin   0xfffff80004a60cb0  getty
  735     1   735     0  Ss+     ttyin   0xfffff800049fc0b0  getty
  734     1   734     0  Ss+     ttyin   0xfffff800049fc4b0  getty
  732     1    24     0  S+      piperd  0xfffff800250b9ba0  logger
  731   730    24     0  S+      nanslp  0xffffffff8252ed91  sleep
  730     1    24     0  S+      wait    0xfffff80004bc6000  sh
  686     1   686     0  Ss      nanslp  0xffffffff8252ed90  cron
  682     1   682     0  Ss      select  0xfffff8001cf7b6c0  sshd
  495     1   495     0  Ss      select  0xfffff8001cf7b740  syslogd
  424     1   424     0  Ss      select  0xfffff8001cf7bbc0  devd
  423     1   423    65  Ss      select  0xfffff8001cf7b940  dhclient
  338     1   338     0  Ss      select  0xfffff8001cf7b9c0  dhclient
  335     1   335     0  Ss      select  0xfffff8001cf723c0  dhclient
   23     0     0     0  DL      vlruwt  0xfffff80004a77000  [vnlru]
   22     0     0     0  DL      syncer  0xffffffff8261c138  [syncer]
   21     0     0     0  DL      (threaded)                  [bufdaemon]
100070                   D       qsleep  0xffffffff8261b200  [bufdaemon]
100077                   D       -       0xffffffff8200ac80  [bufspacedaemon-0]
100088                   D       sdflush 0xfffff800041818e8  [/ worker]
   20     0     0     0  DL      psleep  0xffffffff826426c8  [vmdaemon]
   19     0     0     0  DL      (threaded)                  [pagedaemon]
100068                   D       psleep  0xffffffff82636b38  [dom0]
100075                   D       launds  0xffffffff82636b44  [laundry: dom0]
100076                   D       umarcl  0xffffffff814def40  [uma]
   18     0     0     0  DL      -       0xffffffff82363278  [rand_harvestq]
   17     0     0     0  DL      waiting 0xffffffff82c14818  [sctp_iterator]
   16     0     0     0  DL      pftm    0xffffffff82e05390  [pf purge]
   15     0     0     0  DL      -       0xffffffff8261a7dc  [soaiod4]
    9     0     0     0  DL      -       0xffffffff8261a7dc  [soaiod3]
    8     0     0     0  DL      -       0xffffffff8261a7dc  [soaiod2]
    7     0     0     0  DL      -       0xffffffff8261a7dc  [soaiod1]
    6     0     0     0  DL      (threaded)                  [cam]
100034                   D       -       0xffffffff8223afc0  [doneq0]
100067                   D       -       0xffffffff8223ae90  [scanner]
    5     0     0     0  DL      crypto_ 0xfffff80004351d90  [crypto returns 1]
    4     0     0     0  DL      crypto_ 0xfffff80004351d30  [crypto returns 0]
    3     0     0     0  DL      crypto_ 0xffffffff82634030  [crypto]
   14     0     0     0  DL      seqstat 0xfffff8000432a888  [sequencer 00]
   13     0     0     0  DL      (threaded)                  [geom]
100025                   D       -       0xffffffff8250e620  [g_event]
100026                   D       -       0xffffffff8250e628  [g_up]
100027                   D       -       0xffffffff8250e630  [g_down]
    2     0     0     0  DL      (threaded)                  [KTLS]
100018                   D       -       0xfffff800042b3300  [thr_0]
100019                   D       -       0xfffff800042b3380  [thr_1]
   12     0     0     0  WL      (threaded)                  [intr]
100011                   I                                   [swi5: fast taskq]
100014                   I                                   [swi6: task queue]
100015                   I                                   [swi6: Giant taskq]
100020                   I                                   [swi1: netisr 0]
100021                   I                                   [swi4: clock (0)]
100022                   I                                   [swi4: clock (1)]
100023                   I                                   [swi3: vm]
100035                   I                                   [irq24: virtio_pci0]
100036                   I                                   [irq25: virtio_pci0]
100037                   I                                   [irq26: virtio_pci0]
100038                   I                                   [irq27: virtio_pci0]
100039                   I                                   [irq28: virtio_pci1]
100040                   I                                   [irq29: virtio_pci1]
100041                   I                                   [irq30: virtio_pci1]
100042                   I                                   [irq31: virtio_pci1]
100043                   I                                   [irq32: virtio_pci1]
100048                   I                                   [irq10: virtio_pci2]
100050                   I                                   [irq1: atkbd0]
100051                   I                                   [irq12: psm0]
100052                   I                                   [swi0: uart uart++]
100060                   I                                   [swi1: pf send]
100073                   I                                   [swi1: hpts]
100074                   I                                   [swi1: hpts]
   11     0     0     0  RL      (threaded)                  [idle]
100003                   Run     CPU 0                       [idle: cpu0]
100004                   CanRun                              [idle: cpu1]
    1     0     1     0  SLs     wait    0xfffff80004291528  [init]
   10     0     0     0  DL      audit_w 0xffffffff82634550  [audit]
    0     0     0     0  DLs     (threaded)                  [kernel]
100000                   D       swapin  0xffffffff8250ebb0  [swapper]
100005                   D       -       0xfffff800042b8e00  [if_io_tqg_0]
100006                   D       -       0xfffff800042b8d00  [if_io_tqg_1]
100007                   D       -       0xfffff800042b8c00  [if_config_tqg_0]
100008                   D       -       0xfffff800042b8b00  [softirq_0]
100009                   D       -       0xfffff800042b8a00  [softirq_1]
100010                   D       -       0xfffff80004183200  [in6m_free taskq]
100012                   D       -       0xfffff800042bcd00  [kqueue_ctx taskq]
100013                   D       -       0xfffff800042bcb00  [inm_free taskq]
100016                   D       -       0xfffff800042bc500  [aiod_kick taskq]
100017                   D       -       0xfffff800042bc300  [thread taskq]
100024                   D       -       0xfffff800042d5800  [firmware taskq]
100029                   D       -       0xfffff800042d5100  [crypto_0]
100030                   D       -       0xfffff800042d5100  [crypto_1]
100044                   D       -       0xfffff80004354100  [vtnet0 rxq 0]
100045                   D       -       0xfffff80004354000  [vtnet0 txq 0]
100046                   D       -       0xfffff8000476ae00  [vtnet0 rxq 1]
100047                   D       -       0xfffff8000476ad00  [vtnet0 txq 1]
100049                   D       vtbslp  0xfffff800044a1c00  [virtio_balloon]
100053                   D       -       0xfffff8000477ec00  [mca taskq]
100054                   D       -       0xffffffff81d0b211  [deadlkres]
100062                   D       -       0xfffff80004a54d00  [acpi_task_0]
100063                   D       -       0xfffff80004a54d00  [acpi_task_1]
100064                   D       -       0xfffff80004a54d00  [acpi_task_2]
100066                   D       -       0xfffff80004354900  [CAM taskq]
db> show all locks
db> show malloc
              Type        InUse        MemUse     Requests
           pf_hash            5        11524K            5
            devbuf         4214         4339K         4239
          tcp_hpts            5         3201K            5
             vtbuf           24         1968K           46
         sysctloid        29217         1704K        29281
              kobj          336         1344K          496
            newblk          523         1155K          584
          vfscache            3         1025K            3
               pcb           21          537K           75
          inodedep            8          515K           86
         ufs_quota            1          512K            1
          vfs_hash            1          512K            1
           callout            2          512K            2
              intr            4          472K            4
           subproc          105          210K          828
            acpica         1674          184K        52503
         vnet_data            1          168K            1
           tidhash            3          141K            3
           pagedep            7          130K           61
        tfo_ccache            1          128K            1
               sem            4          106K            4
            DEVFS1          101          101K          110
            linker          254           97K          282
               bus          979           79K         3035
          mtx_pool            2           72K            2
          syncache            1           68K            1
          acpitask            1           64K            1
       ddb_capture            1           64K            1
            module          507           64K          507
              temp           19           33K         1507
         hostcache            1           32K            1
               shm            1           32K            1
               msg            4           30K            4
            DEVFS3          120           30K          130
              umtx          234           30K          234
           kdtrace          149           29K         1595
        gtaskqueue           18           26K           18
              vmem            3           22K            4
            kbdmux            6           22K            6
        DEVFS_RULE           56           20K           56
               BPF           10           18K           10
         ufs_mount            5           17K            6
              proc            3           17K            3
               tty           16           16K           16
           ithread           99           16K           99
            bus-sc           30           14K         1416
            KTRACE          100           13K          100
            ifaddr           32           12K           32
              kenv           92           12K           92
      eventhandler          129           11K          129
              GEOM           60           10K          489
              rman           82           10K          423
              UART           12            9K           12
           devstat            4            9K            4
               rpc            2            8K            2
         bmsafemap            1            8K           44
             shmfd            1            8K            1
       pfs_vncache            1            8K            1
         pfs_nodes           20            8K           20
     audit_evclass          233            8K          291
           CAM DEV            3            6K          510
              cred           21            6K          233
            sglist            5            6K            5
         CAM queue            5            6K         1528
         taskqueue           48            6K           48
       ufs_dirhash           24            5K           24
            plimit           17            5K          322
             ifnet            3            5K            3
                vt           11            5K           11
               UMA          254            5K          254
           memdesc            1            4K            1
               MCA           32            4K           32
          filedesc            1            4K            1
             evdev            4            4K            4
          routetbl           15            4K           15
             hhook           13            4K           13
       ether_multi           40            4K           50
           lltable           11            4K           11
          pf_ifnet            5            3K            6
         in6_multi           25            3K           25
            kqueue           47            3K          772
           pwddesc           47            3K          770
          terminal           11            3K           11
           acpisem           22            3K           22
           session           20            3K           31
              pgrp           20            3K           31
           uidinfo            3            3K            8
        local_apic            1            2K            1
           io_apic            1            2K            1
         ipsec-saq            2            2K            2
             selfd           27            2K        10059
         proc-args           39            2K          468
           CAM XPT           22            2K          543
             lockf           15            2K           22
            Unitno           25            2K           37
       ipsecpolicy            2            2K            2
           acpidev           20            2K           20
               msi            9            2K            9
             clone            9            2K            9
           softdep            1            1K            1
            sahead            1            1K            1
          secasvar            1            1K            1
       vnodemarker            2            1K           10
      NFSD session            1            1K            1
        CAM periph            4            1K          271
            select            7            1K           29
             ipsec            3            1K            3
          indirdep            3            1K            3
             nhops            6            1K            6
         toponodes            6            1K            6
            isadev            6            1K            6
             mount           16            1K           89
          pci_link           10            1K           10
          sctp_ifa            5            1K            7
            ip6ndp            4            1K            5
 encap_export_host           12            1K           12
            crypto            3            1K            3
            diradd            4            1K           36
          in_multi            2            1K            4
              pfil            4            1K            4
           CAM SIM            2            1K            2
              cdev            2            1K            2
    chacha20random            1            1K            1
               osd            3            1K            9
          sctp_ifn            2            1K            7
            dirrem            1            1K           28
               mld            2            1K            2
              igmp            2            1K            2
            vnodes            1            1K            1
              ktls            1            1K            1
      NFSD lckfile            1            1K            1
     NFSD V4client            1            1K            1
             DEVFS            9            1K           10
            feeder            7            1K            7
       inpcbpolicy            6            1K          131
        loginclass            3            1K            7
            prison            6            1K            6
     CAM dev queue            2            1K            2
 CAM I/O Scheduler            1            1K            1
            apmdev            1            1K            1
          atkbddev            2            1K            2
          CAM path            4            1K         1034
          freework            1            1K           26
           tcpfunc            2            1K            2
          procdesc            1            1K            6
          pmchooks            1            1K            1
            DEVFSP            2            1K            2
            soname            4            1K         3051
          filecaps            4            1K           63
               tun            3            1K            3
          nexusdev            5            1K            5
          sctp_vrf            1            1K            1
           entropy            2            1K           35
              vnet            1            1K            1
               pmc            1            1K            1
          acpiintr            1            1K            1
              cpus            2            1K            2
    vnet_data_free            1            1K            1
           Per-cpu            1            1K            1
          p1003.1b            1            1K            1
          pf_table            0            0K            0
           pf_rule            0            0K            0
           pf_altq            0            0K            0
           pf_osfp            0            0K            0
           pf_temp            0            0K            0
        sctp_mcore            0            0K            0
        sctp_socko            0            0K            0
         sctp_iter            0            0K            3
         sctp_mvrf            0            0K            0
         sctp_timw            0            0K            0
         sctp_cpal            0            0K            0
         sctp_cmsg            0            0K            0
         sctp_stre            0            0K            0
         sctp_athi            0            0K            0
         sctp_athm            0            0K            0
         sctp_atky            0            0K            0
         sctp_atcl            0            0K            0
         sctp_a_it            0            0K            3
         sctp_aadr            0            0K            0
         sctp_stro            0            0K            0
         sctp_stri            0            0K            0
          sctp_map            0            0K            0
            nvlist            0            0K            0
          SCSI ENC            0            0K            0
           SCSI sa            0            0K            0
        madt_table            0            0K            2
          smartpqi            0            0K            0
         scsi_pass            0            0K            0
         ciss_data            0            0K            0
            pvscsi            0            0K            0
           ath_hal            0            0K            0
            athdev            0            0K            0
           ata_pci            0            0K            0
           ata_dma            0            0K            0
       ata_generic            0            0K            0
              iavf            0            0K            0
               ixl            0            0K            0
               amr            0            0K            0
           scsi_da            0            0K           69
            ata_da            0            0K            0
           scsi_ch            0            0K            0
           scsi_cd            0            0K            0
            USBdev            0            0K            0
        ice-resmgr            0            0K            0
         ice-osdep            0            0K            0
               ice            0            0K            0
             axgbe            0            0K            0
       fpukern_ctx            0            0K            0
               USB            0            0K            0
          xen_intr            0            0K            0
       AHCI driver            0            0K            0
               agp            0            0K            0
           xen_hvm            0            0K            0
         legacydrv            0            0K            0
            qpidrv            0            0K            0
      dmar_idpgtbl            0            0K            0
          dmar_dom            0            0K            0
          dmar_ctx            0            0K            0
           nvme_da            0            0K            0
           acpipwr            0            0K            0
              isci            0            0K            0
      iommu_dmamap            0            0K            0
            twsbuf            0            0K            0
     hyperv_socket            0            0K            0
           bxe_ilt            0            0K            0
            xenbus            0            0K            0
      twe_commands            0            0K            0
      twa_commands            0            0K            0
       tcp_log_dev            0            0K            0
     vm_fictitious            0            0K            0
      midi buffers            0            0K            0
             mixer            0            0K            0
              ac97            0            0K            0
             hdacc            0            0K            0
              hdac            0            0K            0
              hdaa            0            0K            0
         acpi_perf            0            0K            0
         acpicmbat            0            0K            0
       SIIS driver            0            0K            0
           UMAHash            0            0K            0
           CAM CCB            0            0K         2000
         vm_pgdata            0            0K            0
           jblocks            0            0K            0
          savedino            0            0K            8
          sentinel            0            0K            0
            jfsync            0            0K            0
            jtrunc            0            0K            0
             sbdep            0            0K            3
           jsegdep            0            0K            0
              jseg            0            0K            0
         jfreefrag            0            0K            0
          jfreeblk            0            0K            0
           jnewblk            0            0K            0
            jmvref            0            0K            0
           jremref            0            0K            0
           jaddref            0            0K            0
           freedep            0            0K            0
         newdirblk            0            0K            8
             mkdir            0            0K           16
          freefile            0            0K           25
          freeblks            0            0K           25
          freefrag            0            0K            7
        allocindir            0            0K            0
       allocdirect            0            0K            0
          ufs_trim            0            0K            0
           mactemp            0            0K            0
     audit_trigger            0            0K            0
 audit_pipe_presel            0            0K            0
     audit_pipeent            0            0K            0
        audit_pipe            0            0K            0
      audit_evname            0            0K            0
         audit_bsm            0            0K            0
      audit_gidset            0            0K            0
        audit_text            0            0K            0
        audit_path            0            0K            0
        audit_data            0            0K            0
        audit_cred            0            0K            0
             xform            0            0K            0
               NLM            0            0K            0
    ipsec-spdcache            0            0K            0
         ipsec-reg            0            0K            0
        ipsec-misc            0            0K            0
      ipsecrequest            0            0K            0
            ip6opt            0            0K            3
       ip6_msource            0            0K            0
      ip6_moptions            0            0K            0
       in6_mfilter            0            0K            0
             frag6            0            0K            0
            tcplog            0            0K            0
               LRO            0            0K            0
      newreno data            0            0K            0
        ip_msource            0            0K            0
       ip_moptions            0            0K            0
        in_mfilter            0            0K            0
              ipid            0            0K            0
         80211scan            0            0K            0
      80211ratectl            0            0K            0
        80211power            0            0K            0
       80211nodeie            0            0K            0
         80211node            0            0K            0
      80211mesh_gt            0            0K            0
      80211mesh_rt            0            0K            0
         80211perr            0            0K            0
         80211prep            0            0K            0
         80211preq            0            0K            0
          80211dfs            0            0K            0
       80211crypto            0            0K            0
          80211vap            0            0K            0
             iflib            0            0K            0
              vlan            0            0K            0
               gif            0            0K            0
           ifdescr            0            0K            0
              zlib            0            0K            0
           fadvise            0            0K            0
           VN POLL            0            0K            0
            statfs            0            0K          195
     namei_tracker            0            0K            0
       export_host            0            0K            0
        cl_savebuf            0            0K            3
               PUC            0            0K            0
          ppbusdev            0            0K            0
agtiapi_MemAlloc malloc            0            0K            0
    osti_cacheable            0            0K            0
          tempbuff            0            0K            0
          tempbuff            0            0K            0
ag_tgt_map_t malloc            0            0K            0
ag_slr_map_t malloc            0            0K            0
lDevFlags * malloc            0            0K            0
tiDeviceHandle_t * malloc            0            0K            0
ag_portal_data_t malloc            0            0K            0
ag_device_t malloc            0            0K            0
     STLock malloc            0            0K            0
          CCB List            0            0K            0
            sr_iov            0            0K            0
               OCS            0            0K            0
               OCS            0            0K            0
              nvme            0            0K            0
               nvd            0            0K            0
            netmap            0            0K            0
            mwldev            0            0K            0
            biobuf            0            0K            0
              aios            0            0K            0
               lio            0            0K            0
               acl            0            0K            0
        MVS driver            0            0K            0
          mbuf_tag            0            0K           27
              accf            0            0K            0
               pts            0            0K            0
               iov            0            0K        13033
          ioctlops            0            0K           84
           Witness            0            0K            0
             stack            0            0K            0
     CAM ccb queue            0            0K            0
          mrsasbuf            0            0K            0
          mpt_user            0            0K            0
              sbuf            0            0K          288
          mps_user            0            0K            0
          firmware            0            0K            0
        compressor            0            0K            0
            MPSSAS            0            0K            0
              SWAP            0            0K            0
               mps            0            0K            0
          mpr_user            0            0K            0
         sysctltmp            0            0K          586
            sysctl            0            0K            1
            MPRSAS            0            0K            0
              ekcd            0            0K            0
            dumper            0            0K            0
          sendfile            0            0K            0
              rctl            0            0K            0
               mpr            0            0K            0
            mfibuf            0            0K            0
        md_sectors            0            0K            0
           md_disk            0            0K            0
           malodev            0            0K            0
               LED            0            0K            0
             cache            0            0K            0
          kcovinfo            0            0K            0
      prison_racct            0            0K            0
       Fail Points            0            0K            0
             sigio            0            0K            1
filedesc_to_leader            0            0K            0
               pwd            0            0K            0
       tty console            0            0K            0
          ix_sriov            0            0K            0
        aacraidcam            0            0K            0
                ix            0            0K            0
            ipsbuf            0            0K            0
            iirbuf            0            0K            0
       aacraid_buf            0            0K            0
            aaccam            0            0K            0
        isofs_node            0            0K            0
       isofs_mount            0            0K            0
     tr_raid5_data            0            0K            0
    tr_raid1e_data            0            0K            0
     tr_raid1_data            0            0K            0
     tr_raid0_data            0            0K            0
    tr_concat_data            0            0K            0
       md_sii_data            0            0K            0
   md_promise_data            0            0K            0
    md_nvidia_data            0            0K            0
   md_jmicron_data            0            0K            0
     md_intel_data            0            0K            0
       md_ddf_data            0            0K            0
         raid_data            0            0K           72
     geom_flashmap            0            0K            0
         tmpfs dir            0            0K            0
        tmpfs name            0            0K            0
       tmpfs mount            0            0K            0
           NFS FHA            0            0K            0
         newnfsmnt            0            0K            0
  newnfsclient_req            0            0K            0
   NFSCL layrecall            0            0K            0
     NFSCL session            0            0K            0
     NFSCL sockreq            0            0K            0
     NFSCL devinfo            0            0K            0
     NFSCL flayout            0            0K            0
      NFSCL layout            0            0K            0
     NFSD rollback            0            0K            0
NFSCL diroffdiroff            0            0K            0
       NEWdirectio            0            0K            0
        NEWNFSnode            0            0K            0
         NFSCL lck            0            0K            0
      NFSCL lckown            0            0K            0
      NFSCL client            0            0K            0
       NFSCL deleg            0            0K            0
        NFSCL open            0            0K            0
       NFSCL owner            0            0K            0
            NFS fh            0            0K            0
           NFS req            0            0K            0
     NFSD usrgroup            0            0K            0
       NFSD string            0            0K            0
       NFSD V4lock            0            0K            0
      NFSD V4state            0            0K            0
     NFSD srvcache            0            0K            0
       msdosfs_fat            0            0K            0
     msdosfs_mount            0            0K            0
      msdosfs_node            0            0K            0
            DEVFS4            0            0K            0
            DEVFS2            0            0K            0
            gntdev            0            0K            0
       privcmd_dev            0            0K            0
        evtchn_dev            0            0K            0
          xenstore            0            0K            0
            aacbuf            0            0K            0
               xnb            0            0K            0
              xbbd            0            0K            0
               xbd            0            0K            0
           Balloon            0            0K            0
          sysmouse            0            0K            0
            vtfont            0            0K            0
              zstd            0            0K            0
db> show uma
              Zone   Size    Used    Free    Requests  Sleeps  Bucket  Total Mem    XFree
      mbuf_cluster   2048    9023       1        9023       0     254   18481152        0
       mbuf_packet    256    8192     703       25653       0     254    2277120        0
          BUF TRIE    144     169   13299         651       0      62    1939392        0
        malloc-384    384    4138      12        4140       0      30    1593600        0
       malloc-4096   4096     336       1         496       0       2    1380352        0
        malloc-128    128   10360      56       10655       0     126    1333248        0
              pbuf    832       0     969           0       0       2     806208        0
       UMA Slabs 0    112    5297      28        5297       0     126     596400        0
         FFS inode   1128     483      21         509       0       8     568512        0
        RADIX NODE    144    2059     150       19337       0      62     318096        0
             VNODE    488     512      32         540       0      30     265472        0
      malloc-65536  65536       4       0           4       0       1     262144        0
      malloc-65536  65536       3       1          11       0       1     262144        0
         VM OBJECT    264     894      51       12367       0      30     249480        0
        256 Bucket   2048     101      19        5972       0       8     245760        0
            DEVCTL   1024       0     208         109       0       0     212992        0
       malloc-4096   4096      50       2        1222       0       2     212992        0
            THREAD   1792     110       7         110       0       8     209664        0
         malloc-16     16   12218     282       12279       0     254     200000        0
         malloc-64     64    2751      84        2775       0     254     181440        0
         UMA Zones    768     226       3         226       0      16     175872        0
         malloc-32     32    4771     143        4944       0     254     157248        0
        malloc-128    128    1126      83       24452       0     126     154752        0
        malloc-256    256     555      15         738       0      62     145920        0
              mbuf    256     385     170        1972       0     254     142080        0
      malloc-65536  65536       0       2          45       0       1     131072        0
      malloc-65536  65536       0       2         110       0       1     131072        0
      malloc-65536  65536       2       0           2       0       1     131072        0
       FFS2 dinode    256     483      12         508       0      62     126720        0
       malloc-1024   1024     113       3         123       0      16     118784        0
          ksiginfo    112      44    1000          60       0     126     116928        0
         MAP ENTRY     96     847     329       36990       0     126     112896        0
       S VFS Cache    104     962      91        1006       0     126     109512        0
         vmem btag     56    1661      55        1661       0     254      96096        0
       malloc-4096   4096      23       0          34       0       2      94208        0
       malloc-2048   2048       1      41        2001       0       8      86016        0
          UMA Kegs    384     212       1         212       0      30      81792        0
             g_bio    408       0     190        5651       0      30      77520        0
           VMSPACE   2544      24       6         748       0       4      76320        0
              PROC   1320      46      11         769       0       8      75240        0
       malloc-8192   8192       6       3          34       0       1      73728        0
      malloc-32768  32768       2       0           2       0       1      65536        0
      malloc-32768  32768       0       2         112       0       1      65536        0
         filedesc0   1072      47       9         770       0       8      60032        0
        malloc-256    256     154      71        1187       0      62      57600        0
        128 Bucket   1024      35      20        1251       0      16      56320        0
      malloc-16384  16384       3       0           3       0       1      49152        0
      malloc-16384  16384       2       1           5       0       1      49152        0
       malloc-2048   2048       9      15         516       0       8      49152        0
         32 Bucket    256      49     131         802       0      62      46080        0
         malloc-64     64     544     149       11721       0     254      44352        0
         malloc-64     64     523     170        1604       0     254      44352        0
        malloc-128    128     305      36         434       0     126      43648        0
   mbuf_jumbo_page   4096       0      10          10       0     254      40960        0
            clpbuf    832       0      48         137       0      16      39936        0
        malloc-256    256     136      14         439       0      62      38400        0
           DIRHASH   1024      34       2          34       0      16      36864        0
             NAMEI   1024       0      36       11511       0      16      36864        0
        malloc-384    384      78      12         369       0      30      34560        0
      malloc-32768  32768       1       0           1       0       1      32768        0
      malloc-16384  16384       0       2         136       0       1      32768        0
           pcpu-64     64     452      60         452       0     254      32768        0
            pcpu-8      8    3859     237        3863       0     254      32768        0
         64 Bucket    512      47      17         538       0      30      32768        0
        malloc-256    256      22      83         682       0      62      26880        0
            socket    944      14      14        1187       0     254      26432        0
       malloc-8192   8192       2       1           4       0       1      24576        0
       malloc-8192   8192       2       1          91       0       1      24576        0
       malloc-8192   8192       3       0           3       0       1      24576        0
       malloc-4096   4096       4       2           7       0       2      24576        0
         malloc-64     64     249     129       12972       0     254      24192        0
            ttyinq    160     135      15         300       0      62      24000        0
        malloc-128    128     125      61         168       0     126      23808        0
        malloc-128    128     142      44         494       0     126      23808        0
             tcpcb   1048       3      19           7       0     254      23056        0
           ttyoutq    256      72      18         160       0      62      23040        0
        malloc-384    384       7      53          85       0      30      23040        0
        malloc-384    384      53       7          53       0      30      23040        0
              pipe    744       7      23         278       0      16      22320        0
       malloc-2048   2048       5       5         261       0       8      20480        0
       malloc-1024   1024      18       2          18       0      16      20480        0
         TURNSTILE    136     118      29         118       0      62      19992        0
       Mountpoints   2752       2       5           2       0       4      19264        0
        malloc-256    256      57      18         471       0      62      19200        0
      malloc-16384  16384       1       0           1       0       1      16384        0
      malloc-16384  16384       1       0           1       0       1      16384        0
      malloc-16384  16384       1       0           1       0       1      16384        0
       malloc-8192   8192       2       0           2       0       1      16384        0
       malloc-8192   8192       2       0           2       0       1      16384        0
       malloc-8192   8192       2       0           2       0       1      16384        0
       malloc-1024   1024       6      10         851       0      16      16384        0
       malloc-1024   1024      14       2          14       0      16      16384        0
         malloc-64     64     198      54        1702       0     254      16128        0
         malloc-32     32     277     227         404       0     254      16128        0
          4 Bucket     48       9     327        2957       0     254      16128        0
          8 Bucket     80      55     145        6903       0     126      16000        0
        malloc-128    128      95      29         771       0     126      15872        0
        malloc-256    256      52       8          53       0      62      15360        0
        malloc-256    256      36      24         422       0      62      15360        0
       malloc-4096   4096       3       0           4       0       2      12288        0
       malloc-4096   4096       1       2         199       0       2      12288        0
       malloc-2048   2048       3       3          15       0       8      12288        0
       malloc-1024   1024       2      10          42       0      16      12288        0
       malloc-1024   1024       4       8         141       0      16      12288        0
        malloc-512    512       1      23         119       0      30      12288        0
          2 Bucket     32      54     324       12734       0     254      12096        0
             Files     80      69      81        6302       0     126      12000        0
         udp_inpcb    488       2      22         120       0     254      11712        0
        SLEEPQUEUE     88     118      10         118       0     126      11264        0
       malloc-8192   8192       1       0           1       0       1       8192        0
       malloc-4096   4096       1       1           3       0       2       8192        0
       malloc-2048   2048       2       2           4       0       8       8192        0
       malloc-2048   2048       1       3          33       0       8       8192        0
       malloc-2048   2048       3       1           3       0       8       8192        0
       malloc-2048   2048       3       1         194       0       8       8192        0
       malloc-1024   1024       3       5          17       0      16       8192        0
        malloc-512    512       4      12           4       0      30       8192        0
        malloc-512    512       2      14         510       0      30       8192        0
        malloc-512    512       3      13          44       0      30       8192        0
        malloc-512    512       2      14          10       0      30       8192        0
        malloc-512    512       8       8           8       0      30       8192        0
           rtentry    176      14      32          18       0      62       8096        0
          rl_entry     40      33     169          33       0     254       8080        0
        sctp_laddr     48       0     168           4       0     254       8064        0
             udpcb     32       2     250         120       0     254       8064        0
               PWD     32      10     242          98       0     254       8064        0
         malloc-64     64      11     115          24       0     254       8064        0
         malloc-64     64       2     124          37       0     254       8064        0
         malloc-64     64      28      98          48       0     254       8064        0
         malloc-32     32       8     244          29       0     254       8064        0
         malloc-32     32     100     152         136       0     254       8064        0
         malloc-32     32      47     205        1891       0     254       8064        0
         malloc-32     32      57     195         806       0     254       8064        0
         malloc-32     32       1     251          20       0     254       8064        0
         malloc-32     32      30     222        2654       0     254       8064        0
         16 Bucket    144      32      24         195       0      62       8064        0
      vtnet_tx_hdr     24       0     334        1009       0     254       8016        0
             KNOTE    160       0      50           8       0      62       8000        0
         malloc-16     16      16     484          46       0     254       8000        0
         malloc-16     16      28     472         226       0     254       8000        0
         malloc-16     16      39     461         495       0     254       8000        0
         malloc-16     16     183     317        1238       0     254       8000        0
         malloc-16     16     252     248         254       0     254       8000        0
         malloc-16     16      27     473       23643       0     254       8000        0
        malloc-128    128      15      47          71       0     126       7936        0
        malloc-128    128      43      19        2946       0     126       7936        0
         tcp_inpcb    488       3      13           7       0     254       7808        0
              kenv    258       3      27         998       0      30       7740        0
     routing nhops    256      11      19          18       0      62       7680        0
             unpcb    256       7      23        1039       0     254       7680        0
        malloc-384    384       4      16           5       0      30       7680        0
        malloc-384    384       9      11           9       0      30       7680        0
        malloc-256    256      12      18          42       0      62       7680        0
     FPU_save_area    832       1       8           1       0      16       7488        0
            cpuset    104      28      34          33       0     126       6448        0
 epoch_record pcpu    256       4      12           4       0      62       4096        0
       malloc-1024   1024       3       1           3       0      16       4096        0
        malloc-512    512       1       7           2       0      30       4096        0
           pcpu-16     16       7     249           7       0     254       4096        0
         hostcache     96       1      41           1       0     254       4032        0
          syncache    168       0      24           5       0     254       4032        0
         malloc-16     16       8     242           8       0     254       4000        0
             ripcb    488       1       7           4       0     254       3904        0
       UMA Slabs 1    176       7      15           7       0      62       3872        0
        malloc-384    384       1       9           1       0      30       3840        0
        malloc-384    384       1       9           1       0      30       3840        0
        KMAP ENTRY     96      12      27          12       0       0       3744        0
              vmem   1856       1       1           1       0       8       3712        0
         domainset     40       1      62          28       0     254       2520        0
           SMR CPU     32       2      61           2       0     254       2016        0
        SMR SHARED     24       2      61           2       0     254       1512        0
       FFS1 dinode    128       0       0           0       0     126          0        0
             swblk    136       0       0           0       0      62          0        0
          swpctrie    144       0       0           0       0      62          0        0
   sctp_asconf_ack     48       0       0           0       0     254          0        0
       sctp_asconf     40       0       0           0       0     254          0        0
sctp_stream_msg_out    112       0       0           0       0     254          0        0
        sctp_readq    152       0       0           0       0     254          0        0
        sctp_chunk    152       0       0           0       0     254          0        0
        sctp_raddr    736       0       0           0       0     254          0        0
         sctp_asoc   2288       0       0           0       0     254          0        0
           sctp_ep   1280       0       0           0       0     254          0        0
   pf state scrubs     40       0       0           0       0     254          0        0
   pf frag entries     40       0       0           0       0     254          0        0
          pf frags    248       0       0           0       0      62          0        0
  pf table entries    160       0       0           0       0      62          0        0
pf table entry counters     64       0       0           0       0     254          0        0
   pf source nodes    136       0       0           0       0     254          0        0
     pf state keys     88       0       0           0       0     126          0        0
         pf states    296       0       0           0       0     254          0        0
           pf tags    104       0       0           0       0     126          0        0
          pf mtags     48       0       0           0       0     254          0        0
       tcp_bbr_pcb    832       0       0           0       0      16          0        0
       tcp_bbr_map    128       0       0           0       0     126          0        0
    IPsec SA lft_c     16       0       0           0       0     254          0        0
     udplite_inpcb    488       0       0           0       0     254          0        0
      tcp_log_node    120       0       0           0       0     126          0        0
    tcp_log_bucket    176       0       0           0       0      62          0        0
           tcp_log    416       0       0           0       0     254          0        0
          tcpreass     48       0       0           0       0     254          0        0
tfo_ccache_entries     80       0       0           0       0     126          0        0
               tfo      4       0       0           0       0     254          0        0
          sackhole     32       0       0           0       0     254          0        0
             tcptw     88       0       0           0       0     254          0        0
               ipq     56       0       0           0       0     254          0        0
            itimer    352       0       0           0       0      30          0        0
            AIOLIO    272       0       0           0       0      30          0        0
             AIOCB    744       0       0           0       0      16          0        0
              AIOP     32       0       0           0       0     254          0        0
               AIO    208       0       0           0       0      62          0        0
        TMPFS node    224       0       0           0       0      62          0        0
           NCLNODE    592       0       0           0       0      16          0        0
     LTS VFS Cache    360       0       0           0       0      30          0        0
       L VFS Cache    320       0       0           0       0      30          0        0
     STS VFS Cache    144       0       0           0       0      62          0        0
    crypto_session     72       0       0           0       0     126          0        0
           cryptop    280       0       0           0       0      30          0        0
   IOMMU_MAP_ENTRY    120       0       0           0       0     126          0        0
      ktls_session    192       0       0           0       0      62          0        0
    mbuf_jumbo_16k  16384       0       0           0       0     254          0        0
     mbuf_jumbo_9k   9216       0       0           0       0     254          0        0
      audit_record   1280       0       0           0       0       8          0        0
        MAC labels     40       0       0           0       0     254          0        0
            vnpbuf    832       0       0           0       0      62          0        0
            mdpbuf    832       0       0           0       0       4          0        0
           nfspbuf    832       0       0           0       0      16          0        0
            swwbuf    832       0       0           0       0       8          0        0
            swrbuf    832       0       0           0       0      16          0        0
          umtx_shm     88       0       0           0       0     126          0        0
           umtx pi     96       0       0           0       0     126          0        0
rangeset pctrie nodes    144       0       0           0       0      62          0        0
      malloc-65536  65536       0       0           0       0       1          0        0
      malloc-65536  65536       0       0           0       0       1          0        0
      malloc-65536  65536       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-32768  32768       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
      malloc-16384  16384       0       0           0       0       1          0        0
       malloc-4096   4096       0       0           0       0       2          0        0
        malloc-512    512       0       0           0       0      30          0        0
           pcpu-32     32       0       0           0       0     254          0        0
            pcpu-4      4       0       0           0       0     254          0        0
            fakepg    104       0       0           0       0     126          0        0
          UMA Hash    256       0       0           0       0      62          0        0
Crashes (9):
Manager	Time	Kernel	Commit	Syzkaller	Log	Report	Syz repro	C repro
ci-freebsd-main	2020/11/23 16:55	freebsd	14b48c5d2a17	878fb17a	log	report	syz	C
ci-freebsd-main	2020/11/27 13:07	freebsd	219284e64fa7	486f93ef	log	report
ci-freebsd-main	2020/11/26 10:04	freebsd	6db1a3e80986	2f1cec62	log	report
ci-freebsd-main	2020/11/25 19:48	freebsd	8848cda937bc	2f1cec62	log	report
ci-freebsd-main	2020/11/24 09:28	freebsd	03250e131186	bed14cd9	log	report
ci-freebsd-main	2020/11/24 09:27	freebsd	03250e131186	bed14cd9	log	report
ci-freebsd-main	2020/11/24 08:47	freebsd	03250e131186	bed14cd9	log	report
ci-freebsd-main	2020/11/24 05:19	freebsd	03250e131186	878fb17a	log	report
ci-freebsd-main	2020/11/23 16:29	freebsd	14b48c5d2a17	878fb17a	log	report