======================================================== WARNING: possible irq lock inversion dependency detected 6.4.0-rc7-next-20230622 #1 Not tainted -------------------------------------------------------- kworker/0:1/9 just changed the state of lock: ffffffff85814c18 (blkg_stat_lock){+.-.}-{2:2}, at: __blkcg_rstat_flush.isra.0+0x11f/0x4e0 but this lock was taken by another, HARDIRQ-safe lock in the past: (per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)){-.-.}-{2:2} and interrupts could create inverse lock ordering between them. other info that might help us debug this: Possible interrupt unsafe locking scenario: CPU0 CPU1 ---- ---- lock(blkg_stat_lock); local_irq_disable(); lock(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)); lock(blkg_stat_lock); lock(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)); *** DEADLOCK *** 5 locks held by kworker/0:1/9: #0: ffff888008468d38 ((wq_completion)events){+.+.}-{0:0}, at: process_one_work+0x99d/0x1770 #1: ffff888008d77db0 ((work_completion)(&rdev->wiphy_work)){+.+.}-{0:0}, at: process_one_work+0x9d0/0x1770 #2: ffff888041fa0768 (&rdev->wiphy.mtx){+.+.}-{3:3}, at: cfg80211_wiphy_work+0x2c/0x290 #3: ffffffff8560b5a0 (rcu_callback){....}-{0:0}, at: rcu_core+0x83e/0x2860 #4: ffffffff8560b6c0 (rcu_read_lock){....}-{1:2}, at: __blkcg_rstat_flush.isra.0+0x93/0x4e0 the shortest dependencies between 2nd lock and 1st lock: -> (per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)){-.-.}-{2:2} { IN-HARDIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_updated+0xcb/0x2e0 __cgroup_account_cputime_field+0xa0/0x120 account_system_index_time+0x199/0x2c0 update_process_times+0x26/0x150 tick_sched_handle+0x8e/0x170 tick_sched_timer+0xe6/0x110 __hrtimer_run_queues+0x17f/0xb60 hrtimer_interrupt+0x2ef/0x750 __sysvec_apic_timer_interrupt+0xff/0x380 sysvec_apic_timer_interrupt+0x69/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 unwind_next_frame+0x289/0x2490 arch_stack_walk+0x87/0xf0 stack_trace_save+0x90/0xd0 kasan_save_stack+0x22/0x50 kasan_set_track+0x25/0x30 __kasan_slab_alloc+0x59/0x70 kmem_cache_alloc+0x16b/0x370 seq_open+0x5b/0x180 kernfs_fop_open+0x679/0xe90 do_dentry_open+0x69b/0x15c0 path_openat+0x1859/0x2710 do_filp_open+0x1ba/0x410 do_sys_openat2+0x164/0x1d0 __x64_sys_openat+0x143/0x200 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 IN-SOFTIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_updated+0xcb/0x2e0 __mod_memcg_lruvec_state+0x18d/0x2b0 mod_objcg_mlstate+0x119/0x230 drain_obj_stock+0x23e/0x580 refill_obj_stock+0x12d/0x5c0 kmem_cache_free+0x444/0x4d0 file_free_rcu+0xb5/0x100 rcu_core+0x8c8/0x2860 __do_softirq+0x1b7/0x7d4 run_ksoftirqd+0x29/0x50 smpboot_thread_fn+0x40b/0x930 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 INITIAL USE at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_flush_locked+0x131/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 flush_memcg_stats_dwork+0x9/0x50 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 } ... key at: [] __key.0+0x0/0x40 ... acquired at: _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 blkcg_rstat_flush+0x87/0xb0 cgroup_rstat_flush_locked+0x706/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 mem_cgroup_wb_stats+0x3d8/0x4b0 balance_dirty_pages+0x339/0x2640 balance_dirty_pages_ratelimited_flags+0xbce/0x1130 fault_dirty_shared_page+0x2b3/0x500 do_wp_page+0x3c8/0x3b00 __handle_mm_fault+0xcbc/0x2bf0 handle_mm_fault+0x1af/0xba0 do_user_addr_fault+0x58e/0x12f0 exc_page_fault+0x9c/0x1a0 asm_exc_page_fault+0x26/0x30 -> (blkg_stat_lock){+.-.}-{2:2} { HARDIRQ-ON-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x2860 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 kasan_quarantine_put+0x83/0x1d0 kmem_cache_free+0x10d/0x4d0 skb_release_data.isra.0+0x708/0x840 kfree_skb_reason+0x13a/0x470 ieee80211_iface_work+0x353/0xcb0 cfg80211_wiphy_work+0x1f7/0x290 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 IN-SOFTIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x2860 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 kasan_quarantine_put+0x83/0x1d0 kmem_cache_free+0x10d/0x4d0 skb_release_data.isra.0+0x708/0x840 kfree_skb_reason+0x13a/0x470 ieee80211_iface_work+0x353/0xcb0 cfg80211_wiphy_work+0x1f7/0x290 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 INITIAL USE at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 blkcg_rstat_flush+0x87/0xb0 cgroup_rstat_flush_locked+0x706/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 mem_cgroup_wb_stats+0x3d8/0x4b0 balance_dirty_pages+0x339/0x2640 balance_dirty_pages_ratelimited_flags+0xbce/0x1130 fault_dirty_shared_page+0x2b3/0x500 do_wp_page+0x3c8/0x3b00 __handle_mm_fault+0xcbc/0x2bf0 handle_mm_fault+0x1af/0xba0 do_user_addr_fault+0x58e/0x12f0 exc_page_fault+0x9c/0x1a0 asm_exc_page_fault+0x26/0x30 } ... key at: [] blkg_stat_lock+0x18/0x60 ... acquired at: __lock_acquire+0x8b8/0x6340 lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x2860 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 kasan_quarantine_put+0x83/0x1d0 kmem_cache_free+0x10d/0x4d0 skb_release_data.isra.0+0x708/0x840 kfree_skb_reason+0x13a/0x470 ieee80211_iface_work+0x353/0xcb0 cfg80211_wiphy_work+0x1f7/0x290 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 stack backtrace: CPU: 0 PID: 9 Comm: kworker/0:1 Not tainted 6.4.0-rc7-next-20230622 #1 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-1 04/01/2014 Workqueue: events cfg80211_wiphy_work Call Trace: dump_stack_lvl+0x91/0xf0 print_irq_inversion_bug.part.0+0x3d5/0x570 mark_lock.part.0+0x900/0x2f50 __lock_acquire+0x8b8/0x6340 lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x2860 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 RIP: 0010:kasan_quarantine_put+0x83/0x1d0 Code: 00 00 00 00 48 03 50 10 48 89 50 10 48 89 d3 48 81 fa 00 00 10 00 77 59 41 bd 01 00 00 00 4d 85 e4 74 39 e8 cf 63 d2 ff fb 5b <44> 89 e8 5d 41 5c 41 5d 41 5e e9 8e a4 d2 02 e8 c9 64 d2 ff 48 c7 RSP: 0018:ffff888008d77b68 EFLAGS: 00000202 RAX: 000000000014ecf1 RBX: ffff88803e1bfc00 RCX: ffffffff812cfb6f RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffffffff81810d21 RBP: ffff888008faf280 R08: 0000000000000001 R09: fffffbfff0ef8169 R10: ffffffff877c0b4f R11: 0000000000000001 R12: 0000000000000200 R13: 0000000000000001 R14: ffffffff83677638 R15: ffff88801c6f54c8 kmem_cache_free+0x10d/0x4d0 skb_release_data.isra.0+0x708/0x840 kfree_skb_reason+0x13a/0x470 ieee80211_iface_work+0x353/0xcb0 cfg80211_wiphy_work+0x1f7/0x290 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 loop6: detected capacity change from 0 to 732 loop3: detected capacity change from 0 to 40 ---------------- Code disassembly (best guess): 0: 00 00 add %al,(%rax) 2: 00 00 add %al,(%rax) 4: 48 03 50 10 add 0x10(%rax),%rdx 8: 48 89 50 10 mov %rdx,0x10(%rax) c: 48 89 d3 mov %rdx,%rbx f: 48 81 fa 00 00 10 00 cmp $0x100000,%rdx 16: 77 59 ja 0x71 18: 41 bd 01 00 00 00 mov $0x1,%r13d 1e: 4d 85 e4 test %r12,%r12 21: 74 39 je 0x5c 23: e8 cf 63 d2 ff callq 0xffd263f7 28: fb sti 29: 5b pop %rbx * 2a: 44 89 e8 mov %r13d,%eax <-- trapping instruction 2d: 5d pop %rbp 2e: 41 5c pop %r12 30: 41 5d pop %r13 32: 41 5e pop %r14 34: e9 8e a4 d2 02 jmpq 0x2d2a4c7 39: e8 c9 64 d2 ff callq 0xffd26507 3e: 48 rex.W 3f: c7 .byte 0xc7