======================================================== WARNING: possible irq lock inversion dependency detected 6.4.0-rc7-next-20230623 #1 Not tainted -------------------------------------------------------- systemd-udevd/5589 just changed the state of lock: ffffffff85815818 (blkg_stat_lock){+.-.}-{2:2}, at: __blkcg_rstat_flush.isra.0+0x11f/0x4e0 but this lock was taken by another, HARDIRQ-safe lock in the past: (per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)){-.-.}-{2:2} and interrupts could create inverse lock ordering between them. other info that might help us debug this: Possible interrupt unsafe locking scenario: CPU0 CPU1 ---- ---- lock(blkg_stat_lock); local_irq_disable(); lock(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)); lock(blkg_stat_lock); lock(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)); *** DEADLOCK *** 2 locks held by systemd-udevd/5589: #0: ffffffff8560b5a0 (rcu_callback){....}-{0:0}, at: rcu_core+0x83e/0x28b0 #1: ffffffff8560b6c0 (rcu_read_lock){....}-{1:2}, at: __blkcg_rstat_flush.isra.0+0x93/0x4e0 the shortest dependencies between 2nd lock and 1st lock: -> (per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)){-.-.}-{2:2} { IN-HARDIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_updated+0xcb/0x2e0 __cgroup_account_cputime_field+0xa0/0x120 account_system_index_time+0x199/0x2c0 update_process_times+0x26/0x150 tick_sched_handle+0x8e/0x170 tick_sched_timer+0xe6/0x110 __hrtimer_run_queues+0x17f/0xb60 hrtimer_interrupt+0x2ef/0x750 __sysvec_apic_timer_interrupt+0xff/0x380 sysvec_apic_timer_interrupt+0x69/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 mem_cgroup_move_task+0xc/0x170 cgroup_procs_write_finish+0xeb/0x170 __cgroup_procs_write+0x371/0x6b0 cgroup_procs_write+0x26/0x60 cgroup_file_write+0x1e5/0x7c0 kernfs_fop_write_iter+0x3f7/0x610 vfs_write+0x97d/0xda0 ksys_write+0x122/0x250 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 IN-SOFTIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_updated+0xcb/0x2e0 __mod_memcg_state+0x86/0x160 memcg_account_kmem+0x38/0x70 refill_obj_stock+0x35f/0x5c0 kmem_cache_free+0x444/0x4d0 file_free_rcu+0xb5/0x100 rcu_core+0x8c8/0x28b0 __do_softirq+0x1b7/0x7d4 run_ksoftirqd+0x29/0x50 smpboot_thread_fn+0x40b/0x930 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 INITIAL USE at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_flush_locked+0x131/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 flush_memcg_stats_dwork+0x9/0x50 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 } ... key at: [] __key.0+0x0/0x40 ... acquired at: _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 blkcg_rstat_flush+0x87/0xb0 cgroup_rstat_flush_locked+0x706/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 mem_cgroup_wb_stats+0x3d8/0x4b0 wb_over_bg_thresh+0x30e/0x6a0 wb_workfn+0x486/0xed0 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 -> (blkg_stat_lock){+.-.}-{2:2} { HARDIRQ-ON-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x28b0 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 __sanitizer_cov_trace_pc+0x3b/0x70 ___bpf_prog_run+0x2b6c/0x78c0 __bpf_prog_run32+0x9d/0xe0 __seccomp_filter+0x23f/0x1080 __secure_computing+0x195/0x300 syscall_trace_enter.constprop.0+0x94/0x1d0 do_syscall_64+0x1a/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 IN-SOFTIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x28b0 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 __sanitizer_cov_trace_pc+0x3b/0x70 ___bpf_prog_run+0x2b6c/0x78c0 __bpf_prog_run32+0x9d/0xe0 __seccomp_filter+0x23f/0x1080 __secure_computing+0x195/0x300 syscall_trace_enter.constprop.0+0x94/0x1d0 do_syscall_64+0x1a/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 INITIAL USE at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 blkcg_rstat_flush+0x87/0xb0 cgroup_rstat_flush_locked+0x706/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 mem_cgroup_wb_stats+0x3d8/0x4b0 wb_over_bg_thresh+0x30e/0x6a0 wb_workfn+0x486/0xed0 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 } ... key at: [] blkg_stat_lock+0x18/0x60 ... acquired at: __lock_acquire+0x8b8/0x6340 lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x28b0 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 __sanitizer_cov_trace_pc+0x3b/0x70 ___bpf_prog_run+0x2b6c/0x78c0 __bpf_prog_run32+0x9d/0xe0 __seccomp_filter+0x23f/0x1080 __secure_computing+0x195/0x300 syscall_trace_enter.constprop.0+0x94/0x1d0 do_syscall_64+0x1a/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 stack backtrace: CPU: 1 PID: 5589 Comm: systemd-udevd Not tainted 6.4.0-rc7-next-20230623 #1 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-1 04/01/2014 Call Trace: dump_stack_lvl+0x91/0xf0 print_irq_inversion_bug.part.0+0x3d5/0x570 mark_lock.part.0+0x900/0x2f50 __lock_acquire+0x8b8/0x6340 lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x28b0 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 RIP: 0010:__sanitizer_cov_trace_pc+0x3b/0x70 Code: 81 e1 00 01 00 00 65 48 8b 14 25 c0 8a 03 00 a9 00 01 ff 00 74 0e 85 c9 74 35 8b 82 9c 14 00 00 85 c0 74 2b 8b 82 78 14 00 00 <83> f8 02 75 20 48 8b 8a 80 14 00 00 8b 92 7c 14 00 00 48 8b 01 48 RSP: 0018:ffff88804467fc58 EFLAGS: 00000246 RAX: 0000000000000000 RBX: dffffc0000000000 RCX: 0000000000000000 RDX: ffff888016a33680 RSI: ffffffff815ac43c RDI: 0000000000000006 RBP: ffff88804467fd28 R08: 0000000000000006 R09: 00000000c000003e R10: 0000000040000003 R11: 0000000000000000 R12: 0000000040000003 R13: ffff88804467fd60 R14: 00000000c000003e R15: ffffc900006a7068 ___bpf_prog_run+0x2b6c/0x78c0 __bpf_prog_run32+0x9d/0xe0 __seccomp_filter+0x23f/0x1080 __secure_computing+0x195/0x300 syscall_trace_enter.constprop.0+0x94/0x1d0 do_syscall_64+0x1a/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 RIP: 0033:0x7fd07a099be7 Code: 25 00 00 41 00 3d 00 00 41 00 74 47 64 8b 04 25 18 00 00 00 85 c0 75 6b 44 89 e2 48 89 ee bf 9c ff ff ff b8 01 01 00 00 0f 05 <48> 3d 00 f0 ff ff 0f 87 95 00 00 00 48 8b 4c 24 28 64 48 2b 0c 25 RSP: 002b:00007ffc27f94100 EFLAGS: 00000246 ORIG_RAX: 0000000000000101 RAX: ffffffffffffffda RBX: 000055c237d86c00 RCX: 00007fd07a099be7 RDX: 0000000000080000 RSI: 00007ffc27f94340 RDI: 00000000ffffff9c RBP: 00007ffc27f94340 R08: 0000000000000008 R09: 0000000000000001 R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000080000 R13: 000055c237d86c00 R14: 0000000000000001 R15: 000055c237d874f0 ---------------- Code disassembly (best guess): 0: 81 e1 00 01 00 00 and $0x100,%ecx 6: 65 48 8b 14 25 c0 8a mov %gs:0x38ac0,%rdx d: 03 00 f: a9 00 01 ff 00 test $0xff0100,%eax 14: 74 0e je 0x24 16: 85 c9 test %ecx,%ecx 18: 74 35 je 0x4f 1a: 8b 82 9c 14 00 00 mov 0x149c(%rdx),%eax 20: 85 c0 test %eax,%eax 22: 74 2b je 0x4f 24: 8b 82 78 14 00 00 mov 0x1478(%rdx),%eax * 2a: 83 f8 02 cmp $0x2,%eax <-- trapping instruction 2d: 75 20 jne 0x4f 2f: 48 8b 8a 80 14 00 00 mov 0x1480(%rdx),%rcx 36: 8b 92 7c 14 00 00 mov 0x147c(%rdx),%edx 3c: 48 8b 01 mov (%rcx),%rax 3f: 48 rex.W