======================================================== WARNING: possible irq lock inversion dependency detected 6.4.0-rc7-next-20230622 #1 Not tainted -------------------------------------------------------- systemd-udevd/102 just changed the state of lock: ffffffff85814c18 (blkg_stat_lock){+.-.}-{2:2}, at: __blkcg_rstat_flush.isra.0+0x11f/0x4e0 but this lock was taken by another, HARDIRQ-safe lock in the past: (per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)){-.-.}-{2:2} and interrupts could create inverse lock ordering between them. other info that might help us debug this: Possible interrupt unsafe locking scenario: CPU0 CPU1 ---- ---- lock(blkg_stat_lock); local_irq_disable(); lock(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)); lock(blkg_stat_lock); lock(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)); *** DEADLOCK *** 3 locks held by systemd-udevd/102: #0: ffff8880389d0668 (&p->lock){+.+.}-{3:3}, at: seq_read_iter+0xe7/0x1300 #1: ffffffff8560b5a0 (rcu_callback){....}-{0:0}, at: rcu_core+0x83e/0x2860 #2: ffffffff8560b6c0 (rcu_read_lock){....}-{1:2}, at: __blkcg_rstat_flush.isra.0+0x93/0x4e0 the shortest dependencies between 2nd lock and 1st lock: -> (per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)){-.-.}-{2:2} { IN-HARDIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_updated+0xcb/0x2e0 __cgroup_account_cputime_field+0xa0/0x120 account_system_index_time+0x199/0x2c0 update_process_times+0x26/0x150 tick_sched_handle+0x8e/0x170 tick_sched_timer+0xe6/0x110 __hrtimer_run_queues+0x17f/0xb60 hrtimer_interrupt+0x2ef/0x750 __sysvec_apic_timer_interrupt+0xff/0x380 sysvec_apic_timer_interrupt+0x69/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 __sanitizer_cov_trace_pc+0xb/0x70 seq_read_iter+0xa9c/0x1300 kernfs_fop_read_iter+0x4d4/0x690 vfs_read+0x4b5/0x8f0 ksys_read+0x122/0x250 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 IN-SOFTIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_updated+0xcb/0x2e0 __cgroup_account_cputime+0x75/0xc0 update_curr+0x350/0x6d0 dequeue_task_fair+0x20e/0x14a0 load_balance+0xcb4/0x2790 rebalance_domains+0x66c/0xc00 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 unwind_next_frame+0x48b/0x2490 arch_stack_walk+0x87/0xf0 stack_trace_save+0x90/0xd0 kasan_save_stack+0x22/0x50 kasan_set_track+0x25/0x30 __kasan_slab_alloc+0x59/0x70 kmem_cache_alloc+0x16b/0x370 __create_object+0x3c/0xc90 kmem_cache_alloc+0x20b/0x370 getname_flags.part.0+0x50/0x4f0 getname+0x92/0xe0 do_sys_openat2+0xec/0x1d0 __x64_sys_openat+0x143/0x200 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 INITIAL USE at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_flush_locked+0x131/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 flush_memcg_stats_dwork+0x9/0x50 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 } ... key at: [] __key.0+0x0/0x40 ... acquired at: _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 blkcg_rstat_flush+0x87/0xb0 cgroup_rstat_flush_locked+0x706/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 mem_cgroup_wb_stats+0x3d8/0x4b0 wb_over_bg_thresh+0x30e/0x6a0 wb_workfn+0x486/0xed0 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 -> (blkg_stat_lock){+.-.}-{2:2} { HARDIRQ-ON-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x2860 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 try_charge_memcg+0x275/0x12f0 obj_cgroup_charge+0x1d0/0x530 __kmem_cache_alloc_node+0xa3/0x320 __kmalloc_node+0x4d/0x160 kvmalloc_node+0xaa/0x200 seq_read_iter+0x820/0x1300 kernfs_fop_read_iter+0x4d4/0x690 vfs_read+0x4b5/0x8f0 ksys_read+0x122/0x250 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 IN-SOFTIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x2860 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 try_charge_memcg+0x275/0x12f0 obj_cgroup_charge+0x1d0/0x530 __kmem_cache_alloc_node+0xa3/0x320 __kmalloc_node+0x4d/0x160 kvmalloc_node+0xaa/0x200 seq_read_iter+0x820/0x1300 kernfs_fop_read_iter+0x4d4/0x690 vfs_read+0x4b5/0x8f0 ksys_read+0x122/0x250 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 INITIAL USE at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 blkcg_rstat_flush+0x87/0xb0 cgroup_rstat_flush_locked+0x706/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 mem_cgroup_wb_stats+0x3d8/0x4b0 wb_over_bg_thresh+0x30e/0x6a0 wb_workfn+0x486/0xed0 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 } ... key at: [] blkg_stat_lock+0x18/0x60 ... acquired at: __lock_acquire+0x8b8/0x6340 lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x2860 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 try_charge_memcg+0x275/0x12f0 obj_cgroup_charge+0x1d0/0x530 __kmem_cache_alloc_node+0xa3/0x320 __kmalloc_node+0x4d/0x160 kvmalloc_node+0xaa/0x200 seq_read_iter+0x820/0x1300 kernfs_fop_read_iter+0x4d4/0x690 vfs_read+0x4b5/0x8f0 ksys_read+0x122/0x250 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 stack backtrace: CPU: 1 PID: 102 Comm: systemd-udevd Not tainted 6.4.0-rc7-next-20230622 #1 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-1 04/01/2014 Call Trace: dump_stack_lvl+0x91/0xf0 print_irq_inversion_bug.part.0+0x3d5/0x570 mark_lock.part.0+0x900/0x2f50 __lock_acquire+0x8b8/0x6340 lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x2860 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 RIP: 0010:try_charge_memcg+0x275/0x12f0 Code: 85 6e 0e 00 00 49 c7 46 28 00 00 00 00 48 c7 c6 90 5c 83 81 4c 89 f7 e8 49 41 aa ff 48 85 db 74 06 e8 3f 14 d0 ff fb 40 84 ed <0f> 85 7d 03 00 00 0f 1f 44 00 00 8b 2c 24 4c 8d 74 24 78 49 8d 9d RSP: 0018:ffff8880168f7a18 EFLAGS: 00000202 RAX: 0000000001b345a9 RBX: 0000000000000200 RCX: ffffffff812cfb6f RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffffffff81835cb1 RBP: 0000000000000001 R08: 0000000000000001 R09: fffffbfff0ef816c R10: ffffffff877c0b67 R11: 0000000000000001 R12: 0000000000000003 R13: ffff888016fb4000 R14: ffff88806cf36160 R15: ffff88806cf36188 obj_cgroup_charge+0x1d0/0x530 __kmem_cache_alloc_node+0xa3/0x320 __kmalloc_node+0x4d/0x160 kvmalloc_node+0xaa/0x200 seq_read_iter+0x820/0x1300 kernfs_fop_read_iter+0x4d4/0x690 vfs_read+0x4b5/0x8f0 ksys_read+0x122/0x250 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 RIP: 0033:0x7fd9b8b8ce8e Code: c0 e9 b6 fe ff ff 50 48 8d 3d 6e 18 0a 00 e8 b9 e7 01 00 66 0f 1f 84 00 00 00 00 00 64 8b 04 25 18 00 00 00 85 c0 75 14 0f 05 <48> 3d 00 f0 ff ff 77 5a c3 66 0f 1f 84 00 00 00 00 00 48 83 ec 28 RSP: 002b:00007ffc5551ef48 EFLAGS: 00000246 ORIG_RAX: 0000000000000000 RAX: ffffffffffffffda RBX: 0000563ab1ac1cc0 RCX: 00007fd9b8b8ce8e RDX: 0000000000001000 RSI: 0000563ab1abf4a0 RDI: 000000000000000f RBP: 00007fd9b8c5e4a0 R08: 000000000000000f R09: 00007fd9b8c5cbe0 R10: 0000000000000040 R11: 0000000000000246 R12: 0000000000001001 R13: 0000563ab1abf4a0 R14: 0000000000000d68 R15: 00007fd9b8c5d8a0 loop6: detected capacity change from 0 to 40 loop7: detected capacity change from 0 to 40 loop6: detected capacity change from 0 to 40 loop7: detected capacity change from 0 to 40 loop6: detected capacity change from 0 to 40 loop0: detected capacity change from 0 to 256 FAT-fs (loop0): bogus number of FAT structure FAT-fs (loop0): Can't find a valid FAT filesystem warning: checkpointing journal with EXT4_IOC_CHECKPOINT_FLAG_ZEROOUT can be slow cgroup: fork rejected by pids controller in /syz6 warning: checkpointing journal with EXT4_IOC_CHECKPOINT_FLAG_ZEROOUT can be slow warning: checkpointing journal with EXT4_IOC_CHECKPOINT_FLAG_ZEROOUT can be slow warning: checkpointing journal with EXT4_IOC_CHECKPOINT_FLAG_ZEROOUT can be slow warning: checkpointing journal with EXT4_IOC_CHECKPOINT_FLAG_ZEROOUT can be slow warning: checkpointing journal with EXT4_IOC_CHECKPOINT_FLAG_ZEROOUT can be slow warning: checkpointing journal with EXT4_IOC_CHECKPOINT_FLAG_ZEROOUT can be slow ---------------- Code disassembly (best guess): 0: 85 6e 0e test %ebp,0xe(%rsi) 3: 00 00 add %al,(%rax) 5: 49 c7 46 28 00 00 00 movq $0x0,0x28(%r14) c: 00 d: 48 c7 c6 90 5c 83 81 mov $0xffffffff81835c90,%rsi 14: 4c 89 f7 mov %r14,%rdi 17: e8 49 41 aa ff callq 0xffaa4165 1c: 48 85 db test %rbx,%rbx 1f: 74 06 je 0x27 21: e8 3f 14 d0 ff callq 0xffd01465 26: fb sti 27: 40 84 ed test %bpl,%bpl * 2a: 0f 85 7d 03 00 00 jne 0x3ad <-- trapping instruction 30: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 35: 8b 2c 24 mov (%rsp),%ebp 38: 4c 8d 74 24 78 lea 0x78(%rsp),%r14 3d: 49 rex.WB 3e: 8d .byte 0x8d 3f: 9d popfq