loop5: detected capacity change from 0 to 2247 ======================================================== WARNING: possible irq lock inversion dependency detected 6.4.0-rc7-next-20230623 #1 Not tainted -------------------------------------------------------- systemd-udevd/5677 just changed the state of lock: ffffffff85815818 (blkg_stat_lock){+.-.}-{2:2}, at: __blkcg_rstat_flush.isra.0+0x11f/0x4e0 but this lock was taken by another, HARDIRQ-safe lock in the past: (per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)){-.-.}-{2:2} and interrupts could create inverse lock ordering between them. other info that might help us debug this: Possible interrupt unsafe locking scenario: CPU0 CPU1 ---- ---- lock(blkg_stat_lock); local_irq_disable(); lock(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)); lock(blkg_stat_lock); lock(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)); *** DEADLOCK *** 3 locks held by systemd-udevd/5677: #0: ffff8880087849e0 (&root->kernfs_iattr_rwsem){++++}-{3:3}, at: kernfs_iop_permission+0xbc/0x120 #1: ffffffff8560b5a0 (rcu_callback){....}-{0:0}, at: rcu_core+0x83e/0x28b0 #2: ffffffff8560b6c0 (rcu_read_lock){....}-{1:2}, at: __blkcg_rstat_flush.isra.0+0x93/0x4e0 the shortest dependencies between 2nd lock and 1st lock: -> (per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)){-.-.}-{2:2} { IN-HARDIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_updated+0xcb/0x2e0 __cgroup_account_cputime_field+0xa0/0x120 account_system_index_time+0x199/0x2c0 update_process_times+0x26/0x150 tick_sched_handle+0x8e/0x170 tick_sched_timer+0xe6/0x110 __hrtimer_run_queues+0x17f/0xb60 hrtimer_interrupt+0x2ef/0x750 __sysvec_apic_timer_interrupt+0xff/0x380 sysvec_apic_timer_interrupt+0x69/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 __orc_find+0x8f/0xf0 unwind_next_frame+0x2b1/0x2490 arch_stack_walk+0x87/0xf0 stack_trace_save+0x90/0xd0 kasan_save_stack+0x22/0x50 kasan_set_track+0x25/0x30 __kasan_slab_alloc+0x59/0x70 kmem_cache_alloc+0x16b/0x370 security_inode_alloc+0x38/0x160 inode_init_always+0xbb2/0xea0 alloc_inode+0x84/0x240 iget_locked+0x199/0x660 kernfs_get_inode+0x4b/0x520 kernfs_iop_lookup+0x1e9/0x330 lookup_open.isra.0+0x745/0x1400 path_openat+0x96c/0x2710 do_filp_open+0x1ba/0x410 do_sys_openat2+0x164/0x1d0 __x64_sys_openat+0x143/0x200 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 IN-SOFTIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_updated+0xcb/0x2e0 __cgroup_account_cputime+0x75/0xc0 update_curr+0x350/0x6d0 dequeue_task_fair+0x20e/0x14a0 load_balance+0xcb4/0x2790 rebalance_domains+0x66c/0xc00 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 lock_acquire+0x1c7/0x4c0 __is_insn_slot_addr+0x41/0x290 kernel_text_address+0x48/0xc0 __kernel_text_address+0xd/0x40 unwind_get_return_address+0x59/0xa0 arch_stack_walk+0x9d/0xf0 stack_trace_save+0x90/0xd0 set_track_prepare+0x74/0xd0 __create_object+0x3b2/0xc90 kmem_cache_alloc+0x20b/0x370 getname_flags.part.0+0x50/0x4f0 getname+0x92/0xe0 do_sys_openat2+0xec/0x1d0 __x64_sys_openat+0x143/0x200 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 INITIAL USE at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_flush_locked+0x131/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 flush_memcg_stats_dwork+0x9/0x50 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 } ... key at: [] __key.0+0x0/0x40 ... acquired at: _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 blkcg_rstat_flush+0x87/0xb0 cgroup_rstat_flush_locked+0x706/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 flush_memcg_stats_dwork+0x9/0x50 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 -> (blkg_stat_lock){+.-.}-{2:2} { HARDIRQ-ON-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x28b0 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 kernfs_refresh_inode+0x32/0x3a0 kernfs_iop_permission+0xc7/0x120 inode_permission.part.0+0x29a/0x520 link_path_walk.part.0+0x85e/0xd90 path_openat+0x254/0x2710 do_filp_open+0x1ba/0x410 do_sys_openat2+0x164/0x1d0 __x64_sys_openat+0x143/0x200 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 IN-SOFTIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x28b0 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 kernfs_refresh_inode+0x32/0x3a0 kernfs_iop_permission+0xc7/0x120 inode_permission.part.0+0x29a/0x520 link_path_walk.part.0+0x85e/0xd90 path_openat+0x254/0x2710 do_filp_open+0x1ba/0x410 do_sys_openat2+0x164/0x1d0 __x64_sys_openat+0x143/0x200 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 INITIAL USE at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 blkcg_rstat_flush+0x87/0xb0 cgroup_rstat_flush_locked+0x706/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 flush_memcg_stats_dwork+0x9/0x50 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 } ... key at: [] blkg_stat_lock+0x18/0x60 ... acquired at: __lock_acquire+0x8b8/0x6340 lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x28b0 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 kernfs_refresh_inode+0x32/0x3a0 kernfs_iop_permission+0xc7/0x120 inode_permission.part.0+0x29a/0x520 link_path_walk.part.0+0x85e/0xd90 path_openat+0x254/0x2710 do_filp_open+0x1ba/0x410 do_sys_openat2+0x164/0x1d0 __x64_sys_openat+0x143/0x200 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 stack backtrace: CPU: 1 PID: 5677 Comm: systemd-udevd Not tainted 6.4.0-rc7-next-20230623 #1 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-1 04/01/2014 Call Trace: dump_stack_lvl+0x91/0xf0 print_irq_inversion_bug.part.0+0x3d5/0x570 mark_lock.part.0+0x900/0x2f50 __lock_acquire+0x8b8/0x6340 lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x28b0 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 RIP: 0010:kernfs_refresh_inode+0x32/0x3a0 Code: 41 54 49 89 f4 53 48 89 fb e8 5a d0 a3 ff 48 8d bb a0 00 00 00 48 b8 00 00 00 00 00 fc ff df 48 89 fa 48 c1 ea 03 80 3c 02 00 <0f> 85 0c 03 00 00 48 8d bb 9a 00 00 00 4c 8b ab a0 00 00 00 48 b8 RSP: 0018:ffff88801862f978 EFLAGS: 00000246 RAX: dffffc0000000000 RBX: ffff888008ccc3a0 RCX: 0000000000000000 RDX: 1ffff11001199888 RSI: ffffffff81a88ee6 RDI: ffff888008ccc440 RBP: ffff88801862f998 R08: 0000000000000001 R09: ffffed10010f092f R10: ffff88800878497f R11: 0000000000000001 R12: ffff8880143722c8 R13: ffff888008784970 R14: ffff888008ccc3a0 R15: ffff8880143722ca kernfs_iop_permission+0xc7/0x120 inode_permission.part.0+0x29a/0x520 link_path_walk.part.0+0x85e/0xd90 path_openat+0x254/0x2710 do_filp_open+0x1ba/0x410 do_sys_openat2+0x164/0x1d0 __x64_sys_openat+0x143/0x200 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 RIP: 0033:0x7ff6137ba767 Code: 25 00 00 41 00 3d 00 00 41 00 74 47 64 8b 04 25 18 00 00 00 85 c0 75 6b 44 89 e2 48 89 ee bf 9c ff ff ff b8 01 01 00 00 0f 05 <48> 3d 00 f0 ff ff 0f 87 95 00 00 00 48 8b 4c 24 28 64 48 2b 0c 25 RSP: 002b:00007ffee16d2870 EFLAGS: 00000246 ORIG_RAX: 0000000000000101 RAX: ffffffffffffffda RBX: 000055fd6dd3e300 RCX: 00007ff6137ba767 RDX: 0000000000080000 RSI: 000055fd6dd008f0 RDI: 00000000ffffff9c RBP: 000055fd6dd008f0 R08: 000055fd6dd008f0 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000080000 R13: 0000000000000000 R14: 0000000000000000 R15: 00007ff6132216c0 ---------------- Code disassembly (best guess): 0: 41 54 push %r12 2: 49 89 f4 mov %rsi,%r12 5: 53 push %rbx 6: 48 89 fb mov %rdi,%rbx 9: e8 5a d0 a3 ff callq 0xffa3d068 e: 48 8d bb a0 00 00 00 lea 0xa0(%rbx),%rdi 15: 48 b8 00 00 00 00 00 movabs $0xdffffc0000000000,%rax 1c: fc ff df 1f: 48 89 fa mov %rdi,%rdx 22: 48 c1 ea 03 shr $0x3,%rdx 26: 80 3c 02 00 cmpb $0x0,(%rdx,%rax,1) * 2a: 0f 85 0c 03 00 00 jne 0x33c <-- trapping instruction 30: 48 8d bb 9a 00 00 00 lea 0x9a(%rbx),%rdi 37: 4c 8b ab a0 00 00 00 mov 0xa0(%rbx),%r13 3e: 48 rex.W 3f: b8 .byte 0xb8