======================================================== WARNING: possible irq lock inversion dependency detected 6.4.0-rc7-next-20230623 #1 Not tainted -------------------------------------------------------- kworker/u4:0/6063 just changed the state of lock: ffffffff85815818 (blkg_stat_lock){+.-.}-{2:2}, at: __blkcg_rstat_flush.isra.0+0x11f/0x4e0 but this lock was taken by another, HARDIRQ-safe lock in the past: (per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)){-.-.}-{2:2} and interrupts could create inverse lock ordering between them. other info that might help us debug this: Possible interrupt unsafe locking scenario: CPU0 CPU1 ---- ---- lock(blkg_stat_lock); local_irq_disable(); lock(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)); lock(blkg_stat_lock); lock(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)); *** DEADLOCK *** 4 locks held by kworker/u4:0/6063: #0: ffff88801a6163c8 (&sig->cred_guard_mutex){+.+.}-{3:3}, at: bprm_execve+0xb2/0x1940 #1: ffffffff8560b6c0 (rcu_read_lock){....}-{1:2}, at: filemap_get_read_batch+0x0/0x890 #2: ffffffff8560b5a0 (rcu_callback){....}-{0:0}, at: rcu_core+0x83e/0x28b0 #3: ffffffff8560b6c0 (rcu_read_lock){....}-{1:2}, at: __blkcg_rstat_flush.isra.0+0x93/0x4e0 the shortest dependencies between 2nd lock and 1st lock: -> (per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)){-.-.}-{2:2} { IN-HARDIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_updated+0xcb/0x2e0 __cgroup_account_cputime_field+0xa0/0x120 account_system_index_time+0x199/0x2c0 update_process_times+0x26/0x150 tick_sched_handle+0x8e/0x170 tick_sched_timer+0xe6/0x110 __hrtimer_run_queues+0x17f/0xb60 hrtimer_interrupt+0x2ef/0x750 __sysvec_apic_timer_interrupt+0xff/0x380 sysvec_apic_timer_interrupt+0x69/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 ___slab_alloc+0x227/0x1190 kmem_cache_alloc+0x34e/0x370 __create_object+0x3c/0xc90 __kmem_cache_alloc_node+0x206/0x320 kmalloc_trace+0x26/0xc0 cgroup_file_open+0x93/0x370 kernfs_fop_open+0x94e/0xe90 do_dentry_open+0x69b/0x15c0 path_openat+0x1859/0x2710 do_filp_open+0x1ba/0x410 do_sys_openat2+0x164/0x1d0 __x64_sys_openat+0x143/0x200 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 IN-SOFTIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_updated+0xcb/0x2e0 __cgroup_account_cputime+0x75/0xc0 update_curr+0x350/0x6d0 dequeue_task_fair+0x20e/0x14a0 load_balance+0xcb4/0x2790 rebalance_domains+0x66c/0xc00 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 lock_acquire+0x1c7/0x4c0 avc_has_perm_noaudit+0x6c/0x3a0 avc_has_perm+0xa7/0x1a0 inode_has_perm+0x162/0x1c0 selinux_file_open+0x306/0x3f0 security_file_open+0x4f/0xc0 do_dentry_open+0x547/0x15c0 path_openat+0x1859/0x2710 do_filp_open+0x1ba/0x410 do_sys_openat2+0x164/0x1d0 __x64_sys_openat+0x143/0x200 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 INITIAL USE at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_flush_locked+0x131/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 flush_memcg_stats_dwork+0x9/0x50 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 } ... key at: [] __key.0+0x0/0x40 ... acquired at: _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 blkcg_rstat_flush+0x87/0xb0 cgroup_rstat_flush_locked+0x706/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 flush_memcg_stats_dwork+0x9/0x50 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 -> (blkg_stat_lock){+.-.}-{2:2} { HARDIRQ-ON-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x28b0 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 lock_is_held_type+0x2b/0x120 xas_reload+0x256/0x440 filemap_get_read_batch+0x23f/0x890 filemap_get_pages+0x1cb/0x1560 filemap_read+0x32c/0xb90 generic_file_read_iter+0x2f8/0x400 ext4_file_read_iter+0x18c/0x4b0 __kernel_read+0x2cf/0x880 kernel_read+0xc3/0x1b0 bprm_execve+0x71c/0x1940 kernel_execve+0x3ee/0x510 call_usermodehelper_exec_async+0x263/0x4e0 ret_from_fork+0x2c/0x50 IN-SOFTIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x28b0 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 lock_is_held_type+0x2b/0x120 xas_reload+0x256/0x440 filemap_get_read_batch+0x23f/0x890 filemap_get_pages+0x1cb/0x1560 filemap_read+0x32c/0xb90 generic_file_read_iter+0x2f8/0x400 ext4_file_read_iter+0x18c/0x4b0 __kernel_read+0x2cf/0x880 kernel_read+0xc3/0x1b0 bprm_execve+0x71c/0x1940 kernel_execve+0x3ee/0x510 call_usermodehelper_exec_async+0x263/0x4e0 ret_from_fork+0x2c/0x50 INITIAL USE at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 blkcg_rstat_flush+0x87/0xb0 cgroup_rstat_flush_locked+0x706/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 flush_memcg_stats_dwork+0x9/0x50 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 } ... key at: [] blkg_stat_lock+0x18/0x60 ... acquired at: __lock_acquire+0x8b8/0x6340 lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x28b0 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 lock_is_held_type+0x2b/0x120 xas_reload+0x256/0x440 filemap_get_read_batch+0x23f/0x890 filemap_get_pages+0x1cb/0x1560 filemap_read+0x32c/0xb90 generic_file_read_iter+0x2f8/0x400 ext4_file_read_iter+0x18c/0x4b0 __kernel_read+0x2cf/0x880 kernel_read+0xc3/0x1b0 bprm_execve+0x71c/0x1940 kernel_execve+0x3ee/0x510 call_usermodehelper_exec_async+0x263/0x4e0 ret_from_fork+0x2c/0x50 stack backtrace: CPU: 0 PID: 6063 Comm: kworker/u4:0 Not tainted 6.4.0-rc7-next-20230623 #1 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-1 04/01/2014 Call Trace: dump_stack_lvl+0x91/0xf0 print_irq_inversion_bug.part.0+0x3d5/0x570 mark_lock.part.0+0x900/0x2f50 __lock_acquire+0x8b8/0x6340 lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x28b0 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 RIP: 0010:lock_is_held_type+0x2b/0x120 Code: 0f 1e fa 8b 0d b6 78 81 01 41 b8 ff ff ff ff 85 c9 0f 84 de 00 00 00 65 8b 05 f1 e5 af 7b 85 c0 0f 85 cf 00 00 00 41 57 41 56 <41> 55 41 54 55 53 65 4c 8b 24 25 c0 8a 03 00 48 83 ec 08 41 8b 94 RSP: 0018:ffff888043a7f7b0 EFLAGS: 00000246 RAX: 0000000000000000 RBX: ffff888043a7f838 RCX: 0000000000000001 RDX: ffff888043fad1c0 RSI: 00000000ffffffff RDI: ffff888015ccc068 RBP: ffff8880088c3c80 R08: 00000000ffffffff R09: 0000000000000000 R10: 0000000000000001 R11: 0000000000000001 R12: ffffea00001ba380 R13: ffff888015ccc050 R14: 0000000000000001 R15: ffff888043a7f838 xas_reload+0x256/0x440 filemap_get_read_batch+0x23f/0x890 filemap_get_pages+0x1cb/0x1560 filemap_read+0x32c/0xb90 generic_file_read_iter+0x2f8/0x400 ext4_file_read_iter+0x18c/0x4b0 __kernel_read+0x2cf/0x880 kernel_read+0xc3/0x1b0 bprm_execve+0x71c/0x1940 kernel_execve+0x3ee/0x510 call_usermodehelper_exec_async+0x263/0x4e0 ret_from_fork+0x2c/0x50 ---------------- Code disassembly (best guess): 0: 0f 1e fa nop %edx 3: 8b 0d b6 78 81 01 mov 0x18178b6(%rip),%ecx # 0x18178bf 9: 41 b8 ff ff ff ff mov $0xffffffff,%r8d f: 85 c9 test %ecx,%ecx 11: 0f 84 de 00 00 00 je 0xf5 17: 65 8b 05 f1 e5 af 7b mov %gs:0x7bafe5f1(%rip),%eax # 0x7bafe60f 1e: 85 c0 test %eax,%eax 20: 0f 85 cf 00 00 00 jne 0xf5 26: 41 57 push %r15 28: 41 56 push %r14 * 2a: 41 55 push %r13 <-- trapping instruction 2c: 41 54 push %r12 2e: 55 push %rbp 2f: 53 push %rbx 30: 65 4c 8b 24 25 c0 8a mov %gs:0x38ac0,%r12 37: 03 00 39: 48 83 ec 08 sub $0x8,%rsp 3d: 41 rex.B 3e: 8b .byte 0x8b 3f: 94 xchg %eax,%esp