======================================================== WARNING: possible irq lock inversion dependency detected 6.4.0-rc7-next-20230622 #1 Not tainted -------------------------------------------------------- sshd/235 just changed the state of lock: ffffffff85814c18 (blkg_stat_lock){+.-.}-{2:2}, at: __blkcg_rstat_flush.isra.0+0x11f/0x4e0 but this lock was taken by another, HARDIRQ-safe lock in the past: (per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)){-.-.}-{2:2} and interrupts could create inverse lock ordering between them. other info that might help us debug this: Possible interrupt unsafe locking scenario: CPU0 CPU1 ---- ---- lock(blkg_stat_lock); local_irq_disable(); lock(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)); lock(blkg_stat_lock); lock(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)); *** DEADLOCK *** 3 locks held by sshd/235: #0: ffff888014708d30 (sk_lock-AF_INET){+.+.}-{0:0}, at: tcp_sendmsg+0x21/0x50 #1: ffffffff8560b5a0 (rcu_callback){....}-{0:0}, at: rcu_core+0x83e/0x2860 #2: ffffffff8560b6c0 (rcu_read_lock){....}-{1:2}, at: __blkcg_rstat_flush.isra.0+0x93/0x4e0 the shortest dependencies between 2nd lock and 1st lock: -> (per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)){-.-.}-{2:2} { IN-HARDIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_updated+0xcb/0x2e0 __cgroup_account_cputime_field+0xa0/0x120 account_system_index_time+0x199/0x2c0 update_process_times+0x26/0x150 tick_sched_handle+0x8e/0x170 tick_sched_timer+0xe6/0x110 __hrtimer_run_queues+0x17f/0xb60 hrtimer_interrupt+0x2ef/0x750 __sysvec_apic_timer_interrupt+0xff/0x380 sysvec_apic_timer_interrupt+0x69/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 __orc_find+0x6f/0xf0 unwind_next_frame+0x2b1/0x2490 __unwind_start+0x513/0x7c0 arch_stack_walk+0x63/0xf0 stack_trace_save+0x90/0xd0 set_track_prepare+0x74/0xd0 __create_object+0x3b2/0xc90 kmem_cache_alloc+0x20b/0x370 security_file_alloc+0x38/0x170 init_file+0x98/0x1f0 alloc_empty_file+0x94/0x1d0 path_openat+0xd8/0x2710 do_filp_open+0x1ba/0x410 do_sys_openat2+0x164/0x1d0 __x64_sys_openat+0x143/0x200 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 IN-SOFTIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_updated+0xcb/0x2e0 __cgroup_account_cputime+0x75/0xc0 update_curr+0x350/0x6d0 dequeue_task_fair+0x20e/0x14a0 load_balance+0xcb4/0x2790 rebalance_domains+0x66c/0xc00 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 default_idle+0xf/0x20 default_idle_call+0x67/0xa0 do_idle+0x305/0x3c0 cpu_startup_entry+0x18/0x20 start_secondary+0x1ba/0x210 secondary_startup_64_no_verify+0x179/0x17b INITIAL USE at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_flush_locked+0x131/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 flush_memcg_stats_dwork+0x9/0x50 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 } ... key at: [] __key.0+0x0/0x40 ... acquired at: _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 blkcg_rstat_flush+0x87/0xb0 cgroup_rstat_flush_locked+0x706/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 flush_memcg_stats_dwork+0x9/0x50 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 -> (blkg_stat_lock){+.-.}-{2:2} { HARDIRQ-ON-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x2860 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 _raw_spin_unlock_irqrestore+0x34/0x50 kmem_cache_alloc_node+0x238/0x3b0 __alloc_skb+0x28c/0x330 tcp_stream_alloc_skb+0x36/0x580 tcp_sendmsg_locked+0xcbd/0x3120 tcp_sendmsg+0x2f/0x50 inet_sendmsg+0xb9/0x150 sock_write_iter+0x4ab/0x550 vfs_write+0x97d/0xda0 ksys_write+0x1eb/0x250 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 IN-SOFTIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x2860 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 _raw_spin_unlock_irqrestore+0x34/0x50 kmem_cache_alloc_node+0x238/0x3b0 __alloc_skb+0x28c/0x330 tcp_stream_alloc_skb+0x36/0x580 tcp_sendmsg_locked+0xcbd/0x3120 tcp_sendmsg+0x2f/0x50 inet_sendmsg+0xb9/0x150 sock_write_iter+0x4ab/0x550 vfs_write+0x97d/0xda0 ksys_write+0x1eb/0x250 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 INITIAL USE at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 blkcg_rstat_flush+0x87/0xb0 cgroup_rstat_flush_locked+0x706/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 flush_memcg_stats_dwork+0x9/0x50 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 } ... key at: [] blkg_stat_lock+0x18/0x60 ... acquired at: __lock_acquire+0x8b8/0x6340 lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x2860 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 _raw_spin_unlock_irqrestore+0x34/0x50 kmem_cache_alloc_node+0x238/0x3b0 __alloc_skb+0x28c/0x330 tcp_stream_alloc_skb+0x36/0x580 tcp_sendmsg_locked+0xcbd/0x3120 tcp_sendmsg+0x2f/0x50 inet_sendmsg+0xb9/0x150 sock_write_iter+0x4ab/0x550 vfs_write+0x97d/0xda0 ksys_write+0x1eb/0x250 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 stack backtrace: CPU: 0 PID: 235 Comm: sshd Not tainted 6.4.0-rc7-next-20230622 #1 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-1 04/01/2014 Call Trace: dump_stack_lvl+0x91/0xf0 print_irq_inversion_bug.part.0+0x3d5/0x570 mark_lock.part.0+0x900/0x2f50 __lock_acquire+0x8b8/0x6340 lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x2860 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 RIP: 0010:_raw_spin_unlock_irqrestore+0x34/0x50 Code: c7 18 53 48 89 f3 48 8b 74 24 10 e8 56 16 da fc 48 89 ef e8 1e 94 da fc 80 e7 02 74 06 e8 44 e9 ff fc fb 65 ff 0d 14 03 b0 7b <74> 07 5b 5d e9 03 2a 00 00 0f 1f 44 00 00 5b 5d e9 f7 29 00 00 0f RSP: 0018:ffff88801018f8d8 EFLAGS: 00000286 RAX: 000000000034057d RBX: 0000000000000246 RCX: ffffffff812cfb6f RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffffffff845387ac RBP: ffffffff856d8ee0 R08: 0000000000000001 R09: fffffbfff0ef8171 R10: ffffffff877c0b8f R11: 0000000000000001 R12: ffff888008faf140 R13: 0000000000000000 R14: 0000000000000cc0 R15: 0000000000000cc0 kmem_cache_alloc_node+0x238/0x3b0 __alloc_skb+0x28c/0x330 tcp_stream_alloc_skb+0x36/0x580 tcp_sendmsg_locked+0xcbd/0x3120 tcp_sendmsg+0x2f/0x50 inet_sendmsg+0xb9/0x150 sock_write_iter+0x4ab/0x550 vfs_write+0x97d/0xda0 ksys_write+0x1eb/0x250 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 RIP: 0033:0x7f8908749f33 Code: 8b 15 61 ef 0c 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b7 0f 1f 00 64 8b 04 25 18 00 00 00 85 c0 75 14 b8 01 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 55 c3 0f 1f 40 00 48 83 ec 28 48 89 54 24 18 RSP: 002b:00007fffe541b158 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 RAX: ffffffffffffffda RBX: 000000000000010c RCX: 00007f8908749f33 RDX: 000000000000010c RSI: 000055a5c32dba40 RDI: 0000000000000004 RBP: 000055a5c329fe70 R08: 0000000000000140 R09: 00007fffe5494080 R10: 00007fffe54940f0 R11: 0000000000000246 R12: 000055a5c1a8e768 R13: 0000000000000000 R14: 0000000000000004 R15: 000055a5c1a43a80 ---------------- Code disassembly (best guess), 1 bytes skipped: 0: 18 53 48 sbb %dl,0x48(%rbx) 3: 89 f3 mov %esi,%ebx 5: 48 8b 74 24 10 mov 0x10(%rsp),%rsi a: e8 56 16 da fc callq 0xfcda1665 f: 48 89 ef mov %rbp,%rdi 12: e8 1e 94 da fc callq 0xfcda9435 17: 80 e7 02 and $0x2,%bh 1a: 74 06 je 0x22 1c: e8 44 e9 ff fc callq 0xfcffe965 21: fb sti 22: 65 ff 0d 14 03 b0 7b decl %gs:0x7bb00314(%rip) # 0x7bb0033d * 29: 74 07 je 0x32 <-- trapping instruction 2b: 5b pop %rbx 2c: 5d pop %rbp 2d: e9 03 2a 00 00 jmpq 0x2a35 32: 0f 1f 44 00 00 nopl 0x0(%rax,%rax,1) 37: 5b pop %rbx 38: 5d pop %rbp 39: e9 f7 29 00 00 jmpq 0x2a35 3e: 0f .byte 0xf