I/O error, dev sr0, sector 0 op 0x0:(READ) flags 0x0 phys_seg 1 prio class 2 Buffer I/O error on dev sr0, logical block 0, async page read ======================================================== WARNING: possible irq lock inversion dependency detected 6.4.0-rc7-next-20230623 #1 Not tainted -------------------------------------------------------- kworker/0:1H/36 just changed the state of lock: ffffffff85815818 (blkg_stat_lock){+.-.}-{2:2}, at: __blkcg_rstat_flush.isra.0+0x11f/0x4e0 but this lock was taken by another, HARDIRQ-safe lock in the past: (per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)){-.-.}-{2:2} and interrupts could create inverse lock ordering between them. other info that might help us debug this: Possible interrupt unsafe locking scenario: CPU0 CPU1 ---- ---- lock(blkg_stat_lock); local_irq_disable(); lock(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)); lock(blkg_stat_lock); lock(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)); *** DEADLOCK *** 5 locks held by kworker/0:1H/36: #0: ffff888008eed138 ((wq_completion)kblockd){+.+.}-{0:0}, at: process_one_work+0x99d/0x1770 #1: ffff88800bacfdb0 ((work_completion)(&(&hctx->run_work)->work)){+.+.}-{0:0}, at: process_one_work+0x9d0/0x1770 #2: ffffffff8560b6c0 (rcu_read_lock){....}-{1:2}, at: blk_mq_run_work_fn+0x1b5/0x390 #3: ffffffff8560b5a0 (rcu_callback){....}-{0:0}, at: rcu_core+0x83e/0x28b0 #4: ffffffff8560b6c0 (rcu_read_lock){....}-{1:2}, at: __blkcg_rstat_flush.isra.0+0x93/0x4e0 the shortest dependencies between 2nd lock and 1st lock: -> (per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu)){-.-.}-{2:2} { IN-HARDIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_updated+0xcb/0x2e0 __cgroup_account_cputime_field+0xa0/0x120 account_system_index_time+0x199/0x2c0 update_process_times+0x26/0x150 tick_sched_handle+0x8e/0x170 tick_sched_timer+0xe6/0x110 __hrtimer_run_queues+0x17f/0xb60 hrtimer_interrupt+0x2ef/0x750 __sysvec_apic_timer_interrupt+0xff/0x380 sysvec_apic_timer_interrupt+0x69/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 __sanitizer_cov_trace_pc+0x20/0x70 kernfs_seq_next+0x17e/0x420 seq_read_iter+0xadd/0x1300 kernfs_fop_read_iter+0x4d4/0x690 vfs_read+0x4b5/0x8f0 ksys_read+0x122/0x250 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 IN-SOFTIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_updated+0xcb/0x2e0 __cgroup_account_cputime+0x75/0xc0 update_curr+0x350/0x6d0 dequeue_task_fair+0x20e/0x14a0 load_balance+0xcb4/0x2790 rebalance_domains+0x66c/0xc00 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 vma_prepare+0x141/0x11a0 __split_vma+0x43e/0x770 do_vmi_align_munmap+0x2aa/0x12e0 do_vmi_munmap+0x1f1/0x2b0 mmap_region+0x1fe/0x26c0 do_mmap+0x833/0xea0 vm_mmap_pgoff+0x1a8/0x3b0 ksys_mmap_pgoff+0x3af/0x500 do_syscall_64+0x3f/0x90 entry_SYSCALL_64_after_hwframe+0x6e/0xd8 INITIAL USE at: lock_acquire+0x19a/0x4c0 _raw_spin_lock_irqsave+0x3a/0x60 cgroup_rstat_flush_locked+0x131/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 flush_memcg_stats_dwork+0x9/0x50 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 } ... key at: [] __key.0+0x0/0x40 ... acquired at: _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 blkcg_rstat_flush+0x87/0xb0 cgroup_rstat_flush_locked+0x706/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 mem_cgroup_wb_stats+0x3d8/0x4b0 balance_dirty_pages+0x339/0x2640 balance_dirty_pages_ratelimited_flags+0xbce/0x1130 fault_dirty_shared_page+0x2b3/0x500 do_wp_page+0x3c8/0x3b00 __handle_mm_fault+0xcbc/0x2d20 handle_mm_fault+0x1af/0xba0 do_user_addr_fault+0x58e/0x12f0 exc_page_fault+0x9c/0x1a0 asm_exc_page_fault+0x26/0x30 -> (blkg_stat_lock){+.-.}-{2:2} { HARDIRQ-ON-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x28b0 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 vprintk_store+0x180/0x990 vprintk_emit+0x154/0x560 vprintk+0x88/0xa0 _printk+0xbf/0x100 buffer_io_error+0xd9/0x110 end_buffer_async_read+0x304/0x3a0 end_bio_bh_io_sync+0xde/0x130 bio_endio+0x51a/0x620 blk_update_request+0x397/0x12a0 blk_mq_end_request+0x59/0x630 blk_mq_dispatch_rq_list+0xc5c/0x1dc0 __blk_mq_sched_dispatch_requests+0xcd8/0x1520 blk_mq_sched_dispatch_requests+0x10a/0x190 blk_mq_run_work_fn+0x206/0x390 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 IN-SOFTIRQ-W at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x28b0 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 vprintk_store+0x180/0x990 vprintk_emit+0x154/0x560 vprintk+0x88/0xa0 _printk+0xbf/0x100 buffer_io_error+0xd9/0x110 end_buffer_async_read+0x304/0x3a0 end_bio_bh_io_sync+0xde/0x130 bio_endio+0x51a/0x620 blk_update_request+0x397/0x12a0 blk_mq_end_request+0x59/0x630 blk_mq_dispatch_rq_list+0xc5c/0x1dc0 __blk_mq_sched_dispatch_requests+0xcd8/0x1520 blk_mq_sched_dispatch_requests+0x10a/0x190 blk_mq_run_work_fn+0x206/0x390 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 INITIAL USE at: lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 blkcg_rstat_flush+0x87/0xb0 cgroup_rstat_flush_locked+0x706/0xd80 cgroup_rstat_flush+0x37/0x50 do_flush_stats+0x97/0xf0 mem_cgroup_wb_stats+0x3d8/0x4b0 balance_dirty_pages+0x339/0x2640 balance_dirty_pages_ratelimited_flags+0xbce/0x1130 fault_dirty_shared_page+0x2b3/0x500 do_wp_page+0x3c8/0x3b00 __handle_mm_fault+0xcbc/0x2d20 handle_mm_fault+0x1af/0xba0 do_user_addr_fault+0x58e/0x12f0 exc_page_fault+0x9c/0x1a0 asm_exc_page_fault+0x26/0x30 } ... key at: [] blkg_stat_lock+0x18/0x60 ... acquired at: __lock_acquire+0x8b8/0x6340 lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x28b0 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 vprintk_store+0x180/0x990 vprintk_emit+0x154/0x560 vprintk+0x88/0xa0 _printk+0xbf/0x100 buffer_io_error+0xd9/0x110 end_buffer_async_read+0x304/0x3a0 end_bio_bh_io_sync+0xde/0x130 bio_endio+0x51a/0x620 blk_update_request+0x397/0x12a0 blk_mq_end_request+0x59/0x630 blk_mq_dispatch_rq_list+0xc5c/0x1dc0 __blk_mq_sched_dispatch_requests+0xcd8/0x1520 blk_mq_sched_dispatch_requests+0x10a/0x190 blk_mq_run_work_fn+0x206/0x390 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 stack backtrace: CPU: 0 PID: 36 Comm: kworker/0:1H Not tainted 6.4.0-rc7-next-20230623 #1 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-1 04/01/2014 Workqueue: kblockd blk_mq_run_work_fn Call Trace: dump_stack_lvl+0x91/0xf0 print_irq_inversion_bug.part.0+0x3d5/0x570 mark_lock.part.0+0x900/0x2f50 __lock_acquire+0x8b8/0x6340 lock_acquire+0x19a/0x4c0 _raw_spin_lock+0x2b/0x40 __blkcg_rstat_flush.isra.0+0x11f/0x4e0 __blkg_release+0xfa/0x3b0 rcu_core+0x8c8/0x28b0 __do_softirq+0x1b7/0x7d4 irq_exit_rcu+0x93/0xc0 sysvec_apic_timer_interrupt+0x6e/0x90 asm_sysvec_apic_timer_interrupt+0x1a/0x20 RIP: 0010:vprintk_store+0x180/0x990 Code: e8 75 c3 1b 00 41 80 fd 03 0f 86 cf 00 00 00 e8 26 c8 1b 00 48 89 ee 31 ff 45 31 ff e8 e9 c3 1b 00 48 85 ed 0f 85 4e 04 00 00 0b c8 1b 00 48 b8 00 00 00 00 00 fc ff df 48 03 44 24 10 48 c7 RSP: 0018:ffff88800bacf580 EFLAGS: 00000202 RAX: 0000000000005b3d RBX: ffff88806ce287e5 RCX: ffffffff812d1fbf RDX: 0000000000000000 RSI: 0000000000000000 RDI: ffffffff81309b88 RBP: 0000000000000200 R08: 0000000000000001 R09: fffffbfff0ef9b69 R10: ffffffff877cdb4f R11: 0000000000000001 R12: 000000000000003d R13: 0000000000000002 R14: ffffffff8552e698 R15: 000000000000003d vprintk_emit+0x154/0x560 vprintk+0x88/0xa0 _printk+0xbf/0x100 buffer_io_error+0xd9/0x110 end_buffer_async_read+0x304/0x3a0 end_bio_bh_io_sync+0xde/0x130 bio_endio+0x51a/0x620 blk_update_request+0x397/0x12a0 blk_mq_end_request+0x59/0x630 blk_mq_dispatch_rq_list+0xc5c/0x1dc0 __blk_mq_sched_dispatch_requests+0xcd8/0x1520 blk_mq_sched_dispatch_requests+0x10a/0x190 blk_mq_run_work_fn+0x206/0x390 process_one_work+0xabf/0x1770 worker_thread+0x64f/0x12a0 kthread+0x33f/0x440 ret_from_fork+0x2c/0x50 sr 1:0:0:0: [sr0] tag#0 unaligned transfer I/O error, dev sr0, sector 1 op 0x0:(READ) flags 0x0 phys_seg 7 prio class 2 Buffer I/O error on dev sr0, logical block 1, async page read sr 1:0:0:0: [sr0] tag#0 unaligned transfer I/O error, dev sr0, sector 0 op 0x0:(READ) flags 0x0 phys_seg 1 prio class 2 sr 1:0:0:0: [sr0] tag#0 unaligned transfer I/O error, dev sr0, sector 1 op 0x0:(READ) flags 0x0 phys_seg 1 prio class 2 sr 1:0:0:0: [sr0] tag#0 unaligned transfer I/O error, dev sr0, sector 2 op 0x0:(READ) flags 0x0 phys_seg 1 prio class 2 sr 1:0:0:0: [sr0] tag#0 unaligned transfer I/O error, dev sr0, sector 3 op 0x0:(READ) flags 0x0 phys_seg 1 prio class 2 sr 1:0:0:0: [sr0] tag#0 unaligned transfer I/O error, dev sr0, sector 4 op 0x0:(READ) flags 0x0 phys_seg 1 prio class 2 sr 1:0:0:0: [sr0] tag#0 unaligned transfer I/O error, dev sr0, sector 5 op 0x0:(READ) flags 0x0 phys_seg 1 prio class 2 sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 unaligned transfer sr 1:0:0:0: [sr0] tag#0 FAILED Result: hostbyte=DID_OK driverbyte=DRIVER_OK cmd_age=0s sr 1:0:0:0: [sr0] tag#0 Sense Key : Not Ready [current] sr 1:0:0:0: [sr0] tag#0 Add. Sense: Medium not present sr 1:0:0:0: [sr0] tag#0 CDB: Read(10) 28 00 00 00 00 00 00 00 40 00 ---------------- Code disassembly (best guess): 0: e8 75 c3 1b 00 callq 0x1bc37a 5: 41 80 fd 03 cmp $0x3,%r13b 9: 0f 86 cf 00 00 00 jbe 0xde f: e8 26 c8 1b 00 callq 0x1bc83a 14: 48 89 ee mov %rbp,%rsi 17: 31 ff xor %edi,%edi 19: 45 31 ff xor %r15d,%r15d 1c: e8 e9 c3 1b 00 callq 0x1bc40a 21: 48 85 ed test %rbp,%rbp 24: 0f 85 4e 04 00 00 jne 0x478 * 2a: e8 0b c8 1b 00 callq 0x1bc83a <-- trapping instruction 2f: 48 b8 00 00 00 00 00 movabs $0xdffffc0000000000,%rax 36: fc ff df 39: 48 03 44 24 10 add 0x10(%rsp),%rax 3e: 48 rex.W 3f: c7 .byte 0xc7