Messages in this thread | | | From | Vicente Bergas <> | Subject | Re: d lookup: Unable to handle kernel paging request | Date | Wed, 19 Jun 2019 14:42:16 +0200 |
| |
On Tuesday, June 18, 2019 8:35:48 PM CEST, Al Viro wrote: > On Tue, May 28, 2019 at 11:38:43AM +0200, Vicente Bergas wrote: >> On Wednesday, May 22, 2019 6:29:46 PM CEST, Al Viro wrote: ... > > __d_lookup() running into &dentry->d_hash == 0x01000000 at some > point in hash chain > and trying to look at ->d_name.hash: > >> pc : __d_lookup+0x58/0x198 >> lr : d_lookup+0x38/0x68 >> sp : ffff000012663b90 >> x29: ffff000012663b90 x28: ffff000012663d58 x27: 0000000000000000 x26: >> ffff8000ae7cc900 x25: 0000000000000001 x24: ffffffffffffffff x23: ... > > __d_lookup_rcu() running into &dentry->d_hash == 0x01000000 at > some point in hash > chain and trying to look at ->d_seq: > >> pc : __d_lookup_rcu+0x68/0x198 >> lr : lookup_fast+0x44/0x2e8 >> sp : ffff0000130b3b60 >> x29: ffff0000130b3b60 x28: 00000000ce99d070 x27: ffffffffffffffff x26: >> 0000000000000026 x25: ffff8000ecec6030 x24: ffff0000130b3c2c x23: ... > > __d_lookup_rcu() running into &dentry->d_hash == > 0x0000880001000000 at some point > in hash chain and trying to look at ->d_seq: > >> pc : __d_lookup_rcu+0x68/0x198 >> lr : lookup_fast+0x44/0x2e8 >> sp : ffff00001325ba90 >> x29: ffff00001325ba90 x28: 00000000ce99f075 x27: ffffffffffffffff x26: >> 0000000000000007 x25: ffff8000ecec402a x24: ffff00001325bb5c x23: ... > > ditto > >> pc : __d_lookup_rcu+0x68/0x198 >> lr : lookup_fast+0x44/0x2e8 >> sp : ffff000012a3ba90 >> x29: ffff000012a3ba90 x28: 00000000ce99f075 x27: ffffffffffffffff x26: >> 0000000000000007 x25: ffff8000ecec702a x24: ffff000012a3bb5c x23: ... > > ditto > >> pc : __d_lookup_rcu+0x68/0x198 >> lr : lookup_fast+0x44/0x2e8 >> sp : ffff0000132bba90 >> x29: ffff0000132bba90 x28: 00000000ce99e1a6 x27: ffffffffffffffff x26: >> 000000000000000c x25: ffff8000f21dd036 x24: ffff0000132bbb5c x23: ... > > ... and ditto: > >> pc : __d_lookup_rcu+0x68/0x198 >> lr : lookup_fast+0x44/0x2e8 >> sp : ffff000013263a90 >> x29: ffff000013263a90 x28: 00000000ce99e1a6 x27: ffffffffffffffff x26: >> 000000000000000c x25: ffff8000f0a6f036 x24: ffff000013263b5c x23: ... > > > All of those run under rcu_read_lock() and no dentry with DCACHE_NORCU has > ever been inserted into a hash chain, so it doesn't look like a plain > use-after-free. Could you try something like the following to see a bit > more about where it comes from? > > So far it looks like something is buggering a forward reference > in hash chain in a fairly specific way - the values seen had been > 00000000010000000 and > 00008800010000000. Does that smell like anything from arm64-specific > data structures (PTE, etc.)? > > Alternatively, we might've gone off rails a step (or more) before, > with the previous iteration going through bogus, but at least mapped > address - the one that has never been a dentry in the first place. > > > diff --git a/fs/dcache.c b/fs/dcache.c > index c435398f2c81..cb555edb5b55 100644 > --- a/fs/dcache.c > +++ b/fs/dcache.c > @@ -2114,6 +2114,22 @@ static inline bool d_same_name(const > struct dentry *dentry, > name) == 0; > } > > +static void dump(struct dentry *dentry) > +{ > + int i; > + if (!dentry) { > + printk(KERN_ERR "list fucked in head"); > + return; > + } > + printk(KERN_ERR "fucked dentry[%p]: d_hash.next = %p, flags = > %x, count = %d", > + dentry, dentry->d_hash.next, dentry->d_flags, > + dentry->d_lockref.count > + ); > + for (i = 0; i < sizeof(struct dentry); i++) > + printk(KERN_CONT "%c%02x", i & 31 ? ' ' : '\n', > + ((unsigned char *)dentry)[i]); > +} > + > /** > * __d_lookup_rcu - search for a dentry (racy, store-free) > * @parent: parent dentry > @@ -2151,7 +2167,7 @@ struct dentry *__d_lookup_rcu(const > struct dentry *parent, > const unsigned char *str = name->name; > struct hlist_bl_head *b = d_hash(hashlen_hash(hashlen)); > struct hlist_bl_node *node; > - struct dentry *dentry; > + struct dentry *dentry, *last = NULL; > > /* > * Note: There is significant duplication with __d_lookup_rcu which is > @@ -2176,6 +2192,10 @@ struct dentry *__d_lookup_rcu(const > struct dentry *parent, > hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { > unsigned seq; > > + if (unlikely((u32)(unsigned long)&dentry->d_hash == 0x01000000)) > + dump(last); > + last = dentry; > + > seqretry: > /* > * The dentry sequence count protects us from concurrent > @@ -2274,7 +2294,7 @@ struct dentry *__d_lookup(const struct > dentry *parent, const struct qstr *name) > struct hlist_bl_head *b = d_hash(hash); > struct hlist_bl_node *node; > struct dentry *found = NULL; > - struct dentry *dentry; > + struct dentry *dentry, *last = NULL; > > /* > * Note: There is significant duplication with __d_lookup_rcu which is > @@ -2300,6 +2320,10 @@ struct dentry *__d_lookup(const struct > dentry *parent, const struct qstr *name) > > hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { > > + if (unlikely((u32)(unsigned long)&dentry->d_hash == 0x01000000)) > + dump(last); > + last = dentry; > + > if (dentry->d_name.hash != hash) > continue;
Hi Al, i have been running the distro-provided kernel the last few weeks and had no issues at all. https://archlinuxarm.org/packages/aarch64/linux-aarch64 It is from the v5.1 branch and is compiled with gcc 8.3.
IIRC, i also tested https://archlinuxarm.org/packages/aarch64/linux-aarch64-rc v5.2-rc1 and v5.2-rc2 (which at that time where compiled with gcc 8.2) with no issues.
This week tested v5.2-rc4 and v5.2-rc5 from archlinuxarm but there are regressions unrelated to d_lookup.
At this point i was convinced it was a gcc 9.1 issue and had nothing to do with the kernel, but anyways i gave your patch a try. The tested kernel is v5.2-rc5-224-gbed3c0d84e7e and it has been compiled with gcc 8.3. The sentinel you put there has triggered! So, it is not a gcc 9.1 issue.
In any case, i have no idea if those addresses are arm64-specific in any way.
Regards, Vicenç.
list fucked in head Unable to handle kernel paging request at virtual address 0000000000fffffc Mem abort info: ESR = 0x96000004 Exception class = DABT (current EL), IL = 32 bits SET = 0, FnV = 0 EA = 0, S1PTW = 0 Data abort info: ISV = 0, ISS = 0x00000004 CM = 0, WnR = 0 user pgtable: 4k pages, 48-bit VAs, pgdp=000000005c989000 [0000000000fffffc] pgd=0000000000000000 Internal error: Oops: 96000004 [#1] SMP CPU: 4 PID: 2427 Comm: git Not tainted 5.2.0-rc5 #1 Hardware name: Sapphire-RK3399 Board (DT) pstate: 60000005 (nZCv daif -PAN -UAO) pc : __d_lookup_rcu+0x90/0x1e8 lr : __d_lookup_rcu+0x84/0x1e8 sp : ffff000013413a90 x29: ffff000013413a90 x28: ffff000013413b5c x27: 0000000000000002 x26: ffff000013413c78 x25: 0000001ac1084259 x24: 0000000000fffff8 x23: 0000000001000000 x22: ffff8000586ed9c0 x21: ffff8000586ed9c0 x20: 0000000000fffff8 x19: 0000000001000000 x18: 0000000000000000 x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000 x14: 0000000000000000 x13: 0000000000000000 x12: 0000000000000000 x11: 0000000000000000 x10: ffffffffffffffff x9 : 00000000c1084259 x8 : ffff800054f31032 x7 : 000000000000001a x6 : ffff00001090454b x5 : 0000000000000013 x4 : 0000000000000000 x3 : 0000000000000000 x2 : 00000000ffffffff x1 : 00008000e6f46000 x0 : 0000000000000013 Call trace: __d_lookup_rcu+0x90/0x1e8 lookup_fast+0x44/0x300 walk_component+0x34/0x2e0 path_lookupat.isra.13+0x5c/0x1e0 filename_lookup.part.19+0x6c/0xe8 user_path_at_empty+0x4c/0x60 vfs_statx+0x78/0xd8 __se_sys_newfstatat+0x24/0x48 __arm64_sys_newfstatat+0x18/0x20 el0_svc_handler+0x94/0x138 el0_svc+0x8/0xc Code: 94000753 294c1fe9 9280000a f94037e8 (b85fc263) ---[ end trace 93a444e9b6bc67e8 ]--- list fucked in head Unable to handle kernel paging request at virtual address 0000000000fffffc Mem abort info: ESR = 0x96000004 Exception class = DABT (current EL), IL = 32 bits SET = 0, FnV = 0 EA = 0, S1PTW = 0 Data abort info: ISV = 0, ISS = 0x00000004 CM = 0, WnR = 0 user pgtable: 4k pages, 48-bit VAs, pgdp=000000005c989000 [0000000000fffffc] pgd=0000000000000000 Internal error: Oops: 96000004 [#2] SMP CPU: 5 PID: 2424 Comm: git Tainted: G D 5.2.0-rc5 #1 Hardware name: Sapphire-RK3399 Board (DT) pstate: 60000005 (nZCv daif -PAN -UAO) pc : __d_lookup_rcu+0x90/0x1e8 lr : __d_lookup_rcu+0x84/0x1e8 sp : ffff0000133fba90 x29: ffff0000133fba90 x28: ffff0000133fbb5c x27: 0000000000000002 x26: ffff0000133fbc78 x25: 0000001ac1084259 x24: 0000000000fffff8 x23: 0000000001000000 x22: ffff8000586ed9c0 x21: ffff8000586ed9c0 x20: 0000000000fffff8 x19: 0000000001000000 x18: 0000000000000000 x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000 x14: 0000000000000000 x13: 0000000000000000 x12: 0000000000000000 x11: 0000000000000000 x10: ffffffffffffffff x9 : 00000000c1084259 x8 : ffff800079d7b032 x7 : 000000000000001a x6 : ffff00001090454b x5 : 0000000000000013 x4 : 0000000000000000 x3 : 0000000000000000 x2 : 00000000ffffffff x1 : 00008000e6f5a000 x0 : 0000000000000013 Call trace: __d_lookup_rcu+0x90/0x1e8 lookup_fast+0x44/0x300 walk_component+0x34/0x2e0 path_lookupat.isra.13+0x5c/0x1e0 filename_lookup.part.19+0x6c/0xe8 user_path_at_empty+0x4c/0x60 vfs_statx+0x78/0xd8 __se_sys_newfstatat+0x24/0x48 __arm64_sys_newfstatat+0x18/0x20 el0_svc_handler+0x94/0x138 el0_svc+0x8/0xc Code: 94000753 294c1fe9 9280000a f94037e8 (b85fc263) ---[ end trace 93a444e9b6bc67e9 ]--- list fucked in head Unable to handle kernel paging request at virtual address 0000880000fffffc Mem abort info: ESR = 0x96000004 Exception class = DABT (current EL), IL = 32 bits SET = 0, FnV = 0 EA = 0, S1PTW = 0 Data abort info: ISV = 0, ISS = 0x00000004 CM = 0, WnR = 0 user pgtable: 4k pages, 48-bit VAs, pgdp=000000003ba5d000 [0000880000fffffc] pgd=0000000000000000 Internal error: Oops: 96000004 [#3] SMP CPU: 2 PID: 2659 Comm: git Tainted: G D 5.2.0-rc5 #1 Hardware name: Sapphire-RK3399 Board (DT) pstate: 60000005 (nZCv daif -PAN -UAO) pc : __d_lookup_rcu+0x90/0x1e8 lr : __d_lookup_rcu+0x84/0x1e8 sp : ffff0000135cba90 x29: ffff0000135cba90 x28: ffff0000135cbb5c x27: 0000000000000000 x26: ffff0000135cbc78 x25: 00000010cb63a9bb x24: 0000880000fffff8 x23: 0000000001000000 x22: ffff80003be53180 x21: ffff80003be53180 x20: 0000880000fffff8 x19: 0000880001000000 x18: 0000000000000000 x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000 x14: 0000000000000000 x13: 0000000000000000 x12: 0000000000000000 x11: 0000000000000000 x10: ffffffffffffffff x9 : 00000000cb63a9bb x8 : ffff8000f094102e x7 : 0000000000000010 x6 : ffff00001090454b x5 : 0000000000000013 x4 : 0000000000000000 x3 : 0000000000000000 x2 : 00000000ffffffff x1 : 00008000e6f1e000 x0 : 0000000000000013 Call trace: __d_lookup_rcu+0x90/0x1e8 lookup_fast+0x44/0x300 walk_component+0x34/0x2e0 path_lookupat.isra.13+0x5c/0x1e0 filename_lookup.part.19+0x6c/0xe8 user_path_at_empty+0x4c/0x60 vfs_statx+0x78/0xd8 __se_sys_newfstatat+0x24/0x48 __arm64_sys_newfstatat+0x18/0x20 el0_svc_handler+0x94/0x138 el0_svc+0x8/0xc Code: 94000753 294c1fe9 9280000a f94037e8 (b85fc263) ---[ end trace 93a444e9b6bc67ea ]--- list fucked in head Unable to handle kernel paging request at virtual address 0000880000fffffc Mem abort info: ESR = 0x96000004 Exception class = DABT (current EL), IL = 32 bits SET = 0, FnV = 0 EA = 0, S1PTW = 0 Data abort info: ISV = 0, ISS = 0x00000004 CM = 0, WnR = 0 user pgtable: 4k pages, 48-bit VAs, pgdp=000000003ba5d000 [0000880000fffffc] pgd=0000000000000000 Internal error: Oops: 96000004 [#4] SMP CPU: 4 PID: 2658 Comm: git Tainted: G D 5.2.0-rc5 #1 Hardware name: Sapphire-RK3399 Board (DT) pstate: 60000005 (nZCv daif -PAN -UAO) pc : __d_lookup_rcu+0x90/0x1e8 lr : __d_lookup_rcu+0x84/0x1e8 sp : ffff00001363ba90 x29: ffff00001363ba90 x28: ffff00001363bb5c x27: 0000000000000000 x26: ffff00001363bc78 x25: 00000010cb63a9bb x24: 0000880000fffff8 x23: 0000000001000000 x22: ffff80003be53180 x21: ffff80003be53180 x20: 0000880000fffff8 x19: 0000880001000000 x18: 0000000000000000 x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000 x14: 0000000000000000 x13: 0000000000000000 x12: 0000000000000000 x11: 0000000000000000 x10: ffffffffffffffff x9 : 00000000cb63a9bb x8 : ffff80004b94d02e x7 : 0000000000000010 x6 : ffff00001090454b x5 : 0000000000000013 x4 : 0000000000000000 x3 : 0000000000000000 x2 : 00000000ffffffff x1 : 00008000e6f46000 x0 : 0000000000000013 Call trace: __d_lookup_rcu+0x90/0x1e8 lookup_fast+0x44/0x300 walk_component+0x34/0x2e0 path_lookupat.isra.13+0x5c/0x1e0 filename_lookup.part.19+0x6c/0xe8 user_path_at_empty+0x4c/0x60 vfs_statx+0x78/0xd8 __se_sys_newfstatat+0x24/0x48 __arm64_sys_newfstatat+0x18/0x20 el0_svc_handler+0x94/0x138 el0_svc+0x8/0xc Code: 94000753 294c1fe9 9280000a f94037e8 (b85fc263) ---[ end trace 93a444e9b6bc67eb ]--- list fucked in head Unable to handle kernel paging request at virtual address 0000000001000018 Mem abort info: ESR = 0x96000004 Exception class = DABT (current EL), IL = 32 bits SET = 0, FnV = 0 EA = 0, S1PTW = 0 Data abort info: ISV = 0, ISS = 0x00000004 CM = 0, WnR = 0 user pgtable: 4k pages, 48-bit VAs, pgdp=000000002091a000 [0000000001000018] pgd=0000000000000000 Internal error: Oops: 96000004 [#5] SMP CPU: 4 PID: 3205 Comm: update_all_gits Tainted: G D 5.2.0-rc5 #1 Hardware name: Sapphire-RK3399 Board (DT) pstate: 60000005 (nZCv daif -PAN -UAO) pc : __d_lookup+0x88/0x1d8 lr : __d_lookup+0x7c/0x1d8 sp : ffff000013dabaa0 x29: ffff000013dabaa0 x28: ffff000013dabbd8 x27: ffff00001076f0f8 x26: ffff8000f2808780 x25: 0000000000fffff8 x24: 0000000001000000 x23: 00000000cb639d51 x22: 0000000000000000 x21: 0000000000000001 x20: 0000000000fffff8 x19: 0000000001000000 x18: 0000000000000000 x17: 0000000000000000 x16: 0000000000000000 x15: ffffffffffffffff x14: ffff000010898508 x13: ffff000013dabbf8 x12: ffff000013dabbed x11: 0000000000000001 x10: ffff000013daba60 x9 : ffff000013daba60 x8 : ffff000013daba60 x7 : ffff000013daba60 x6 : ffffffffffffffff x5 : 0000000000000000 x4 : 0000000000000000 x3 : 0000000000000000 x2 : 00000000ffffffff x1 : 00008000e6f46000 x0 : 0000000000000013 Call trace: __d_lookup+0x88/0x1d8 d_lookup+0x34/0x68 d_hash_and_lookup+0x50/0x68 proc_flush_task+0x9c/0x198 release_task.part.3+0x68/0x4b8 wait_consider_task+0x91c/0x9b0 do_wait+0x120/0x1e0 kernel_wait4+0x7c/0x140 __se_sys_wait4+0x68/0xa8 __arm64_sys_wait4+0x18/0x20 el0_svc_handler+0x94/0x138 el0_svc+0x8/0xc Code: 940006db b9406fe5 92800006 d503201f (b9402282) ---[ end trace 93a444e9b6bc67ec ]--- list fucked in head Unable to handle kernel paging request at virtual address 0000880101000018 Mem abort info: ESR = 0x96000004 Exception class = DABT (current EL), IL = 32 bits SET = 0, FnV = 0 EA = 0, S1PTW = 0 Data abort info: ISV = 0, ISS = 0x00000004 CM = 0, WnR = 0 user pgtable: 4k pages, 48-bit VAs, pgdp=000000009bddd000 [0000880101000018] pgd=0000000000000000 Internal error: Oops: 96000004 [#6] SMP CPU: 5 PID: 3978 Comm: tar Tainted: G D 5.2.0-rc5 #1 Hardware name: Sapphire-RK3399 Board (DT) pstate: 60000005 (nZCv daif -PAN -UAO) pc : __d_lookup+0x88/0x1d8 lr : __d_lookup+0x7c/0x1d8 sp : ffff000014dc3b90 x29: ffff000014dc3b90 x28: ffff000014dc3d58 x27: ffff000014dc3d48 x26: ffff8000a77becc0 x25: 0000880100fffff8 x24: 0000000001000000 x23: 00000000c1086fd8 x22: 0000000000000000 x21: 0000000000000001 x20: 0000880100fffff8 x19: 0000880101000000 x18: 0000000000000000 x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000 x14: 0000000000000000 x13: 0000000000000000 x12: 0000000000000000 x11: 0000000000000000 x10: ffff000014dc3b50 x9 : ffff000014dc3b50 x8 : ffff000014dc3b50 x7 : ffff000014dc3b50 x6 : ffffffffffffffff x5 : 0000000000000000 x4 : 0000000000000000 x3 : 0000000000000000 x2 : 00000000ffffffff x1 : 00008000e6f5a000 x0 : 0000000000000013 Call trace: __d_lookup+0x88/0x1d8 d_lookup+0x34/0x68 path_openat+0x528/0xfd0 do_filp_open+0x60/0xc0 do_sys_open+0x164/0x200 __arm64_sys_openat+0x20/0x28 el0_svc_handler+0x94/0x138 el0_svc+0x8/0xc Code: 940006db b9406fe5 92800006 d503201f (b9402282) ---[ end trace 93a444e9b6bc67ed ]---
0000000000002d10 <__d_lookup_rcu>: { 2d10: a9b97bfd stp x29, x30, [sp, #-112]! return dentry_hashtable + (hash >> d_hash_shift); 2d14: 90000003 adrp x3, 0 <d_shrink_del> 2d18: 91000065 add x5, x3, #0x0 { 2d1c: 910003fd mov x29, sp 2d20: a90153f3 stp x19, x20, [sp, #16] 2d24: a90363f7 stp x23, x24, [sp, #48] 2d28: a9046bf9 stp x25, x26, [sp, #64] const unsigned char *str = name->name; 2d2c: a9402039 ldp x25, x8, [x1] return dentry_hashtable + (hash >> d_hash_shift); 2d30: f9400064 ldr x4, [x3] 2d34: b94008a3 ldr w3, [x5, #8] 2d38: 1ac32723 lsr w3, w25, w3 __READ_ONCE_SIZE; 2d3c: f8637893 ldr x19, [x4, x3, lsl #3] hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { 2d40: f27ffa73 ands x19, x19, #0xfffffffffffffffe 2d44: 54000420 b.eq 2dc8 <__d_lookup_rcu+0xb8> // b.none if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0) 2d48: d360ff27 lsr x7, x25, #32 2d4c: 2a1903e9 mov w9, w25 2d50: aa0103fa mov x26, x1 2d54: a90573fb stp x27, x28, [sp, #80] 2d58: aa0203fc mov x28, x2 2d5c: 120008fb and w27, w7, #0x7 if (unlikely((u32)(unsigned long)&dentry->d_hash == 0x01000000)) 2d60: 52a02017 mov w23, #0x1000000 // #16777216 mask = bytemask_from_count(tcount); 2d64: 9280000a mov x10, #0xffffffffffffffff // #-1 2d68: a9025bf5 stp x21, x22, [sp, #32] 2d6c: aa0003f5 mov x21, x0 struct dentry *dentry, *last = NULL; 2d70: d2800000 mov x0, #0x0 // #0 2d74: d503201f nop hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { 2d78: d1002274 sub x20, x19, #0x8 if (unlikely((u32)(unsigned long)&dentry->d_hash == 0x01000000)) 2d7c: 6b17027f cmp w19, w23 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { 2d80: aa1403f8 mov x24, x20 if (unlikely((u32)(unsigned long)&dentry->d_hash == 0x01000000)) 2d84: 540000e1 b.ne 2da0 <__d_lookup_rcu+0x90> // b.any 2d88: 290c1fe9 stp w9, w7, [sp, #96] 2d8c: f90037e8 str x8, [sp, #104] dump(last); 2d90: 94000000 bl 0 <d_shrink_del> 2d94: 294c1fe9 ldp w9, w7, [sp, #96] 2d98: 9280000a mov x10, #0xffffffffffffffff // #-1 2d9c: f94037e8 ldr x8, [sp, #104] 2da0: b85fc263 ldur w3, [x19, #-4] smp_rmb(); 2da4: d50339bf dmb ishld if (dentry->d_parent != parent) 2da8: f9400e80 ldr x0, [x20, #24] 2dac: eb15001f cmp x0, x21 2db0: 540001a0 b.eq 2de4 <__d_lookup_rcu+0xd4> // b.none 2db4: f9400273 ldr x19, [x19] hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { 2db8: aa1403e0 mov x0, x20 2dbc: b5fffdf3 cbnz x19, 2d78 <__d_lookup_rcu+0x68> 2dc0: a9425bf5 ldp x21, x22, [sp, #32] 2dc4: a94573fb ldp x27, x28, [sp, #80] return NULL; 2dc8: d2800018 mov x24, #0x0 // #0 } 2dcc: aa1803e0 mov x0, x24 2dd0: a94153f3 ldp x19, x20, [sp, #16] 2dd4: a94363f7 ldp x23, x24, [sp, #48] 2dd8: a9446bf9 ldp x25, x26, [sp, #64] 2ddc: a8c77bfd ldp x29, x30, [sp], #112 2de0: d65f03c0 ret if (d_unhashed(dentry)) 2de4: f9400660 ldr x0, [x19, #8] 2de8: b4fffe60 cbz x0, 2db4 <__d_lookup_rcu+0xa4> if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) { 2dec: b94002a0 ldr w0, [x21] return ret & ~1; 2df0: 121f7876 and w22, w3, #0xfffffffe 2df4: 370804a0 tbnz w0, #1, 2e88 <__d_lookup_rcu+0x178> if (dentry->d_name.hash_len != hashlen) 2df8: f9401280 ldr x0, [x20, #32] 2dfc: eb19001f cmp x0, x25 2e00: 54fffda1 b.ne 2db4 <__d_lookup_rcu+0xa4> // b.any 2e04: f9401264 ldr x4, [x19, #32] const unsigned char *cs = READ_ONCE(dentry->d_name.name); 2e08: 2a0703e5 mov w5, w7 2e0c: cb040101 sub x1, x8, x4 2e10: 14000006 b 2e28 <__d_lookup_rcu+0x118> cs += sizeof(unsigned long); 2e14: 91002084 add x4, x4, #0x8 if (unlikely(a != b)) 2e18: eb06001f cmp x0, x6 2e1c: 54fffcc1 b.ne 2db4 <__d_lookup_rcu+0xa4> // b.any if (!tcount) 2e20: 710020a5 subs w5, w5, #0x8 2e24: 54000160 b.eq 2e50 <__d_lookup_rcu+0x140> // b.none cs += sizeof(unsigned long); 2e28: 8b010083 add x3, x4, x1 if (tcount < sizeof(unsigned long)) 2e2c: 6b1b00bf cmp w5, w27 static inline unsigned long load_unaligned_zeropad(const void *addr) { unsigned long ret, offset;
/* Load word from unaligned pointer addr */ asm( 2e30: f9400066 ldr x6, [x3]
static __no_kasan_or_inline unsigned long read_word_at_a_time(const void *addr) { kasan_check_read(addr, 1); return *(unsigned long *)addr; 2e34: f9400080 ldr x0, [x4] 2e38: 54fffee1 b.ne 2e14 <__d_lookup_rcu+0x104> // b.any mask = bytemask_from_count(tcount); 2e3c: 531d7361 lsl w1, w27, #3 return unlikely(!!((a ^ b) & mask)); 2e40: ca060000 eor x0, x0, x6 mask = bytemask_from_count(tcount); 2e44: 9ac12141 lsl x1, x10, x1 if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0) 2e48: ea21001f bics xzr, x0, x1 2e4c: 54fffb41 b.ne 2db4 <__d_lookup_rcu+0xa4> // b.any *seqp = seq; 2e50: b9000396 str w22, [x28] } 2e54: aa1803e0 mov x0, x24 2e58: a94153f3 ldp x19, x20, [sp, #16] return dentry; 2e5c: a9425bf5 ldp x21, x22, [sp, #32] } 2e60: a94363f7 ldp x23, x24, [sp, #48] 2e64: a9446bf9 ldp x25, x26, [sp, #64] return dentry; 2e68: a94573fb ldp x27, x28, [sp, #80] } 2e6c: a8c77bfd ldp x29, x30, [sp], #112 2e70: d65f03c0 ret if (d_unhashed(dentry)) 2e74: f9400660 ldr x0, [x19, #8] 2e78: 121f7876 and w22, w3, #0xfffffffe 2e7c: b4fff9c0 cbz x0, 2db4 <__d_lookup_rcu+0xa4> if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) { 2e80: b94002a0 ldr w0, [x21] 2e84: 360ffba0 tbz w0, #1, 2df8 <__d_lookup_rcu+0xe8> if (dentry->d_name.hash != hashlen_hash(hashlen)) 2e88: b9402280 ldr w0, [x20, #32] 2e8c: 6b00013f cmp w9, w0 2e90: 54fff921 b.ne 2db4 <__d_lookup_rcu+0xa4> // b.any tlen = dentry->d_name.len; 2e94: b9402681 ldr w1, [x20, #36] tname = dentry->d_name.name; 2e98: f9401682 ldr x2, [x20, #40] smp_rmb(); 2e9c: d50339bf dmb ishld return unlikely(s->sequence != start); 2ea0: b85fc260 ldur w0, [x19, #-4] if (read_seqcount_retry(&dentry->d_seq, seq)) { 2ea4: 6b0002df cmp w22, w0 2ea8: 54000100 b.eq 2ec8 <__d_lookup_rcu+0x1b8> // b.none 2eac: d503203f yield __READ_ONCE_SIZE; 2eb0: b85fc263 ldur w3, [x19, #-4] smp_rmb(); 2eb4: d50339bf dmb ishld if (dentry->d_parent != parent) 2eb8: f9400e80 ldr x0, [x20, #24] 2ebc: eb15001f cmp x0, x21 2ec0: 54fff7a1 b.ne 2db4 <__d_lookup_rcu+0xa4> // b.any 2ec4: 17ffffec b 2e74 <__d_lookup_rcu+0x164> if (parent->d_op->d_compare(dentry, 2ec8: f94032a4 ldr x4, [x21, #96] 2ecc: aa1a03e3 mov x3, x26 2ed0: aa1403e0 mov x0, x20 2ed4: 290c1fe9 stp w9, w7, [sp, #96] 2ed8: f90037e8 str x8, [sp, #104] 2edc: f9400c84 ldr x4, [x4, #24] 2ee0: d63f0080 blr x4 2ee4: 9280000a mov x10, #0xffffffffffffffff // #-1 2ee8: 294c1fe9 ldp w9, w7, [sp, #96] 2eec: f94037e8 ldr x8, [sp, #104] 2ef0: 34fffb00 cbz w0, 2e50 <__d_lookup_rcu+0x140> 2ef4: 17ffffb0 b 2db4 <__d_lookup_rcu+0xa4>
0000000000002ef8 <__d_lookup>: { 2ef8: a9b97bfd stp x29, x30, [sp, #-112]! return dentry_hashtable + (hash >> d_hash_shift); 2efc: 90000002 adrp x2, 0 <d_shrink_del> 2f00: 91000044 add x4, x2, #0x0 { 2f04: 910003fd mov x29, sp 2f08: a90153f3 stp x19, x20, [sp, #16] 2f0c: a90363f7 stp x23, x24, [sp, #48] 2f10: a9046bf9 stp x25, x26, [sp, #64] return dentry_hashtable + (hash >> d_hash_shift); 2f14: f9400043 ldr x3, [x2] unsigned int hash = name->hash; 2f18: b9400037 ldr w23, [x1] return dentry_hashtable + (hash >> d_hash_shift); 2f1c: b9400882 ldr w2, [x4, #8] 2f20: 1ac226e2 lsr w2, w23, w2 2f24: f8627873 ldr x19, [x3, x2, lsl #3] hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { 2f28: f27ffa73 ands x19, x19, #0xfffffffffffffffe 2f2c: 54000520 b.eq 2fd0 <__d_lookup+0xd8> // b.none 2f30: aa0003fa mov x26, x0 2f34: a90573fb stp x27, x28, [sp, #80] 2f38: aa0103fc mov x28, x1 2f3c: d2800002 mov x2, #0x0 // #0 if (unlikely((u32)(unsigned long)&dentry->d_hash == 0x01000000)) 2f40: 52a02018 mov w24, #0x1000000 // #16777216 smp_store_release(&lock->locked, 0); 2f44: 52800005 mov w5, #0x0 // #0 mask = bytemask_from_count(tcount); 2f48: 92800006 mov x6, #0xffffffffffffffff // #-1 2f4c: a9025bf5 stp x21, x22, [sp, #32] 2f50: d2800016 mov x22, #0x0 // #0 2f54: 52800035 mov w21, #0x1 // #1 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { 2f58: d1002274 sub x20, x19, #0x8 if (unlikely((u32)(unsigned long)&dentry->d_hash == 0x01000000)) 2f5c: 6b18027f cmp w19, w24 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { 2f60: aa1403f9 mov x25, x20 if (unlikely((u32)(unsigned long)&dentry->d_hash == 0x01000000)) 2f64: 540000e1 b.ne 2f80 <__d_lookup+0x88> // b.any dump(last); 2f68: aa0203e0 mov x0, x2 2f6c: b9006fe5 str w5, [sp, #108] 2f70: 94000000 bl 0 <d_shrink_del> 2f74: b9406fe5 ldr w5, [sp, #108] 2f78: 92800006 mov x6, #0xffffffffffffffff // #-1 2f7c: d503201f nop if (dentry->d_name.hash != hash) 2f80: b9402282 ldr w2, [x20, #32] 2f84: 6b17005f cmp w2, w23 2f88: 540001a1 b.ne 2fbc <__d_lookup+0xc4> // b.any 2f8c: 9101427b add x27, x19, #0x50 2f90: f9800371 prfm pstl1strm, [x27] 2f94: 885fff61 ldaxr w1, [x27] 2f98: 4a160020 eor w0, w1, w22 2f9c: 35000060 cbnz w0, 2fa8 <__d_lookup+0xb0> 2fa0: 88007f75 stxr w0, w21, [x27] 2fa4: 35ffff80 cbnz w0, 2f94 <__d_lookup+0x9c> 2fa8: 35000521 cbnz w1, 304c <__d_lookup+0x154> if (dentry->d_parent != parent) 2fac: f9400e80 ldr x0, [x20, #24] 2fb0: eb1a001f cmp x0, x26 2fb4: 540001c0 b.eq 2fec <__d_lookup+0xf4> // b.none 2fb8: 089fff65 stlrb w5, [x27] 2fbc: f9400273 ldr x19, [x19] hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { 2fc0: aa1403e2 mov x2, x20 2fc4: b5fffcb3 cbnz x19, 2f58 <__d_lookup+0x60> 2fc8: a9425bf5 ldp x21, x22, [sp, #32] 2fcc: a94573fb ldp x27, x28, [sp, #80] struct dentry *found = NULL; 2fd0: d2800019 mov x25, #0x0 // #0 } 2fd4: aa1903e0 mov x0, x25 2fd8: a94153f3 ldp x19, x20, [sp, #16] 2fdc: a94363f7 ldp x23, x24, [sp, #48] 2fe0: a9446bf9 ldp x25, x26, [sp, #64] 2fe4: a8c77bfd ldp x29, x30, [sp], #112 2fe8: d65f03c0 ret if (d_unhashed(dentry)) 2fec: f9400660 ldr x0, [x19, #8] 2ff0: b4fffe40 cbz x0, 2fb8 <__d_lookup+0xc0> if (likely(!(parent->d_flags & DCACHE_OP_COMPARE))) { 2ff4: b9400340 ldr w0, [x26] 2ff8: b9402681 ldr w1, [x20, #36] 2ffc: 37080340 tbnz w0, #1, 3064 <__d_lookup+0x16c> if (dentry->d_name.len != name->len) 3000: b9400783 ldr w3, [x28, #4] 3004: 6b01007f cmp w3, w1 3008: 54fffd81 b.ne 2fb8 <__d_lookup+0xc0> // b.any return dentry_cmp(dentry, name->name, name->len) == 0; 300c: f9400787 ldr x7, [x28, #8] static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount) 3010: 12000868 and w8, w3, #0x7 3014: f9401264 ldr x4, [x19, #32] 3018: cb0400e7 sub x7, x7, x4 301c: 14000003 b 3028 <__d_lookup+0x130> if (!tcount) 3020: 71002063 subs w3, w3, #0x8 3024: 54000380 b.eq 3094 <__d_lookup+0x19c> // b.none cs += sizeof(unsigned long); 3028: 8b070082 add x2, x4, x7 if (tcount < sizeof(unsigned long)) 302c: 6b08007f cmp w3, w8 3030: f9400041 ldr x1, [x2] return *(unsigned long *)addr; 3034: f9400080 ldr x0, [x4] 3038: 540003e0 b.eq 30b4 <__d_lookup+0x1bc> // b.none cs += sizeof(unsigned long); 303c: 91002084 add x4, x4, #0x8 if (unlikely(a != b)) 3040: eb01001f cmp x0, x1 3044: 54fffee0 b.eq 3020 <__d_lookup+0x128> // b.none 3048: 17ffffdc b 2fb8 <__d_lookup+0xc0> queued_spin_lock_slowpath(lock, val); 304c: aa1b03e0 mov x0, x27 3050: b9006fe5 str w5, [sp, #108] 3054: 94000000 bl 0 <queued_spin_lock_slowpath> 3058: b9406fe5 ldr w5, [sp, #108] 305c: 92800006 mov x6, #0xffffffffffffffff // #-1 3060: 17ffffd3 b 2fac <__d_lookup+0xb4> return parent->d_op->d_compare(dentry, 3064: f9403340 ldr x0, [x26, #96] 3068: aa1c03e3 mov x3, x28 306c: f9401682 ldr x2, [x20, #40] 3070: b9006fe5 str w5, [sp, #108] 3074: f9400c04 ldr x4, [x0, #24] 3078: aa1403e0 mov x0, x20 307c: d63f0080 blr x4 name) == 0; 3080: 7100001f cmp w0, #0x0 3084: 1a9f17e0 cset w0, eq // eq = none 3088: 92800006 mov x6, #0xffffffffffffffff // #-1 308c: b9406fe5 ldr w5, [sp, #108] if (!d_same_name(dentry, parent, name)) 3090: 34fff940 cbz w0, 2fb8 <__d_lookup+0xc0> dentry->d_lockref.count++; 3094: b9405e80 ldr w0, [x20, #92] smp_store_release(&lock->locked, 0); 3098: 52800001 mov w1, #0x0 // #0 309c: 11000400 add w0, w0, #0x1 30a0: b9005e80 str w0, [x20, #92] 30a4: 089fff61 stlrb w1, [x27] preempt_enable(); 30a8: a9425bf5 ldp x21, x22, [sp, #32] 30ac: a94573fb ldp x27, x28, [sp, #80] 30b0: 17ffffc9 b 2fd4 <__d_lookup+0xdc> mask = bytemask_from_count(tcount); 30b4: 531d7062 lsl w2, w3, #3 return unlikely(!!((a ^ b) & mask)); 30b8: ca010000 eor x0, x0, x1 mask = bytemask_from_count(tcount); 30bc: 9ac220c2 lsl x2, x6, x2 30c0: ea22001f bics xzr, x0, x2 30c4: 1a9f17e0 cset w0, eq // eq = none if (!d_same_name(dentry, parent, name)) 30c8: 34fff780 cbz w0, 2fb8 <__d_lookup+0xc0> 30cc: 17fffff2 b 3094 <__d_lookup+0x19c>
00000000000030d0 <d_lookup>: { 30d0: a9bd7bfd stp x29, x30, [sp, #-48]! 30d4: 910003fd mov x29, sp 30d8: a90153f3 stp x19, x20, [sp, #16] 30dc: 90000014 adrp x20, 0 <d_shrink_del> 30e0: 91000294 add x20, x20, #0x0 30e4: a9025bf5 stp x21, x22, [sp, #32] 30e8: aa0003f6 mov x22, x0 30ec: aa0103f5 mov x21, x1 30f0: 1400000a b 3118 <d_lookup+0x48> smp_rmb(); 30f4: d50339bf dmb ishld dentry = __d_lookup(parent, name); 30f8: aa1503e1 mov x1, x21 30fc: aa1603e0 mov x0, x22 3100: 94000000 bl 2ef8 <__d_lookup> if (dentry) 3104: b5000120 cbnz x0, 3128 <d_lookup+0x58> smp_rmb(); 3108: d50339bf dmb ishld } while (read_seqretry(&rename_lock, seq)); 310c: b9400281 ldr w1, [x20] 3110: 6b13003f cmp w1, w19 3114: 540000a0 b.eq 3128 <d_lookup+0x58> // b.none __READ_ONCE_SIZE; 3118: b9400293 ldr w19, [x20] if (unlikely(ret & 1)) { 311c: 3607fed3 tbz w19, #0, 30f4 <d_lookup+0x24> 3120: d503203f yield 3124: 17fffffd b 3118 <d_lookup+0x48> } 3128: a94153f3 ldp x19, x20, [sp, #16] 312c: a9425bf5 ldp x21, x22, [sp, #32] 3130: a8c37bfd ldp x29, x30, [sp], #48 3134: d65f03c0 ret
0000000000003138 <d_hash_and_lookup>: { 3138: a9be7bfd stp x29, x30, [sp, #-32]! 313c: 910003fd mov x29, sp 3140: a90153f3 stp x19, x20, [sp, #16] 3144: aa0103f3 mov x19, x1 3148: aa0003f4 mov x20, x0 name->hash = full_name_hash(dir, name->name, name->len); 314c: b9400422 ldr w2, [x1, #4] 3150: f9400421 ldr x1, [x1, #8] 3154: 94000000 bl 0 <full_name_hash> 3158: b9000260 str w0, [x19] if (dir->d_flags & DCACHE_OP_HASH) { 315c: b9400280 ldr w0, [x20] 3160: 360000e0 tbz w0, #0, 317c <d_hash_and_lookup+0x44> int err = dir->d_op->d_hash(dir, name); 3164: f9403282 ldr x2, [x20, #96] 3168: aa1303e1 mov x1, x19 316c: aa1403e0 mov x0, x20 3170: f9400842 ldr x2, [x2, #16] 3174: d63f0040 blr x2 if (unlikely(err < 0)) 3178: 37f800e0 tbnz w0, #31, 3194 <d_hash_and_lookup+0x5c> return d_lookup(dir, name); 317c: aa1303e1 mov x1, x19 3180: aa1403e0 mov x0, x20 3184: 94000000 bl 30d0 <d_lookup> } 3188: a94153f3 ldp x19, x20, [sp, #16] 318c: a8c27bfd ldp x29, x30, [sp], #32 3190: d65f03c0 ret return ERR_PTR(err); 3194: 93407c00 sxtw x0, w0 3198: 17fffffc b 3188 <d_hash_and_lookup+0x50> 319c: d503201f nop
Disassembly of section .text.unlikely:
0000000000000000 <dump>: { 0: a9bc7bfd stp x29, x30, [sp, #-64]! 4: 910003fd mov x29, sp 8: a90153f3 stp x19, x20, [sp, #16] c: a9025bf5 stp x21, x22, [sp, #32] 10: f9001bf7 str x23, [sp, #48] if (!dentry) { 14: b50000a0 cbnz x0, 28 <dump+0x28> printk(KERN_ERR "list fucked in head"); 18: 90000000 adrp x0, 0 <dump> 1c: 91000000 add x0, x0, #0x0 20: 94000000 bl 0 <printk> return; 24: 14000016 b 7c <dump+0x7c> printk(KERN_ERR "fucked dentry[%p]: d_hash.next = %p, flags = %x, count = %d", 28: aa0003f3 mov x19, x0 printk(KERN_CONT "%c%02x", i & 31 ? ' ' : '\n', 2c: 90000015 adrp x21, 0 <dump> printk(KERN_ERR "fucked dentry[%p]: d_hash.next = %p, flags = %x, count = %d", 30: 90000000 adrp x0, 0 <dump> 34: aa1303e1 mov x1, x19 38: 91000000 add x0, x0, #0x0 3c: d2800014 mov x20, #0x0 // #0 40: b9400263 ldr w3, [x19] printk(KERN_CONT "%c%02x", i & 31 ? ' ' : '\n', 44: 52800157 mov w23, #0xa // #10 printk(KERN_ERR "fucked dentry[%p]: d_hash.next = %p, flags = %x, count = %d", 48: b9405e64 ldr w4, [x19, #92] printk(KERN_CONT "%c%02x", i & 31 ? ' ' : '\n', 4c: 52800416 mov w22, #0x20 // #32 printk(KERN_ERR "fucked dentry[%p]: d_hash.next = %p, flags = %x, count = %d", 50: f9400662 ldr x2, [x19, #8] printk(KERN_CONT "%c%02x", i & 31 ? ' ' : '\n', 54: 910002b5 add x21, x21, #0x0 printk(KERN_ERR "fucked dentry[%p]: d_hash.next = %p, flags = %x, count = %d", 58: 94000000 bl 0 <printk> printk(KERN_CONT "%c%02x", i & 31 ? ' ' : '\n', 5c: 38746a62 ldrb w2, [x19, x20] 60: f240129f tst x20, #0x1f 64: 1a9602e1 csel w1, w23, w22, eq // eq = none 68: 91000694 add x20, x20, #0x1 6c: aa1503e0 mov x0, x21 70: 94000000 bl 0 <printk> for (i = 0; i < sizeof(struct dentry); i++) 74: f103029f cmp x20, #0xc0 78: 54ffff21 b.ne 5c <dump+0x5c> // b.any } 7c: a94153f3 ldp x19, x20, [sp, #16] 80: a9425bf5 ldp x21, x22, [sp, #32] 84: f9401bf7 ldr x23, [sp, #48] 88: a8c47bfd ldp x29, x30, [sp], #64 8c: d65f03c0 ret
| |