lkml.org 
[lkml]   [2010]   [Jan]   [11]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    /
    Date
    From
    SubjectRe: [RFC] [PATCH 4/7] Uprobes Implementation
    On Mon, Jan 11, 2010 at 05:55:53PM +0530, Srikar Dronamraju wrote:
    > Uprobes Implementation
    >
    > Uprobes Infrastructure enables user to dynamically establish
    > probepoints in user applications and collect information by executing
    > a handler functions when the probepoints are hit.
    > Please refer Documentation/uprobes.txt for more details.
    >
    > This patch provides the core implementation of uprobes.
    > This patch builds on utrace infrastructure.
    >
    > You need to follow this up with the uprobes patch for your
    > architecture.

    Good to see this!!! Several questions interspersed below.

    Thanx, Paul

    > Signed-off-by: Jim Keniston <jkenisto@us.ibm.com>
    > Signed-off-by: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
    > ---
    > arch/Kconfig | 12
    > include/linux/uprobes.h | 292 ++++++
    > kernel/Makefile | 1
    > kernel/uprobes_core.c | 2017 ++++++++++++++++++++++++++++++++++++++++++++++++
    > 4 files changed, 2322 insertions(+)
    >
    > Index: new_uprobes.git/arch/Kconfig
    > ===================================================================
    > --- new_uprobes.git.orig/arch/Kconfig
    > +++ new_uprobes.git/arch/Kconfig
    > @@ -66,6 +66,16 @@ config UBP
    > in user applications. This service is used by components
    > such as uprobes. If in doubt, say "N".
    >
    > +config UPROBES
    > + bool "User-space probes (EXPERIMENTAL)"
    > + depends on UTRACE && MODULES && UBP
    > + depends on HAVE_UPROBES
    > + help
    > + Uprobes enables kernel modules to establish probepoints
    > + in user applications and execute handler functions when
    > + the probepoints are hit. For more information, refer to
    > + Documentation/uprobes.txt. If in doubt, say "N".
    > +
    > config HAVE_EFFICIENT_UNALIGNED_ACCESS
    > bool
    > help
    > @@ -115,6 +125,8 @@ config HAVE_KPROBES
    > config HAVE_KRETPROBES
    > bool
    >
    > +config HAVE_UPROBES
    > + def_bool n
    > #
    > # An arch should select this if it provides all these things:
    > #
    > Index: new_uprobes.git/include/linux/uprobes.h
    > ===================================================================
    > --- /dev/null
    > +++ new_uprobes.git/include/linux/uprobes.h
    > @@ -0,0 +1,292 @@
    > +#ifndef _LINUX_UPROBES_H
    > +#define _LINUX_UPROBES_H
    > +/*
    > + * Userspace Probes (UProbes)
    > + * include/linux/uprobes.h
    > + *
    > + * This program is free software; you can redistribute it and/or modify
    > + * it under the terms of the GNU General Public License as published by
    > + * the Free Software Foundation; either version 2 of the License, or
    > + * (at your option) any later version.
    > + *
    > + * This program is distributed in the hope that it will be useful,
    > + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    > + * GNU General Public License for more details.
    > + *
    > + * You should have received a copy of the GNU General Public License
    > + * along with this program; if not, write to the Free Software
    > + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
    > + *
    > + * Copyright (C) IBM Corporation, 2006, 2009
    > + */
    > +#include <linux/types.h>
    > +#include <linux/list.h>
    > +
    > +struct pt_regs;
    > +
    > +/* This is what the user supplies us. */
    > +struct uprobe {
    > + /*
    > + * The pid of the probed process. Currently, this can be the
    > + * thread ID (task->pid) of any active thread in the process.
    > + */
    > + pid_t pid;
    > +
    > + /* Location of the probepoint */
    > + unsigned long vaddr;
    > +
    > + /* Handler to run when the probepoint is hit */
    > + void (*handler)(struct uprobe*, struct pt_regs*);
    > +
    > + /*
    > + * This function, if non-NULL, will be called upon completion of
    > + * an ASYNCHRONOUS registration (i.e., one initiated by a uprobe
    > + * handler). reg = 1 for register, 0 for unregister.
    > + */
    > + void (*registration_callback)(struct uprobe *u, int reg, int result);
    > +
    > + /* Reserved for use by uprobes */
    > + void *kdata;
    > +};
    > +
    > +#if defined(CONFIG_UPROBES)
    > +extern int register_uprobe(struct uprobe *u);
    > +extern void unregister_uprobe(struct uprobe *u);
    > +#else
    > +static inline int register_uprobe(struct uprobe *u)
    > +{
    > + return -ENOSYS;
    > +}
    > +static inline void unregister_uprobe(struct uprobe *u)
    > +{
    > +}
    > +#endif /* CONFIG_UPROBES */
    > +
    > +#ifdef UPROBES_IMPLEMENTATION
    > +
    > +#include <linux/mutex.h>
    > +#include <linux/rwsem.h>
    > +#include <linux/wait.h>
    > +#include <asm/atomic.h>
    > +#include <linux/ubp.h>
    > +#include <linux/ubp_xol.h>
    > +#include <asm/uprobes.h>
    > +
    > +struct utrace_engine;
    > +struct task_struct;
    > +struct pid;
    > +
    > +enum uprobe_probept_state {
    > + UPROBE_INSERTING, /* process quiescing prior to insertion */
    > + UPROBE_BP_SET, /* breakpoint in place */
    > + UPROBE_REMOVING, /* process quiescing prior to removal */
    > + UPROBE_DISABLED /* removal completed */
    > +};
    > +
    > +enum uprobe_task_state {
    > + UPTASK_QUIESCENT,
    > + UPTASK_SLEEPING, /* See utask_fake_quiesce(). */
    > + UPTASK_RUNNING,
    > + UPTASK_BP_HIT,
    > + UPTASK_SSTEP
    > +};
    > +
    > +enum uprobe_ssil_state {
    > + SSIL_DISABLE,
    > + SSIL_CLEAR,
    > + SSIL_SET
    > +};
    > +
    > +#define UPROBE_HASH_BITS 5
    > +#define UPROBE_TABLE_SIZE (1 << UPROBE_HASH_BITS)
    > +
    > +/*
    > + * uprobe_process -- not a user-visible struct.
    > + * A uprobe_process represents a probed process. A process can have
    > + * multiple probepoints (each represented by a uprobe_probept) and
    > + * one or more threads (each represented by a uprobe_task).
    > + */
    > +struct uprobe_process {
    > + /*
    > + * rwsem is write-locked for any change to the uprobe_process's
    > + * graph (including uprobe_tasks, uprobe_probepts, and uprobe_kimgs) --
    > + * e.g., due to probe [un]registration or special events like exit.
    > + * It's read-locked during the whole time we process a probepoint hit.
    > + */
    > + struct rw_semaphore rwsem;
    > +
    > + /* Table of uprobe_probepts registered for this process */
    > + /* TODO: Switch to list_head[] per Ingo. */
    > + struct hlist_head uprobe_table[UPROBE_TABLE_SIZE];
    > +
    > + /* List of uprobe_probepts awaiting insertion or removal */
    > + struct list_head pending_uprobes;
    > +
    > + /* List of uprobe_tasks in this task group */
    > + struct list_head thread_list;
    > + int nthreads;
    > + int n_quiescent_threads;
    > +
    > + /* this goes on the uproc_table */
    > + struct hlist_node hlist;
    > +
    > + /*
    > + * All threads (tasks) in a process share the same uprobe_process.
    > + */
    > + struct pid *tg_leader;
    > + pid_t tgid;
    > +
    > + /* Threads in UTASK_SLEEPING state wait here to be roused. */
    > + wait_queue_head_t waitq;
    > +
    > + /*
    > + * We won't free the uprobe_process while...
    > + * - any register/unregister operations on it are in progress; or
    > + * - any uprobe_report_* callbacks are running; or
    > + * - uprobe_table[] is not empty; or
    > + * - any tasks are UTASK_SLEEPING in the waitq;
    > + * refcount reflects this. We do NOT ref-count tasks (threads),
    > + * since once the last thread has exited, the rest is academic.
    > + */
    > + atomic_t refcount;
    > +
    > + /*
    > + * finished = 1 means the process is execing or the last thread
    > + * is exiting, and we're cleaning up the uproc. If the execed
    > + * process is probed, a new uproc will be created.
    > + */
    > + bool finished;
    > +
    > + /*
    > + * 1 to single-step out of line; 0 for inline. This can drop to
    > + * 0 if we can't set up the XOL area, but never goes from 0 to 1.
    > + */
    > + bool sstep_out_of_line;
    > +
    > + /*
    > + * Manages slots for instruction-copies to be single-stepped
    > + * out of line.
    > + */
    > + void *xol_area;
    > +};
    > +
    > +/*
    > + * uprobe_kimg -- not a user-visible struct.
    > + * Holds implementation-only per-uprobe data.
    > + * uprobe->kdata points to this.
    > + */
    > +struct uprobe_kimg {
    > + struct uprobe *uprobe;
    > + struct uprobe_probept *ppt;
    > +
    > + /*
    > + * -EBUSY while we're waiting for all threads to quiesce so the
    > + * associated breakpoint can be inserted or removed.
    > + * 0 if the the insert/remove operation has succeeded, or -errno
    > + * otherwise.
    > + */
    > + int status;
    > +
    > + /* on ppt's list */
    > + struct list_head list;
    > +};
    > +
    > +/*
    > + * uprobe_probept -- not a user-visible struct.
    > + * A probepoint, at which several uprobes can be registered.
    > + * Guarded by uproc->rwsem.
    > + */
    > +struct uprobe_probept {
    > + /* breakpoint/XOL details */
    > + struct ubp_bkpt ubp;
    > +
    > + /* The uprobe_kimg(s) associated with this uprobe_probept */
    > + struct list_head uprobe_list;
    > +
    > + enum uprobe_probept_state state;
    > +
    > + /* The parent uprobe_process */
    > + struct uprobe_process *uproc;
    > +
    > + /*
    > + * ppt goes in the uprobe_process->uprobe_table when registered --
    > + * even before the breakpoint has been inserted.
    > + */
    > + struct hlist_node ut_node;
    > +
    > + /*
    > + * ppt sits in the uprobe_process->pending_uprobes queue while
    > + * awaiting insertion or removal of the breakpoint.
    > + */
    > + struct list_head pd_node;
    > +
    > + /* [un]register_uprobe() waits 'til bkpt inserted/removed */
    > + wait_queue_head_t waitq;
    > +
    > + /*
    > + * ssil_lock, ssilq and ssil_state are used to serialize
    > + * single-stepping inline, so threads don't clobber each other
    > + * swapping the breakpoint instruction in and out. This helps
    > + * prevent crashing the probed app, but it does NOT prevent
    > + * probe misses while the breakpoint is swapped out.
    > + * ssilq - threads wait for their chance to single-step inline.
    > + */
    > + spinlock_t ssil_lock;
    > + wait_queue_head_t ssilq;
    > + enum uprobe_ssil_state ssil_state;
    > +};
    > +
    > +/*
    > + * uprobe_utask -- not a user-visible struct.
    > + * Corresponds to a thread in a probed process.
    > + * Guarded by uproc->rwsem.
    > + */
    > +struct uprobe_task {
    > + /* Lives in the global utask_table */
    > + struct hlist_node hlist;
    > +
    > + /* Lives on the thread_list for the uprobe_process */
    > + struct list_head list;
    > +
    > + struct task_struct *tsk;
    > + struct pid *pid;
    > +
    > + /* The utrace engine for this task */
    > + struct utrace_engine *engine;
    > +
    > + /* Back pointer to the associated uprobe_process */
    > + struct uprobe_process *uproc;
    > +
    > + enum uprobe_task_state state;
    > +
    > + /*
    > + * quiescing = 1 means this task has been asked to quiesce.
    > + * It may not be able to comply immediately if it's hit a bkpt.
    > + */
    > + bool quiescing;
    > +
    > + /* Set before running handlers; cleared after single-stepping. */
    > + struct uprobe_probept *active_probe;
    > +
    > + /* Saved address of copied original instruction */
    > + long singlestep_addr;
    > +
    > + struct ubp_task_arch_info arch_info;
    > +
    > + /*
    > + * Unexpected error in probepoint handling has left task's
    > + * text or stack corrupted. Kill task ASAP.
    > + */
    > + bool doomed;
    > +
    > + /* [un]registrations initiated by handlers must be asynchronous. */
    > + struct list_head deferred_registrations;
    > +
    > + /* Delay handler-destined signals 'til after single-step done. */
    > + struct list_head delayed_signals;
    > +};
    > +
    > +#endif /* UPROBES_IMPLEMENTATION */
    > +
    > +#endif /* _LINUX_UPROBES_H */
    > Index: new_uprobes.git/kernel/Makefile
    > ===================================================================
    > --- new_uprobes.git.orig/kernel/Makefile
    > +++ new_uprobes.git/kernel/Makefile
    > @@ -104,6 +104,7 @@ obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_b
    > obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o
    > obj-$(CONFIG_UBP) += ubp_core.o
    > obj-$(CONFIG_UBP_XOL) += ubp_xol.o
    > +obj-$(CONFIG_UPROBES) += uprobes_core.o
    >
    > ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y)
    > # According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is
    > Index: new_uprobes.git/kernel/uprobes_core.c
    > ===================================================================
    > --- /dev/null
    > +++ new_uprobes.git/kernel/uprobes_core.c
    > @@ -0,0 +1,2017 @@
    > +/*
    > + * Userspace Probes (UProbes)
    > + * kernel/uprobes_core.c
    > + *
    > + * This program is free software; you can redistribute it and/or modify
    > + * it under the terms of the GNU General Public License as published by
    > + * the Free Software Foundation; either version 2 of the License, or
    > + * (at your option) any later version.
    > + *
    > + * This program is distributed in the hope that it will be useful,
    > + * but WITHOUT ANY WARRANTY; without even the implied warranty of
    > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    > + * GNU General Public License for more details.
    > + *
    > + * You should have received a copy of the GNU General Public License
    > + * along with this program; if not, write to the Free Software
    > + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
    > + *
    > + * Copyright (C) IBM Corporation, 2006, 2009
    > + */
    > +#include <linux/types.h>
    > +#include <linux/hash.h>
    > +#include <linux/init.h>
    > +#include <linux/module.h>
    > +#include <linux/sched.h>
    > +#include <linux/rcupdate.h>
    > +#include <linux/err.h>
    > +#include <linux/kref.h>
    > +#include <linux/utrace.h>
    > +#include <linux/regset.h>
    > +#define UPROBES_IMPLEMENTATION 1
    > +#include <linux/uprobes.h>
    > +#include <linux/tracehook.h>
    > +#include <linux/string.h>
    > +#include <linux/uaccess.h>
    > +#include <linux/errno.h>
    > +#include <linux/mman.h>
    > +
    > +#define UPROBE_SET_FLAGS 1
    > +#define UPROBE_CLEAR_FLAGS 0
    > +
    > +#define MAX_XOL_SLOTS 1024
    > +
    > +static int utask_fake_quiesce(struct uprobe_task *utask);
    > +static int uprobe_post_ssout(struct uprobe_task *utask,
    > + struct uprobe_probept *ppt, struct pt_regs *regs);
    > +
    > +typedef void (*uprobe_handler_t)(struct uprobe*, struct pt_regs*);
    > +
    > +/*
    > + * Table of currently probed processes, hashed by task-group leader's
    > + * struct pid.
    > + */
    > +static struct hlist_head uproc_table[UPROBE_TABLE_SIZE];
    > +
    > +/* Protects uproc_table during uprobe (un)registration */
    > +static DEFINE_MUTEX(uproc_mutex);
    > +
    > +/* Table of uprobe_tasks, hashed by task_struct pointer. */
    > +static struct hlist_head utask_table[UPROBE_TABLE_SIZE];
    > +static DEFINE_SPINLOCK(utask_table_lock);
    > +
    > +/* p_uprobe_utrace_ops = &uprobe_utrace_ops. Fwd refs are a pain w/o this. */
    > +static const struct utrace_engine_ops *p_uprobe_utrace_ops;
    > +
    > +struct deferred_registration {
    > + struct list_head list;
    > + struct uprobe *uprobe;
    > + int regflag; /* 0 - unregister, 1 - register */
    > +};
    > +
    > +/*
    > + * Calling a signal handler cancels single-stepping, so uprobes delays
    > + * calling the handler, as necessary, until after single-stepping is completed.
    > + */
    > +struct delayed_signal {
    > + struct list_head list;
    > + siginfo_t info;
    > +};
    > +
    > +static u16 ubp_strategies;
    > +
    > +static struct uprobe_task *uprobe_find_utask(struct task_struct *tsk)
    > +{
    > + struct hlist_head *head;
    > + struct hlist_node *node;
    > + struct uprobe_task *utask;
    > + unsigned long flags;
    > +
    > + head = &utask_table[hash_ptr(tsk, UPROBE_HASH_BITS)];
    > + spin_lock_irqsave(&utask_table_lock, flags);
    > + hlist_for_each_entry(utask, node, head, hlist) {
    > + if (utask->tsk == tsk) {
    > + spin_unlock_irqrestore(&utask_table_lock, flags);
    > + return utask;
    > + }
    > + }
    > + spin_unlock_irqrestore(&utask_table_lock, flags);
    > + return NULL;
    > +}
    > +
    > +static void uprobe_hash_utask(struct uprobe_task *utask)
    > +{
    > + struct hlist_head *head;
    > + unsigned long flags;
    > +
    > + INIT_HLIST_NODE(&utask->hlist);
    > + head = &utask_table[hash_ptr(utask->tsk, UPROBE_HASH_BITS)];
    > + spin_lock_irqsave(&utask_table_lock, flags);
    > + hlist_add_head(&utask->hlist, head);
    > + spin_unlock_irqrestore(&utask_table_lock, flags);
    > +}
    > +
    > +static void uprobe_unhash_utask(struct uprobe_task *utask)
    > +{
    > + unsigned long flags;
    > +
    > + spin_lock_irqsave(&utask_table_lock, flags);
    > + hlist_del(&utask->hlist);
    > + spin_unlock_irqrestore(&utask_table_lock, flags);
    > +}
    > +
    > +static inline void uprobe_get_process(struct uprobe_process *uproc)
    > +{
    > + atomic_inc(&uproc->refcount);
    > +}
    > +
    > +/*
    > + * Decrement uproc's refcount in a situation where we "know" it can't
    > + * reach zero. It's OK to call this with uproc locked. Compare with
    > + * uprobe_put_process().
    > + */
    > +static inline void uprobe_decref_process(struct uprobe_process *uproc)
    > +{
    > + if (atomic_dec_and_test(&uproc->refcount))
    > + BUG();
    > +}
    > +
    > +/*
    > + * Runs with the uproc_mutex held. Returns with uproc ref-counted and
    > + * write-locked.
    > + *
    > + * Around exec time, briefly, it's possible to have one (finished) uproc
    > + * for the old image and one for the new image. We find the latter.
    > + */
    > +static struct uprobe_process *uprobe_find_process(struct pid *tg_leader)
    > +{
    > + struct uprobe_process *uproc;
    > + struct hlist_head *head;
    > + struct hlist_node *node;
    > +
    > + head = &uproc_table[hash_ptr(tg_leader, UPROBE_HASH_BITS)];
    > + hlist_for_each_entry(uproc, node, head, hlist) {
    > + if (uproc->tg_leader == tg_leader && !uproc->finished) {
    > + uprobe_get_process(uproc);
    > + down_write(&uproc->rwsem);
    > + return uproc;
    > + }
    > + }
    > + return NULL;
    > +}
    > +
    > +/*
    > + * In the given uproc's hash table of probepoints, find the one with the
    > + * specified virtual address. Runs with uproc->rwsem locked.
    > + */
    > +static struct uprobe_probept *uprobe_find_probept(struct uprobe_process *uproc,
    > + unsigned long vaddr)
    > +{
    > + struct uprobe_probept *ppt;
    > + struct hlist_node *node;
    > + struct hlist_head *head = &uproc->uprobe_table[hash_long(vaddr,
    > + UPROBE_HASH_BITS)];
    > +
    > + hlist_for_each_entry(ppt, node, head, ut_node) {
    > + if (ppt->ubp.vaddr == vaddr && ppt->state != UPROBE_DISABLED)
    > + return ppt;
    > + }
    > + return NULL;
    > +}
    > +
    > +/*
    > + * Save a copy of the original instruction (so it can be single-stepped
    > + * out of line), insert the breakpoint instruction, and awake
    > + * register_uprobe().
    > + */
    > +static void uprobe_insert_bkpt(struct uprobe_probept *ppt,
    > + struct task_struct *tsk)
    > +{
    > + struct uprobe_kimg *uk;
    > + int result;
    > +
    > + if (tsk)
    > + result = ubp_insert_bkpt(tsk, &ppt->ubp);
    > + else
    > + /* No surviving tasks associated with ppt->uproc */
    > + result = -ESRCH;
    > + ppt->state = (result ? UPROBE_DISABLED : UPROBE_BP_SET);
    > + list_for_each_entry(uk, &ppt->uprobe_list, list)
    > + uk->status = result;
    > + wake_up_all(&ppt->waitq);
    > +}
    > +
    > +/*
    > + * Check if task has just stepped on a trap instruction at the
    > + * indicated address. If it has indeed stepped on that address,
    > + * then reset Instruction Pointer for the task.
    > + *
    > + * tsk should either be current thread or already quiesced thread.
    > + */
    > +static inline void reset_thread_ip(struct task_struct *tsk,
    > + struct pt_regs *regs, unsigned long addr)
    > +{
    > + if ((ubp_get_bkpt_addr(regs) == addr) &&
    > + !test_tsk_thread_flag(tsk, TIF_SINGLESTEP))
    > + ubp_set_ip(regs, addr);
    > +}
    > +
    > +/*
    > + * ppt's breakpoint has been removed. If any threads are in the middle of
    > + * single-stepping at this probepoint, fix things up so they can proceed.
    > + * If any threads have just hit breakpoint but are yet to start
    > + * pre-processing, reset their instruction pointers.
    > + *
    > + * Runs with all of ppt->uproc's threads quiesced and ppt->uproc->rwsem
    > + * write-locked
    > + */
    > +static inline void adjust_trapped_thread_ip(struct uprobe_probept *ppt)
    > +{
    > + struct uprobe_process *uproc = ppt->uproc;
    > + struct uprobe_task *utask;
    > + struct pt_regs *regs;
    > +
    > + list_for_each_entry(utask, &uproc->thread_list, list) {
    > + regs = task_pt_regs(utask->tsk);
    > + if (utask->active_probe != ppt) {
    > + reset_thread_ip(utask->tsk, regs, ppt->ubp.vaddr);
    > + continue;
    > + }
    > +
    > + /*
    > + * Current thread cannot have an active breakpoint
    > + * and still request for a breakpoint removal. The
    > + * above case is handled by utask_fake_quiesce().
    > + */
    > + BUG_ON(utask->tsk == current);
    > +
    > +#ifdef CONFIG_UBP_XOL
    > + if (instruction_pointer(regs) == ppt->ubp.xol_vaddr)
    > + /* adjust the ip to breakpoint addr. */
    > + ubp_set_ip(regs, ppt->ubp.vaddr);
    > + else
    > + /* adjust the ip to next instruction. */
    > + uprobe_post_ssout(utask, ppt, regs);
    > +#endif
    > + }
    > +}
    > +
    > +static void uprobe_remove_bkpt(struct uprobe_probept *ppt,
    > + struct task_struct *tsk)
    > +{
    > + if (tsk) {
    > + if (ubp_remove_bkpt(tsk, &ppt->ubp) != 0) {
    > + printk(KERN_ERR
    > + "Error removing uprobe at pid %d vaddr %#lx:"
    > + " can't restore original instruction\n",
    > + tsk->tgid, ppt->ubp.vaddr);
    > + /*
    > + * This shouldn't happen, since we were previously
    > + * able to write the breakpoint at that address.
    > + * There's not much we can do besides let the
    > + * process die with a SIGTRAP the next time the
    > + * breakpoint is hit.
    > + */
    > + }
    > + adjust_trapped_thread_ip(ppt);
    > + if (ppt->ubp.strategy & UBP_HNT_INLINE) {
    > + unsigned long flags;
    > + spin_lock_irqsave(&ppt->ssil_lock, flags);
    > + ppt->ssil_state = SSIL_DISABLE;
    > + wake_up_all(&ppt->ssilq);
    > + spin_unlock_irqrestore(&ppt->ssil_lock, flags);
    > + }
    > + }
    > + /* Wake up unregister_uprobe(). */
    > + ppt->state = UPROBE_DISABLED;
    > + wake_up_all(&ppt->waitq);
    > +}
    > +
    > +/*
    > + * Runs with all of uproc's threads quiesced and uproc->rwsem write-locked.
    > + * As specified, insert or remove the breakpoint instruction for each
    > + * uprobe_probept on uproc's pending list.
    > + * tsk = one of the tasks associated with uproc -- NULL if there are
    > + * no surviving threads.
    > + * It's OK for uproc->pending_uprobes to be empty here. It can happen
    > + * if a register and an unregister are requested (by different probers)
    > + * simultaneously for the same pid/vaddr.
    > + */
    > +static void handle_pending_uprobes(struct uprobe_process *uproc,
    > + struct task_struct *tsk)
    > +{
    > + struct uprobe_probept *ppt, *tmp;
    > +
    > + list_for_each_entry_safe(ppt, tmp, &uproc->pending_uprobes, pd_node) {
    > + switch (ppt->state) {
    > + case UPROBE_INSERTING:
    > + uprobe_insert_bkpt(ppt, tsk);
    > + break;
    > + case UPROBE_REMOVING:
    > + uprobe_remove_bkpt(ppt, tsk);
    > + break;
    > + default:
    > + BUG();
    > + }
    > + list_del(&ppt->pd_node);
    > + }
    > +}
    > +
    > +static void utask_adjust_flags(struct uprobe_task *utask, int set,
    > + unsigned long flags)
    > +{
    > + unsigned long newflags, oldflags;
    > +
    > + oldflags = utask->engine->flags;
    > + newflags = oldflags;
    > + if (set)
    > + newflags |= flags;
    > + else
    > + newflags &= ~flags;
    > + /*
    > + * utrace_barrier[_pid] is not appropriate here. If we're
    > + * adjusting current, it's not needed. And if we're adjusting
    > + * some other task, we're holding utask->uproc->rwsem, which
    > + * could prevent that task from completing the callback we'd
    > + * be waiting on.
    > + */
    > + if (newflags != oldflags) {
    > + if (utrace_set_events_pid(utask->pid, utask->engine,
    > + newflags) != 0)
    > + /* We don't care. */
    > + ;
    > + }
    > +}
    > +
    > +static inline void clear_utrace_quiesce(struct uprobe_task *utask, bool resume)
    > +{
    > + utask_adjust_flags(utask, UPROBE_CLEAR_FLAGS, UTRACE_EVENT(QUIESCE));
    > + if (resume) {
    > + if (utrace_control_pid(utask->pid, utask->engine,
    > + UTRACE_RESUME) != 0)
    > + /* We don't care. */
    > + ;
    > + }
    > +}
    > +
    > +/* Opposite of quiesce_all_threads(). Same locking applies. */
    > +static void rouse_all_threads(struct uprobe_process *uproc)
    > +{
    > + struct uprobe_task *utask;
    > +
    > + list_for_each_entry(utask, &uproc->thread_list, list) {
    > + if (utask->quiescing) {
    > + utask->quiescing = false;
    > + if (utask->state == UPTASK_QUIESCENT) {
    > + utask->state = UPTASK_RUNNING;
    > + uproc->n_quiescent_threads--;
    > + clear_utrace_quiesce(utask, true);
    > + }
    > + }
    > + }
    > + /* Wake any threads that decided to sleep rather than quiesce. */
    > + wake_up_all(&uproc->waitq);
    > +}
    > +
    > +/*
    > + * If all of uproc's surviving threads have quiesced, do the necessary
    > + * breakpoint insertions or removals, un-quiesce everybody, and return 1.
    > + * tsk is a surviving thread, or NULL if there is none. Runs with
    > + * uproc->rwsem write-locked.
    > + */
    > +static int check_uproc_quiesced(struct uprobe_process *uproc,
    > + struct task_struct *tsk)
    > +{
    > + if (uproc->n_quiescent_threads >= uproc->nthreads) {
    > + handle_pending_uprobes(uproc, tsk);
    > + rouse_all_threads(uproc);
    > + return 1;
    > + }
    > + return 0;
    > +}
    > +
    > +/* Direct the indicated thread to quiesce. */
    > +static void uprobe_stop_thread(struct uprobe_task *utask)
    > +{
    > + int result;
    > +
    > + /*
    > + * As with utask_adjust_flags, calling utrace_barrier_pid below
    > + * could deadlock.
    > + */
    > + BUG_ON(utask->tsk == current);
    > + result = utrace_control_pid(utask->pid, utask->engine, UTRACE_STOP);
    > + if (result == 0) {
    > + /* Already stopped. */
    > + utask->state = UPTASK_QUIESCENT;
    > + utask->uproc->n_quiescent_threads++;
    > + } else if (result == -EINPROGRESS) {
    > + if (utask->tsk->state & TASK_INTERRUPTIBLE) {
    > + /*
    > + * Task could be in interruptible wait for a long
    > + * time -- e.g., if stopped for I/O. But we know
    > + * it's not going to run user code before all
    > + * threads quiesce, so pretend it's quiesced.
    > + * This avoids terminating a system call via
    > + * UTRACE_INTERRUPT.
    > + */
    > + utask->state = UPTASK_QUIESCENT;
    > + utask->uproc->n_quiescent_threads++;
    > + } else {
    > + /*
    > + * Task will eventually stop, but it may be a long time.
    > + * Don't wait.
    > + */
    > + result = utrace_control_pid(utask->pid, utask->engine,
    > + UTRACE_INTERRUPT);
    > + if (result != 0)
    > + /* We don't care. */
    > + ;
    > + }
    > + }
    > +}
    > +
    > +/*
    > + * Quiesce all threads in the specified process -- e.g., prior to
    > + * breakpoint insertion. Runs with uproc->rwsem write-locked.
    > + * Returns false if all threads have died.
    > + */
    > +static bool quiesce_all_threads(struct uprobe_process *uproc,
    > + struct uprobe_task **cur_utask_quiescing)
    > +{
    > + struct uprobe_task *utask;
    > + struct task_struct *survivor = NULL; /* any survivor */
    > + bool survivors = false;
    > +
    > + *cur_utask_quiescing = NULL;
    > + list_for_each_entry(utask, &uproc->thread_list, list) {
    > + if (!survivors) {
    > + survivor = pid_task(utask->pid, PIDTYPE_PID);
    > + if (survivor)
    > + survivors = true;
    > + }
    > + if (!utask->quiescing) {
    > + /*
    > + * If utask is currently handling a probepoint, it'll
    > + * check utask->quiescing and quiesce when it's done.
    > + */
    > + utask->quiescing = true;
    > + if (utask->tsk == current)
    > + *cur_utask_quiescing = utask;
    > + else if (utask->state == UPTASK_RUNNING) {
    > + utask_adjust_flags(utask, UPROBE_SET_FLAGS,
    > + UTRACE_EVENT(QUIESCE));
    > + uprobe_stop_thread(utask);
    > + }
    > + }
    > + }
    > + /*
    > + * If all the (other) threads are already quiesced, it's up to the
    > + * current thread to do the necessary work.
    > + */
    > + check_uproc_quiesced(uproc, survivor);
    > + return survivors;
    > +}
    > +
    > +/* Called with utask->uproc write-locked. */
    > +static void uprobe_free_task(struct uprobe_task *utask, bool in_callback)
    > +{
    > + struct deferred_registration *dr, *d;
    > + struct delayed_signal *ds, *ds2;
    > + int result;
    > +
    > + if (utask->engine && (utask->tsk != current || !in_callback)) {
    > + /*
    > + * No other tasks in this process should be running
    > + * uprobe_report_* callbacks. (If they are, utrace_barrier()
    > + * here could deadlock.)
    > + */
    > + result = utrace_control_pid(utask->pid, utask->engine,
    > + UTRACE_DETACH);
    > + BUG_ON(result == -EINPROGRESS);
    > + }
    > + put_pid(utask->pid); /* null pid OK */
    > +
    > + uprobe_unhash_utask(utask);
    > + list_del(&utask->list);
    > + list_for_each_entry_safe(dr, d, &utask->deferred_registrations, list) {
    > + list_del(&dr->list);
    > + kfree(dr);
    > + }
    > +
    > + list_for_each_entry_safe(ds, ds2, &utask->delayed_signals, list) {
    > + list_del(&ds->list);
    > + kfree(ds);
    > + }
    > +
    > + kfree(utask);
    > +}
    > +
    > +/*
    > + * Dismantle uproc and all its remaining uprobe_tasks.
    > + * in_callback = 1 if the caller is a uprobe_report_* callback who will
    > + * handle the UTRACE_DETACH operation.
    > + * Runs with uproc_mutex held; called with uproc->rwsem write-locked.
    > + */
    > +static void uprobe_free_process(struct uprobe_process *uproc, int in_callback)
    > +{
    > + struct uprobe_task *utask, *tmp;
    > +
    > + if (!hlist_unhashed(&uproc->hlist))
    > + hlist_del(&uproc->hlist);
    > + list_for_each_entry_safe(utask, tmp, &uproc->thread_list, list)
    > + uprobe_free_task(utask, in_callback);
    > + put_pid(uproc->tg_leader);
    > + if (uproc->xol_area)
    > + xol_put_area(uproc->xol_area);
    > + up_write(&uproc->rwsem); /* So kfree doesn't complain */
    > + kfree(uproc);
    > +}
    > +
    > +/*
    > + * Decrement uproc's ref count. If it's zero, free uproc and return
    > + * 1. Else return 0. If uproc is locked, don't call this; use
    > + * uprobe_decref_process().
    > + */
    > +static int uprobe_put_process(struct uprobe_process *uproc, bool in_callback)
    > +{
    > + int freed = 0;
    > +
    > + if (atomic_dec_and_test(&uproc->refcount)) {
    > + mutex_lock(&uproc_mutex);
    > + down_write(&uproc->rwsem);
    > + if (unlikely(atomic_read(&uproc->refcount) != 0)) {
    > + /*
    > + * The works because uproc_mutex is held any
    > + * time the ref count can go from 0 to 1 -- e.g.,
    > + * register_uprobe() sneaks in with a new probe.
    > + */
    > + up_write(&uproc->rwsem);
    > + } else {
    > + uprobe_free_process(uproc, in_callback);
    > + freed = 1;
    > + }
    > + mutex_unlock(&uproc_mutex);
    > + }
    > + return freed;
    > +}
    > +
    > +static struct uprobe_kimg *uprobe_mk_kimg(struct uprobe *u)
    > +{
    > + struct uprobe_kimg *uk = kzalloc(sizeof *uk,
    > + GFP_USER);
    > +
    > + if (unlikely(!uk))
    > + return ERR_PTR(-ENOMEM);
    > + u->kdata = uk;
    > + uk->uprobe = u;
    > + uk->ppt = NULL;
    > + INIT_LIST_HEAD(&uk->list);
    > + uk->status = -EBUSY;
    > + return uk;
    > +}
    > +
    > +/*
    > + * Allocate a uprobe_task object for p and add it to uproc's list.
    > + * Called with p "got" and uproc->rwsem write-locked. Called in one of
    > + * the following cases:
    > + * - before setting the first uprobe in p's process
    > + * - we're in uprobe_report_clone() and p is the newly added thread
    > + * Returns:
    > + * - pointer to new uprobe_task on success
    > + * - NULL if t dies before we can utrace_attach it
    > + * - negative errno otherwise
    > + */
    > +static struct uprobe_task *uprobe_add_task(struct pid *p,
    > + struct uprobe_process *uproc)
    > +{
    > + struct uprobe_task *utask;
    > + struct utrace_engine *engine;
    > + struct task_struct *t = pid_task(p, PIDTYPE_PID);

    What keeps the task_struct referenced by "t" from disappearing at this
    point?

    > +
    > + if (!t)
    > + return NULL;
    > + utask = kzalloc(sizeof *utask, GFP_USER);
    > + if (unlikely(utask == NULL))
    > + return ERR_PTR(-ENOMEM);
    > +
    > + utask->pid = p;
    > + utask->tsk = t;
    > + utask->state = UPTASK_RUNNING;
    > + utask->quiescing = false;
    > + utask->uproc = uproc;
    > + utask->active_probe = NULL;
    > + utask->doomed = false;
    > + INIT_LIST_HEAD(&utask->deferred_registrations);
    > + INIT_LIST_HEAD(&utask->delayed_signals);
    > + INIT_LIST_HEAD(&utask->list);
    > + list_add_tail(&utask->list, &uproc->thread_list);
    > + uprobe_hash_utask(utask);
    > +
    > + engine = utrace_attach_pid(p, UTRACE_ATTACH_CREATE,
    > + p_uprobe_utrace_ops, utask);
    > + if (IS_ERR(engine)) {
    > + long err = PTR_ERR(engine);
    > + printk("uprobes: utrace_attach_task failed, returned %ld\n",
    > + err);
    > + uprobe_free_task(utask, 0);
    > + if (err == -ESRCH)
    > + return NULL;
    > + return ERR_PTR(err);
    > + }
    > + utask->engine = engine;
    > + /*
    > + * Always watch for traps, clones, execs and exits. Caller must
    > + * set any other engine flags.
    > + */
    > + utask_adjust_flags(utask, UPROBE_SET_FLAGS,
    > + UTRACE_EVENT(SIGNAL) | UTRACE_EVENT(SIGNAL_IGN) |
    > + UTRACE_EVENT(SIGNAL_CORE) | UTRACE_EVENT(EXEC) |
    > + UTRACE_EVENT(CLONE) | UTRACE_EVENT(EXIT));
    > + /*
    > + * Note that it's OK if t dies just after utrace_attach, because
    > + * with the engine in place, the appropriate report_* callback
    > + * should handle it after we release uproc->rwsem.
    > + */
    > + utrace_engine_put(engine);
    > + return utask;
    > +}
    > +
    > +/*
    > + * start_pid is the pid for a thread in the probed process. Find the
    > + * next thread that doesn't have a corresponding uprobe_task yet. Return
    > + * a ref-counted pid for that task, if any, else NULL.
    > + */
    > +static struct pid *find_next_thread_to_add(struct uprobe_process *uproc,
    > + struct pid *start_pid)
    > +{
    > + struct task_struct *t, *start;
    > + struct uprobe_task *utask;
    > + struct pid *pid = NULL;
    > +
    > + rcu_read_lock();
    > + start = pid_task(start_pid, PIDTYPE_PID);
    > + t = start;
    > + if (t) {
    > + do {
    > + if (unlikely(t->flags & PF_EXITING))
    > + goto dont_add;
    > + list_for_each_entry(utask, &uproc->thread_list, list) {

    Doesn't this need to be list_for_each_entry_rcu()?

    Or do you have ->thread_list protected elsewise?

    > + if (utask->tsk == t)
    > + /* Already added */
    > + goto dont_add;
    > + }
    > + /* Found thread/task to add. */
    > + pid = get_pid(task_pid(t));
    > + break;
    > +dont_add:
    > + t = next_thread(t);
    > + } while (t != start);
    > + }
    > + rcu_read_unlock();

    Now that we are outside of rcu_read_lock()'s protection, the task
    indicated by "pid" might disappear, and the value of "pid" might well
    be reused. Is this really OK?

    > + return pid;
    > +}
    > +
    > +/* Runs with uproc_mutex held; returns with uproc->rwsem write-locked. */
    > +static struct uprobe_process *uprobe_mk_process(struct pid *tg_leader)
    > +{
    > + struct uprobe_process *uproc;
    > + struct uprobe_task *utask;
    > + struct pid *add_me;
    > + int i;
    > + long err;
    > +
    > + uproc = kzalloc(sizeof *uproc, GFP_USER);
    > + if (unlikely(uproc == NULL))
    > + return ERR_PTR(-ENOMEM);
    > +
    > + /* Initialize fields */
    > + atomic_set(&uproc->refcount, 1);
    > + init_rwsem(&uproc->rwsem);
    > + down_write(&uproc->rwsem);
    > + init_waitqueue_head(&uproc->waitq);
    > + for (i = 0; i < UPROBE_TABLE_SIZE; i++)
    > + INIT_HLIST_HEAD(&uproc->uprobe_table[i]);
    > + INIT_LIST_HEAD(&uproc->pending_uprobes);
    > + INIT_LIST_HEAD(&uproc->thread_list);
    > + uproc->nthreads = 0;
    > + uproc->n_quiescent_threads = 0;
    > + INIT_HLIST_NODE(&uproc->hlist);
    > + uproc->tg_leader = get_pid(tg_leader);
    > + uproc->tgid = pid_task(tg_leader, PIDTYPE_PID)->tgid;
    > + uproc->finished = false;
    > +
    > +#ifdef CONFIG_UBP_XOL
    > + if (!(ubp_strategies & UBP_HNT_INLINE))
    > + uproc->sstep_out_of_line = true;
    > + else
    > +#endif
    > + uproc->sstep_out_of_line = false;
    > +
    > + /*
    > + * Create and populate one utask per thread in this process. We
    > + * can't call uprobe_add_task() while holding RCU lock, so we:
    > + * 1. rcu_read_lock()
    > + * 2. Find the next thread, add_me, in this process that's not
    > + * already on uproc's thread_list.
    > + * 3. rcu_read_unlock()
    > + * 4. uprobe_add_task(add_me, uproc)
    > + * Repeat 1-4 'til we have utasks for all threads.
    > + */
    > + add_me = tg_leader;
    > + while ((add_me = find_next_thread_to_add(uproc, add_me)) != NULL) {
    > + utask = uprobe_add_task(add_me, uproc);
    > + if (IS_ERR(utask)) {
    > + err = PTR_ERR(utask);
    > + goto fail;
    > + }
    > + if (utask)
    > + uproc->nthreads++;
    > + }
    > +
    > + if (uproc->nthreads == 0) {
    > + /* All threads -- even p -- are dead. */
    > + err = -ESRCH;
    > + goto fail;
    > + }
    > + return uproc;
    > +
    > +fail:
    > + uprobe_free_process(uproc, 0);
    > + return ERR_PTR(err);
    > +}
    > +
    > +/*
    > + * Creates a uprobe_probept and connects it to uk and uproc. Runs with
    > + * uproc->rwsem write-locked.
    > + */
    > +static struct uprobe_probept *uprobe_add_probept(struct uprobe_kimg *uk,
    > + struct uprobe_process *uproc)
    > +{
    > + struct uprobe_probept *ppt;
    > +
    > + ppt = kzalloc(sizeof *ppt, GFP_USER);
    > + if (unlikely(ppt == NULL))
    > + return ERR_PTR(-ENOMEM);
    > + init_waitqueue_head(&ppt->waitq);
    > + init_waitqueue_head(&ppt->ssilq);
    > + spin_lock_init(&ppt->ssil_lock);
    > + ppt->ssil_state = SSIL_CLEAR;
    > +
    > + /* Connect to uk. */
    > + INIT_LIST_HEAD(&ppt->uprobe_list);
    > + list_add_tail(&uk->list, &ppt->uprobe_list);
    > + uk->ppt = ppt;
    > + uk->status = -EBUSY;
    > + ppt->ubp.vaddr = uk->uprobe->vaddr;
    > + ppt->ubp.xol_vaddr = 0;
    > +
    > + /* Connect to uproc. */
    > + if (!uproc->sstep_out_of_line)
    > + ppt->ubp.strategy = UBP_HNT_INLINE;
    > + else
    > + ppt->ubp.strategy = ubp_strategies;
    > + ppt->state = UPROBE_INSERTING;
    > + ppt->uproc = uproc;
    > + INIT_LIST_HEAD(&ppt->pd_node);
    > + list_add_tail(&ppt->pd_node, &uproc->pending_uprobes);
    > + INIT_HLIST_NODE(&ppt->ut_node);
    > + hlist_add_head(&ppt->ut_node,
    > + &uproc->uprobe_table[hash_long(ppt->ubp.vaddr,
    > + UPROBE_HASH_BITS)]);
    > + uprobe_get_process(uproc);
    > + return ppt;
    > +}
    > +
    > +/*
    > + * Runs with ppt->uproc write-locked. Frees ppt and decrements the ref
    > + * count on ppt->uproc (but ref count shouldn't hit 0).
    > + */
    > +static void uprobe_free_probept(struct uprobe_probept *ppt)
    > +{
    > + struct uprobe_process *uproc = ppt->uproc;
    > +
    > + xol_free_insn_slot(ppt->ubp.xol_vaddr, uproc->xol_area);
    > + hlist_del(&ppt->ut_node);
    > + kfree(ppt);
    > + uprobe_decref_process(uproc);
    > +}
    > +
    > +static void uprobe_free_kimg(struct uprobe_kimg *uk)
    > +{
    > + uk->uprobe->kdata = NULL;
    > + kfree(uk);
    > +}
    > +
    > +/*
    > + * Runs with uprobe_process write-locked.
    > + * Note that we never free uk->uprobe, because the user owns that.
    > + */
    > +static void purge_uprobe(struct uprobe_kimg *uk)
    > +{
    > + struct uprobe_probept *ppt = uk->ppt;
    > +
    > + list_del(&uk->list);
    > + uprobe_free_kimg(uk);
    > + if (list_empty(&ppt->uprobe_list))
    > + uprobe_free_probept(ppt);
    > +}
    > +
    > +/*
    > + * Runs with utask->uproc locked.
    > + * read lock if called from uprobe handler.
    > + * else write lock.
    > + * Returns -EINPROGRESS on success.
    > + * Returns -EBUSY if a request for defer registration already exists.
    > + * Returns 0 if we have deferred request for both register/unregister.
    > + *
    > + */
    > +static int defer_registration(struct uprobe *u, int regflag,
    > + struct uprobe_task *utask)
    > +{
    > + struct deferred_registration *dr, *d;
    > +
    > + /* Check if we already have such a defer request */
    > + list_for_each_entry_safe(dr, d, &utask->deferred_registrations, list) {
    > + if (dr->uprobe == u) {
    > + if (dr->regflag != regflag) {
    > + /* same as successful register + unregister */
    > + list_del(&dr->list);
    > + kfree(dr);
    > + return 0;
    > + } else
    > + /* we already have identical request */
    > + return -EBUSY;
    > + }
    > + }
    > +
    > + /* We have a new unique request */
    > + dr = kmalloc(sizeof(struct deferred_registration), GFP_USER);
    > + if (!dr)
    > + return -ENOMEM;
    > + dr->uprobe = u;
    > + dr->regflag = regflag;
    > + INIT_LIST_HEAD(&dr->list);
    > + list_add_tail(&dr->list, &utask->deferred_registrations);
    > + return -EINPROGRESS;
    > +}
    > +
    > +/*
    > + * Given a numeric thread ID, return a ref-counted struct pid for the
    > + * task-group-leader thread.
    > + */
    > +static struct pid *uprobe_get_tg_leader(pid_t p)
    > +{
    > + struct pid *pid = NULL;
    > +
    > + rcu_read_lock();
    > + if (current->nsproxy)
    > + pid = find_vpid(p);
    > + if (pid) {
    > + struct task_struct *t = pid_task(pid, PIDTYPE_PID);
    > + if (t)
    > + pid = task_tgid(t);
    > + else
    > + pid = NULL;
    > + }
    > + rcu_read_unlock();

    What happens if the thread disappears at this point? We are outside of
    rcu_read_lock() protection, so all the structures could potentially be
    freed up by other CPUs, especially if this CPU takes an interrupt or is
    preempted.

    > + return get_pid(pid); /* null pid OK here */
    > +}
    > +
    > +/* See Documentation/uprobes.txt. */
    > +int register_uprobe(struct uprobe *u)
    > +{
    > + struct uprobe_task *cur_utask, *cur_utask_quiescing = NULL;
    > + struct uprobe_process *uproc;
    > + struct uprobe_probept *ppt;
    > + struct uprobe_kimg *uk;
    > + struct pid *p;
    > + int ret = 0, uproc_is_new = 0;
    > + bool survivors;
    > +#ifndef CONFIG_UBP_XOL
    > + struct task_struct *tsk;
    > +#endif
    > +
    > + if (!u || !u->handler)
    > + return -EINVAL;
    > +
    > + p = uprobe_get_tg_leader(u->pid);
    > + if (!p)
    > + return -ESRCH;
    > +
    > + cur_utask = uprobe_find_utask(current);
    > + if (cur_utask && cur_utask->active_probe) {
    > + /*
    > + * Called from handler; cur_utask->uproc is read-locked.
    > + * Do this registration later.
    > + */
    > + put_pid(p);
    > + return defer_registration(u, 1, cur_utask);
    > + }
    > +
    > + /* Get the uprobe_process for this pid, or make a new one. */
    > + mutex_lock(&uproc_mutex);
    > + uproc = uprobe_find_process(p);
    > +
    > + if (uproc) {
    > + struct uprobe_task *utask;
    > +
    > + mutex_unlock(&uproc_mutex);
    > + list_for_each_entry(utask, &uproc->thread_list, list) {
    > + if (!utask->active_probe)
    > + continue;
    > + /*
    > + * utask is at a probepoint, but has dropped
    > + * uproc->rwsem to single-step. If utask is
    > + * stopped, then it's probably because some
    > + * other engine has asserted UTRACE_STOP;
    > + * that engine may not allow UTRACE_RESUME
    > + * until register_uprobe() returns. But, for
    > + * reasons we won't go into here, utask wants
    > + * to finish with utask->active_probe before
    > + * allowing handle_pending_uprobes() to run
    > + * (via utask_fake_quiesce()). So we defer this
    > + * registration operation; it will be run after
    > + * utask->active_probe is taken care of.
    > + */
    > + BUG_ON(utask->state != UPTASK_SSTEP);
    > + if (task_is_stopped_or_traced(utask->tsk)) {
    > + ret = defer_registration(u, 1, utask);
    > + goto fail_uproc;
    > + }
    > + }
    > + } else {
    > + uproc = uprobe_mk_process(p);
    > + if (IS_ERR(uproc)) {
    > + ret = (int) PTR_ERR(uproc);
    > + mutex_unlock(&uproc_mutex);
    > + goto fail_tsk;
    > + }
    > + /* Hold uproc_mutex until we've added uproc to uproc_table. */
    > + uproc_is_new = 1;
    > + }
    > +
    > +#ifdef CONFIG_UBP_XOL
    > + ret = xol_validate_vaddr(p, u->vaddr, uproc->xol_area);
    > +#else
    > + tsk = pid_task(p, PIDTYPE_PID);
    > + ret = ubp_validate_insn_addr(tsk, u->vaddr);
    > +#endif
    > + if (ret < 0)
    > + goto fail_uproc;
    > +
    > + if (u->kdata) {
    > + /*
    > + * Probe is already/still registered. This is the only
    > + * place we return -EBUSY to the user.
    > + */
    > + ret = -EBUSY;
    > + goto fail_uproc;
    > + }
    > +
    > + uk = uprobe_mk_kimg(u);
    > + if (IS_ERR(uk)) {
    > + ret = (int) PTR_ERR(uk);
    > + goto fail_uproc;
    > + }
    > +
    > + /* See if we already have a probepoint at the vaddr. */
    > + ppt = (uproc_is_new ? NULL : uprobe_find_probept(uproc, u->vaddr));
    > + if (ppt) {
    > + /* Breakpoint is already in place, or soon will be. */
    > + uk->ppt = ppt;
    > + list_add_tail(&uk->list, &ppt->uprobe_list);
    > + switch (ppt->state) {
    > + case UPROBE_INSERTING:
    > + uk->status = -EBUSY; /* in progress */
    > + if (uproc->tg_leader == task_tgid(current)) {
    > + cur_utask_quiescing = cur_utask;
    > + BUG_ON(!cur_utask_quiescing);
    > + }
    > + break;
    > + case UPROBE_REMOVING:
    > + /* Wait! Don't remove that bkpt after all! */
    > + ppt->state = UPROBE_BP_SET;
    > + /* Remove from pending list. */
    > + list_del(&ppt->pd_node);
    > + /* Wake unregister_uprobe(). */
    > + wake_up_all(&ppt->waitq);
    > + /*FALLTHROUGH*/
    > + case UPROBE_BP_SET:
    > + uk->status = 0;
    > + break;
    > + default:
    > + BUG();
    > + }
    > + up_write(&uproc->rwsem);
    > + put_pid(p);
    > + if (uk->status == 0) {
    > + uprobe_decref_process(uproc);
    > + return 0;
    > + }
    > + goto await_bkpt_insertion;
    > + } else {
    > + ppt = uprobe_add_probept(uk, uproc);
    > + if (IS_ERR(ppt)) {
    > + ret = (int) PTR_ERR(ppt);
    > + goto fail_uk;
    > + }
    > + }
    > +
    > + if (uproc_is_new) {
    > + hlist_add_head(&uproc->hlist,
    > + &uproc_table[hash_ptr(uproc->tg_leader,
    > + UPROBE_HASH_BITS)]);
    > + mutex_unlock(&uproc_mutex);
    > + }
    > + put_pid(p);
    > + survivors = quiesce_all_threads(uproc, &cur_utask_quiescing);
    > +
    > + if (!survivors) {
    > + purge_uprobe(uk);
    > + up_write(&uproc->rwsem);
    > + uprobe_put_process(uproc, false);
    > + return -ESRCH;
    > + }
    > + up_write(&uproc->rwsem);
    > +
    > +await_bkpt_insertion:
    > + if (cur_utask_quiescing)
    > + /* Current task is probing its own process. */
    > + (void) utask_fake_quiesce(cur_utask_quiescing);
    > + else
    > + wait_event(ppt->waitq, ppt->state != UPROBE_INSERTING);
    > + ret = uk->status;
    > + if (ret != 0) {
    > + down_write(&uproc->rwsem);
    > + purge_uprobe(uk);
    > + up_write(&uproc->rwsem);
    > + }
    > + uprobe_put_process(uproc, false);
    > + return ret;
    > +
    > +fail_uk:
    > + uprobe_free_kimg(uk);
    > +
    > +fail_uproc:
    > + if (uproc_is_new) {
    > + uprobe_free_process(uproc, 0);
    > + mutex_unlock(&uproc_mutex);
    > + } else {
    > + up_write(&uproc->rwsem);
    > + uprobe_put_process(uproc, false);
    > + }
    > +
    > +fail_tsk:
    > + put_pid(p);
    > + return ret;
    > +}
    > +EXPORT_SYMBOL_GPL(register_uprobe);
    > +
    > +/* See Documentation/uprobes.txt. */
    > +void unregister_uprobe(struct uprobe *u)
    > +{
    > + struct pid *p;
    > + struct uprobe_process *uproc;
    > + struct uprobe_kimg *uk;
    > + struct uprobe_probept *ppt;
    > + struct uprobe_task *cur_utask, *cur_utask_quiescing = NULL;
    > + struct uprobe_task *utask;
    > +
    > + if (!u)
    > + return;
    > + p = uprobe_get_tg_leader(u->pid);
    > + if (!p)
    > + return;
    > +
    > + cur_utask = uprobe_find_utask(current);
    > + if (cur_utask && cur_utask->active_probe) {
    > + /* Called from handler; uproc is read-locked; do this later */
    > + put_pid(p);
    > + (void) defer_registration(u, 0, cur_utask);
    > + return;
    > + }
    > +
    > + /*
    > + * Lock uproc before walking the graph, in case the process we're
    > + * probing is exiting.
    > + */
    > + mutex_lock(&uproc_mutex);
    > + uproc = uprobe_find_process(p);
    > + mutex_unlock(&uproc_mutex);
    > + put_pid(p);
    > + if (!uproc)
    > + return;
    > +
    > + list_for_each_entry(utask, &uproc->thread_list, list) {
    > + if (!utask->active_probe)
    > + continue;
    > +
    > + /* See comment in register_uprobe(). */
    > + BUG_ON(utask->state != UPTASK_SSTEP);
    > + if (task_is_stopped_or_traced(utask->tsk)) {
    > + (void) defer_registration(u, 0, utask);
    > + goto done;
    > + }
    > + }
    > + uk = (struct uprobe_kimg *)u->kdata;
    > + if (!uk)
    > + /*
    > + * This probe was never successfully registered, or
    > + * has already been unregistered.
    > + */
    > + goto done;
    > + if (uk->status == -EBUSY)
    > + /* Looks like register or unregister is already in progress. */
    > + goto done;
    > + ppt = uk->ppt;
    > +
    > + list_del(&uk->list);
    > + uprobe_free_kimg(uk);
    > +
    > + if (!list_empty(&ppt->uprobe_list))
    > + goto done;
    > +
    > + /*
    > + * The last uprobe at ppt's probepoint is being unregistered.
    > + * Queue the breakpoint for removal.
    > + */
    > + ppt->state = UPROBE_REMOVING;
    > + list_add_tail(&ppt->pd_node, &uproc->pending_uprobes);
    > +
    > + (void) quiesce_all_threads(uproc, &cur_utask_quiescing);
    > + up_write(&uproc->rwsem);
    > + if (cur_utask_quiescing)
    > + /* Current task is probing its own process. */
    > + (void) utask_fake_quiesce(cur_utask_quiescing);
    > + else
    > + wait_event(ppt->waitq, ppt->state != UPROBE_REMOVING);
    > +
    > + if (likely(ppt->state == UPROBE_DISABLED)) {
    > + down_write(&uproc->rwsem);
    > + uprobe_free_probept(ppt);
    > + /* else somebody else's register_uprobe() resurrected ppt. */
    > + up_write(&uproc->rwsem);
    > + }
    > + uprobe_put_process(uproc, false);
    > + return;
    > +
    > +done:
    > + up_write(&uproc->rwsem);
    > + uprobe_put_process(uproc, false);
    > +}
    > +EXPORT_SYMBOL_GPL(unregister_uprobe);
    > +
    > +/* Find a surviving thread in uproc. Runs with uproc->rwsem locked. */
    > +static struct task_struct *find_surviving_thread(struct uprobe_process *uproc)
    > +{
    > + struct uprobe_task *utask;
    > +
    > + list_for_each_entry(utask, &uproc->thread_list, list) {
    > + if (!(utask->tsk->flags & PF_EXITING))
    > + return utask->tsk;
    > + }
    > + return NULL;
    > +}
    > +
    > +/*
    > + * Run all the deferred_registrations previously queued by the current utask.
    > + * Runs with no locks or mutexes held. The current utask's uprobe_process
    > + * is ref-counted, so it won't disappear as the result of unregister_u*probe()
    > + * called here.
    > + */
    > +static void uprobe_run_def_regs(struct list_head *drlist)
    > +{
    > + struct deferred_registration *dr, *d;
    > +
    > + list_for_each_entry_safe(dr, d, drlist, list) {
    > + int result = 0;
    > + struct uprobe *u = dr->uprobe;
    > +
    > + if (dr->regflag)
    > + result = register_uprobe(u);
    > + else
    > + unregister_uprobe(u);
    > + if (u && u->registration_callback)
    > + u->registration_callback(u, dr->regflag, result);
    > + list_del(&dr->list);
    > + kfree(dr);
    > + }
    > +}
    > +
    > +/*
    > + * utrace engine report callbacks
    > + */
    > +
    > +/*
    > + * We've been asked to quiesce, but aren't in a position to do so.
    > + * This could happen in either of the following cases:
    > + *
    > + * 1) Our own thread is doing a register or unregister operation --
    > + * e.g., as called from a uprobe handler or a non-uprobes utrace
    > + * callback. We can't wait_event() for ourselves in [un]register_uprobe().
    > + *
    > + * 2) We've been asked to quiesce, but we hit a probepoint first. Now
    > + * we're in the report_signal callback, having handled the probepoint.
    > + * We'd like to just turn on UTRACE_EVENT(QUIESCE) and coast into
    > + * quiescence. Unfortunately, it's possible to hit a probepoint again
    > + * before we quiesce. When processing the SIGTRAP, utrace would call
    > + * uprobe_report_quiesce(), which must decline to take any action so
    > + * as to avoid removing the uprobe just hit. As a result, we could
    > + * keep hitting breakpoints and never quiescing.
    > + *
    > + * So here we do essentially what we'd prefer to do in uprobe_report_quiesce().
    > + * If we're the last thread to quiesce, handle_pending_uprobes() and
    > + * rouse_all_threads(). Otherwise, pretend we're quiescent and sleep until
    > + * the last quiescent thread handles that stuff and then wakes us.
    > + *
    > + * Called and returns with no mutexes held. Returns 1 if we free utask->uproc,
    > + * else 0.
    > + */
    > +static int utask_fake_quiesce(struct uprobe_task *utask)
    > +{
    > + struct uprobe_process *uproc = utask->uproc;
    > + enum uprobe_task_state prev_state = utask->state;
    > +
    > + down_write(&uproc->rwsem);
    > +
    > + /* In case we're somehow set to quiesce for real... */
    > + clear_utrace_quiesce(utask, false);
    > +
    > + if (uproc->n_quiescent_threads == uproc->nthreads-1) {
    > + /* We're the last thread to "quiesce." */
    > + handle_pending_uprobes(uproc, utask->tsk);
    > + rouse_all_threads(uproc);
    > + up_write(&uproc->rwsem);
    > + return 0;
    > + } else {
    > + utask->state = UPTASK_SLEEPING;
    > + uproc->n_quiescent_threads++;
    > + up_write(&uproc->rwsem);
    > + /* We ref-count sleepers. */
    > + uprobe_get_process(uproc);
    > +
    > + wait_event(uproc->waitq, !utask->quiescing);
    > +
    > + down_write(&uproc->rwsem);
    > + utask->state = prev_state;
    > + uproc->n_quiescent_threads--;
    > + up_write(&uproc->rwsem);
    > +
    > + /*
    > + * If uproc's last uprobe has been unregistered, and
    > + * unregister_uprobe() woke up before we did, it's up
    > + * to us to free uproc.
    > + */
    > + return uprobe_put_process(uproc, false);
    > + }
    > +}
    > +
    > +/* Prepare to single-step ppt's probed instruction inline. */
    > +static void uprobe_pre_ssin(struct uprobe_task *utask,
    > + struct uprobe_probept *ppt, struct pt_regs *regs)
    > +{
    > + unsigned long flags;
    > +
    > + if (unlikely(ppt->ssil_state == SSIL_DISABLE)) {
    > + reset_thread_ip(utask->tsk, regs, ppt->ubp.vaddr);
    > + return;
    > + }
    > + spin_lock_irqsave(&ppt->ssil_lock, flags);
    > + while (ppt->ssil_state == SSIL_SET) {
    > + spin_unlock_irqrestore(&ppt->ssil_lock, flags);
    > + up_read(&utask->uproc->rwsem);
    > + wait_event(ppt->ssilq, ppt->ssil_state != SSIL_SET);
    > + down_read(&utask->uproc->rwsem);
    > + spin_lock_irqsave(&ppt->ssil_lock, flags);
    > + }
    > + if (unlikely(ppt->ssil_state == SSIL_DISABLE)) {
    > + /*
    > + * While waiting to single step inline, breakpoint has
    > + * been removed. Thread continues as if nothing happened.
    > + */
    > + spin_unlock_irqrestore(&ppt->ssil_lock, flags);
    > + reset_thread_ip(utask->tsk, regs, ppt->ubp.vaddr);
    > + return;
    > + }
    > + ppt->ssil_state = SSIL_SET;
    > + spin_unlock_irqrestore(&ppt->ssil_lock, flags);
    > +
    > + if (unlikely(ubp_pre_sstep(utask->tsk, &ppt->ubp,
    > + &utask->arch_info, regs) != 0)) {
    > + printk(KERN_ERR "Failed to temporarily restore original "
    > + "instruction for single-stepping: "
    > + "pid/tgid=%d/%d, vaddr=%#lx\n",
    > + utask->tsk->pid, utask->tsk->tgid, ppt->ubp.vaddr);
    > + utask->doomed = true;
    > + }
    > +}
    > +
    > +/* Prepare to continue execution after single-stepping inline. */
    > +static void uprobe_post_ssin(struct uprobe_task *utask,
    > + struct uprobe_probept *ppt, struct pt_regs *regs)
    > +{
    > + unsigned long flags;
    > +
    > + if (unlikely(ubp_post_sstep(utask->tsk, &ppt->ubp,
    > + &utask->arch_info, regs) != 0))
    > + printk("Couldn't restore bp: pid/tgid=%d/%d, addr=%#lx\n",
    > + utask->tsk->pid, utask->tsk->tgid, ppt->ubp.vaddr);
    > + spin_lock_irqsave(&ppt->ssil_lock, flags);
    > + if (likely(ppt->ssil_state == SSIL_SET)) {
    > + ppt->ssil_state = SSIL_CLEAR;
    > + wake_up(&ppt->ssilq);
    > + }
    > + spin_unlock_irqrestore(&ppt->ssil_lock, flags);
    > +}
    > +
    > +#ifdef CONFIG_UBP_XOL
    > +/*
    > + * This architecture wants to do single-stepping out of line, but now we've
    > + * discovered that it can't -- typically because we couldn't set up the XOL
    > + * vma. Make all probepoints use inline single-stepping.
    > + */
    > +static void uproc_cancel_xol(struct uprobe_process *uproc)
    > +{
    > + down_write(&uproc->rwsem);
    > + if (likely(uproc->sstep_out_of_line)) {
    > + /* No other task beat us to it. */
    > + int i;
    > + struct uprobe_probept *ppt;
    > + struct hlist_node *node;
    > + struct hlist_head *head;
    > + for (i = 0; i < UPROBE_TABLE_SIZE; i++) {
    > + head = &uproc->uprobe_table[i];
    > + hlist_for_each_entry(ppt, node, head, ut_node) {
    > + if (!(ppt->ubp.strategy & UBP_HNT_INLINE))
    > + ubp_cancel_xol(current, &ppt->ubp);
    > + }
    > + }
    > + /* Do this last, so other tasks don't proceed too soon. */
    > + uproc->sstep_out_of_line = false;
    > + }
    > + up_write(&uproc->rwsem);
    > +}
    > +
    > +/* Prepare to single-step ppt's probed instruction out of line. */
    > +static int uprobe_pre_ssout(struct uprobe_task *utask,
    > + struct uprobe_probept *ppt, struct pt_regs *regs)
    > +{
    > + if (!ppt->ubp.xol_vaddr)
    > + ppt->ubp.xol_vaddr = xol_get_insn_slot(&ppt->ubp,
    > + ppt->uproc->xol_area);
    > + if (unlikely(!ppt->ubp.xol_vaddr)) {
    > + ubp_cancel_xol(utask->tsk, &ppt->ubp);
    > + return -1;
    > + }
    > + utask->singlestep_addr = ppt->ubp.xol_vaddr;
    > + return ubp_pre_sstep(utask->tsk, &ppt->ubp, &utask->arch_info, regs);
    > +}
    > +
    > +/* Prepare to continue execution after single-stepping out of line. */
    > +static int uprobe_post_ssout(struct uprobe_task *utask,
    > + struct uprobe_probept *ppt, struct pt_regs *regs)
    > +{
    > + int ret;
    > +
    > + ret = ubp_post_sstep(utask->tsk, &ppt->ubp, &utask->arch_info, regs);
    > + return ret;
    > +}
    > +#endif
    > +
    > +/*
    > + * If this thread is supposed to be quiescing, mark it quiescent; and
    > + * if it was the last thread to quiesce, do the work we quiesced for.
    > + * Runs with utask->uproc->rwsem write-locked. Returns true if we can
    > + * let this thread resume.
    > + */
    > +static bool utask_quiesce(struct uprobe_task *utask)
    > +{
    > + if (utask->quiescing) {
    > + if (utask->state != UPTASK_QUIESCENT) {
    > + utask->state = UPTASK_QUIESCENT;
    > + utask->uproc->n_quiescent_threads++;
    > + }
    > + return check_uproc_quiesced(utask->uproc, current);
    > + } else {
    > + clear_utrace_quiesce(utask, false);
    > + return true;
    > + }
    > +}
    > +
    > +/*
    > + * Delay delivery of the indicated signal until after single-step.
    > + * Otherwise single-stepping will be cancelled as part of calling
    > + * the signal handler.
    > + */
    > +static void uprobe_delay_signal(struct uprobe_task *utask, siginfo_t *info)
    > +{
    > + struct delayed_signal *ds;
    > +
    > + ds = kmalloc(sizeof(*ds), GFP_USER);
    > + if (ds) {
    > + ds->info = *info;
    > + INIT_LIST_HEAD(&ds->list);
    > + list_add_tail(&ds->list, &utask->delayed_signals);
    > + }
    > +}
    > +
    > +static void uprobe_inject_delayed_signals(struct list_head *delayed_signals)
    > +{
    > + struct delayed_signal *ds, *tmp;
    > +
    > + list_for_each_entry_safe(ds, tmp, delayed_signals, list) {
    > + send_sig_info(ds->info.si_signo, &ds->info, current);
    > + list_del(&ds->list);
    > + kfree(ds);
    > + }
    > +}
    > +
    > +/*
    > + * Verify from Instruction Pointer if singlestep has indeed occurred.
    > + * If Singlestep has occurred, then do post singlestep fix-ups.
    > + */
    > +static bool validate_and_post_sstep(struct uprobe_task *utask,
    > + struct pt_regs *regs,
    > + struct uprobe_probept *ppt)
    > +{
    > + unsigned long vaddr = instruction_pointer(regs);
    > +
    > + if (ppt->ubp.strategy & UBP_HNT_INLINE) {
    > + /*
    > + * If we have singlestepped, Instruction pointer cannot
    > + * be same as virtual address of probepoint.
    > + */
    > + if (vaddr == ppt->ubp.vaddr)
    > + return false;
    > + uprobe_post_ssin(utask, ppt, regs);
    > +#ifdef CONFIG_UBP_XOL
    > + } else {
    > + /*
    > + * If we have executed out of line, Instruction pointer
    > + * cannot be same as virtual address of XOL slot.
    > + */
    > + if (vaddr == ppt->ubp.xol_vaddr)
    > + return false;
    > + uprobe_post_ssout(utask, ppt, regs);
    > +#endif
    > + }
    > + return true;
    > +}
    > +
    > +/*
    > + * Helper routine for uprobe_report_signal().
    > + * We get called here with:
    > + * state = UPTASK_RUNNING => we are here due to a breakpoint hit
    > + * - Read-lock the process
    > + * - Figure out which probepoint, based on regs->IP
    > + * - Set state = UPTASK_BP_HIT
    > + * - Invoke handler for each uprobe at this probepoint
    > + * - Reset regs->IP to beginning of the insn, if necessary
    > + * - Start watching for quiesce events, in case another
    > + * engine cancels our UTRACE_SINGLESTEP with a
    > + * UTRACE_STOP.
    > + * - Set singlestep in motion (UTRACE_SINGLESTEP),
    > + * with state = UPTASK_SSTEP
    > + * - Read-unlock the process
    > + *
    > + * state = UPTASK_SSTEP => here after single-stepping
    > + * - Read-lock the process
    > + * - Validate we are here per the state machine
    > + * - Clean up after single-stepping
    > + * - Set state = UPTASK_RUNNING
    > + * - Read-unlock the process
    > + * - If it's time to quiesce, take appropriate action.
    > + * - If the handler(s) we ran called [un]register_uprobe(),
    > + * complete those via uprobe_run_def_regs().
    > + *
    > + * state = ANY OTHER STATE
    > + * - Not our signal, pass it on (UTRACE_RESUME)
    > + */
    > +static u32 uprobe_handle_signal(u32 action,
    > + struct uprobe_task *utask,
    > + struct pt_regs *regs,
    > + siginfo_t *info,
    > + const struct k_sigaction *orig_ka)
    > +{
    > + struct uprobe_probept *ppt;
    > + struct uprobe_process *uproc;
    > + struct uprobe_kimg *uk;
    > + unsigned long probept;
    > + enum utrace_resume_action resume_action;
    > + enum utrace_signal_action signal_action = utrace_signal_action(action);
    > +
    > + uproc = utask->uproc;
    > +
    > + /*
    > + * We may need to re-assert UTRACE_SINGLESTEP if this signal
    > + * is not associated with the breakpoint.
    > + */
    > + if (utask->state == UPTASK_SSTEP)
    > + resume_action = UTRACE_SINGLESTEP;
    > + else
    > + resume_action = UTRACE_RESUME;
    > + /*
    > + * This might be UTRACE_SIGNAL_REPORT request but some other
    > + * engine's callback might have changed the signal action to
    > + * something other than UTRACE_SIGNAL_REPORT. Use orig_ka to figure
    > + * out such cases.
    > + */
    > + if (unlikely(signal_action == UTRACE_SIGNAL_REPORT) || !orig_ka) {
    > + /* This thread was quiesced using UTRACE_INTERRUPT. */
    > + bool done_quiescing;
    > + if (utask->active_probe)
    > + /*
    > + * We'll fake quiescence after we're done
    > + * processing the probepoint.
    > + */
    > + return UTRACE_SIGNAL_IGN | resume_action;
    > +
    > + down_write(&uproc->rwsem);
    > + done_quiescing = utask_quiesce(utask);
    > + up_write(&uproc->rwsem);
    > + if (done_quiescing)
    > + resume_action = UTRACE_RESUME;
    > + else
    > + resume_action = UTRACE_STOP;
    > + return UTRACE_SIGNAL_IGN | resume_action;
    > + }
    > +
    > + /*
    > + * info will be null if we're called with action=UTRACE_SIGNAL_HANDLER,
    > + * which means that single-stepping has been disabled so a signal
    > + * handler can be called in the probed process. That should never
    > + * happen because we intercept and delay handled signals (action =
    > + * UTRACE_RESUME) until after we're done single-stepping.
    > + */
    > + BUG_ON(!info);
    > + if (signal_action == UTRACE_SIGNAL_DELIVER && utask->active_probe &&
    > + info->si_signo != SSTEP_SIGNAL) {
    > + uprobe_delay_signal(utask, info);
    > + return UTRACE_SIGNAL_IGN | UTRACE_SINGLESTEP;
    > + }
    > +
    > + if (info->si_signo != BREAKPOINT_SIGNAL &&
    > + info->si_signo != SSTEP_SIGNAL)
    > + goto no_interest;
    > +
    > + switch (utask->state) {
    > + case UPTASK_RUNNING:
    > + if (info->si_signo != BREAKPOINT_SIGNAL)
    > + goto no_interest;
    > +
    > +#ifdef CONFIG_UBP_XOL
    > + /*
    > + * Set up the XOL area if it's not already there. We
    > + * do this here because we have to do it before
    > + * handling the first probepoint hit, the probed
    > + * process has to do it, and this may be the first
    > + * time our probed process runs uprobes code. We need
    > + * the XOL area for the uretprobe trampoline even if
    > + * this architectures doesn't single-step out of line.
    > + */
    > + if (uproc->sstep_out_of_line && !uproc->xol_area) {
    > + uproc->xol_area = xol_get_area(uproc->tg_leader);
    > + if (unlikely(uproc->sstep_out_of_line) &&
    > + unlikely(!uproc->xol_area))
    > + uproc_cancel_xol(uproc);
    > + }
    > +#endif
    > +
    > + down_read(&uproc->rwsem);
    > + /* Don't quiesce while running handlers. */
    > + clear_utrace_quiesce(utask, false);
    > + probept = ubp_get_bkpt_addr(regs);
    > + ppt = uprobe_find_probept(uproc, probept);
    > + if (!ppt) {
    > + up_read(&uproc->rwsem);
    > + goto no_interest;
    > + }
    > + utask->active_probe = ppt;
    > + utask->state = UPTASK_BP_HIT;
    > +
    > + if (likely(ppt->state == UPROBE_BP_SET)) {
    > + list_for_each_entry(uk, &ppt->uprobe_list, list) {
    > + struct uprobe *u = uk->uprobe;
    > + if (u->handler)
    > + u->handler(u, regs);
    > + }
    > + }
    > +
    > +#ifdef CONFIG_UBP_XOL
    > + if ((ppt->ubp.strategy & UBP_HNT_INLINE) ||
    > + uprobe_pre_ssout(utask, ppt, regs) != 0)
    > +#endif
    > + uprobe_pre_ssin(utask, ppt, regs);
    > + if (unlikely(utask->doomed)) {
    > + utask->active_probe = NULL;
    > + utask->state = UPTASK_RUNNING;
    > + up_read(&uproc->rwsem);
    > + goto no_interest;
    > + }
    > + utask->state = UPTASK_SSTEP;
    > + /* In case another engine cancels our UTRACE_SINGLESTEP... */
    > + utask_adjust_flags(utask, UPROBE_SET_FLAGS,
    > + UTRACE_EVENT(QUIESCE));
    > + /* Don't deliver this signal to the process. */
    > + resume_action = UTRACE_SINGLESTEP;
    > + signal_action = UTRACE_SIGNAL_IGN;
    > +
    > + up_read(&uproc->rwsem);
    > + break;
    > +
    > + case UPTASK_SSTEP:
    > + if (info->si_signo != SSTEP_SIGNAL)
    > + goto no_interest;
    > +
    > + down_read(&uproc->rwsem);
    > + ppt = utask->active_probe;
    > + BUG_ON(!ppt);
    > +
    > + /*
    > + * Havent singlestepped yet? then re-assert
    > + * UTRACE_SINGLESTEP.
    > + */
    > + if (!validate_and_post_sstep(utask, regs, ppt)) {
    > + up_read(&uproc->rwsem);
    > + goto no_interest;
    > + }
    > +
    > + /* No further need to re-assert UTRACE_SINGLESTEP. */
    > + clear_utrace_quiesce(utask, false);
    > +
    > + utask->active_probe = NULL;
    > + utask->state = UPTASK_RUNNING;
    > + if (unlikely(utask->doomed)) {
    > + up_read(&uproc->rwsem);
    > + goto no_interest;
    > + }
    > +
    > + if (utask->quiescing) {
    > + int uproc_freed;
    > + up_read(&uproc->rwsem);
    > + uproc_freed = utask_fake_quiesce(utask);
    > + BUG_ON(uproc_freed);
    > + } else
    > + up_read(&uproc->rwsem);
    > +
    > + /*
    > + * We hold a ref count on uproc, so this should never
    > + * make utask or uproc disappear.
    > + */
    > + uprobe_run_def_regs(&utask->deferred_registrations);
    > +
    > + uprobe_inject_delayed_signals(&utask->delayed_signals);
    > +
    > + resume_action = UTRACE_RESUME;
    > + signal_action = UTRACE_SIGNAL_IGN;
    > + break;
    > + default:
    > + goto no_interest;
    > + }
    > +
    > +no_interest:
    > + return signal_action | resume_action;
    > +}
    > +
    > +/*
    > + * Signal callback:
    > + */
    > +static u32 uprobe_report_signal(u32 action,
    > + struct utrace_engine *engine,
    > + struct pt_regs *regs,
    > + siginfo_t *info,
    > + const struct k_sigaction *orig_ka,
    > + struct k_sigaction *return_ka)
    > +{
    > + struct uprobe_task *utask;
    > + struct uprobe_process *uproc;
    > + bool doomed;
    > + enum utrace_resume_action report_action;
    > +
    > + utask = (struct uprobe_task *)rcu_dereference(engine->data);

    Are we really in an RCU read-side critical section here?

    > + BUG_ON(!utask);
    > + uproc = utask->uproc;
    > +
    > + /* Keep uproc intact until just before we return. */
    > + uprobe_get_process(uproc);
    > + report_action = uprobe_handle_signal(action, utask, regs, info,
    > + orig_ka);
    > + doomed = utask->doomed;
    > +
    > + if (uprobe_put_process(uproc, true))
    > + report_action = utrace_signal_action(report_action) |
    > + UTRACE_DETACH;
    > + if (doomed)
    > + do_exit(SIGSEGV);
    > + return report_action;
    > +}
    > +
    > +/*
    > + * Quiesce callback: The associated process has one or more breakpoint
    > + * insertions or removals pending. If we're the last thread in this
    > + * process to quiesce, do the insertion(s) and/or removal(s).
    > + */
    > +static u32 uprobe_report_quiesce(u32 action,
    > + struct utrace_engine *engine,
    > + unsigned long event)
    > +{
    > + struct uprobe_task *utask;
    > + struct uprobe_process *uproc;
    > + bool done_quiescing = false;
    > +
    > + utask = (struct uprobe_task *)rcu_dereference(engine->data);

    Are we really in an RCU read-side critical section here?

    > + BUG_ON(!utask);
    > +
    > + if (utask->state == UPTASK_SSTEP)
    > + /*
    > + * We got a breakpoint trap and tried to single-step,
    > + * but somebody else's report_signal callback overrode
    > + * our UTRACE_SINGLESTEP with a UTRACE_STOP. Try again.
    > + */
    > + return UTRACE_SINGLESTEP;
    > +
    > + BUG_ON(utask->active_probe);
    > + uproc = utask->uproc;
    > + down_write(&uproc->rwsem);
    > + done_quiescing = utask_quiesce(utask);
    > + up_write(&uproc->rwsem);
    > + return done_quiescing ? UTRACE_RESUME : UTRACE_STOP;
    > +}
    > +
    > +/*
    > + * uproc's process is exiting or exec-ing. Runs with uproc->rwsem
    > + * write-locked. Caller must ref-count uproc before calling this
    > + * function, to ensure that uproc doesn't get freed in the middle of
    > + * this.
    > + */
    > +static void uprobe_cleanup_process(struct uprobe_process *uproc)
    > +{
    > + struct hlist_node *pnode1, *pnode2;
    > + struct uprobe_kimg *uk, *unode;
    > + struct uprobe_probept *ppt;
    > + struct hlist_head *head;
    > + int i;
    > +
    > + uproc->finished = true;
    > + for (i = 0; i < UPROBE_TABLE_SIZE; i++) {
    > + head = &uproc->uprobe_table[i];
    > + hlist_for_each_entry_safe(ppt, pnode1, pnode2, head, ut_node) {
    > + if (ppt->state == UPROBE_INSERTING ||
    > + ppt->state == UPROBE_REMOVING) {
    > + /*
    > + * This task is (exec/exit)ing with
    > + * a [un]register_uprobe pending.
    > + * [un]register_uprobe will free ppt.
    > + */
    > + ppt->state = UPROBE_DISABLED;
    > + list_del(&ppt->pd_node);
    > + list_for_each_entry_safe(uk, unode,
    > + &ppt->uprobe_list, list)
    > + uk->status = -ESRCH;
    > + wake_up_all(&ppt->waitq);
    > + } else if (ppt->state == UPROBE_BP_SET) {
    > + list_for_each_entry_safe(uk, unode,
    > + &ppt->uprobe_list, list) {
    > + list_del(&uk->list);
    > + uprobe_free_kimg(uk);
    > + }
    > + uprobe_free_probept(ppt);
    > + /* else */
    > + /*
    > + * If ppt is UPROBE_DISABLED, assume that
    > + * [un]register_uprobe() has been notified
    > + * and will free it soon.
    > + */
    > + }
    > + }
    > + }
    > +}
    > +
    > +static u32 uprobe_exec_exit(struct utrace_engine *engine,
    > + struct task_struct *tsk, int exit)
    > +{
    > + struct uprobe_process *uproc;
    > + struct uprobe_probept *ppt;
    > + struct uprobe_task *utask;
    > + bool utask_quiescing;
    > +
    > + utask = (struct uprobe_task *)rcu_dereference(engine->data);

    Are we really in an RCU read-side critical section here?

    > + uproc = utask->uproc;
    > + uprobe_get_process(uproc);
    > +
    > + ppt = utask->active_probe;
    > + if (ppt) {
    > + printk(KERN_WARNING "Task handler called %s while at uprobe"
    > + " probepoint: pid/tgid = %d/%d, probepoint"
    > + " = %#lx\n", (exit ? "exit" : "exec"),
    > + tsk->pid, tsk->tgid, ppt->ubp.vaddr);
    > + /*
    > + * Mutex cleanup depends on where do_execve()/do_exit() was
    > + * called and on ubp strategy (XOL vs. SSIL).
    > + */
    > + if (ppt->ubp.strategy & UBP_HNT_INLINE) {
    > + switch (utask->state) {
    > + unsigned long flags;
    > + case UPTASK_SSTEP:
    > + spin_lock_irqsave(&ppt->ssil_lock, flags);
    > + ppt->ssil_state = SSIL_CLEAR;
    > + wake_up(&ppt->ssilq);
    > + spin_unlock_irqrestore(&ppt->ssil_lock, flags);
    > + break;
    > + default:
    > + break;
    > + }
    > + }
    > + if (utask->state == UPTASK_BP_HIT) {
    > + /* uprobe handler called do_exit()/do_execve(). */
    > + up_read(&uproc->rwsem);
    > + uprobe_decref_process(uproc);
    > + }
    > + }
    > +
    > + down_write(&uproc->rwsem);
    > + utask_quiescing = utask->quiescing;
    > + uproc->nthreads--;
    > + if (utrace_set_events_pid(utask->pid, engine, 0))
    > + /* We don't care. */
    > + ;
    > + uprobe_free_task(utask, 1);
    > + if (uproc->nthreads) {
    > + /*
    > + * In case other threads are waiting for us to quiesce...
    > + */
    > + if (utask_quiescing)
    > + (void) check_uproc_quiesced(uproc,
    > + find_surviving_thread(uproc));
    > + } else
    > + /*
    > + * We were the last remaining thread - clean up the uprobe
    > + * remnants a la unregister_uprobe(). We don't have to
    > + * remove the breakpoints, though.
    > + */
    > + uprobe_cleanup_process(uproc);
    > +
    > + up_write(&uproc->rwsem);
    > + uprobe_put_process(uproc, true);
    > + return UTRACE_DETACH;
    > +}
    > +
    > +/*
    > + * Exit callback: The associated task/thread is exiting.
    > + */
    > +static u32 uprobe_report_exit(u32 action,
    > + struct utrace_engine *engine,
    > + long orig_code, long *code)
    > +{
    > + return uprobe_exec_exit(engine, current, 1);
    > +}
    > +/*
    > + * Clone callback: The current task has spawned a thread/process.
    > + * Utrace guarantees that parent and child pointers will be valid
    > + * for the duration of this callback.
    > + *
    > + * NOTE: For now, we don't pass on uprobes from the parent to the
    > + * child. We now do the necessary clearing of breakpoints in the
    > + * child's address space.
    > + *
    > + * TODO:
    > + * - Provide option for child to inherit uprobes.
    > + */
    > +static u32 uprobe_report_clone(u32 action,
    > + struct utrace_engine *engine,
    > + unsigned long clone_flags,
    > + struct task_struct *child)
    > +{
    > + struct uprobe_process *uproc;
    > + struct uprobe_task *ptask, *ctask;
    > +
    > + ptask = (struct uprobe_task *)rcu_dereference(engine->data);

    Are we really in an RCU read-side critical section here?

    > + uproc = ptask->uproc;
    > +
    > + /*
    > + * Lock uproc so no new uprobes can be installed 'til all
    > + * report_clone activities are completed.
    > + */
    > + mutex_lock(&uproc_mutex);
    > + down_write(&uproc->rwsem);
    > +
    > + if (clone_flags & CLONE_THREAD) {
    > + /* New thread in the same process. */
    > + ctask = uprobe_find_utask(child);
    > + if (unlikely(ctask)) {
    > + /*
    > + * uprobe_mk_process() ran just as this clone
    > + * happened, and has already accounted for the
    > + * new child.
    > + */
    > + } else {
    > + struct pid *child_pid = get_pid(task_pid(child));
    > + BUG_ON(!child_pid);
    > + ctask = uprobe_add_task(child_pid, uproc);
    > + BUG_ON(!ctask);
    > + if (IS_ERR(ctask))
    > + goto done;
    > + uproc->nthreads++;
    > + /*
    > + * FIXME: Handle the case where uproc is quiescing
    > + * (assuming it's possible to clone while quiescing).
    > + */
    > + }
    > + } else {
    > + /*
    > + * New process spawned by parent. Remove the probepoints
    > + * in the child's text.
    > + *
    > + * Its not necessary to quiesce the child as we are assured
    > + * by utrace that this callback happens *before* the child
    > + * gets to run userspace.
    > + *
    > + * We also hold the uproc->rwsem for the parent - so no
    > + * new uprobes will be registered 'til we return.
    > + */
    > + int i;
    > + struct uprobe_probept *ppt;
    > + struct hlist_node *node;
    > + struct hlist_head *head;
    > +
    > + for (i = 0; i < UPROBE_TABLE_SIZE; i++) {
    > + head = &uproc->uprobe_table[i];
    > + hlist_for_each_entry(ppt, node, head, ut_node) {
    > + if (ubp_remove_bkpt(child, &ppt->ubp) != 0) {
    > + /* Ratelimit this? */
    > + printk(KERN_ERR "Pid %d forked %d;"
    > + " failed to remove probepoint"
    > + " at %#lx in child\n",
    > + current->pid, child->pid,
    > + ppt->ubp.vaddr);
    > + }
    > + }
    > + }
    > + }
    > +
    > +done:
    > + up_write(&uproc->rwsem);
    > + mutex_unlock(&uproc_mutex);
    > + return UTRACE_RESUME;
    > +}
    > +
    > +/*
    > + * Exec callback: The associated process called execve() or friends
    > + *
    > + * The new program is about to start running and so there is no
    > + * possibility of a uprobe from the previous user address space
    > + * to be hit.
    > + *
    > + * NOTE:
    > + * Typically, this process would have passed through the clone
    > + * callback, where the necessary action *should* have been
    > + * taken. However, if we still end up at this callback:
    > + * - We don't have to clear the uprobes - memory image
    > + * will be overlaid.
    > + * - We have to free up uprobe resources associated with
    > + * this process.
    > + */
    > +static u32 uprobe_report_exec(u32 action,
    > + struct utrace_engine *engine,
    > + const struct linux_binfmt *fmt,
    > + const struct linux_binprm *bprm,
    > + struct pt_regs *regs)
    > +{
    > + return uprobe_exec_exit(engine, current, 0);
    > +}
    > +
    > +static const struct utrace_engine_ops uprobe_utrace_ops = {
    > + .report_quiesce = uprobe_report_quiesce,
    > + .report_signal = uprobe_report_signal,
    > + .report_exit = uprobe_report_exit,
    > + .report_clone = uprobe_report_clone,
    > + .report_exec = uprobe_report_exec
    > +};
    > +
    > +static int __init init_uprobes(void)
    > +{
    > + int ret, i;
    > +
    > + ubp_strategies = UBP_HNT_TSKINFO;
    > + ret = ubp_init(&ubp_strategies);
    > + if (ret != 0) {
    > + printk(KERN_ERR "Can't start uprobes: ubp_init() returned %d\n",
    > + ret);
    > + return ret;
    > + }
    > + for (i = 0; i < UPROBE_TABLE_SIZE; i++) {
    > + INIT_HLIST_HEAD(&uproc_table[i]);
    > + INIT_HLIST_HEAD(&utask_table[i]);
    > + }
    > +
    > + p_uprobe_utrace_ops = &uprobe_utrace_ops;
    > + return 0;
    > +}
    > +
    > +static void __exit exit_uprobes(void)
    > +{
    > +}
    > +
    > +module_init(init_uprobes);
    > +module_exit(exit_uprobes);
    > --
    > To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
    > the body of a message to majordomo@vger.kernel.org
    > More majordomo info at http://vger.kernel.org/majordomo-info.html
    > Please read the FAQ at http://www.tux.org/lkml/


    \
     
     \ /
      Last update: 2010-01-12 03:05    [W:0.314 / U:0.128 seconds]
    ©2003-2016 Jasper Spaans. hosted at Digital OceanAdvertise on this site