crash vmcore

　　目前问题为：内核出现coredump 需要分析coredump， 根据堆栈分析为内核唤醒内核进程/线程的时候，在内核太发生缺页中断触发panic

目前可以参考以前的以下文章：copy_from_user以及缺页中断     缺页中断分析

　　根据crash 我们可以拿到函数调用栈也就是栈地址，但是栈数据怎么获取呢？ 其分布是怎样的呢？ 函数出入的参数是怎样的呢？

有如下命令

bt -f /* 打印函数栈数据 */
/* 函数栈内自底向上，自右向左存储数据。
1.右下角为第一个数据：返回到 上一级函数的继续执行地址。 
2.左下角为第二个数据，当前函数的栈底地址，返回时使用。
*/

 

X86-64有16个64位寄存器，分别是：

%rax，%rbx，%rcx，%rdx，%esi，%edi，%rbp，%rsp，%r8，%r9，%r10，%r11，%r12，%r13，%r14，%r15。

其中：

• %rax 作为函数返回值使用。
• %rsp 栈指针寄存器，指向栈顶
• %rdi，%rsi，%rdx，%rcx，%r8，%r9 用作函数参数，依次对应第1参数，第2参数。。。
• %rbx，%rbp，%r12，%r13，%14，%15 用作数据存储，遵循被调用者使用规则，简单说就是随便用，调用子函数之前要备份它，以防他被修改
• %r10，%r11 用作数据存储，遵循调用者使用规则，简单说就是使用之前要先保存原值

根据分析 ： 

static int
select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag, int wake_flags)

其中 函数调用的第二栈地址为0xffff8810792b9a40  也许是错的只能慢慢整了

 

 详细结果如下：

其中：sd_flag = 0； wake_flags = 1

第一个参数：

struct rq ffff88107fc73480 -x
struct rq {
  lock = {
    raw_lock = {
      slock = 0x821081f
    }
  }, 
  nr_running = 0x0, 
  cpu_load = {0x132, 0x99, 0x4d, 0x27, 0x14}, 
  last_load_update_tick = 0x10019e92e, 
  skip_clock_update = 0x0, 
  load = {
    weight = 0x0, 
    inv_weight = 0x0
  }, 
  nr_load_updates = 0x1e7de7, 
  nr_switches = 0x63517, 
  cfs = {
    load = {
      weight = 0x0, 
      inv_weight = 0x0
    }, 
    nr_running = 0x0, 
    exec_clock = 0x0, 
    min_vruntime = 0xd64aeb9f6, 
    tasks_timeline = {
      rb_node = 0x0
    }, 
    rb_leftmost = 0x0, 
    tasks = {
      next = 0xffff88107fc73520, 
      prev = 0xffff88107fc73520
    }, 
    balance_iterator = 0x0, 
    curr = 0x0, 
    next = 0x0, 
    last = 0x0, 
    skip = 0x0, 
    nr_spread_over = 0x0, 
    rq = 0xffff88107fc73480, 
    on_list = 0x1, 
    leaf_cfs_rq_list = {
      next = 0xffff88107fc73c70, 
      prev = 0xffff8810362b0d88
    }, 
    tg = 0xffffffff81a9f6f0 <root_task_group>, 
    task_weight = 0x0, 
    h_load = 0x3c3, 
    load_avg = 0x0, 
    load_period = 0x0, 
    load_stamp = 0x1, 
    load_last = 0x0, 
    load_unacc_exec_time = 0x42b221936, 
    load_contribution = 0x0
  }, 
  rt = {
    active = {
      bitmap = {0x0, 0x1000000000}, 
      queue = {{
          next = 0xffff88107fc735d8, 
          prev = 0xffff88107fc735d8
        }, {
          next = 0xffff88107fc735e8, 
          prev = 0xffff88107fc735e8
        }, {
          next = 0xffff88107fc735f8, 
          prev = 0xffff88107fc735f8
        }, {
          next = 0xffff88107fc73608, 
          prev = 0xffff88107fc73608
        }, {
          next = 0xffff88107fc73618, 
          prev = 0xffff88107fc73618
        }, {
          next = 0xffff88107fc73628, 
          prev = 0xffff88107fc73628
        }, {
          next = 0xffff88107fc73638, 
          prev = 0xffff88107fc73638
        }, {
          next = 0xffff88107fc73648, 
          prev = 0xffff88107fc73648
        }, {
          next = 0xffff88107fc73658, 
          prev = 0xffff88107fc73658
        }, {
          next = 0xffff88107fc73668, 
          prev = 0xffff88107fc73668
        }, {
          next = 0xffff88107fc73678, 
          prev = 0xffff88107fc73678
        }, {
          next = 0xffff88107fc73688, 
          prev = 0xffff88107fc73688
        }, {
          next = 0xffff88107fc73698, 
          prev = 0xffff88107fc73698
        }, {
          next = 0xffff88107fc736a8, 
          prev = 0xffff88107fc736a8
        }, {
          next = 0xffff88107fc736b8, 
          prev = 0xffff88107fc736b8
        }, {
          next = 0xffff88107fc736c8, 
          prev = 0xffff88107fc736c8
        }, {
          next = 0xffff88107fc736d8, 
          prev = 0xffff88107fc736d8
        }, {
          next = 0xffff88107fc736e8, 
          prev = 0xffff88107fc736e8
        }, {
          next = 0xffff88107fc736f8, 
          prev = 0xffff88107fc736f8
        }, {
          next = 0xffff88107fc73708, 
          prev = 0xffff88107fc73708
        }, {
          next = 0xffff88107fc73718, 
          prev = 0xffff88107fc73718
        }, {
          next = 0xffff88107fc73728, 
          prev = 0xffff88107fc73728
        }, {
          next = 0xffff88107fc73738, 
          prev = 0xffff88107fc73738
        }, {
          next = 0xffff88107fc73748, 
          prev = 0xffff88107fc73748
        }, {
          next = 0xffff88107fc73758, 
          prev = 0xffff88107fc73758
        }, {
          next = 0xffff88107fc73768, 
          prev = 0xffff88107fc73768
        }, {
          next = 0xffff88107fc73778, 
          prev = 0xffff88107fc73778
        }, {
          next = 0xffff88107fc73788, 
          prev = 0xffff88107fc73788
        }, {
          next = 0xffff88107fc73798, 
          prev = 0xffff88107fc73798
        }, {
          next = 0xffff88107fc737a8, 
          prev = 0xffff88107fc737a8
        }, {
          next = 0xffff88107fc737b8, 
          prev = 0xffff88107fc737b8
        }, {
          next = 0xffff88107fc737c8, 
          prev = 0xffff88107fc737c8
        }, {
          next = 0xffff88107fc737d8, 
          prev = 0xffff88107fc737d8
        }, {
          next = 0xffff88107fc737e8, 
          prev = 0xffff88107fc737e8
        }, {
          next = 0xffff88107fc737f8, 
          prev = 0xffff88107fc737f8
        }, {
          next = 0xffff88107fc73808, 
          prev = 0xffff88107fc73808
        }, {
          next = 0xffff88107fc73818, 
          prev = 0xffff88107fc73818
        }, {
          next = 0xffff88107fc73828, 
          prev = 0xffff88107fc73828
        }, {
          next = 0xffff88107fc73838, 
          prev = 0xffff88107fc73838
        }, {
          next = 0xffff88107fc73848, 
          prev = 0xffff88107fc73848
        }, {
          next = 0xffff88107fc73858, 
          prev = 0xffff88107fc73858
        }, {
          next = 0xffff88107fc73868, 
          prev = 0xffff88107fc73868
        }, {
          next = 0xffff88107fc73878, 
          prev = 0xffff88107fc73878
        }, {
          next = 0xffff88107fc73888, 
          prev = 0xffff88107fc73888
        }, {
          next = 0xffff88107fc73898, 
          prev = 0xffff88107fc73898
        }, {
          next = 0xffff88107fc738a8, 
          prev = 0xffff88107fc738a8
        }, {
          next = 0xffff88107fc738b8, 
          prev = 0xffff88107fc738b8
        }, {
          next = 0xffff88107fc738c8, 
          prev = 0xffff88107fc738c8
        }, {
          next = 0xffff88107fc738d8, 
          prev = 0xffff88107fc738d8
        }, {
          next = 0xffff88107fc738e8, 
          prev = 0xffff88107fc738e8
        }, {
          next = 0xffff88107fc738f8, 
          prev = 0xffff88107fc738f8
        }, {
          next = 0xffff88107fc73908, 
          prev = 0xffff88107fc73908
        }, {
          next = 0xffff88107fc73918, 
          prev = 0xffff88107fc73918
        }, {
          next = 0xffff88107fc73928, 
          prev = 0xffff88107fc73928
        }, {
          next = 0xffff88107fc73938, 
          prev = 0xffff88107fc73938
        }, {
          next = 0xffff88107fc73948, 
          prev = 0xffff88107fc73948
        }, {
          next = 0xffff88107fc73958, 
          prev = 0xffff88107fc73958
        }, {
          next = 0xffff88107fc73968, 
          prev = 0xffff88107fc73968
        }, {
          next = 0xffff88107fc73978, 
          prev = 0xffff88107fc73978
        }, {
          next = 0xffff88107fc73988, 
          prev = 0xffff88107fc73988
        }, {
          next = 0xffff88107fc73998, 
          prev = 0xffff88107fc73998
        }, {
          next = 0xffff88107fc739a8, 
          prev = 0xffff88107fc739a8
        }, {
          next = 0xffff88107fc739b8, 
          prev = 0xffff88107fc739b8
        }, {
          next = 0xffff88107fc739c8, 
          prev = 0xffff88107fc739c8
        }, {
          next = 0xffff88107fc739d8, 
          prev = 0xffff88107fc739d8
        }, {
          next = 0xffff88107fc739e8, 
          prev = 0xffff88107fc739e8
        }, {
          next = 0xffff88107fc739f8, 
          prev = 0xffff88107fc739f8
        }, {
          next = 0xffff88107fc73a08, 
          prev = 0xffff88107fc73a08
        }, {
          next = 0xffff88107fc73a18, 
          prev = 0xffff88107fc73a18
        }, {
          next = 0xffff88107fc73a28, 
          prev = 0xffff88107fc73a28
        }, {
          next = 0xffff88107fc73a38, 
          prev = 0xffff88107fc73a38
        }, {
          next = 0xffff88107fc73a48, 
          prev = 0xffff88107fc73a48
        }, {
          next = 0xffff88107fc73a58, 
          prev = 0xffff88107fc73a58
        }, {
          next = 0xffff88107fc73a68, 
          prev = 0xffff88107fc73a68
        }, {
          next = 0xffff88107fc73a78, 
          prev = 0xffff88107fc73a78
        }, {
          next = 0xffff88107fc73a88, 
          prev = 0xffff88107fc73a88
        }, {
          next = 0xffff88107fc73a98, 
          prev = 0xffff88107fc73a98
        }, {
          next = 0xffff88107fc73aa8, 
          prev = 0xffff88107fc73aa8
        }, {
          next = 0xffff88107fc73ab8, 
          prev = 0xffff88107fc73ab8
        }, {
          next = 0xffff88107fc73ac8, 
          prev = 0xffff88107fc73ac8
        }, {
          next = 0xffff88107fc73ad8, 
          prev = 0xffff88107fc73ad8
        }, {
          next = 0xffff88107fc73ae8, 
          prev = 0xffff88107fc73ae8
        }, {
          next = 0xffff88107fc73af8, 
          prev = 0xffff88107fc73af8
        }, {
          next = 0xffff88107fc73b08, 
          prev = 0xffff88107fc73b08
        }, {
          next = 0xffff88107fc73b18, 
          prev = 0xffff88107fc73b18
        }, {
          next = 0xffff88107fc73b28, 
          prev = 0xffff88107fc73b28
        }, {
          next = 0xffff88107fc73b38, 
          prev = 0xffff88107fc73b38
        }, {
          next = 0xffff88107fc73b48, 
          prev = 0xffff88107fc73b48
        }, {
          next = 0xffff88107fc73b58, 
          prev = 0xffff88107fc73b58
        }, {
          next = 0xffff88107fc73b68, 
          prev = 0xffff88107fc73b68
        }, {
          next = 0xffff88107fc73b78, 
          prev = 0xffff88107fc73b78
        }, {
          next = 0xffff88107fc73b88, 
          prev = 0xffff88107fc73b88
        }, {
          next = 0xffff88107fc73b98, 
          prev = 0xffff88107fc73b98
        }, {
          next = 0xffff88107fc73ba8, 
          prev = 0xffff88107fc73ba8
        }, {
          next = 0xffff88107fc73bb8, 
          prev = 0xffff88107fc73bb8
        }, {
          next = 0xffff88107fc73bc8, 
          prev = 0xffff88107fc73bc8
        }, {
          next = 0xffff88107fc73bd8, 
          prev = 0xffff88107fc73bd8
        }, {
          next = 0xffff88107fc73be8, 
          prev = 0xffff88107fc73be8
        }, {
          next = 0xffff88107fc73bf8, 
          prev = 0xffff88107fc73bf8
        }, {
          next = 0xffff88107fc73c08, 
          prev = 0xffff88107fc73c08
        }}
    }, 
    rt_nr_running = 0x0, 
    highest_prio = {
      curr = 0x64, 
      next = 0x64
    }, 
    rt_nr_migratory = 0x0, 
    rt_nr_total = 0x0, 
    overloaded = 0x0, 
    pushable_tasks = {
      node_list = {
        next = 0xffff88107fc73c40, 
        prev = 0xffff88107fc73c40
      }
    }, 
    rt_throttled = 0x0, 
    rt_time = 0x0, 
    rt_runtime = 0x389fd980, 
    rt_runtime_lock = {
      raw_lock = {
        slock = 0x1f801f8
      }
    }
  }, 
  leaf_cfs_rq_list = {
    next = 0xffff88103af91b88, 
    prev = 0xffff88107fc73570
  }, 
  nr_uninterruptible = 0x0, 
  curr = 0xffff8810796c0d20, 
  idle = 0xffff8810796c0d20, 
  stop = 0xffff881079567620, 
  next_balance = 0x10019e930, 
  prev_mm = 0x0, 
  clock = 0x1d06193af03, 
  clock_task = 0x1d06193af03, 
  nr_iowait = {
    counter = 0x0
  }, 
  rd = 0xffff8820792bc000, 
  sd = 0xffff88107fc6f240, 
  cpu_power = 0x400, 
  idle_at_tick = 0x0, 
  post_schedule = 0x0, 
  active_balance = 0x0, 
  push_cpu = 0x7, 
  active_balance_work = {
    list = {
      next = 0xffff88107fc73cf0, 
      prev = 0xffff88107fc73cf0
    }, 
    fn = 0xffffffff81043b54 <active_load_balance_cpu_stop>, 
    arg = 0xffff88107fc73480, 
    done = 0x0
  }, 
  cpu = 0x3, 
  online = 0x1, 
  avg_load_per_task = 0x1e1, 
  rt_avg = 0x1, 
  age_stamp = 0x1d06176c900, 
  idle_stamp = 0x1d06193aaa6, 
  avg_idle = 0x8b80d, 
  calc_load_update = 0x10019f230, 
  calc_load_active = 0x0, 
  hrtick_csd_pending = 0x0, 
  hrtick_csd = {
    list = {
      next = 0x0, 
      prev = 0x0
    }, 
    func = 0xffffffff8103e71c <__hrtick_start>, 
    info = 0xffff88107fc73480, 
    flags = 0x0, 
    priv = 0x0
  }, 
  hrtick_timer = {
    node = {
      node = {
        rb_parent_color = 0xffff88107fc73d88, 
        rb_right = 0x0, 
        rb_left = 0x0
      }, 
      expires = {
        tv64 = 0x0
      }
    }, 
    _softexpires = {
      tv64 = 0x0
    }, 
    function = 0xffffffff8103d264 <hrtick>, 
    base = 0xffff88107fc0fac8, 
    state = 0x0
  }
}

第二参数：

 struct task_struct ffff8810792b9a40 -x
struct task_struct {
  state = 0x100, 
  stack = 0xffff8810360ea000, 
  usage = {
    counter = 0x2
  }, 
  flags = 0x402040, 
  ptrace = 0x0, 
  lock_depth = 0xffffffff, 
  prio = 0x78, 
  static_prio = 0x78, 
  normal_prio = 0x78, 
  rt_priority = 0x0, 
  sched_class = 0xffffffff81602c30 <fair_sched_class>, 
  se = {
    load = {
      weight = 0x400, 
      inv_weight = 0x400000
    }, 
    run_node = {
      rb_parent_color = 0x1, 
      rb_right = 0x0, 
      rb_left = 0x0
    }, 
    group_node = {
      next = 0xffff8810792b9aa0, 
      prev = 0xffff8810792b9aa0
    }, 
    on_rq = 0x0, 
    exec_start = 0x1d06193aaa6, 
    sum_exec_runtime = 0xa2f02e4, 
    vruntime = 0xffffffffff49d69d, 
    prev_sum_exec_runtime = 0xa2e609d, 
    nr_migrations = 0x2, 
    parent = 0xffff88103e840a00, 
    cfs_rq = 0xffff88103af91b00, 
    my_q = 0x0
  }, 
  rt = {
    run_list = {
      next = 0xffff8810792b9af8, 
      prev = 0xffff8810792b9af8
    }, 
    timeout = 0x0, 
    time_slice = 0x3e8, 
    nr_cpus_allowed = 0x1, 
    back = 0x0
  }, 
  fpu_counter = 0x1, 
  policy = 0x0, 
  cpus_allowed = {
    bits = {0x8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}
  }, 
  sched_info = {
    pcount = 0x4fdf, 
    run_delay = 0x749c3d6, 
    last_arrival = 0x1d06193085f, 
    last_queued = 0x0
  }, 
  tasks = {
    next = 0xffff882077945698, 
    prev = 0xffff8810792214f8
  }, 
  pushable_tasks = {
    prio = 0x8c, 
    prio_list = {
      next = 0xffff8810792b9ba0, 
      prev = 0xffff8810792b9ba0
    }, 
    node_list = {
      next = 0xffff8810792b9bb0, 
      prev = 0xffff8810792b9bb0
    }
  }, 
  mm = 0xffff8820779b5780, 
  active_mm = 0xffff8820779b5780, 
  brk_randomized = 0x0, 
  rss_stat = {
    events = 0xd, 
    count = {0x5, 0x0, 0x0}
  }, 
  exit_state = 0x0, 
  exit_code = 0x0, 
  exit_signal = 0xffffffff, 
  pdeath_signal = 0x0, 
  personality = 0x0, 
  did_exec = 0x0, 
  in_execve = 0x0, 
  in_iowait = 0x0, 
  sched_reset_on_fork = 0x0, 
  pid = 0x188d, 
  tgid = 0x1803, 
  real_parent = 0xffff8810792213b0, 
  parent = 0xffff8810792213b0, 
  children = {
    next = 0xffff8810792b9c18, 
    prev = 0xffff8810792b9c18
  }, 
  sibling = {
    next = 0xffff8810792b9c28, 
    prev = 0xffff8810792b9c28
  }, 
  group_leader = 0xffff8820779420d0, 
  ptraced = {
    next = 0xffff8810792b9c40, 
    prev = 0xffff8810792b9c40
  }, 
  ptrace_entry = {
    next = 0xffff8810792b9c50, 
    prev = 0xffff8810792b9c50
  }, 
  pids = {{
      node = {
        next = 0x0, 
        pprev = 0xffff88103f06b208
      }, 
      pid = 0xffff88103f06b200
    }, {
      node = {
        next = 0xffff8810792215e8, 
        pprev = 0xffff88103f04cb10
      }, 
      pid = 0xffff88103f04cb00
    }, {
      node = {
        next = 0xffff881079221600, 
        pprev = 0xffff88103f04cb18
      }, 
      pid = 0xffff88103f04cb00
    }}, 
  thread_group = {
    next = 0xffff88103f3bde48, 
    prev = 0xffff88103f3beb68
  }, 
  vfork_done = 0x0, 
  set_child_tid = 0x0, 
  clear_child_tid = 0x7f6f900019d0, 
  utime = 0x1c, 
  stime = 0x43, 
  utimescaled = 0x1c, 
  stimescaled = 0x43, 
  gtime = 0x0, 
  prev_utime = 0x0, 
  prev_stime = 0x0, 
  nvcsw = 0x4fdf, 
  nivcsw = 0x0, 
  start_time = {
    tv_sec = 0x2b, 
    tv_nsec = 0x787e451
  }, 
  real_start_time = {
    tv_sec = 0x2b, 
    tv_nsec = 0x787e451
  }, 
  min_flt = 0x35, 
  maj_flt = 0x8, 
  cputime_expires = {
    utime = 0x0, 
    stime = 0x0, 
    sum_exec_runtime = 0x0
  }, 
  cpu_timers = {{
      next = 0xffff8810792b9d60, 
      prev = 0xffff8810792b9d60
    }, {
      next = 0xffff8810792b9d70, 
      prev = 0xffff8810792b9d70
    }, {
      next = 0xffff8810792b9d80, 
      prev = 0xffff8810792b9d80
    }}, 
  real_cred = 0xffff88203eb7bec0, 
  cred = 0xffff88203eb7bec0, 
  replacement_session_keyring = 0x0, 
  comm = "wafd0000000000000000000000", 
  link_count = 0x0, 
  total_link_count = 0x0, 
  sysvsem = {
    undo_list = 0xffff882077d3e440
  }, 
  thread = {
    tls_array = {{
        {
          {
            a = 0x0, 
            b = 0x0
          }, 
          {
            limit0 = 0x0, 
            base0 = 0x0, 
            base1 = 0x0, 
            type = 0x0, 
            s = 0x0, 
            dpl = 0x0, 
            p = 0x0, 
            limit = 0x0, 
            avl = 0x0, 
            l = 0x0, 
            d = 0x0, 
            g = 0x0, 
            base2 = 0x0
          }
        }
      }, {
        {
          {
            a = 0x0, 
            b = 0x0
          }, 
          {
            limit0 = 0x0, 
            base0 = 0x0, 
            base1 = 0x0, 
            type = 0x0, 
            s = 0x0, 
            dpl = 0x0, 
            p = 0x0, 
            limit = 0x0, 
            avl = 0x0, 
            l = 0x0, 
            d = 0x0, 
            g = 0x0, 
            base2 = 0x0
          }
        }
      }, {
        {
          {
            a = 0x0, 
            b = 0x0
          }, 
          {
            limit0 = 0x0, 
            base0 = 0x0, 
            base1 = 0x0, 
            type = 0x0, 
            s = 0x0, 
            dpl = 0x0, 
            p = 0x0, 
            limit = 0x0, 
            avl = 0x0, 
            l = 0x0, 
            d = 0x0, 
            g = 0x0, 
            base2 = 0x0
          }
        }
      }}, 
    sp0 = 0xffff8810360ec000, 
    sp = 0xffff8810360ebcf8, 
    usersp = 0x7f6f8fff0620, 
    es = 0x0, 
    ds = 0x0, 
    fsindex = 0x0, 
    gsindex = 0x0, 
    fs = 0x7f6f90001700, 
    gs = 0x0, 
    ptrace_bps = {0x0, 0x0, 0x0, 0x0}, 
    debugreg6 = 0x0, 
    ptrace_dr7 = 0x0, 
    cr2 = 0x0, 
    trap_no = 0x0, 
    error_code = 0x0, 
    fpu = {
      state = 0xffff88103ad82080
    }, 
    io_bitmap_ptr = 0x0, 
    iopl = 0x0, 
    io_bitmap_max = 0x0
  }, 
  fs = 0xffff882077b2d240, 
  files = 0xffff882077973f40, 
  nsproxy = 0xffffffff81972490 <init_nsproxy>, 
  signal = 0xffff88203c03cc00, 
  sighand = 0xffff88207797a940, 
  blocked = {
    sig = {0xfffffffe7ffbfa37}
  }, 
  real_blocked = {
    sig = {0x0}
  }, 
  saved_sigmask = {
    sig = {0x0}
  }, 
  pending = {
    list = {
      next = 0xffff8810792b9eb8, 
      prev = 0xffff8810792b9eb8
    }, 
    signal = {
      sig = {0x0}
    }
  }, 
  sas_ss_sp = 0x0, 
  sas_ss_size = 0x0, 
  notifier = 0x0, 
  notifier_data = 0x0, 
  notifier_mask = 0x0, 
  audit_context = 0x0, 
  seccomp = {
    mode = 0x0
  }, 
  parent_exec_id = 0x8, 
  self_exec_id = 0x8, 
  alloc_lock = {
    {
      rlock = {
        raw_lock = {
          slock = 0x20002
        }
      }
    }
  }, 
  irqaction = 0x0, 
  pi_lock = {
    raw_lock = {
      slock = 0x0
    }
  }, 
  pi_waiters = {
    node_list = {
      next = 0xffff8810792b9f20, 
      prev = 0xffff8810792b9f20
    }
  }, 
  pi_blocked_on = 0x0, 
  journal_info = 0x0, 
  bio_list = 0x0, 
  plug = 0x0, 
  reclaim_state = 0x0, 
  backing_dev_info = 0x0, 
  io_context = 0xffff88103ac966c0, 
  ptrace_message = 0x0, 
  last_siginfo = 0x0, 
  ioac = {
    rchar = 0xffa3, 
    wchar = 0x1000b, 
    syscr = 0x9d7, 
    syscw = 0xf38, 
    read_bytes = 0x24000, 
    write_bytes = 0x0, 
    cancelled_write_bytes = 0x0
  }, 
  acct_rss_mem1 = 0x109ba098f, 
  acct_vm_mem1 = 0x32aac4109c, 
  acct_timexpd = 0x5f, 
  mems_allowed = {
    bits = {0x3}
  }, 
  mems_allowed_change_disable = 0x0, 
  cpuset_mem_spread_rotor = 0x0, 
  cpuset_slab_spread_rotor = 0x0, 
  cgroups = 0xffffffff81aea2a0 <init_css_set>, 
  cg_list = {
    next = 0xffff8810792b9fe8, 
    prev = 0xffff8810792b9fe8
  }, 
  robust_list = 0x7f6f900019e0, 
  compat_robust_list = 0x0, 
  pi_state_list = {
    next = 0xffff8810792ba008, 
    prev = 0xffff8810792ba008
  }, 
  pi_state_cache = 0x0, 
  perf_event_ctxp = {0x0, 0x0}, 
  perf_event_mutex = {
    count = {
      counter = 0x1
    }, 
    wait_lock = {
      {
        rlock = {
          raw_lock = {
            slock = 0x0
          }
        }
      }
    }, 
    wait_list = {
      next = 0xffff8810792ba038, 
      prev = 0xffff8810792ba038
    }, 
    owner = 0x0
  }, 
  perf_event_list = {
    next = 0xffff8810792ba050, 
    prev = 0xffff8810792ba050
  }, 
  mempolicy = 0x0, 
  il_next = 0x1, 
  pref_node_fork = 0x0, 
  fs_excl = {
    counter = 0x0
  }, 
  rcu = {
    next = 0x0, 
    func = 0x0
  }, 
  splice_pipe = 0x0, 
  delays = 0xffff881079555f50, 
  dirties = {
    events = 0x0, 
    period = 0x0, 
    shift = 0x0, 
    lock = {
      {
        rlock = {
          raw_lock = {
            slock = 0x0
          }
        }
      }
    }
  }, 
  timer_slack_ns = 0xc350, 
  default_timer_slack_ns = 0xc350, 
  scm_work_list = 0x0, 
  ptrace_bp_refcnt = {
    counter = 0x1
  }
}

crash> l *0xffffffff810451b9
0xffffffff810451b9 is in select_task_rq_fair (kernel/sched_fair.c:1676).
1671    kernel/sched_fair.c: No such file or directory.
crash> l *(select_task_rq_fair+115)
0xffffffff810451b9 is in select_task_rq_fair (kernel/sched_fair.c:1676).
1671    in kernel/sched_fair.c

 根据这篇博文：https://blog.csdn.net/pwl999/article/details/106930732

貌似gs 指向的是per_cpu 变量q

gs寄存器在x86平台上主要用于记录per cpu变量的base address，我们可以使用kmem -o命令来查看这个基地址：

crash> kmem -o 
PER-CPU OFFSET VALUES:
  CPU 0: ffff88107fc00000
  CPU 1: ffff88107fc20000
  CPU 2: ffff88107fc40000
  CPU 3: ffff88107fc60000
  CPU 4: ffff88107fc80000
  CPU 5: ffff88107fca0000
  CPU 6: ffff88107fcc0000
  CPU 7: ffff88107fce0000
  CPU 8: ffff88207fc00000
  CPU 9: ffff88207fc20000
 CPU 10: ffff88207fc40000
 CPU 11: ffff88207fc60000
 CPU 12: ffff88207fc80000
 CPU 13: ffff88207fca0000
 CPU 14: ffff88207fcc0000
 CPU 15: ffff88207fce0000
 CPU 16: ffff88107fd00000
 CPU 17: ffff88107fd20000
 CPU 18: ffff88107fd40000
 CPU 19: ffff88107fd60000
 CPU 20: ffff88107fd80000
 CPU 21: ffff88107fda0000
 CPU 22: ffff88107fdc0000
 CPU 23: ffff88107fde0000
 CPU 24: ffff88207fd00000
 CPU 25: ffff88207fd20000
 CPU 26: ffff88207fd40000
 CPU 27: ffff88207fd60000
 CPU 28: ffff88207fd80000
 CPU 29: ffff88207fda0000
 CPU 30: ffff88207fdc0000
 CPU 31: ffff88207fde0000

CPU 9: ffff88207fc20000

src/core/kernel/linux/build/linux-2.6.39/kernel/sched_fair.c: 1663
0xffffffff81045163 <select_task_rq_fair+29>:    mov    %gs:0xdbe0,%eax
0xffffffff8104516b <select_task_rq_fair+37>:    mov    %eax,-0x38(%rbp)

crash> eval ffff88207fc20000 + 0xdbe0
hexadecimal: ffff88207fc2dbe0  
    decimal: 18446612271896648672  (-131801812902944)
      octal: 1777774202017760555740
     binary: 1111111111111111100010000010000001111111110000101101101111100000

 rd ffff88207fc2dbe0
ffff88207fc2dbe0:  0000000000000009                    ........

值为9 确实是cpu 9

int cpu = smp_processor_id();
int prev_cpu = task_cpu(p);

对于perv_cpu 值为3：

struct  task_struct  ffff8810792b9a40
struct task_struct {
  state = 256, 
  stack = 0xffff8810360ea000, 
  usage = {
    counter = 2
  }, 
  flags = 4202560, 
---
} 


struct thread_info   0xffff8810360ea000
struct thread_info {
  task = 0xffff8810792b9a40, 
  exec_domain = 0xffffffff8196ed80 <default_exec_domain>, 
  flags = 0, 
  status = 0, 
  cpu = 3, 
  preempt_count = 0, 
  addr_limit = {
    seg = 140737488351232
  }, 
  restart_block = {
    fn = 0xffffffff81057218 <do_no_restart_syscall>, 
    {
      futex = {
        uaddr = 0x0, 
        val = 0, 
        flags = 0, 
        bitset = 0, 
        time = 0, 
        uaddr2 = 0x0
      }, 
      nanosleep = {
        index = 0, 
        rmtp = 0x0, 
        compat_rmtp = 0x0, 
        expires = 0
      }, 
      poll = {
        ufds = 0x0, 
        nfds = 0, 
        has_timeout = 0, 
        tv_sec = 0, 
        tv_nsec = 0
      }
    }
  }, 
  sysenter_return = 0x0, 
  uaccess_err = 0
}

> dis -rl 0xffffffff810451b9
  rc/core/kernel/linux/build/linux-2.6.39/kernel/sched_fair.c: 1661
0xffffffff81045146 <select_task_rq_fair>:       push   %rbp
0xffffffff81045147 <select_task_rq_fair+1>:     mov    %rsp,%rbp
0xffffffff8104514a <select_task_rq_fair+4>:     push   %r15
0xffffffff8104514c <select_task_rq_fair+6>:     push   %r14
0xffffffff8104514e <select_task_rq_fair+8>:     push   %r13
0xffffffff81045150 <select_task_rq_fair+10>:    push   %r12
0xffffffff81045152 <select_task_rq_fair+12>:    push   %rbx
0xffffffff81045153 <select_task_rq_fair+13>:    mov    %rsi,%rbx
0xffffffff81045156 <select_task_rq_fair+16>:    sub    $0x88,%rsp
0xffffffff8104515d <select_task_rq_fair+23>:    mov    %edx,-0x34(%rbp)
0xffffffff81045160 <select_task_rq_fair+26>:    mov    %ecx,-0x40(%rbp)
  rc/core/kernel/linux/build/linux-2.6.39/kernel/sched_fair.c: 1663
0xffffffff81045163 <select_task_rq_fair+29>:    mov    %gs:0xdbe0,%eax
0xffffffff8104516b <select_task_rq_fair+37>:    mov    %eax,-0x38(%rbp)
  rc/core/kernel/linux/build/linux-2.6.39/include/linux/sched.h: 2501
0xffffffff8104516e <select_task_rq_fair+40>:    mov    0x8(%rsi),%rax
src/core/kernel/linux/build/linux-2.6.39/kernel/sched_fair.c: 1670
0xffffffff81045172 <select_task_rq_fair+44>:    and    $0x10,%edx
0xffffffff81045175 <select_task_rq_fair+47>:    mov    %edx,-0x68(%rbp)
src/core/kernel/linux/build/linux-2.6.39/kernel/sched_fair.c: 1664
0xffffffff81045178 <select_task_rq_fair+50>:    mov    0x18(%rax),%eax
0xffffffff8104517b <select_task_rq_fair+53>:    mov    %eax,-0x48(%rbp)
src/core/kernel/linux/build/linux-2.6.39/kernel/sched_fair.c: 1670
0xffffffff8104517e <select_task_rq_fair+56>:    je     0xffffffff81045199 <select_task_rq_fair+83>
  rc/core/kernel/linux/build/linux-2.6.39/arch/x86/include/asm/bitops.h: 319
0xffffffff81045180 <select_task_rq_fair+58>:    mov    -0x38(%rbp),%edx
0xffffffff81045183 <select_task_rq_fair+61>:    bt     %edx,0xe8(%rsi)
0xffffffff8104518a <select_task_rq_fair+68>:    sbb    %eax,%eax
  rc/core/kernel/linux/build/linux-2.6.39/kernel/sched_fair.c: 1664
0xffffffff8104518c <select_task_rq_fair+70>:    cmp    $0x1,%eax
0xffffffff8104518f <select_task_rq_fair+73>:    mov    -0x48(%rbp),%r14d
0xffffffff81045193 <select_task_rq_fair+77>:    sbb    %edx,%edx
0xffffffff81045195 <select_task_rq_fair+79>:    inc    %edx
0xffffffff81045197 <select_task_rq_fair+81>:    jmp    0xffffffff8104519f <select_task_rq_fair+89>
  rc/core/kernel/linux/build/linux-2.6.39/kernel/sched_fair.c: 1663
0xffffffff81045199 <select_task_rq_fair+83>:    mov    -0x38(%rbp),%r14d
  rc/core/kernel/linux/build/linux-2.6.39/kernel/sched_fair.c: 1666
0xffffffff8104519d <select_task_rq_fair+87>:    xor    %edx,%edx
  rc/core/kernel/linux/build/linux-2.6.39/kernel/sched_fair.c: 1676
0xffffffff8104519f <select_task_rq_fair+89>:    movslq -0x38(%rbp),%rax
0xffffffff810451a3 <select_task_rq_fair+93>:    mov    $0x13480,%r10
  rc/core/kernel/linux/build/linux-2.6.39/kernel/sched_fair.c: 1667
0xffffffff810451aa <select_task_rq_fair+100>:   mov    $0x1,%r8d
  rc/core/kernel/linux/build/linux-2.6.39/kernel/sched_fair.c: 1676
0xffffffff810451b0 <select_task_rq_fair+106>:   xor    %r13d,%r13d
0xffffffff810451b3 <select_task_rq_fair+109>:   xor    %r12d,%r12d
0xffffffff810451b6 <select_task_rq_fair+112>:   mov    %r14d,%ecx
0xffffffff810451b9 <select_task_rq_fair+115>:   mov    -0x7e62bd10(,%rax,8),%rax 

从上述看到结果是： 从cpu 3 切换到cpu9  然后访问per_cpu 变量的rcu 结构； 但是没有使用rcu_lock

目前认为是rcu 使用出错吧

ＰＳ：task 查看当前进程或指定进程task_struct和thread_info的信息

kmen 查看当时系统内存使用信息

files命令

files pid 打印指定进程所打开的文件信息

crash > set 进程id /* 连接需要调试的进程 */
crash> mod -s memdisk /* 导入模块memdisk的符号表 */
crash> mod -s memcon /* 导入模块memcon的符号表 */

Irq
irq [[[index ...] | -u] | -d | -b]
显示中断编号的所有信息
Irq 不加参数，则显示所有的中断
Irq index 显示中断编号为index的所有信息
Irq –u 仅仅显示正在使用的中断

Foreach
foreach [[pid | taskp | name | [kernel | user]] ...] command [flag] [argument]
跟C#中的foreach类似，为多任务准备的。它根据参数指定的任务中去查找command相关的内容。任务可以用pid、taskp、name来指定。如果未指定，则搜索所有的任务。形如：
Foreach bash task 表示搜索任务bash中的task相关数据。

Vtop
vtop [-c [pid | taskp]] [-u|-k] address ...
显示用户或内核虚拟内存所对应的物理内存。其中-u和-k分别表示用户空间和内核空间。

Ptov
ptov address ...
该命令与vtop相反。把物理内存转换成虚拟内存。

Set
set [pid | taskp | [-c cpu] | -p] | [crash_variable [setting]] | -v
1、设置要显示的内容，内容一般以进程为单位。
Set pid 设置当前的内容为pid所代表的进程
Set taskp 设置当前的内容为十六制表示的taskp任务的内容
Set –p 设置当前的内容为panic任务的内容
Set -v 显示crash当前的内部变量
Set 不带参数，表示显示当前任务的内容
2、同时set命令也可以设置当前crash的内部变量
Set scroll on表示开启滚动条。
具体的内部变量可以通过set –v命令获得，也可以通过help set来查看帮助。

Ascii
把一个十六进制表示的字符串转化成ascii表示的字符串
Ascii 不带参数则显示ascii码表
Ascii number number所代表的ascii字符串

Struct
struct struct_name[.member[,member]][-o][-l offset][-rfu] [address | symbol]
[count | -c count]
显示结构体的具体内容（下面只介绍常用的，具体的可通过命令help struct查询）
注：如果crash关键字与name所表示的结构体名称不冲突，可以省略struct关键字。
Struct name 显示name所表示的结构体的具体结构
Struct name.member 显示name所表示的结构体中的member成员
Struct name –o 显示name所表示的结构体的具体结构，同时也显示每个成员的偏移量
注：如果crash关键字与name所表示的结构体名称不冲突，可以省略struct关键字。

Union
union union_name[.member[,member]] [-o][-l offset][-rfu] [address | symbol]
[count | -c count]
显示联合体的具体内容，用法与struct一致。

*
它是一个快捷键，用来取代struct和union。
Struct page == *page
Struct page == *page

P
p [-x|-d][-u] expression
Print的缩写，打印表达式的值。表达式可以为变量，也可以为结构体。
通过命令alias可以查看命令缩写的列表。
Px expression == p –x expression 以十六进制显示expression的值
Pd expression == p –d expression 以十进制显示expression的值
不加参数的print，则根据set设置来显示打印信息。

Whatis
whatis [struct | union | typedef | symbol]
搜索数据或者类型的信息
参数可以是结构体的名称、联合体的名称、宏的名称或内核的符号。

Sym
sym [-l] | [-M] | [-m module] | [-p|-n] | [-q string] | [symbol | vaddr]
把一个标志符转换到它所对应的虚拟地址，或者把虚拟地址转换为它所对应的标志符。
Sym –l 列出所有的标志符及虚拟地址
Sym –M 列出模块标志符的集合
Sym –m module name 列表模块name的虚拟地址
Sym vaddr 显示虚拟地址addr所代表的标志
Sym symbol 显示symbol标志符所表示的虚拟地址
Sym –q string 搜索所有包含string的标志符及虚拟地址

Dis
dis [-r][-l][-u][-b [num]] [address | symbol | (expression)] [count]
disassemble的缩写。把一个命令或者函数分解成汇编代码。
Dis symbol
Dis –l symbol

Bt
bt [-a|-g|-r|-t|-T|-l|-e|-E|-f|-F|-o|-O] [-R ref] [-I ip] [-S sp] [pid | task]
跟踪堆栈的信息。
Bt 无参数则显示当前任务的堆栈信息
Bt –a 以任务为单位，显示每个任务的堆栈信息
Bt –t 显示当前任务的堆栈中所有的文本标识符
Bt –f 显示当前任务的所有堆栈数据，通过用来检查每个函数的参数传递

Dev
dev [-i | -p]
显示数据关联着的块设备分配，包括端口使用、内存使用及PCI设备数据
Dev –I 显示I/O端口使用情况
Dev –p 显示PCI设备数据

Files
files [-l | -d dentry] | [-R reference] [pid | taskp]
显示某任务的打开文件的信息
Files 显示当前任务下所有打开文件的信息
File –l 显示被服务器锁住的文件的信息

Irq
irq [[[index ...] | -u] | -d | -b]
显示中断编号的所有信息
Irq 不加参数，则显示所有的中断
Irq index 显示中断编号为index的所有信息
Irq –u 仅仅显示正在使用的中断

Foreach
foreach [[pid | taskp | name | [kernel | user]] ...] command [flag] [argument]
跟C#中的foreach类似，为多任务准备的。它根据参数指定的任务中去查找command相关的内容。任务可以用pid、taskp、name来指定。如果未指定，则搜索所有的任务。形如：
Foreach bash task 表示搜索任务bash中的task相关数据。

当command为{bt,vm,task,files,net,set,sig,vtop}时，显示的内容与命令中的命令类似，只是加了foreach则显示所有任务，而不是单条任务。形如：
Foreach files 显示所有任务打开的文件

Runq
无参数。显示每个CPU运行队列中的任务。

Alias
alias [alias] [command string]
创建给定的命令的别名，如果未指定参数，则显示创建好的别名列表。
Command string可以是带各种参数的命令。

Mount
mount [-f] [-i] [-n pid|task] [vfsmount|superblock|devname|dirname|inode]
显示挂载的相关信息
Mount 不加参数，则显示所有已挂载的文件系统
Mount –f 显示每个挂载文件系统中已经打开的文件
Mount –I 显示每个挂载文件系统中的dirty inodes

Search
search [-s start] [ -[kKV] | -u | -p ] [-e end | -l length] [-m mask] -[cwh] value ...
搜索在给定范围的用户、内核虚拟内存或者物理内存。如果不指定-l length或-e end，则搜索虚拟内存或者物理内存的结尾。内存地址以十六进制表示。
-u 如果未指定start，则从当前任务的用户内存搜索指定的value
-k 如果未指定start，则从当前任务的内核内存搜索指定的value
-p 如果未指定start，则从当前任务的物理内存搜索指定的value
-c 后面则指定要搜索的字符串，这个搜索中很有用。

Vm
vm [-p | -v | -m | [-R reference] | [-f vm_flags]] [pid | taskp] ...
显示任务的基本虚拟内存信息。
-p 显示虚拟内存及转换后的物理内存信息

Net
net [-a] [[-s | -S] [-R ref] [pid | taskp]] [-n addr]
显示各种网络相关的数据
-a 显示ARP cache
-s 显示指定任务的网络信息
-S 与-s相似，但是显示的信息更为详细
该命令与foreach配合使用，能加快定位的速度。

Vtop
vtop [-c [pid | taskp]] [-u|-k] address ...
显示用户或内核虚拟内存所对应的物理内存。其中-u和-k分别表示用户空间和内核空间。

Ptov
ptov address ...
该命令与vtop相反。把物理内存转换成虚拟内存。


Btop
btop address ...
把一个十六进制表示的地址转换成它的分页号。

Ptob
ptob page_number ...
该命令与btop相反，是把一个分页号转换成地址。


Sig
sig [[-l] | [-s sigset]] | [-g] [pid | taskp] ...
显示一个或者多个任务的signal-handling数据
-l 列出信息的编号及名字
-g 显示指定任务线程组中所有的signal-handling数据

Waitq
waitq [ symbol ] | [ struct.member struct_addr ] | [ address ]
列出在等待队列中的所有任务。参数可以指定队列的名称、内存地址等。

Pte
pte contents ...
把一个十六进制表示的页表项转换为物理页地址和页的位设置

Swap
无参数。显示已经配置好的交换设备的信息。

Wr
wr [-u|-k|-p] [-8|-16|-32|-64] [address|symbol] value
根据参数指定的写内存。在定位系统出错的地方时，一般不使用该命令。

Eval
eval [-b][-l] (expression) | value
计算表达式的值，及把计算结果或者值显示为16、10、8和2进制。表达式可以有运算符，包括加减乘除移位等。
-b 统计2进制位数为1的索引编号。

List
list [[-o] offset] [-e end] [-s struct[.member[,member]]] [-H] start
显示链表的内容

Mach
mach [-cm]
显示机器的一些信息，如CPU主频等。
-c 显示每个CPU的结构体信息
-m 显示物理内存每段的映射

Log
log [-m]
显示内核的日志，以时间的先后顺序排列
-m 在每个消息前添加该消息的日志等级

Sys
sys [-c [name|number]] config
显示特殊系统的数据。不指定参数，则显示crash启动时打印的系统数据。
-c [name|number] 如果不指定参数，则显示所有的系统调用。否则搜索指定的系统调用。
Config 显示内核的配置。不过必须把CONFIG_IKCONFIG编进内核

Rd
rd [-dDsSupxmf][-8|-16|-32|-64][-o offs][-e addr] [address|symbol] [count]
显示指定内存的内容。缺少的输出格式是十六进制输出
-d 以十进制方式输出
-D 以十进制无符号输出
-8 只输出最后8位
-16 只输出最后16位
-32 只输出最后32位
-64 只输出最后64位
-o offs 开始地址的偏移量
-e addr 显示内存，直到到过地址addr为止
Address 开始的内存地址，以十六进制表示
Symbol 开始地址的标识符
Count 按多少位显示内存地址。如addr=1234，count=8，则显示34 12

Task
task [-R member[,member]] [pid | taskp] ...
显示指定内容或者进程的task_struct的内容。不指定参数则显示当前内容的task_struct的内容。
Pid 进程的pid
Taskp 十六进制表示的task_struct指针。
-R member

Extend
extend [shared-object ...] | [-u [shared-object ...]]
动态装载或卸载crash额外的动态链接库。

Repeat
repeat [-seconds] command
每隔seconds重复一次命令command，无限期的执行下去。

Timer
无参数。按时间的先后顺序显示定时器队列的数据。

Gdb
gdb command ...
用GDB执行命令command。

PS：

顺便把 公司以前的内核 段错误给改了

<1>[217758.819517] BUG: unable to handle kernel NULL pointer dereference at
0000000000000068
<1>[217758.819533] IP: [<ffffffff81547714>] ip6_dst_lookup_tail+0x34/0xb5
<4>[217758.819552] PGD 11a74d067 PUD 11aae6067 PMD 0 
<0>[217758.819564] Oops: 0000 [#1] SMP 
<0>[217758.819572] last sysfs file: /sys/devices/system/cpu/online
<4>[217758.819581] CPU 1 
<4>[217758.819585] Modules linked in: ixgbe igb virtio_net e1000 e1000e
<4>[217758.819604] 
<4>[217758.819612] Pid: 14975, comm: python Not tainted
2.6.39-gentoo-r3-wafg2-33331 #18 NSFocus 1U/1U
<4>[217758.819625] RIP: 0010:[<ffffffff81547714>]  [<ffffffff81547714>]
ip6_dst_lookup_tail+0x34/0xb5
<4>[217758.819642] RSP: 0000:ffff88013fc837c0  EFLAGS: 00010206
<4>[217758.819649] RAX: 0000000000000000 RBX: ffff88013fc83808 RCX:
0000000000000009
<4>[217758.819657] RDX: ffff880139e5c000 RSI: 0000000000000000 RDI:
ffffffff81553364
<4>[217758.819665] RBP: ffff88013fc837f0 R08: 0000000000000000 R09:
ffffffff8198f998
<4>[217758.819673] R10: 00000000af99f324 R11: 00000000ff000002 R12:
ffffffff81b05040
<4>[217758.819682] R13: ffff88013fc83860 R14: ffff88013a781d00 R15:
ffff88013fc83910
<4>[217758.819692] FS:  00007ffd377ae700(0000) GS:ffff88013fc80000(0000)
knlGS:0000000000000000
<4>[217758.819701] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
<4>[217758.819709] CR2: 0000000000000068 CR3: 000000010ee9d000 CR4:
00000000000006e0
<4>[217758.819718] DR0: 0000000000000000 DR1: 0000000000000000 DR2:
0000000000000000
<4>[217758.819726] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7:
0000000000000400
<4>[217758.819736] Process python (pid: 14975, threadinfo ffff8800a94d6000,
task ffff8800b2abf7f0)
<0>[217758.819743] Stack:
<4>[217758.819749]  ffff880100000000 000000148103c71e ffff88013fc83860
ffff88013a781d00
<4>[217758.819764]  0000000000000000 ffff8800b287e800 ffff88013fc83830
ffffffff815478dc
<4>[217758.819779]  0000000000000286 0000000000000000 ffff88013a781d00

正好是 偏移0x68  对应error