root@ubuntu:~/nginx-1.16.1# ps -elf | grep nginx 1 S root 2739 1 0 80 0 - 2265 sigsus 20:09 ? 00:00:00 nginx: master process ./objs/nginx -c conf/nginx.conf 5 S root 2740 2739 0 80 0 - 2468 ep_pol 20:09 ? 00:00:00 nginx: worker process 0 S root 3492 41645 0 80 0 - 1096 pipe_w 20:11 pts/2 00:00:00 grep nginx root@ubuntu:~/nginx-1.16.1#
gdb attach worker进程
root@ubuntu:~# gdb attach 2740 GNU gdb (Ubuntu 8.1-0ubuntu3.2) 8.1.0.20180409-git Copyright (C) 2018 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details.
(gdb) b ngx_quic_read_handler Breakpoint 1 at 0xaaaae2e9bd64: file src/event/ngx_event_quic.c, line 229. (gdb) c Continuing. Breakpoint 1, ngx_quic_read_handler (rev=0xaaab145576f0) at src/event/ngx_event_quic.c:229 229 { (gdb) bt #0 ngx_quic_read_handler (rev=0xaaab145576f0) at src/event/ngx_event_quic.c:229 #1 0x0000aaaae2e8af78 in ngx_event_recvmsg (ev=0xaaab145575d0) at src/event/ngx_event_udp.c:263 #2 0x0000aaaae2e93a40 in ngx_epoll_process_events (cycle=<optimized out>, timer=<optimized out>, flags=1) at src/event/modules/ngx_epoll_module.c:902 #3 0x0000aaaae2e89a44 in ngx_process_events_and_timers (cycle=cycle@entry=0xaaab1452f8e0) at src/event/ngx_event.c:242 #4 0x0000aaaae2e91a04 in ngx_worker_process_cycle (cycle=0xaaab1452f8e0, data=<optimized out>) at src/os/unix/ngx_process_cycle.c:750 #5 0x0000aaaae2e8fd68 in ngx_spawn_process (cycle=cycle@entry=0xaaab1452f8e0, proc=proc@entry=0xaaaae2e91964 <ngx_worker_process_cycle>, data=data@entry=0x0, name=name@entry=0xaaaae31a5408 "worker process", respawn=respawn@entry=-3) at src/os/unix/ngx_process.c:199 #6 0x0000aaaae2e910a0 in ngx_start_worker_processes (cycle=cycle@entry=0xaaab1452f8e0, n=1, type=type@entry=-3) at src/os/unix/ngx_process_cycle.c:359 #7 0x0000aaaae2e92258 in ngx_master_process_cycle (cycle=0xaaab1452f8e0) at src/os/unix/ngx_process_cycle.c:131 #8 0x0000aaaae2e69d78 in main (argc=0, argv=<optimized out>) at src/core/nginx.c:382 (gdb) clearall Undefined command: "clearall". Try "help". (gdb) dis b (gdb) break info Function "info" not defined. Make breakpoint pending on future shared library load? (y or [n]) Quit (gdb) quit A debugging session is active. Inferior 1 [process 2740] will be detached. Quit anyway? (y or n) y Detaching from program: /root/nginx-1.16.1/objs/nginx, process 2740 root@ubuntu:~#
ngx_event_recvmsg
通过我们知道ngx_event_core_module模块的init_process函数ngx_event_process_init()会为每个监听套接字的读事件注册处理函数ngx_event_accept(TCP)或者ngx_event_recvmsg(UDP)
static ngx_int_t ngx_event_process_init(ngx_cycle_t *cycle) { …… ls = cycle->listening.elts; for (i = 0; i < cycle->listening.nelts; i++) { //对每个监听描述符创建一个对应的connection对象，封装了描述符fd c = ngx_get_connection(ls[i].fd, cycle->log); //拿到监听描述符的读事件 rev = c->read; //表示accept新连接 rev->accept = 1; //读事件的hander,type为流时走tcp的accept，否则直接走udp的recvmsg rev->handler = (c->type == SOCK_STREAM) ? ngx_event_accept : ngx_event_recvmsg; //将读事件添加到IO多路复用模型中，当采用epoll模型时ngx_add_event就是ngx_epoll_add_event //udp，当事件触发时会调用ngx_event_recvmsg if (ngx_add_event(rev, NGX_READ_EVENT, 0) == NGX_ERROR) { return NGX_ERROR; } } }
ngx_event_recvmsg
udp读事件的handler函数ngx_event_recvmsg：
// src/event/ngx_event_udp.c void ngx_event_recvmsg(ngx_event_t *ev){ struct msghdr msg; struct iovec iov[1]; struct msghdr msg; ngx_buf_t buf; static u_char buffer[65535]; //静态读缓冲区 …… lc = ev->data; ls = lc->listening; do { ngx_memzero(&msg, sizeof(struct msghdr)); iov[0].iov_base = (void *) buffer; iov[0].iov_len = sizeof(buffer); …… msg.msg_iov = iov; msg.msg_iovlen = 1; //读取报文，读出的数据存放在iov[0].iov_base也即是buffer中 n = recvmsg(lc->fd, &msg, 0); //拿到源ip和源端口 sockaddr = msg.msg_name; socklen = msg.msg_namelen; //本地监听目的地址和端口 local_sockaddr = ls->sockaddr; local_socklen = ls->socklen; //ls代带有一个存放连接的红黑树，从红黑树找是否已经有相同四元组的连接 c = ngx_lookup_udp_connection(ls, sockaddr, socklen, local_sockaddr, local_socklen); if (c) { …… ngx_memzero(&buf, sizeof(ngx_buf_t)); buf.pos = buffer; buf.last = buffer + n; //找到则调用读事件的handler处理数据读取事件 rev = c->read; //最终，c->udp->buffer指向了读取的数据缓冲区 c->udp->buffer = &buf; rev->ready = 1; //ready为1，后面会用到 //此时handler为ngx_stream_session_handler@src/stream/ngx_stream_handler.c rev->handler(rev); if (c->udp) { c->udp->buffer = NULL; } rev->ready = 0; goto next; } //找不到则新建一个connection对象 c = ngx_get_connection(lc->fd, ev->log); if (c == NULL) { return; } c->shared = 1; c->type = SOCK_DGRAM; c->socklen = socklen; …… //将本次读到的缓冲区buffer中的数据拷贝追加到临时缓冲区c->buffer后面 c->buffer = ngx_create_temp_buf(c->pool, n); c->buffer->last = ngx_cpymem(c->buffer->last, buffer, n); //设置连接的recv和send操作函数 //ngx_udp_shared_recv是直接从c->udp->buffer拷贝出数据 c->recv = ngx_udp_shared_recv; c->send = ngx_udp_send; c->send_chain = ngx_udp_send_chain; //将新的对象插入到管理连接的红黑树中 if (ngx_insert_udp_connection(c) != NGX_OK) { ngx_close_accepted_udp_connection(c); return; } //调用监听handler处理新连接事件， //此时handler为ngx_stream_init_connection@src/stream/ngx_stream_handler.c ls->handler(c); next: if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) { ev->available -= n; } } while (ev->available); }
RUST_LOG="debug" ./target/debug/examples/http3-client https://10.10.16.82:443
(gdb) p ls $3 = (ngx_listening_t *) 0xaaab1452ff90 (gdb) p *ls $4 = {fd = 6, sockaddr = 0xaaab1454e020, socklen = 16, addr_text_max_len = 15, addr_text = {len = 11, data = 0xaaab1454e030 "0.0.0.0:443"}, type = 2, backlog = 511, rcvbuf = -1, sndbuf = -1, keepidle = 0, keepintvl = 0, keepcnt = 0, handler = 0xaaaae2ea942c <ngx_http_init_connection>, servers = 0xaaab1454e040, log = {log_level = 4, file = 0xaaab1452fc38, connection = 0, disk_full_time = 0, handler = 0x0, data = 0x0, writer = 0x0, wdata = 0x0, action = 0x0, next = 0x0}, logp = 0xaaab1452f8f8, pool_size = 512, post_accept_buffer_size = 0, post_accept_timeout = 60000, previous = 0x0, connection = 0xffffac7d8010, rbtree = {root = 0xaaab14530088, sentinel = 0xaaab14530088, insert = 0xaaaae2e8b380 <ngx_udp_rbtree_insert_value>}, sentinel = {key = 0, left = 0x0, right = 0x0, parent = 0x0, color = 0 ' 00', data = 0 ' 00'}, worker = 0, open = 1, remain = 0, ignore = 0, bound = 0, inherited = 0, nonblocking_accept = 0, listen = 0, nonblocking = 0, shared = 0, addr_ntop = 1, wildcard = 1, ipv6only = 1, reuseport = 1, add_reuseport = 0, keepalive = 0, deferred_accept = 0, delete_deferred = 0, add_deferred = 0, quic = 1, fastopen = -1} (gdb) n
root@ubuntu:~# tcpdump -i lo udp and port 443 -xx -Xs 0 tcpdump: verbose output suppressed, use -v or -vv for full protocol decode listening on lo, link-type EN10MB (Ethernet), capture size 262144 bytes 14:25:23.649797 IP ubuntu.39128 > ubuntu.443: UDP, length 1200 0x0000: 4500 04cc c17f 4000 4011 3fea 0a0a 1052 E.....@.@.?....R 0x0010: 0a0a 1052 98d8 01bb 04b8 3981 c2ff 0000 ...R......9..... 0x0020: 1d10 9db3 28b7 e938 ee56 b314 c326 4df7 ....(..8.V...&M. 0x0030: 7934 14b1 9b9f a635 821b 57d7 d820 6ac0 y4.....5..W...j. 0x0040: 72cb 4466 e6d9 3600 4105 113a 48b6 15b4 r.Df..6.A..:H... 0x0050: b660 e387 acb9 7853 a1b1 02d4 1e5d 8f72 .`....xS.....].r 0x0060: f263 f3ad 7cd5 31cb c61c a1e5 27c5 42c8 .c..|.1.....'.B. 0x0070: 41ec 3edf 580c 667a de8f 0e81 c0fe c19d A.>.X.fz........ 0x0080: 2474 3ad2 76a9 924c 6bce 5109 d2e8 6ec9 $t:.v..Lk.Q...n. 0x0090: c691 0eef 6597 1345 bff2 e241 0ea6 5a40 ....e..E...A..Z@ 0x00a0: 7d30 ba27 c78a ddd3 daf9 41da 040d 41da }0.'......A...A. 0x00b0: 954c ad8c 0869 9074 4e7b b27e 3bf9 7ab7 .L...i.tN{.~;.z. 0x00c0: 8f9e 4475 537f 745f 02fa 2bc8 d843 4b72 ..DuS.t_..+..CKr 0x00d0: dbde 47eb 0c8f e2f4 02ec 9d29 42ee 38df ..G........)B.8. 0x00e0: b318 5cdb 45e2 a33e 3b21 7aa2 4aad e29c ...E..>;!z.J... 0x00f0: 54d4 fe4b f436 1c82 0eb5 7021 698c dafc T..K.6....p!i... 0x0100: 0ca5 24ff 129f bc06 02a4 aad0 bff0 fbb8 ..$............. 0x0110: f018 df88 3714 5f7c 562b 9911 6aee 70c4 ....7._|V+..j.p. 0x0120: 148d 8a68 7873 118e 1b18 11ce 946c 8e6d ...hxs.......l.m 0x0130: 364c 7157 0c88 1ae0 b8a5 7e4e 3a48 8427 6LqW......~N:H.' 0x0140: 2341 8c27 8dc9 be2a 0c42 d51b 4fa7 8500 #A.'...*.B..O...