static void
ngx_http_upstream_send_request(ngx_http_request_t *r, ngx_http_upstream_t *u,
ngx_uint_t do_write) //向上游服务器发送请求 当一次发送不完,通过ngx_http_upstream_send_request_handler再次触发发送
{
ngx_int_t rc;
ngx_connection_t *c;
c = u->peer.connection; //向上游服务器的连接信息
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
"http upstream send request");
if (u->state->connect_time == (ngx_msec_t) -1) {//更新upstream 连接时间耗费状态信息
u->state->connect_time = ngx_current_msec - u->state->response_time;
}
/*通过getsockopt测试与上游服务器的tcp连接是否异常
* BSDs and Linux return 0 and set a pending error in err
* Solaris returns -1 and sets errno
if (getsockopt(c->fd, SOL_SOCKET, SO_ERROR, (void *) &err, &len) == -1)*/
if (!u->request_sent && ngx_http_upstream_test_connect(c) != NGX_OK) { //测试连接失败
ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_ERROR);//如果测试失败,调用ngx_http_upstream_next函数,这个函数可能再次调用peer.get调用别的连接。
return;
}
c->log->action = "sending request to upstream";
rc = ngx_http_upstream_send_request_body(r, u, do_write);
if (rc == NGX_ERROR) {
/* 若返回值rc=NGX_ERROR,表示当前连接上出错, 将错误信息传递给ngx_http_upstream_next方法, 该方法根据错误信息决定
是否重新向上游其他服务器发起连接; 并return从当前函数返回; */
ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_ERROR);
return;
}
if (rc >= NGX_HTTP_SPECIAL_RESPONSE) {//满足http错误状态码条件 结束请求 同时响应结果到请求端
ngx_http_upstream_finalize_request(r, u, rc);
return;
}
/*
若返回值rc = NGX_AGAIN,表示请求数据并未完全发送, 即有剩余的请求数据保存在output中,但此时,写事件已经不可写,
则调用ngx_add_timer方法把当前连接上的写事件添加到定时器机制, 并调用ngx_handle_write_event方法将写事件注册到epoll事件机制中;
*/ //通过ngx_http_upstream_read_request_handler进行再次epoll write
if (rc == NGX_AGAIN) {//协议栈缓冲区已满,需要等待发送数据出去后出发epoll可写,从而继续write
if (!c->write->ready) {
//这里加定时器的原因是,例如我把数据扔到协议栈了,并且协议栈已经满了,但是对方就是不接受数据,造成数据一直在协议栈缓存中
//因此只要数据发送出去,就会触发epoll继续写,从而在下面两行删除写超时定时器
ngx_add_timer(c->write, u->conf->send_timeout, NGX_FUNC_LINE);
//如果超时会执行ngx_http_upstream_send_request_handler,这里面对写超时进行处理
} else if (c->write->timer_set) { //例如ngx_http_upstream_send_request_body发送了三次返回NGX_AGAIN,那么第二次就需要把第一次上面的超时定时器关了,表示发送正常
ngx_del_timer(c->write, NGX_FUNC_LINE);
}
//在连接后端服务器conncet前,有设置ngx_add_conn,里面已经将fd添加到了读写事件中,因此这里实际上只是简单执行下ngx_send_lowat
if (ngx_handle_write_event(c->write, u->conf->send_lowat, NGX_FUNC_LINE) != NGX_OK) {
ngx_http_upstream_finalize_request(r, u,
NGX_HTTP_INTERNAL_SERVER_ERROR);
return;
}
return;
}
/* rc == NGX_OK */
//向后端的数据发送完毕
//当发往后端服务器的数据包过大,需要分多次发送的时候,在上面的if (rc == NGX_AGAIN)中会添加定时器来触发发送,如果协议栈一直不发送数据出去
//就会超时,如果数据最终全部发送出去则需要为最后一次time_write添加删除操作。
//如果发往后端的数据长度后小,则一般不会再上门添加定时器,这里的timer_set肯定为0,所以如果拔掉后端网线,通过ngx_http_upstream_test_connect
//是判断不出后端服务器掉线的,上面的ngx_http_upstream_send_request_body还是会返回成功的,所以这里有个bug
if (c->write->timer_set) { //这里的定时器是ngx_http_upstream_connect中connect返回NGX_AGAIN的时候添加的定时器
/*
2025/04/24 02:54:29[ ngx_event_connect_peer, 32] [debug] 14867#14867: *1 socket 12
2025/04/24 02:54:29[ ngx_epoll_add_connection, 1486] [debug] 14867#14867: *1 epoll add connection: fd:12 ev:80002005
2025/04/24 02:54:29[ ngx_event_connect_peer, 125] [debug] 14867#14867: *1 connect to 127.0.0.1:3666, fd:12 #2
2025/04/24 02:54:29[ ngx_http_upstream_connect, 1549] [debug] 14867#14867: *1 http upstream connect: -2 //返回NGX_AGAIN
2025/04/24 02:54:29[ ngx_event_add_timer, 88] [debug] 14867#14867: *1 <ngx_http_upstream_connect, 1665> event timer add: 12: 60000:1677807811 //这里添加
2025/04/24 02:54:29[ ngx_http_finalize_request, 2526] [debug] 14867#14867: *1 http finalize request: -4, "/test.php?" a:1, c:2
2025/04/24 02:54:29[ ngx_http_close_request, 3789] [debug] 14867#14867: *1 http request count:2 blk:0
2025/04/24 02:54:29[ ngx_worker_process_cycle, 1110] [debug] 14867#14867: worker(14867) cycle again
2025/04/24 02:54:29[ ngx_trylock_accept_mutex, 405] [debug] 14867#14867: accept mutex locked
2025/04/24 02:54:29[ ngx_epoll_process_events, 1614] [debug] 14867#14867: begin to epoll_wait, epoll timer: 60000
2025/04/24 02:54:29[ ngx_epoll_process_events, 1699] [debug] 14867#14867: epoll: fd:11 epoll-out(ev:0004) d:B27440E8
2025/04/24 02:54:29[ ngx_epoll_process_events, 1772] [debug] 14867#14867: *1 post event AEB44068
2025/04/24 02:54:29[ ngx_epoll_process_events, 1699] [debug] 14867#14867: epoll: fd:12 epoll-out(ev:0004) d:B2744158
2025/04/24 02:54:29[ ngx_epoll_process_events, 1772] [debug] 14867#14867: *1 post event AEB44098
2025/04/24 02:54:29[ ngx_process_events_and_timers, 371] [debug] 14867#14867: epoll_wait timer range(delta): 2
2025/04/24 02:54:29[ ngx_event_process_posted, 65] [debug] 14867#14867: posted event AEB44068
2025/04/24 02:54:29[ ngx_event_process_posted, 67] [debug] 14867#14867: *1 delete posted event AEB44068
2025/04/24 02:54:29[ ngx_http_request_handler, 2400] [debug] 14867#14867: *1 http run request: "/test.php?"
2025/04/24 02:54:29[ngx_http_upstream_check_broken_connection, 1335] [debug] 14867#14867: *1 http upstream check client, write event:1, "/test.php"
2025/04/24 02:54:29[ngx_http_upstream_check_broken_connection, 1458] [debug] 14867#14867: *1 http upstream recv(): -1 (11: Resource temporarily unavailable)
2025/04/24 02:54:29[ ngx_event_process_posted, 65] [debug] 14867#14867: posted event AEB44098
2025/04/24 02:54:29[ ngx_event_process_posted, 67] [debug] 14867#14867: *1 delete posted event AEB44098
2025/04/24 02:54:29[ ngx_http_upstream_handler, 1295] [debug] 14867#14867: *1 http upstream request: "/test.php?"
2025/04/24 02:54:29[ngx_http_upstream_send_request_handler, 2210] [debug] 14867#14867: *1 http upstream send request handler
2025/04/24 02:54:29[ ngx_http_upstream_send_request, 2007] [debug] 14867#14867: *1 http upstream send request
2025/04/24 02:54:29[ngx_http_upstream_send_request_body, 2095] [debug] 14867#14867: *1 http upstream send request body
2025/04/24 02:54:29[ ngx_chain_writer, 690] [debug] 14867#14867: *1 chain writer buf fl:0 s:968
2025/04/24 02:54:29[ ngx_chain_writer, 704] [debug] 14867#14867: *1 chain writer in: 080EC838
2025/04/24 02:54:29[ ngx_writev, 192] [debug] 14867#14867: *1 writev: 968 of 968
2025/04/24 02:54:29[ ngx_chain_writer, 740] [debug] 14867#14867: *1 chain writer out: 00000000
2025/04/24 02:54:29[ ngx_event_del_timer, 39] [debug] 14867#14867: *1 <ngx_http_upstream_send_request, 2052> event timer del: 12: 1677807811//这里删除
2025/04/24 02:54:29[ ngx_event_add_timer, 88] [debug] 14867#14867: *1 <ngx_http_upstream_send_request, 2075> event timer add: 12: 60000:1677807813
*/
ngx_del_timer(c->write, NGX_FUNC_LINE);
}
/* 若返回值 rc = NGX_OK,表示已经发送完全部请求数据, 准备接收来自上游服务器的响应报文,则执行以下程序; */
if (c->tcp_nopush == NGX_TCP_NOPUSH_SET) {
if (ngx_tcp_push(c->fd) == NGX_ERROR) {
ngx_log_error(NGX_LOG_CRIT, c->log, ngx_socket_errno,
ngx_tcp_push_n " failed");
ngx_http_upstream_finalize_request(r, u,
NGX_HTTP_INTERNAL_SERVER_ERROR);
return;
}
c->tcp_nopush = NGX_TCP_NOPUSH_UNSET;
}
u->write_event_handler = ngx_http_upstream_dummy_handler; //数据已经在前面全部发往后端服务器了,所以不需要再做写处理
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "send out chain data to uppeer server OK");
//在连接后端服务器conncet前,有设置ngx_add_conn,里面已经将fd添加到了读写事件中,因此这里实际上只是简单执行下ngx_send_lowat
if (ngx_handle_write_event(c->write, 0, NGX_FUNC_LINE) != NGX_OK) {
ngx_http_upstream_finalize_request(r, u,
NGX_HTTP_INTERNAL_SERVER_ERROR);
return;
}
//这回数据已经发送了,可以准备接收了,设置接收后端应答的超时定时器。
/*
该定时器在收到后端应答数据后删除,见ngx_event_pipe
if (rev->timer_set) {
ngx_del_timer(rev, NGX_FUNC_LINE);
}
*/
ngx_add_timer(c->read, u->conf->read_timeout, NGX_FUNC_LINE); //如果超时在该函数检测ngx_http_upstream_process_header
if (c->read->ready) { //读事件触发 准备处理http头部信息---此时c 表示链接上游服务器的connect可读,也就是上游服务器回复了响应报文
ngx_http_upstream_process_header(r, u);
return;
}
}