root@ubuntu:~/posix_quic# g++ -v Using built-in specs. COLLECT_GCC=g++ COLLECT_LTO_WRAPPER=/usr/lib/gcc/aarch64-linux-gnu/5/lto-wrapper Target: aarch64-linux-gnu Configured with: ../src/configure -v --with-pkgversion='Ubuntu/Linaro 5.5.0-12ubuntu1' --with-bugurl=file:///usr/share/doc/gcc-5/README.Bugs --enable-languages=c,ada,c++,go,d,fortran,objc,obj-c++ --prefix=/usr --program-suffix=-5 --enable-shared --enable-linker-build-id --libexecdir=/usr/lib --without-included-gettext --enable-threads=posix --libdir=/usr/lib --enable-nls --with-sysroot=/ --enable-clocale=gnu --enable-libstdcxx-debug --enable-libstdcxx-time=yes --with-default-libstdcxx-abi=new --enable-gnu-unique-object --disable-libquadmath --enable-plugin --enable-default-pie --with-system-zlib --enable-multiarch --enable-fix-cortex-a53-843419 --disable-werror --enable-checking=release --build=aarch64-linux-gnu --host=aarch64-linux-gnu --target=aarch64-linux-gnu Thread model: posix gcc version 5.5.0 20171010 (Ubuntu/Linaro 5.5.0-12ubuntu1) root@ubuntu:~/posix_quic# gcc -v Using built-in specs. COLLECT_GCC=gcc COLLECT_LTO_WRAPPER=/usr/lib/gcc/aarch64-linux-gnu/5/lto-wrapper Target: aarch64-linux-gnu Configured with: ../src/configure -v --with-pkgversion='Ubuntu/Linaro 5.5.0-12ubuntu1' --with-bugurl=file:///usr/share/doc/gcc-5/README.Bugs --enable-languages=c,ada,c++,go,d,fortran,objc,obj-c++ --prefix=/usr --program-suffix=-5 --enable-shared --enable-linker-build-id --libexecdir=/usr/lib --without-included-gettext --enable-threads=posix --libdir=/usr/lib --enable-nls --with-sysroot=/ --enable-clocale=gnu --enable-libstdcxx-debug --enable-libstdcxx-time=yes --with-default-libstdcxx-abi=new --enable-gnu-unique-object --disable-libquadmath --enable-plugin --enable-default-pie --with-system-zlib --enable-multiarch --enable-fix-cortex-a53-843419 --disable-werror --enable-checking=release --build=aarch64-linux-gnu --host=aarch64-linux-gnu --target=aarch64-linux-gnu Thread model: posix gcc version 5.5.0 20171010 (Ubuntu/Linaro 5.5.0-12ubuntu1) root@ubuntu:~/posix_quic#
安装golang rust 环境
cmake .
make $(nproc)
root@ubuntu:~/posix_quic# mkdir build root@ubuntu:~/posix_quic# cd build root@ubuntu:~/posix_quic/build# cmake .. ------------ Options ------------- CMAKE_BUILD_TYPE: RELEASE CMAKE_COMMAND: /usr/bin/cmake ------------ Cxx flags ------------- CMAKE_CXX_FLAGS_RELEASE: -g -DNDEBUG -O3 -fno-strict-aliasing ------------------------------------ ------------ Options ------------- CMAKE_BUILD_TYPE: RELEASE CMAKE_COMMAND: /usr/bin/cmake ------------ Cxx flags ------------- CMAKE_CXX_FLAGS_RELEASE: -g -O3 -fno-strict-aliasing ------------------------------------ -- Configuring done -- Generating done -- Build files have been written to: /root/posix_quic root@ubuntu:~/posix_quic/build# make -j$(nproc) make: *** No targets specified and no makefile found. Stop. root@ubuntu:~/posix_quic/build# make -j $(nproc) make: *** No targets specified and no makefile found. Stop. root@ubuntu:~/posix_quic/build# ls root@ubuntu:~/posix_quic/build# ls
root@ubuntu:~/posix_quic/test/client# cmake . -- The C compiler identification is GNU 5.5.0 -- The CXX compiler identification is GNU 5.5.0 -- Check for working C compiler: /usr/bin/cc -- Check for working C compiler: /usr/bin/cc -- works -- Detecting C compiler ABI info -- Detecting C compiler ABI info - done -- Detecting C compile features -- Detecting C compile features - done -- Check for working CXX compiler: /usr/bin/c++ -- Check for working CXX compiler: /usr/bin/c++ -- works -- Detecting CXX compiler ABI info -- Detecting CXX compiler ABI info - done -- Detecting CXX compile features -- Detecting CXX compile features - done ------------ Options ------------- CMAKE_BUILD_TYPE: RELEASE CMAKE_COMMAND: /usr/bin/cmake ------------ Cxx flags ------------- CMAKE_CXX_FLAGS_RELEASE: -g -O3 ------------------------------------ -- Configuring done -- Generating done -- Build files have been written to: /root/posix_quic/test/client root@ubuntu:~/posix_quic/test/client# make Scanning dependencies of target client [ 50%] Building CXX object CMakeFiles/client.dir/src/client.cpp.o [100%] Linking CXX executable client /usr/bin/ld: cannot find -lposix_quic /usr/bin/ld: cannot find -lquic /usr/bin/ld: cannot find -ldecrepit collect2: error: ld returned 1 exit status CMakeFiles/client.dir/build.make:94: recipe for target 'client' failed make[2]: *** [client] Error 1 CMakeFiles/Makefile2:67: recipe for target 'CMakeFiles/client.dir/all' failed make[1]: *** [CMakeFiles/client.dir/all] Error 2 Makefile:83: recipe for target 'all' failed make: *** [all] Error 2 root@ubuntu:~/posix_quic/test/client# ls CMakeCache.txt CMakeFiles cmake_install.cmake CMakeLists.txt Makefile src root@ubuntu:~/posix_quic/test/client#
root@ubuntu:~/posix_quic/test/client# grep posix_quic * CMakeCache.txt:# For build in directory: /root/posix_quic/test/client CMakeCache.txt:client_BINARY_DIR:STATIC=/root/posix_quic/test/client CMakeCache.txt:client_SOURCE_DIR:STATIC=/root/posix_quic/test/client CMakeCache.txt:CMAKE_CACHEFILE_DIR:INTERNAL=/root/posix_quic/test/client CMakeCache.txt:CMAKE_HOME_DIRECTORY:INTERNAL=/root/posix_quic/test/client grep: CMakeFiles: Is a directory cmake_install.cmake:# Install script for directory: /root/posix_quic/test/client cmake_install.cmake:file(WRITE "/root/posix_quic/test/client/${CMAKE_INSTALL_MANIFEST}" CMakeLists.txt:target_link_libraries(${TARGET} posix_quic quic ssl crypto decrepit protobuf -pthread -Wl,--whole-archive -lpthread -Wl,--no-whole-archive dl -static) Makefile:CMAKE_SOURCE_DIR = /root/posix_quic/test/client Makefile:CMAKE_BINARY_DIR = /root/posix_quic/test/client Makefile: $(CMAKE_COMMAND) -E cmake_progress_start /root/posix_quic/test/client/CMakeFiles /root/posix_quic/test/client/CMakeFiles/progress.marks Makefile: $(CMAKE_COMMAND) -E cmake_progress_start /root/posix_quic/test/client/CMakeFiles 0
重新编译
root@ubuntu:~/posix_quic# mkdir build root@ubuntu:~/posix_quic# cd build root@ubuntu:~/posix_quic/build# cmake .. ------------ Options ------------- CMAKE_BUILD_TYPE: RELEASE CMAKE_COMMAND: /usr/bin/cmake ------------ Cxx flags ------------- CMAKE_CXX_FLAGS_RELEASE: -g -DNDEBUG -O3 -fno-strict-aliasing ------------------------------------ ------------ Options ------------- CMAKE_BUILD_TYPE: RELEASE CMAKE_COMMAND: /usr/bin/cmake ------------ Cxx flags ------------- CMAKE_CXX_FLAGS_RELEASE: -g -O3 -fno-strict-aliasing ------------------------------------ -- Configuring done -- Generating done -- Build files have been written to: /root/posix_quic root@ubuntu:~/posix_quic/build# make -j$(nproc) make: *** No targets specified and no makefile found. Stop. root@ubuntu:~/posix_quic/build# make -j $(nproc) make: *** No targets specified and no makefile found. Stop. root@ubuntu:~/posix_quic/build# ls root@ubuntu:~/posix_quic/build# ls
root@ubuntu:~/posix_quic/build# ls
CMakeCache.txt CMakeFiles cmake_install.cmake libquic Makefile simple
root@ubuntu:~/posix_quic/build# make -j$(nproc)
make install
-- Installing: /usr/local/include/posix_quic/google/google/protobuf/proto3_arena_lite_unittest.cc -- Installing: /usr/local/include/posix_quic/google/google/protobuf/map_field_inl.h -- Installing: /usr/local/include/posix_quic/google/google/protobuf/reflection_internal.h -- Installing: /usr/local/include/posix_quic/google/google/protobuf/extension_set.h -- Installing: /usr/local/include/posix_quic/google/google/protobuf/unittest_mset.proto -- Installing: /usr/local/include/posix_quic/google/google/protobuf/wrappers.pb.cc -- Installing: /usr/local/include/posix_quic/google/google/protobuf/duration.pb.h -- Installing: /usr/local/include/posix_quic/google/google/protobuf/any.cc -- Installing: /usr/local/include/posix_quic/google/google/protobuf/api.proto -- Installing: /usr/local/include/posix_quic/google/google/protobuf/generated_message_util.h -- Installing: /usr/local/include/posix_quic/google/google/protobuf/message_unittest.cc -- Installing: /usr/local/include/posix_quic/google/google/protobuf/descriptor_unittest.cc -- Installing: /usr/local/include/posix_quic/google/google/protobuf/any.h -- Installing: /usr/local/include/posix_quic/google/google/protobuf/wire_format.h -- Installing: /usr/local/include/posix_quic/google/google/protobuf/descriptor.proto root@ubuntu:~/posix_quic/build#
编译server
root@ubuntu:~/posix_quic/test/server# cmake . ------------ Options ------------- CMAKE_BUILD_TYPE: RELEASE CMAKE_COMMAND: /usr/bin/cmake ------------ Cxx flags ------------- CMAKE_CXX_FLAGS_RELEASE: -g -O3 ------------------------------------ -- Configuring done -- Generating done -- Build files have been written to: /root/posix_quic/test/server root@ubuntu:~/posix_quic/test/server# make -j$(nproc) [ 50%] Linking CXX executable server /root/posix_quic/test/server/../../build/libquic/third_party/boringssl/src/crypto/libcrypto.a(socket_helper.c.o): In function `bio_ip_and_port_to_socket_and_addr': /root/posix_quic/libquic/third_party/boringssl/src/crypto/bio/socket_helper.c:53: warning: Using 'getaddrinfo' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking [100%] Built target server root@ubuntu:~/posix_quic/test/server#
日志
posix_quic/src/debug.h
int main() { debug_mask = dbg_all & ~dbg_timer; QuicEpoller ep = QuicCreateEpoll(); assert(ep >= 0); QuicSocket socket = QuicCreateSocket(); assert(socket > 0); int res; struct sockaddr_in addr; addr.sin_family = AF_INET; addr.sin_port = htons(9700); addr.sin_addr.s_addr = inet_addr("127.0.0.1"); res = QuicBind(socket, (struct sockaddr*)&addr, sizeof(addr)); CHECK_RES(res, "bind"); struct epoll_event ev; ev.data.fd = socket; //ev.events = EPOLLIN | EPOLLOUT; ev.events = EPOLLIN ; res = QuicEpollCtl(ep, EPOLL_CTL_ADD, socket, &ev); CHECK_RES(res, "epoll_ctl"); for (;;) { res = doLoop(ep, socket); if (res != 0) return res; } }
编译client
root@ubuntu:~/posix_quic/build# cd ../test/ root@ubuntu:~/posix_quic/test# ls client empty_client ping server root@ubuntu:~/posix_quic/test# cd client/ root@ubuntu:~/posix_quic/test/client# ls CMakeCache.txt CMakeFiles cmake_install.cmake CMakeLists.txt Makefile src root@ubuntu:~/posix_quic/test/client# make [ 50%] Linking CXX executable client /root/posix_quic/test/client/../../build/libquic/third_party/boringssl/src/crypto/libcrypto.a(socket_helper.c.o): In function `bio_ip_and_port_to_socket_and_addr': /root/posix_quic/libquic/third_party/boringssl/src/crypto/bio/socket_helper.c:53: warning: Using 'getaddrinfo' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking [100%] Built target client root@ubuntu:~/posix_quic/test/client# ls client CMakeCache.txt CMakeFiles cmake_install.cmake CMakeLists.txt Makefile src root@ubuntu:~/posix_quic/test/client#
int main() { debug_mask = dbg_user & ~dbg_timer; QuicEpoller ep = QuicCreateEpoll(); assert(ep >= 0); QuicSocket socket = QuicCreateSocket(); assert(socket > 0); conn = socket; int res; struct sockaddr_in addr; addr.sin_family = AF_INET; addr.sin_port = htons(9700); addr.sin_addr.s_addr = inet_addr("127.0.0.1"); res = QuicConnect(socket, (struct sockaddr*)&addr, sizeof(addr)); assert(errno == EINPROGRESS); assert(res == -1); struct epoll_event ev; ev.data.fd = socket; ev.events = EPOLLIN | EPOLLOUT; res = QuicEpollCtl(ep, EPOLL_CTL_ADD, socket, &ev); CHECK_RES(res, "epoll_ctl"); for (;;) { res = doLoop(ep); if (res != 0) return res; } }
运行server
int main() { debug_mask = dbg_fd & ~dbg_timer; //debug_mask = dbg_all & ~dbg_timer; QuicEpoller ep = QuicCreateEpoll(); assert(ep >= 0); QuicSocket socket = QuicCreateSocket(); assert(socket > 0); int res; struct sockaddr_in addr; addr.sin_family = AF_INET; addr.sin_port = htons(9700); addr.sin_addr.s_addr = inet_addr("127.0.0.1"); res = QuicBind(socket, (struct sockaddr*)&addr, sizeof(addr)); CHECK_RES(res, "bind"); struct epoll_event ev; ev.data.fd = socket; //ev.events = EPOLLIN | EPOLLOUT; ev.events = EPOLLIN ; res = QuicEpollCtl(ep, EPOLL_CTL_ADD, socket, &ev); CHECK_RES(res, "epoll_ctl"); for (;;) { res = doLoop(ep, socket); if (res != 0) return res; } }
root@ubuntu:~/posix_quic/test/server# ./server [17:06:44.113958]fd_manager.h:21:(Put) [C=0] Put Epoll fd = 3 [17:06:44.120620]fd_manager.h:21:(Put) [C=0] Put Socket fd = 1 [17:06:53.057576]fd_manager.h:21:(Put) [C=0] Put Socket fd = 2 [17:06:53.060913]server.cpp:58:(doLoop) [C=0] QuicEpoller trigger: fd = 1, category = Socket, events = EPOLLIN [17:06:53.060946]server.cpp:79:(doLoop) [C=0] Accept Socket fd=1, newSocket=2 [17:06:53.060960]server.cpp:81:(doLoop) [C=0] No Accept Socket. fd=1 [17:06:53.061626]fd_manager.h:21:(Put) [C=15966964779770058277] Put Stream fd = 3 [17:06:53.061667]server.cpp:58:(doLoop) [C=0] QuicEpoller trigger: fd = 2, category = Socket, events = EPOLLIN [17:06:53.061682]server.cpp:95:(doLoop) [C=0] Accept Stream fd=2, newSocket=3 [17:06:53.061695]server.cpp:39:(OnRead) [C=0] recv(len=11): Hello quic! [17:06:53.061745]server.cpp:99:(doLoop) [C=0] No Accept Stream. fd=2 [17:06:53.061757]server.cpp:58:(doLoop) [C=0] QuicEpoller trigger: fd = 3, category = Stream, events = EPOLLIN [17:06:53.062273]server.cpp:58:(doLoop) [C=0] QuicEpoller trigger: fd = 3, category = Stream, events = EPOLLIN [17:06:53.062325]server.cpp:58:(doLoop) [C=0] QuicEpoller trigger: fd = 3, category = Stream, events = EPOLLERR [17:06:53.062400]server.cpp:116:(doLoop) [C=0] Close Stream fd=3 [17:06:53.062411]fd_manager.h:36:(Delete) [C=0] Del Stream fd = 3 [17:07:05.063036]server.cpp:58:(doLoop) [C=0] QuicEpoller trigger: fd = 2, category = Socket, events = EPOLLERR [17:07:05.063082]server.cpp:113:(doLoop) [C=0] Close Socket fd=2 [17:07:05.063108]fd_manager.h:36:(Delete) [C=0] Del Socket fd = 2
root@ubuntu:~/posix_quic/test/server# ./server >> server.txt ^C root@ubuntu:~/posix_quic/test/server# grep 'Hello quic!' server.txt [11:57:00.898844]server.cpp:39:(OnRead) [C=0] recv(len=11): Hello quic! root@ubuntu:~/posix_quic/test/server#
多线程
(gdb) info threads Id Target Id Frame * 1 Thread 0x824000 (LWP 36006) "client" 0x000000000042ff6c in net::QuicConnection::SendStreamData(unsigned int, unsigned long, unsigned long, net::StreamSendingState) () 2 Thread 0xffffbf6fe010 (LWP 36009) "client" 0x000000000060c11c in futex_abstimed_wait_cancelable (private=<optimized out>, abstime=0xffffbf6fd7d8, expected=0, futex_word=0x837d78) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 3 Thread 0xffffbeefd010 (LWP 36010) "client" 0x000000000061001c in __nanosleep (requested_time=requested_time@entry=0xffffbeefc7d8, remaining=remaining@entry=0x0) at ../sysdeps/unix/sysv/linux/nanosleep.c:28 (gdb) treahd 2 Undefined command: "treahd". Try "help". (gdb) thread 2 [Switching to thread 2 (Thread 0xffffbf6fe010 (LWP 36009))] #0 0x000000000060c11c in futex_abstimed_wait_cancelable (private=<optimized out>, abstime=0xffffbf6fd7d8, expected=0, futex_word=0x837d78) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 205 ../sysdeps/unix/sysv/linux/futex-internal.h: No such file or directory. (gdb) bt #0 0x000000000060c11c in futex_abstimed_wait_cancelable (private=<optimized out>, abstime=0xffffbf6fd7d8, expected=0, futex_word=0x837d78) at ../sysdeps/unix/sysv/linux/futex-internal.h:205 #1 __pthread_cond_wait_common (abstime=0xffffbf6fd7d8, mutex=0x837d80, cond=0x837d50) at pthread_cond_wait.c:539 #2 __pthread_cond_timedwait (cond=0x837d50, mutex=0x837d80, abstime=0xffffbf6fd7d8) at pthread_cond_wait.c:667 #3 0x00000000004062b8 in __gthread_cond_timedwait (__abs_timeout=0xffffbf6fd7d8, __mutex=<optimized out>, __cond=0x837d50) at /usr/include/aarch64-linux-gnu/c++/5/bits/gthr-default.h:871 #4 std::condition_variable::__wait_until_impl<std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=..., __lock=..., this=0x837d50) at /usr/include/c++/5/condition_variable:165 #5 std::condition_variable::wait_until<std::chrono::duration<long, std::ratio<1l, 1000000000l> > > (__atime=..., __lock=..., this=0x837d50) at /usr/include/c++/5/condition_variable:105 #6 std::condition_variable::wait_for<long, std::ratio<1l, 1000l> > (__rtime=..., __lock=..., this=0x837d50) at /usr/include/c++/5/condition_variable:137 #7 posix_quic::QuicEpollerEntry::<lambda()>::operator() (__closure=<optimized out>) at /root/posix_quic/src/epoller_entry.cpp:59 #8 std::_Bind_simple<posix_quic::QuicEpollerEntry::QuicEpollerEntry()::<lambda()>()>::_M_invoke<> (this=<optimized out>) at /usr/include/c++/5/functional:1531 #9 std::_Bind_simple<posix_quic::QuicEpollerEntry::QuicEpollerEntry()::<lambda()>()>::operator() (this=<optimized out>) at /usr/include/c++/5/functional:1520 #10 std::thread::_Impl<std::_Bind_simple<posix_quic::QuicEpollerEntry::QuicEpollerEntry()::<lambda()>()> >::_M_run(void) (this=0x837f70) at /usr/include/c++/5/thread:115 #11 0x0000000000654f98 in execute_native_thread_routine () #12 0x0000000000605b30 in start_thread (arg=0xfffffffff65f) at pthread_create.c:463 #13 0x00000000006f3afc in thread_start () (gdb) thread 3 [Switching to thread 3 (Thread 0xffffbeefd010 (LWP 36010))] #0 0x000000000061001c in __nanosleep (requested_time=requested_time@entry=0xffffbeefc7d8, remaining=remaining@entry=0x0) at ../sysdeps/unix/sysv/linux/nanosleep.c:28 28 ../sysdeps/unix/sysv/linux/nanosleep.c: No such file or directory. (gdb) bt #0 0x000000000061001c in __nanosleep (requested_time=requested_time@entry=0xffffbeefc7d8, remaining=remaining@entry=0x0) at ../sysdeps/unix/sysv/linux/nanosleep.c:28 #1 0x0000000000424110 in std::this_thread::sleep_for<long, std::ratio<1l, 1000l> > (__rtime=...) at /usr/include/c++/5/thread:293 #2 posix_quic::FastSteadyClock::ThreadRun () at /root/posix_quic/src/clock_impl.h:33 #3 0x0000000000654f98 in execute_native_thread_routine () #4 0x0000000000605b30 in start_thread (arg=0xfffffffff66f) at pthread_create.c:463 #5 0x00000000006f3afc in thread_start () (gdb) thread 1 [Switching to thread 1 (Thread 0x824000 (LWP 36006))] #0 0x000000000042ff6c in net::QuicConnection::SendStreamData(unsigned int, unsigned long, unsigned long, net::StreamSendingState) () (gdb) bt
我从x86到arm64迁移，遇到tsc 的问题，x86平台的实现如下：
#define rdtsc(low,high) __asm__ __volatile(“rdtsc”:”=a”(low), “=d”(high))
inline static uint64_t rdtsc() { uint64_t var; asm volatile("mrs %0, CNTVCT_EL0" : "=r"(var)); //uint32_t high, low; //__asm__ __volatile__( // "rdtsc" : "=a" (low), "=d" (high) // ); //return ((uint64_t)high << 32) | low; }
(gdb) b QuicSocketEntry::NewQuicSocketEntry Breakpoint 1 at 0x41b9b0: file /root/posix_quic/src/socket_entry.cpp, line 28. (gdb) b QuicSocketEntry::OnIncomingStream Breakpoint 2 at 0x41a948: QuicSocketEntry::OnIncomingStream. (2 locations) (gdb) b FdManager::Pute Function "FdManager::Pute" not defined. Make breakpoint pending on future shared library load? (y or [n]) n (gdb) b FdManager::Put Function "FdManager::Put" not defined. Make breakpoint pending on future shared library load? (y or [n]) n (gdb) b fd_manager.h:24 Breakpoint 3 at 0x409fc8: file /root/posix_quic/src/fd_manager.h, line 24. (gdb) b fd_manager.h:18 Breakpoint 4 at 0x4097b0: fd_manager.h:18. (8 locations) (gdb) r Starting program: /root/posix_quic/test/client/client [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1". [New Thread 0xffffbf6fe010 (LWP 10607)] Thread 1 "client" hit Breakpoint 4, posix_quic::FdManager<std::shared_ptr<posix_quic::QuicEpollerEntry> >::Put (entry=<error reading variable: access outside bounds of object referenced via synthetic pointer>, fd=3, this=<optimized out>) at /root/posix_quic/src/fd_manager.h:21 21 DebugPrint(dbg_fd, "Put %s fd = %d", entry->DebugTypeInfo(), fd); (gdb) bt #0 posix_quic::FdManager<std::shared_ptr<posix_quic::QuicEpollerEntry> >::Put (entry=<error reading variable: access outside bounds of object referenced via synthetic pointer>, fd=3, this=<optimized out>) at /root/posix_quic/src/fd_manager.h:21 #1 posix_quic::QuicEpollerEntry::NewQuicEpollerEntry () at /root/posix_quic/src/epoller_entry.cpp:99 #2 0x000000000040ec68 in posix_quic::QuicCreateEpoll () at /root/posix_quic/src/quic_socket.cpp:430 #3 0x0000000000400930 in main () at /root/posix_quic/test/client/src/client.cpp:118 (gdb) c Continuing. Thread 1 "client" hit Breakpoint 1, posix_quic::QuicSocketEntry::NewQuicSocketEntry (isServer=isServer@entry=false, id=id@entry=18446744073709551615) at /root/posix_quic/src/socket_entry.cpp:28 28 { (gdb) bt #0 posix_quic::QuicSocketEntry::NewQuicSocketEntry (isServer=isServer@entry=false, id=id@entry=18446744073709551615) at /root/posix_quic/src/socket_entry.cpp:28 #1 0x000000000040ea60 in posix_quic::QuicCreateSocket () at /root/posix_quic/src/quic_socket.cpp:10 #2 0x000000000040093c in main () at /root/posix_quic/test/client/src/client.cpp:121 (gdb) c Continuing. [New Thread 0xffffbeefd010 (LWP 10759)] Thread 1 "client" hit Breakpoint 4, posix_quic::FdManager<std::shared_ptr<posix_quic::EntryBase> >::Put (entry=..., fd=1, this=<optimized out>) at /root/posix_quic/src/fd_manager.h:21 21 DebugPrint(dbg_fd, "Put %s fd = %d", entry->DebugTypeInfo(), fd); (gdb) bt #0 posix_quic::FdManager<std::shared_ptr<posix_quic::EntryBase> >::Put (entry=..., fd=1, this=<optimized out>) at /root/posix_quic/src/fd_manager.h:21 #1 posix_quic::QuicSocketEntry::NewQuicSocketEntry (isServer=isServer@entry=false, id=<optimized out>, id@entry=18446744073709551615) at /root/posix_quic/src/socket_entry.cpp:60 #2 0x000000000040ea60 in posix_quic::QuicCreateSocket () at /root/posix_quic/src/quic_socket.cpp:10 #3 0x000000000040093c in main () at /root/posix_quic/test/client/src/client.cpp:121 (gdb) c Continuing.
QuicSocketEntryPtr socket = QuicSocketEntry::NewQuicSocketEntry(true, connectionId); fwd.h:61:typedef std::weak_ptr<QuicStreamEntry> QuicStreamEntryWeakPtr; fwd.h:62:typedef std::shared_ptr<QuicStreamEntry> QuicStreamEntryPtr; void QuicSocketEntry::OnIncomingStream(QuartcStreamInterface* stream) { QuicStreamEntryPtr streamPtr = QuicStreamEntry::NewQuicStream(shared_from_this(), stream); std::unique_lock<std::mutex> lock(acceptStreamsMtx_); acceptStreams_.push_back(streamPtr); SetReadable(true); }
streamId 通过Alloc()
struct FdFactory { public: int Alloc() { std::unique_lock<std::mutex> lock(mtx_); if (free_.empty()) return ++acq_; int fd = free_.front(); free_.pop(); return fd; } void Free(int fd) { std::unique_lock<std::mutex> lock(mtx_); free struct FdFactory { public: int Alloc() { std::unique_lock<std::mutex> lock(mtx_); if (free_.empty()) return ++acq_; int fd = free_.front(); free_.pop(); return fd; } void Free(int fd) { std::unique_lock<std::mutex> lock(mtx_); free_.push(fd); } private: std::mutex mtx_; int acq_ = 0; std::queue<int> free_; // TODO: 高效的可重用的fd分配算法, 阻止回绕 }; }_.push(fd); } private: std::mutex mtx_; int acq_ = 0; std::queue<int> free_; // TODO: 高效的可重用的fd分配算法, 阻止回绕 }; }
QuicSocketEntryPtr QuicSocketEntry::NewQuicSocketEntry(bool isServer, QuicConnectionId id) { if (id == INVALID_QUIC_CONNECTION_ID) id = QuicRandom::GetInstance()->RandUint64(); std::shared_ptr<QuicTaskRunnerProxy> taskRunnerProxy(new QuicTaskRunnerProxy); std::shared_ptr<QuartcFactory> factory(new QuartcFactory( QuartcFactoryConfig{taskRunnerProxy.get(), &QuicClockImpl::getInstance()})); int fd = GetFdFactory().Alloc();
(gdb) bt #0 posix_quic::FdFactory::Alloc (this=0x838b00) at /root/posix_quic/src/fd_factory.h:14 #1 posix_quic::QuicSocketEntry::NewQuicSocketEntry (isServer=isServer@entry=false, id=8537857, id@entry=18446744073709551615) at /root/posix_quic/src/socket_entry.cpp:37 #2 0x000000000040ea60 in posix_quic::QuicCreateSocket () at /root/posix_quic/src/quic_socket.cpp:10 #3 0x000000000040093c in main () at /root/posix_quic/test/client/src/client.cpp:123
packet_transport.h:19: std::shared_ptr<int> udpSocket_;
int QuicSocketEntry::SetUdpSocket(int fd) { if (udpSocket_) { errno = EINVAL; return -1; } int val = ::fcntl(fd, F_GETFL, 0); if (val == -1) { ::close(fd); return -1; } int res = ::fcntl(fd, F_SETFL, val | O_NONBLOCK); if (res == -1) { ::close(fd); return -1; } val = GetOpt(sockopt_udp_rmem); if (!val) val = kDefaultUdpRecvMem; res = setsockopt(fd, SOL_SOCKET, SO_RCVBUF, (const char*)&val, sizeof(int)); if (res == -1) { ::close(fd); return -1; } val = GetOpt(sockopt_udp_wmem); if (!val) val = kDefaultUdpWriteMem; res = setsockopt(fd, SOL_SOCKET, SO_SNDBUF, (const char*)&val, sizeof(int)); if (res == -1) { ::close(fd); return -1; } udpSocket_.reset(new int(fd), [](int* p){ ::close(*p); delete p; }); socketState_ = QuicSocketState_Inited; GetConnectionManager().Put(*udpSocket_, impl_->connection_id(), Fd(), true); return 0; }
PosixQuicPacketTransport::Set
(gdb) bt #0 posix_quic::PosixQuicPacketTransport::Set (this=0x839000, udpSocket=..., address=...) at /root/posix_quic/src/packet_transport.cpp:6 #1 0x0000000000419dc4 in posix_quic::QuicSocketEntry::Connect (this=this@entry=0x83ba80, addr=addr@entry=0xfffffffff968, addrlen=addrlen@entry=16) at /root/posix_quic/src/socket_entry.cpp:275 #2 0x00000000004100bc in posix_quic::QuicConnect (sock=sock@entry=1, addr=0xfffffffff968, addr@entry=0xfffffffff988, addrlen=addrlen@entry=16) at /root/posix_quic/src/quic_socket.cpp:98 #3 0x0000000000400980 in main () at /root/posix_quic/test/client/src/client.cpp:134 (gdb)
QuicStreamEntry::QuicStreamEntry & QuicSocketEntry::QuicSocketEntry
(gdb) b QuicSocketEntry::QuicSocketEntry Breakpoint 1 at 0x416a78: file /root/posix_quic/src/socket_entry.cpp, line 17. (gdb) b QuicStreamEntry::QuicStreamEntry Breakpoint 2 at 0x41ff48: file /root/posix_quic/src/stream_entry.cpp, line 5. (gdb) r Starting program: /root/posix_quic/test/client/client [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1". [New Thread 0xffffbf6fe010 (LWP 64014)] [19:43:47.598336]fd_manager.h:21:(Put) [C=0] Put Epoll fd = 3 [New Thread 0xffffbeefd010 (LWP 64015)] Thread 1 "client" hit Breakpoint 1, posix_quic::QuicSocketEntry::QuicSocketEntry (this=0x83ba80, session=0x83a530) at /root/posix_quic/src/socket_entry.cpp:17 17 QuicSocketEntry::QuicSocketEntry(QuartcSession * session) (gdb) bt #0 posix_quic::QuicSocketEntry::QuicSocketEntry (this=0x83ba80, session=0x83a530) at /root/posix_quic/src/socket_entry.cpp:17 #1 0x000000000041bbec in posix_quic::QuicSocketEntry::NewQuicSocketEntry (isServer=isServer@entry=false, id=0, id@entry=18446744073709551615) at /root/posix_quic/src/socket_entry.cpp:49 #2 0x000000000040ea60 in posix_quic::QuicCreateSocket () at /root/posix_quic/src/quic_socket.cpp:10 #3 0x000000000040093c in main () at /root/posix_quic/test/client/src/client.cpp:122 (gdb) c Continuing. [19:43:56.723558]fd_manager.h:21:(Put) [C=0] Put Socket fd = 1 [19:43:56.726291]client.cpp:47:(doLoop) [C=0] QuicEpoller trigger: fd = 1, category = Socket, events = EPOLLOUT [19:43:56.726310]client.cpp:51:(doLoop) [C=0] Connected. Thread 1 "client" hit Breakpoint 2, posix_quic::QuicStreamEntry::QuicStreamEntry (this=0x84e390, socketEntry=..., stream=0x84e370) at /root/posix_quic/src/stream_entry.cpp:5 5 QuicStreamEntry::QuicStreamEntry(QuicSocketEntryPtr socketEntry, QuartcStreamInterface* stream) (gdb) bt #0 posix_quic::QuicStreamEntry::QuicStreamEntry (this=0x84e390, socketEntry=..., stream=0x84e370) at /root/posix_quic/src/stream_entry.cpp:5 #1 0x000000000041a2e0 in posix_quic::QuicStreamEntry::NewQuicStream<std::shared_ptr<posix_quic::QuicSocketEntry>, net::QuartcStream*&> (this=<optimized out>, this=<optimized out>) at /root/posix_quic/src/stream_entry.h:42 #2 posix_quic::QuicSocketEntry::CreateStream (this=this@entry=0x83ba80) at /root/posix_quic/src/socket_entry.cpp:383 #3 0x0000000000411628 in posix_quic::QuicCreateStream (sock=sock@entry=1) at /root/posix_quic/src/quic_socket.cpp:171 #4 0x00000000004020b0 in doLoop (ep=3158323, ep@entry=3) at /root/posix_quic/test/client/src/client.cpp:54 #5 0x00000000004009c8 in main () at /root/posix_quic/test/client/src/client.cpp:144
QuicStreamEntryPtr QuicSocketEntry::CreateStream() { if (!IsConnected()) { errno = ENOTCONN; return QuicStreamEntryPtr(); } std::unique_lock<std::recursive_mutex> lock(*mtx_); QuartcStream* stream = impl_->CreateOutgoingDynamicStream(); return QuicStreamEntry::NewQuicStream(shared_from_this(), stream); }
typedef int QuicSocket
QuicStream QuicCreateStream(QuicSocket sock) { auto socket = EntryBase::GetFdManager().Get(sock); if (!socket || socket->Category() != EntryCategory::Socket) { DebugPrint(dbg_api, "sock = %d, return = -1, errno = EBADF", sock); errno = EBADF; return -1; } auto stream = ((QuicSocketEntry*)socket.get())->CreateStream(); if (!stream) { DebugPrint(dbg_api, "sock = %d, return = -1, errno = %d", sock, errno); return -1; } DebugPrint(dbg_api, "sock = %d, newStream = %d", sock, stream->Fd()); return stream->Fd(); }
QuicSocketEntryPtr QuicSocketEntry::AcceptSocket() { if (socketState_ != QuicSocketState_Binded) { errno = EINVAL; return QuicSocketEntryPtr(); } std::unique_lock<std::mutex> lock(acceptSocketsMtx_); if (acceptSockets_.empty()) { errno = EAGAIN; return QuicSocketEntryPtr(); } auto ptr = acceptSockets_.front(); acceptSockets_.pop_front(); return ptr; } QuicStreamEntryPtr QuicSocketEntry::AcceptStream() { if (!IsConnected()) { errno = ENOTCONN; return QuicStreamEntryPtr(); } std::unique_lock<std::mutex> lock(acceptStreamsMtx_); while (!acceptStreams_.empty()) { QuicStreamEntryPtr stream = acceptStreams_.front(); acceptStreams_.pop_front(); return stream; } errno = EAGAIN; return QuicStreamEntryPtr(); }
创建QuicStreamFrame
(gdb) bt #0 0x00000000004a7968 in net::QuicStreamFrame::QuicStreamFrame(unsigned int, bool, unsigned long, unsigned short) () #1 0x00000000004adfd4 in net::QuicPacketCreator::CreateStreamFrame(unsigned int, unsigned long, unsigned long, unsigned long, bool, net::QuicFrame*) () #2 0x00000000004aea60 in net::QuicPacketCreator::ConsumeData(unsigned int, unsigned long, unsigned long, unsigned long, bool, bool, net::QuicFrame*) () #3 0x0000000000441cd4 in net::QuicPacketGenerator::ConsumeData(unsigned int, unsigned long, unsigned long, net::StreamSendingState) () #4 0x0000000000430034 in net::QuicConnection::SendStreamData(unsigned int, unsigned long, unsigned long, net::StreamSendingState) ()
ConnectionManager pk FdManager
FdManager：：PUT fd管理
Thread 1 "client" hit Breakpoint 1, posix_quic::FdManager<std::shared_ptr<posix_quic::QuicEpollerEntry> >::Put (entry=<error reading variable: access outside bounds of object referenced via synthetic pointer>, fd=3, this=<optimized out>) at /root/posix_quic/src/fd_manager.h:21 21 DebugPrint(dbg_fd, "Put %s fd = %d", entry->DebugTypeInfo(), fd); (gdb) bt #0 posix_quic::FdManager<std::shared_ptr<posix_quic::QuicEpollerEntry> >::Put (entry=<error reading variable: access outside bounds of object referenced via synthetic pointer>, fd=3, this=<optimized out>) at /root/posix_quic/src/fd_manager.h:21 #1 posix_quic::QuicEpollerEntry::NewQuicEpollerEntry () at /root/posix_quic/src/epoller_entry.cpp:99 #2 0x000000000040ec68 in posix_quic::QuicCreateEpoll () at /root/posix_quic/src/quic_socket.cpp:430 #3 0x0000000000400930 in main () at /root/posix_quic/test/client/src/client.cpp:120 (gdb)
class QuicEpollerEntry : public FdBase
class Event : public FdBase
// non-blocking quic fd. class QuicSocketEntry : public EntryBase, public std::enable_shared_from_this<QuicSocketEntry>, private QuartcSessionInterface::Delegate class QuicStreamEntry : public EntryBase, public std::enable_shared_from_this<QuicStreamEntry>, private QuartcStreamInterface::Delegate
class FdBase { public: virtual ~FdBase() {} void SetFd(int fd) { fd_ = fd; } int Fd() const { return fd_; } private: int fd_ = 0; };
static QuicStreamEntryPtr NewQuicStream(Args && ... args) { int fd = GetFdFactory().Alloc(); QuicStreamEntryPtr ptr(new QuicStreamEntry(std::forward<Args>(args)...)); ptr->SetFd(fd); GetFdManager().Put(fd, ptr); return ptr; }
ConnectionManager
struct ConnectionManager { public: typedef std::unordered_map<QuicConnectionId, QuicSocket> UdpConnectionMap; typedef std::unordered_map<UdpSocket, UdpConnectionMap> ConnectionMap; typedef std::unordered_map<UdpSocket, QuicSocket> OwnerMap; void Put(UdpSocket udpSocket, QuicConnectionId connectionId, QuicSocket quicSocket, bool owner) { std::unique_lock<std::mutex> lock(mtx_); connections_[udpSocket][connectionId] = quicSocket; if (owner) { owners_[udpSocket] = quicSocket; } } QuicSocket Get(UdpSocket udpSocket, QuicConnectionId connectionId) { std::unique_lock<std::mutex> lock(mtx_); auto it1 = connections_.find(udpSocket); if (connections_.end() == it1) return -1; auto it2 = it1->second.find(connectionId); if (it1->second.end() == it2) return -1; return it2->second; } QuicSocket GetOwner(UdpSocket udpSocket) { std::unique_lock<std::mutex> lock(mtx_); auto it = owners_.find(udpSocket); if (it != owners_.end()) return it->second; return -1; } bool Delete(UdpSocket udpSocket, QuicConnectionId connectionId, QuicSocket quicSocket) { std::unique_lock<std::mutex> lock(mtx_); auto it1 = connections_.find(udpSocket); if (connections_.end() == it1) return false; auto it2 = it1->second.find(connectionId); if (it1->second.end() == it2) return false; it1->second.erase(it2); if (it1->second.empty()) connections_.erase(it1); auto it = owners_.find(udpSocket); if (it != owners_.end() && it->second == quicSocket) { owners_.erase(it); } return true; } private: std::mutex mtx_; ConnectionMap connections_; OwnerMap owners_; };
typedef std::unordered_map<QuicConnectionId, QuicSocket> UdpConnectionMap;
typedef std::unordered_map<UdpSocket, UdpConnectionMap> ConnectionMap;
BBR 流控
QuicSocketEntryPtr QuicSocketEntry::NewQuicSocketEntry(bool isServer, QuicConnectionId id) { if (id == INVALID_QUIC_CONNECTION_ID) id = QuicRandom::GetInstance()->RandUint64(); std::shared_ptr<QuicTaskRunnerProxy> taskRunnerProxy(new QuicTaskRunnerProxy); std::shared_ptr<QuartcFactory> factory(new QuartcFactory( QuartcFactoryConfig{taskRunnerProxy.get(), &QuicClockImpl::getInstance()})); int fd = GetFdFactory().Alloc(); std::shared_ptr<PosixQuicPacketTransport> packetTransport(new PosixQuicPacketTransport); QuartcFactoryInterface::QuartcSessionConfig config; config.is_server = isServer; config.unique_remote_server_id = ""; config.packet_transport = packetTransport.get(); config.congestion_control = QuartcCongestionControl::kBBR; config.connection_id = id; config.max_idle_time_before_crypto_handshake_secs = 10; config.max_time_before_crypto_handshake_secs = 10; QuartcSession* session = (QuartcSession*)factory->CreateQuartcSession(config).release(); QuicSocketEntryPtr sptr(new QuicSocketEntry(session)); taskRunnerProxy->Initialize(id, sptr->mtx_); sptr->SetFd(fd); sptr->factory_ = factory; sptr->packetTransport_ = packetTransport; sptr->taskRunnerProxy_ = taskRunnerProxy; session->SetDelegate(sptr.get()); session->connection()->set_debug_visitor(&sptr->connectionVisitor_); sptr->connectionVisitor_.Bind(session->connection(), &sptr->opts_, sptr.get(), sptr->packetTransport_); GetFdManager().Put(fd, sptr); return sptr; } The latest Google code for the BBR congestion control algorithm is: BBR v2 Alpha/Preview release ---------------------------- - For Linux TCP BBR: https://github.com/google/bbr/blob/v2alpha/net/ipv4/tcp_bbr2.c - For QUIC BBR: https://cs.chromium.org/chromium/src/net/third_party/quiche/src/quic/core/congestion_control/bbr2_sender.cc https://cs.chromium.org/chromium/src/net/third_party/quiche/src/quic/core/congestion_control/bbr2_sender.h BBR v1 release ---------------------------- - For Linux TCP BBR: https://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next.git/tree/net/ipv4/tcp_bbr.c - For QUIC BBR: https://cs.chromium.org/chromium/src/net/third_party/quiche/src/quic/core/congestion_control/bbr_sender.cc https://cs.chromium.org/chromium/src/net/third_party/quiche/src/quic/core/congestion_control/bbr_sender.h --- a/net/quic/core/quic_sent_packet_manager.cc +++ b/net/quic/core/quic_sent_packet_manager.cc @@ -117,6 +117,9 @@ void QuicSentPacketManager::SetFromConfig(const QuicConfig& config) { } } else if (config.HasClientRequestedIndependentOption(kBYTE, perspective_)) { SetSendAlgorithm(kCubic); + } else if (FLAGS_quic_reloadable_flag_quic_default_to_bbr && + config.HasClientRequestedIndependentOption(kQBIC, perspective_)) { + SetSendAlgorithm(kCubicBytes); } else if (FLAGS_quic_reloadable_flag_quic_enable_pcc && config.HasClientRequestedIndependentOption(kTPCC, perspective_)) { SetSendAlgorithm(kPCC);
sendto
int PosixQuicPacketTransport::Write(const char* buffer, size_t buf_len) { if (!udpSocket_) { DebugPrint(dbg_write, "buffer length = %d, no udp socket. return = -1", (int)buf_len); return -1; } if (!address_.IsInitialized()) { DebugPrint(dbg_write, "Udp socket = %d, buffer length = %d, address not initialized. return = -1", *udpSocket_, (int)buf_len); return -1; } struct sockaddr_storage addr = address_.generic_address(); retry_sendto: int res = ::sendto(*udpSocket_, buffer, buf_len, 0, (const struct sockaddr*)&addr, sizeof(addr)); if (res == -1 && errno == EINTR) goto retry_sendto; DebugPrint(dbg_write, "sento %s Udp socket = %d, buffer length = %d, return = %d, errno = %d", address_.ToString().c_str(), *udpSocket_, (int)buf_len, res, errno); // 告诉协议栈, udp总是可用的, 以便于切网也可以正常重传 return buf_len; }
#0 0x0000000000433e70 in net::QuicConnection::WritePacket(net::SerializedPacket*) () #1 0x0000000000434eac in net::QuicConnection::OnSerializedPacket(net::SerializedPacket*) () #2 0x00000000004ad984 in net::QuicPacketCreator::OnSerializedPacket() () #3 0x00000000004aeff0 in net::QuicPacketCreator::Flush() [clone .part.54] () #4 0x0000000000441ef8 in net::QuicPacketGenerator::ConsumeData(unsigned int, unsigned long, unsigned long, net::StreamSendingState) () #5 0x0000000000430034 in net::QuicConnection::SendStreamData(unsigned int, unsigned long, unsigned long, net::StreamSendingState) () #6 0x0000000000447de0 in net::QuicSession::WritevData(net::QuicStream*, unsigned int, unsigned long, unsigned long, net::StreamSendingState) ()
socket EntryCategory
EntryCategory GetCategory(int fd) { auto socket = EntryBase::GetFdManager().Get(fd); if (!socket) return EntryCategory::Invalid; return socket->Category(); } enum class EntryCategory : int8_t { Invalid, Socket, Stream, }; class EntryBase : public Event { public: virtual EntryCategory Category() const = 0; virtual std::shared_ptr<int> NativeUdpFd() const = 0; virtual int Close() = 0; static FdFactory & GetFdFactory(); static FdManager<EntryPtr> & GetFdManager(); const char* DebugTypeInfo() override { return EntryCategory2Str((int)Category()); }; virtual std::string GetDebugInfo(int indent) = 0; }; class QuicSocketEntry : public EntryBase, public std::enable_shared_from_this<QuicSocketEntry>, private QuartcSessionInterface::Delegate { public: explicit QuicSocketEntry(QuartcSession * session); ~QuicSocketEntry(); EntryCategory Category() const override { return EntryCategory::Socket; } } class QuicStreamEntry : public EntryBase, public std::enable_shared_from_this<QuicStreamEntry>, private QuartcStreamInterface::Delegate { public: QuicStreamEntry(QuicSocketEntryPtr socketEntry, QuartcStreamInterface* stream); EntryCategory Category() const override { return EntryCategory::Stream; } }
ssh握手
(gdb) b CryptoConnect Breakpoint 1 at 0x4a9818 (3 locations) (gdb) b WriteOrBufferData Breakpoint 2 at 0x451884 (gdb) r Starting program: /root/posix_quic/test/client/client [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1". [New Thread 0xffffbf6fe010 (LWP 38231)] [New Thread 0xffffbeefd010 (LWP 38232)] Thread 1 "client" hit Breakpoint 1, 0x00000000004a9818 in net::QuicCryptoClientStream::CryptoConnect() () (gdb) bt #0 0x00000000004a9818 in net::QuicCryptoClientStream::CryptoConnect() () #1 0x0000000000419f48 in posix_quic::QuicSocketEntry::Connect (this=this@entry=0x83b080, addr=addr@entry=0xfffffffff978, addrlen=addrlen@entry=16) at /root/posix_quic/src/socket_entry.cpp:282 #2 0x00000000004100bc in posix_quic::QuicConnect (sock=sock@entry=1, addr=0xfffffffff978, addr@entry=0xfffffffff998, addrlen=addrlen@entry=16) at /root/posix_quic/src/quic_socket.cpp:98 #3 0x0000000000400980 in main () at /root/posix_quic/test/client/src/client.cpp:132 (gdb) c Continuing. Thread 1 "client" hit Breakpoint 1, 0x00000000004cc5d0 in net::QuicCryptoClientHandshaker::CryptoConnect() () (gdb) bt #0 0x00000000004cc5d0 in net::QuicCryptoClientHandshaker::CryptoConnect() () #1 0x0000000000419f48 in posix_quic::QuicSocketEntry::Connect (this=this@entry=0x83b080, addr=addr@entry=0xfffffffff978, addrlen=addrlen@entry=16) at /root/posix_quic/src/socket_entry.cpp:282 #2 0x00000000004100bc in posix_quic::QuicConnect (sock=sock@entry=1, addr=0xfffffffff978, addr@entry=0xfffffffff998, addrlen=addrlen@entry=16) at /root/posix_quic/src/quic_socket.cpp:98 #3 0x0000000000400980 in main () at /root/posix_quic/test/client/src/client.cpp:132 (gdb) c Continuing. Thread 1 "client" hit Breakpoint 2, 0x0000000000451884 in net::QuicStream::WriteOrBufferData(base::BasicStringPiece<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, bool, net::QuicReferenceCountedPointer<net::QuicAckListenerInterface>) () (gdb) bt #0 0x0000000000451884 in net::QuicStream::WriteOrBufferData(base::BasicStringPiece<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, bool, net::QuicReferenceCountedPointer<net::QuicAckListenerInterface>) () #1 0x00000000004ccadc in net::QuicCryptoHandshaker::SendHandshakeMessage(net::CryptoHandshakeMessage const&) () #2 0x00000000004cad2c in net::QuicCryptoClientHandshaker::DoSendCHLO(net::QuicCryptoClientConfig::CachedState*) () #3 0x00000000004cc1ec in net::QuicCryptoClientHandshaker::DoHandshakeLoop(net::CryptoHandshakeMessage const*) () #4 0x00000000004cc5d8 in net::QuicCryptoClientHandshaker::CryptoConnect() () #5 0x0000000000419f48 in posix_quic::QuicSocketEntry::Connect (this=this@entry=0x83b080, addr=addr@entry=0xfffffffff978, addrlen=addrlen@entry=16) at /root/posix_quic/src/socket_entry.cpp:282 #6 0x00000000004100bc in posix_quic::QuicConnect (sock=sock@entry=1, addr=0xfffffffff978, addr@entry=0xfffffffff998, addrlen=addrlen@entry=16) at /root/posix_quic/src/quic_socket.cpp:98 #7 0x0000000000400980 in main () at /root/posix_quic/test/client/src/client.cpp:132 (gdb) c
#0 0x0000000000463288 in net::QuartcSession::StartCryptoHandshake() () #1 0x0000000000419f48 in posix_quic::QuicSocketEntry::Connect (this=this@entry=0x83b080, addr=addr@entry=0xfffffffff978, addrlen=addrlen@entry=16) at /root/posix_quic/src/socket_entry.cpp:282 #2 0x00000000004100bc in posix_quic::QuicConnect (sock=sock@entry=1, addr=0xfffffffff978, addr@entry=0xfffffffff998, addrlen=addrlen@entry=16) at /root/posix_quic/src/quic_socket.cpp:98 #3 0x0000000000400980 in main () at /root/posix_quic/test/client/src/client.cpp:132
void QuartcSession::StartCryptoHandshake() { if (perspective_ == Perspective::IS_CLIENT) { QuicServerId server_id(unique_remote_server_id_, kQuicServerPort, /*privacy_mode_enabled=*/false); QuicCryptoClientStream* crypto_stream = new QuicCryptoClientStream( server_id, this, quic_crypto_client_config_->proof_verifier()->CreateDefaultContext(), quic_crypto_client_config_.get(), this); crypto_stream_.reset(crypto_stream); QuicSession::Initialize(); crypto_stream->CryptoConnect(); } else { quic_compressed_certs_cache_.reset(new QuicCompressedCertsCache( QuicCompressedCertsCache::kQuicCompressedCertsCacheSize)); bool use_stateless_rejects_if_peer_supported = false; QuicCryptoServerStream* crypto_stream = new QuicCryptoServerStream( quic_crypto_server_config_.get(), quic_compressed_certs_cache_.get(), use_stateless_rejects_if_peer_supported, this, &stream_helper_); crypto_stream_.reset(crypto_stream); QuicSession::Initialize(); } }
root@ubuntu:~/posix_quic/test/client# gdb ./client GNU gdb (Ubuntu 8.1-0ubuntu3.2) 8.1.0.20180409-git Copyright (C) 2018 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "aarch64-linux-gnu". Type "show configuration" for configuration details. For bug reporting instructions, please see: <http://www.gnu.org/software/gdb/bugs/>. Find the GDB manual and other documentation resources online at: <http://www.gnu.org/software/gdb/documentation/>. For help, type "help". Type "apropos word" to search for commands related to "word"... Reading symbols from ./client...done. (gdb) b WriteOrBufferData Breakpoint 1 at 0x451884 (gdb) r Starting program: /root/posix_quic/test/client/client [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/aarch64-linux-gnu/libthread_db.so.1". [New Thread 0xffffbf6fe010 (LWP 62596)] [New Thread 0xffffbeefd010 (LWP 62597)] Thread 1 "client" hit Breakpoint 1, 0x0000000000451884 in net::QuicStream::WriteOrBufferData(base::BasicStringPiece<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, bool, net::QuicReferenceCountedPointer<net::QuicAckListenerInterface>) () (gdb) bt #0 0x0000000000451884 in net::QuicStream::WriteOrBufferData(base::BasicStringPiece<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, bool, net::QuicReferenceCountedPointer<net::QuicAckListenerInterface>) () #1 0x00000000004ccadc in net::QuicCryptoHandshaker::SendHandshakeMessage(net::CryptoHandshakeMessage const&) () #2 0x00000000004cad2c in net::QuicCryptoClientHandshaker::DoSendCHLO(net::QuicCryptoClientConfig::CachedState*) () #3 0x00000000004cc1ec in net::QuicCryptoClientHandshaker::DoHandshakeLoop(net::CryptoHandshakeMessage const*) () #4 0x00000000004cc5d8 in net::QuicCryptoClientHandshaker::CryptoConnect() () #5 0x0000000000419f48 in posix_quic::QuicSocketEntry::Connect (this=this@entry=0x83b080, addr=addr@entry=0xfffffffff968, addrlen=addrlen@entry=16) at /root/posix_quic/src/socket_entry.cpp:282 #6 0x00000000004100bc in posix_quic::QuicConnect (sock=sock@entry=1, addr=0xfffffffff968, addr@entry=0xfffffffff988, addrlen=addrlen@entry=16) at /root/posix_quic/src/quic_socket.cpp:98 #7 0x0000000000400980 in main () at /root/posix_quic/test/client/src/client.cpp:132 (gdb)
QuicStream::WritevData
Thread 1 "client" hit Breakpoint 1, 0x0000000000451efc in net::QuicStream::WritevData(iovec const*, int, bool) () (gdb) bt #0 0x0000000000451efc in net::QuicStream::WritevData(iovec const*, int, bool) () #1 0x00000000004650cc in net::QuartcStream::WritevData(iovec const*, int, bool) () #2 0x000000000041f350 in posix_quic::QuicStreamEntry::Writev (this=this@entry=0x84dbc0, iov=0xb00cb4775b5ab200, iov@entry=0xffffffff8ff8, iov_count=iov_count@entry=1, fin=fin@entry=false) at /root/posix_quic/src/stream_entry.cpp:33 #3 0x0000000000412b4c in posix_quic::QuicWritev (stream=stream@entry=2, iov=0xffffffff8ff8, iov@entry=0xffffffff9008, iov_count=iov_count@entry=1, fin=fin@entry=false) at /root/posix_quic/src/quic_socket.cpp:236 #4 0x0000000000412eb0 in posix_quic::QuicWrite (stream=stream@entry=2, data=data@entry=0xffffffff9118, length=length@entry=11, fin=fin@entry=false) at /root/posix_quic/src/quic_socket.cpp:246 #5 0x0000000000402114 in doLoop (ep=3487025, ep@entry=3) at /root/posix_quic/test/client/src/client.cpp:65 #6 0x00000000004009c8 in main () at /root/posix_quic/test/client/src/client.cpp:143 (gdb)
QuicSession::WritevData
Thread 1 "client" hit Breakpoint 2, 0x0000000000447cec in net::QuicSession::WritevData(net::QuicStream*, unsigned int, unsigned long, unsigned long, net::StreamSendingState) () (gdb) bt #0 0x0000000000447cec in net::QuicSession::WritevData(net::QuicStream*, unsigned int, unsigned long, unsigned long, net::StreamSendingState) () #1 0x00000000004516cc in net::QuicStream::WriteBufferedData() () #2 0x0000000000452420 in net::QuicStream::WritevData(iovec const*, int, bool) () #3 0x00000000004650cc in net::QuartcStream::WritevData(iovec const*, int, bool) () #4 0x000000000041f350 in posix_quic::QuicStreamEntry::Writev (this=this@entry=0x84dbc0, iov=0xb00cb4775b5ab200, iov@entry=0xffffffff8ff8, iov_count=iov_count@entry=1, fin=fin@entry=false) at /root/posix_quic/src/stream_entry.cpp:33 #5 0x0000000000412b4c in posix_quic::QuicWritev (stream=stream@entry=2, iov=0xffffffff8ff8, iov@entry=0xffffffff9008, iov_count=iov_count@entry=1, fin=fin@entry=false) at /root/posix_quic/src/quic_socket.cpp:236 #6 0x0000000000412eb0 in posix_quic::QuicWrite (stream=stream@entry=2, data=data@entry=0xffffffff9118, length=length@entry=11, fin=fin@entry=false) at /root/posix_quic/src/quic_socket.cpp:246 #7 0x0000000000402114 in doLoop (ep=3487025, ep@entry=3) at /root/posix_quic/test/client/src/client.cpp:65 #8 0x00000000004009c8 in main () at /root/posix_quic/test/client/src/client.cpp:143 (gdb)
(gdb) bt #0 posix_quic::PosixQuicPacketTransport::Write (this=0x34ab3200, buffer=0xffffcb386d40 "r %266215214377220 02206 310355336D357w351333304@ 02", buf_len=38) at /root/posix_quic/src/packet_transport.cpp:18 #1 0x0000000000465ee0 in net::QuartcPacketWriter::WritePacket(char const*, unsigned long, net::QuicIpAddress const&, net::QuicSocketAddress const&, net::PerPacketOptions*) () #2 0x00000000004368b4 in net::QuicConnection::WritePacket(net::SerializedPacket*) [clone .part.191] [clone .constprop.259] () #3 0x0000000000437e5c in net::QuicConnection::OnSerializedPacket(net::SerializedPacket*) () #4 0x00000000004af934 in net::QuicPacketCreator::OnSerializedPacket() () #5 0x00000000004b0fa0 in net::QuicPacketCreator::Flush() [clone .part.54] () #6 0x0000000000444fe0 in net::QuicPacketGenerator::Flush() () #7 0x0000000000432c00 in net::QuicConnection::ScopedPacketFlusher::~ScopedPacketFlusher() ()
(gdb) bt #0 posix_quic::PosixQuicPacketTransport::Write (this=0x34abf390, buffer=0xffffcb383200 "252r273366224264221353 01, 355343335344301364277244 64 71240 01 01kREJ", buf_len=1200) at /root/posix_quic/src/packet_transport.cpp:18 #1 0x0000000000465ee0 in net::QuartcPacketWriter::WritePacket(char const*, unsigned long, net::QuicIpAddress const&, net::QuicSocketAddress const&, net::PerPacketOptions*) () #2 0x00000000004368b4 in net::QuicConnection::WritePacket(net::SerializedPacket*) [clone .part.191] [clone .constprop.259] () #3 0x0000000000437e5c in net::QuicConnection::OnSerializedPacket(net::SerializedPacket*) () #4 0x00000000004af934 in net::QuicPacketCreator::OnSerializedPacket() () #5 0x00000000004b0fa0 in net::QuicPacketCreator::Flush() [clone .part.54] () #6 0x0000000000444ea8 in net::QuicPacketGenerator::ConsumeData(unsigned int, unsigned long, unsigned long, net::StreamSendingState) () #7 0x0000000000432fe4 in net::QuicConnection::SendStreamData(unsigned int, unsigned long, unsigned long, net::StreamSendingState) () #8 0x000000000044ad90 in net::QuicSession::WritevData(net::QuicStream*, unsigned int, unsigned long, unsigned long, net::StreamSendingState) () #9 0x000000000044ad90 in net::QuicSession::WritevData(net::QuicStream*, unsigned int, unsigned long, unsigned long, net::StreamSendingState) () #10 0x000000000044ad90 in net::QuicSession::WritevData(net::QuicStream*, unsigned int, unsigned long, unsigned long, net::StreamSendingState) ()
WriteResult QuartcPacketWriter::WritePacket( const char* buffer, size_t buf_len, const QuicIpAddress& self_address, const QuicSocketAddress& peer_address, PerPacketOptions* options) { DCHECK(packet_transport_); int bytes_written = packet_transport_->Write(buffer, buf_len); if (bytes_written <= 0) { writable_ = false; return WriteResult(WRITE_STATUS_BLOCKED, EWOULDBLOCK); } return WriteResult(WRITE_STATUS_OK, bytes_written); }
packet_transport_初始化
QuartcPacketWriter::QuartcPacketWriter( QuartcSessionInterface::PacketTransport* packet_transport, QuicByteCount max_packet_size) : packet_transport_(packet_transport), max_packet_size_(max_packet_size) {}
(gdb) bt #0 0x0000000000462f44 in net::QuartcPacketWriter::QuartcPacketWriter(net::QuartcSessionInterface::PacketTransport*, unsigned long) () #1 0x0000000000462758 in net::QuartcFactory::CreateQuicConnection(net::QuartcFactoryInterface::QuartcSessionConfig const&, net::Perspective) () #2 0x00000000004628dc in net::QuartcFactory::CreateQuartcSession(net::QuartcFactoryInterface::QuartcSessionConfig const&) () #3 0x000000000041bb7c in posix_quic::QuicSocketEntry::NewQuicSocketEntry (isServer=isServer@entry=false, id=0, id@entry=18446744073709551615) at /root/posix_quic/src/socket_entry.cpp:47 #4 0x000000000040ea10 in posix_quic::QuicCreateSocket () at /root/posix_quic/src/quic_socket.cpp:10 #5 0x000000000040093c in main () at /root/posix_quic/test/server/src/server.cpp:132 (gdb) quit
class PosixQuicPacketTransport : public QuartcSessionInterface::PacketTransport { public: void Set(std::shared_ptr<int> udpSocket, QuicSocketAddress const& address); void UpdatePeerAddress(QuicSocketAddress const& address); int Write(const char* buffer, size_t buf_len) override; private: std::shared_ptr<int> udpSocket_; QuicSocketAddress address_; };
// Send and receive packets, like a virtual UDP socket. For example, this // could be implemented by WebRTC's IceTransport. class PacketTransport { public: virtual ~PacketTransport() {} // Called by the QuartcPacketWriter when writing packets to the network. // Return the number of written bytes. Return 0 if the write is blocked. virtual int Write(const char* buffer, size_t buf_len) = 0; };
std::shared_ptr<QuartcFactory> factory(new QuartcFactory( QuartcFactoryConfig{taskRunnerProxy.get(), &QuicClockImpl::getInstance()})); int fd = GetFdFactory().Alloc(); std::shared_ptr<PosixQuicPacketTransport> packetTransport(new PosixQuicPacketTransport); QuartcFactoryInterface::QuartcSessionConfig config; config.is_server = isServer; config.unique_remote_server_id = ""; config.packet_transport = packetTransport.get(); config.congestion_control = QuartcCongestionControl::kBBR; config.connection_id = id; config.max_idle_time_before_crypto_handshake_secs = 10; config.max_time_before_crypto_handshake_secs = 10; QuartcSession* session = (QuartcSession*)factory->CreateQuartcSession(config).release();
recv
if (!owner) { // listen socket已关闭, 不再接受新连接 DebugPrint(dbg_epoll | dbg_read | dbg_accept, " -> recvfrom. Udp socket = %d, connectionId = %lu is a new Connection. Not found owner, so ignore it!", udpFd, connectionId); continue; } DebugPrint(dbg_epoll | dbg_read | dbg_accept, " -> recvfrom. Udp socket = %d, connectionId = %lu is a new Connection.", udpFd, connectionId); QuicSocketEntryPtr socket = QuicSocketEntry::NewQuicSocketEntry(true, connectionId); socket->OnSyn(owner, peerAddress); socket->ProcessUdpPacket(selfAddress, peerAddress, QuicReceivedPacket(&udpRecvBuf_[0], bytes, QuicClockImpl::getInstance().Now())); continue; } DebugPrint(dbg_epoll | dbg_read, " -> recvfrom. Udp socket = %d, connectionId = %lu is exists.", udpFd, connectionId);
QuicStreamEntry::OnReceived
Thread 1 "server" hit Breakpoint 1, non-virtual thunk to posix_quic::QuicStreamEntry::OnReceived(net::QuartcStreamInterface*, char const*, unsigned long) () at /root/posix_quic/src/stream_entry.h:54 54 void OnReceived(QuartcStreamInterface* stream, const char* data, size_t size) override; (gdb) bt #0 non-virtual thunk to posix_quic::QuicStreamEntry::OnReceived(net::QuartcStreamInterface*, char const*, unsigned long) () at /root/posix_quic/src/stream_entry.h:54 #1 0x00000000004653a4 in net::QuartcStream::OnDataAvailable() () #2 0x000000000045687c in net::QuicStreamSequencer::OnStreamFrame(net::QuicStreamFrame const&) () #3 0x0000000000450ac8 in net::QuicStream::OnStreamFrame(net::QuicStreamFrame const&) () #4 0x000000000044ca24 in net::QuicSession::OnStreamFrame(net::QuicStreamFrame const&) () #5 0x000000000043201c in net::QuicConnection::OnStreamFrame(net::QuicStreamFrame const&) () #6 0x000000000043fc74 in net::QuicFramer::ProcessFrameData(net::QuicDataReader*, net::QuicPacketHeader const&) [clone .part.162] [clone .constprop.172] () #7 0x0000000000440208 in net::QuicFramer::ProcessDataPacket(net::QuicDataReader*, net::QuicPacketHeader*, net::QuicEncryptedPacket const&, char*, unsigned long) [clone .part.163] [clone .constprop.166] () #8 0x0000000000440634 in net::QuicFramer::ProcessPacket(net::QuicEncryptedPacket const&) () #9 0x0000000000434908 in net::QuicConnection::ProcessUdpPacket(net::QuicSocketAddress const&, net::QuicSocketAddress const&, net::QuicReceivedPacket const&) () #10 0x00000000004171a8 in posix_quic::QuicSocketEntry::ProcessUdpPacket (this=this@entry=0x84ba50, self_address=..., peer_address=..., packet=...) at /root/posix_quic/src/socket_entry.cpp:415 #11 0x0000000000406fa0 in posix_quic::QuicEpollerEntry::Wait (this=this@entry=0x833950, events=0x461740 <net::QuicSocketAddressImpl::QuicSocketAddressImpl(sockaddr_storage const&)+48>, events@entry=0xffffffffb908, maxevents=65535, maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/epoller_entry.cpp:376 #12 0x000000000041577c in posix_quic::QuicEpollWait (epfd=3, epfd@entry=0, events=events@entry=0xffffffffb908, maxevents=maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/quic_socket.cpp:494 #13 0x0000000000401f74 in doLoop (ep=0, ep@entry=3, listenSock=0, listenSock@entry=1) at /root/posix_quic/test/server/src/server.cpp:48 #14 0x00000000004009ac in main () at /root/posix_quic/test/server/src/server.cpp:153 (gdb)
void QuicSocketEntry::ProcessUdpPacket(const QuicSocketAddress& self_address, const QuicSocketAddress& peer_address, const QuicReceivedPacket& packet) { TlsConnectionIdGuard guard(impl_->connection_id()); DebugPrint(dbg_connect | dbg_accept | dbg_write | dbg_read, "fd = %d, packet length = %d", Fd(), (int)packet.length()); std::unique_lock<std::recursive_mutex> lock(*mtx_); impl_->FlushWrites(); impl_->ProcessUdpPacket(self_address, peer_address, packet); }
(gdb) bt #0 posix_quic::QuicSocketEntry::QuicSocketEntry (this=0x837a80, session=0x836530) at /root/posix_quic/src/socket_entry.cpp:17 #1 0x000000000041bb9c in posix_quic::QuicSocketEntry::NewQuicSocketEntry (isServer=isServer@entry=false, id=0, id@entry=18446744073709551615) at /root/posix_quic/src/socket_entry.cpp:49 #2 0x000000000040ea10 in posix_quic::QuicCreateSocket () at /root/posix_quic/src/quic_socket.cpp:10 #3 0x000000000040093c in main () at /root/posix_quic/test/server/src/server.cpp:132 (gdb)
QuicSocketEntry::QuicSocketEntry(QuartcSession * session) : mtx_(std::make_shared<std::recursive_mutex>()), impl_(session) { }
QuartcSession* session = (QuartcSession*)factory->CreateQuartcSession(config).release(); QuicSocketEntryPtr sptr(new QuicSocketEntry(session));
ProcessUdpPacket
libquic/net/quic/core/quic_session.cc:380:void QuicSession::ProcessUdpPacket(
void QuicSession::ProcessUdpPacket(const QuicSocketAddress& self_address, const QuicSocketAddress& peer_address, const QuicReceivedPacket& packet) { connection_->ProcessUdpPacket(self_address, peer_address, packet); }
void QuicConnection::ProcessUdpPacket(const QuicSocketAddress& self_address, const QuicSocketAddress& peer_address, const QuicReceivedPacket& packet) { if (!connected_) { return; } QUIC_DVLOG(2) << ENDPOINT << "Received encrypted " << packet.length() << " bytes:" << std::endl << QuicTextUtils::HexDump( QuicStringPiece(packet.data(), packet.length())); QUIC_BUG_IF(current_packet_data_ != nullptr) << "ProcessUdpPacket must not be called while processing a packet."; if (debug_visitor_ != nullptr) { debug_visitor_->OnPacketReceived(self_address, peer_address, packet); } last_size_ = packet.length(); current_packet_data_ = packet.data(); last_packet_destination_address_ = self_address; last_packet_source_address_ = peer_address; if (!self_address_.IsInitialized()) { self_address_ = last_packet_destination_address_; } if (!direct_peer_address_.IsInitialized()) { direct_peer_address_ = last_packet_source_address_; } if (!effective_peer_address_.IsInitialized()) { const QuicSocketAddress effective_peer_addr = GetEffectivePeerAddressFromCurrentPacket(); // effective_peer_address_ must be initialized at the beginning of the // first packet processed(here). If effective_peer_addr is uninitialized, // just set effective_peer_address_ to the direct peer address. effective_peer_address_ = effective_peer_addr.IsInitialized() ? effective_peer_addr : direct_peer_address_; } stats_.bytes_received += packet.length(); ++stats_.packets_received; // Ensure the time coming from the packet reader is within 2 minutes of now. if (std::abs((packet.receipt_time() - clock_->ApproximateNow()).ToSeconds()) > 2 * 60) { QUIC_BUG << "Packet receipt time:" << packet.receipt_time().ToDebuggingValue() << " too far from current time:" << clock_->ApproximateNow().ToDebuggingValue(); } time_of_last_received_packet_ = packet.receipt_time(); QUIC_DVLOG(1) << ENDPOINT << "time of last received packet: " << time_of_last_received_packet_.ToDebuggingValue(); ScopedPacketFlusher flusher(this); if (!framer_.ProcessPacket(packet)) { // If we are unable to decrypt this packet, it might be // because the CHLO or SHLO packet was lost. if (framer_.error() == QUIC_DECRYPTION_FAILURE) { ++stats_.undecryptable_packets_received; if (encryption_level_ != ENCRYPTION_FORWARD_SECURE && undecryptable_packets_.size() < max_undecryptable_packets_) { QueueUndecryptablePacket(packet); } else if (debug_visitor_ != nullptr) { debug_visitor_->OnUndecryptablePacket(); } } QUIC_DVLOG(1) << ENDPOINT << "Unable to process packet. Last packet processed: " << last_header_.packet_number; current_packet_data_ = nullptr; is_current_packet_connectivity_probing_ = false; MaybeProcessCoalescedPackets(); return; } ++stats_.packets_processed; QUIC_DLOG_IF(INFO, active_effective_peer_migration_type_ != NO_CHANGE) << "sent_packet_manager_.GetLargestObserved() = " << sent_packet_manager_.GetLargestObserved() << ", highest_packet_sent_before_effective_peer_migration_ = " << highest_packet_sent_before_effective_peer_migration_; if (active_effective_peer_migration_type_ != NO_CHANGE && sent_packet_manager_.GetLargestObserved().IsInitialized() && (!highest_packet_sent_before_effective_peer_migration_.IsInitialized() || sent_packet_manager_.GetLargestObserved() > highest_packet_sent_before_effective_peer_migration_)) { if (perspective_ == Perspective::IS_SERVER) { OnEffectivePeerMigrationValidated(); } } MaybeProcessCoalescedPackets(); MaybeProcessUndecryptablePackets(); MaybeSendInResponseToPacket(); SetPingAlarm(); current_packet_data_ = nullptr; is_current_packet_connectivity_probing_ = false; }
delegate_->OnReceived
void QuartcStream::OnDataAvailable() { struct iovec iov; while (sequencer()->GetReadableRegions(&iov, 1) == 1) { DCHECK(delegate_); delegate_->OnReceived(this, reinterpret_cast<const char*>(iov.iov_base), iov.iov_len); sequencer()->MarkConsumed(iov.iov_len); } // All the data has been received if the sequencer is closed. // Notify the delegate by calling the callback function one more time with // iov_len = 0. if (sequencer()->IsClosed()) { OnFinRead(); delegate_->OnReceived(this, reinterpret_cast<const char*>(iov.iov_base), 0); } }
关闭server 和重传
root@ubuntu:~/posix_quic/test/server# tcpdump -i lo udp -ennvv tcpdump: listening on lo, link-type EN10MB (Ethernet), capture size 262144 bytes 19:27:54.673665 00:00:00:00:00:00 > 00:00:00:00:00:00, ethertype IPv4 (0x0800), length 1242: (tos 0x0, ttl 64, id 9481, offset 0, flags [DF], proto UDP (17), length 1228) 127.0.0.1.54019 > 127.0.0.1.9700: [bad udp cksum 0x02cc -> 0x2e47!] UDP, length 1200 19:27:54.925886 00:00:00:00:00:00 > 00:00:00:00:00:00, ethertype IPv4 (0x0800), length 1242: (tos 0x0, ttl 64, id 9506, offset 0, flags [DF], proto UDP (17), length 1228) 127.0.0.1.54019 > 127.0.0.1.9700: [bad udp cksum 0x02cc -> 0xc19d!] UDP, length 1200 19:27:55.228133 00:00:00:00:00:00 > 00:00:00:00:00:00, ethertype IPv4 (0x0800), length 1242: (tos 0x0, ttl 64, id 9554, offset 0, flags [DF], proto UDP (17), length 1228) 127.0.0.1.54019 > 127.0.0.1.9700: [bad udp cksum 0x02cc -> 0x208f!] UDP, length 1200
(gdb) bt #0 0x0000000000447cec in net::QuicSession::WritevData(net::QuicStream*, unsigned int, unsigned long, unsigned long, net::StreamSendingState) () #1 0x00000000004aaf9c in net::QuicCryptoStream::RetransmitStreamData(unsigned long, unsigned long, bool) () #2 0x000000000044a14c in net::QuicSession::RetransmitFrames(std::vector<net::QuicFrame, std::allocator<net::QuicFrame> > const&, net::TransmissionType) () #3 0x0000000000445c00 in net::QuicSentPacketManager::MarkForRetransmission(unsigned long, net::TransmissionType) () #4 0x00000000004460b0 in net::QuicSentPacketManager::RetransmitCryptoPackets() () #5 0x00000000004343ec in net::QuicConnection::OnRetransmissionTimeout() () #6 0x000000000046220c in net::(anonymous namespace)::QuartcAlarm::Run() () #7 0x0000000000421b44 in posix_quic::QuicTaskRunner::RunOnce (this=0x814000 <tunable_list+448>, this@entry=0x837ce0) at /root/posix_quic/src/task_runner.cpp:80 #8 0x00000000004065a8 in posix_quic::QuicEpollerEntry::Wait (this=this@entry=0x837950, events=0x5b20297325283a64, events@entry=0xffffffff9108, maxevents=624587557, maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/epoller_entry.cpp:272 #9 0x00000000004157d4 in posix_quic::QuicEpollWait (epfd=3, epfd@entry=0, events=events@entry=0xffffffff9108, maxevents=maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/quic_socket.cpp:494 #10 0x0000000000401e04 in doLoop (ep=0, ep@entry=3) at /root/posix_quic/test/client/src/client.cpp:37 #11 0x00000000004009cc in main () at /root/posix_quic/test/client/src/client.cpp:143 (gdb)
QuicEpollerEntry::AddInner
Thread 1 "server" hit Breakpoint 1, posix_quic::QuicEpollerEntry::AddInner (this=this@entry=0x836950, fd=fd@entry=1, event=event@entry=0xfffffffff948) at /root/posix_quic/src/epoller_entry.cpp:115 115 { (gdb) bt #0 posix_quic::QuicEpollerEntry::AddInner (this=this@entry=0x836950, fd=fd@entry=1, event=event@entry=0xfffffffff948) at /root/posix_quic/src/epoller_entry.cpp:115 #1 0x0000000000408958 in posix_quic::QuicEpollerEntry::Add (this=this@entry=0x836950, fd=fd@entry=1, event=event@entry=0xfffffffff948) at /root/posix_quic/src/epoller_entry.cpp:110 #2 0x00000000004154f0 in posix_quic::QuicEpollCtl (epfd=epfd@entry=3, op=op@entry=1, quicFd=quicFd@entry=1, event=0xfffffffff948, event@entry=0xfffffffff968) at /root/posix_quic/src/quic_socket.cpp:462 #3 0x00000000004009a0 in main () at /root/posix_quic/test/server/src/server.cpp:147 (gdb)
server bug
int main() { debug_mask = dbg_all & ~dbg_timer; QuicEpoller ep = QuicCreateEpoll(); assert(ep >= 0); QuicSocket socket = QuicCreateSocket(); assert(socket > 0); int res; struct sockaddr_in addr; addr.sin_family = AF_INET; addr.sin_port = htons(9700); addr.sin_addr.s_addr = inet_addr("127.0.0.1"); res = QuicBind(socket, (struct sockaddr*)&addr, sizeof(addr)); CHECK_RES(res, "bind"); struct epoll_event ev; ev.data.fd = socket; ev.events = EPOLLIN | EPOLLOUT; res = QuicEpollCtl(ep, EPOLL_CTL_ADD, socket, &ev); CHECK_RES(res, "epoll_ctl"); for (;;) { res = doLoop(ep, socket); if (res != 0) return res; } }
ev.events = EPOLLIN | EPOLLOUT;有问题，服务端不停出发out事件
Bbr算法
ndAlgorithmInterface::Create
(gdb) bt #0 0x0000000000489190 in net::SendAlgorithmInterface::Create(net::QuicClock const*, net::RttStats const*, net::QuicUnackedPacketMap const*, net::CongestionControlType, net::QuicRandom*, net::QuicConnectionStats*, unsigned long) () #1 0x0000000000444200 in net::QuicSentPacketManager::SetSendAlgorithm(net::CongestionControlType) () #2 0x000000000042f254 in net::QuicConnection::QuicConnection(unsigned long, net::QuicSocketAddress, net::QuicConnectionHelperInterface*, net::QuicAlarmFactory*, net::QuicPacketWriter*, bool, net::Perspective, std::vector<net::ParsedQuicVersion, std::allocator<net::ParsedQuicVersion> > const&) () #3 0x0000000000462814 in net::QuartcFactory::CreateQuicConnection(net::QuartcFactoryInterface::QuartcSessionConfig const&, net::Perspective) () #4 0x000000000046292c in net::QuartcFactory::CreateQuartcSession(net::QuartcFactoryInterface::QuartcSessionConfig const&) () #5 0x000000000041bbcc in posix_quic::QuicSocketEntry::NewQuicSocketEntry (isServer=isServer@entry=false, id=0, id@entry=18446744073709551615) at /root/posix_quic/src/socket_entry.cpp:47 #6 0x000000000040ea60 in posix_quic::QuicCreateSocket () at /root/posix_quic/src/quic_socket.cpp:10 #7 0x000000000040093c in main () at /root/posix_quic/test/client/src/client.cpp:121 (gdb)
CalculateCongestionWindow
(gdb) bt #0 0x00000000004c54f8 in net::BbrSender::CalculateCongestionWindow(unsigned long) () #1 0x00000000004c59bc in net::BbrSender::OnCongestionEvent(bool, unsigned long, net::QuicTime, std::vector<net::AckedPacket, std::allocator<net::AckedPacket> > const&, std::vector<net::LostPacket, std::allocator<net::LostPacket> > const&) () #2 0x0000000000443820 in net::QuicSentPacketManager::MaybeInvokeCongestionEvent(bool, unsigned long, net::QuicTime) () #3 0x0000000000446700 in net::QuicSentPacketManager::PostProcessAfterMarkingPacketHandled(net::QuicAckFrame const&, net::QuicTime, bool, unsigned long) () #4 0x00000000004468ec in net::QuicSentPacketManager::OnIncomingAck(net::QuicAckFrame const&, net::QuicTime) () #5 0x0000000000433530 in net::QuicConnection::OnAckFrame(net::QuicAckFrame const&) () #6 0x000000000043fd08 in net::QuicFramer::ProcessFrameData(net::QuicDataReader*, net::QuicPacketHeader const&) [clone .part.162] [clone .constprop.172] () #7 0x0000000000440258 in net::QuicFramer::ProcessDataPacket(net::QuicDataReader*, net::QuicPacketHeader*, net::QuicEncryptedPacket const&, char*, unsigned long) [clone .part.163] [clone .constprop.166] () #8 0x0000000000440684 in net::QuicFramer::ProcessPacket(net::QuicEncryptedPacket const&) () #9 0x0000000000434958 in net::QuicConnection::ProcessUdpPacket(net::QuicSocketAddress const&, net::QuicSocketAddress const&, net::QuicReceivedPacket const&) () #10 0x00000000004171f8 in posix_quic::QuicSocketEntry::ProcessUdpPacket (this=this@entry=0x83b080, self_address=..., peer_address=..., packet=...) at /root/posix_quic/src/socket_entry.cpp:415 #11 0x0000000000406ff0 in posix_quic::QuicEpollerEntry::Wait (this=this@entry=0x837950, events=0x461790 <net::QuicSocketAddressImpl::QuicSocketAddressImpl(sockaddr_storage const&)+48>, events@entry=0xffffffff9128, maxevents=65535, maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/epoller_entry.cpp:376 #12 0x00000000004157cc in posix_quic::QuicEpollWait (epfd=3, epfd@entry=0, events=events@entry=0xffffffff9128, maxevents=maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/quic_socket.cpp:494 #13 0x0000000000401dfc in doLoop (ep=0, ep@entry=3) at /root/posix_quic/test/client/src/client.cpp:37 #14 0x00000000004009c8 in main () at /root/posix_quic/test/client/src/client.cpp:143 (gdb)
(gdb) bt #0 0x00000000004c3bb0 in net::BbrSender::OnPacketSent(net::QuicTime, unsigned long, unsigned long, unsigned long, net::HasRetransmittableData) () #1 0x0000000000488d40 in net::PacingSender::OnPacketSent(net::QuicTime, unsigned long, unsigned long, unsigned long, net::HasRetransmittableData) () #2 0x00000000004454ac in net::QuicSentPacketManager::OnPacketSent(net::SerializedPacket*, unsigned long, net::QuicTime, net::TransmissionType, net::HasRetransmittableData) () #3 0x00000000004339d4 in net::QuicConnection::WritePacket(net::SerializedPacket*) [clone .part.191] [clone .constprop.259] () #4 0x0000000000434eac in net::QuicConnection::OnSerializedPacket(net::SerializedPacket*) () #5 0x00000000004ad984 in net::QuicPacketCreator::OnSerializedPacket() () #6 0x00000000004aeff0 in net::QuicPacketCreator::Flush() [clone .part.54] () #7 0x0000000000441ef8 in net::QuicPacketGenerator::ConsumeData(unsigned int, unsigned long, unsigned long, net::StreamSendingState) () #8 0x0000000000430034 in net::QuicConnection::SendStreamData(unsigned int, unsigned long, unsigned long, net::StreamSendingState) () #9 0x0000000000447de0 in net::QuicSession::WritevData(net::QuicStream*, unsigned int, unsigned long, unsigned long, net::StreamSendingState) () #10 0x0000000000447de0 in net::QuicSession::WritevData(net::QuicStream*, unsigned int, unsigned long, unsigned long, net::StreamSendingState) ()
BbrSender::OnPacketSent
(gdb) bt #0 0x00000000004c3bb0 in net::BbrSender::OnPacketSent(net::QuicTime, unsigned long, unsigned long, unsigned long, net::HasRetransmittableData) () #1 0x0000000000488d40 in net::PacingSender::OnPacketSent(net::QuicTime, unsigned long, unsigned long, unsigned long, net::HasRetransmittableData) () #2 0x00000000004454ac in net::QuicSentPacketManager::OnPacketSent(net::SerializedPacket*, unsigned long, net::QuicTime, net::TransmissionType, net::HasRetransmittableData) () #3 0x00000000004339d4 in net::QuicConnection::WritePacket(net::SerializedPacket*) [clone .part.191] [clone .constprop.259] () #4 0x0000000000434eac in net::QuicConnection::OnSerializedPacket(net::SerializedPacket*) () #5 0x00000000004ad984 in net::QuicPacketCreator::OnSerializedPacket() () #6 0x00000000004aeff0 in net::QuicPacketCreator::Flush() [clone .part.54] () #7 0x0000000000441ef8 in net::QuicPacketGenerator::ConsumeData(unsigned int, unsigned long, unsigned long, net::StreamSendingState) () #8 0x0000000000430034 in net::QuicConnection::SendStreamData(unsigned int, unsigned long, unsigned long, net::StreamSendingState) () #9 0x0000000000447de0 in net::QuicSession::WritevData(net::QuicStream*, unsigned int, unsigned long, unsigned long, net::StreamSendingState) () #10 0x0000000000447de0 in net::QuicSession::WritevData(net::QuicStream*, unsigned int, unsigned long, unsigned long, net::StreamSendingState) ()
如果是ACK包，则会对ack进行处理，并进行拥塞算法的运算。
if (!ProcessAckFrame(reader, frame_type, &frame)) {
return RaiseError(QUIC_INVALID_ACK_DATA);
}
if (!visitor_->OnAckFrame(frame)) {
QUIC_DVLOG(1) << ENDPOINT
<< "Visitor asked to stop further processing.";
// Returning true since there was no parsing error.
return true;
}(gdb) bt #0 posix_quic::QuicConnectionVisitor::OnAckFrame (this=0x83b238, frame=...) at /root/posix_quic/src/connection_visitor.cpp:109 #1 0x000000000043331c in net::QuicConnection::OnAckFrame(net::QuicAckFrame const&) () #2 0x000000000043fd08 in net::QuicFramer::ProcessFrameData(net::QuicDataReader*, net::QuicPacketHeader const&) [clone .part.162] [clone .constprop.172] () #3 0x0000000000440258 in net::QuicFramer::ProcessDataPacket(net::QuicDataReader*, net::QuicPacketHeader*, net::QuicEncryptedPacket const&, char*, unsigned long) [clone .part.163] [clone .constprop.166] () #4 0x0000000000440684 in net::QuicFramer::ProcessPacket(net::QuicEncryptedPacket const&) () #5 0x0000000000434958 in net::QuicConnection::ProcessUdpPacket(net::QuicSocketAddress const&, net::QuicSocketAddress const&, net::QuicReceivedPacket const&) () #6 0x00000000004171f8 in posix_quic::QuicSocketEntry::ProcessUdpPacket (this=this@entry=0x83b080, self_address=..., peer_address=..., packet=...) at /root/posix_quic/src/socket_entry.cpp:415 #7 0x0000000000406ff0 in posix_quic::QuicEpollerEntry::Wait (this=this@entry=0x837950, events=0x461790 <net::QuicSocketAddressImpl::QuicSocketAddressImpl(sockaddr_storage const&)+48>, events@entry=0xffffffff9128, maxevents=65535, maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/epoller_entry.cpp:376 #8 0x00000000004157cc in posix_quic::QuicEpollWait (epfd=3, epfd@entry=0, events=events@entry=0xffffffff9128, maxevents=maxevents@entry=1024, timeout=timeout@entry=6000) at /root/posix_quic/src/quic_socket.cpp:494 #9 0x0000000000401dfc in doLoop (ep=0, ep@entry=3) at /root/posix_quic/test/client/src/client.cpp:37 #10 0x00000000004009c8 in main () at /root/posix_quic/test/client/src/client.cpp:143 (gdb)
总结
void Put(UdpSocket udpSocket, QuicConnectionId connectionId, QuicSocket quicSocket, bool owner) { std::unique_lock<std::mutex> lock(mtx_); connections_[udpSocket][connectionId] = quicSocket; if (owner) { owners_[udpSocket] = quicSocket; } }
typedef int QuicSocket;
QuicSocket quicSocket = QuicSocketEntry::GetConnectionManager().Get(udpFd, connectionId);
EntryPtr entry = QuicSocketEntry::GetFdManager().Get(quicSocket);
// 2.根据ConnectionId来分派, 给指定的QuicSocket QuicSocket quicSocket = QuicSocketEntry::GetConnectionManager().Get(udpFd, connectionId); if (quicSocket == -1) { // 新连接 if (!owner) { QuicSocket ownerFd = QuicSocketEntry::GetConnectionManager().GetOwner(udpFd); if (ownerFd >= 0) { EntryPtr ownerBase = QuicSocketEntry::GetFdManager().Get(ownerFd); assert(ownerBase->Category() == EntryCategory::Socket); owner = std::dynamic_pointer_cast<QuicSocketEntry>(ownerBase); } } if (!owner) { // listen socket已关闭, 不再接受新连接 DebugPrint(dbg_epoll | dbg_read | dbg_accept, " -> recvfrom. Udp socket = %d, connectionId = %lu is a new Connection. Not found owner, so ignore it!", udpFd, connectionId); continue; } DebugPrint(dbg_epoll | dbg_read | dbg_accept, " -> recvfrom. Udp socket = %d, connectionId = %lu is a new Connection.", udpFd, connectionId); QuicSocketEntryPtr socket = QuicSocketEntry::NewQuicSocketEntry(true, connectionId); socket->OnSyn(owner, peerAddress); socket->ProcessUdpPacket(selfAddress, peerAddress, QuicReceivedPacket(&udpRecvBuf_[0], bytes, QuicClockImpl::getInstance().Now())); continue; } DebugPrint(dbg_epoll | dbg_read, " -> recvfrom. Udp socket = %d, connectionId = %lu is exists.", udpFd, connectionId); EntryPtr entry = QuicSocketEntry::GetFdManager().Get(quicSocket); assert(entry->Category() == EntryCategory::Socket); QuicSocketEntry* socket = (QuicSocketEntry*)entry.get(); //share_ptr的get方法 socket->ProcessUdpPacket(selfAddress, peerAddress, QuicReceivedPacket(&udpRecvBuf_[0], bytes, QuicClockImpl::getInstance().Now()));
替换x86 rdtsc汇编指令
https://support.huaweicloud.com/codeprtr-kunpenggrf/kunpengtaishanporting_12_0030.html
QUIC协议和HTTP3.0技术研究
https://www.cnblogs.com/zhangxu1998cn/p/14342545.html
(gdb) bt#0 0x0000000000489190 in net::SendAlgorithmInterface::Create(net::QuicClock const*, net::RttStats const*, net::QuicUnackedPacketMap const*, net::CongestionControlType, net::QuicRandom*, net::QuicConnectionStats*, unsigned long) ()#1 0x0000000000444200 in net::QuicSentPacketManager::SetSendAlgorithm(net::CongestionControlType) ()#2 0x000000000042f254 in net::QuicConnection::QuicConnection(unsigned long, net::QuicSocketAddress, net::QuicConnectionHelperInterface*, net::QuicAlarmFactory*, net::QuicPacketWriter*, bool, net::Perspective, std::vector<net::ParsedQuicVersion, std::allocator<net::ParsedQuicVersion> > const&) ()#3 0x0000000000462814 in net::QuartcFactory::CreateQuicConnection(net::QuartcFactoryInterface::QuartcSessionConfig const&, net::Perspective) ()#4 0x000000000046292c in net::QuartcFactory::CreateQuartcSession(net::QuartcFactoryInterface::QuartcSessionConfig const&) ()#5 0x000000000041bbcc in posix_quic::QuicSocketEntry::NewQuicSocketEntry (isServer=isServer@entry=false, id=0, id@entry=18446744073709551615) at /root/posix_quic/src/socket_entry.cpp:47#6 0x000000000040ea60 in posix_quic::QuicCreateSocket () at /root/posix_quic/src/quic_socket.cpp:10#7 0x000000000040093c in main () at /root/posix_quic/test/client/src/client.cpp:121(gdb)