diff --git a/src/common/steamnetworkingsockets_messages.proto b/src/common/steamnetworkingsockets_messages.proto index c226bb2..9f9452f 100644 --- a/src/common/steamnetworkingsockets_messages.proto +++ b/src/common/steamnetworkingsockets_messages.proto @@ -101,6 +101,9 @@ message CMsgSteamDatagramLinkLifetimeStats optional uint64 packets_recv_out_of_order = 9; optional uint64 packets_recv_duplicate = 10; optional uint64 packets_recv_lurch = 11; + repeated uint64 multipath_packets_recv_sequenced = 12; // If multipath send or receive was used, then the samples for how many we received on each path + repeated uint64 multipath_packets_recv_later = 13; // How often we received a sequenced packet only this path, but another path one the race and so we discarded this one + optional uint32 multipath_send_enabled = 14; /// Histogram of connection quality. Here we count up the number /// of connection quality measurement intervals (about 5 seconds) diff --git a/src/steamnetworkingsockets/clientlib/csteamnetworkingsockets.cpp b/src/steamnetworkingsockets/clientlib/csteamnetworkingsockets.cpp index be68917..816a11e 100644 --- a/src/steamnetworkingsockets/clientlib/csteamnetworkingsockets.cpp +++ b/src/steamnetworkingsockets/clientlib/csteamnetworkingsockets.cpp @@ -84,6 +84,9 @@ DEFINE_CONNECTON_DEFAULT_CONFIGVAL( int32, MTU_PacketSize, 1300, k_cbSteamNetwor DEFINE_CONNECTON_DEFAULT_CONFIGVAL( int32, Unencrypted, 0, 0, 3 ); DEFINE_CONNECTON_DEFAULT_CONFIGVAL( int32, SymmetricConnect, 0, 0, 1 ); DEFINE_CONNECTON_DEFAULT_CONFIGVAL( int32, LocalVirtualPort, -1, -1, INT32_MAX ); +#ifdef STEAMNETWORKINGSOCKETS_ENABLE_DUALWIFI +DEFINE_CONNECTON_DEFAULT_CONFIGVAL( int32, DualWifi_Enable, 1, 0, k_nDualWifiEnable_MAX ); +#endif DEFINE_CONNECTON_DEFAULT_CONFIGVAL( int32, LogLevel_AckRTT, k_ESteamNetworkingSocketsDebugOutputType_Warning, k_ESteamNetworkingSocketsDebugOutputType_Error, k_ESteamNetworkingSocketsDebugOutputType_Everything ); DEFINE_CONNECTON_DEFAULT_CONFIGVAL( int32, LogLevel_PacketDecode, k_ESteamNetworkingSocketsDebugOutputType_Warning, k_ESteamNetworkingSocketsDebugOutputType_Error, k_ESteamNetworkingSocketsDebugOutputType_Everything ); DEFINE_CONNECTON_DEFAULT_CONFIGVAL( int32, LogLevel_Message, k_ESteamNetworkingSocketsDebugOutputType_Warning, k_ESteamNetworkingSocketsDebugOutputType_Error, k_ESteamNetworkingSocketsDebugOutputType_Everything ); diff --git a/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_connections.cpp b/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_connections.cpp index 021c007..28704df 100644 --- a/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_connections.cpp +++ b/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_connections.cpp @@ -1060,7 +1060,8 @@ void CSteamNetworkConnectionBase::RecvNonDataSequencedPacket( int64 nPktNum, Ste SNP_RecordReceivedPktNum( nPktNum, usecNow, false ); // Update general sequence number/stats tracker for the end-to-end flow. - m_statsEndToEnd.TrackProcessSequencedPacket( nPktNum, usecNow, 0 ); + int idxMultiPath = 0; // Assume for now + m_statsEndToEnd.TrackProcessSequencedPacket( nPktNum, usecNow, 0, idxMultiPath ); } bool CSteamNetworkConnectionBase::BThinkCryptoReady( SteamNetworkingMicroseconds usecNow ) @@ -2062,7 +2063,7 @@ bool CSteamNetworkConnectionBase::DecryptDataChunk( uint16 nWireSeqNum, int cbPa AssertMsg1( m_statsEndToEnd.m_nMaxRecvPktNum > 0 || m_statsEndToEnd.m_nPeerProtocolVersion < 10, "[%s] packet number not properly initialized!", GetDescription() ); // Get the full end-to-end packet number, check if we should process it - ctx.m_nPktNum = m_statsEndToEnd.ExpandWirePacketNumberAndCheck( nWireSeqNum ); + ctx.m_nPktNum = m_statsEndToEnd.ExpandWirePacketNumberAndCheck( nWireSeqNum, ctx.m_idxMultiPath ); if ( ctx.m_nPktNum <= 0 ) { @@ -2118,8 +2119,8 @@ bool CSteamNetworkConnectionBase::DecryptDataChunk( uint16 nWireSeqNum, int cbPa // The assumption is that we either have a bug or some weird thing, // or that somebody is spoofing / tampering. If it's the latter // we don't want to magnify the impact of their efforts - SpewWarningRateLimited( ctx.m_usecNow, "[%s] Packet %lld (0x%x) decrypt failed (tampering/spoofing/bug)!", - GetDescription(), (long long)ctx.m_nPktNum, (unsigned)nWireSeqNum ); + SpewWarningRateLimited( ctx.m_usecNow, "[%s] Packet %lld (0x%x) decrypt failed (tampering/spoofing/bug)! mpath%d", + GetDescription(), (long long)ctx.m_nPktNum, (unsigned)nWireSeqNum, ctx.m_idxMultiPath ); // Update raw packet counters numbers, but do not update any logical state such as reply timeouts, etc m_statsEndToEnd.m_recv.ProcessPacket( cbPacketSize ); @@ -3985,8 +3986,9 @@ void CSteamNetworkConnectionPipe::FakeRecvStats( SteamNetworkingMicroseconds use // And the peer receiving it immediately. And assume every packet represents // a ping measurement. - int64 nPktNum = m_statsEndToEnd.ExpandWirePacketNumberAndCheck( nWirePktNum ); - m_statsEndToEnd.TrackProcessSequencedPacket( nPktNum, usecNow, -1 ); + int idxMultiPath = 0; + int64 nPktNum = m_statsEndToEnd.ExpandWirePacketNumberAndCheck( nWirePktNum, idxMultiPath ); + m_statsEndToEnd.TrackProcessSequencedPacket( nPktNum, usecNow, -1, idxMultiPath ); m_statsEndToEnd.TrackRecvPacket( cbPktSize, usecNow ); m_statsEndToEnd.m_ping.ReceivedPing( 0, usecNow ); } diff --git a/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_connections.h b/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_connections.h index 96eec7b..ac82aaf 100644 --- a/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_connections.h +++ b/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_connections.h @@ -114,6 +114,9 @@ struct RecvPacketContext_t /// Jitter measurement, if present //int m_usecTimeSinceLast; + /// For dual path, is this the primary or secondary path? + int m_idxMultiPath = 0; + // // Output of DecryptDataChunk // diff --git a/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_lowlevel.cpp b/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_lowlevel.cpp index badaddc..4bc0d30 100644 --- a/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_lowlevel.cpp +++ b/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_lowlevel.cpp @@ -970,6 +970,10 @@ public: nullptr // lpCompletionRoutine ); bool bResult = ( r == 0 ); + if ( !bResult ) + { + SpewWarning( "WSASendTo %s failed, returned %d, last error=0x%x\n", CUtlNetAdrRender( adrTo ).String(), r, GetLastSocketError() ); + } #else msghdr msg; msg.msg_name = (sockaddr *)&destAddress; @@ -1052,6 +1056,11 @@ public: } } + #ifdef STEAMNETWORKINGSOCKETS_ENABLE_DUALWIFI + CRawUDPSocketImpl *m_pDualWifiPartner = nullptr; + virtual IRawUDPSocket *GetDualWifiSecondarySocket( int nEnableSetting ) override; + #endif + private: void InternalAddToCleanupQueue(); @@ -1369,6 +1378,31 @@ void CRawUDPSocketImpl::Close() // Mark the callback as detached, and put us in the queue for cleanup when it's safe. InternalAddToCleanupQueue(); + // Check for Dual Wifi + #ifdef STEAMNETWORKINGSOCKETS_ENABLE_DUALWIFI + if ( m_pDualWifiPartner ) + { + Assert( m_pDualWifiPartner->m_pDualWifiPartner == this ); + + if ( m_eDualWifiStatus == k_EDualWifi_Primary ) + { + Assert( m_pDualWifiPartner->m_eDualWifiStatus == k_EDualWifi_Secondary ); + m_pDualWifiPartner->InternalAddToCleanupQueue(); + } + else + { + // People shouldn't do this, but if they do, let's not crash + AssertMsg( false, "Closed secondary dual Wifi socket directly?" ); + } + + m_pDualWifiPartner->m_eDualWifiStatus = k_EDualWifi_Done; + m_pDualWifiPartner->m_pDualWifiPartner = nullptr; + + m_eDualWifiStatus = k_EDualWifi_Done; + m_pDualWifiPartner = nullptr; + } + #endif + // Make sure we don't delay doing this too long if ( s_bManualPollMode || ( s_pThreadSteamDatagram && s_pThreadSteamDatagram->get_id() != std::this_thread::get_id() ) ) { @@ -1382,7 +1416,7 @@ void CRawUDPSocketImpl::Close() } } -static SOCKET OpenUDPSocketBoundToSockAddr( const void *sockaddr, size_t len, SteamDatagramErrMsg &errMsg, int *pnIPv6AddressFamilies ) +static SOCKET OpenUDPSocketBoundToSockAddr( const void *sockaddr, size_t len, SteamDatagramErrMsg &errMsg, int *pnIPv6AddressFamilies, int nBindInterface = -1 ) { unsigned int opt; @@ -1468,7 +1502,46 @@ static SOCKET OpenUDPSocketBoundToSockAddr( const void *sockaddr, size_t len, St } } - // Bind it to specific desired port and/or interfaces + // Bind to particular interface + if ( nBindInterface >= 0 ) + { + #ifdef _WIN32 + Assert( nBindInterface != 0 ); // 0 is reserved, invalid value in Windows. + + // Bind to particular interface for IPv4 + if ( inaddr->sin_family == AF_INET || ( pnIPv6AddressFamilies && ( *pnIPv6AddressFamilies & k_nAddressFamily_IPv4 ) ) ) + { + // WARNING: interface index should be in network byte order for IPPROTO_IP + const DWORD value = htonl(nBindInterface); + const int length = sizeof(value); + const auto error = setsockopt( sock , IPPROTO_IP, IP_UNICAST_IF, reinterpret_cast(&value), length); + if (ERROR_SUCCESS != error) + { + V_sprintf_safe( errMsg, "sockopt(IP_PROTO_IP, IP_UNICAST_IF, %d) failed with error code 0x%08X.", nBindInterface, GetLastSocketError() ); + closesocket( sock ); + return INVALID_SOCKET; + } + SpewVerbose( "sockopt(IP_PROTO_IP, IP_UNICAST_IF, %d) OK\n", nBindInterface ); + } + + // Bind to particular interface for IPv6 + if ( inaddr->sin_family == AF_INET6 || ( pnIPv6AddressFamilies && ( *pnIPv6AddressFamilies & k_nAddressFamily_IPv6 ) ) ) + { + // WARNING: interface index should be in host byte order for IPPROTO_IPV6 + auto length = static_cast(sizeof(nBindInterface)); + const auto error = setsockopt( sock, IPPROTO_IPV6, IPV6_UNICAST_IF, reinterpret_cast(&nBindInterface), length); + if (ERROR_SUCCESS != error) + { + V_sprintf_safe( errMsg, "sockopt(IPPROTO_IPV6, IPV6_UNICAST_IF, %d) failed with error code 0x%08X.", nBindInterface, GetLastSocketError() ); + closesocket( sock ); + return INVALID_SOCKET; + } + SpewVerbose( "sockopt(IPPROTO_IPV6, IPV6_UNICAST_IF, %d) OK\n", nBindInterface ); + } + #endif + } + + // Bind it to specific desired local port/IP if ( bind( sock, (struct sockaddr *)sockaddr, (socklen_t)len ) == -1 ) { V_sprintf_safe( errMsg, "Failed to bind socket. Error code 0x%08X.", GetLastSocketError() ); @@ -1480,7 +1553,7 @@ static SOCKET OpenUDPSocketBoundToSockAddr( const void *sockaddr, size_t len, St return sock; } -static CRawUDPSocketImpl *OpenRawUDPSocketInternal( CRecvPacketCallback callback, SteamDatagramErrMsg &errMsg, const SteamNetworkingIPAddr *pAddrLocal, int *pnAddressFamilies ) +static CRawUDPSocketImpl *OpenRawUDPSocketInternal( CRecvPacketCallback callback, SteamDatagramErrMsg &errMsg, const SteamNetworkingIPAddr *pAddrLocal, int *pnAddressFamilies, int nBindInterface = -1 ) { // Creating a socket *should* be fast, but sometimes the OS might need to do some work. // We shouldn't do this too often, give it a little extra time. @@ -1552,7 +1625,7 @@ static CRawUDPSocketImpl *OpenRawUDPSocketInternal( CRecvPacketCallback callback // Try to get socket int nIPv6AddressFamilies = nAddressFamilies; - sock = OpenUDPSocketBoundToSockAddr( &address6, sizeof(address6), errMsg, &nIPv6AddressFamilies ); + sock = OpenUDPSocketBoundToSockAddr( &address6, sizeof(address6), errMsg, &nIPv6AddressFamilies, nBindInterface ); if ( sock == INVALID_SOCKET ) { @@ -1580,7 +1653,7 @@ static CRawUDPSocketImpl *OpenRawUDPSocketInternal( CRecvPacketCallback callback address4.sin_port = BigWord( addrLocal.m_port ); // Try to get socket - sock = OpenUDPSocketBoundToSockAddr( &address4, sizeof(address4), errMsg, nullptr ); + sock = OpenUDPSocketBoundToSockAddr( &address4, sizeof(address4), errMsg, nullptr, nBindInterface ); // If we failed, well, we have no other options left to try. if ( sock == INVALID_SOCKET ) @@ -2014,6 +2087,398 @@ static void ProcessDeferredOperations() ProcessPendingDestroyClosedRawUDPSockets(); } +///////////////////////////////////////////////////////////////////////////// +// +// Dual Wifi band support +// +///////////////////////////////////////////////////////////////////////////// + +#ifdef STEAMNETWORKINGSOCKETS_ENABLE_DUALWIFI + +#include +static int s_ifaceDualWifiSecondary = -1; // -1 means we haven't tried yet. Any other negative value means, we tried and failed +static HANDLE wlanHandle = INVALID_HANDLE_VALUE; + +static HMODULE hModuleWlanAPI = NULL; + +static +DWORD +(WINAPI *pWlanOpenHandle)( + DWORD dwClientVersion, + PVOID pReserved, + PDWORD pdwNegotiatedVersion, + PHANDLE phClientHandle +); + +static +DWORD +(WINAPI *pWlanCloseHandle)( + HANDLE hClientHandle, + PVOID pReserved +); + +static +DWORD +(WINAPI *pWlanEnumInterfaces)( + HANDLE hClientHandle, + PVOID pReserved, + PWLAN_INTERFACE_INFO_LIST *ppInterfaceList +); + +static +DWORD +( WINAPI *pWlanSetInterface)( + HANDLE hClientHandle, + CONST GUID *pInterfaceGuid, + WLAN_INTF_OPCODE OpCode, + DWORD dwDataSize, + CONST PVOID pData, + PVOID pReserved +); + +static +DWORD +( WINAPI *pWlanQueryInterface)( + HANDLE hClientHandle, + CONST GUID *pInterfaceGuid, + WLAN_INTF_OPCODE OpCode, + PVOID pReserved, + PDWORD pdwDataSize, + PVOID *ppData, + PWLAN_OPCODE_VALUE_TYPE pWlanOpcodeValueType +); + + + +void DualWifiShutdown() +{ + if ( wlanHandle != INVALID_HANDLE_VALUE ) + { + (*pWlanCloseHandle)( wlanHandle, nullptr ); + wlanHandle = INVALID_HANDLE_VALUE; + } + s_ifaceDualWifiSecondary = -1; +} + +// !KLUDGE! Pasted from an early version of wlan.h. +namespace wlan_new +{ +#ifdef __midl +// use the 4-byte enum +typedef [v1_enum] enum _WLAN_INTF_OPCODE { +#else +typedef enum _WLAN_INTF_OPCODE { +#endif + wlan_intf_opcode_autoconf_start = 0x000000000, + wlan_intf_opcode_autoconf_enabled, + wlan_intf_opcode_background_scan_enabled, + wlan_intf_opcode_media_streaming_mode, + wlan_intf_opcode_radio_state, + wlan_intf_opcode_bss_type, + wlan_intf_opcode_interface_state, + wlan_intf_opcode_current_connection, + wlan_intf_opcode_channel_number, + wlan_intf_opcode_supported_infrastructure_auth_cipher_pairs, + wlan_intf_opcode_supported_adhoc_auth_cipher_pairs, + wlan_intf_opcode_supported_country_or_region_string_list, + wlan_intf_opcode_current_operation_mode, + wlan_intf_opcode_supported_safe_mode, + wlan_intf_opcode_certified_safe_mode, + wlan_intf_opcode_hosted_network_capable, + wlan_intf_opcode_management_frame_protection_capable, + wlan_intf_opcode_secondary_sta_interfaces, + wlan_intf_opcode_secondary_sta_synchronized_connections, + wlan_intf_opcode_autoconf_end = 0x0fffffff, + wlan_intf_opcode_msm_start = 0x10000100, + wlan_intf_opcode_statistics, + wlan_intf_opcode_rssi, + wlan_intf_opcode_msm_end = 0x1fffffff, + wlan_intf_opcode_security_start = 0x20010000, + wlan_intf_opcode_security_end = 0x2fffffff, + wlan_intf_opcode_ihv_start = 0x30000000, + wlan_intf_opcode_ihv_end = 0x3fffffff +} WLAN_INTF_OPCODE, *PWLAN_INTF_OPCODE; +} +const WLAN_INTF_OPCODE wlan_intf_opcode_secondary_sta_synchronized_connections = (WLAN_INTF_OPCODE)wlan_new::wlan_intf_opcode_secondary_sta_synchronized_connections; +const WLAN_INTF_OPCODE wlan_intf_opcode_secondary_sta_interfaces = (WLAN_INTF_OPCODE)wlan_new::wlan_intf_opcode_secondary_sta_interfaces; + +static void DualWifiInitFailed( const char *fmt, ... ) +{ + va_list ap; + va_start( ap, fmt ); + char buf[ 512 ]; + V_vsprintf_safe( buf, fmt, ap ); + SpewMsg( "DualWifi not detected. We won't try again. %s\n", buf ); + DualWifiShutdown(); + s_ifaceDualWifiSecondary = -2; // but remember that we failed +} + +static int ConvertInterfaceGuidToIndex(const GUID& interfaceGuid) +{ + // + // These functions were added with Vista, so load dynamically + // in case + // + + typedef + NETIO_STATUS + (NETIOAPI_API_*FnConvertInterfaceGuidToLuid)( + _In_ CONST GUID *InterfaceGuid, + _Out_ PNET_LUID InterfaceLuid + ); + typedef + NETIO_STATUS + (NETIOAPI_API_*FnConvertInterfaceLuidToIndex)( + _In_ CONST NET_LUID *InterfaceLuid, + _Out_ PNET_IFINDEX InterfaceIndex + ); + + static HMODULE hModule = LoadLibraryA( "Iphlpapi.dll" ); + static FnConvertInterfaceGuidToLuid pConvertInterfaceGuidToLuid = hModule ? (FnConvertInterfaceGuidToLuid)GetProcAddress( hModule, "ConvertInterfaceGuidToLuid" ) : nullptr; + static FnConvertInterfaceLuidToIndex pConvertInterfaceLuidToIndex = hModule ? (FnConvertInterfaceLuidToIndex)GetProcAddress( hModule, "ConvertInterfaceLuidToIndex" ) : nullptr;; + if ( !pConvertInterfaceGuidToLuid || !pConvertInterfaceLuidToIndex ) + { + AssertMsg( false, "How did I get here?" ); + return -1; + } + + NET_LUID interfaceLuid{}; + auto error = (pConvertInterfaceGuidToLuid)(&interfaceGuid, &interfaceLuid); + if ( ERROR_SUCCESS != error ) + { + AssertMsg( false, "ConvertInterfaceGuidToLuid failed 0x%x", error ); + return -1; + } + + NET_IFINDEX interfaceIndex = 0; + error = (*pConvertInterfaceLuidToIndex)(&interfaceLuid, &interfaceIndex); + if ( ERROR_SUCCESS != error ) + { + AssertMsg( false, "ConvertInterfaceLuidToIndex failed 0x%x", error ); + return -1; + } + + return static_cast(interfaceIndex); +} + +class RenderGUID +{ + char buf[64]; +public: + RenderGUID( const GUID &guid ) + { + // https://stackoverflow.com/a/18114061/8004137 + V_sprintf_safe( buf, "{%08lX-%04hX-%04hX-%02hhX%02hhX-%02hhX%02hhX%02hhX%02hhX%02hhX%02hhX}", + guid.Data1, guid.Data2, guid.Data3, + guid.Data4[0], guid.Data4[1], guid.Data4[2], guid.Data4[3], + guid.Data4[4], guid.Data4[5], guid.Data4[6], guid.Data4[7]); + } + const char *c_str() const { return buf; } +}; + +template +bool MyGetProcAddress( F& fn, HMODULE hm, const char *pszName ) +{ + if ( hm == NULL ) + fn = nullptr; + else + fn = (F)GetProcAddress( hm, pszName ); + return fn != nullptr; +} + +static int GetDualWifiSecondaryInterfaceIndex( int nSimulateMode ) +{ + // First time attempt? + // FIXME - it's not clear to me when I should retry + if ( s_ifaceDualWifiSecondary != -1 ) + return s_ifaceDualWifiSecondary; + + // Dynamically load wlanapi.dll the first time. + if ( hModuleWlanAPI == NULL ) + { + hModuleWlanAPI = LoadLibraryA( "wlanapi.dll" ); + if ( + !MyGetProcAddress( pWlanOpenHandle, hModuleWlanAPI, "WlanOpenHandle" ) + || !MyGetProcAddress( pWlanCloseHandle, hModuleWlanAPI, "WlanCloseHandle" ) + || !MyGetProcAddress( pWlanEnumInterfaces, hModuleWlanAPI, "WlanEnumInterfaces" ) + || !MyGetProcAddress( pWlanSetInterface, hModuleWlanAPI, "WlanSetInterface" ) + || !MyGetProcAddress( pWlanQueryInterface, hModuleWlanAPI, "WlanQueryInterface" ) + ) { + DualWifiInitFailed( "Failed to load wlanAPI.DLL" ); + return -1; + } + } + + // First time we need to open Wlan session + if ( wlanHandle == INVALID_HANDLE_VALUE ) + { + DWORD clientVersion = 2; // Vista+ APIs + DWORD curVersion = 0; + DWORD error = (*pWlanOpenHandle)(clientVersion, nullptr, &curVersion, &wlanHandle ); + if ( ERROR_SUCCESS != error || wlanHandle == INVALID_HANDLE_VALUE ) + { + DualWifiInitFailed( "WlanOpenHandle failed 0x%x.", error ); + return -1; + } + } + + PWLAN_INTERFACE_INFO_LIST primaryInterfaceList = nullptr; + DWORD error = (*pWlanEnumInterfaces)(wlanHandle, nullptr, &primaryInterfaceList); + if ( ERROR_SUCCESS != error || !primaryInterfaceList ) + { + DualWifiInitFailed( "WlanEnumInterfaces failed 0x%x.", error ); + return -1; + } + + if ( nSimulateMode == k_nDualWifiEnable_DoNotEnumerate ) + { + DualWifiInitFailed( "Not really checking for capable adapters as per DualWifi_Enable=%d", nSimulateMode ); + s_ifaceDualWifiSecondary = -1; + return -1; + } + + // Look for the first Wireless interface that we can enable DualSTA for + for ( DWORD idxPrimary = 0 ; idxPrimary < primaryInterfaceList->dwNumberOfItems ; ++idxPrimary ) + { + const GUID primaryInterfaceGuid = primaryInterfaceList->InterfaceInfo[idxPrimary].InterfaceGuid; + + // Get adapter name in UTF8 + char szInterfaceDescription[ 256 ]; + memset( szInterfaceDescription, 0, sizeof(szInterfaceDescription) ); + WideCharToMultiByte( + CP_UTF8, // codepage + 0, // flags + primaryInterfaceList->InterfaceInfo[idxPrimary].strInterfaceDescription, -1, // input string and length + szInterfaceDescription, sizeof(szInterfaceDescription)-1, // output buffer and SIZE in bytes + nullptr, nullptr // no default char, and we don't care if it was used + ); + + // Try to enable the feature on this adapter. This is where most adapters should fail. + BOOL enable = TRUE; + error = (*pWlanSetInterface)( + wlanHandle, &primaryInterfaceGuid, wlan_intf_opcode_secondary_sta_synchronized_connections, sizeof(BOOL), static_cast(&enable), nullptr); + if ( ERROR_SUCCESS != error ) + { + SpewVerbose( "Dual Wifi support not detected on adapter '%s' (wlan_intf_opcode_secondary_sta_synchronized_connections returned 0x%x)\n", szInterfaceDescription, error ); + continue; + } + + // Feature is detected! + SpewMsg( "Dual Wifi support enabled successfully on adapter '%s'\n", szInterfaceDescription ); + + PWLAN_INTERFACE_INFO_LIST secondaryInterfaceList = nullptr; + DWORD dataSize = 0; + error = (*pWlanQueryInterface)( + wlanHandle, + &primaryInterfaceGuid, + wlan_intf_opcode_secondary_sta_interfaces, + nullptr, + &dataSize, + reinterpret_cast(&secondaryInterfaceList), + nullptr); + if ( ERROR_SUCCESS != error || !secondaryInterfaceList ) + { + AssertMsg( false, "wlan_intf_opcode_secondary_sta_synchronized_connections succeeded, but wlan_intf_opcode_secondary_sta_interfaces failed 0x%x?", error ); + continue; + } + + for ( DWORD idxSecondary = 0 ; idxSecondary < secondaryInterfaceList->dwNumberOfItems ; ++idxSecondary ) + { + s_ifaceDualWifiSecondary = ConvertInterfaceGuidToIndex( secondaryInterfaceList->InterfaceInfo[ idxSecondary ].InterfaceGuid ); + if ( s_ifaceDualWifiSecondary >= 0 ) + { + SpewMsg( "Primary DualSTA interfaces %s matched to secondary interface %s, index %d\n", + RenderGUID( primaryInterfaceGuid ).c_str(), RenderGUID( secondaryInterfaceList->InterfaceInfo[ idxSecondary ].InterfaceGuid ).c_str(), + s_ifaceDualWifiSecondary ); + return s_ifaceDualWifiSecondary; + } + } + + AssertMsg( false, "Could not find secondary wifi adapter, even though wlan_intf_opcode_secondary_sta_synchronized_connections succeeded" ); + } + + // Failed. This should be common + DualWifiInitFailed( "Didn't find any Dual-Wifi-capable Wifi adapters" ); + return -1; +} + +IRawUDPSocket *CRawUDPSocketImpl::GetDualWifiSecondarySocket( int nEnableSetting ) +{ + SteamDatagramErrMsg errMsg; + + switch ( m_eDualWifiStatus ) + { + case k_EDualWifi_NotAttempted: + { + Assert( m_pDualWifiPartner == nullptr ); + + // Locate the secondary interface, if any. + int ifaceIndex = -1; + if ( nEnableSetting == k_nDualWifiEnable_ForceSimulate ) + { + SpewMsg( "Not actually checking for Dual-wifi support, just creating simulating support, as per DualWifi_Enable=%d\n", nEnableSetting ); + } + else + { + ifaceIndex = GetDualWifiSecondaryInterfaceIndex( nEnableSetting ); + if ( ifaceIndex < 0 && nEnableSetting == k_nDualWifiEnable_Enable ) + { + // not found, and we don't want to simulate support. + // This will be common! Don't retry. + m_eDualWifiStatus = k_EDualWifi_Done; + break; + } + } + + if ( nEnableSetting == k_nDualWifiEnable_DoNotBind ) + { + SpewMsg( "Not actually creating secondary socket as per DualWifi_Enable=%d\n", nEnableSetting ); + m_eDualWifiStatus = k_EDualWifi_Done; + break; + } + + // Create the second socket, binding it the interface as appropriate + int nTempAddressFamiles = m_nAddressFamilies; + m_pDualWifiPartner = OpenRawUDPSocketInternal( m_callback, errMsg, nullptr, &nTempAddressFamiles, ifaceIndex ); + if ( !m_pDualWifiPartner ) + { + SpewWarning( "Failed to create dual Wifi secondary socket. %s\n", errMsg ); + m_eDualWifiStatus = k_EDualWifi_Done; // Don't retry + break; + } + + m_eDualWifiStatus = k_EDualWifi_Primary; + m_pDualWifiPartner->m_eDualWifiStatus = k_EDualWifi_Secondary; + m_pDualWifiPartner->m_pDualWifiPartner = this; + + SpewMsg( "Created %sdual Wifi secondary socket OK. Primary local address is %s, secondary is %s\n", + ifaceIndex < 0 ? "simulated " : "", + SteamNetworkingIPAddrRender( m_boundAddr ).c_str(), SteamNetworkingIPAddrRender( m_pDualWifiPartner->m_boundAddr ).c_str() ); + return m_pDualWifiPartner; + } + + case k_EDualWifi_Primary: + Assert( m_pDualWifiPartner ); + return m_pDualWifiPartner; + + default: + case k_EDualWifi_Secondary: + Assert( false ); + break; + + case k_EDualWifi_Done: + break; + } + + return nullptr; +} + +#else + +void DualWifiShutdown() {} + +#endif + ///////////////////////////////////////////////////////////////////////////// // // Service thread @@ -2748,6 +3213,9 @@ void SteamNetworkingSocketsLowLevelDecRef() // Shutdown event tracing ETW_Kill(); + // Shutdown Dual wifi support + DualWifiShutdown(); + // Nuke sockets and COM #ifdef _WIN32 ::timeEndPeriod( 1 ); diff --git a/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_lowlevel.h b/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_lowlevel.h index 86f8f8e..1eb8aea 100644 --- a/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_lowlevel.h +++ b/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_lowlevel.h @@ -121,10 +121,33 @@ public: /// get any further callbacks. virtual void Close() = 0; + /// Check if Dual Wifi support is available, and if so, return the secondary socket. + /// The first time this is called, we will check for support and attempt to open a socket. + /// Thereafter, we will merely return the result of the first attempt + enum EDualWifiStatus + { + k_EDualWifi_NotAttempted, + k_EDualWifi_Done, // Failed or closed; don't try again + k_EDualWifi_Primary, // We're the primary. m_pDualWifiPartner is the secondary + k_EDualWifi_Secondary, // We're the secondary. m_pDualWifiPartner is the primary + }; + #ifdef STEAMNETWORKINGSOCKETS_ENABLE_DUALWIFI + virtual IRawUDPSocket *GetDualWifiSecondarySocket( int nEnableSetting ) = 0; + inline bool IsDualWifiSecondary() const { return m_eDualWifiStatus == k_EDualWifi_Secondary; } + #else + inline IRawUDPSocket *GetDualWifiSecondarySocket( int nEnableSetting ) { return nullptr; } + inline bool IsDualWifiSecondary() const { return false; } + #endif + /// The local address we ended up binding to SteamNetworkingIPAddr m_boundAddr; protected: + + #ifdef STEAMNETWORKINGSOCKETS_ENABLE_DUALWIFI + EDualWifiStatus m_eDualWifiStatus = k_EDualWifi_NotAttempted; + #endif + IRawUDPSocket(); virtual ~IRawUDPSocket(); }; diff --git a/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_snp.cpp b/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_snp.cpp index 0657bb6..60d5f1f 100644 --- a/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_snp.cpp +++ b/src/steamnetworkingsockets/clientlib/steamnetworkingsockets_snp.cpp @@ -1215,7 +1215,7 @@ bool CSteamNetworkConnectionBase::ProcessPlainTextDataChunk( int usecTimeSinceLa // // Also, note that order of operations is important. This call must // happen after the SNP_RecordReceivedPktNum call above - m_statsEndToEnd.TrackProcessSequencedPacket( nPktNum, usecNow, usecTimeSinceLast ); + m_statsEndToEnd.TrackProcessSequencedPacket( nPktNum, usecNow, usecTimeSinceLast, ctx.m_idxMultiPath ); // Packet can be processed further return true; diff --git a/src/steamnetworkingsockets/steamnetworking_stats.h b/src/steamnetworkingsockets/steamnetworking_stats.h index 7541b6a..d11e3ec 100644 --- a/src/steamnetworkingsockets/steamnetworking_stats.h +++ b/src/steamnetworkingsockets/steamnetworking_stats.h @@ -191,6 +191,11 @@ struct SteamDatagramLinkLifetimeStats int64 m_nPktsRecvDuplicate; int64 m_nPktsRecvSequenceNumberLurch; + // Multipath stats + bool m_bMultiPathSendEnabled; + int64 m_nMultiPathRecvSeq[2]; + int64 m_nMultiPathRecvLater[2]; + // SNP message counters int64 m_nMessagesSentReliable; int64 m_nMessagesSentUnreliable; diff --git a/src/steamnetworkingsockets/steamnetworking_statsutils.h b/src/steamnetworkingsockets/steamnetworking_statsutils.h index ca5f7f3..02b6c6e 100644 --- a/src/steamnetworkingsockets/steamnetworking_statsutils.h +++ b/src/steamnetworkingsockets/steamnetworking_statsutils.h @@ -580,6 +580,12 @@ struct LinkStatsTrackerBase m_usecWhenTimeoutStarted = 0; } + // For multi-path, we track some extra stats + uint64 m_recvPktNumberMaskMultiPath[2][2]; // Bitmask that we have received on either path + int64 m_nMultiPathRecvLater[2]; + int64 m_nMultiPathRecvSeq[2]; + bool m_bMultiPathSendEnabled; + // // Quality metrics stats // @@ -874,10 +880,11 @@ protected: // Hooks that derived classes may override when we process a packet // and it meets certain characteristics - inline void InternalProcessSequencedPacket_Count() + inline void InternalProcessSequencedPacket_Count( int idxMultiPath ) { m_seqPktCounters.OnRecv(); ++m_nPktsRecvSequenced; + ++m_nMultiPathRecvSeq[ idxMultiPath ]; } void InternalProcessSequencedPacket_OutOfOrder( int64 nPktNum ); inline void InternalProcessSequencedPacket_Duplicate() @@ -1115,7 +1122,7 @@ struct LinkStatsTracker final : public TLinkStatsTracker /// This expands the wire packet number to its full value, /// and checks if it is a duplicate or out of range. /// Stats are also updated - int64 ExpandWirePacketNumberAndCheck( uint16 nWireSeqNum ) + int64 ExpandWirePacketNumberAndCheck( uint16 nWireSeqNum, int idxMultiPath ) { int16 nGap = (int16)( nWireSeqNum - (uint16)TLinkStatsTracker::m_nMaxRecvPktNum ); int64 nPktNum = TLinkStatsTracker::m_nMaxRecvPktNum + nGap; @@ -1125,7 +1132,7 @@ struct LinkStatsTracker final : public TLinkStatsTracker constexpr int N = V_ARRAYSIZE(TLinkStatsTracker::m_arDebugHistoryRecvSeqNum); COMPILE_TIME_ASSERT( ( N & (N-1) ) == 0 ); TLinkStatsTracker::m_arDebugHistoryRecvSeqNum[ TLinkStatsTracker::m_nPktsRecvSequenced & (N-1) ] = nPktNum; - TLinkStatsTracker::InternalProcessSequencedPacket_Count(); + TLinkStatsTracker::InternalProcessSequencedPacket_Count( idxMultiPath ); // Packet number is increasing? // (Maybe by a lot -- we don't handle that here.) @@ -1145,8 +1152,23 @@ struct LinkStatsTracker final : public TLinkStatsTracker uint64 bit = uint64{1} << ( nPktNum & 63 ); if ( TLinkStatsTracker::m_recvPktNumberMask[ idxRecvBitmask ] & bit ) { - // Duplicate - TLinkStatsTracker::InternalProcessSequencedPacket_Duplicate(); + // Duplicate. But if we haven't received it through this path yet, + // it's just because the packet got to us through the other path first. + if ( TLinkStatsTracker::m_recvPktNumberMaskMultiPath[idxMultiPath][idxRecvBitmask] & bit ) + { + // Yes a true duplicate. (This will be the typical case, + // when dual-path is not available.) + TLinkStatsTracker::InternalProcessSequencedPacket_Duplicate(); + } + else + { + // The other path beat us + ++TLinkStatsTracker::m_nMultiPathRecvLater[ idxMultiPath ]; + + // Mark that we got it on this path, too + TLinkStatsTracker::m_recvPktNumberMaskMultiPath[idxMultiPath][idxRecvBitmask] |= bit; + } + return 0; } @@ -1158,18 +1180,18 @@ struct LinkStatsTracker final : public TLinkStatsTracker /// Same as ExpandWirePacketNumberAndCheck, but if this is the first sequenced /// packet we have ever received, initialize the packet number - int64 ExpandWirePacketNumberAndCheckMaybeInitialize( uint16 nWireSeqNum ) + int64 ExpandWirePacketNumberAndCheckMaybeInitialize( uint16 nWireSeqNum, int idxMultiPath ) { if ( unlikely( TLinkStatsTracker::m_nMaxRecvPktNum == 0 ) ) TLinkStatsTracker::ResetMaxRecvPktNumForIncomingWirePktNum( nWireSeqNum ); - return ExpandWirePacketNumberAndCheck( nWireSeqNum ); + return ExpandWirePacketNumberAndCheck( nWireSeqNum, idxMultiPath ); } /// Called when we have processed a packet with a sequence number, to update estimated /// number of dropped packets, etc. This MUST only be called after we have /// called ExpandWirePacketNumberAndCheck, to ensure that the packet number is not a /// duplicate or out of range. - inline void TrackProcessSequencedPacket( int64 nPktNum, SteamNetworkingMicroseconds usecNow, int usecSenderTimeSincePrev ) + inline void TrackProcessSequencedPacket( int64 nPktNum, SteamNetworkingMicroseconds usecNow, int usecSenderTimeSincePrev, int idxMultiPath ) { Assert( nPktNum > 0 ); @@ -1183,18 +1205,25 @@ struct LinkStatsTracker final : public TLinkStatsTracker { // Crossed to the next 64-packet block. Shift bitmasks forward by one. TLinkStatsTracker::m_recvPktNumberMask[0] = TLinkStatsTracker::m_recvPktNumberMask[1]; + TLinkStatsTracker::m_recvPktNumberMaskMultiPath[0][0] = TLinkStatsTracker::m_recvPktNumberMaskMultiPath[0][1]; + TLinkStatsTracker::m_recvPktNumberMaskMultiPath[1][0] = TLinkStatsTracker::m_recvPktNumberMaskMultiPath[1][1]; } else { // Large packet number jump, we skipped a whole block TLinkStatsTracker::m_recvPktNumberMask[0] = 0; + TLinkStatsTracker::m_recvPktNumberMaskMultiPath[0][0] = 0; + TLinkStatsTracker::m_recvPktNumberMaskMultiPath[1][0] = 0; } TLinkStatsTracker::m_recvPktNumberMask[1] = 0; + TLinkStatsTracker::m_recvPktNumberMaskMultiPath[0][1] = 0; + TLinkStatsTracker::m_recvPktNumberMaskMultiPath[1][1] = 0; idxRecvBitmask = 1; } uint64 bit = uint64{1} << ( nPktNum & 63 ); Assert( !( TLinkStatsTracker::m_recvPktNumberMask[ idxRecvBitmask ] & bit ) ); // Should not have already been marked! We should have already discarded duplicates TLinkStatsTracker::m_recvPktNumberMask[ idxRecvBitmask ] |= bit; + TLinkStatsTracker::m_recvPktNumberMaskMultiPath[idxMultiPath][idxRecvBitmask] |= bit; // Check for dropped packet. Since we hope that by far the most common // case will be packets delivered in order, we optimize this logic diff --git a/src/steamnetworkingsockets/steamnetworkingsockets_internal.h b/src/steamnetworkingsockets/steamnetworkingsockets_internal.h index d8481f9..8ee7d5d 100644 --- a/src/steamnetworkingsockets/steamnetworkingsockets_internal.h +++ b/src/steamnetworkingsockets/steamnetworkingsockets_internal.h @@ -80,6 +80,23 @@ #endif #endif +//// Don't enable DualSTA support, even on Windows. It's not useful +//// right now because it only works for relayed connections. We'll want +//// to enable this once it's working for plain IP connections, too. +//#ifdef _WINDOWS +// #define STEAMNETWORKINGSOCKETS_ENABLE_DUALWIFI +//#endif + +enum EDualWifiEnable { + k_nDualWifiEnable_Disable = 0, + k_nDualWifiEnable_Enable = 1, // + k_nDualWifiEnable_DoNotEnumerate = 2, // Enumerate primary adapters, but don't actually try to enable any Dual Wifi support + k_nDualWifiEnable_DoNotBind = 3, // Try to turn on Dual Wifi and locate the secondary adapter, but don't actually bind + k_nDualWifiEnable_ForceSimulate = 4, // Don't really do any DualWifi work, just open up another "regular" socket + + k_nDualWifiEnable_MAX = k_nDualWifiEnable_ForceSimulate // Maximum legal value +}; + /// Enumerate different kinds of transport that can be used enum ESteamNetTransportKind { @@ -809,6 +826,9 @@ struct ConnectionConfig ConfigValue m_EnableDiagnosticsUI; #endif + #ifdef STEAMNETWORKINGSOCKETS_ENABLE_DUALWIFI + ConfigValue m_DualWifi_Enable; + #endif ConfigValue m_LogLevel_AckRTT; ConfigValue m_LogLevel_PacketDecode; diff --git a/src/steamnetworkingsockets/steamnetworkingsockets_stats.cpp b/src/steamnetworkingsockets/steamnetworkingsockets_stats.cpp index 02a698d..0adb60b 100644 --- a/src/steamnetworkingsockets/steamnetworkingsockets_stats.cpp +++ b/src/steamnetworkingsockets/steamnetworkingsockets_stats.cpp @@ -154,6 +154,9 @@ void LinkStatsTrackerBase::InitInternal( SteamNetworkingMicroseconds usecNow ) m_flInPacketsWeirdSequencePct = -1.0f; m_usecMaxJitterPreviousInterval = -1; m_nPktsRecvSequenced = 0; + m_nMultiPathRecvSeq[0] = m_nMultiPathRecvSeq[1] = 0; + m_nMultiPathRecvLater[0] = m_nMultiPathRecvLater[1] = 0; + m_bMultiPathSendEnabled = false; m_nDebugPktsRecvInOrder = 0; m_nPktsRecvDroppedAccumulator = 0; m_nPktsRecvOutOfOrderAccumulator = 0; @@ -314,6 +317,9 @@ void LinkStatsTrackerBase::InitMaxRecvPktNum( int64 nPktNum ) else m_recvPktNumberMask[1] = ( (uint64)1 << nBitsToSet ) - 1; + m_recvPktNumberMaskMultiPath[0][0] = m_recvPktNumberMaskMultiPath[1][0] = m_recvPktNumberMask[0]; + m_recvPktNumberMaskMultiPath[0][1] = m_recvPktNumberMaskMultiPath[1][1] = m_recvPktNumberMask[1]; + m_nDebugLastInitMaxRecvPktNum = nPktNum; } @@ -555,6 +561,10 @@ void LinkStatsTrackerBase::GetLifetimeStats( SteamDatagramLinkLifetimeStats &s ) s.m_nPktsRecvDuplicate = PktsRecvDuplicate(); s.m_nPktsRecvSequenceNumberLurch = PktsRecvLurch(); + s.m_bMultiPathSendEnabled = m_bMultiPathSendEnabled; + s.m_nMultiPathRecvSeq[0] = m_nMultiPathRecvSeq[0]; s.m_nMultiPathRecvSeq[1] = m_nMultiPathRecvSeq[1]; + s.m_nMultiPathRecvLater[0] = m_nMultiPathRecvLater[0]; s.m_nMultiPathRecvLater[1] = m_nMultiPathRecvLater[1]; + s.m_qualityHistogram = m_qualityHistogram; s.m_nQualityNtile50th = m_qualitySample.NumSamples() < 2 ? -1 : m_qualitySample.GetPercentile( .50f ); @@ -823,6 +833,16 @@ void LinkStatsLifetimeStructToMsg( const SteamDatagramLinkLifetimeStats &s, CMsg msg.set_packets_recv_duplicate( s.m_nPktsRecvDuplicate ); msg.set_packets_recv_lurch( s.m_nPktsRecvSequenceNumberLurch ); + if ( s.m_bMultiPathSendEnabled ) + msg.set_multipath_send_enabled( 1 ); + if ( s.m_bMultiPathSendEnabled || s.m_nMultiPathRecvSeq[1] > 0 ) + { + msg.add_multipath_packets_recv_sequenced( s.m_nMultiPathRecvSeq[0] ); + msg.add_multipath_packets_recv_sequenced( s.m_nMultiPathRecvSeq[1] ); + msg.add_multipath_packets_recv_later( s.m_nMultiPathRecvLater[0] ); + msg.add_multipath_packets_recv_later( s.m_nMultiPathRecvLater[1] ); + } + #define SET_HISTOGRAM( mbr, field ) if ( mbr > 0 ) msg.set_ ## field( mbr ); #define SET_NTILE( mbr, field ) if ( mbr >= 0 ) msg.set_ ## field( mbr ); @@ -917,6 +937,12 @@ void LinkStatsLifetimeMsgToStruct( const CMsgSteamDatagramLinkLifetimeStats &msg s.m_nPktsRecvDuplicate = msg.packets_recv_duplicate(); s.m_nPktsRecvSequenceNumberLurch = msg.packets_recv_lurch(); + s.m_bMultiPathSendEnabled = msg.multipath_send_enabled() > 0; + s.m_nMultiPathRecvSeq[0] = ( msg.multipath_packets_recv_sequenced_size() > 0 ) ? msg.multipath_packets_recv_sequenced(0) : 0; + s.m_nMultiPathRecvSeq[1] = ( msg.multipath_packets_recv_sequenced_size() > 1 ) ? msg.multipath_packets_recv_sequenced(1) : 0; + s.m_nMultiPathRecvLater[0] = ( msg.multipath_packets_recv_later_size() > 0 ) ? msg.multipath_packets_recv_later(0) : 0; + s.m_nMultiPathRecvLater[1] = ( msg.multipath_packets_recv_later_size() > 1 ) ? msg.multipath_packets_recv_later(1) : 0; + #define SET_HISTOGRAM( mbr, field ) mbr = msg.field(); #define SET_NTILE( mbr, field ) mbr = ( msg.has_ ## field() ? msg.field() : -1 ); @@ -1059,6 +1085,7 @@ void LinkStatsPrintLifetimeToBuf( const char *pszLeader, const SteamDatagramLink buf.Printf( "%sTotals\n", pszLeader ); buf.Printf( "%s Sent:%11s pkts %15s bytes\n", pszLeader, NumberPrettyPrinter( stats.m_nPacketsSent ).String(), NumberPrettyPrinter( stats.m_nBytesSent ).String() ); buf.Printf( "%s Recv:%11s pkts %15s bytes\n", pszLeader, NumberPrettyPrinter( stats.m_nPacketsRecv ).String(), NumberPrettyPrinter( stats.m_nBytesRecv ).String() ); + if ( stats.m_nPktsRecvSequenced > 0 ) { buf.Printf( "%s Recv w seq:%11s pkts\n", pszLeader, NumberPrettyPrinter( stats.m_nPktsRecvSequenced ).String() ); @@ -1067,6 +1094,17 @@ void LinkStatsPrintLifetimeToBuf( const char *pszLeader, const SteamDatagramLink buf.Printf( "%s OutOfOrder:%11s pkts%7.2f%%\n", pszLeader, NumberPrettyPrinter( stats.m_nPktsRecvOutOfOrder ).String(), stats.m_nPktsRecvOutOfOrder * flToPct ); buf.Printf( "%s Duplicate :%11s pkts%7.2f%%\n", pszLeader, NumberPrettyPrinter( stats.m_nPktsRecvDuplicate ).String(), stats.m_nPktsRecvDuplicate * flToPct ); buf.Printf( "%s SeqLurch :%11s pkts%7.2f%%\n", pszLeader, NumberPrettyPrinter( stats.m_nPktsRecvSequenceNumberLurch ).String(), stats.m_nPktsRecvSequenceNumberLurch * flToPct ); + + if ( stats.m_nMultiPathRecvSeq[1] > 0 ) + { + AssertMsg( stats.m_nMultiPathRecvSeq[0] + stats.m_nMultiPathRecvSeq[1] == stats.m_nPktsRecvSequenced, "multipath seq bookkeeping error %lld + %lld != %lld", + (long long)stats.m_nMultiPathRecvSeq[0], (long long)stats.m_nMultiPathRecvSeq[1], (long long)stats.m_nPktsRecvSequenced ); + + buf.Printf( "%s Pth0 w seq:%11s pkts\n", pszLeader, NumberPrettyPrinter( stats.m_nMultiPathRecvSeq[0] ).String() ); + buf.Printf( "%s Pth1 w seq:%11s pkts\n", pszLeader, NumberPrettyPrinter( stats.m_nMultiPathRecvSeq[1] ).String() ); + buf.Printf( "%s Pth0 later:%11s pkts\n", pszLeader, NumberPrettyPrinter( stats.m_nMultiPathRecvLater[0] ).String() ); + buf.Printf( "%s Pth1 later:%11s pkts\n", pszLeader, NumberPrettyPrinter( stats.m_nMultiPathRecvLater[1] ).String() ); + } } // Do we have enough ping samples such that the distribution might be interesting