diff --git a/src/common/steamnetworkingsockets_messages.proto b/src/common/steamnetworkingsockets_messages.proto index 02932e7..796607c 100644 --- a/src/common/steamnetworkingsockets_messages.proto +++ b/src/common/steamnetworkingsockets_messages.proto @@ -99,6 +99,7 @@ message CMsgSteamDatagramLinkLifetimeStats optional uint64 packets_recv_sequenced = 7; optional uint64 packets_recv_dropped = 8; optional uint64 packets_recv_out_of_order = 9; + optional uint64 packets_recv_out_of_order_corrected = 15; optional uint64 packets_recv_duplicate = 10; optional uint64 packets_recv_lurch = 11; repeated uint64 multipath_packets_recv_sequenced = 12; // If multipath send or receive was used, then the samples for how many we received on each path diff --git a/src/steamnetworkingsockets/steamnetworking_stats.h b/src/steamnetworkingsockets/steamnetworking_stats.h index 3389b22..aae8eac 100644 --- a/src/steamnetworkingsockets/steamnetworking_stats.h +++ b/src/steamnetworkingsockets/steamnetworking_stats.h @@ -222,6 +222,7 @@ struct SteamDatagramLinkLifetimeStats int64 m_nPktsRecvSequenced; // packets that we received that had a sequence number. int64 m_nPktsRecvDropped; int64 m_nPktsRecvOutOfOrder; + int64 m_nPktsRecvOutOfOrderCorrected; int64 m_nPktsRecvDuplicate; int64 m_nPktsRecvSequenceNumberLurch; diff --git a/src/steamnetworkingsockets/steamnetworking_statsutils.h b/src/steamnetworkingsockets/steamnetworking_statsutils.h index 283fa06..dca2ee4 100644 --- a/src/steamnetworkingsockets/steamnetworking_statsutils.h +++ b/src/steamnetworkingsockets/steamnetworking_statsutils.h @@ -25,6 +25,8 @@ class CMsgSteamDatagramConnectionQuality; // Internal stuff goes in a private namespace namespace SteamNetworkingSocketsLib { +class CPossibleOutOfOrderPacket; + /// Default interval for link stats rate measurement const SteamNetworkingMicroseconds k_usecSteamDatagramLinkStatsDefaultInterval = 5 * k_nMillion; @@ -417,9 +419,10 @@ struct SequencedPacketCounters { int m_nRecv; // packets successfully received containing a sequence number int m_nDropped; // packets assumed to be dropped in the current interval - int m_nOutOfOrder; // any sequence number deviation other than a simple dropped packet. (Most recent interval.) - int m_nLurch; // any sequence number deviation other than a simple dropped packet. (Most recent interval.) - int m_nDuplicate; // any sequence number deviation other than a simple dropped packet. (Most recent interval.) + int m_nOutOfOrder; // sequence number was smaller than previous packet. (And we were not able to correct it.) + int m_nOutOfOrderCorrected; // out of order, but we corrected it at the packet level + int m_nLurch; // any sequence number deviation not accounted for by other fields. + int m_nDuplicate; // duplicate sequence number. int m_usecMaxJitter; void Reset() @@ -427,6 +430,7 @@ struct SequencedPacketCounters m_nRecv = 0; m_nDropped = 0; m_nOutOfOrder = 0; + m_nOutOfOrderCorrected = 0; m_nLurch = 0; m_nDuplicate = 0; m_usecMaxJitter = -1; @@ -437,6 +441,7 @@ struct SequencedPacketCounters m_nRecv += x.m_nRecv; m_nDropped += x.m_nDropped; m_nOutOfOrder += m_nOutOfOrder; + m_nOutOfOrderCorrected += x.m_nOutOfOrderCorrected; m_nLurch += m_nLurch; m_nDuplicate += x.m_nDuplicate; m_usecMaxJitter = std::max( m_usecMaxJitter, x.m_usecMaxJitter ); @@ -482,6 +487,12 @@ struct SequencedPacketCounters if ( m_nDropped > 0 ) // Might have marked it in the previous interval. Our stats will be slightly off in this case. Not worth it to try to get this exactly right. --m_nDropped; } + inline void OnOutOfOrderCorrected() + { + ++m_nOutOfOrderCorrected; + // NOTE - current out-of-order correction code does not increment the + // dropped count for the skipped packet while we await for it to arrive + } }; /// Rough classification of the amount of activity we expect on a link. This informs @@ -518,7 +529,7 @@ enum class ELinkActivityLevel /// all of these small functions if appropriate, and no virtual function dispatch is used. struct LinkStatsTrackerBase { - + ~LinkStatsTrackerBase(); /// What version is the peer running? It's 0 if we don't know yet. uint32 m_nPeerProtocolVersion; @@ -616,10 +627,12 @@ struct LinkStatsTrackerBase int64 m_nPktsRecvSequenced; int64 m_nPktsRecvDroppedAccumulator; int64 m_nPktsRecvOutOfOrderAccumulator; + int64 m_nPktsRecvOutOfOrderCorrectedAccumulator; int64 m_nPktsRecvDuplicateAccumulator; int64 m_nPktsRecvLurchAccumulator; inline int64 PktsRecvDropped() const { return m_nPktsRecvDroppedAccumulator + m_seqPktCounters.m_nDropped; } inline int64 PktsRecvOutOfOrder() const { return m_nPktsRecvOutOfOrderAccumulator + m_seqPktCounters.m_nOutOfOrder; } + inline int64 PktsRecvOutOfOrderCorrected() const { return m_nPktsRecvOutOfOrderCorrectedAccumulator + m_seqPktCounters.m_nOutOfOrderCorrected; } inline int64 PktsRecvDuplicate() const { return m_nPktsRecvDuplicateAccumulator + m_seqPktCounters.m_nDuplicate; } inline int64 PktsRecvLurch() const { return m_nPktsRecvLurchAccumulator + m_seqPktCounters.m_nLurch; } @@ -810,6 +823,11 @@ struct LinkStatsTrackerBase ++m_qualityHistogram.m_arBuckets[ eBucket ]; } + /// Get the pending out-of-order packet on this flow, if any + CPossibleOutOfOrderPacket *GetPossibleOutOfOrderPacket() const { return m_pPossibleOutOfOrderPacket; } + + virtual void ProcessSequencedPacket_OutOfOrderCorrected(); + protected: // Make sure it's used as abstract base. Note that we require you to call Init() // with a timestamp value, so the constructor is empty by default. @@ -983,6 +1001,7 @@ protected: } private: + friend class CPossibleOutOfOrderPacket; // Number of lifetime sequenced packets received, and overall packets sent, // the last time the peer acked stats @@ -1004,6 +1023,10 @@ private: void UpdateInterval( SteamNetworkingMicroseconds usecNow ); void StartNextInterval( SteamNetworkingMicroseconds usecNow ); + + // Certain flows have the ability to detect and + // repair a single out of order packet. + CPossibleOutOfOrderPacket *m_pPossibleOutOfOrderPacket = nullptr; }; struct LinkStatsTrackerEndToEnd : public LinkStatsTrackerBase diff --git a/src/steamnetworkingsockets/steamnetworkingsockets_internal.h b/src/steamnetworkingsockets/steamnetworkingsockets_internal.h index 2f9c4b7..b90b3dc 100644 --- a/src/steamnetworkingsockets/steamnetworkingsockets_internal.h +++ b/src/steamnetworkingsockets/steamnetworkingsockets_internal.h @@ -241,6 +241,7 @@ struct SteamDatagramLinkStats; struct SteamDatagramLinkLifetimeStats; struct SteamDatagramLinkInstantaneousStats; struct SteamNetworkingDetailedConnectionStatus; +struct LinkStatsTrackerBase; // An identity operator that always returns its operand. // NOTE: std::hash is an identity operator on many compilers @@ -1001,6 +1002,38 @@ inline ESteamNetworkingFakeIPType GetIPv4FakeIPType( uint32 nIPv4 ) inline ESteamNetworkingFakeIPType GetIPv4FakeIPType( uint32 nIPv4 ) { return k_ESteamNetworkingFakeIPType_NotFake; } #endif +// Helper class to store a packet that was received when it looks +// like the packet might have been delivered out of order. +class CPossibleOutOfOrderPacket +{ +public: + + // "Timeout" when this packet should be dispatched if we haven't + // received the skipped packet + SteamNetworkingMicroseconds m_usecFlush = 0; + + // Detach from our owner and destroy this object. + void Destroy(); + + // Detach from our owner + void Detach(); + + LinkStatsTrackerBase *GetOwner() const { return m_pOwner; } + void SetOwner( LinkStatsTrackerBase *pOwner ); + +protected: + + // Link stats tracker that owns us + LinkStatsTrackerBase *m_pOwner = nullptr; + + virtual void DoDestroy(); + + inline CPossibleOutOfOrderPacket() {} + + // Destructor is protected, you should call Destroy() + virtual ~CPossibleOutOfOrderPacket(); +}; + } // namespace SteamNetworkingSocketsLib #include diff --git a/src/steamnetworkingsockets/steamnetworkingsockets_stats.cpp b/src/steamnetworkingsockets/steamnetworkingsockets_stats.cpp index c69d6b9..1b454c0 100644 --- a/src/steamnetworkingsockets/steamnetworkingsockets_stats.cpp +++ b/src/steamnetworkingsockets/steamnetworkingsockets_stats.cpp @@ -138,6 +138,59 @@ int PingTracker::WorstPingInRecentSample() const return nResult; } +LinkStatsTrackerBase::~LinkStatsTrackerBase() +{ + if ( unlikely( m_pPossibleOutOfOrderPacket != nullptr ) ) + { + m_pPossibleOutOfOrderPacket->Destroy(); + Assert( m_pPossibleOutOfOrderPacket == nullptr ); + } +} + +void CPossibleOutOfOrderPacket::Destroy() +{ + Detach(); + DoDestroy(); +} + +void CPossibleOutOfOrderPacket::Detach() +{ + if ( m_pOwner ) + { + Assert( m_pOwner->m_pPossibleOutOfOrderPacket == this ); + m_pOwner->m_pPossibleOutOfOrderPacket = nullptr; + m_pOwner = nullptr; + } +} + +void CPossibleOutOfOrderPacket::DoDestroy() +{ + // Assume we were allocated on the heap + delete this; +} + +void CPossibleOutOfOrderPacket::SetOwner( LinkStatsTrackerBase *pOwner ) +{ + if ( m_pOwner ) + { + Assert( false ); + Detach(); + } + if ( pOwner->m_pPossibleOutOfOrderPacket ) + { + Assert( false ); + pOwner->m_pPossibleOutOfOrderPacket->Detach(); + } + + m_pOwner = pOwner; + m_pOwner->m_pPossibleOutOfOrderPacket = this; +} + +CPossibleOutOfOrderPacket::~CPossibleOutOfOrderPacket() +{ + Assert( m_pOwner == nullptr ); +} + void LinkStatsTrackerBase::InitInternal( SteamNetworkingMicroseconds usecNow ) { m_nPeerProtocolVersion = 0; @@ -161,6 +214,7 @@ void LinkStatsTrackerBase::InitInternal( SteamNetworkingMicroseconds usecNow ) m_nPktsRecvSequenced = 0; m_nPktsRecvDroppedAccumulator = 0; m_nPktsRecvOutOfOrderAccumulator = 0; + m_nPktsRecvOutOfOrderCorrectedAccumulator = 0; m_nPktsRecvDuplicateAccumulator = 0; m_nPktsRecvLurchAccumulator = 0; m_qualitySample.Clear(); @@ -206,6 +260,7 @@ void LinkStatsTrackerBase::StartNextInterval( SteamNetworkingMicroseconds usecNo { m_nPktsRecvDroppedAccumulator += m_seqPktCounters.m_nDropped; m_nPktsRecvOutOfOrderAccumulator += m_seqPktCounters.m_nOutOfOrder; + m_nPktsRecvOutOfOrderCorrectedAccumulator += m_seqPktCounters.m_nOutOfOrderCorrected; m_nPktsRecvDuplicateAccumulator += m_seqPktCounters.m_nDuplicate; m_nPktsRecvLurchAccumulator += m_seqPktCounters.m_nLurch; m_seqPktCounters.Reset(); @@ -382,6 +437,11 @@ void LinkStatsTrackerBase::InternalProcessSequencedPacket_OutOfOrder( int64 nPkt m_seqPktCounters.OnOutOfOrder(); } +void LinkStatsTrackerBase::ProcessSequencedPacket_OutOfOrderCorrected() +{ + m_seqPktCounters.OnOutOfOrderCorrected(); +} + bool LinkStatsTrackerBase::BCheckHaveDataToSendInstantaneous( SteamNetworkingMicroseconds usecNow ) { Assert( m_eActivityLevel == ELinkActivityLevel::Active ); @@ -553,6 +613,7 @@ void LinkStatsTrackerBase::GetLifetimeStats( SteamDatagramLinkLifetimeStats &s ) s.m_nPktsRecvSequenced = m_nPktsRecvSequenced; s.m_nPktsRecvDropped = PktsRecvDropped(); s.m_nPktsRecvOutOfOrder = PktsRecvOutOfOrder(); + s.m_nPktsRecvOutOfOrderCorrected = PktsRecvOutOfOrderCorrected(); s.m_nPktsRecvDuplicate = PktsRecvDuplicate(); s.m_nPktsRecvSequenceNumberLurch = PktsRecvLurch(); @@ -825,6 +886,7 @@ void LinkStatsLifetimeStructToMsg( const SteamDatagramLinkLifetimeStats &s, CMsg msg.set_packets_recv_sequenced( s.m_nPktsRecvSequenced ); msg.set_packets_recv_dropped( s.m_nPktsRecvDropped ); msg.set_packets_recv_out_of_order( s.m_nPktsRecvOutOfOrder ); + msg.set_packets_recv_out_of_order_corrected( s.m_nPktsRecvOutOfOrderCorrected ); msg.set_packets_recv_duplicate( s.m_nPktsRecvDuplicate ); msg.set_packets_recv_lurch( s.m_nPktsRecvSequenceNumberLurch ); @@ -929,6 +991,7 @@ void LinkStatsLifetimeMsgToStruct( const CMsgSteamDatagramLinkLifetimeStats &msg s.m_nPktsRecvSequenced = msg.packets_recv_sequenced(); s.m_nPktsRecvDropped = msg.packets_recv_dropped(); s.m_nPktsRecvOutOfOrder = msg.packets_recv_out_of_order(); + s.m_nPktsRecvOutOfOrderCorrected = msg.packets_recv_out_of_order_corrected(); s.m_nPktsRecvDuplicate = msg.packets_recv_duplicate(); s.m_nPktsRecvSequenceNumberLurch = msg.packets_recv_lurch(); @@ -1087,6 +1150,7 @@ void LinkStatsPrintLifetimeToBuf( const char *pszLeader, const SteamDatagramLink float flToPct = 100.0f / ( stats.m_nPktsRecvSequenced + stats.m_nPktsRecvDropped ); buf.Printf( "%s Dropped :%11s pkts%7.2f%%\n", pszLeader, NumberPrettyPrinter( stats.m_nPktsRecvDropped ).String(), stats.m_nPktsRecvDropped * flToPct ); buf.Printf( "%s OutOfOrder:%11s pkts%7.2f%%\n", pszLeader, NumberPrettyPrinter( stats.m_nPktsRecvOutOfOrder ).String(), stats.m_nPktsRecvOutOfOrder * flToPct ); + buf.Printf( "%s OOOFixed :%11s pkts%7.2f%%\n", pszLeader, NumberPrettyPrinter( stats.m_nPktsRecvOutOfOrderCorrected ).String(), stats.m_nPktsRecvOutOfOrderCorrected * flToPct ); buf.Printf( "%s Duplicate :%11s pkts%7.2f%%\n", pszLeader, NumberPrettyPrinter( stats.m_nPktsRecvDuplicate ).String(), stats.m_nPktsRecvDuplicate * flToPct ); buf.Printf( "%s SeqLurch :%11s pkts%7.2f%%\n", pszLeader, NumberPrettyPrinter( stats.m_nPktsRecvSequenceNumberLurch ).String(), stats.m_nPktsRecvSequenceNumberLurch * flToPct );