[active-standby] Probe the link in suspend timeout

Signed-off-by: Longxiang Lyu <[email protected]>
sonic-net · Jan 11, 2024 · 193a40e · 193a40e
1 parent 6586abf
commit 193a40e
Show file tree

Hide file tree

Showing 8 changed files with 109 additions and 1 deletion.
diff --git a/src/link_manager/LinkManagerStateMachineActiveStandby.cpp b/src/link_manager/LinkManagerStateMachineActiveStandby.cpp
@@ -366,6 +366,9 @@ void ActiveStandbyStateMachine::handleSwssBladeIpv4AddressUpdate(boost::asio::ip
             mProbePeerTorFnPtr = boost::bind(
                 &link_prober::LinkProber::probePeerTor, mLinkProberPtr.get()
             );
+            mDetectLinkFnPtr = boost::bind(
+                &link_prober::LinkProber::detectLink, mLinkProberPtr.get()
+            );
             mSuspendTxFnPtr = boost::bind(
                 &link_prober::LinkProber::suspendTxProbes, mLinkProberPtr.get(), boost::placeholders::_1
             );
@@ -829,6 +832,8 @@ void ActiveStandbyStateMachine::handleSuspendTimerExpiry()
         CompositeState currState = mCompositeState;
         enterMuxWaitState(mCompositeState);
         LOGINFO_MUX_STATE_TRANSITION(mMuxPortConfig.getPortName(), currState, mCompositeState);
+        // detect if link/server recovers
+        mDetectLinkFnPtr();
     } else {
         mUnknownActiveUpBackoffFactor = 1;
     }

diff --git a/src/link_manager/LinkManagerStateMachineActiveStandby.h b/src/link_manager/LinkManagerStateMachineActiveStandby.h
@@ -755,6 +755,17 @@ class ActiveStandbyStateMachine: public LinkManagerStateMachineBase,
     */
     void setProbePeerTorFnPtr(boost::function<void ()> probePeerTorFnPtr) {mProbePeerTorFnPtr = probePeerTorFnPtr;};
 
+    /**
+    *@method setDetectLinkFnPtr
+    *
+    *@brief set new DetectLinkFnPtr for the state machine. This method is used for testing
+    *
+    *@param detectLinkFnPtr (in)  pointer to new DetectLinkFnPtr
+    *
+    *@return none
+    */
+    void setDetectLinkFnPtr(boost::function<void ()> detectLinkFnPtr) {mDetectLinkFnPtr = detectLinkFnPtr;};
+
     /**
     *@method setSuspendTxFnPtr
     *
@@ -852,6 +863,7 @@ class ActiveStandbyStateMachine: public LinkManagerStateMachineBase,
     boost::function<void ()> mInitializeProberFnPtr;
     boost::function<void ()> mStartProbingFnPtr;
     boost::function<void ()> mProbePeerTorFnPtr;
+    boost::function<void ()> mDetectLinkFnPtr;
     boost::function<void (uint32_t suspendTime_msec)> mSuspendTxFnPtr;
     boost::function<void ()> mResumeTxFnPtr;
     boost::function<void ()> mSendPeerSwitchCommandFnPtr;

diff --git a/src/link_prober/LinkProber.cpp b/src/link_prober/LinkProber.cpp
@@ -229,11 +229,35 @@ void LinkProber::updateEthernetFrame()
 // send an early HB to peer ToR
 //
 void LinkProber::probePeerTor()
+{
+    detectLink(1);
+}
+
+//
+// ---> detectLink(uint32_t probeCount);
+//
+// send HBs to detect the link status
+//
+void LinkProber::detectLink(uint32_t probeCount)
 {
     boost::asio::io_service &ioService = mStrand.context();
-    ioService.post(mStrand.wrap(boost::bind(&LinkProber::sendHeartbeat, this)));
+    for (uint32_t i = 0; i < probeCount; ++i)
+    {
+        ioService.post(mStrand.wrap(boost::bind(&LinkProber::sendHeartbeat, this)));
+    }
 }
 
+//
+// ---> detectLink();
+//
+// send HBs to detect the link status
+//
+void LinkProber::detectLink()
+{
+    detectLink(mMuxPortConfig.getPositiveStateChangeRetryCount());
+}
+
+
 //
 // ---> sendPeerSwitchCommand();
 //

diff --git a/src/link_prober/LinkProber.h b/src/link_prober/LinkProber.h
@@ -167,6 +167,26 @@ class LinkProber
     */
     void probePeerTor();
 
+    /**
+    *@method detectLink
+    *
+    *@brief detect link status
+    *
+    *@param probeCount (in) HB count
+    *
+    *@return none
+    */
+    void detectLink(uint32_t probeCount);
+
+    /**
+    *@method detectLink
+    *
+    *@brief detect link status
+    *
+    *@return none
+    */
+    void detectLink();
+
     /**
     *@method sendPeerSwitchCommand
     *

diff --git a/test/FakeLinkProber.cpp b/test/FakeLinkProber.cpp
@@ -106,6 +106,13 @@ void FakeLinkProber::probePeerTor()
     mProbePeerTorCallCount++;
 }
 
+void FakeLinkProber::detectLink()
+{
+    MUXLOGINFO("");
+
+    mDetectLinkCallCount++;
+}
+
 void FakeLinkProber::suspendTxProbes(uint32_t suspendTime_msec)
 {
     MUXLOGINFO("");

diff --git a/test/FakeLinkProber.h b/test/FakeLinkProber.h
@@ -43,6 +43,7 @@ class FakeLinkProber
     void startProbing();
     void updateEthernetFrame();
     void probePeerTor();
+    void detectLink();
     void suspendTxProbes(uint32_t suspendTime_msec);
     void resumeTxProbes();
     void sendPeerSwitchCommand();
@@ -62,6 +63,7 @@ class FakeLinkProber
     uint32_t mUpdateEthernetFrameCallCount = 0;
     uint32_t mProbePeerTorCallCount = 0;
     uint32_t mSuspendTxProbeCallCount = 0;
+    uint32_t mDetectLinkCallCount = 0;
     uint32_t mResumeTxProbeCallCount = 0;
     uint32_t mSendPeerSwitchCommand = 0;
     uint32_t mSendPeerProbeCommand = 0;

diff --git a/test/FakeMuxPort.cpp b/test/FakeMuxPort.cpp
@@ -117,6 +117,9 @@ inline void FakeMuxPort::initLinkProberActiveStandby()
     getActiveStandbyStateMachinePtr()->setProbePeerTorFnPtr(
         boost::bind(&FakeLinkProber::probePeerTor, mFakeLinkProber.get())
     );
+    getActiveStandbyStateMachinePtr()->setDetectLinkFnPtr(
+        boost::bind(&FakeLinkProber::detectLink, mFakeLinkProber.get())
+    );
     getActiveStandbyStateMachinePtr()->setSuspendTxFnPtr(
         boost::bind(&FakeLinkProber::suspendTxProbes, mFakeLinkProber.get(), boost::placeholders::_1)
     );

diff --git a/test/LinkManagerStateMachineTest.cpp b/test/LinkManagerStateMachineTest.cpp
@@ -1494,4 +1494,39 @@ TEST_F(LinkManagerStateMachineTest, OrchagentRollback)
     EXPECT_EQ(mDbInterfacePtr->mSetMuxStateInvokeCount, 2);
 }
 
+TEST_F(LinkManagerStateMachineTest, ProbeLinkInSuspendTimeout)
+{
+    setMuxActive();
+    VALIDATE_STATE(Active, Active, Up);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mSuspendTxProbeCallCount, 0);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mResumeTxProbeCallCount, 1);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mDetectLinkCallCount, 0);
+
+    postLinkProberEvent(link_prober::LinkProberState::Unknown);
+    VALIDATE_STATE(Unknown, Active, Up);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mSuspendTxProbeCallCount, 1);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mResumeTxProbeCallCount, 1);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mDetectLinkCallCount, 0);
+
+    postSuspendTimerExpiredEvent(2);
+    VALIDATE_STATE(Unknown, Wait, Up);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mSuspendTxProbeCallCount, 1);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mResumeTxProbeCallCount, 1);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mDetectLinkCallCount, 1);
+
+    // peer doesn't take the link
+    handleProbeMuxState("active", 3);
+    VALIDATE_STATE(Unknown, Active, Up);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mSuspendTxProbeCallCount, 2);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mResumeTxProbeCallCount, 1);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mDetectLinkCallCount, 1);
+
+    // link resumes, take back the link and resume heartbeat
+    postLinkProberEvent(link_prober::LinkProberState::Active);
+    VALIDATE_STATE(Active, Active, Up);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mSuspendTxProbeCallCount, 2);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mResumeTxProbeCallCount, 2);
+    EXPECT_EQ(mFakeMuxPort.mFakeLinkProber->mDetectLinkCallCount, 1);
+}
+
 } /* namespace test */