[active-standby] Add oscillation logic when there is no heartbeat on both sides

src/DbInterface.cpp

@@ -47,7 +47,7 @@ std::vector<std::string> DbInterface::mMuxState = {"active", "standby", "unknown
 std::vector<std::string> DbInterface::mMuxLinkmgrState = {"uninitialized", "unhealthy", "healthy"};
 std::vector<std::string> DbInterface::mMuxMetrics = {"start", "end"};
 std::vector<std::string> DbInterface::mLinkProbeMetrics = {"link_prober_unknown_start", "link_prober_unknown_end", "link_prober_wait_start", "link_prober_active_start", "link_prober_standby_start"};
-std::vector<std::string> DbInterface::mActiveStandbySwitchCause = {"Peer_Heartbeat_Missing" , "Peer_Link_Down" , "Tlv_Switch_Active_Command" , "Link_Down" , "Transceiver_Daemon_Timeout" , "Matching_Hardware_State" , "Config_Mux_Mode", "Hardware_State_Unknown"};
+std::vector<std::string> DbInterface::mActiveStandbySwitchCause = {"Peer_Heartbeat_Missing" , "Peer_Link_Down" , "Tlv_Switch_Active_Command" , "Link_Down" , "Transceiver_Daemon_Timeout" , "Matching_Hardware_State" , "Config_Mux_Mode", "Hardware_State_Unknown", "Timed_Oscillation"};
 
 //
 // ---> DbInterface(mux::MuxManager *muxManager);

src/common/MuxConfig.h

@@ -274,6 +274,33 @@ class MuxConfig
     */
     inline uint32_t getSuspendTimeout_msec() const {return (mNegativeStateChangeRetryCount + 1) * mTimeoutIpv4_msec;};
 
+    /**
+    *@method getOscillationInterval_sec
+    *
+    *@brief getter for mux state oscillation interval
+    *
+    *@return oscillation interval
+    */
+    inline uint32_t getOscillationInterval_sec() const {return mOscillationTimeout_sec;};
+
+    /**
+    *@method setOscillationInterval_sec
+    *
+    *@brief setter for mux state oscillation interval
+    *
+    *@param oscillationInterval_sec (in) oscillation interval
+    *@param force (in) force set the oscillation interval
+    *
+    *@return none
+    */
+    inline void setOscillationInterval_sec(uint32_t oscillationInterval_sec, bool force=false) {
+        if (force || oscillationInterval_sec > 300) {
+            mOscillationTimeout_sec = oscillationInterval_sec;
+        } else {
+            mOscillationTimeout_sec = 300;
+        }
+    };
+
     /**
     *@method getMuxStateChangeRetryCount
     *
@@ -406,6 +433,7 @@ class MuxConfig
 
 private:
     uint8_t mNumberOfThreads = 5;
+    uint32_t mOscillationTimeout_sec = 300;
     uint32_t mTimeoutIpv4_msec = 100;
     uint32_t mTimeoutIpv6_msec = 1000;
     uint32_t mPositiveStateChangeRetryCount = 1;

src/common/MuxPortConfig.h

@@ -192,6 +192,15 @@ class MuxPortConfig
     */
     inline uint32_t getLinkWaitTimeout_msec() const {return mMuxConfig.getSuspendTimeout_msec();};
 
+    /**
+    *@method getOscillationInterval_sec
+    *
+    *@brief getter for mux state oscillation interval
+    *
+    *@return oscillation interval
+    */
+    inline uint32_t getOscillationInterval_sec() const {return mMuxConfig.getOscillationInterval_sec();};
+
     /**
     *@method getMuxStateChangeRetryCount
     *

src/link_manager/LinkManagerStateMachineActiveStandby.cpp

@@ -54,12 +54,15 @@ ActiveStandbyStateMachine::ActiveStandbyStateMachine(
         {link_prober::LinkProberState::Label::Unknown, mux_state::MuxState::Label::Wait, link_state::LinkState::Label::Down}
     ),
     mDeadlineTimer(strand.context()),
-    mWaitTimer(strand.context())
+    mWaitTimer(strand.context()),
+    mOscillationTimer(strand.context())
 {
     assert(muxPortPtr != nullptr);
     mMuxStateMachine.setWaitStateCause(mux_state::WaitState::WaitStateCause::SwssUpdate);
     mMuxPortPtr->setMuxLinkmgrState(mLabel);
     initializeTransitionFunctionTable();
+
+    mOscillationTimer.expires_from_now(boost::posix_time::seconds(1)); 
 }
 
 //
@@ -1037,6 +1040,41 @@ void ActiveStandbyStateMachine::handleMuxWaitTimeout(boost::system::error_code e
     }
 }
 
+//
+// ---> startOscillationTimer();
+//
+// when there is no icmp heartbeat, start a timer to oscillate between active and standby
+//
+void ActiveStandbyStateMachine::startOscillationTimer()
+{
+    // Note: This timer is started when Mux state is active and link prober is in wait state.
+    mOscillationTimer.expires_from_now(boost::posix_time::seconds(
+        mMuxPortConfig.getOscillationInterval_sec()
+    ));
+    mOscillationTimer.async_wait(boost::asio::bind_executor(getStrand(), boost::bind(
+            &ActiveStandbyStateMachine::handleOscillationTimeout,
+            this,
+            boost::asio::placeholders::error
+    )));
+}
+
+//
+// ---> handleOscillationTimeout(boost::system::error_code errorCode);
+//
+// handle when oscillation timer expires
+//
+void ActiveStandbyStateMachine::handleOscillationTimeout(boost::system::error_code errorCode)
+{
+    MUXLOGDEBUG(mMuxPortConfig.getPortName());
+
+    if (errorCode == boost::system::errc::success &&
+        ps(mCompositeState) == link_prober::LinkProberState::Label::Wait &&
+        ms(mCompositeState) == mux_state::MuxState::Label::Active &&
+        ls(mCompositeState) == link_state::LinkState::Label::Up) {
+            switchMuxState(link_manager::ActiveStandbyStateMachine::SwitchCause::TimedOscillation, mCompositeState, mux_state::MuxState::Label::Standby);
+    }
+}
+
 //
 // ---> initLinkProberState(CompositeState &compositeState, bool forceReset);
 //
@@ -1262,6 +1300,10 @@ void ActiveStandbyStateMachine::LinkProberWaitMuxActiveLinkUpTransitionFunction(
     if (mWaitActiveUpCount++ & 0x1) {
         mSuspendTxFnPtr(mMuxPortConfig.getLinkWaitTimeout_msec());
     }
+
+    if (mOscillationTimer.expires_at() < boost::posix_time::microsec_clock::local_time()) {
+        startOscillationTimer();
+    }
 }
 
 //

src/link_manager/LinkManagerStateMachineActiveStandby.h

@@ -73,6 +73,7 @@ class ActiveStandbyStateMachine: public LinkManagerStateMachineBase,
         MatchingHardwareState,
         ConfigMuxMode,
         HarewareStateUnknown,
+        TimedOscillation,
 
         Count
     };
@@ -454,6 +455,26 @@ class ActiveStandbyStateMachine: public LinkManagerStateMachineBase,
     */
     void handleMuxWaitTimeout(boost::system::error_code errorCode);
 
+    /**
+    *@method startOscillationTimer
+    *
+    *@brief when there is no icmp heartbeat, start a timer to oscillate between active and standby
+    *
+    *@return none
+    */
+    void startOscillationTimer();
+
+    /**
+    *@method handleOscillationTimeout
+    *
+    *@brief handle when oscillation timer expires
+    *
+    *@param errorCode (in)          timer error code
+    *
+    *@return none
+    */
+    void handleOscillationTimeout(boost::system::error_code errorCode);
+
     /**
     *@method initLinkProberState
     *
@@ -826,6 +847,7 @@ class ActiveStandbyStateMachine: public LinkManagerStateMachineBase,
 
     boost::asio::deadline_timer mDeadlineTimer;
     boost::asio::deadline_timer mWaitTimer;
+    boost::asio::deadline_timer mOscillationTimer;
 
     boost::function<void ()> mInitializeProberFnPtr;
     boost::function<void ()> mStartProbingFnPtr;

test/LinkManagerStateMachineTest.cpp

@@ -49,6 +49,7 @@ LinkManagerStateMachineTest::LinkManagerStateMachineTest() :
     mMuxConfig.setPositiveStateChangeRetryCount(mPositiveUpdateCount);
     mMuxConfig.setMuxStateChangeRetryCount(mPositiveUpdateCount);
     mMuxConfig.setLinkStateChangeRetryCount(mPositiveUpdateCount);
+    mMuxConfig.setOscillationInterval_sec(1,true);
 }
 
 void LinkManagerStateMachineTest::runIoService(uint32_t count)
@@ -1435,5 +1436,34 @@ TEST_F(LinkManagerStateMachineTest, MuxStandbyLinkProberStandbyLinkDownLinkUpRes
     VALIDATE_STATE(Standby, Standby, Up);
 }
 
+TEST_F(LinkManagerStateMachineTest, TimedOscillation)
+{
+    setMuxStandby();
+
+    // set icmp timeout to be 500ms otherwise it will probe mux state endlessly and get no chance to do timed oscillation
+    mMuxConfig.setTimeoutIpv4_msec(500);
+
+    postLinkProberEvent(link_prober::LinkProberState::Unknown, 2);
+    VALIDATE_STATE(Wait, Wait, Up);
+    EXPECT_EQ(mDbInterfacePtr->mSetMuxStateInvokeCount, 1);
+    EXPECT_EQ(mDbInterfacePtr->mLastPostedSwitchCause, link_manager::ActiveStandbyStateMachine::SwitchCause::PeerHeartbeatMissing);
+
+    // swss notification
+    handleMuxState("active", 3);
+    VALIDATE_STATE(Wait, Active, Up);
+
+    runIoService(2);
+    VALIDATE_STATE(Wait, Wait, Up);
+
+    handleProbeMuxState("active", 3);
+    VALIDATE_STATE(Wait, Active, Up);
+
+    runIoService(2);
+    VALIDATE_STATE(Wait, Wait, Up);
+    EXPECT_EQ(mDbInterfacePtr->mLastPostedSwitchCause, link_manager::ActiveStandbyStateMachine::SwitchCause::TimedOscillation);
+
+    mMuxConfig.setTimeoutIpv4_msec(10);
+}
+
 
 } /* namespace test */