[Tunnelbroker] retry FCM notif after receiving 401 Unauthorized

Summary: [ENG-9442](https://linear.app/comm/issue/ENG-9442/investigate-tunnelbrokerstagingfcmerroralarm).

Test Plan:
1. Android notifs works.
2. Simulate 401 and see if token was regenerated.

Reviewers: bartek

Reviewed By: bartek

Subscribers: ashoat, tomek

Differential Revision: https://phab.comm.dev/D13586

services/tunnelbroker/src/notifs/fcm/mod.rs

@@ -45,50 +45,57 @@ impl FCMClient {
     let token = message.token.clone();
     debug!("Sending FCM notif to {}", token);
 
-    let mut headers = HeaderMap::new();
-    headers.insert(
-      reqwest::header::CONTENT_TYPE,
-      HeaderValue::from_static("application/json"),
-    );
-
-    let bearer = self.token.get_auth_bearer().await?;
-    headers.insert(AUTHORIZATION, HeaderValue::from_str(&bearer)?);
-
-    let url = format!(
-      "https://fcm.googleapis.com/v1/projects/{}/messages:send",
-      self.config.project_id
-    );
-
     let msg_wrapper = FCMMessageWrapper { message };
     let payload = serde_json::to_string(&msg_wrapper).unwrap();
+    let mut is_retry = false;
 
-    let response = self
-      .http_client
-      .post(&url)
-      .headers(headers)
-      .body(payload)
-      .send()
-      .await?;
+    loop {
+      let mut headers = HeaderMap::new();
+      headers.insert(
+        reqwest::header::CONTENT_TYPE,
+        HeaderValue::from_static("application/json"),
+      );
+      let bearer = self.token.get_auth_bearer(is_retry).await?;
+      headers.insert(AUTHORIZATION, HeaderValue::from_str(&bearer)?);
 
-    match response.status() {
-      StatusCode::OK => {
-        debug!("Successfully sent FCM notif to {}", token);
-        Ok(())
-      }
-      error_status => {
-        let body = response
-          .text()
-          .await
-          .unwrap_or_else(|error| format!("Error occurred: {}", error));
-        error!(
-          errorType = error_types::FCM_ERROR,
-          "Failed sending FCM notification to: {}. Status: {}. Body: {}",
-          token,
-          error_status,
-          body
-        );
-        let fcm_error = FCMErrorResponse::from_status(error_status, body);
-        Err(FCMError(fcm_error))
+      let url = format!(
+        "https://fcm.googleapis.com/v1/projects/{}/messages:send",
+        self.config.project_id
+      );
+
+      let response = self
+        .http_client
+        .post(&url)
+        .headers(headers)
+        .body(payload.clone())
+        .send()
+        .await?;
+
+      match response.status() {
+        StatusCode::OK => {
+          debug!("Successfully sent FCM notif to {}", token);
+          return Ok(());
+        }
+        StatusCode::UNAUTHORIZED if !is_retry => {
+          is_retry = true;
+          debug!("Retrying after first 401 to regenerate token.");
+          continue;
+        }
+        error_status => {
+          let body = response
+            .text()
+            .await
+            .unwrap_or_else(|error| format!("Error occurred: {}", error));
+          error!(
+            errorType = error_types::FCM_ERROR,
+            "Failed sending FCM notification to: {}. Status: {}. Body: {}",
+            token,
+            error_status,
+            body
+          );
+          let fcm_error = FCMErrorResponse::from_status(error_status, body);
+          return Err(FCMError(fcm_error));
+        }
       }
     }
   }

services/tunnelbroker/src/notifs/fcm/token.rs

@@ -40,8 +40,11 @@ impl FCMToken {
     })
   }
 
-  pub async fn get_auth_bearer(&self) -> Result<String, Error> {
-    if self.fcm_token_needs_generation().await {
+  pub async fn get_auth_bearer(
+    &self,
+    force_regenerate: bool,
+  ) -> Result<String, Error> {
+    if force_regenerate || self.fcm_token_needs_generation().await {
       self.generate_fcm_token().await?;
     }