From 76cacbfa9f98c0a32b52e45b9a05f684c71ec6b2 Mon Sep 17 00:00:00 2001
From: Greg Burri <greg.burri@gmail.com>
Date: Tue, 12 Dec 2023 07:50:33 +0100
Subject: [PATCH] Send an email after a certain amount of error

---
 src/main.rs | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/src/main.rs b/src/main.rs
index 60d0724..41a6312 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,5 +1,9 @@
 /*
  * API Reference: https://ethereum.github.io/beacon-APIs/
+ *
+ * TODO:
+ *  - Check that some processes (nethermind, lightouse) do not exceed a percentage
+ *    of total memory. For example 40 %.
  */
 
 use std::{
@@ -24,7 +28,12 @@ mod config;
 const FILE_CONF: &str = "config.ron";
 const CHECK_PERIOD: Duration = Duration::from_secs(10); // 10 s.
 const PING_TIMEOUT: Duration = Duration::from_secs(10); // 10 s.
+
 const NUMBER_SUCCESSIVE_ERRORS_SEND_EMAIL: i32 = 2; // Send an email after 2 successive errors.
+
+// Send an emergency email after 10 successive errors.
+const NUMBER_SUCCESSIVE_ERRORS_SEND_EMEGENCY_EMAIL: i32 = 10;
+
 const EMAIL_RESEND_PERIOD: Duration = Duration::from_secs(2 * 60 * 60); // 2 h.
 const STATE_PRINT_PERIOD: Duration = Duration::from_secs(15 * 60); // 15 min.
 const BASE_URI: &str = "http://localhost:5052/eth/v1/";
@@ -56,7 +65,7 @@ fn main() -> Result<()> {
             .and(check_alive_error_mutex.lock().unwrap().as_ref())
         {
             error_count += 1;
-            println!("Error (error_count={}): {:?}", error_count, error);
+            println!("Error {:?} (error_count={}): {}", error, error_count, error);
             if error_count >= NUMBER_SUCCESSIVE_ERRORS_SEND_EMAIL
                 && time::Instant::now() - time_last_email_send >= EMAIL_RESEND_PERIOD
             {
@@ -75,6 +84,25 @@ fn main() -> Result<()> {
                         time_last_email_send = time::Instant::now();
                     }
                 }
+            } else if error_count == NUMBER_SUCCESSIVE_ERRORS_SEND_EMEGENCY_EMAIL {
+                // Send e-mail.
+                println!("Sending EMERGENCY email...");
+                match send_email(
+                    "[EMERGENCY] Watchdog: Staking error",
+                    &format!(
+                        "[EMERGENCY] Error: {}\n\nNumber of error: {}",
+                        error, error_count
+                    ),
+                    &config.smtp_relay_address,
+                    &config.smtp_login,
+                    &config.smtp_password,
+                ) {
+                    Err(email_error) => println!("Error sending email: {:?}", email_error),
+                    _ => {
+                        println!("Email successfully sent");
+                        time_last_email_send = time::Instant::now();
+                    }
+                }
             }
         } else {
             if error_count > 0 {
-- 
2.45.1