add: add instance watcher support

module/instance_watcher.py

@@ -0,0 +1,86 @@
+import threading
+import time
+from dataclasses import dataclass
+from typing import Dict
+
+from module.config.config import AzurLaneConfig
+from module.counter import MaxCounter
+from module.notify import handle_notify
+from module.webui.process_manager import ProcessManager
+
+
+@dataclass
+class InstanceSetting:
+    counter: MaxCounter
+    enable_notify: bool
+    notify_config: str
+
+
+class InstanceWatcher:
+    instance = None
+
+    def __init__(self):
+        self.watcher: threading.Thread = None
+        self.instances: Dict[str, InstanceSetting] = dict()
+
+    def start(self):
+        if self.watcher is not None:
+            if self.watcher.is_alive():
+                return
+        self.watcher = threading.Thread(target=self.watcher_thread, daemon=True)
+        self.watcher.start()
+
+    def check_instances(self):
+        ins_has_triggered = []
+        for name, setting in self.instances.items():
+            ins = ProcessManager.get_manager(name)
+            if not ins.alive:
+                if setting.counter.count_once(throw=False):
+                    ins.start("alas")
+                    if setting.enable_notify:
+                        handle_notify(
+                            setting.notify_config,
+                            title=f"Alas <{ins.config_name}> instance auto restarted",
+                            content=f"Critical error occurred, instance restarted",
+                        )
+                else:
+                    ins_has_triggered.append(name)
+                    handle_notify(
+                        setting.notify_config,
+                        title=f"Alas <{ins.config_name}> instance restarted too many times",
+                        content=f"Too many critical error occurred, instance restarted too many times",
+                    )
+
+        for i in ins_has_triggered:
+            self.remove_instance(i)
+
+    def try_add_instance(self, name):
+        config = AzurLaneConfig(name)
+        full_config = config.full_config
+        if full_config.Restart_InstanceRestart_Enable:
+            setting = InstanceSetting(
+                counter=MaxCounter(full_config.Restart_InstanceRestart_MaxRetryTimes),
+                enable_notify=full_config.Restart_InstanceRestart_Notify,
+                notify_config=full_config.Alas_Error_OnePushConfig
+            )
+            self.instances[name] = setting
+
+    def remove_instance(self, name):
+        try:
+            self.instances.pop(name)
+        except Exception:
+            ...
+
+    def watcher_thread(self):
+        while 1:
+            time.sleep(60)
+            try:
+                self.check_instances()
+            except Exception:
+                ...
+
+    @staticmethod
+    def get_instance() -> "InstanceWatcher":
+        if InstanceWatcher.instance is None:
+            InstanceWatcher.instance = InstanceWatcher()
+        return InstanceWatcher.instance

module/webui/app.py

@@ -1525,3 +1525,119 @@ def app():
     )
 
     return app
+
+
+from module.instance_watcher import InstanceWatcher
+
+
+class AlasGUI(AlasGUI):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.instance_watcher = InstanceWatcher.get_instance()
+
+    def on_start_on(self):
+        self.instance_watcher.remove_instance(self.alas.config_name)
+        self.alas.stop()
+
+    def on_start_off(self):
+        self.alas.start(None, updater.event)
+        self.instance_watcher.try_add_instance(self.alas.config_name)
+
+    @use_scope("content", clear=True)
+    def alas_overview(self) -> None:
+        self.init_menu(name="Overview")
+        self.set_title(t(f"Gui.MenuAlas.Overview"))
+
+        put_scope("overview", [put_scope("schedulers"), put_scope("logs")])
+
+        with use_scope("schedulers"):
+            put_scope(
+                "scheduler-bar",
+                [
+                    put_text(t("Gui.Overview.Scheduler")).style(
+                        "font-size: 1.25rem; margin: auto .5rem auto;"
+                    ),
+                    put_scope("scheduler_btn"),
+                ],
+            )
+            put_scope(
+                "running",
+                [
+                    put_text(t("Gui.Overview.Running")),
+                    put_html('<hr class="hr-group">'),
+                    put_scope("running_tasks"),
+                ],
+            )
+            put_scope(
+                "pending",
+                [
+                    put_text(t("Gui.Overview.Pending")),
+                    put_html('<hr class="hr-group">'),
+                    put_scope("pending_tasks"),
+                ],
+            )
+            put_scope(
+                "waiting",
+                [
+                    put_text(t("Gui.Overview.Waiting")),
+                    put_html('<hr class="hr-group">'),
+                    put_scope("waiting_tasks"),
+                ],
+            )
+
+        switch_scheduler = BinarySwitchButton(
+            label_on=t("Gui.Button.Stop"),
+            label_off=t("Gui.Button.Start"),
+            onclick_on=self.on_start_on,
+            onclick_off=self.on_start_off,
+            get_state=lambda: self.alas.alive,
+            color_on="off",
+            color_off="on",
+            scope="scheduler_btn",
+        )
+
+        log = RichLog("log")
+
+        with use_scope("logs"):
+            put_scope(
+                "log-bar",
+                [
+                    put_text(t("Gui.Overview.Log")).style(
+                        "font-size: 1.25rem; margin: auto .5rem auto;"
+                    ),
+                    put_scope(
+                        "log-bar-btns",
+                        [
+                            put_scope("log_scroll_btn"),
+                        ],
+                    ),
+                ],
+            )
+            put_scope("log", [put_html("")])
+
+        log.console.width = log.get_width()
+
+        switch_log_scroll = BinarySwitchButton(
+            label_on=t("Gui.Button.ScrollON"),
+            label_off=t("Gui.Button.ScrollOFF"),
+            onclick_on=lambda: log.set_scroll(False),
+            onclick_off=lambda: log.set_scroll(True),
+            get_state=lambda: log.keep_bottom,
+            color_on="on",
+            color_off="off",
+            scope="log_scroll_btn",
+        )
+
+        self.task_handler.add(switch_scheduler.g(), 1, True)
+        self.task_handler.add(switch_log_scroll.g(), 1, True)
+        self.task_handler.add(self.alas_update_overview_task, 10, True)
+        self.task_handler.add(log.put_log(self.alas), 0.25, True)
+
+
+old_app = app
+
+
+def app():
+    ret = old_app()
+    InstanceWatcher.get_instance().start()
+    return ret