add: post process and plot draw to resource report script.

fix: I hate args

fix: allow config name only use

opt: allow no resample rate args

fix: copy mistakes

opt: reduce dpi of plots

Add: action_points , limit record counts for each mode

fix: requirements upd

upd: requirements

requirements-in.txt

@@ -25,6 +25,7 @@ pydantic
 aiofiles
 prettytable==2.2.1
 anyio==1.3.1
+pandas==1.0.5
 
 # Pushing
 onepush==1.4.0

requirements.txt

@@ -4,7 +4,6 @@
 #
 #    pip-compile --annotation-style=line --output-file=requirements.txt requirements-in.txt
 #
-
 adbutils==0.11.0          # via -r requirements-in.txt, uiautomator2
 aiofiles==0.7.0           # via -r requirements-in.txt
 alas-webapp==0.3.7        # via -r requirements-in.txt
@@ -46,10 +45,11 @@ lz4==3.1.3                # via -r requirements-in.txt
 matplotlib==3.4.3         # via gluoncv
 msgpack==1.0.3            # via zerorpc
 mxnet==1.6.0              # via -r requirements-in.txt, cnocr
-numpy==1.16.6             # via -r requirements-in.txt, cnocr, gluoncv, imageio, matplotlib, mxnet, opencv-python, scipy
+numpy==1.16.6             # via -r requirements-in.txt, cnocr, gluoncv, imageio, matplotlib, mxnet, opencv-python, pandas, scipy
 onepush==1.4.0            # via -r requirements-in.txt
 opencv-python==4.5.3.56   # via -r requirements-in.txt
 packaging==20.9           # via deprecation, uiautomator2
+pandas==1.0.5             # via -r requirements-in.txt
 pillow==8.3.2             # via -r requirements-in.txt, cnocr, gluoncv, imageio, matplotlib, uiautomator2
 portalocker==2.3.2        # via gluoncv
 prettytable==2.2.1        # via -r requirements-in.txt
@@ -57,14 +57,15 @@ progress==1.6             # via uiautomator2
 psutil==5.9.3             # via -r requirements-in.txt
 py==1.10.0                # via retry
 pycparser==2.21           # via cffi
-pycryptodome==3.9.9       # via onepush
+pycryptodome==3.9.9       # via -r requirements-in.txt, onepush
 pydantic==1.10.2          # via -r requirements-in.txt
 pyelftools==0.27          # via apkutils2
 pygments==2.12.0          # via rich
 pyparsing==2.4.7          # via matplotlib, packaging
 pypresence==4.2.1         # via -r requirements-in.txt
-python-dateutil==2.8.2    # via matplotlib
+python-dateutil==2.8.2    # via matplotlib, pandas
 python-dotenv==0.19.0     # via uvicorn
+pytz==2025.2              # via pandas
 pywebio==1.6.2            # via -r requirements-in.txt
 pywin32==301              # via portalocker
 pyyaml==5.4.1             # via -r requirements-in.txt, uvicorn
@@ -74,7 +75,7 @@ retry==0.9.2              # via adbutils, uiautomator2
 retrying==1.3.3           # via -r requirements-in.txt
 rich==11.2.0              # via -r requirements-in.txt
 scipy==1.4.1              # via -r requirements-in.txt, gluoncv
-six==1.16.0               # via adbutils, cycler, python-dateutil, retrying, uiautomator2
+six==1.16.0               # via adbutils, python-dateutil, retrying, uiautomator2
 sniffio==1.2.0            # via anyio
 starlette==0.14.2         # via -r requirements-in.txt
 tornado==6.1              # via pywebio

res_report.py

@@ -1,22 +1,40 @@
 import csv
 import sys
-from datetime import datetime
+from datetime import datetime, timedelta
 import glob
 import os
 from typing import *
+from copy import deepcopy
+import pandas as pd
+import matplotlib.pyplot as plt
+from matplotlib.dates import DateFormatter
+
+from typing import Dict
 
 
 class ResourcesReport:
     class Record:
-        def __init__(self, time_str, value_str):
-            self.time = datetime.strptime(time_str, "%Y-%m-%d %H:%M:%S.%f")
-            self.value = int(value_str)
+        def __init__(self, time, value):
+            if isinstance(time, datetime):
+                self.time = time
+            if isinstance(time, str):
+                self.time = datetime.strptime(time, "%Y-%m-%d %H:%M:%S.%f")
+            self.value = float(value)
 
     CSV_HEADER = ["Time", "Value"]
 
     def __init__(self, config):
-        self.config = config
-
+        self.records_to_add : Dict = {}
+        self.config: Dict = config
+        if self.config['resample_rate'] == 'H':
+            self.config['timedelta'] = timedelta(hours=1)
+        elif self.config['resample_rate'] == 'D':
+            self.config['timedelta'] = timedelta(days=1)
+        elif self.config['resample_rate'] == 'M':
+            self.config['timedelta'] = timedelta(minutes=1)
+        else:
+            exit("Invalid sample rate")
+        self.action_point_csv_records: List[ResourcesReport.Record] = list()
         self.oil_csv_records: List[ResourcesReport.Record] = list()
         self.gem_csv_records: List[ResourcesReport.Record] = list()
         self.coin_csv_records: List[ResourcesReport.Record] = list()
@@ -25,6 +43,17 @@ class ResourcesReport:
         self.yellow_coin_csv_records: List[ResourcesReport.Record] = list()
         self.purple_coin_csv_records: List[ResourcesReport.Record] = list()
         self.player_exp_csv_records: List[ResourcesReport.Record] = list()
+        self.records_dict : Dict[str: List[ResourcesReport.Record]] = {
+            'oil'           :self.oil_csv_records,
+            'gem'           :self.gem_csv_records,
+            'coin'          :self.coin_csv_records,
+            'cube'          :self.cube_csv_records,
+            'pt'            :self.pt_csv_records,
+            'yellow_coin'   :self.yellow_coin_csv_records,
+            'purple_coin'   :self.purple_coin_csv_records,
+            'player_exp'    :self.player_exp_csv_records,
+            'action_point'  :self.action_point_csv_records,
+        }
 
     def process_lines(self, lines):
         for line in lines:
@@ -82,20 +111,183 @@ class ResourcesReport:
                 time_str, value_str = line_splited[0], line_splited[2].rstrip().split(" ")[2].split("/")[0]
                 self.player_exp_csv_records.append(self.Record(time_str, value_str))
 
+            #  action_points
+            if line.find(" | INFO | Action points:") != -1:
+                line_splited = line.split(" | ")
+                time_str, value_str = line_splited[0], line_splited[2].rstrip().split(" ")[2]
+                value_str = value_str.split("(")[1].split(")")[0]
+                self.action_point_csv_records.append(self.Record(time_str, value_str))
+
     def sort_records(self):
-        self.oil_csv_records.sort(key=lambda x: x.time)
-        self.gem_csv_records.sort(key=lambda x: x.time)
-        self.coin_csv_records.sort(key=lambda x: x.time)
-        self.cube_csv_records.sort(key=lambda x: x.time)
-        self.pt_csv_records.sort(key=lambda x: x.time)
-        self.yellow_coin_csv_records.sort(key=lambda x: x.time)
-        self.purple_coin_csv_records.sort(key=lambda x: x.time)
-        self.player_exp_csv_records.sort(key=lambda x: x.time)
+        for name,records in self.records_dict.items():
+            records.sort(key=lambda x: x.time)
+
+
+    def resample(self, records, name):
+
+
+        # 重要！！！我不知道怎么动起来的，搞不懂就别动！！！
+
+
+        if self.config['resample_rate'] == 'D':
+            start_time = datetime(records[0].time.year, records[0].time.month, records[0].time.day)+self.config['timedelta']
+        elif self.config['resample_rate'] == 'H':
+            start_time = datetime(records[0].time.year, records[0].time.month, records[0].time.day, records[0].time.hour)+self.config['timedelta']
+        elif self.config['resample_rate'] == 'M':
+            start_time = datetime(records[0].time.year, records[0].time.month, records[0].time.day, records[0].time.hour, records[0].time.minute)+self.config['timedelta']
+        _ = start_time
+        last_record = records[0]
+        records_copy = records.copy()
+        for record in records_copy:
+            if record.time > _ >= last_record.time:
+                while True:
+                    if _ >= record.time:
+                        break
+                    if _ - self.config['timedelta']< last_record.time <= _:
+                        _record = self.Record(_, last_record.value)
+                        records.append(_record)
+                        last_record = record
+
+                    _ += self.config['timedelta']
+                _record = self.Record(_, record.value)
+                records.remove(record)
+                records.append(_record)
+                _ += self.config['timedelta']
+                continue
+            if record.time <= _:
+                if record == records_copy[-1]:
+                    _record = self.Record(_, record.value)
+                    records.remove(record)
+                    records.append(_record)
+                    break
+                last_record = record
+                records.remove(record)
+                continue
+
+        if name == 'gem':
+            d = 0
+            records_copy = deepcopy(records)
+            l = len(records)
+            for _ in range(1,l):
+                record=records[_]
+                last_record = records[_-1]
+                delta_v = record.value - last_record.value
+                if delta_v > 150 or delta_v < 0:
+                    d -= delta_v
+                records_copy[_].value = records_copy[_].value +d
+            self.records_to_add.update({'gem_modified' : records_copy})
+
+        if name == 'pt':
+            new_records = []
+            new_count = 1
+            l = len(records)
+            for _ in range(1,l):
+                record=records[_]
+                last_record = records[_-1]
+                new_records.append(last_record)
+                if record.value < last_record.value:
+                    self.records_to_add.update({f'pt_event_{new_count}': new_records})
+                    new_count += 1
+                    new_records = []
+                    last_record = record
+            new_records.append(last_record)
+            if new_count > 1:
+                self.records_to_add.update({f'pt_event_{new_count}' : new_records})
+
+
+
+    def pre_draw_reshape(self, records):
+        inserted = 0
+        records_copy = records.copy()
+        for _ in range(1,len(records_copy)):
+            if records_copy[_].time - records_copy[_-1].time > self.config['timedelta']:
+                time_delta = (records_copy[_].time - records_copy[_-1].time)//self.config['timedelta']
+                value_delta = (records_copy[_].value - records_copy[_-1].value)/time_delta
+                for __ in range(1,time_delta):
+                    records.insert(_+inserted, self.Record(records_copy[_-1].time  + __*self.config['timedelta'] ,
+                                                           records_copy[_-1].value + __*value_delta))
+                    inserted += 1
+
+    def draw_plt(self, records, name):
+        png_dir = f"./report/{self.config['config']}/PNG"
+        if not os.path.exists(png_dir):
+            os.makedirs(png_dir)
+        df = pd.DataFrame({
+            "time": [record.time for record in records],
+            name.capitalize(): [record.value for record in records]
+        })
+        df.set_index("time", inplace=True)
+
+        width_needed = len(records) * 0.0125 if self.config['config'] == "H" else len(records) * 0.3
+        width_max = 2^32
+        if width_needed > width_max:
+            width = width_max
+        else:
+            width = width_needed
+        width = max(10, width)
+        height = width/2
+        ax = df.plot(
+            kind="line",
+            style="-",
+            figsize=(width, height),
+            title=f"{name.capitalize()} Over Time",
+            grid=True,
+        )
+        ax.set_xlabel("Time")
+        ax.set_ylabel(name.capitalize())
+
+        plt.xticks(rotation=45)
+        plt.tight_layout()
+
+        plt.savefig(f"{png_dir}/{name}.png", dpi=150, bbox_inches="tight")
 
     def post_process(self):
-        print(f"post process")
-
+        print(f"Post process - Sorting")
         self.sort_records()
+        for name, records in self.records_dict.items():
+            if not records:
+                continue
+            print(f"Post process - Resampling {name}")
+            self.resample(records, name)
+            if self.config['resample_rate'] == "M":
+                if len(records) >= 24 * 60:
+                #         and name in [
+                #     'oil',
+                #     'coin',
+                #     'gem',
+                #     'gem_modified',
+                # ]:
+                    self.config['resample_rate'] = "H"
+                    self.config['timedelta'] = timedelta(hours=1)
+                    self.resample(records, name)
+                    print(f'WARNING      - {name.capitalize()} has too many records, forced resample to hourly statistics.')
+                    self.config['resample_rate'] = "M"
+                    self.config['timedelta'] = timedelta(minutes=1)
+            if self.config['resample_rate'] == "H":
+                if len(records)>=60*24:
+                        # and name in [
+                        #             'oil',
+                        #             'coin',
+                        #             'gem',
+                        #             'gem_modified',
+                        #             ]:
+                    self.config['resample_rate'] = "D"
+                    self.config['timedelta'] = timedelta(days=1)
+                    self.resample(records, name)
+                    print(f'WARNING      - {name.capitalize()} has too many records, forced resample to daily statistics.')
+                    self.config['resample_rate'] = "H"
+                    self.config['timedelta'] = timedelta(hours=1)
+        self.records_dict.update(self.records_to_add)
+        # if 'pt_event_1' in self.records_dict.keys():
+        #      self.records_dict.pop('pt')
+
+        for name, records in self.records_dict.items():
+            if not records:
+                continue
+
+            print(f"Post process - Plot Drawing {name}")
+            self.draw_plt(records, name)
+
 
     def dump_single(self, f, l):
         with open(f, mode="w", encoding="utf-8", newline="") as fd:
@@ -107,26 +299,25 @@ class ResourcesReport:
             ])
 
     def dump_to_file(self):
-        config_report_dir = f"./report/{self.config}"
+        config_report_dir = f"./report/{self.config['config']}"
         if not os.path.exists(config_report_dir):
             os.makedirs(config_report_dir)
+        print(f"Finishing    - Generate report to {config_report_dir}")
 
-        print(f"gen report to {config_report_dir}")
+        csv_dir = f"./report/{self.config['config']}/CSV"
+        if not os.path.exists(csv_dir):
+            os.makedirs(csv_dir)
 
-        self.dump_single(f"{config_report_dir}/oil.csv", self.oil_csv_records)
-        self.dump_single(f"{config_report_dir}/gem.csv", self.gem_csv_records)
-        self.dump_single(f"{config_report_dir}/coin.csv", self.coin_csv_records)
-        self.dump_single(f"{config_report_dir}/cube.csv", self.cube_csv_records)
-        self.dump_single(f"{config_report_dir}/pt.csv", self.pt_csv_records)
-        self.dump_single(f"{config_report_dir}/yellow_coin.csv", self.yellow_coin_csv_records)
-        self.dump_single(f"{config_report_dir}/purple_coin.csv", self.purple_coin_csv_records)
-        self.dump_single(f"{config_report_dir}/player_exp.csv", self.player_exp_csv_records)
+        for name, records in self.records_dict.items():
+            self.dump_single(f"{csv_dir}/{name}.csv", records)
+
+        print(f"Finished     - Report generated")
 
     def gen_res_report(self):
-        l = glob.glob(f"./log/*_{self.config}.txt")
+        l = glob.glob(f"./log/*_{self.config['config']}.txt")
 
         for log in l:
-            print(f"process {log}")
+            print(f"Line process - {log} - {os.stat(log).st_size} Bytes")
 
             with open(log, mode="r", encoding="utf-8") as f:
                 lines = f.readlines()
@@ -136,11 +327,63 @@ class ResourcesReport:
 
 
 def main():
-    if len(sys.argv) == 1:
-        print("Please specify the configuration name !")
+    if len(sys.argv) < 2:
+        print("Invalid args.")
         exit(-1)
+    _ = 1
+    conf = {
+        "config": "",
+        "resample_rate": ""
+    }
+    while _ < len(sys.argv):
+        arg = sys.argv[_]
+        if arg in ["--config-name", "--config", "-c"]:
+            if _+1 >=  len(sys.argv):
+                exit("Invalid args.")
+            conf["config"] = sys.argv[_+1]
+            _ += 2
+            continue
+        if arg in ["--resample-rate","-r"]:
+            if _+1 >=  len(sys.argv):
+                exit("Invalid resample rate.")
+            conf["resample_rate"] = sys.argv[_+1]
+            _ += 2
+            continue
+        if arg.upper() in ["M","D","H","HOUR","MINUTE","DAY","24"] and len(sys.argv) >2 and conf["resample_rate"] == ""\
+           and  _+1 < len(sys.argv) and sys.argv[_+1].upper() not in ["M","D","HOUR","MINUTE","DAY","24"]\
+           and sys.argv[_+1] not in ["--resample-rate","-r"]:
+            conf["resample_rate"] = arg
+            _ += 1
+            continue
+        if arg.upper() in ["M","D","H","HOUR","MINUTE","DAY","24"] and conf["config"] != ""\
+           and _+1 == len(sys.argv) :
+            conf["resample_rate"] = arg
+            _ += 1
+            continue
+        if conf['config'] == "":
+            conf["config"] = arg
+            _ += 1
+        else:
+            exit("Invalid args.")
+    if conf["resample_rate"].upper() in ["M",'MINUTE']:
+        conf["resample_rate"] = 'M'
+    elif conf["resample_rate"].upper() in ["H",'HOUR','']:
+        conf["resample_rate"] = 'H'
+    elif conf["resample_rate"] .upper() in ["D","DAY",'24']:
+        conf["resample_rate"] = 'D'
+    else:
+        exit("Invalid resample rate.")
 
-    reporter = ResourcesReport(sys.argv[1])
+    rate ={
+        'M' : 'Per Minute',
+        'D' : 'Per Day',
+        'H' : 'Per Hour',
+    }
+
+    print(f"Starting with config: \n"
+          f"Config name  : {conf['config']}\n"
+          f"Resample rate: {rate[conf['resample_rate']]}")
+    reporter = ResourcesReport(conf)
     reporter.gen_res_report()