From 8a9e30371ad8d43ee1970854b56a2ee69d78e483 Mon Sep 17 00:00:00 2001 From: ZuoSiZhu <2839299264@qq.com> Date: Sun, 25 May 2025 05:18:52 +0800 Subject: [PATCH] add: post process and plot draw to resource report script. fix: I hate args fix: allow config name only use opt: allow no resample rate args fix: copy mistakes opt: reduce dpi of plots Add: action_points , limit record counts for each mode fix: requirements upd upd: requirements --- requirements-in.txt | 1 + requirements.txt | 11 +- res_report.py | 305 +++++++++++++++++++++++++++++++++++++++----- 3 files changed, 281 insertions(+), 36 deletions(-) diff --git a/requirements-in.txt b/requirements-in.txt index 3dedf41d9..b5338cd46 100644 --- a/requirements-in.txt +++ b/requirements-in.txt @@ -25,6 +25,7 @@ pydantic aiofiles prettytable==2.2.1 anyio==1.3.1 +pandas==1.0.5 # Pushing onepush==1.4.0 diff --git a/requirements.txt b/requirements.txt index 72ed5c457..d520be5ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ # # pip-compile --annotation-style=line --output-file=requirements.txt requirements-in.txt # - adbutils==0.11.0 # via -r requirements-in.txt, uiautomator2 aiofiles==0.7.0 # via -r requirements-in.txt alas-webapp==0.3.7 # via -r requirements-in.txt @@ -46,10 +45,11 @@ lz4==3.1.3 # via -r requirements-in.txt matplotlib==3.4.3 # via gluoncv msgpack==1.0.3 # via zerorpc mxnet==1.6.0 # via -r requirements-in.txt, cnocr -numpy==1.16.6 # via -r requirements-in.txt, cnocr, gluoncv, imageio, matplotlib, mxnet, opencv-python, scipy +numpy==1.16.6 # via -r requirements-in.txt, cnocr, gluoncv, imageio, matplotlib, mxnet, opencv-python, pandas, scipy onepush==1.4.0 # via -r requirements-in.txt opencv-python==4.5.3.56 # via -r requirements-in.txt packaging==20.9 # via deprecation, uiautomator2 +pandas==1.0.5 # via -r requirements-in.txt pillow==8.3.2 # via -r requirements-in.txt, cnocr, gluoncv, imageio, matplotlib, uiautomator2 portalocker==2.3.2 # via gluoncv prettytable==2.2.1 # via -r requirements-in.txt @@ -57,14 +57,15 @@ progress==1.6 # via uiautomator2 psutil==5.9.3 # via -r requirements-in.txt py==1.10.0 # via retry pycparser==2.21 # via cffi -pycryptodome==3.9.9 # via onepush +pycryptodome==3.9.9 # via -r requirements-in.txt, onepush pydantic==1.10.2 # via -r requirements-in.txt pyelftools==0.27 # via apkutils2 pygments==2.12.0 # via rich pyparsing==2.4.7 # via matplotlib, packaging pypresence==4.2.1 # via -r requirements-in.txt -python-dateutil==2.8.2 # via matplotlib +python-dateutil==2.8.2 # via matplotlib, pandas python-dotenv==0.19.0 # via uvicorn +pytz==2025.2 # via pandas pywebio==1.6.2 # via -r requirements-in.txt pywin32==301 # via portalocker pyyaml==5.4.1 # via -r requirements-in.txt, uvicorn @@ -74,7 +75,7 @@ retry==0.9.2 # via adbutils, uiautomator2 retrying==1.3.3 # via -r requirements-in.txt rich==11.2.0 # via -r requirements-in.txt scipy==1.4.1 # via -r requirements-in.txt, gluoncv -six==1.16.0 # via adbutils, cycler, python-dateutil, retrying, uiautomator2 +six==1.16.0 # via adbutils, python-dateutil, retrying, uiautomator2 sniffio==1.2.0 # via anyio starlette==0.14.2 # via -r requirements-in.txt tornado==6.1 # via pywebio diff --git a/res_report.py b/res_report.py index e1bdfd649..688837aab 100644 --- a/res_report.py +++ b/res_report.py @@ -1,22 +1,40 @@ import csv import sys -from datetime import datetime +from datetime import datetime, timedelta import glob import os from typing import * +from copy import deepcopy +import pandas as pd +import matplotlib.pyplot as plt +from matplotlib.dates import DateFormatter + +from typing import Dict class ResourcesReport: class Record: - def __init__(self, time_str, value_str): - self.time = datetime.strptime(time_str, "%Y-%m-%d %H:%M:%S.%f") - self.value = int(value_str) + def __init__(self, time, value): + if isinstance(time, datetime): + self.time = time + if isinstance(time, str): + self.time = datetime.strptime(time, "%Y-%m-%d %H:%M:%S.%f") + self.value = float(value) CSV_HEADER = ["Time", "Value"] def __init__(self, config): - self.config = config - + self.records_to_add : Dict = {} + self.config: Dict = config + if self.config['resample_rate'] == 'H': + self.config['timedelta'] = timedelta(hours=1) + elif self.config['resample_rate'] == 'D': + self.config['timedelta'] = timedelta(days=1) + elif self.config['resample_rate'] == 'M': + self.config['timedelta'] = timedelta(minutes=1) + else: + exit("Invalid sample rate") + self.action_point_csv_records: List[ResourcesReport.Record] = list() self.oil_csv_records: List[ResourcesReport.Record] = list() self.gem_csv_records: List[ResourcesReport.Record] = list() self.coin_csv_records: List[ResourcesReport.Record] = list() @@ -25,6 +43,17 @@ class ResourcesReport: self.yellow_coin_csv_records: List[ResourcesReport.Record] = list() self.purple_coin_csv_records: List[ResourcesReport.Record] = list() self.player_exp_csv_records: List[ResourcesReport.Record] = list() + self.records_dict : Dict[str: List[ResourcesReport.Record]] = { + 'oil' :self.oil_csv_records, + 'gem' :self.gem_csv_records, + 'coin' :self.coin_csv_records, + 'cube' :self.cube_csv_records, + 'pt' :self.pt_csv_records, + 'yellow_coin' :self.yellow_coin_csv_records, + 'purple_coin' :self.purple_coin_csv_records, + 'player_exp' :self.player_exp_csv_records, + 'action_point' :self.action_point_csv_records, + } def process_lines(self, lines): for line in lines: @@ -82,20 +111,183 @@ class ResourcesReport: time_str, value_str = line_splited[0], line_splited[2].rstrip().split(" ")[2].split("/")[0] self.player_exp_csv_records.append(self.Record(time_str, value_str)) + # action_points + if line.find(" | INFO | Action points:") != -1: + line_splited = line.split(" | ") + time_str, value_str = line_splited[0], line_splited[2].rstrip().split(" ")[2] + value_str = value_str.split("(")[1].split(")")[0] + self.action_point_csv_records.append(self.Record(time_str, value_str)) + def sort_records(self): - self.oil_csv_records.sort(key=lambda x: x.time) - self.gem_csv_records.sort(key=lambda x: x.time) - self.coin_csv_records.sort(key=lambda x: x.time) - self.cube_csv_records.sort(key=lambda x: x.time) - self.pt_csv_records.sort(key=lambda x: x.time) - self.yellow_coin_csv_records.sort(key=lambda x: x.time) - self.purple_coin_csv_records.sort(key=lambda x: x.time) - self.player_exp_csv_records.sort(key=lambda x: x.time) + for name,records in self.records_dict.items(): + records.sort(key=lambda x: x.time) + + + def resample(self, records, name): + + + # 重要！！！我不知道怎么动起来的，搞不懂就别动！！！ + + + if self.config['resample_rate'] == 'D': + start_time = datetime(records[0].time.year, records[0].time.month, records[0].time.day)+self.config['timedelta'] + elif self.config['resample_rate'] == 'H': + start_time = datetime(records[0].time.year, records[0].time.month, records[0].time.day, records[0].time.hour)+self.config['timedelta'] + elif self.config['resample_rate'] == 'M': + start_time = datetime(records[0].time.year, records[0].time.month, records[0].time.day, records[0].time.hour, records[0].time.minute)+self.config['timedelta'] + _ = start_time + last_record = records[0] + records_copy = records.copy() + for record in records_copy: + if record.time > _ >= last_record.time: + while True: + if _ >= record.time: + break + if _ - self.config['timedelta']< last_record.time <= _: + _record = self.Record(_, last_record.value) + records.append(_record) + last_record = record + + _ += self.config['timedelta'] + _record = self.Record(_, record.value) + records.remove(record) + records.append(_record) + _ += self.config['timedelta'] + continue + if record.time <= _: + if record == records_copy[-1]: + _record = self.Record(_, record.value) + records.remove(record) + records.append(_record) + break + last_record = record + records.remove(record) + continue + + if name == 'gem': + d = 0 + records_copy = deepcopy(records) + l = len(records) + for _ in range(1,l): + record=records[_] + last_record = records[_-1] + delta_v = record.value - last_record.value + if delta_v > 150 or delta_v < 0: + d -= delta_v + records_copy[_].value = records_copy[_].value +d + self.records_to_add.update({'gem_modified' : records_copy}) + + if name == 'pt': + new_records = [] + new_count = 1 + l = len(records) + for _ in range(1,l): + record=records[_] + last_record = records[_-1] + new_records.append(last_record) + if record.value < last_record.value: + self.records_to_add.update({f'pt_event_{new_count}': new_records}) + new_count += 1 + new_records = [] + last_record = record + new_records.append(last_record) + if new_count > 1: + self.records_to_add.update({f'pt_event_{new_count}' : new_records}) + + + + def pre_draw_reshape(self, records): + inserted = 0 + records_copy = records.copy() + for _ in range(1,len(records_copy)): + if records_copy[_].time - records_copy[_-1].time > self.config['timedelta']: + time_delta = (records_copy[_].time - records_copy[_-1].time)//self.config['timedelta'] + value_delta = (records_copy[_].value - records_copy[_-1].value)/time_delta + for __ in range(1,time_delta): + records.insert(_+inserted, self.Record(records_copy[_-1].time + __*self.config['timedelta'] , + records_copy[_-1].value + __*value_delta)) + inserted += 1 + + def draw_plt(self, records, name): + png_dir = f"./report/{self.config['config']}/PNG" + if not os.path.exists(png_dir): + os.makedirs(png_dir) + df = pd.DataFrame({ + "time": [record.time for record in records], + name.capitalize(): [record.value for record in records] + }) + df.set_index("time", inplace=True) + + width_needed = len(records) * 0.0125 if self.config['config'] == "H" else len(records) * 0.3 + width_max = 2^32 + if width_needed > width_max: + width = width_max + else: + width = width_needed + width = max(10, width) + height = width/2 + ax = df.plot( + kind="line", + style="-", + figsize=(width, height), + title=f"{name.capitalize()} Over Time", + grid=True, + ) + ax.set_xlabel("Time") + ax.set_ylabel(name.capitalize()) + + plt.xticks(rotation=45) + plt.tight_layout() + + plt.savefig(f"{png_dir}/{name}.png", dpi=150, bbox_inches="tight") def post_process(self): - print(f"post process") - + print(f"Post process - Sorting") self.sort_records() + for name, records in self.records_dict.items(): + if not records: + continue + print(f"Post process - Resampling {name}") + self.resample(records, name) + if self.config['resample_rate'] == "M": + if len(records) >= 24 * 60: + # and name in [ + # 'oil', + # 'coin', + # 'gem', + # 'gem_modified', + # ]: + self.config['resample_rate'] = "H" + self.config['timedelta'] = timedelta(hours=1) + self.resample(records, name) + print(f'WARNING - {name.capitalize()} has too many records, forced resample to hourly statistics.') + self.config['resample_rate'] = "M" + self.config['timedelta'] = timedelta(minutes=1) + if self.config['resample_rate'] == "H": + if len(records)>=60*24: + # and name in [ + # 'oil', + # 'coin', + # 'gem', + # 'gem_modified', + # ]: + self.config['resample_rate'] = "D" + self.config['timedelta'] = timedelta(days=1) + self.resample(records, name) + print(f'WARNING - {name.capitalize()} has too many records, forced resample to daily statistics.') + self.config['resample_rate'] = "H" + self.config['timedelta'] = timedelta(hours=1) + self.records_dict.update(self.records_to_add) + # if 'pt_event_1' in self.records_dict.keys(): + # self.records_dict.pop('pt') + + for name, records in self.records_dict.items(): + if not records: + continue + + print(f"Post process - Plot Drawing {name}") + self.draw_plt(records, name) + def dump_single(self, f, l): with open(f, mode="w", encoding="utf-8", newline="") as fd: @@ -107,26 +299,25 @@ class ResourcesReport: ]) def dump_to_file(self): - config_report_dir = f"./report/{self.config}" + config_report_dir = f"./report/{self.config['config']}" if not os.path.exists(config_report_dir): os.makedirs(config_report_dir) + print(f"Finishing - Generate report to {config_report_dir}") - print(f"gen report to {config_report_dir}") + csv_dir = f"./report/{self.config['config']}/CSV" + if not os.path.exists(csv_dir): + os.makedirs(csv_dir) - self.dump_single(f"{config_report_dir}/oil.csv", self.oil_csv_records) - self.dump_single(f"{config_report_dir}/gem.csv", self.gem_csv_records) - self.dump_single(f"{config_report_dir}/coin.csv", self.coin_csv_records) - self.dump_single(f"{config_report_dir}/cube.csv", self.cube_csv_records) - self.dump_single(f"{config_report_dir}/pt.csv", self.pt_csv_records) - self.dump_single(f"{config_report_dir}/yellow_coin.csv", self.yellow_coin_csv_records) - self.dump_single(f"{config_report_dir}/purple_coin.csv", self.purple_coin_csv_records) - self.dump_single(f"{config_report_dir}/player_exp.csv", self.player_exp_csv_records) + for name, records in self.records_dict.items(): + self.dump_single(f"{csv_dir}/{name}.csv", records) + + print(f"Finished - Report generated") def gen_res_report(self): - l = glob.glob(f"./log/*_{self.config}.txt") + l = glob.glob(f"./log/*_{self.config['config']}.txt") for log in l: - print(f"process {log}") + print(f"Line process - {log} - {os.stat(log).st_size} Bytes") with open(log, mode="r", encoding="utf-8") as f: lines = f.readlines() @@ -136,11 +327,63 @@ class ResourcesReport: def main(): - if len(sys.argv) == 1: - print("Please specify the configuration name !") + if len(sys.argv) < 2: + print("Invalid args.") exit(-1) + _ = 1 + conf = { + "config": "", + "resample_rate": "" + } + while _ < len(sys.argv): + arg = sys.argv[_] + if arg in ["--config-name", "--config", "-c"]: + if _+1 >= len(sys.argv): + exit("Invalid args.") + conf["config"] = sys.argv[_+1] + _ += 2 + continue + if arg in ["--resample-rate","-r"]: + if _+1 >= len(sys.argv): + exit("Invalid resample rate.") + conf["resample_rate"] = sys.argv[_+1] + _ += 2 + continue + if arg.upper() in ["M","D","H","HOUR","MINUTE","DAY","24"] and len(sys.argv) >2 and conf["resample_rate"] == ""\ + and _+1 < len(sys.argv) and sys.argv[_+1].upper() not in ["M","D","HOUR","MINUTE","DAY","24"]\ + and sys.argv[_+1] not in ["--resample-rate","-r"]: + conf["resample_rate"] = arg + _ += 1 + continue + if arg.upper() in ["M","D","H","HOUR","MINUTE","DAY","24"] and conf["config"] != ""\ + and _+1 == len(sys.argv) : + conf["resample_rate"] = arg + _ += 1 + continue + if conf['config'] == "": + conf["config"] = arg + _ += 1 + else: + exit("Invalid args.") + if conf["resample_rate"].upper() in ["M",'MINUTE']: + conf["resample_rate"] = 'M' + elif conf["resample_rate"].upper() in ["H",'HOUR','']: + conf["resample_rate"] = 'H' + elif conf["resample_rate"] .upper() in ["D","DAY",'24']: + conf["resample_rate"] = 'D' + else: + exit("Invalid resample rate.") - reporter = ResourcesReport(sys.argv[1]) + rate ={ + 'M' : 'Per Minute', + 'D' : 'Per Day', + 'H' : 'Per Hour', + } + + print(f"Starting with config: \n" + f"Config name : {conf['config']}\n" + f"Resample rate: {rate[conf['resample_rate']]}") + reporter = ResourcesReport(conf) reporter.gen_res_report()