-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
22 changed files
with
162 additions
and
54 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
# -*- coding: utf-8 -*- | ||
|
||
help: ## Show this help message | ||
@perl -nle'print $& if m{^[a-zA-Z0-9_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-40s\033[0m %s\n", $$1, $$2}' | ||
|
||
|
||
update-gzip: ## Update all *.tsv.gz file | ||
python ./bin/update_gzip.py | ||
|
||
|
||
restore-tsv: ## Restore all *.tsv file | ||
python ./bin/restore_tsv.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
# -*- coding: utf-8 -*- | ||
|
||
from wotlkdoc.docs.dataframe import restore_tsv | ||
|
||
restore_tsv() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
# -*- coding: utf-8 -*- | ||
|
||
from wotlkdoc.docs.dataframe import update_gzip | ||
|
||
update_gzip() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
polars | ||
polars==0.14.5 | ||
pypinyin==0.47.1 | ||
rstobj==0.0.7 | ||
boto3==1.24.66 | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,54 +1,52 @@ | ||
# -*- coding: utf-8 -*- | ||
|
||
import logging | ||
|
||
try: | ||
from . import consumable | ||
from . import faction | ||
from . import mount | ||
from . import gps | ||
from . import images | ||
from ..df_to_list_table import df_to_list_table | ||
|
||
doc_data = dict( | ||
# consumable | ||
lt_consumable_vanilla=consumable.lt_consumable_vanilla, | ||
lt_consumable_tbc=consumable.lt_consumable_tbc, | ||
lt_consumable_wlk=consumable.lt_consumable_wlk, | ||
rst_list_item_enhancement=consumable.rst_list_item_enhancement, | ||
|
||
lt_faction_vanilla=faction.lt_faction_vanilla, | ||
lt_faction_tbc=faction.lt_faction_tbc, | ||
lt_faction_wlk=faction.lt_faction_wlk, | ||
lt_mount=mount.lt_mount, | ||
|
||
# gps zone | ||
lt_list_east_map_gps=gps.lt_list_east_map_gps, | ||
lt_list_kali_map_gps=gps.lt_list_kali_map_gps, | ||
lt_list_tbc_map_gps=gps.lt_list_tbc_map_gps, | ||
lt_list_wlk_map_gps=gps.lt_list_wlk_map_gps, | ||
|
||
# gps city | ||
lt_list_main_city_gps=gps.lt_list_main_city_gps, | ||
|
||
# gps class trainer | ||
lt_list_class_trainer_gsp=gps.lt_list_class_trainer_gsp, | ||
|
||
# gps trade skill trainer | ||
lt_list_trade_skill_trainer_gsp=gps.lt_list_trade_skill_trainer_gsp, | ||
|
||
# gps instance, dungeon, raid | ||
list_of_lt_list_instance_gps=gps.list_of_lt_list_instance_gps, | ||
|
||
func_df_to_list_table=df_to_list_table, | ||
|
||
# image links | ||
# maps | ||
img_map_tbc=images.img_map_tbc, | ||
img_map_wlk=images.img_map_wlk, | ||
img_map_cities=images.img_map_cities, | ||
) | ||
|
||
|
||
except Exception as e: | ||
logging.warning(str(e)) | ||
# import logging | ||
# | ||
# try: | ||
# from . import consumable | ||
# from . import faction | ||
# from . import mount | ||
# from . import gps | ||
# from . import images | ||
# from ..df_to_list_table import df_to_list_table | ||
# | ||
# doc_data = dict( | ||
# # consumable | ||
# lt_consumable_vanilla=consumable.lt_consumable_vanilla, | ||
# lt_consumable_tbc=consumable.lt_consumable_tbc, | ||
# lt_consumable_wlk=consumable.lt_consumable_wlk, | ||
# rst_list_item_enhancement=consumable.rst_list_item_enhancement, | ||
# | ||
# lt_faction_vanilla=faction.lt_faction_vanilla, | ||
# lt_faction_tbc=faction.lt_faction_tbc, | ||
# lt_faction_wlk=faction.lt_faction_wlk, | ||
# lt_mount=mount.lt_mount, | ||
# | ||
# # gps zone | ||
# lt_list_east_map_gps=gps.lt_list_east_map_gps, | ||
# lt_list_kali_map_gps=gps.lt_list_kali_map_gps, | ||
# lt_list_tbc_map_gps=gps.lt_list_tbc_map_gps, | ||
# lt_list_wlk_map_gps=gps.lt_list_wlk_map_gps, | ||
# | ||
# # gps city | ||
# lt_list_main_city_gps=gps.lt_list_main_city_gps, | ||
# | ||
# # gps class trainer | ||
# lt_list_class_trainer_gsp=gps.lt_list_class_trainer_gsp, | ||
# | ||
# # gps trade skill trainer | ||
# lt_list_trade_skill_trainer_gsp=gps.lt_list_trade_skill_trainer_gsp, | ||
# | ||
# # gps instance, dungeon, raid | ||
# list_of_lt_list_instance_gps=gps.list_of_lt_list_instance_gps, | ||
# | ||
# func_df_to_list_table=df_to_list_table, | ||
# | ||
# # image links | ||
# # maps | ||
# img_map_tbc=images.img_map_tbc, | ||
# img_map_wlk=images.img_map_wlk, | ||
# img_map_cities=images.img_map_cities, | ||
# ) | ||
# except Exception as e: | ||
# logging.warning(str(e)) |
Binary file not shown.
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
# -*- coding: utf-8 -*- | ||
|
||
""" | ||
本模块提供了对数据文件进行编辑, 管理, 引用的工作流. | ||
1. 把数据文件 check in 到代码库的时候, 存的是 .tsv.gz 文件. 你可以用 ``make update_gzip`` | ||
命令进行批量更新. | ||
2. 你编辑数据内容的时候, 是用 ``make restore`` 命令将 .tsv.gz 恢复成 .tsv 文件 | ||
(如果 .tsv 已经存在则认为你正在编辑, 不进行恢复), 然后 copy paste .tsv 文件的内容 | ||
到 Google Sheet 中编辑. 满意了再用 #1 中的步骤更新, 并 check in 到代码库. | ||
3. 对数据的 DataFrame 进行引用的时候用的是 :class:`TsvGzReader`` 类, 初始化的时候 | ||
只 pass 一个 ``reader = TsvGzReader(__file__)`` 参数, 让它能定位到你当前目录. | ||
然后用 df = reader.read_df("filename.tsv.gz") 来读取 DataFrame 数据. | ||
""" | ||
|
||
import gzip | ||
import attr | ||
from attrs_mate import AttrsClass | ||
from pathlib_mate import Path | ||
import polars as pl | ||
|
||
|
||
def gzip_compress(f_in: str, f_out: str): | ||
with gzip.open(f_out, "wb") as f: | ||
f.write(Path(f_in).read_bytes()) | ||
|
||
|
||
def gzip_decompress(f_in: str, f_out: str): | ||
with gzip.open(f_in, "rb") as f: | ||
Path(f_out).write_bytes(f.read()) | ||
|
||
|
||
dir_here = Path.dir_here(__file__) | ||
|
||
|
||
def update_gzip(): | ||
for path in dir_here.select_by_ext(recursive=True, ext=".tsv"): | ||
path_gz = Path(path.abspath + ".gz") | ||
gzip_compress(f_in=path.abspath, f_out=path_gz.abspath) | ||
|
||
|
||
def restore_tsv(): | ||
for path_gz in dir_here.select_by_ext(recursive=True, ext=".gz"): | ||
path_tsv = path_gz.change(new_basename=path_gz.fname) | ||
if not path_tsv.exists(): | ||
gzip_decompress(f_in=path_gz.abspath, f_out=path_tsv.abspath) | ||
|
||
|
||
@attr.s | ||
class TsvGzReader(AttrsClass): | ||
file: str = attr.ib() | ||
|
||
@property | ||
def dir_here(self) -> Path: | ||
return Path(self.file).parent | ||
|
||
def read_df(self, filename: str) -> pl.DataFrame: | ||
p = self.dir_here / filename | ||
return pl.read_csv(p.abspath, sep="\t") | ||
|
||
|
||
def example(): | ||
reader = TsvGzReader(__file__) | ||
print(reader.read_df("example-items.tsv.gz")) |
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Empty file.