feat(mis): charge_record 表增加字段 user_id 及 metadata 以存储更多的信息 (#1072)

## 一、charge_record 表增加字段 user_id 及 metadata charge 接口增加可选入参 user_id charge 接口增加可选入参 metadata ## 二、在各消费记录页面中增加以下内容 1.查询结果中的用户显示 2.按用户ID检索 3.增加可选导出用户ID功能 ![image](https://github.com/PKUHPC/SCOW/assets/43978285/1c1a43d3-de3e-491c-879e-c2bbd89dcedb) ![image](https://github.com/PKUHPC/SCOW/assets/43978285/6ae69f70-5622-4fb3-bba0-7d6dc4e36344) ![image](https://github.com/PKUHPC/SCOW/assets/43978285/dd8e42e5-d162-47b2-85ff-32aabc6c6cb5) ## 三、mis.yaml中增加jobChargeMetadata配置 ``` # 自定义消费表单其他存储内容 jobChargeMetadata: # 可选。需要保存的作业的字段，字段参考src/entities/JobInfo savedFields: ["idJob", "account"] # 可选。定义显示出来的格式。如果不配置，直接显示上面保存的字段的信息 displayFormats: # i18n格式，在不同语言下，把元数据按不同格式显示出来 # 或字符串格式，直接显示字符串 i18n: default: "账户 {{ account }} 的作业ID {{ idJob }} 的计费" en: "The billing for jobId {{ idJob }} of Account {{ account }}" zh_cn: "账户 {{ account }} 的作业ID {{ idJob }} 的计费" ``` ## metadata前端展示效果 ### 1.没有配置jobChargeMetadata中的saveFields，不展示相关信息 ![image](https://github.com/PKUHPC/SCOW/assets/43978285/bd67cc4e-4c69-4244-bd77-7979940b6b11) ### 2.已配置jobChargeMetadata中的saveFields ### (1).displayFormats的类型为i18n对象时 ![image](https://github.com/PKUHPC/SCOW/assets/43978285/cdd8628f-9a6b-4e16-a40c-23fff8a383ae) ![image](https://github.com/PKUHPC/SCOW/assets/43978285/b66c76bd-fac0-4aa9-87cd-abfad52deecf) ### (2).displayFormats的类型为字符串时（支持使用占位符 {{ 属性名 }}） ![image](https://github.com/PKUHPC/SCOW/assets/43978285/5f1e1603-492b-4a1c-a5ea-5d3c70ffa9b3) ### (3).没有配置displayFormats时 ![image](https://github.com/PKUHPC/SCOW/assets/43978285/18873592-98da-412f-9bc1-e31d979254a8) ### 3.特殊情况：配置的jobChargeMetadata发生改动之前保存的信息与当前displayFormats占位属性不一致，则之前没有保存过的占位属性显示为"-" ![image](https://github.com/PKUHPC/SCOW/assets/43978285/bc6d2acf-59f3-4d76-a9b6-3b90e871f7b9) ### 上述字段及接口的修改，测试时需要注意的地方为 1. 同步成功作业扣费时 2. 修改作业价格时 3. 其他服务调用scow计费api时 --------- Co-authored-by: picca Sun <piccasun@gmail.com>
PKUHPC · Jan 30, 2024 · afc3350 · afc3350
1 parent c1ce9e4
commit afc3350
Show file tree

Hide file tree

Showing 36 changed files with 559 additions and 47 deletions.
diff --git a/.changeset/late-snakes-fold.md b/.changeset/late-snakes-fold.md
@@ -0,0 +1,6 @@
+---
+"@scow/lib-config": patch
+"@scow/config": patch
+---
+
+在 mis.yaml 中增加 jobChargeMetadata 可选配置可记录需要存储的扣费作业的字段信息
diff --git a/.changeset/loud-doors-rescue.md b/.changeset/loud-doors-rescue.md
@@ -0,0 +1,5 @@
+---
+"@scow/mis-server": patch
+---
+
+charge_record 表增加字段 user_id 及 metadata, 以及增加了 time,tenant,account,user_id,type 各字段的索引
diff --git a/.changeset/nasty-garlics-help.md b/.changeset/nasty-garlics-help.md
@@ -0,0 +1,6 @@
+---
+"@scow/mis-server": patch
+"@scow/mis-web": patch
+---
+
+增加消费记录中用户的显示、筛选及导出功能
diff --git a/.changeset/slimy-bulldogs-allow.md b/.changeset/slimy-bulldogs-allow.md
@@ -0,0 +1,5 @@
+---
+"@scow/grpc-api": minor
+---
+
+charge接口增加可选入参user_id,metadata, 消费记录查询接口增加查询参数user_ids,增加返回值user_id与metadata
diff --git a/apps/cli/assets/init-full/config/mis.yaml b/apps/cli/assets/init-full/config/mis.yaml
@@ -116,3 +116,40 @@ createUser:
 
 # 用户是否可以修改作业时限配置
 # allowUserChangeJobTimeLimit: true
+
+# # 新增自定义收费规则
+# customAmountStrategies:
+#   # 计费方式id，请勿重复，重复的话后面的计费方式将会覆盖前面的，
+#   # 定义后不能更改，自定义计费项会记录所使用的计费方式id，并存储到数据库里，
+#   # 其作用与"max-cpusAlloc-mem", "max-gpu-cpusAlloc", "gpu", "cpusAlloc"等同，
+#   # 若当前生效的自定义计费项使用了某自定义计费方式id，你不能删除它，否则系统将无法启动，如要删除，请先使该计费项失效
+#   - id: "strategy1"
+#       # 可选，新的计量方式的显示名称，如不填写将使用id的内容
+#     name: "自定义收费计算方式1"
+#       # 可选，计量方式描述
+#     comment: "自定义收费计算方式1，运行时间低于3分钟以下的作业不计费，大于或等于3分钟的按照gpu或cpu用量计算"
+#       # 脚本文件路径，不包含config/scripts前缀，如my-strategy.js即等于config/scripts/my-strategy.js
+#       # 支持commonjs或者esm，内容不支持动态修改，修改后需重启系统
+#       # 自定义计量方式的文件应该默认导出一个如下签名的函数:
+#       # type CustomAmountStrategyFunction = (jobInfo: JobInfo) => number | Promise<number>;
+#       # JobInfo为apps/mis-server/src/bl/PriceMap.ts中的JobInfo类型，提供作业的用量信息
+#     script: "my-strategy.js"
+
+# 自定义可查询消费类型
+# customChargeTypes: ["月租", "存储费"]
+
+# #自定义消费记录中的作业相关的存储内容
+# jobChargeMetadata:
+#   # 可选。需要保存的作业的字段，字段参考src/entities/JobInfo
+#   savedFields: ["idJob", "cluster"]
+
+#   # 可选。定义显示出来的格式。如果不配置，直接显示上面保存的字段的信息
+#   displayFormats:
+#     # i18n格式，根据系统语言显示不同的信息
+#     # 或字符串格式，直接显示字符串
+#     # 利用 {{ 属性名 }} 使用上述savedFields中保存的属性值
+#     i18n:
+#       default: "集群 {{ cluster }} 的作业ID {{ idJob }} 的计费"
+#       en: "The billing for  jobId {{ idJob }} of Cluster {{ cluster }}"
+#       zh_cn: "集群 {{ cluster }} 的作业ID {{ idJob }} 的计费"
+
diff --git a/apps/mis-server/src/bl/charging.ts b/apps/mis-server/src/bl/charging.ts
@@ -22,6 +22,7 @@ import { Tenant } from "src/entities/Tenant";
 import { UserAccount } from "src/entities/UserAccount";
 import { ClusterPlugin } from "src/plugins/clusters";
 import { callHook } from "src/plugins/hookClient";
+import { AnyJson } from "src/utils/types";
 
 interface PayRequest {
   target: Tenant | Loaded<Account, "tenant">;
@@ -85,20 +86,24 @@ type ChargeRequest = {
   amount: Decimal;
   comment: string;
   type: string;
+  userId?: string;
+  metadata?: AnyJson;
 };
 
 export async function charge(
   request: ChargeRequest, em: SqlEntityManager,
   logger: Logger, clusterPlugin: ClusterPlugin,
 ) {
-  const { target, amount, comment, type } = request;
+  const { target, amount, comment, type, userId, metadata } = request;
 
   const record = new ChargeRecord({
     time: new Date(),
     type,
     target,
     comment,
     amount,
+    userId,
+    metadata,
   });
 
   em.persist(record);

diff --git a/apps/mis-server/src/entities/ChargeRecord.ts b/apps/mis-server/src/entities/ChargeRecord.ts
@@ -15,6 +15,7 @@ import { Decimal } from "@scow/lib-decimal";
 import { Account } from "src/entities/Account";
 import { Tenant } from "src/entities/Tenant";
 import { DecimalType } from "src/utils/decimal";
+import { type AnyJson } from "src/utils/types";
 
 @Entity()
 @Index({ name: "query_info", properties: ["time", "tenantName", "accountName", "type"] })
@@ -23,15 +24,23 @@ export class ChargeRecord {
   @PrimaryKey()
     id!: number;
 
+  @Index({ name: "time" })
   @Property()
     time: Date;
 
+  @Index()
   @Property()
     tenantName: string;
 
+  @Index()
   @Property({ nullable: true })
     accountName?: string;
 
+  @Index()
+  @Property({ nullable: true })
+    userId?: string;
+
+  @Index()
   @Property()
     type: string;
 
@@ -41,13 +50,18 @@ export class ChargeRecord {
   @Property()
     comment: string;
 
+  @Property({ type: "json", nullable: true })
+    metadata?: AnyJson;
+
   constructor(init: {
     id?: number;
     time: Date,
     type: string;
     target: Tenant | Account;
+    userId?: string;
     comment: string;
     amount: Decimal,
+    metadata?: AnyJson,
   }) {
     if (init.id) {
       this.id = init.id;
@@ -60,8 +74,10 @@ export class ChargeRecord {
       this.tenantName = init.target.tenant.getProperty("name");
       this.accountName = init.target.accountName;
     }
+    this.userId = init.userId;
     this.comment = init.comment;
     this.amount = init.amount;
+    this.metadata = init.metadata;
   }
 
 }
diff --git a/apps/mis-server/src/migrations/.snapshot-scow_server.json b/apps/mis-server/src/migrations/.snapshot-scow_server.json
@@ -97,6 +97,15 @@
           "nullable": true,
           "mappedType": "string"
         },
+        "user_id": {
+          "name": "user_id",
+          "type": "varchar(255)",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": true,
+          "mappedType": "string"
+        },
         "type": {
           "name": "type",
           "type": "varchar(255)",
@@ -125,10 +134,64 @@
           "primary": false,
           "nullable": false,
           "mappedType": "string"
+        },
+        "metadata": {
+          "name": "metadata",
+          "type": "json",
+          "unsigned": false,
+          "autoincrement": false,
+          "primary": false,
+          "nullable": true,
+          "mappedType": "json"
         }
       },
       "name": "charge_record",
       "indexes": [
+        {
+          "columnNames": [
+            "time"
+          ],
+          "composite": false,
+          "keyName": "time",
+          "primary": false,
+          "unique": false
+        },
+        {
+          "columnNames": [
+            "tenant_name"
+          ],
+          "composite": false,
+          "keyName": "charge_record_tenant_name_index",
+          "primary": false,
+          "unique": false
+        },
+        {
+          "columnNames": [
+            "account_name"
+          ],
+          "composite": false,
+          "keyName": "charge_record_account_name_index",
+          "primary": false,
+          "unique": false
+        },
+        {
+          "columnNames": [
+            "user_id"
+          ],
+          "composite": false,
+          "keyName": "charge_record_user_id_index",
+          "primary": false,
+          "unique": false
+        },
+        {
+          "columnNames": [
+            "type"
+          ],
+          "composite": false,
+          "keyName": "charge_record_type_index",
+          "primary": false,
+          "unique": false
+        },
         {
           "keyName": "static_info",
           "columnNames": [

diff --git a/apps/mis-server/src/migrations/Migration20240118021127.ts b/apps/mis-server/src/migrations/Migration20240118021127.ts
@@ -0,0 +1,32 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20240118021127 extends Migration {
+
+  async up(): Promise<void> {
+    this.addSql('alter table `charge_record` add `user_id` varchar(255) null, add `metadata` json null;');
+    this.addSql('alter table `charge_record` add index `time`(`time`);');
+    this.addSql('alter table `charge_record` add index `charge_record_tenant_name_index`(`tenant_name`);');
+    this.addSql('alter table `charge_record` add index `charge_record_account_name_index`(`account_name`);');
+    this.addSql('alter table `charge_record` add index `charge_record_user_id_index`(`user_id`);');
+    this.addSql('alter table `charge_record` add index `charge_record_type_index`(`type`);');
+
+    this.addSql('alter table `tenant` modify `create_time` DATETIME(6) not null default current_timestamp(6);');
+
+    this.addSql('alter table `user` modify `create_time` DATETIME(6) not null default current_timestamp(6);');
+  }
+
+  async down(): Promise<void> {
+    this.addSql('alter table `charge_record` drop index `time`;');
+    this.addSql('alter table `charge_record` drop index `charge_record_tenant_name_index`;');
+    this.addSql('alter table `charge_record` drop index `charge_record_account_name_index`;');
+    this.addSql('alter table `charge_record` drop index `charge_record_user_id_index`;');
+    this.addSql('alter table `charge_record` drop index `charge_record_type_index`;');
+    this.addSql('alter table `charge_record` drop `user_id`;');
+    this.addSql('alter table `charge_record` drop `metadata`;');
+
+    this.addSql('alter table `tenant` modify `create_time` DATETIME(6) not null default current_timestamp(0);');
+
+    this.addSql('alter table `user` modify `create_time` DATETIME(6) not null default current_timestamp(0);');
+  }
+
+}
diff --git a/apps/mis-server/src/services/charging.ts b/apps/mis-server/src/services/charging.ts
@@ -14,7 +14,8 @@ import { ensureNotUndefined, plugin } from "@ddadaal/tsgrpc-server";
 import { ServiceError, status } from "@grpc/grpc-js";
 import { LockMode, QueryOrder, raw } from "@mikro-orm/core";
 import { Decimal, decimalToMoney, moneyToNumber, numberToMoney } from "@scow/lib-decimal";
-import { ChargingServiceServer, ChargingServiceService } from "@scow/protos/build/server/charging";
+import { ChargeRecord as ChargeRecordProto,
+  ChargingServiceServer, ChargingServiceService } from "@scow/protos/build/server/charging";
 import { charge, pay } from "src/bl/charging";
 import { misConfig } from "src/config/mis";
 import { Account } from "src/entities/Account";
@@ -30,7 +31,6 @@ import {
 import { CHARGE_TYPE_OTHERS } from "src/utils/constants";
 import { DEFAULT_PAGE_SIZE, paginationProps } from "src/utils/orm";
 
-
 export const chargingServiceServer = plugin((server) => {
 
   server.addService<ChargingServiceServer>(ChargingServiceService, {
@@ -105,7 +105,8 @@ export const chargingServiceServer = plugin((server) => {
 
     charge: async ({ request, em, logger }) => {
 
-      const { accountName, type, amount, comment, tenantName } = ensureNotUndefined(request, ["amount"]);
+      const { accountName, type, amount, comment, tenantName, userId, metadata }
+        = ensureNotUndefined(request, ["amount"]);
 
       const reply = await em.transactional(async (em) => {
         const target = accountName !== undefined
@@ -134,6 +135,8 @@ export const chargingServiceServer = plugin((server) => {
           comment,
           target,
           type,
+          userId,
+          metadata,
         }, em, logger, server.ext);
       });
 
@@ -264,6 +267,7 @@ export const chargingServiceServer = plugin((server) => {
           index: x.id,
           time: x.time.toISOString(),
           type: x.type,
+          userId: x.userId,
         })),
         total: decimalToMoney(records.reduce((prev, curr) => prev.plus(curr.amount), new Decimal(0))),
       }];
@@ -396,7 +400,7 @@ export const chargingServiceServer = plugin((server) => {
        * @returns
        */
     getPaginatedChargeRecords: async ({ request, em }) => {
-      const { startTime, endTime, type, target, page, pageSize }
+      const { startTime, endTime, type, target, userIds, page, pageSize }
       = ensureNotUndefined(request, ["startTime", "endTime"]);
 
       const searchParam = getChargesTargetSearchParam(target);
@@ -407,21 +411,27 @@ export const chargingServiceServer = plugin((server) => {
         time: { $gte: startTime, $lte: endTime },
         ...searchType,
         ...searchParam,
+        ...(userIds.length > 0 ? { userId: { $in: userIds } } : {}),
       }, {
         ...paginationProps(page, pageSize || DEFAULT_PAGE_SIZE),
         orderBy: { time: QueryOrder.DESC },
       });
 
       return [{
-        results: records.map((x) => ({
-          tenantName: x.tenantName,
-          accountName: x.accountName,
-          amount: decimalToMoney(x.amount),
-          comment: x.comment,
-          index: x.id,
-          time: x.time.toISOString(),
-          type: x.type,
-        })),
+        results: records.map((x) => {
+          return {
+            tenantName: x.tenantName,
+            accountName: x.accountName,
+            amount: decimalToMoney(x.amount),
+            comment: x.comment,
+            index: x.id,
+            time: x.time.toISOString(),
+            type: x.type,
+            userId: x.userId,
+            metadata: x.metadata as ChargeRecordProto["metadata"] ?? undefined,
+          };
+
+        }),
       }];
     },
 
@@ -436,7 +446,7 @@ export const chargingServiceServer = plugin((server) => {
    * @returns
    */
     getChargeRecordsTotalCount: async ({ request, em }) => {
-      const { startTime, endTime, type, target }
+      const { startTime, endTime, type, target, userIds }
       = ensureNotUndefined(request, ["startTime", "endTime"]);
 
       const searchParam = getChargesTargetSearchParam(target);
@@ -451,6 +461,7 @@ export const chargingServiceServer = plugin((server) => {
             time: { $gte: startTime, $lte: endTime },
             ...searchType,
             ...searchParam,
+            ...(userIds.length > 0 ? { userId: { $in: userIds } } : {}),
           })
           .execute("get");