fix(mis): 修复数据统计时区误差问题 (#1094)

### 现状目前数据统计功能在web侧得时间日期筛选，进入到server端及数据库会以utc 时间来进行搜索，但大部分用户其实是希望以自己所在时区进行日期筛选。举例：关于1-17到1-23号的用户活跃数的统计如果以0时区进行统计： ![img_v3_027c_c7a6c6e5-f096-47dc-85e3-3b79e15fc33g](https://github.com/PKUHPC/SCOW/assets/130351655/6251e5c7-4e96-4372-99d4-c16e4d24fece) 但是数据库里其实 2024-01-22 的记录是八时区的01-23号产生的(2024-01-22 20:56:09.789000)，所以正确的结果应该如下（sql将0时区转为8时区后计算) ![img_v3_027c_8ab313d4-fdaf-4799-bcfc-5c17314d1f8g](https://github.com/PKUHPC/SCOW/assets/130351655/14d13d8d-da97-4bfa-b980-06559a42021d) ### 改动 GetDailyCharge,GetDailyPay,GetNewJobCount,GetNewUserCount,GetActiveUserCount 接口增加时区参数，在数据库进行搜索时以传参时区进行转换（没传就默认为UTC时区），客户端侧通过Intl.DateTimeFormat().resolvedOptions().timeZone 获取当前时区。
PKUHPC · Feb 4, 2024 · 443187e · 443187e
1 parent efcd9a8
commit 443187e
Show file tree

Hide file tree

Showing 33 changed files with 602 additions and 109 deletions.
diff --git a/.changeset/happy-dingos-care.md b/.changeset/happy-dingos-care.md
@@ -0,0 +1,5 @@
+---
+"@scow/grpc-api": minor
+---
+
+GetDailyCharge,GetDailyPay,GetNewJobCount,GetNewUserCount,GetActiveUserCount 接口新增 time_zone 参数以及返回类型由时间戳改为 date
diff --git a/.changeset/hot-tigers-dream.md b/.changeset/hot-tigers-dream.md
@@ -0,0 +1,8 @@
+---
+"@scow/audit-server": patch
+"@scow/mis-server": patch
+"@scow/mis-web": patch
+"@scow/lib-server": patch
+---
+
+修复数据统计相关功能时区转换问题
diff --git a/apps/audit-server/package.json b/apps/audit-server/package.json
@@ -33,6 +33,7 @@
     "@mikro-orm/seeder": "6.0.5",
     "@scow/config": "workspace:*",
     "@scow/lib-config": "workspace:*",
+    "@scow/lib-server": "workspace:*",
     "@scow/lib-decimal": "workspace:*",
     "@scow/lib-operation-log": "workspace:*",
     "@scow/utils": "workspace:*",

diff --git a/apps/audit-server/src/services/statistic.ts b/apps/audit-server/src/services/statistic.ts
@@ -13,6 +13,7 @@
 import { ensureNotUndefined, plugin } from "@ddadaal/tsgrpc-server";
 import { QueryOrder, raw } from "@mikro-orm/core";
 import { OperationType } from "@scow/lib-operation-log";
+import { checktTimeZone, convertToDateMessage } from "@scow/lib-server/build/date";
 import { StatisticServiceServer, StatisticServiceService } from "@scow/protos/build/audit/statistic";
 import { OperationLog } from "src/entities/OperationLog";
 
@@ -21,24 +22,28 @@ export const statisticServiceServer = plugin((server) => {
 
   server.addService<StatisticServiceServer>(StatisticServiceService, {
 
-    getActiveUserCount: async ({ request, em }) => {
+    getActiveUserCount: async ({ request, em, logger }) => {
 
-      const { startTime, endTime } = ensureNotUndefined(request, ["startTime", "endTime"]);
+      const { startTime, endTime, timeZone = "UTC" } = ensureNotUndefined(request, ["startTime", "endTime"]);
 
-      const qb = em.createQueryBuilder(OperationLog, "o");
+      checktTimeZone(timeZone);
 
+      const qb = em.createQueryBuilder(OperationLog, "o");
       qb
-        .select([raw("DATE(o.operation_time) as date"), raw("COUNT(DISTINCT o.operator_user_id) as userCount")])
+        .select([
+          raw("DATE(CONVERT_TZ(o.operation_time, 'UTC', ?)) as date", [timeZone]),
+          raw("COUNT(DISTINCT o.operator_user_id) as userCount"),
+        ])
         .where({ operationTime: { $gte: startTime } })
         .andWhere({ operationTime: { $lte: endTime } })
-        .groupBy(raw("DATE(o.operation_time)"))
-        .orderBy({ [raw("DATE(o.operation_time)")]: QueryOrder.DESC });
+        .groupBy(raw("date"))
+        .orderBy({ [raw("date")]: QueryOrder.DESC });
 
       const records: {date: string, userCount: number}[] = await qb.execute();
 
       return [{
         results: records.map((record) => ({
-          date: record.date,
+          date: convertToDateMessage(record.date, logger),
           count: record.userCount,
         })),
       }];
@@ -72,8 +77,8 @@ export const statisticServiceServer = plugin((server) => {
         .where({ operationTime: { $gte: startTime } })
         .andWhere({ operationTime: { $lte: endTime } })
         .andWhere({ [raw("JSON_EXTRACT(meta_data, '$.$case')")]: { $in: portalOperationType } })
-        .groupBy(raw("JSON_EXTRACT(meta_data, '$.$case')"))
-        .orderBy({ [raw("COUNT(*)")]: QueryOrder.DESC });
+        .groupBy(raw("operationType"))
+        .orderBy({ [raw("count")]: QueryOrder.DESC });
 
       const results: {operationType: string, count: number}[] = await qb.execute();
 
@@ -126,8 +131,8 @@ export const statisticServiceServer = plugin((server) => {
         .where({ operationTime: { $gte: startTime } })
         .andWhere({ operationTime: { $lte: endTime } })
         .andWhere({ [raw("JSON_EXTRACT(meta_data, '$.$case')")]: { $in: misOperationType } })
-        .groupBy(raw("JSON_EXTRACT(meta_data, '$.$case')"))
-        .orderBy({ [raw("COUNT(*)")]: QueryOrder.DESC });
+        .groupBy(raw("operationType"))
+        .orderBy({ [raw("count")]: QueryOrder.DESC });
 
       const results: {operationType: string, count: number}[] = await qb.execute();
 

diff --git a/apps/audit-server/tests/statistic/statistic.test.ts b/apps/audit-server/tests/statistic/statistic.test.ts
@@ -13,6 +13,7 @@
 import { asyncClientCall } from "@ddadaal/tsgrpc-client";
 import { Server } from "@ddadaal/tsgrpc-server";
 import { ChannelCredentials } from "@grpc/grpc-js";
+import { dayjsToDateMessage } from "@scow/lib-server/build/date";
 import { StatisticServiceClient } from "@scow/protos/build/audit/statistic";
 import dayjs from "dayjs";
 import utc from "dayjs/plugin/utc";
@@ -43,7 +44,7 @@ afterEach(async () => {
   await server.close();
 });
 
-it("get active user count correctly", async () => {
+it("get active user count correctly in UTC+8 timezone", async () => {
 
   const em = server.ext.orm.em.fork();
 
@@ -65,13 +66,14 @@ it("get active user count correctly", async () => {
   const resp = await asyncClientCall(client, "getActiveUserCount", {
     startTime: today.toISOString(),
     endTime: endDay.toISOString(),
+    timeZone: "Asia/Shanghai",
   });
 
+  const nowInUtcPlus8 = now.utcOffset(8);
+
   expect(resp.results).toMatchObject([
     {
-      // 应该期待返回的结果的日期为日志的operationTime的UTC日期的开始时间
-      // date: today.toISOString(),
-      date: now.utcOffset(0).startOf("day").toISOString(),
+      date: dayjsToDateMessage(nowInUtcPlus8),
       count: 10,
     },
   ]);

diff --git a/apps/mis-server/src/services/charging.ts b/apps/mis-server/src/services/charging.ts
@@ -14,6 +14,7 @@ import { ensureNotUndefined, plugin } from "@ddadaal/tsgrpc-server";
 import { ServiceError, status } from "@grpc/grpc-js";
 import { LockMode, QueryOrder, raw } from "@mikro-orm/core";
 import { Decimal, decimalToMoney, moneyToNumber, numberToMoney } from "@scow/lib-decimal";
+import { checktTimeZone, convertToDateMessage } from "@scow/lib-server/build/date";
 import { ChargeRecord as ChargeRecordProto,
   ChargingServiceServer, ChargingServiceService } from "@scow/protos/build/server/charging";
 import { charge, pay } from "src/bl/charging";
@@ -303,29 +304,34 @@ export const chargingServiceServer = plugin((server) => {
       ];
     },
 
-    getDailyCharge: async ({ request, em }) => {
+    getDailyCharge: async ({ request, em, logger }) => {
 
-      const { startTime, endTime } = ensureNotUndefined(request, ["startTime", "endTime"]);
+      const { startTime, endTime, timeZone = "UTC" } = ensureNotUndefined(request, ["startTime", "endTime"]);
+
+      checktTimeZone(timeZone);
 
       const qb = em.createQueryBuilder(ChargeRecord, "cr");
 
       qb
-        .select([raw("DATE(cr.time) as date"), raw("SUM(cr.amount) as totalAmount")])
+        .select([
+          raw("DATE(CONVERT_TZ(cr.time, 'UTC', ?)) as date", [timeZone]),
+          raw("SUM(cr.amount) as totalAmount"),
+        ])
         .where({ time: { $gte: startTime } })
         .andWhere({ time: { $lte: endTime } })
         .andWhere({ accountName: { $ne: null } })
-        .groupBy(raw("DATE(cr.time)"))
-        .orderBy({ [raw("DATE(cr.time)")]: QueryOrder.DESC });
+        .groupBy(raw("date"))
+        .orderBy({ [raw("date")]: QueryOrder.DESC });
 
       const records: {date: string, totalAmount: number}[] = await queryWithCache({
         em,
-        queryKeys: ["get_daily_charge", `${startTime}`, `${endTime}`],
+        queryKeys: ["get_daily_charge", `${startTime}`, `${endTime}`, `${timeZone}`],
         queryQb: qb,
       });
 
       return [{
         results: records.map((record) => ({
-          date: new Date(record.date).toISOString(),
+          date: convertToDateMessage(record.date, logger),
           amount: numberToMoney(record.totalAmount),
         })),
       }];
@@ -361,29 +367,34 @@ export const chargingServiceServer = plugin((server) => {
       ];
     },
 
-    getDailyPay: async ({ request, em }) => {
+    getDailyPay: async ({ request, em, logger }) => {
+
+      const { startTime, endTime, timeZone = "UTC" } = ensureNotUndefined(request, ["startTime", "endTime"]);
 
-      const { startTime, endTime } = ensureNotUndefined(request, ["startTime", "endTime"]);
+      checktTimeZone(timeZone);
 
       const qb = em.createQueryBuilder(PayRecord, "pr");
 
       qb
-        .select([raw("DATE(pr.time) as date"), raw("SUM(pr.amount) as totalAmount")])
+        .select([
+          raw("DATE(CONVERT_TZ(pr.time, 'UTC', ?)) as date", [timeZone]),
+          raw("SUM(pr.amount) as totalAmount"),
+        ])
         .where({ time: { $gte: startTime } })
         .andWhere({ time: { $lte: endTime } })
         .andWhere({ accountName: { $ne: null } })
-        .groupBy(raw("DATE(pr.time)"))
-        .orderBy({ [raw("DATE(pr.time)")]: QueryOrder.DESC });
+        .groupBy(raw("date"))
+        .orderBy({ [raw("date")]: QueryOrder.DESC });
 
       const records: {date: string, totalAmount: number}[] = await queryWithCache({
         em,
-        queryKeys: ["get_daily_charge", `${startTime}`, `${endTime}`],
+        queryKeys: ["get_daily_pay", `${startTime}`, `${endTime}`, `${timeZone}`],
         queryQb: qb,
       });
 
       return [{
         results: records.map((record) => ({
-          date: new Date(record.date).toISOString(),
+          date: convertToDateMessage(record.date, logger),
           amount: numberToMoney(record.totalAmount),
         })),
       }];

diff --git a/apps/mis-server/src/services/job.ts b/apps/mis-server/src/services/job.ts
@@ -17,6 +17,7 @@ import { Status } from "@grpc/grpc-js/build/src/constants";
 import { FilterQuery, QueryOrder, raw, UniqueConstraintViolationException } from "@mikro-orm/core";
 import { Decimal, decimalToMoney, moneyToNumber } from "@scow/lib-decimal";
 import { jobInfoToRunningjob } from "@scow/lib-scheduler-adapter";
+import { checktTimeZone, convertToDateMessage } from "@scow/lib-server/build/date";
 import { ChargeRecord } from "@scow/protos/build/server/charging";
 import {
   GetJobsResponse,
@@ -412,15 +413,19 @@ export const jobServiceServer = plugin((server) => {
     },
 
     getNewJobCount: async ({ request, em }) => {
-      const { startTime, endTime } = ensureNotUndefined(request, ["startTime", "endTime"]);
+      const { startTime, endTime, timeZone = "UTC" } = ensureNotUndefined(request, ["startTime", "endTime"]);
+
+      checktTimeZone(timeZone);
 
       const qb = em.createQueryBuilder(JobInfoEntity, "j");
       qb
-        .select([raw("DATE(j.time_submit) as date"), raw("COUNT(*) as count")])
+        .select([
+          raw("DATE(CONVERT_TZ(j.time_submit, 'UTC', ?)) as date", [timeZone]),
+          raw("COUNT(*) as count")])
         .where({ timeSubmit: { $gte: startTime } })
         .andWhere({ timeSubmit: { $lte: endTime } })
-        .groupBy(raw("DATE(j.time_submit)"))
-        .orderBy({ [raw("DATE(j.time_submit)")]: QueryOrder.DESC });
+        .groupBy(raw("date"))
+        .orderBy({ [raw("date")]: QueryOrder.DESC });
 
       const results: {date: string, count: number}[] = await queryWithCache({
         em,
@@ -429,7 +434,10 @@ export const jobServiceServer = plugin((server) => {
       });
       return [
         {
-          results,
+          results: results.map((record) => ({
+            date: convertToDateMessage(record.date, logger),
+            count: record.count,
+          })),
         },
       ];
     },

diff --git a/apps/mis-server/src/services/user.ts b/apps/mis-server/src/services/user.ts
@@ -19,6 +19,7 @@ import { addUserToAccount, changeEmail as libChangeEmail, createUser, getCapabil
 }
   from "@scow/lib-auth";
 import { decimalToMoney } from "@scow/lib-decimal";
+import { checktTimeZone, convertToDateMessage } from "@scow/lib-server/build/date";
 import {
   AccountStatus,
   GetAccountUsersResponse,
@@ -762,22 +763,27 @@ export const userServiceServer = plugin((server) => {
       return [{}];
     },
 
-    getNewUserCount: async ({ request, em }) => {
-      const { startTime, endTime } = ensureNotUndefined(request, ["startTime", "endTime"]);
+    getNewUserCount: async ({ request, em, logger }) => {
+      const { startTime, endTime, timeZone = "UTC" } = ensureNotUndefined(request, ["startTime", "endTime"]);
+
+      checktTimeZone(timeZone);
 
       const qb = em.createQueryBuilder(User, "u");
       qb
-        .select([raw("DATE(u.create_time) as date"), raw("count(*) as count")])
+        .select([raw("DATE(CONVERT_TZ(u.create_time, 'UTC', ?)) as date", [timeZone]), raw("count(*) as count")])
         .where({ createTime: { $gte: startTime } })
         .andWhere({ createTime: { $lte: endTime } })
-        .groupBy(raw("DATE(u.create_time)"))
-        .orderBy({ [raw("DATE(u.create_time)")]: QueryOrder.DESC });
+        .groupBy(raw("date"))
+        .orderBy({ [raw("date")]: QueryOrder.DESC });
 
       const results: {date: string, count: number}[] = await qb.execute();
 
       return [
         {
-          results,
+          results: results.map((record) => ({
+            date: convertToDateMessage(record.date, logger),
+            count: record.count,
+          })),
         },
       ];
     },

diff --git a/apps/mis-server/tests/admin/user.test.ts b/apps/mis-server/tests/admin/user.test.ts
@@ -16,9 +16,11 @@ import { ChannelCredentials } from "@grpc/grpc-js";
 import { Status } from "@grpc/grpc-js/build/src/constants";
 import { Loaded } from "@mikro-orm/core";
 import { createUser } from "@scow/lib-auth";
+import { dayjsToDateMessage } from "@scow/lib-server/build/date";
 import { GetAllUsersRequest_UsersSortField, PlatformRole, platformRoleFromJSON,
   SortDirection, TenantRole, UserServiceClient } from "@scow/protos/build/server/user";
 import dayjs from "dayjs";
+import utc from "dayjs/plugin/utc";
 import { createServer } from "src/app";
 import { authUrl } from "src/config";
 import { Account } from "src/entities/Account";
@@ -31,6 +33,7 @@ import { reloadEntity } from "src/utils/orm";
 import { insertInitialData } from "tests/data/data";
 import { dropDatabase } from "tests/data/helpers";
 
+dayjs.extend(utc);
 
 const anotherTenant = "anotherTenant";
 
@@ -566,7 +569,7 @@ it("change an inexistent user email", async () => {
   expect(reply.code).toBe(Status.NOT_FOUND);
 });
 
-it("get new user count", async () => {
+it("get new user count in UTC+8 timezone", async () => {
 
   const em = server.ext.orm.em.fork();
   const today = dayjs();
@@ -607,12 +610,19 @@ it("get new user count", async () => {
   const info = await asyncClientCall(client, "getNewUserCount", {
     startTime: twoDaysBefore.startOf("day").toISOString(),
     endTime: today.endOf("day").toISOString(),
+    timeZone: "Asia/Shanghai",
   });
 
+  const todyInUtcPlus8 = today.utcOffset(8);
+
+  const yesterdayInUtcPlus8 = yesterday.utcOffset(8);
+
+  const twoDaysBeforeInUtcPlus8 = twoDaysBefore.utcOffset(8);
+
   expect(info.results).toMatchObject([
-    { date: today.startOf("day").toISOString(), count: 30 },
-    { date: yesterday.startOf("day").toISOString(), count: 20 },
-    { date: twoDaysBefore.startOf("day").toISOString(), count: 10 },
+    { date: dayjsToDateMessage(todyInUtcPlus8), count: 30 },
+    { date:dayjsToDateMessage(yesterdayInUtcPlus8), count: 20 },
+    { date:dayjsToDateMessage(twoDaysBeforeInUtcPlus8), count: 10 },
   ]);
 
 });