From b14b1be5b07198665fabaa31f203193aae69e292 Mon Sep 17 00:00:00 2001 From: BEDev24 Date: Fri, 13 Sep 2024 11:55:54 +0200 Subject: [PATCH] merge from develop to staging --- backend/example.env | 4 +- ...2-governance-title-abstrant-len-removal.ts | 29 + .../facade/constitution.facade.ts | 2 - .../governance/facade/governance.facade.ts | 7 +- .../interfaces/icip100.interface.ts | 1 - backend/src/ipfs/services/ipfs.service.ts | 3 +- backend/src/s3/service/s3.service.ts | 7 +- docker-compose.yaml | 5 +- frontend/example.env | 8 +- frontend/messages/de.json | 10 +- frontend/messages/en.json | 10 +- frontend/src/app/[locale]/my-actions/page.tsx | 14 +- .../components/atoms/OutlinedLightButton.tsx | 15 +- .../src/components/molecules/TableDivider.tsx | 6 +- .../GovActionTable/GovActionTableRow.tsx | 344 +++---- .../organisms/GovernanceActions.tsx | 32 +- .../organisms/Modals/GovActionModal.tsx | 24 +- .../src/components/organisms/MyActions.tsx | 41 +- .../organisms/VotesTable/VotesTableRow.tsx | 303 ++++--- frontend/src/constants/paths.ts | 6 +- frontend/src/context/topBanner.tsx | 16 +- frontend/src/context/usersnap.tsx | 23 +- ipfs-service/example.env | 15 +- ipfs-service/package-lock.json | 848 +++++++++++++++++- ipfs-service/package.json | 11 +- ipfs-service/src/app.controller.ts | 8 +- ipfs-service/src/app.module.ts | 14 +- ipfs-service/src/app.service.ts | 79 +- ipfs-service/src/bullmq/bullmq.module.ts | 31 + .../src/constants/bullmq.constants.ts | 2 + ipfs-service/src/main.ts | 23 +- .../processors/provide-to-dht.processor.ts | 40 + .../producers/provide-to-dht.producer.ts | 30 + ipfs-service/src/util/logger-factory.ts | 37 + worker-service/src/common/common-service.ts | 12 +- .../entities/gov-action-proposal.entity.ts | 2 - .../gov-action-proposal.processor.ts | 28 +- .../queues/processors/vote.processor.ts | 28 +- .../services/gov-action-proposal.service.ts | 9 +- .../src/governance/services/vote.service.ts | 3 +- 40 files changed, 1626 insertions(+), 504 deletions(-) create mode 100644 backend/migrations/1726048356332-governance-title-abstrant-len-removal.ts create mode 100644 ipfs-service/src/bullmq/bullmq.module.ts create mode 100644 ipfs-service/src/constants/bullmq.constants.ts create mode 100644 ipfs-service/src/queues/processors/provide-to-dht.processor.ts create mode 100644 ipfs-service/src/queues/producers/provide-to-dht.producer.ts create mode 100644 ipfs-service/src/util/logger-factory.ts diff --git a/backend/example.env b/backend/example.env index d2ec7064..d15953a9 100644 --- a/backend/example.env +++ b/backend/example.env @@ -31,13 +31,15 @@ EMAIL_FROM=example@email.from NAME_FROM=Example Name From #minio -MINIO_ENDPOINT=minio +MINIO_ENDPOINT=localhost MINIO_PORT=9000 MINIO_ACCESS_KEY=minio MINIO_SECRET_KEY=minio123 MINIO_USE_SSL=false MINIO_BUCKET=cc-portal +S3_BASE_URL=http://localhost:9000 + # IPFS Service IPFS_SERVICE_URL=http://ipfs-service:3001 diff --git a/backend/migrations/1726048356332-governance-title-abstrant-len-removal.ts b/backend/migrations/1726048356332-governance-title-abstrant-len-removal.ts new file mode 100644 index 00000000..91abd829 --- /dev/null +++ b/backend/migrations/1726048356332-governance-title-abstrant-len-removal.ts @@ -0,0 +1,29 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class GovTitleAbstractLenRemoval1726048356332 + implements MigrationInterface +{ + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `begin; + ALTER TABLE gov_action_proposals + ALTER COLUMN title TYPE VARCHAR; + + ALTER TABLE gov_action_proposals + alter COLUMN abstract type VARCHAR; + commit;`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `begin; + ALTER TABLE gov_action_proposals + ALTER COLUMN title TYPE VARCHAR(80); + + ALTER TABLE gov_action_proposals + alter COLUMN abstract type VARCHAR(2500); + commit;`, + ); + } +} diff --git a/backend/src/constitution/facade/constitution.facade.ts b/backend/src/constitution/facade/constitution.facade.ts index 91892be8..6f31def6 100644 --- a/backend/src/constitution/facade/constitution.facade.ts +++ b/backend/src/constitution/facade/constitution.facade.ts @@ -3,7 +3,6 @@ import { ConstitutionRedisService } from 'src/redis/service/constitution-redis.s import { ConstitutionResponse } from '../api/response/constitution.response'; import { ConstitutionMapper } from '../mapper/constitution.mapper'; import { ConstitutionDto } from 'src/redis/dto/constitution.dto'; -import { ConstitutionService } from '../services/constitution.service'; import { IpfsService } from 'src/ipfs/services/ipfs.service'; import { ConstitutionMetadataResponse } from '../api/response/constitution-metadata.response'; import { ConstitutionIpnsUrlResponse } from '../api/response/constitutio-ipns-url.response'; @@ -14,7 +13,6 @@ export class ConstitutionFacade { constructor( private readonly constitutionRedisService: ConstitutionRedisService, - private readonly constitutionService: ConstitutionService, private readonly ipfsService: IpfsService, ) {} diff --git a/backend/src/governance/facade/governance.facade.ts b/backend/src/governance/facade/governance.facade.ts index c51f661a..f0f2f38b 100644 --- a/backend/src/governance/facade/governance.facade.ts +++ b/backend/src/governance/facade/governance.facade.ts @@ -45,7 +45,11 @@ export class GovernanceFacade { const response = await this.governanceService.addRationale(rationaleDto); return GovernanceMapper.rationaleDtoToResponse(response); } - + /** + * Creates a CIP 100 compatible JSON object. + * CIP 100 example: + * https://github.com/cardano-foundation/CIPs/blob/master/CIP-0100/example.body.json + **/ private async createRationaleJsonCip100( rationaleRequest: RationaleRequest, ): Promise { @@ -97,7 +101,6 @@ export class GovernanceFacade { hashAlgorithm: CIP100.hashAlgorithm, authors: [], body: { - references: [], comment: rationaleRequest.content, }, }; diff --git a/backend/src/governance/interfaces/icip100.interface.ts b/backend/src/governance/interfaces/icip100.interface.ts index 60f19b74..62c30a46 100644 --- a/backend/src/governance/interfaces/icip100.interface.ts +++ b/backend/src/governance/interfaces/icip100.interface.ts @@ -69,6 +69,5 @@ interface WitnessContext { } interface BodyContent { - references: any[]; comment: string; } diff --git a/backend/src/ipfs/services/ipfs.service.ts b/backend/src/ipfs/services/ipfs.service.ts index 30054ab2..3bf86bea 100644 --- a/backend/src/ipfs/services/ipfs.service.ts +++ b/backend/src/ipfs/services/ipfs.service.ts @@ -52,7 +52,8 @@ export class IpfsService { const formData = new FormData(); formData.append('file', blob, file.originalname); - const apiLink = this.configService.getOrThrow('IPFS_SERVICE_URL') + '/ipfs'; + const apiLink = + this.configService.getOrThrow('IPFS_SERVICE_URL') + '/ipfs/file'; const requestConfig = { headers: { 'Content-Type': 'multipart/form-data', diff --git a/backend/src/s3/service/s3.service.ts b/backend/src/s3/service/s3.service.ts index 059e7b3d..9031b76e 100644 --- a/backend/src/s3/service/s3.service.ts +++ b/backend/src/s3/service/s3.service.ts @@ -23,7 +23,9 @@ export class S3Service { const buffer = Buffer.from(bufferArray); const fullName = this.getFullFileName(context, fileName); - await this.client.putObject(this.bucketName, fullName, buffer, file.size); + await this.client.putObject(this.bucketName, fullName, buffer, file.size, { + 'Content-Type': file.mimetype, + }); const fileUrl = await this.getFileUrl(context, fileName); return fileUrl; @@ -34,7 +36,8 @@ export class S3Service { fileName: string, ): Promise { const fullName = this.getFullFileName(context, fileName); - return await this.client.presignedUrl('GET', this.bucketName, fullName); + const fileUrl = `${this.configService.get('S3_BASE_URL')}/${this.bucketName}/${fullName}`; + return fileUrl; } async deleteFile(fileName: string) { diff --git a/docker-compose.yaml b/docker-compose.yaml index 34d39536..6e855821 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -37,7 +37,7 @@ services: - "5001:5001" - "8080:8080" volumes: - - ipfs_data:/app/ + - ipfs_data:/ipfs/datastore networks: - bloxico_local extra_hosts: @@ -75,6 +75,9 @@ services: image: redis/redis-stack:latest container_name: cache restart: always + environment: + - REDIS_PASSWORD=your-redis-password + command: redis-server --requirepass ${REDIS_PASSWORD} ports: - "6379:6379" volumes: diff --git a/frontend/example.env b/frontend/example.env index 713b4eb5..7ae18bfb 100644 --- a/frontend/example.env +++ b/frontend/example.env @@ -1 +1,7 @@ -NEXT_PUBLIC_API_URL=http://backend:1337 # Container name goes here \ No newline at end of file +NEXT_PUBLIC_API_URL=http://backend:1337 # Container name goes here +NEXT_PUBLIC_USERSNAP_SPACE_API_KEY= +NEXT_PUBLIC_USERSNAP_PROJECT_API_KEY = +NEXT_PUBLIC_HIDDEN_USERSNAP_PROJECT_IDS= +NEXT_PUBLIC_IS_MAINNET=false + + diff --git a/frontend/messages/de.json b/frontend/messages/de.json index 07970556..18830c42 100644 --- a/frontend/messages/de.json +++ b/frontend/messages/de.json @@ -243,15 +243,15 @@ }, "members": { "title": "No members found", - "description": "There is currently no members. Please check back later for updates" + "description": "There are currently no members. Please check back later for updates" }, "latestUpdates": { "title": "No votes found", - "description": "There is currently no votes. Please check back later for updates" + "description": "There are currently no votes. Please check back later for updates" }, "myActions": { "title": "No votes found", - "description": "There is currently no votes. Please check back later for updates" + "description": "There are currently no votes. Please check back later for updates" }, "constitution": { "title": "No Constitution available", @@ -259,11 +259,11 @@ }, "constitutionMetadata": { "title": "No revisions available", - "description": "There is currently no revisions available. Please check back later for updates" + "description": "There are currently no revisions available. Please check back later for updates" }, "governanceAction": { "title": "No governance actions available", - "description": "There is currently no governance actions. Please check back later for updates" + "description": "There are currently no governance actions. Please check back later for updates" } }, "Snackbar": { diff --git a/frontend/messages/en.json b/frontend/messages/en.json index 6ec76645..fbae8799 100644 --- a/frontend/messages/en.json +++ b/frontend/messages/en.json @@ -243,15 +243,15 @@ }, "members": { "title": "No members found", - "description": "There is currently no members. Please check back later for updates" + "description": "There are currently no members. Please check back later for updates" }, "latestUpdates": { "title": "No votes found", - "description": "There is currently no votes. Please check back later for updates" + "description": "There are currently no votes. Please check back later for updates" }, "myActions": { "title": "No votes found", - "description": "There is currently no votes. Please check back later for updates" + "description": "There are currently no votes. Please check back later for updates" }, "constitution": { "title": "No Constitution available", @@ -259,11 +259,11 @@ }, "constitutionMetadata": { "title": "No revisions available", - "description": "There is currently no revisions available. Please check back later for updates" + "description": "There are currently no revisions available. Please check back later for updates" }, "governanceAction": { "title": "No governance actions available", - "description": "There is currently no governance actions. Please check back later for updates" + "description": "There are currently no governance actions. Please check back later for updates" } }, "Snackbar": { diff --git a/frontend/src/app/[locale]/my-actions/page.tsx b/frontend/src/app/[locale]/my-actions/page.tsx index d4a789c7..15e134c4 100644 --- a/frontend/src/app/[locale]/my-actions/page.tsx +++ b/frontend/src/app/[locale]/my-actions/page.tsx @@ -1,15 +1,15 @@ -import React, { Suspense } from "react"; +import { Suspense } from "react"; -import { unstable_setRequestLocale } from "next-intl/server"; // Import function to set the request-specific locale (unstable API). -import { Footer, TopNav, MyActions } from "@organisms"; -import { Loading } from "@molecules"; import { decodeUserToken, getUserVotes } from "@/lib/api"; import { ContentWrapper } from "@atoms"; +import { Loading } from "@molecules"; +import { Footer, MyActions, TopNav } from "@organisms"; import { isResponseErrorI } from "@utils"; +import { unstable_setRequestLocale } from "next-intl/server"; // Import function to set the request-specific locale (unstable API). export default async function MyActionsPage({ params: { locale }, - searchParams, + searchParams }) { unstable_setRequestLocale(locale); // Sets the locale for the request. Use cautiously due to its unstable nature. const user = await decodeUserToken(); @@ -19,10 +19,9 @@ export default async function MyActionsPage({ govActionType: searchParams?.govActionType, vote: searchParams?.vote, sortBy: searchParams?.sortBy, - userId: user?.userId, + userId: user?.userId }); const hasError = isResponseErrorI(userActions); - return ( <> @@ -32,6 +31,7 @@ export default async function MyActionsPage({ actions={!hasError && userActions?.data} paginationMeta={!hasError && userActions?.meta} error={hasError && userActions.error} + userId={user?.userId} /> ; diff --git a/frontend/src/components/atoms/OutlinedLightButton.tsx b/frontend/src/components/atoms/OutlinedLightButton.tsx index e36dc721..819357d1 100644 --- a/frontend/src/components/atoms/OutlinedLightButton.tsx +++ b/frontend/src/components/atoms/OutlinedLightButton.tsx @@ -1,8 +1,7 @@ "use client"; -import React from "react"; -import { Button, SxProps, Theme } from "@mui/material"; import { customPalette } from "@consts"; +import { Button, SxProps, Theme } from "@mui/material"; import { ButtonProps } from "@mui/material/Button"; interface Props extends ButtonProps { @@ -23,11 +22,13 @@ export const OutlinedLightButton = ({ color: customPalette.textBlack, fontWeight: 400, fontSize: 12, - whiteSpace: "nowrap", + lineHeight: "18px", + wordBreak: "break-word", + "&": { - pointerEvents: nonInteractive ? "none" : "all", - }, - }; + pointerEvents: nonInteractive ? "none" : "all" + } + } as React.CSSProperties; return ( - - - diff --git a/frontend/src/components/organisms/GovernanceActions.tsx b/frontend/src/components/organisms/GovernanceActions.tsx index c3e2c25a..60af9d9f 100644 --- a/frontend/src/components/organisms/GovernanceActions.tsx +++ b/frontend/src/components/organisms/GovernanceActions.tsx @@ -1,27 +1,27 @@ "use client"; -import React, { useCallback, useEffect, useState } from "react"; -import { Box } from "@mui/material"; -import { NotFound } from "./NotFound"; -import { countSelectedFilters, isEmpty } from "@utils"; -import { DataActionsBar } from "../molecules"; +import { getGovernanceActions } from "@/lib/api"; +import { GovernanceActionTableI, PaginationMeta } from "@/lib/requests"; import { GOVERNANCE_ACTIONS_FILTERS, GOVERNANCE_ACTIONS_SORTING, MY_ACTIONS_TABS, - PATHS, + PATHS } from "@consts"; -import { PageTitleTabs } from "./PageTitleTabs"; -import { GovActionTable } from "./GovActionTable"; -import { GovernanceActionTableI, PaginationMeta } from "@/lib/requests"; -import { getGovernanceActions } from "@/lib/api"; -import { usePagination, useManageQueryParams } from "@hooks"; -import { ShowMoreButton } from "../atoms"; +import { useManageQueryParams, usePagination } from "@hooks"; +import { Box } from "@mui/material"; +import { countSelectedFilters, isEmpty } from "@utils"; import { useRouter } from "next/navigation"; +import { useCallback, useEffect, useState } from "react"; +import { ShowMoreButton } from "../atoms"; +import { DataActionsBar } from "../molecules"; +import { GovActionTable } from "./GovActionTable"; +import { NotFound } from "./NotFound"; +import { PageTitleTabs } from "./PageTitleTabs"; export const GovernanceActions = ({ actions, - paginationMeta, + paginationMeta }: { actions: GovernanceActionTableI[]; paginationMeta: PaginationMeta; @@ -43,7 +43,7 @@ export const GovernanceActions = ({ : null, status: chosenFilters.status?.length > 0 ? chosenFilters.status?.join(",") : null, - sortBy: chosenSorting || null, + sortBy: chosenSorting || null }; const { data, pagination, isLoading, loadMore } = usePagination( @@ -70,10 +70,10 @@ export const GovernanceActions = ({ paddingBottom={4} display="flex" justifyContent={{ xxs: "flex-start", md: "space-between" }} - flexDirection={{ xxs: "column", md: "row" }} + flexDirection={{ xxs: "column", md: "column", lg: "row" }} alignItems={{ xxs: "left", md: "center" }} > - + router.push(tab.value)} diff --git a/frontend/src/components/organisms/Modals/GovActionModal.tsx b/frontend/src/components/organisms/Modals/GovActionModal.tsx index 38e15acb..46d531ba 100644 --- a/frontend/src/components/organisms/Modals/GovActionModal.tsx +++ b/frontend/src/components/organisms/Modals/GovActionModal.tsx @@ -1,29 +1,29 @@ "use client"; -import React, { useEffect, useState } from "react"; +import { useSnackbar } from "@/context/snackbar"; +import { getGovernanceMetadata } from "@/lib/api"; import { - ModalWrapper, - ModalHeader, - Typography, Button, + ModalHeader, + ModalWrapper, OutlinedLightButton, + Typography } from "@atoms"; import { customPalette, IMAGES } from "@consts"; -import { useTranslations } from "next-intl"; import { useModal } from "@context"; +import { useScreenDimension } from "@hooks"; +import { CopyCard, Loading } from "@molecules"; import { Box } from "@mui/material"; import { GovActionModalState } from "@organisms"; -import { CopyCard, Loading } from "@molecules"; -import { GovActionMetadata } from "../types"; -import { getGovernanceMetadata } from "@/lib/api"; import { formatDisplayDate, getProposalTypeLabel, getShortenedGovActionId, - isResponseErrorI, + isResponseErrorI } from "@utils"; -import { useSnackbar } from "@/context/snackbar"; -import { useScreenDimension } from "@hooks"; +import { useTranslations } from "next-intl"; +import { useEffect, useState } from "react"; +import { GovActionMetadata } from "../types"; export const GovActionModal = () => { const t = useTranslations("Modals"); @@ -83,7 +83,7 @@ export const GovActionModal = () => { diff --git a/frontend/src/components/organisms/MyActions.tsx b/frontend/src/components/organisms/MyActions.tsx index b7f494ea..3fea4807 100644 --- a/frontend/src/components/organisms/MyActions.tsx +++ b/frontend/src/components/organisms/MyActions.tsx @@ -1,35 +1,37 @@ "use client"; -import React, { useCallback, useEffect, useState } from "react"; -import { Box } from "@mui/material"; -import { useTranslations } from "next-intl"; -import { NotFound } from "./NotFound"; -import { VotesTable } from "./VotesTable"; -import { countSelectedFilters, isEmpty } from "@utils"; -import { DataActionsBar } from "../molecules"; +import { useModal } from "@/context"; +import { getUserVotes } from "@/lib/api"; +import { PaginationMeta, VotesTableI } from "@/lib/requests"; import { LATEST_UPDATES_FILTERS, LATEST_UPDATES_SORTING, MY_ACTIONS_TABS, - PATHS, + PATHS } from "@consts"; +import { useManageQueryParams, usePagination } from "@hooks"; +import { Box } from "@mui/material"; +import { countSelectedFilters, isEmpty } from "@utils"; +import { useTranslations } from "next-intl"; +import { useRouter } from "next/navigation"; +import { useCallback, useEffect, useState } from "react"; +import { ShowMoreButton } from "../atoms"; +import { DataActionsBar } from "../molecules"; +import { NotFound } from "./NotFound"; import { PageTitleTabs } from "./PageTitleTabs"; -import { useModal } from "@/context"; -import { PaginationMeta, VotesTableI } from "@/lib/requests"; import { OpenPreviewReasoningModal } from "./types"; -import { getUserVotes } from "@/lib/api"; -import { usePagination, useManageQueryParams } from "@hooks"; -import { ShowMoreButton } from "../atoms"; -import { useRouter } from "next/navigation"; +import { VotesTable } from "./VotesTable"; export const MyActions = ({ actions, paginationMeta, error, + userId }: { actions: VotesTableI[]; paginationMeta: PaginationMeta; error?: string; + userId?: string; }) => { const t = useTranslations("MyActions"); const router = useRouter(); @@ -54,9 +56,9 @@ export const MyActions = ({ submit_time: null, //todo, update BE response end_time: action.gov_action_proposal_end_time, vote_submit_time: action.vote_submit_time, - vote: action.value, - }, - }, + vote: action.value + } + } }); }; @@ -68,6 +70,7 @@ export const MyActions = ({ : null, vote: chosenFilters.vote?.length > 0 ? chosenFilters.vote?.join(",") : null, sortBy: chosenSorting || null, + userId }; const { data, pagination, isLoading, loadMore } = usePagination( @@ -94,10 +97,10 @@ export const MyActions = ({ paddingBottom={4} display="flex" justifyContent={{ xxs: "flex-start", md: "space-between" }} - flexDirection={{ xxs: "column", md: "row" }} + flexDirection={{ xxs: "column", md: "column", lg: "row" }} alignItems={{ xxs: "left", md: "center" }} > - + router.push(tab.value)} diff --git a/frontend/src/components/organisms/VotesTable/VotesTableRow.tsx b/frontend/src/components/organisms/VotesTable/VotesTableRow.tsx index ef14f5c4..3600e49d 100644 --- a/frontend/src/components/organisms/VotesTable/VotesTableRow.tsx +++ b/frontend/src/components/organisms/VotesTable/VotesTableRow.tsx @@ -1,18 +1,15 @@ "use client"; -import React from "react"; -import { Card, TableDivider } from "@molecules"; -import { Box, Grid } from "@mui/material"; -import { UserAvatar, UserBasicInfo } from "@molecules"; +import { VotesTableI } from "@/lib/requests"; import { Button, OutlinedLightButton, Typography, VotePill } from "@atoms"; -import { GovActionModalState } from "../types"; import { customPalette, ICONS } from "@consts"; +import { useModal } from "@context"; +import { Card, TableDivider, UserAvatar, UserBasicInfo } from "@molecules"; +import { Box, Grid } from "@mui/material"; +import { getProposalTypeLabel, truncateText } from "@utils"; import { useTranslations } from "next-intl"; import Image from "next/image"; -import { truncateText } from "@utils"; -import { getProposalTypeLabel } from "@utils"; -import { useModal } from "@context"; -import { VotesTableI } from "@/lib/requests"; +import { GovActionModalState } from "../types"; interface Props { votes: VotesTableI; @@ -25,7 +22,7 @@ export const VotesTableRow = ({ votes, disabled, actionTitle, - onActionClick, + onActionClick }: Props) => { const t = useTranslations("LatestUpdates"); const { openModal } = useModal(); @@ -37,14 +34,14 @@ export const VotesTableRow = ({ reasoning_comment, gov_action_proposal_id, gov_action_proposal_title, - gov_action_proposal_type, + gov_action_proposal_type } = votes; const openGAModal = () => { openModal({ type: "govActionModal", state: { - id: gov_action_proposal_id, - }, + id: gov_action_proposal_id + } }); }; @@ -53,152 +50,178 @@ export const VotesTableRow = ({ item mb={3} sx={{ - opacity: disabled && 0.5, + opacity: disabled && 0.5 }} > - - + + - - - - - - - - + + + + + + - - - + + + + + - {t("govAction")} - + + {t("govAction")} + - - } + + } + > + {gov_action_proposal_title + ? gov_action_proposal_title + : t("notAvailable")} + + + + + + - {gov_action_proposal_title - ? truncateText(gov_action_proposal_title, 40) - : t("notAvailable")} - + + {t("govActionCategoryShort")} + + + {getProposalTypeLabel(gov_action_proposal_type)} + + - - - + + - {t("govActionCategoryShort")} - - - {getProposalTypeLabel(gov_action_proposal_type)} - + + {t("voted")} + + + + + - - - + + + - {t("voted")} - - - - + + {t("rationale")} + + + {reasoning_comment + ? truncateText(reasoning_comment, 100) + : t("notAvailable")} + + - - - + - - - diff --git a/frontend/src/constants/paths.ts b/frontend/src/constants/paths.ts index cc110112..1c1a12d6 100644 --- a/frontend/src/constants/paths.ts +++ b/frontend/src/constants/paths.ts @@ -8,13 +8,13 @@ export const PATHS = { logout: "/logout", admin: { home: "/admin", - dashboard: "/admin/dashboard", - }, + dashboard: "/admin/dashboard" + } }; export const EXTERNAL_LINKS = { guides: - "https://app.gitbook.com/o/Prbm1mtkwSsGWSvG1Bfd/s/IOUshfMdffqF4RObLhje/legal/terms-and-conditions", + "https://docs.gov.tools/about/what-is-the-constitutional-committee-portal" }; export const adminProtectedPath = PATHS.admin.dashboard; diff --git a/frontend/src/context/topBanner.tsx b/frontend/src/context/topBanner.tsx index 718cfd9a..375c2425 100644 --- a/frontend/src/context/topBanner.tsx +++ b/frontend/src/context/topBanner.tsx @@ -1,19 +1,19 @@ "use client"; -import { createContext, useContext, useEffect, useState } from "react"; import { Alert, Snackbar, SnackbarOrigin } from "@mui/material"; import { getItemFromSessionStorage, setItemToSessionStorage, - TOP_BANER, + TOP_BANER } from "@utils"; import { useTranslations } from "next-intl"; +import { createContext, useContext, useEffect, useState } from "react"; interface TopBannerContextI {} const defaultPosition = { vertical: "top", - horizontal: "center", + horizontal: "center" } as SnackbarOrigin; /** @@ -31,8 +31,12 @@ export function TopBannerContextProvider({ children }) { const t = useTranslations("TopBanner"); useEffect(() => { - let returningUser = getItemFromSessionStorage(TOP_BANER); - setDisplayPopUp(!returningUser); + if (process.env.NEXT_PUBLIC_IS_MAINNET === "true") { + setDisplayPopUp(false); + } else { + let returningUser = getItemFromSessionStorage(TOP_BANER); + setDisplayPopUp(!returningUser); + } }, [displayPopUp]); const handleClose = ( @@ -62,7 +66,7 @@ export function TopBannerContextProvider({ children }) { variant="filled" sx={{ minWidth: { xxs: "100vw" }, - backgroundColor: "#DEA029", + backgroundColor: "#DEA029" }} > {t("message")} diff --git a/frontend/src/context/usersnap.tsx b/frontend/src/context/usersnap.tsx index 8adba605..7d81596d 100644 --- a/frontend/src/context/usersnap.tsx +++ b/frontend/src/context/usersnap.tsx @@ -13,12 +13,23 @@ export const UsersnapProvider = ({ const [usersnapApi, setUsersnapApi] = useState(null); useEffect(() => { - loadSpace(process.env.NEXT_PUBLIC_USERSNAP_GLOBAL_API_KEY).then( - (api: SpaceApi) => { - api.init(initParams); - setUsersnapApi(api); - } - ); + if (process.env.NEXT_PUBLIC_USERSNAP_SPACE_API_KEY) { + const hiddenProjects = process.env.NEXT_PUBLIC_HIDDEN_USERSNAP_PROJECT_IDS?.split("||") || [] + loadSpace(process.env.NEXT_PUBLIC_USERSNAP_SPACE_API_KEY).then( + (api: SpaceApi) => { + api.init(initParams); + setUsersnapApi(api); + const hideHiddenProjects = () => { + hiddenProjects.forEach(p => { + api.hide(p); + }); + }; + hideHiddenProjects(); + api.on("close", hideHiddenProjects) + api.on("submit", hideHiddenProjects) + } + ); + } }, [initParams]); return ( diff --git a/ipfs-service/example.env b/ipfs-service/example.env index 62a72fb3..8da0740b 100644 --- a/ipfs-service/example.env +++ b/ipfs-service/example.env @@ -5,4 +5,17 @@ LISTEN_QUIC_ADDRESS='/ip4/0.0.0.0/udp/0/quic-v1' IPFS_PUBLIC_URL='https://ipfs.io/ipfs/' IPNS_PUBLIC_URL='https://ipfs.io/ipns/' -IPNS_CONSTITUTION_KEY_NAME='some-key-name' \ No newline at end of file +IPNS_CONSTITUTION_KEY_NAME='some-key-name' + +# Redis +REDIS_HOST=localhost +REDIS_PORT=6379 +REDIS_PASSWORD=password + +## DHT QUEUE ## +# Attempts +DHT_QUEUE_ATTEMPTS=10 +# Backoff (exponential, fixed) +DHT_QUEUE_BACKOFF_TYPE=exponential +# Backoff delay (ms) +DHT_QUEUE_BACKOFF_DELAY=1000 \ No newline at end of file diff --git a/ipfs-service/package-lock.json b/ipfs-service/package-lock.json index 7077e8c1..bed1315f 100644 --- a/ipfs-service/package-lock.json +++ b/ipfs-service/package-lock.json @@ -10,24 +10,33 @@ "license": "UNLICENSED", "dependencies": { "@aws-sdk/client-s3": "^3.552.0", + "@bull-board/nestjs": "^5.17.1", "@chainsafe/libp2p-gossipsub": "^13.0.0", "@helia/ipns": "^7.2.0", "@helia/unixfs": "^3.0.3", + "@nestjs/bullmq": "^10.1.1", "@nestjs/common": "^10.0.0", "@nestjs/config": "^3.2.2", "@nestjs/core": "^10.0.0", "@nestjs/platform-express": "^10.0.0", + "@types/morgan": "^1.9.7", "@types/multer": "^1.4.11", "blockstore-fs": "^1.1.10", "blockstore-s3": "^1.0.15", + "bull": "^4.12.3", + "bullmq": "^5.7.9", "datastore-level": "^10.1.8", "datastore-s3": "^11.1.11", "helia": "^4.0.1", "i": "^0.3.7", + "morgan": "^1.10.0", "multer": "^1.4.5-lts.1", + "nest-winston": "^1.9.4", "npm": "^10.5.2", "reflect-metadata": "^0.2.1", - "rxjs": "^7.8.1" + "rxjs": "^7.8.1", + "winston": "^3.11.0", + "winston-daily-rotate-file": "^5.0.0" }, "devDependencies": { "@nestjs/cli": "^10.0.0", @@ -3265,6 +3274,59 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, + "node_modules/@bull-board/api": { + "version": "5.21.4", + "resolved": "https://registry.npmjs.org/@bull-board/api/-/api-5.21.4.tgz", + "integrity": "sha512-mPWYIe7ucSKjvJuMGaYhY2A8miirFHtOnvhywxgXZi4342QiqfQCN/ZJiGqCWXN6yIzOQszopePlgemETupoaA==", + "license": "MIT", + "peer": true, + "dependencies": { + "redis-info": "^3.0.8" + }, + "peerDependencies": { + "@bull-board/ui": "5.21.4" + } + }, + "node_modules/@bull-board/express": { + "version": "5.21.4", + "resolved": "https://registry.npmjs.org/@bull-board/express/-/express-5.21.4.tgz", + "integrity": "sha512-naB1h6edTfw698x1W2MKMd8yc49Vu6X+SyJr48FGX2O0x4oeYKZHf2ZfSs6zMVnCMnsycoASiGck085np/9d0w==", + "license": "MIT", + "peer": true, + "dependencies": { + "@bull-board/api": "5.21.4", + "@bull-board/ui": "5.21.4", + "ejs": "^3.1.10", + "express": "^4.19.2" + } + }, + "node_modules/@bull-board/nestjs": { + "version": "5.21.4", + "resolved": "https://registry.npmjs.org/@bull-board/nestjs/-/nestjs-5.21.4.tgz", + "integrity": "sha512-pJaKf7wr/NnjUryrRuJb7jeO4mS8ejNY9GYY5k99RrEgqHkzzKaD8F9QmjsI6ZRfTKANfP7ALVXmPnjRHqSd8A==", + "license": "MIT", + "dependencies": { + "@nestjs/bull-shared": "^10.0.0" + }, + "peerDependencies": { + "@bull-board/api": "^5.21.4", + "@bull-board/express": "^5.21.4", + "@nestjs/common": "^9.0.0 || ^10.0.0", + "@nestjs/core": "^9.0.0 || ^10.0.0", + "reflect-metadata": "^0.1.13 || ^0.2.0", + "rxjs": "^7.8.1" + } + }, + "node_modules/@bull-board/ui": { + "version": "5.21.4", + "resolved": "https://registry.npmjs.org/@bull-board/ui/-/ui-5.21.4.tgz", + "integrity": "sha512-z2otIDhN4EpKRXeBjZZrqbOPtX5pFwkeEs6uGG77/4m6bSjUl+RTRFpzBSVxrwctqxge+S+CNOCmfDiDMGqi8Q==", + "license": "MIT", + "peer": true, + "dependencies": { + "@bull-board/api": "5.21.4" + } + }, "node_modules/@chainsafe/as-chacha20poly1305": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/@chainsafe/as-chacha20poly1305/-/as-chacha20poly1305-0.1.0.tgz", @@ -3386,6 +3448,17 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@dabh/diagnostics": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", + "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", + "license": "MIT", + "dependencies": { + "colorspace": "1.1.x", + "enabled": "2.0.x", + "kuler": "^2.0.0" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", @@ -3649,6 +3722,12 @@ "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", "dev": true }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==", + "license": "MIT" + }, "node_modules/@ipld/dag-cbor": { "version": "9.2.0", "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-9.2.0.tgz", @@ -4745,6 +4824,84 @@ "node": ">=8" } }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", + "integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", + "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", + "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", + "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", + "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", + "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@multiformats/base-x": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@multiformats/base-x/-/base-x-4.0.1.tgz", @@ -4817,6 +4974,46 @@ "npm": ">=7.0.0" } }, + "node_modules/@nestjs/bull-shared": { + "version": "10.2.1", + "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-10.2.1.tgz", + "integrity": "sha512-zvnTvSq6OJ92omcsFUwaUmPbM3PRgWkIusHPB5TE3IFS7nNdM3OwF+kfe56sgKjMtQQMe/56lok0S04OtPMX5Q==", + "license": "MIT", + "dependencies": { + "tslib": "2.6.3" + }, + "peerDependencies": { + "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", + "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0" + } + }, + "node_modules/@nestjs/bull-shared/node_modules/tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==", + "license": "0BSD" + }, + "node_modules/@nestjs/bullmq": { + "version": "10.2.1", + "resolved": "https://registry.npmjs.org/@nestjs/bullmq/-/bullmq-10.2.1.tgz", + "integrity": "sha512-nDR0hDabmtXt5gsb5R786BJsGIJoWh/79sVmRETXf4S45+fvdqG1XkCKAeHF9TO9USodw9m+XBNKysTnkY41gw==", + "license": "MIT", + "dependencies": { + "@nestjs/bull-shared": "^10.2.1", + "tslib": "2.6.3" + }, + "peerDependencies": { + "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", + "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0", + "bullmq": "^3.0.0 || ^4.0.0 || ^5.0.0" + } + }, + "node_modules/@nestjs/bullmq/node_modules/tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==", + "license": "0BSD" + }, "node_modules/@nestjs/cli": { "version": "10.3.2", "resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-10.3.2.tgz", @@ -7412,6 +7609,15 @@ "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==" }, + "node_modules/@types/morgan": { + "version": "1.9.9", + "resolved": "https://registry.npmjs.org/@types/morgan/-/morgan-1.9.9.tgz", + "integrity": "sha512-iRYSDKVaC6FkGSpEVVIvrRGw0DfJMiQzIn3qr2G5B3C//AWkulhXgaBd7tS9/J79GWSYMTHGs7PfI5b3Y8m+RQ==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/multer": { "version": "1.4.11", "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.11.tgz", @@ -7516,6 +7722,12 @@ "@types/superagent": "^8.1.0" } }, + "node_modules/@types/triple-beam": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.5.tgz", + "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==", + "license": "MIT" + }, "node_modules/@types/ws": { "version": "8.5.10", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.10.tgz", @@ -8216,6 +8428,12 @@ "node": ">=4" } }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, "node_modules/async-limiter": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz", @@ -8434,6 +8652,24 @@ } ] }, + "node_modules/basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.1.2" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/basic-auth/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -8693,6 +8929,48 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "node_modules/bull": { + "version": "4.16.2", + "resolved": "https://registry.npmjs.org/bull/-/bull-4.16.2.tgz", + "integrity": "sha512-VCy33UdPGiIoZHDTrslGXKXWxcIUHNH5Z82pihr8HicbIfAH4SHug1HxlwKEbibVv85hq8rJ9tKAW/cuxv2T0A==", + "license": "MIT", + "dependencies": { + "cron-parser": "^4.2.1", + "get-port": "^5.1.1", + "ioredis": "^5.3.2", + "lodash": "^4.17.21", + "msgpackr": "^1.10.1", + "semver": "^7.5.2", + "uuid": "^8.3.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/bull/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/bullmq": { + "version": "5.12.14", + "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.12.14.tgz", + "integrity": "sha512-mcSQHq9EY+DKtAP6XSmkP+0f1ifFithcpLTwo8WmUauArE9dxk45Gae3Fls1Nwf0Er9MoaDhPcglfe6LV/XCOg==", + "license": "MIT", + "dependencies": { + "cron-parser": "^4.6.0", + "ioredis": "^5.4.1", + "msgpackr": "^1.10.1", + "node-abort-controller": "^3.1.1", + "semver": "^7.5.4", + "tslib": "^2.0.0", + "uuid": "^9.0.0" + } + }, "node_modules/busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -9095,6 +9373,15 @@ "node": ">=6" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -9111,6 +9398,16 @@ "integrity": "sha512-lHl4d5/ONEbLlJvaJNtsF/Lz+WvB07u2ycqTYbdrq7UypDXailES4valYb2eWiJFxZlVmpGekfqoxQhzyFdT4Q==", "dev": true }, + "node_modules/color": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", + "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==", + "license": "MIT", + "dependencies": { + "color-convert": "^1.9.3", + "color-string": "^1.6.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -9127,12 +9424,47 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "license": "MIT", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "node_modules/color/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "license": "MIT", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "license": "MIT" + }, "node_modules/colorette": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.4.0.tgz", "integrity": "sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==", "peer": true }, + "node_modules/colorspace": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz", + "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==", + "license": "MIT", + "dependencies": { + "color": "^3.1.3", + "text-hex": "1.0.x" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -9470,6 +9802,18 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, + "node_modules/cron-parser": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", + "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", + "license": "MIT", + "dependencies": { + "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -9829,6 +10173,22 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, + "node_modules/ejs": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", + "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "jake": "^10.8.5" + }, + "bin": { + "ejs": "bin/cli.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/electron-to-chromium": { "version": "1.4.731", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.731.tgz", @@ -9851,6 +10211,12 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, + "node_modules/enabled": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", + "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==", + "license": "MIT" + }, "node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -10501,6 +10867,12 @@ "bser": "2.1.1" } }, + "node_modules/fecha": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", + "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==", + "license": "MIT" + }, "node_modules/figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", @@ -10537,6 +10909,48 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/file-stream-rotator": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/file-stream-rotator/-/file-stream-rotator-0.6.1.tgz", + "integrity": "sha512-u+dBid4PvZw17PmDeRcNOtCP9CCK/9lRN2w+r1xIS7yOL9JFrIBKTvrYsxT4P0pGtThYTn++QS5ChHaUov3+zQ==", + "license": "MIT", + "dependencies": { + "moment": "^2.29.1" + } + }, + "node_modules/filelist": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", + "integrity": "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "minimatch": "^5.0.1" + } + }, + "node_modules/filelist/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "license": "MIT", + "peer": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/filelist/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "license": "ISC", + "peer": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -10772,6 +11186,12 @@ "node": ">=0.4.0" } }, + "node_modules/fn.name": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", + "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==", + "license": "MIT" + }, "node_modules/foreground-child": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", @@ -10969,6 +11389,18 @@ "node": ">=8.0.0" } }, + "node_modules/get-port": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz", + "integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -11476,6 +11908,30 @@ "loose-envify": "^1.0.0" } }, + "node_modules/ioredis": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz", + "integrity": "sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==", + "license": "MIT", + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -12186,6 +12642,25 @@ "@pkgjs/parseargs": "^0.11.0" } }, + "node_modules/jake": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/jake/-/jake-10.9.2.tgz", + "integrity": "sha512-2P4SQ0HrLQ+fw6llpLnOaGAvN2Zu6778SJMrCUwns4fOoG9ayrTiZk3VV8sCPkVZF8ab0zksVpS8FDY5pRCNBA==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "async": "^3.2.3", + "chalk": "^4.0.2", + "filelist": "^1.0.4", + "minimatch": "^3.1.2" + }, + "bin": { + "jake": "bin/cli.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", @@ -13090,6 +13565,12 @@ "node": ">=6" } }, + "node_modules/kuler": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", + "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==", + "license": "MIT" + }, "node_modules/level": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/level/-/level-8.0.1.tgz", @@ -13240,6 +13721,18 @@ "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", "peer": true }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "license": "MIT" + }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -13273,6 +13766,32 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/logform": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.6.1.tgz", + "integrity": "sha512-CdaO738xRapbKIMVn2m4F6KTj4j7ooJ8POVnebSgKo3KBz5axNXRAL7ZdRjIV6NOr2Uf4vjtRkxrFETOioCqSA==", + "license": "MIT", + "dependencies": { + "@colors/colors": "1.6.0", + "@types/triple-beam": "^1.3.2", + "fecha": "^4.2.0", + "ms": "^2.1.1", + "safe-stable-stringify": "^2.3.1", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/logform/node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "license": "MIT", + "engines": { + "node": ">=0.1.90" + } + }, "node_modules/logkitty": { "version": "0.7.1", "resolved": "https://registry.npmjs.org/logkitty/-/logkitty-0.7.1.tgz", @@ -13411,6 +13930,15 @@ "yallist": "^3.0.2" } }, + "node_modules/luxon": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz", + "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/magic-string": { "version": "0.30.5", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.5.tgz", @@ -14164,6 +14692,58 @@ "node": ">=10" } }, + "node_modules/moment": { + "version": "2.30.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", + "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/morgan": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.10.0.tgz", + "integrity": "sha512-AbegBVI4sh6El+1gNwvD5YIck7nSA36weD7xvIxG4in80j/UoK8AEGaWnnz8v1GxonMCltmlNs5ZKbGvl9b1XQ==", + "license": "MIT", + "dependencies": { + "basic-auth": "~2.0.1", + "debug": "2.6.9", + "depd": "~2.0.0", + "on-finished": "~2.3.0", + "on-headers": "~1.0.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/morgan/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/morgan/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/morgan/node_modules/on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/mortice": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/mortice/-/mortice-3.0.4.tgz", @@ -14179,6 +14759,37 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/msgpackr": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.0.tgz", + "integrity": "sha512-I8qXuuALqJe5laEBYoFykChhSXLikZmUhccjGsPuSJ/7uPip2TJ7lwdIQwWSAi0jGZDXv4WOP8Qg65QZRuXxXw==", + "license": "MIT", + "optionalDependencies": { + "msgpackr-extract": "^3.0.2" + } + }, + "node_modules/msgpackr-extract": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", + "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "dependencies": { + "node-gyp-build-optional-packages": "5.2.2" + }, + "bin": { + "download-msgpackr-prebuilds": "bin/download-prebuilds.js" + }, + "optionalDependencies": { + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", + "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" + } + }, "node_modules/multer": { "version": "1.4.5-lts.1", "license": "MIT", @@ -14306,6 +14917,19 @@ "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==" }, + "node_modules/nest-winston": { + "version": "1.9.7", + "resolved": "https://registry.npmjs.org/nest-winston/-/nest-winston-1.9.7.tgz", + "integrity": "sha512-pTTgImRgv7urojsDvaTlenAjyJNLj7ywamfjzrhWKhLhp80AKLYNwf103dVHeqZWe+nzp/vd9DGRs/UN/YadOQ==", + "license": "MIT", + "dependencies": { + "fast-safe-stringify": "^2.1.1" + }, + "peerDependencies": { + "@nestjs/common": "^5.0.0 || ^6.6.0 || ^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0", + "winston": "^3.0.0" + } + }, "node_modules/netmask": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", @@ -14420,6 +15044,21 @@ "node-gyp-build-test": "build-test.js" } }, + "node_modules/node-gyp-build-optional-packages": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", + "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", + "license": "MIT", + "optional": true, + "dependencies": { + "detect-libc": "^2.0.1" + }, + "bin": { + "node-gyp-build-optional-packages": "bin.js", + "node-gyp-build-optional-packages-optional": "optional.js", + "node-gyp-build-optional-packages-test": "build-test.js" + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -17077,6 +17716,15 @@ "node": ">=0.10.0" } }, + "node_modules/object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, "node_modules/object-inspect": { "version": "1.13.1", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", @@ -17109,7 +17757,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", - "peer": true, "engines": { "node": ">= 0.8" } @@ -17122,6 +17769,15 @@ "wrappy": "1" } }, + "node_modules/one-time": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", + "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", + "license": "MIT", + "dependencies": { + "fn.name": "1.x.x" + } + }, "node_modules/onetime": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", @@ -18163,6 +18819,37 @@ "node": ">= 0.10" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-info": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/redis-info/-/redis-info-3.1.0.tgz", + "integrity": "sha512-ER4L9Sh/vm63DkIE0bkSjxluQlioBiBgf5w1UuldaW/3vPcecdljVDisZhmnCMvsxHNiARTTDDHGg9cGwTfrKg==", + "license": "MIT", + "peer": true, + "dependencies": { + "lodash": "^4.17.11" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "license": "MIT", + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/reflect-metadata": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", @@ -18538,6 +19225,15 @@ } ] }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -18904,6 +19600,21 @@ "simple-concat": "^1.0.0" } }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, + "node_modules/simple-swizzle/node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", + "license": "MIT" + }, "node_modules/sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", @@ -18993,6 +19704,15 @@ "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" }, + "node_modules/stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/stack-utils": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", @@ -19039,6 +19759,12 @@ "node": ">=8" } }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -19502,6 +20228,12 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/text-hex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==", + "license": "MIT" + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -19609,6 +20341,15 @@ "tree-kill": "cli.js" } }, + "node_modules/triple-beam": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", + "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==", + "license": "MIT", + "engines": { + "node": ">= 14.0.0" + } + }, "node_modules/truncate-utf8-bytes": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz", @@ -20228,6 +20969,109 @@ "integrity": "sha512-iBdZ57RDvnOR9AGBhML2vFZf7h8vmBjhoaZqODJBFWHVtKkDmKuHai3cx5PgVMrX5YDNp27AofYbAwctSS+vhQ==", "peer": true }, + "node_modules/winston": { + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.14.2.tgz", + "integrity": "sha512-CO8cdpBB2yqzEf8v895L+GNKYJiEq8eKlHU38af3snQBQ+sdAIUepjMSguOIJC7ICbzm0ZI+Af2If4vIJrtmOg==", + "license": "MIT", + "dependencies": { + "@colors/colors": "^1.6.0", + "@dabh/diagnostics": "^2.0.2", + "async": "^3.2.3", + "is-stream": "^2.0.0", + "logform": "^2.6.0", + "one-time": "^1.0.0", + "readable-stream": "^3.4.0", + "safe-stable-stringify": "^2.3.1", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.7.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/winston-daily-rotate-file": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/winston-daily-rotate-file/-/winston-daily-rotate-file-5.0.0.tgz", + "integrity": "sha512-JDjiXXkM5qvwY06733vf09I2wnMXpZEhxEVOSPenZMii+g7pcDcTBt2MRugnoi8BwVSuCT2jfRXBUy+n1Zz/Yw==", + "license": "MIT", + "dependencies": { + "file-stream-rotator": "^0.6.1", + "object-hash": "^3.0.0", + "triple-beam": "^1.4.1", + "winston-transport": "^4.7.0" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "winston": "^3" + } + }, + "node_modules/winston-transport": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.7.1.tgz", + "integrity": "sha512-wQCXXVgfv/wUPOfb2x0ruxzwkcZfxcktz6JIMUaPLmcNhO4bZTwA/WtDWK74xV3F2dKu8YadrFv0qhwYjVEwhA==", + "license": "MIT", + "dependencies": { + "logform": "^2.6.1", + "readable-stream": "^3.6.2", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/winston-transport/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/winston/node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "license": "MIT", + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/winston/node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/winston/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/wrap-ansi": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", diff --git a/ipfs-service/package.json b/ipfs-service/package.json index c8c8ad8c..84e9c68e 100644 --- a/ipfs-service/package.json +++ b/ipfs-service/package.json @@ -19,7 +19,10 @@ "test:e2e": "NODE_OPTIONS=--experimental-vm-modules jest --config ./test/jest-e2e.json" }, "dependencies": { + "@bull-board/nestjs": "^5.17.1", + "@types/morgan": "^1.9.7", "@aws-sdk/client-s3": "^3.552.0", + "morgan": "^1.10.0", "@chainsafe/libp2p-gossipsub": "^13.0.0", "@helia/ipns": "^7.2.0", "@helia/unixfs": "^3.0.3", @@ -27,6 +30,7 @@ "@nestjs/config": "^3.2.2", "@nestjs/core": "^10.0.0", "@nestjs/platform-express": "^10.0.0", + "@nestjs/bullmq": "^10.1.1", "@types/multer": "^1.4.11", "blockstore-fs": "^1.1.10", "blockstore-s3": "^1.0.15", @@ -37,7 +41,12 @@ "multer": "^1.4.5-lts.1", "npm": "^10.5.2", "reflect-metadata": "^0.2.1", - "rxjs": "^7.8.1" + "rxjs": "^7.8.1", + "winston": "^3.11.0", + "bull": "^4.12.3", + "bullmq": "^5.7.9", + "nest-winston": "^1.9.4", + "winston-daily-rotate-file": "^5.0.0" }, "devDependencies": { "@nestjs/cli": "^10.0.0", diff --git a/ipfs-service/src/app.controller.ts b/ipfs-service/src/app.controller.ts index 666c95c9..18471ad5 100644 --- a/ipfs-service/src/app.controller.ts +++ b/ipfs-service/src/app.controller.ts @@ -22,9 +22,9 @@ export class AppController { } @UseInterceptors(FileInterceptor('file')) - @Post() - async addDoc(@UploadedFile() file: Express.Multer.File): Promise { - return await this.appService.addDoc(file); + @Post('file') + async addFile(@UploadedFile() file: Express.Multer.File): Promise { + return await this.appService.addFile(file); } @Post('json') @@ -34,7 +34,7 @@ export class AppController { @Get(':cid') async getDoc(@Param('cid') cid: string): Promise { - const doc = await this.appService.getDoc(cid); + const doc = await this.appService.getDocByCid(cid); if (!doc) { throw new NotFoundException(`Document with cid: ${cid} not found`); } diff --git a/ipfs-service/src/app.module.ts b/ipfs-service/src/app.module.ts index b70e2554..fab71986 100644 --- a/ipfs-service/src/app.module.ts +++ b/ipfs-service/src/app.module.ts @@ -2,10 +2,20 @@ import { Module } from '@nestjs/common'; import { AppController } from './app.controller.js'; import { AppService } from './app.service.js'; import { ConfigModule } from '@nestjs/config'; +import { BullModule } from '@nestjs/bullmq'; +import { BullmqModule } from './bullmq/bullmq.module.js'; +import { ProvideToDHTProcessor } from './queues/processors/provide-to-dht.processor.js'; +import { ProvideToDHTProducer } from './queues/producers/provide-to-dht.producer.js'; +import { QUEUE_NAME_PROVIDE_TO_DHT } from './constants/bullmq.constants.js'; @Module({ - imports: [ConfigModule.forRoot({ isGlobal: true })], + imports: [ + ConfigModule.forRoot({ isGlobal: true }), + BullmqModule, + BullModule.registerQueue({ + name: QUEUE_NAME_PROVIDE_TO_DHT, + }),], controllers: [AppController], - providers: [AppService], + providers: [AppService, ProvideToDHTProcessor, ProvideToDHTProducer], }) export class AppModule {} diff --git a/ipfs-service/src/app.service.ts b/ipfs-service/src/app.service.ts index 94a04430..d566e24b 100644 --- a/ipfs-service/src/app.service.ts +++ b/ipfs-service/src/app.service.ts @@ -6,23 +6,19 @@ import { } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { createHelia } from 'helia'; -import type { HeliaLibp2p } from 'helia'; import { CID } from 'multiformats/cid'; -import { createLibp2p } from 'libp2p'; import { bootstrap } from '@libp2p/bootstrap'; import { identify } from '@libp2p/identify'; import { webSockets } from '@libp2p/websockets'; -import { all } from '@libp2p/websockets/filters'; import { tcp } from '@libp2p/tcp'; import { noise } from '@chainsafe/libp2p-noise'; import { yamux } from '@chainsafe/libp2p-yamux'; -import { mplex } from '@libp2p/mplex'; import { unixfs } from '@helia/unixfs'; import { FsBlockstore } from 'blockstore-fs'; import { LevelDatastore } from 'datastore-level'; import { IPNS, ipns } from '@helia/ipns'; import { keychain, type Keychain } from '@libp2p/keychain'; -import { kadDHT, removePrivateAddressesMapper } from '@libp2p/kad-dht'; +import { kadDHT } from '@libp2p/kad-dht'; import { ipnsSelector } from 'ipns/selector'; import { ipnsValidator } from 'ipns/validator'; import { dcutr } from '@libp2p/dcutr'; @@ -31,8 +27,6 @@ import { ping } from '@libp2p/ping'; import { uPnPNAT } from '@libp2p/upnp-nat'; import { mdns } from '@libp2p/mdns'; import { createDelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client'; -//import { gossipsub } from '@chainsafe/libp2p-gossipsub'; -//import { webRTC, webRTCDirect } from '@libp2p/webrtc'; import { circuitRelayTransport, circuitRelayServer, @@ -41,6 +35,7 @@ import { IpfsMapper } from './mapper/ipfs.mapper.js'; import { IpfsDto } from './dto/ipfs.dto.js'; import { PeerId } from '@libp2p/interface'; import { config } from 'dotenv'; +import { ProvideToDHTProducer } from './queues/producers/provide-to-dht.producer.js'; config(); const libp2pOptions = { @@ -112,7 +107,7 @@ export class AppService implements OnModuleInit { private ipnsPeerId: PeerId; private logger = new Logger(AppService.name); - constructor(private readonly configService: ConfigService) {} + constructor(private readonly provideToDHTProducer: ProvideToDHTProducer) {} async onModuleInit() { console.log(`Initialization helia...`); @@ -168,7 +163,7 @@ export class AppService implements OnModuleInit { this.logger.log(`IPNS PeerID: ${this.ipnsPeerId}`); } - async addDoc(file: Express.Multer.File): Promise { + async addFile(file: Express.Multer.File): Promise { try { this.fs = unixfs(this.helia); const fileBuffer = Buffer.from(file.buffer); @@ -182,8 +177,8 @@ export class AppService implements OnModuleInit { const ret1 = this.helia.pins.add(cid); ret1.next().then((res) => this.logger.log(`Pinned: ${res.value}`)); - // Announce CID to the DHT - this.provideCidtoDHT(cid); + // Announce CID to the DHT via queue + await this.provideToDHTProducer.addToQueue(cid.toString()); // Publish the name await this.ipns.publish(this.ipnsPeerId, cid); @@ -202,19 +197,6 @@ export class AppService implements OnModuleInit { } } - async getDoc(cid: string): Promise { - this.fs = unixfs(this.helia); - const decoder = new TextDecoder(); - let text = ''; - - for await (const chunk of this.fs.cat(cid)) { - text += decoder.decode(chunk, { - stream: true, - }); - } - return IpfsMapper.ipfsToIpfsDto(cid, text); - } - async addJson(json: string): Promise { try { this.fs = unixfs(this.helia); @@ -226,8 +208,8 @@ export class AppService implements OnModuleInit { const ret1 = this.helia.pins.add(cid); ret1.next().then((res) => this.logger.log(`Pinned json: ${res.value}`)); - // Announce CID to the DHT - this.provideCidtoDHT(cid); + // Announce CID to the DHT via queue + await this.provideToDHTProducer.addToQueue(cid.toString()); const url = process.env.IPFS_PUBLIC_URL + cid.toString(); @@ -238,32 +220,29 @@ export class AppService implements OnModuleInit { } } - private provideCidtoDHT(cid, retryDelay = 5000) { - let attempt = 0; - let errCode = null; - - const attemptToProvide = async () => { - try { - await this.helia.libp2p.contentRouting.provide(cid); - this.logger.log(`Announced CID to the DHT: ${cid.toString()}`); - } catch (error) { - this.logger.error(`Error announcing CID to the DHT: ${error}`); - errCode = error.code; - if (errCode === 'ERR_QUERY_ABORTED') { - attempt++; - this.logger.log(`Retrying... (${attempt})`); - await new Promise((resolve) => setTimeout(resolve, retryDelay)); - attemptToProvide(); // Retry - } else { - this.logger.log(`CID: ${cid} has not been announced`); - this.logger.error(error); - throw new InternalServerErrorException(error); - } - } - }; + async getDocByCid(cidString: string): Promise { + try { + this.fs = unixfs(this.helia); + const decoder = new TextDecoder(); + const cid = CID.parse(cidString); - attemptToProvide(); + let text = ''; + for await (const chunk of this.fs.cat(cid)) { + text += decoder.decode(chunk, { + stream: true, + }); + } + return IpfsMapper.ipfsToIpfsDto(cidString, text); + } catch (error) { + this.logger.error(`Failed to get doc by CID - ${cidString} error: ${error}`); + return null; } +} + + async provideCidtoDHTViaQueue(cid: CID) { + await this.helia.libp2p.contentRouting.provide(cid); + this.logger.log(`Announced CID to the DHT: ${cid.toString()}`); +} async getIpnsUrl(): Promise { if (!this.ipnsPeerId) { diff --git a/ipfs-service/src/bullmq/bullmq.module.ts b/ipfs-service/src/bullmq/bullmq.module.ts new file mode 100644 index 00000000..8dd4d1fd --- /dev/null +++ b/ipfs-service/src/bullmq/bullmq.module.ts @@ -0,0 +1,31 @@ +import { BullBoardModule } from "@bull-board/nestjs"; +import { BullModule } from "@nestjs/bullmq"; +import { Module } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; +import { QUEUE_NAME_PROVIDE_TO_DHT } from "../constants/bullmq.constants.js"; +import { BullMQAdapter } from "@bull-board/api/bullMQAdapter.js"; +import { ExpressAdapter } from "@bull-board/express"; + +@Module({ + imports: [ + BullModule.forRootAsync({ + useFactory: async (configService: ConfigService) => ({ + connection: { + host: configService.getOrThrow("REDIS_HOST"), + port: configService.getOrThrow("REDIS_PORT"), + password: configService.getOrThrow("REDIS_PASSWORD"), + }, + }), + inject: [ConfigService], + }), + BullBoardModule.forRoot({ + route: "/bull-board", + adapter: ExpressAdapter, + }), + BullBoardModule.forFeature({ + name: QUEUE_NAME_PROVIDE_TO_DHT, + adapter: BullMQAdapter, + }), + ], +}) +export class BullmqModule {} diff --git a/ipfs-service/src/constants/bullmq.constants.ts b/ipfs-service/src/constants/bullmq.constants.ts new file mode 100644 index 00000000..b2df7f12 --- /dev/null +++ b/ipfs-service/src/constants/bullmq.constants.ts @@ -0,0 +1,2 @@ +export const QUEUE_NAME_PROVIDE_TO_DHT = "provide_to_dht_queue"; +export const JOB_NAME_PROVIDE_TO_DHT = "provide_to_dht_job"; diff --git a/ipfs-service/src/main.ts b/ipfs-service/src/main.ts index c589b826..91f07a94 100644 --- a/ipfs-service/src/main.ts +++ b/ipfs-service/src/main.ts @@ -1,12 +1,27 @@ import { NestFactory } from '@nestjs/core'; import { AppModule } from './app.module.js'; -import { EventEmitter } from 'events'; +import morgan from 'morgan'; +import { LoggerFactory } from './util/logger-factory.js'; +import { LoggerService } from '@nestjs/common'; + async function bootstrap() { - // Increase the max listeners for EventEmitters - EventEmitter.defaultMaxListeners = 50; // Set this to the desired number - const app = await NestFactory.create(AppModule); + + const logger: LoggerService = LoggerFactory('IPFS Service'); + + const morganFormat = + ':method :url :status :res[content-length] - :response-time ms'; + app.use( + morgan(morganFormat, { + stream: { + write: (message) => { + logger.log(message); + }, + }, + }), +); + await app.listen(3001); } bootstrap().catch((err) => { diff --git a/ipfs-service/src/queues/processors/provide-to-dht.processor.ts b/ipfs-service/src/queues/processors/provide-to-dht.processor.ts new file mode 100644 index 00000000..af9048e3 --- /dev/null +++ b/ipfs-service/src/queues/processors/provide-to-dht.processor.ts @@ -0,0 +1,40 @@ +import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq'; +import { Job } from 'bullmq'; +import { + JOB_NAME_PROVIDE_TO_DHT, + QUEUE_NAME_PROVIDE_TO_DHT, +} from '../../constants/bullmq.constants.js'; +import { Logger } from '@nestjs/common'; +import { AppService } from '../../app.service.js'; +import { CID } from 'multiformats/cid'; + +@Processor(QUEUE_NAME_PROVIDE_TO_DHT) +export class ProvideToDHTProcessor extends WorkerHost { + protected readonly logger = new Logger(ProvideToDHTProcessor.name); + constructor(private readonly appService: AppService) { + super(); + } + + async process(job: Job): Promise { + switch (job.name) { + case JOB_NAME_PROVIDE_TO_DHT: { + const cid = CID.parse(job.data); + this.logger.debug('Job triggered: Provide to DHT - CID:', cid.toString()); + try { + await this.appService.provideCidtoDHTViaQueue(cid); + } catch (error) { + this.logger.error(`Error processing job id: ${job.id}, name: ${job.name}. - Error: ${error}`); + throw error; + } + } + } + } + + @OnWorkerEvent('completed') + onCompleted(job: Job) { + const { id, name, queueName, finishedOn, returnvalue } = job; + const completionTime = finishedOn ? new Date(finishedOn).toISOString() : ''; + this.logger.log( + `Job Finished - id: ${id}, name: ${name} in queue ${queueName} on ${completionTime}.`,); + } +} diff --git a/ipfs-service/src/queues/producers/provide-to-dht.producer.ts b/ipfs-service/src/queues/producers/provide-to-dht.producer.ts new file mode 100644 index 00000000..4d890a90 --- /dev/null +++ b/ipfs-service/src/queues/producers/provide-to-dht.producer.ts @@ -0,0 +1,30 @@ +import { InjectQueue } from '@nestjs/bullmq'; +import { Injectable } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { Queue } from 'bullmq'; +import { + JOB_NAME_PROVIDE_TO_DHT, + QUEUE_NAME_PROVIDE_TO_DHT, +} from '../../constants/bullmq.constants.js'; +import { randomUUID } from 'crypto'; + +@Injectable() +export class ProvideToDHTProducer { + constructor( + @InjectQueue(QUEUE_NAME_PROVIDE_TO_DHT) private readonly provideToDHTQueue: Queue, + private readonly configService: ConfigService + ) {} + + async addToQueue(inputData: string) { + const job = await this.provideToDHTQueue.add(JOB_NAME_PROVIDE_TO_DHT, inputData, { + jobId: randomUUID(), + removeOnComplete: true, + removeOnFail: false, + attempts: this.configService.getOrThrow('DHT_QUEUE_ATTEMPTS'), + backoff: { + type: this.configService.getOrThrow('DHT_QUEUE_BACKOFF_TYPE'), + delay: this.configService.getOrThrow('DHT_QUEUE_BACKOFF_DELAY') }, + }); + return job; + } +} diff --git a/ipfs-service/src/util/logger-factory.ts b/ipfs-service/src/util/logger-factory.ts new file mode 100644 index 00000000..e6e21ed1 --- /dev/null +++ b/ipfs-service/src/util/logger-factory.ts @@ -0,0 +1,37 @@ +import { format, transports } from 'winston'; +const { json, timestamp } = format; +// require('winston-daily-rotate-file'); +import { + WinstonModule, + utilities as nestWinstonModuleUtilities, +} from 'nest-winston'; +// eslint-disable-next-line @typescript-eslint/no-unused-vars +import DailyRotateFile from 'winston-daily-rotate-file'; + +export const LoggerFactory = (appName: string) => { + //DailyRotateFile func() + const fileRotateTransport = new DailyRotateFile({ + filename: 'logs/logs-%DATE%.log', + datePattern: 'YYYY-MM-DD', + maxFiles: '14d', + }); + + const consoleFormat = format.combine( + timestamp({ + format: 'MMM-DD-YYYY HH:mm:ss', + }), + json(), + nestWinstonModuleUtilities.format.nestLike(appName, { + colors: true, + prettyPrint: true, + }), + ); + + return WinstonModule.createLogger({ + level: 'info', + transports: [ + fileRotateTransport, + new transports.Console({ format: consoleFormat }), + ], + }); +}; diff --git a/worker-service/src/common/common-service.ts b/worker-service/src/common/common-service.ts index ace3ca5e..fff75a1d 100644 --- a/worker-service/src/common/common-service.ts +++ b/worker-service/src/common/common-service.ts @@ -86,6 +86,10 @@ export abstract class CommonService { } } + /** + * The response data structure example is located on this link: + * https://github.com/cardano-foundation/CIPs/blob/master/CIP-0108/examples/no-confidence.jsonld + **/ async getGovActionProposalFromUrl( url: string, ): Promise> { @@ -95,14 +99,14 @@ export abstract class CommonService { const title = jsonData.body?.title; const abstract = jsonData.body?.abstract; const govActionProposal: Partial = { - title: title?.['@value'], - abstract: abstract?.['@value'], + title: title, + abstract: abstract, govMetadataUrl: url, }; return govActionProposal; } catch (e) { - this.logger.log( - `There has been an exception when fetching data for governance action proposal: ${e}`, + this.logger.warn( + `Error when fetching GAP metadata from url ${url}; Message: ${e}`, ); return null; } diff --git a/worker-service/src/governance/entities/gov-action-proposal.entity.ts b/worker-service/src/governance/entities/gov-action-proposal.entity.ts index 484298da..4972ed9f 100644 --- a/worker-service/src/governance/entities/gov-action-proposal.entity.ts +++ b/worker-service/src/governance/entities/gov-action-proposal.entity.ts @@ -21,7 +21,6 @@ export class GovActionProposal extends CommonEntity { @Column({ name: 'title', type: 'varchar', - length: 80, nullable: true, }) title: string; @@ -29,7 +28,6 @@ export class GovActionProposal extends CommonEntity { @Column({ name: 'abstract', type: 'varchar', - length: 2500, nullable: true, }) abstract: string; diff --git a/worker-service/src/governance/queues/processors/gov-action-proposal.processor.ts b/worker-service/src/governance/queues/processors/gov-action-proposal.processor.ts index dde7dab3..56597e54 100644 --- a/worker-service/src/governance/queues/processors/gov-action-proposal.processor.ts +++ b/worker-service/src/governance/queues/processors/gov-action-proposal.processor.ts @@ -7,30 +7,32 @@ import { import { Logger } from '@nestjs/common'; import { GovActionProposalService } from '../../services/gov-action-proposal.service'; + @Processor(QUEUE_NAME_DB_SYNC_GOV_ACTIONS) export class GovActionsProposalProcessor extends WorkerHost { protected readonly logger = new Logger(GovActionsProposalProcessor.name); constructor( - private readonly govActionProposalService: GovActionProposalService, + private readonly govActionProposalService: GovActionProposalService ) { super(); } async process(job: Job): Promise { switch (job.name) { case JOB_NAME_GOV_ACTIONS_SYNC: { - this.logger.debug('Data from db-sync for gov action proposals job'); - return job.data; + try { + this.logger.debug( + `Processing GAP - amount of GAPs to be processed, ${job.data?.length}`, + ); + await this.govActionProposalService.storeGovActionProposalData( + job.data, + ); + } catch (error) { + this.logger.error( + `Error processing job id: ${job.id}, name: ${job.name}. - Error: ${error}`, + ); + throw error; + } } } } - - @OnWorkerEvent('completed') - onCompleted(job: Job) { - const { id, name, queueName, finishedOn, returnvalue } = job; - const completionTime = finishedOn ? new Date(finishedOn).toISOString() : ''; - this.govActionProposalService.storeGovActionProposalData(returnvalue); - this.logger.log( - `Job id: ${id}, name: ${name} completed in queue ${queueName} on ${completionTime}.`, - ); - } } diff --git a/worker-service/src/governance/queues/processors/vote.processor.ts b/worker-service/src/governance/queues/processors/vote.processor.ts index c8f946af..7cdf2a6c 100644 --- a/worker-service/src/governance/queues/processors/vote.processor.ts +++ b/worker-service/src/governance/queues/processors/vote.processor.ts @@ -6,6 +6,7 @@ import { } from '../../../common/constants/bullmq.constants'; import { Logger } from '@nestjs/common'; import { VoteService } from '../../services/vote.service'; +import { VoteRequest } from 'src/governance/dto/vote.request'; @Processor(QUEUE_NAME_DB_SYNC_VOTES) export class VoteProcessor extends WorkerHost { @@ -17,19 +18,22 @@ export class VoteProcessor extends WorkerHost { async process(job: Job): Promise { switch (job.name) { case JOB_NAME_VOTE_SYNC: { - this.logger.debug('Data from db-sync for votes job'); - return job.data; + try { + const voteRequests: VoteRequest[] = job.data; + const addresses = voteRequests.map( + (voteRequest) => voteRequest.hotAddress, + ); + this.logger.debug( + `Processing votes for addresses:, ${JSON.stringify(addresses)}`, + ); + await this.voteService.storeVoteData(voteRequests); + } catch (error) { + this.logger.error( + `Error processing job id: ${job.id}, name: ${job.name}. - Error: ${error}`, + ); + throw error; + } } } } - - @OnWorkerEvent('completed') - onCompleted(job: Job) { - const { id, name, queueName, finishedOn, returnvalue } = job; - const completionTime = finishedOn ? new Date(finishedOn).toISOString() : ''; - this.voteService.storeVoteData(returnvalue); - this.logger.log( - `Job id: ${id}, name: ${name} completed in queue ${queueName} on ${completionTime}.`, - ); - } } diff --git a/worker-service/src/governance/services/gov-action-proposal.service.ts b/worker-service/src/governance/services/gov-action-proposal.service.ts index a28d8366..93309977 100644 --- a/worker-service/src/governance/services/gov-action-proposal.service.ts +++ b/worker-service/src/governance/services/gov-action-proposal.service.ts @@ -33,16 +33,9 @@ export class GovActionProposalService extends CommonService { private readonly configService: ConfigService, ) { super(dataSource); - // this.cronInterval = - // this.configService.get('GOV_ACTION_PROPOSALS_JOB_FREQUENCY') || - // '0 * * * * *'; this.logger = new Logger(GovActionProposalService.name); } - // getCronExpression(): string { - // return this.cronInterval; - // } - async storeGovActionProposalData( govActionProposalRequests: GovActionProposalRequest[], ): Promise { @@ -95,7 +88,7 @@ export class GovActionProposalService extends CommonService { ); const results: GovActionProposalRequest[] = []; - dbData.forEach((govActionProposal) => { + dbData?.forEach((govActionProposal) => { results.push( GovActionProposalMapper.dbSyncToGovActionProposalRequest( govActionProposal, diff --git a/worker-service/src/governance/services/vote.service.ts b/worker-service/src/governance/services/vote.service.ts index b7df82b1..cac690b2 100644 --- a/worker-service/src/governance/services/vote.service.ts +++ b/worker-service/src/governance/services/vote.service.ts @@ -162,12 +162,13 @@ export class VoteService extends CommonService { try { const prefix = '\\x'; // prefix for each hot address const addresses = [...mapHotAddresses.keys()].map((key) => prefix + key); + this.logger.debug(`Addresses for fetching from db sync: ${addresses}`); const dbData = await this.getDataFromSqlFile( SQL_FILE_PATH.GET_VOTES, addresses, ); const results: VoteRequest[] = []; - dbData.forEach((vote) => { + dbData?.forEach((vote) => { results.push(VoteMapper.dbSyncToVoteRequest(vote, mapHotAddresses)); }); return results;