Skip to content

Commit

Permalink
fix(setup): revert transfer of object and add region to config
Browse files Browse the repository at this point in the history
  • Loading branch information
ctrlc03 committed Jul 19, 2023
1 parent c407bee commit 690da25
Show file tree
Hide file tree
Showing 10 changed files with 15 additions and 208 deletions.
27 changes: 1 addition & 26 deletions packages/actions/src/helpers/functions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -435,29 +435,4 @@ export const finalizeCeremony = async (functions: Functions, ceremonyId: string)
await cf({
ceremonyId
})
}

/**
* Transfer an object between two buckets
* @param functions <Functions> - the Firebase cloud functions object instance.
* @param originBucketName <string> - the name of the origin bucket.
* @param originObjectKey <string> - the key of the origin object.
* @param destinationBucketName <string> - the name of the destination bucket.
* @param destinationObjectKey <string> - the key of the destination object.
*/
export const transferObject = async (
functions: Functions,
originBucketName: string,
originObjectKey: string,
destinationBucketName: string,
destinationObjectKey: string
) => {
const cf = httpsCallable(functions, commonTerms.cloudFunctionsNames.transferObject)

await cf({
originBucketName,
originObjectKey,
destinationBucketName,
destinationObjectKey
})
}
}
2 changes: 1 addition & 1 deletion packages/actions/src/helpers/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ export const parseCeremonyFile = async (path: string, cleanup: boolean = false):

try {
await s3.send(new HeadObjectCommand({
Bucket: circuitData.artifacts.bucket,
Bucket: artifacts.bucket,
Key: r1csPath
}))
} catch (error: any) {
Expand Down
3 changes: 1 addition & 2 deletions packages/actions/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,7 @@ export {
verifyContribution,
checkAndPrepareCoordinatorForFinalization,
finalizeCircuit,
finalizeCeremony,
transferObject
finalizeCeremony
} from "./helpers/functions"
export { toHex, blake512FromPath, computeSHA256ToHex, compareHashes } from "./helpers/crypto"
export {
Expand Down
1 change: 1 addition & 0 deletions packages/actions/test/data/artifacts/ceremonySetup.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
},
"artifacts": {
"bucket": "test-qfi-p0tion-development-environment",
"region": "us-east-1",
"r1csStoragePath": "circuits/circuit/circuit.r1cs",
"wasmStoragePath": "circuits/circuit/circuit.wasm"
},
Expand Down
58 changes: 2 additions & 56 deletions packages/actions/test/unit/storage.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@ import {
sleep,
cleanUpRecursively,
mockCeremoniesCleanup,
generatePseudoRandomStringOfNumbers,
uploadFileToS3
} from "../utils/index"
import { fakeCeremoniesData, fakeCircuitsData, fakeUsersData } from "../data/samples"
import {
Expand All @@ -36,12 +34,12 @@ import {
commonTerms,
genesisZkeyIndex,
checkIfObjectExist,
generateGetObjectPreSignedUrl
generateGetObjectPreSignedUrl,
} from "../../src/index"
import { TestingEnvironment } from "../../src/types/enums"
import { ChunkWithUrl, ETagWithPartNumber } from "../../src/types/index"
import { getChunksAndPreSignedUrls, getWasmStorageFilePath, uploadParts } from "../../src/helpers/storage"
import { completeMultiPartUpload, openMultiPartUpload, transferObject } from "../../src/helpers/functions"
import { completeMultiPartUpload, openMultiPartUpload } from "../../src/helpers/functions"

chai.use(chaiAsPromised)

Expand Down Expand Up @@ -595,58 +593,6 @@ describe("Storage", () => {
await cleanUpRecursively(adminFirestore, fakeCeremoniesData.fakeCeremonyOpenedFixed.uid)
})
})

describe("transferObject", () => {
// we need two buckets - source and destination
const sourceBucketName = randomBytes(10).toString()
const destinationBucketName = randomBytes(10).toString()
const objectKey = "test.txt"
fs.writeFileSync(objectKey, "test")

beforeAll(async () => {
// login as coordinator
await signInWithEmailAndPassword(userAuth, users[1].data.email, passwords[1])
// create the buckets and upload the file
await createS3Bucket(userFunctions, sourceBucketName)
await createS3Bucket(userFunctions, destinationBucketName)
await uploadFileToS3(
sourceBucketName,
objectKey,
objectKey
)
})

it("should successfully transfer an object between buckets", async () => {
await expect(transferObject(
userFunctions,
sourceBucketName,
objectKey,
destinationBucketName,
objectKey
)).to.be.fulfilled
})

it("should transfer an object between buckets in different regions", async () => {})
it("should throw when trying to transfer an object that does not exist", async () => {
await expect(transferObject(
userFunctions,
sourceBucketName,
"i-dont-exist.txt",
destinationBucketName,
objectKey
)).to.be.rejected
})

afterAll(async () => {
// delete the buckets
await deleteObjectFromS3(sourceBucketName, objectKey)
await deleteObjectFromS3(destinationBucketName, objectKey)
await deleteBucket(sourceBucketName)
await deleteBucket(destinationBucketName)

fs.unlinkSync(objectKey)
})
})
}

describe("getR1csStorageFilePath", () => {
Expand Down
3 changes: 1 addition & 2 deletions packages/backend/src/functions/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,7 @@ export {
generateGetObjectPreSignedUrl,
startMultiPartUpload,
generatePreSignedUrlsParts,
completeMultiPartUpload,
transferObject
completeMultiPartUpload
} from "./storage"
export { checkAndRemoveBlockingContributor, resumeContributionAfterTimeoutExpiration } from "./timeout"

Expand Down
58 changes: 1 addition & 57 deletions packages/backend/src/functions/storage.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
import * as functions from "firebase-functions"
import admin from "firebase-admin"
import {
S3Client,
CopyObjectCommand,
GetObjectCommand,
CreateMultipartUploadCommand,
UploadPartCommand,
Expand Down Expand Up @@ -32,8 +30,7 @@ import {
CompleteMultiPartUploadData,
CreateBucketData,
GeneratePreSignedUrlsPartsData,
StartMultiPartUploadData,
TransferObjectData
StartMultiPartUploadData
} from "../types/index"

dotenv.config()
Expand Down Expand Up @@ -231,59 +228,6 @@ export const createBucket = functions
}
})

/**
* Transfer a public object from one bucket to another.
*/
export const transferObject = functions
.runWith({
memory: "512MB"
})
.https.onCall(async (data: TransferObjectData, context: functions.https.CallableContext) => {
// Check if the user has the coordinator claim.
if (!context.auth || !context.auth.token.coordinator) logAndThrowError(COMMON_ERRORS.CM_NOT_COORDINATOR_ROLE)

if (
!data.sourceBucketName ||
!data.sourceObjectKey ||
!data.destinationBucketName ||
!data.destinationObjectKey ||
!data.sourceRegion
) logAndThrowError(COMMON_ERRORS.CM_MISSING_OR_WRONG_INPUT_DATA)

// Connect to S3 client.
const S3 = await getS3Client()

const copyParams = {
Bucket: data.destinationBucketName,
CopySource: `${data.sourceBucketName}/${encodeURIComponent(data.sourceObjectKey)}`,
Key: data.destinationObjectKey,
}

const command = new CopyObjectCommand(copyParams)

try {
// Execute S3 command.
await S3.send(command)

printLog(
`The object was copied from ${data.sourceBucketName} to ${data.destinationBucketName}`,
LogLevel.LOG
)
} catch (error: any) {
// eslint-disable-next-line @typescript-eslint/no-shadow
if (error.$metadata.httpStatusCode === 403) logAndThrowError(SPECIFIC_ERRORS.SE_STORAGE_MISSING_PERMISSIONS)

if (error.$metadata.httpStatusCode !== 200) {
const commonError = COMMON_ERRORS.CM_INVALID_REQUEST
const additionalDetails = error.toString()

logAndThrowError(makeError(commonError.code, commonError.message, additionalDetails))
}
}

logAndThrowError(SPECIFIC_ERRORS.SE_STORAGE_TRASNSFER_FAILED)
})

/**
* Check if a specified object exist in a given AWS S3 bucket.
* @returns <Promise<boolean>> - true if the object exist in the given bucket; otherwise false.
Expand Down
5 changes: 0 additions & 5 deletions packages/backend/src/lib/errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -115,11 +115,6 @@ export const SPECIFIC_ERRORS = {
"Unable to delete the AWS S3 object from the provided ceremony bucket.",
"This could happen if the local file or the bucket do not exist."
),
SE_STORAGE_TRASNSFER_FAILED: makeError(
"failed-precondition",
"Unable to transfer the circuit artifacts to the ceremony bucket.",
"This could happen if the files or the bucket do not exist, or the permissions are not public."
),
SE_CONTRIBUTE_NO_CEREMONY_CIRCUITS: makeError(
"not-found",
"There is no circuit associated with the ceremony.",
Expand Down
17 changes: 0 additions & 17 deletions packages/backend/src/types/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,23 +33,6 @@ export type BucketAndObjectKeyData = {
objectKey: string
}

/**
* Group all the necessary data needed for running the `transferObject` cloud function.
* @typedef {Object} TransferObjectData
* @property {string} sourceRegion - the region of the source bucket.
* @property {string} sourceBucketName - the name of the source bucket.
* @property {string} sourceObjectKey - the unique key to identify the object inside the given AWS S3 source bucket.
* @property {string} destinationBucketName - the name of the destination bucket.
* @property {string} destinationObjectKey - the unique key to identify the object inside the given AWS S3 destination bucket.
*/
export type TransferObjectData = {
sourceRegion: string
sourceBucketName: string
sourceObjectKey: string
destinationBucketName: string
destinationObjectKey: string
}

/**
* Group all the necessary data needed for running the `startMultiPartUpload` cloud function.
* @typedef {Object} StartMultiPartUploadData
Expand Down
49 changes: 7 additions & 42 deletions packages/phase2cli/src/commands/setup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,12 @@

import { zKey } from "snarkjs"
import boxen from "boxen"
import { createWriteStream, Dirent, renameSync, createReadStream } from "fs"
import { createWriteStream, Dirent, renameSync } from "fs"
import { pipeline } from "node:stream"
import { promisify } from "node:util"
import fetch from "node-fetch"
import { Functions } from "firebase/functions"
import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3"
import { getSignedUrl } from "@aws-sdk/s3-request-presigner"
import {
CeremonyTimeoutType,
CircomCompilerData,
Expand Down Expand Up @@ -65,7 +64,6 @@ import {
checkAndMakeNewDirectoryIfNonexistent
} from "../lib/files.js"
import { Readable } from "stream"
import { transferObject } from "@p0tion/actions"

/**
* Handle whatever is needed to obtain the input data for a circuit that the coordinator would like to add to the ceremony.
Expand Down Expand Up @@ -462,37 +460,6 @@ export const handleCircuitArtifactUploadToStorage = async (
spinner.succeed(`Upload of (${theme.text.bold(completeFilename)}) file completed successfully`)
}

/**
* Transfer a file between two buckets
* @param firebaseFunctions <Functions> - the Firebase Cloud Functions instance connected to the current application.
* @param bucketName <string> - the ceremony bucket name.
* @param storageFilePath <string> - the storage (bucket) path where the file should be uploaded.
* @param sourceBucketName <string> - the source bucket name.
* @param sourceObjectKey <string> - the source object key.
* @param completeFilename <string> - the complete filename.
*/
export const handleCircuitArtifactTransferToStorage = async (
firebaseFunctions: Functions,
bucketName: string,
storageFilePath: string,
sourceBucketName: string,
sourceObjectKey: string,
completeFilename: string
) => {
const spinner = customSpinner(`Transfering ${theme.text.bold(completeFilename)} file to ceremony storage...`, `clock`)
spinner.start()

await transferObject(
firebaseFunctions,
sourceBucketName,
bucketName,
sourceObjectKey,
storageFilePath
)

spinner.succeed(`Transfer of (${theme.text.bold(completeFilename)}) file completed successfully`)
}

/**
* Setup command.
* @notice The setup command allows the coordinator of the ceremony to prepare the next ceremony by interacting with the CLI.
Expand Down Expand Up @@ -608,23 +575,21 @@ const setup = async (cmd: { template?: string, auth?: string}) => {
circuit.files.potFilename
)

// Move r1cs between buckets
await handleCircuitArtifactTransferToStorage(
// Upload r1cs to Storage.
await handleCircuitArtifactUploadToStorage(
firebaseFunctions,
ceremonySetupData.circuitArtifacts[index].artifacts.bucket,
ceremonySetupData.circuitArtifacts[index].artifacts.r1csStoragePath,
bucketName,
circuit.files.r1csStoragePath,
r1csLocalPathAndFileName,
circuit.files.r1csFilename
)

// Move wasm between buckets.
await handleCircuitArtifactTransferToStorage(
// Upload wasm to Storage.
await handleCircuitArtifactUploadToStorage(
firebaseFunctions,
ceremonySetupData.circuitArtifacts[index].artifacts.bucket,
ceremonySetupData.circuitArtifacts[index].artifacts.wasmStoragePath,
bucketName,
circuit.files.wasmStoragePath,
r1csLocalPathAndFileName,
circuit.files.wasmFilename
)

Expand Down

0 comments on commit 690da25

Please sign in to comment.