diff --git a/backend/packages/backend/.env.example b/backend/packages/backend/.env.example index 91d7ad1b0..c674eaebb 100644 --- a/backend/packages/backend/.env.example +++ b/backend/packages/backend/.env.example @@ -8,12 +8,11 @@ TEST_ACCOUNT_BALANCE=100000000000 NODE_API_ENDPOINT=http://localhost:3223 -INFURA_PROJECT_ID= -INFURA_PROJECT_SECRET= -LOCAL_IPFS_NODE_URL=http://localhost:5001 -USE_LOCAL_IPFS_NODE=false - -IPFS_ENDPOINT=https://ipfs.infura.io/ipfs/ +S3_API= +S3_ACCESS_KEY_ID= +S3_SECRET_ACCESS_KEY= +S3_BUCKET= +S3_PUBLIC_ENDPOINT=https://voting-static.opensquare.io DEVELOPMENT=true diff --git a/backend/packages/backend/ecosystem.config.js b/backend/packages/backend/ecosystem.config.js index 52cdf819d..45b07ea8e 100644 --- a/backend/packages/backend/ecosystem.config.js +++ b/backend/packages/backend/ecosystem.config.js @@ -1,8 +1,8 @@ module.exports = { apps: [ { - name: "voting-pin-to-ipfs", - script: "src/scripts/pin-to-ipfs.js", + name: "voting-save-to-s3", + script: "src/scripts/save-to-s3.js", log_date_format: "YYYY-MM-DD HH:mm Z", env: { NODE_ENV: "development", diff --git a/backend/packages/backend/package.json b/backend/packages/backend/package.json index f7438b462..6d0bd5160 100644 --- a/backend/packages/backend/package.json +++ b/backend/packages/backend/package.json @@ -7,9 +7,10 @@ "start": "NODE_ENV=production node src/index.js", "lint": "eslint .", "test": "jest src --runInBand", - "pin": "pm2 start ecosystem.config.js --only voting-pin-to-ipfs --env production" + "saveToS3": "pm2 start ecosystem.config.js --only voting-save-to-s3 --env production" }, "dependencies": { + "@aws-sdk/client-s3": "^3.1079.0", "@koa/cors": "^5.0.0", "@koa/multer": "^3.0.0", "@osn/ipfs": "^0.2.0", diff --git a/backend/packages/backend/src/__mocks__/env.js b/backend/packages/backend/src/__mocks__/env.js index 0b8745f42..d1579a4d3 100644 --- a/backend/packages/backend/src/__mocks__/env.js +++ b/backend/packages/backend/src/__mocks__/env.js @@ -1,15 +1,5 @@ const NODE_API_ENDPOINT = "http://127.0.0.1:8081"; -const INFURA_PROJECT_ID = "xxx"; -const INFURA_PROJECT_SECRET = "yyy"; -const LOCAL_IPFS_NODE_URL = "http://ipfs.dotask.cc:5001"; -const USE_LOCAL_IFPS_NODE = false; -const IPFS_ENDPOINT = "https://ipfs.infura.io/ipfs/"; module.exports = { NODE_API_ENDPOINT, - INFURA_PROJECT_ID, - INFURA_PROJECT_SECRET, - LOCAL_IPFS_NODE_URL, - USE_LOCAL_IFPS_NODE, - IPFS_ENDPOINT, }; diff --git a/backend/packages/backend/src/env.js b/backend/packages/backend/src/env.js index 6cea44fcb..66020cb52 100644 --- a/backend/packages/backend/src/env.js +++ b/backend/packages/backend/src/env.js @@ -1,20 +1,6 @@ const NODE_API_ENDPOINT = process.env.NODE_API_ENDPOINT || "http://127.0.0.1:8081"; -const INFURA_PROJECT_ID = process.env.INFURA_PROJECT_ID || ""; -const INFURA_PROJECT_SECRET = process.env.INFURA_PROJECT_SECRET || ""; -const LOCAL_IPFS_NODE_URL = - process.env.LOCAL_IPFS_NODE_URL || "http://ipfs.dotask.cc:5001"; -const IPFS_ENDPOINT = - process.env.IPFS_ENDPOINT || "https://ipfs.infura.io/ipfs/"; -const USE_LOCAL_IFPS_NODE = ["true", "True", "TRUE", "1"].includes( - process.env.USE_LOCAL_IFPS_NODE, -); module.exports = { NODE_API_ENDPOINT, - INFURA_PROJECT_ID, - INFURA_PROJECT_SECRET, - LOCAL_IPFS_NODE_URL, - USE_LOCAL_IFPS_NODE, - IPFS_ENDPOINT, }; diff --git a/backend/packages/backend/src/features/files/file.controller.js b/backend/packages/backend/src/features/files/file.controller.js index 2e28f2650..112d4ec67 100644 --- a/backend/packages/backend/src/features/files/file.controller.js +++ b/backend/packages/backend/src/features/files/file.controller.js @@ -1,24 +1,23 @@ const { HttpError } = require("../../exc"); -const { IPFS_ENDPOINT } = require("../../env"); -const { pinFileToIpfs } = require("../../services/ipfs.service/pin"); +const { saveFileToStorage } = require("../../services/s3.service/saveToS3"); const Megabyte = 1024 * 1024; const trimTailSlash = (url) => url.endsWith("/") ? url.substr(0, url.length - 1) : url; -async function getFile(ctx) { - const { hash } = ctx.params; - ctx.redirect(`${trimTailSlash(IPFS_ENDPOINT)}/${hash}`); -} - -async function getIpfsEndpoint(ctx) { +async function getS3Endpoint(ctx) { ctx.body = { - endpoint: trimTailSlash(IPFS_ENDPOINT), + endpoint: trimTailSlash(process.env.S3_PUBLIC_ENDPOINT), }; } -async function upload(ctx) { +async function getS3File(ctx) { + const { cid } = ctx.params; + ctx.redirect(`${trimTailSlash(process.env.S3_PUBLIC_ENDPOINT)}/${cid}`); +} + +async function uploadFile(ctx) { const file = ctx.request.file; if (!file) { throw new HttpError(400, "File is missing"); @@ -32,10 +31,11 @@ async function upload(ctx) { } try { - const hash = await pinFileToIpfs(file); + const hash = await saveFileToStorage(file); + ctx.body = { - hash, - url: `${trimTailSlash(IPFS_ENDPOINT)}/${hash}`, + cid: hash, + url: `${trimTailSlash(process.env.S3_PUBLIC_ENDPOINT)}/${hash}`, }; } catch (e) { throw new HttpError(500, e.message); @@ -43,7 +43,7 @@ async function upload(ctx) { } module.exports = { - getFile, - getIpfsEndpoint, - upload, + getS3Endpoint, + getS3File, + uploadFile, }; diff --git a/backend/packages/backend/src/features/files/routes.js b/backend/packages/backend/src/features/files/routes.js index cfc515f93..ec7fe2f6d 100644 --- a/backend/packages/backend/src/features/files/routes.js +++ b/backend/packages/backend/src/features/files/routes.js @@ -5,8 +5,8 @@ const fileController = require("./file.controller"); const router = new Router(); const upload = multer(); -router.post("/ipfs/files", upload.single("banner"), fileController.upload); -router.get("/ipfs/files/:hash", fileController.getFile); -router.get("/ipfs/endpoint", fileController.getIpfsEndpoint); +router.post("/s3/files", upload.single("banner"), fileController.uploadFile); +router.get("/s3/files/:cid", fileController.getS3File); +router.get("/s3/endpoint", fileController.getS3Endpoint); module.exports = router; diff --git a/backend/packages/backend/src/features/spaces/common.js b/backend/packages/backend/src/features/spaces/common.js index 700f15099..cc746ec16 100644 --- a/backend/packages/backend/src/features/spaces/common.js +++ b/backend/packages/backend/src/features/spaces/common.js @@ -1,7 +1,8 @@ const { isAddress } = require("@polkadot/util-crypto"); const { HttpError } = require("../../exc"); const { getSpaceCollection } = require("../../mongo"); -const { ipfsAddBuffer } = require("../../services/ipfs.service/ipfs"); +const { uploadBufferToS3 } = require("../../services/s3.service"); +const { cidOfBuffer } = require("../../utils/cid"); const { isSameAddress } = require("../../utils/address"); const dataUriToBuffer = (dataUri) => @@ -11,8 +12,16 @@ async function pinLogo(logo) { let logoCid = null; if (logo) { const buf = await dataUriToBuffer(logo); - const added = await ipfsAddBuffer(buf); - logoCid = added.path; + const rawBuf = Buffer.from(buf); + logoCid = await cidOfBuffer(rawBuf); + + await uploadBufferToS3( + { + buffer: rawBuf, + mimetype: buf.type, + }, + logoCid, + ); } return logoCid; diff --git a/backend/packages/backend/src/index.js b/backend/packages/backend/src/index.js index ea39ff87d..278301f99 100644 --- a/backend/packages/backend/src/index.js +++ b/backend/packages/backend/src/index.js @@ -10,6 +10,15 @@ const cors = require("@koa/cors"); const { reloadSpaces } = require("./spaces"); const socketSetup = require("./socket"); const spaceNotify = require("./scripts/space-notify"); +const { validateEnv } = require("./utils/requireEnv"); + +validateEnv([ + "S3_API", + "S3_ACCESS_KEY_ID", + "S3_SECRET_ACCESS_KEY", + "S3_BUCKET", + "S3_PUBLIC_ENDPOINT", +]); const app = new Koa(); diff --git a/backend/packages/backend/src/scripts/repin-to-ipfs.js b/backend/packages/backend/src/scripts/resave-to-s3.js similarity index 76% rename from backend/packages/backend/src/scripts/repin-to-ipfs.js rename to backend/packages/backend/src/scripts/resave-to-s3.js index c4ed8978a..8bb84c952 100644 --- a/backend/packages/backend/src/scripts/repin-to-ipfs.js +++ b/backend/packages/backend/src/scripts/resave-to-s3.js @@ -8,7 +8,7 @@ const { getCommentCollection, getAppendantCollection, } = require("../mongo"); -const { repinCollectionDataToIpfs } = require("../services/ipfs.service"); +const { resaveCollectionDataToS3 } = require("../services/s3.service/saveToS3"); const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -18,10 +18,10 @@ async function startPin() { const voteCol = await getVoteCollection(); const commentCol = await getCommentCollection(); await Promise.all([ - repinCollectionDataToIpfs(proposalCol), - repinCollectionDataToIpfs(appendantCol), - repinCollectionDataToIpfs(voteCol), - repinCollectionDataToIpfs(commentCol), + resaveCollectionDataToS3(proposalCol), + resaveCollectionDataToS3(appendantCol), + resaveCollectionDataToS3(voteCol), + resaveCollectionDataToS3(commentCol), ]); } diff --git a/backend/packages/backend/src/scripts/pin-to-ipfs.js b/backend/packages/backend/src/scripts/save-to-s3.js similarity index 72% rename from backend/packages/backend/src/scripts/pin-to-ipfs.js rename to backend/packages/backend/src/scripts/save-to-s3.js index 24b1b7ea8..4e6703f90 100644 --- a/backend/packages/backend/src/scripts/pin-to-ipfs.js +++ b/backend/packages/backend/src/scripts/save-to-s3.js @@ -7,7 +7,7 @@ const { getCommentCollection, getAppendantCollection, } = require("../mongo"); -const { pinCollectionDataToIpfs } = require("../services/ipfs.service"); +const { saveCollectionDataToS3 } = require("../services/s3.service/saveToS3"); async function startPin() { const proposalCol = await getProposalCollection(); @@ -15,10 +15,10 @@ async function startPin() { const voteCol = await getVoteCollection(); const commentCol = await getCommentCollection(); await Promise.all([ - pinCollectionDataToIpfs(proposalCol), - pinCollectionDataToIpfs(appendantCol), - pinCollectionDataToIpfs(voteCol), - pinCollectionDataToIpfs(commentCol), + saveCollectionDataToS3(proposalCol), + saveCollectionDataToS3(appendantCol), + saveCollectionDataToS3(voteCol), + saveCollectionDataToS3(commentCol), ]); } diff --git a/backend/packages/backend/src/services/ipfs.service/__mocks__/pin.js b/backend/packages/backend/src/services/ipfs.service/__mocks__/pin.js deleted file mode 100644 index 78c5ba837..000000000 --- a/backend/packages/backend/src/services/ipfs.service/__mocks__/pin.js +++ /dev/null @@ -1,8 +0,0 @@ -async function pinJsonToIpfs(json) { - console.log("Use mock pinJsonToIpfs"); - return "QmZPLzpusNUu6Re4CRhqcw74tEi9Rb9ecYePk2h2Sr2CjG"; -} - -module.exports = { - pinJsonToIpfs, -}; diff --git a/backend/packages/backend/src/services/ipfs.service/index.js b/backend/packages/backend/src/services/ipfs.service/index.js deleted file mode 100644 index 17dc1d5c3..000000000 --- a/backend/packages/backend/src/services/ipfs.service/index.js +++ /dev/null @@ -1,57 +0,0 @@ -const Hash = require("ipfs-only-hash"); -const { pinJsonToIpfs } = require("./pin"); -const pick = require("lodash/pick"); - -async function pinJsonToIpfsWithTimeout(json, timeout) { - const errorMsg = "Pin json to ipfs timeout"; - return await Promise.race([ - new Promise((_, reject) => - setTimeout(() => reject(new Error(errorMsg)), timeout), - ), - pinJsonToIpfs(json), - ]); -} - -async function getObjectBufAndCid(data) { - const jsonData = JSON.stringify(data); - const buf = Buffer.from(jsonData); - const cid = await Hash.of(buf); - return { buf, cid }; -} - -async function pinCollectionDataToIpfs(col) { - const items = await col.find({ pinHash: null }).toArray(); - await pinItemsToIpfs(col, items); -} - -async function repinCollectionDataToIpfs(col) { - const items = await col.find().toArray(); - await pinItemsToIpfs(col, items); -} - -async function pinItemsToIpfs(col, items) { - for (const item of items) { - try { - const toBePin = { - ...pick(item, ["data", "address", "signature", "delegators"]), - // version 2: replace `msg` with `data` - version: "2", - }; - const pinHash = await pinJsonToIpfs(toBePin); - - if (pinHash) { - await col.updateOne({ _id: item._id }, { $set: { pinHash } }); - console.log(`Save pin hash ${pinHash} to: ${item._id}`); - } - } catch (e) { - console.error(e); - } - } -} - -module.exports = { - pinJsonToIpfsWithTimeout, - getObjectBufAndCid, - pinCollectionDataToIpfs, - repinCollectionDataToIpfs, -}; diff --git a/backend/packages/backend/src/services/ipfs.service/ipfs.js b/backend/packages/backend/src/services/ipfs.service/ipfs.js deleted file mode 100644 index 6b29d1cac..000000000 --- a/backend/packages/backend/src/services/ipfs.service/ipfs.js +++ /dev/null @@ -1,51 +0,0 @@ -const Hash = require("ipfs-only-hash"); -const { CID } = require("multiformats/cid"); -const { getClients, ClientOptions, ClientMode } = require("@osn/ipfs"); -const { - INFURA_PROJECT_ID, - INFURA_PROJECT_SECRET, - LOCAL_IPFS_NODE_URL, - USE_LOCAL_IFPS_NODE, -} = require("../../env"); - -let ipfsClient; - -async function getIpfsClient() { - if (!ipfsClient) { - [ipfsClient] = await getClients( - USE_LOCAL_IFPS_NODE ? ClientMode.Local : ClientMode.Infura, - new ClientOptions( - INFURA_PROJECT_ID, - INFURA_PROJECT_SECRET, - LOCAL_IPFS_NODE_URL, - ), - ); - } - - return ipfsClient; -} - -async function ipfsAddJson(data) { - const ipfsClient = await getIpfsClient(); - const added = await ipfsClient.add(JSON.stringify(data)); - return added; -} - -async function ipfsAddBuffer(data) { - const ipfsClient = await getIpfsClient(); - const added = await ipfsClient.add(data); - return added; -} - -async function cidOf(obj) { - const data = JSON.stringify(obj); - const cidV0 = await Hash.of(Buffer.from(data)); - const cid = CID.parse(cidV0); - return cid.toV1().toString(); -} - -module.exports = { - cidOf, - ipfsAddJson, - ipfsAddBuffer, -}; diff --git a/backend/packages/backend/src/services/ipfs.service/pin.js b/backend/packages/backend/src/services/ipfs.service/pin.js deleted file mode 100644 index 96030a5a1..000000000 --- a/backend/packages/backend/src/services/ipfs.service/pin.js +++ /dev/null @@ -1,16 +0,0 @@ -const { ipfsAddBuffer, ipfsAddJson } = require("./ipfs"); - -async function pinJsonToIpfs(json) { - const { path } = await ipfsAddJson(json); - return path; -} - -async function pinFileToIpfs(file) { - const { path } = await ipfsAddBuffer(file.buffer); - return path; -} - -module.exports = { - pinFileToIpfs, - pinJsonToIpfs, -}; diff --git a/backend/packages/backend/src/services/proposal.service/common.js b/backend/packages/backend/src/services/proposal.service/common.js index 0257a5d37..2c8446364 100644 --- a/backend/packages/backend/src/services/proposal.service/common.js +++ b/backend/packages/backend/src/services/proposal.service/common.js @@ -1,7 +1,7 @@ const { getObjectBufAndCid, - pinJsonToIpfsWithTimeout, -} = require("../ipfs.service"); + saveJsonToStorageWithTimeout, +} = require("../s3.service/saveToS3"); const { enhancedSqrtOfBalance } = require("../../utils"); const { getProposalCollection } = require("../../mongo"); const { HttpError } = require("../../exc"); @@ -45,7 +45,7 @@ async function pinData(rawData) { let pinHash = null; try { - pinHash = await pinJsonToIpfsWithTimeout(toBePin, 3000); + pinHash = await saveJsonToStorageWithTimeout(toBePin, 3000); } catch (e) { console.error(e); } diff --git a/backend/packages/backend/src/services/proposal.service/tests/comment.test.js b/backend/packages/backend/src/services/proposal.service/tests/comment.test.js deleted file mode 100644 index dc57901af..000000000 --- a/backend/packages/backend/src/services/proposal.service/tests/comment.test.js +++ /dev/null @@ -1,95 +0,0 @@ -/** - * @jest-environment node - */ - -const { ObjectId } = require("mongodb"); -const { getComments, postComment } = require(".."); -const { getDb } = require("../../../mongo"); -const { karuraConfig } = require("../../../scripts/spaces/karura"); - -jest.mock("../../ipfs.service/pin"); - -describe("Comment Test", () => { - let db; - let proposalCol; - let commentCol; - - beforeAll(async () => { - db = await getDb(); - proposalCol = db.getCollection("proposal"); - commentCol = db.getCollection("comment"); - - await proposalCol.insertMany([ - { - _id: ObjectId("616e454d8f661dee51552e6a"), - space: "karura", - networksConfig: [karuraConfig], - postUid: "36", - title: "Proposal#1", - content: "Proposal#1", - contentType: "markdown", - choiceType: "single", - choices: ["Aye", "Nay"], - startDate: Date.now() - 3600 * 1000, - endDate: Date.now() + 3600 * 1000, - snapshotHeights: { karura: 779147 }, - weightStrategy: ["balance-of", "quadratic-balance-of"], - cid: "QmRgpY9WUuxKkptU6Sj9ow97u5QtFeRYnx4pqXzVBfMapE", - pinHash: "QmRgpY9WUuxKkptU6Sj9ow97u5QtFeRYnx4pqXzVBfMapE", - }, - ]); - }); - - afterAll(async () => { - await commentCol.drop(); - await proposalCol.drop(); - await db.close(); - }); - - test("postComment", async () => { - let proposalCid = "QmRgpY9WUuxKkptU6Sj9ow97u5QtFeRYnx4pqXzVBfMapE", - content = "Nay", - contentType = "Remark", - data = {}, - address = "14uSQFo8WePzpZaBUZHKa8AqDxigH1wjtNDHaRAG8ipSqQS3", - commenterNetwork = "karura", - signature = - "0x4899168d4b6597d4d3fbff99a5a8fabd3d56509973a38e805bf9900f9498c50078bfbf382bb5a182d6e0edb7bddac7bc18c5a2ef54b3896a39200ffcdd99d181"; - - await postComment( - proposalCid, - content, - contentType, - commenterNetwork, - data, - address, - signature - ); - - const result = await getComments( - "QmRgpY9WUuxKkptU6Sj9ow97u5QtFeRYnx4pqXzVBfMapE", - 1, - 10 - ); - expect(result).toMatchObject({ - items: [ - { - _id: new ObjectId("616e6cce1493c854d201ef48"), - proposal: new ObjectId("616e454d8f661dee51552e6a"), - content: "Nay", - contentType: "Remark", - data: {}, - address: "14uSQFo8WePzpZaBUZHKa8AqDxigH1wjtNDHaRAG8ipSqQS3", - signature: - "0x4899168d4b6597d4d3fbff99a5a8fabd3d56509973a38e805bf9900f9498c50078bfbf382bb5a182d6e0edb7bddac7bc18c5a2ef54b3896a39200ffcdd99d181", - height: 1, - cid: "QmZPLzpusNUu6Re4CRhqcw74tEi9Rb9ecYePk2h2Sr2CjG", - pinHash: "QmZPLzpusNUu6Re4CRhqcw74tEi9Rb9ecYePk2h2Sr2CjG", - }, - ], - total: 1, - page: 1, - pageSize: 10, - }); - }); -}); diff --git a/backend/packages/backend/src/services/proposal.service/tests/create.test.js b/backend/packages/backend/src/services/proposal.service/tests/create.test.js deleted file mode 100644 index cd06c5ad5..000000000 --- a/backend/packages/backend/src/services/proposal.service/tests/create.test.js +++ /dev/null @@ -1,98 +0,0 @@ -/** - * @jest-environment node - */ - -jest.mock("../../ipfs.service/pin"); -jest.mock("../../node.service"); - -const { createProposal } = require(".."); -const { getDb } = require("../../../mongo"); -const { reloadSpaces } = require("../../../spaces"); -const { karuraConfig } = require("../../../scripts/spaces/karura"); - -describe("Create Proposal Test", () => { - let db; - let proposalCol; - let spaceCol; - - beforeAll(async () => { - db = await getDb(); - proposalCol = await db.getCollection("proposal"); - spaceCol = db.getCollection("space"); - - await spaceCol.insertMany([karuraConfig]); - - await reloadSpaces(); - }); - - afterAll(async () => { - await proposalCol.drop(); - await spaceCol.drop(); - - await db.close(); - }); - - test("createProposal", async () => { - const space = "karura", - title = "Title", - content = "Content", - contentType = "markdown", - choices = ["Nay", "Aye"], - choiceType = "single", - startDate = Date.now() - 1000 * 3600, - endDate = Date.now() + 1000 * 3600, - snapshotHeights = { karura: 8000, moonriver: 8000, bifrost: 8000 }, - realProposer = null, - proposerNetwork = "karura", - banner = null, - data = {}, - address = "5EgqZkmeq5c2VVb5TYwMkXHWRQ9V5q6pucoeoiUcWE455Vcp", - signature = ""; - - await createProposal({ - space, - title, - content, - contentType, - choiceType, - choices, - startDate, - endDate, - snapshotHeights, - realProposer, - proposerNetwork, - banner, - data, - address, - signature, - }); - - const items = await proposalCol.find({}).toArray(); - expect(items).toMatchObject([ - { - space, - postUid: "1", - networksConfig: { - symbol: karuraConfig.symbol, - decimals: karuraConfig.decimals, - networks: karuraConfig.networks, - }, - title, - content, - contentType, - choiceType, - choices, - startDate, - endDate, - snapshotHeights, - weightStrategy: ["balance-of", "quadratic-balance-of", "biased-voting"], - proposerNetwork: "karura", - banner: null, - data: {}, - address: "5EgqZkmeq5c2VVb5TYwMkXHWRQ9V5q6pucoeoiUcWE455Vcp", - cid: "QmZAxGcgQvryyUM7wUr1t5MGaAGNG5T3SbmAvtabG8KMn8", - signature: "", - }, - ]); - }); -}); diff --git a/backend/packages/backend/src/services/proposal.service/tests/query.test.js b/backend/packages/backend/src/services/proposal.service/tests/query.test.js deleted file mode 100644 index 2ac2d6239..000000000 --- a/backend/packages/backend/src/services/proposal.service/tests/query.test.js +++ /dev/null @@ -1,148 +0,0 @@ -/** - * @jest-environment node - */ - -jest.mock("../../../env"); -jest.mock("../../ipfs.service/pin"); - -const { reloadSpaces } = require("../../../spaces"); -const { ObjectId, Decimal128 } = require("mongodb"); -const { getProposalById } = require(".."); -const { getDb } = require("../../../mongo"); -const { karuraConfig } = require("../../../scripts/spaces/karura"); - -describe("Query Proposal Test", () => { - let db; - let proposalCol; - let voteCol; - let spaceCol; - - beforeAll(async () => { - db = await getDb(); - proposalCol = db.getCollection("proposal"); - voteCol = db.getCollection("vote"); - spaceCol = db.getCollection("space"); - - await spaceCol.insertMany([karuraConfig]); - - await reloadSpaces(); - - await proposalCol.insertMany([ - { - _id: ObjectId("616e454d8f661dee51552e6a"), - space: "karura", - networksConfig: [karuraConfig], - postUid: "36", - title: "Proposal#1", - content: "Proposal#1", - contentType: "markdown", - choiceType: "single", - choices: ["Aye", "Nay"], - startDate: Date.now() - 3600 * 1000, - endDate: Date.now() + 3600 * 1000, - snapshotHeights: { karura: 779147 }, - weightStrategy: ["balance-of", "quadratic-balance-of"], - cid: "QmRgpY9WUuxKkptU6Sj9ow97u5QtFeRYnx4pqXzVBfMapE", - pinHash: "QmRgpY9WUuxKkptU6Sj9ow97u5QtFeRYnx4pqXzVBfMapE", - }, - ]); - - await voteCol.insertMany([ - { - proposal: ObjectId("616e454d8f661dee51552e6a"), - voter: "13d8i62igrsVw2bbRBzMtg7fH298n8exz7Y8y1Ty4K5bFuzf", - address: "13d8i62igrsVw2bbRBzMtg7fH298n8exz7Y8y1Ty4K5bFuzf", - voterNetwork: "karura", - choices: ["Aye"], - cid: "QmXoJvVbwdtBU5JC1ZgzZUtE2eBE92BVjmUsjzK98RJVec", - data: {}, - pinHash: "QmXoJvVbwdtBU5JC1ZgzZUtE2eBE92BVjmUsjzK98RJVec", - remark: "", - signature: - "0x4899168d4b6597d4d3fbff99a5a8fabd3d56509973a38e805bf9900f9498c50078bfbf382bb5a182d6e0edb7bddac7bc18c5a2ef54b3896a39200ffcdd99d181", - weights: { - balanceOf: new Decimal128("11111110000000"), - quadraticBalanceOf: new Decimal128("3333333166666.6625"), - }, - }, - { - proposal: ObjectId("616e454d8f661dee51552e6a"), - voter: "14uSQFo8WePzpZaBUZHKa8AqDxigH1wjtNDHaRAG8ipSqQS3", - address: "14uSQFo8WePzpZaBUZHKa8AqDxigH1wjtNDHaRAG8ipSqQS3", - voterNetwork: "karura", - choices: ["Nay"], - cid: "QmXoJvVbwdtBU5JC1ZgzZUtE2eBE92BVjmUsjzK98RJVec", - data: {}, - pinHash: "QmXoJvVbwdtBU5JC1ZgzZUtE2eBE92BVjmUsjzK98RJVec", - remark: "", - signature: - "0x4899168d4b6597d4d3fbff99a5a8fabd3d56509973a38e805bf9900f9498c50078bfbf382bb5a182d6e0edb7bddac7bc18c5a2ef54b3896a39200ffcdd99d181", - weights: { - balanceOf: new Decimal128("11111110000000"), - quadraticBalanceOf: new Decimal128("3333333166666.6625"), - }, - }, - ]); - }); - - afterAll(async () => { - await voteCol.drop(); - await proposalCol.drop(); - await spaceCol.drop(); - await db.close(); - }); - - test("getProposalById(mongoId)", async () => { - const proposal = await getProposalById("616e454d8f661dee51552e6a"); - - expect(proposal).toMatchObject({ - _id: new ObjectId("616e454d8f661dee51552e6a"), - space: "karura", - networksConfig: [karuraConfig], - postUid: "36", - title: "Proposal#1", - content: "Proposal#1", - contentType: "markdown", - choiceType: "single", - choices: ["Aye", "Nay"], - snapshotHeights: { karura: 779147 }, - weightStrategy: ["balance-of", "quadratic-balance-of"], - cid: "QmRgpY9WUuxKkptU6Sj9ow97u5QtFeRYnx4pqXzVBfMapE", - pinHash: "QmRgpY9WUuxKkptU6Sj9ow97u5QtFeRYnx4pqXzVBfMapE", - votedWeights: { - balanceOf: "22222220000000", - quadraticBalanceOf: "6666666333334", - }, - votesCount: 2, - status: "active", - }); - }); - - test("getProposalById(proposalCid)", async () => { - const proposal = await getProposalById( - "QmRgpY9WUuxKkptU6Sj9ow97u5QtFeRYnx4pqXzVBfMapE" - ); - - expect(proposal).toMatchObject({ - _id: new ObjectId("616e454d8f661dee51552e6a"), - space: "karura", - networksConfig: [karuraConfig], - postUid: "36", - title: "Proposal#1", - content: "Proposal#1", - contentType: "markdown", - choiceType: "single", - choices: ["Aye", "Nay"], - snapshotHeights: { karura: 779147 }, - weightStrategy: ["balance-of", "quadratic-balance-of"], - cid: "QmRgpY9WUuxKkptU6Sj9ow97u5QtFeRYnx4pqXzVBfMapE", - pinHash: "QmRgpY9WUuxKkptU6Sj9ow97u5QtFeRYnx4pqXzVBfMapE", - votedWeights: { - balanceOf: "22222220000000", - quadraticBalanceOf: "6666666333334", - }, - votesCount: 2, - status: "active", - }); - }); -}); diff --git a/backend/packages/backend/src/services/proposal.service/tests/vote.test.js b/backend/packages/backend/src/services/proposal.service/tests/vote.test.js deleted file mode 100644 index 1aa131663..000000000 --- a/backend/packages/backend/src/services/proposal.service/tests/vote.test.js +++ /dev/null @@ -1,111 +0,0 @@ -/** - * @jest-environment node - */ - -jest.mock("../../../env"); -jest.mock("../../ipfs.service/pin"); -jest.mock("../../node.service"); - -const { ObjectId } = require("mongodb"); -const { vote, getVotes } = require(".."); -const { getDb } = require("../../../mongo"); -const { reloadSpaces } = require("../../../spaces"); -const { karuraConfig } = require("../../../scripts/spaces/karura"); - -describe("Vote Test", () => { - let db; - let proposalCol; - let voteCol; - - beforeAll(async () => { - db = await getDb(); - proposalCol = db.getCollection("proposal"); - voteCol = db.getCollection("vote"); - spaceCol = db.getCollection("space"); - - await spaceCol.insertMany([karuraConfig]); - - await reloadSpaces(); - - await proposalCol.insertMany([ - { - _id: ObjectId("616e454d8f661dee51552e6a"), - space: "karura", - networksConfig: [karuraConfig], - postUid: "36", - title: "Proposal#1", - content: "Proposal#1", - contentType: "markdown", - choiceType: "single", - choices: ["Aye", "Nay"], - startDate: Date.now() - 3600 * 1000, - endDate: Date.now() + 3600 * 1000, - snapshotHeights: { karura: 779147 }, - weightStrategy: ["balance-of", "quadratic-balance-of"], - cid: "QmRgpY9WUuxKkptU6Sj9ow97u5QtFeRYnx4pqXzVBfMapE", - pinHash: "QmRgpY9WUuxKkptU6Sj9ow97u5QtFeRYnx4pqXzVBfMapE", - }, - ]); - }); - - afterAll(async () => { - await voteCol.drop(); - await proposalCol.drop(); - await spaceCol.drop(); - - await db.close(); - }); - - test("vote", async () => { - let proposalCid = "QmRgpY9WUuxKkptU6Sj9ow97u5QtFeRYnx4pqXzVBfMapE", - choices = ["Nay"], - remark = "Remark", - realVoter = null, - voterNetwork = "karura", - data = {}, - address = "14uSQFo8WePzpZaBUZHKa8AqDxigH1wjtNDHaRAG8ipSqQS3", - signature = - "0x4899168d4b6597d4d3fbff99a5a8fabd3d56509973a38e805bf9900f9498c50078bfbf382bb5a182d6e0edb7bddac7bc18c5a2ef54b3896a39200ffcdd99d181"; - - await vote( - proposalCid, - choices, - remark, - realVoter, - data, - address, - voterNetwork, - signature - ); - - const result = await getVotes( - "QmRgpY9WUuxKkptU6Sj9ow97u5QtFeRYnx4pqXzVBfMapE", - 1, - 10 - ); - expect(result).toMatchObject({ - items: [ - { - _id: new ObjectId("616e6aeb30a3c9772e936c7c"), - proposal: new ObjectId("616e454d8f661dee51552e6a"), - voter: "14uSQFo8WePzpZaBUZHKa8AqDxigH1wjtNDHaRAG8ipSqQS3", - address: "14uSQFo8WePzpZaBUZHKa8AqDxigH1wjtNDHaRAG8ipSqQS3", - choices: ["Nay"], - cid: "QmZPLzpusNUu6Re4CRhqcw74tEi9Rb9ecYePk2h2Sr2CjG", - data: {}, - pinHash: "QmZPLzpusNUu6Re4CRhqcw74tEi9Rb9ecYePk2h2Sr2CjG", - remark: "Remark", - signature: - "0x4899168d4b6597d4d3fbff99a5a8fabd3d56509973a38e805bf9900f9498c50078bfbf382bb5a182d6e0edb7bddac7bc18c5a2ef54b3896a39200ffcdd99d181", - weights: { - balanceOf: "10000000000000", - quadraticBalanceOf: "3162277660168", - }, - }, - ], - total: 1, - page: 1, - pageSize: 10, - }); - }); -}); diff --git a/backend/packages/backend/src/services/s3.service/index.js b/backend/packages/backend/src/services/s3.service/index.js new file mode 100644 index 000000000..e227c4736 --- /dev/null +++ b/backend/packages/backend/src/services/s3.service/index.js @@ -0,0 +1,66 @@ +const { S3Client, PutObjectCommand } = require("@aws-sdk/client-s3"); +const { requireEnv } = require("../../utils/requireEnv"); +const { cidOf, cidOfBuffer } = require("../../utils/cid"); + +let s3Client; + +function getS3Client() { + if (!s3Client) { + s3Client = new S3Client({ + region: "auto", + endpoint: requireEnv("S3_API"), + credentials: { + accessKeyId: requireEnv("S3_ACCESS_KEY_ID"), + secretAccessKey: requireEnv("S3_SECRET_ACCESS_KEY"), + }, + forcePathStyle: true, + }); + } + + return s3Client; +} + +const trimTailSlash = (url) => + url.endsWith("/") ? url.substr(0, url.length - 1) : url; + +async function uploadBufferToS3(file, key) { + const cid = key || (await cidOfBuffer(file.buffer)); + + await getS3Client().send( + new PutObjectCommand({ + Bucket: requireEnv("S3_BUCKET"), + Key: cid, + Body: file.buffer, + ContentType: file.mimetype, + }), + ); + + return { + cid, + url: `${trimTailSlash(requireEnv("S3_PUBLIC_ENDPOINT"))}/${cid}`, + }; +} + +async function uploadJsonToS3(data, key) { + const cid = key || (await cidOf(data)); + const body = JSON.stringify(data); + + await getS3Client().send( + new PutObjectCommand({ + Bucket: requireEnv("S3_BUCKET"), + Key: cid, + Body: body, + ContentType: "application/json", + }), + ); + + return { + cid, + url: `${trimTailSlash(requireEnv("S3_PUBLIC_ENDPOINT"))}/${cid}`, + }; +} + +module.exports = { + uploadBufferToS3, + uploadJsonToS3, +}; diff --git a/backend/packages/backend/src/services/s3.service/saveToS3.js b/backend/packages/backend/src/services/s3.service/saveToS3.js new file mode 100644 index 000000000..7ec144eac --- /dev/null +++ b/backend/packages/backend/src/services/s3.service/saveToS3.js @@ -0,0 +1,71 @@ +const pick = require("lodash/pick"); +const Hash = require("ipfs-only-hash"); +const { uploadJsonToS3, uploadBufferToS3 } = require("."); + +async function saveJsonToStorage(json) { + const { cid } = await uploadJsonToS3(json); + return cid; +} + +async function saveFileToStorage(file) { + const { cid } = await uploadBufferToS3(file); + return cid; +} + +async function saveJsonToStorageWithTimeout(json, timeout) { + const errorMsg = "Pin json to storage timeout"; + return await Promise.race([ + new Promise((_, reject) => + setTimeout(() => reject(new Error(errorMsg)), timeout), + ), + saveJsonToStorage(json), + ]); +} + +async function getObjectBufAndCid(data) { + const jsonData = JSON.stringify(data); + const buf = Buffer.from(jsonData); + const cid = await Hash.of(buf); + return { buf, cid }; +} + +async function saveItemsToS3(col, items) { + for (const item of items) { + try { + const toBePin = { + ...pick(item, ["data", "address", "signature", "delegators"]), + version: "2", + }; + const result = await uploadJsonToS3(toBePin); + + if (result && result.cid) { + await col.updateOne( + { _id: item._id }, + { $set: { pinHash: result.cid } }, + ); + console.log(`Save pin hash ${result.cid} to: ${item._id}`); + } + } catch (e) { + console.error(e); + } + } +} + +async function saveCollectionDataToS3(col) { + const items = await col.find({ pinHash: null }).toArray(); + await saveItemsToS3(col, items); +} + +async function resaveCollectionDataToS3(col) { + const items = await col.find().toArray(); + await saveItemsToS3(col, items); +} + +module.exports = { + saveFileToStorage, + saveJsonToStorage, + saveJsonToStorageWithTimeout, + getObjectBufAndCid, + saveCollectionDataToS3, + resaveCollectionDataToS3, +}; diff --git a/backend/packages/backend/src/utils/cid.js b/backend/packages/backend/src/utils/cid.js new file mode 100644 index 000000000..cc70a49ff --- /dev/null +++ b/backend/packages/backend/src/utils/cid.js @@ -0,0 +1,17 @@ +const Hash = require("ipfs-only-hash"); +const { CID } = require("multiformats/cid"); + +async function cidOfBuffer(buffer) { + const cidV0 = await Hash.of(buffer); + const cid = CID.parse(cidV0); + return cid.toV1().toString(); +} + +async function cidOf(obj) { + return await cidOfBuffer(Buffer.from(JSON.stringify(obj))); +} + +module.exports = { + cidOf, + cidOfBuffer, +}; diff --git a/backend/packages/backend/src/utils/requireEnv.js b/backend/packages/backend/src/utils/requireEnv.js new file mode 100644 index 000000000..a87fe544f --- /dev/null +++ b/backend/packages/backend/src/utils/requireEnv.js @@ -0,0 +1,20 @@ +function requireEnv(name) { + const value = process.env[name]; + if (!value) { + throw new Error(`Required env ${name} is not defined`); + } + return value; +} + +function validateEnv(names = []) { + const missing = names.filter((name) => !process.env[name]); + + if (missing.length > 0) { + throw new Error(`Missing required env: ${missing.join(", ")}`); + } +} + +module.exports = { + requireEnv, + validateEnv, +}; diff --git a/backend/yarn.lock b/backend/yarn.lock index a4f88a68f..e9fedd75b 100644 --- a/backend/yarn.lock +++ b/backend/yarn.lock @@ -14,6 +14,225 @@ resolved "https://registry.npmmirror.com/@assemblyscript/loader/-/loader-0.9.4.tgz#a483c54c1253656bb33babd464e3154a173e1577" integrity sha512-HazVq9zwTVwGmqdwYzu7WyQ6FQVZ7SwET0KKQuKm55jD0IfUpZgN0OPIiZG3zV1iSrVYcN0bdwLRXI/VNCYsUA== +"@aws-sdk/checksums@^3.1000.16": + version "3.1000.16" + resolved "https://registry.npmmirror.com/@aws-sdk/checksums/-/checksums-3.1000.16.tgz#b53915e554aa0e7015ce463605899f5bf683fb5c" + integrity sha512-EKnvkXSmz3IpA99tCNuI+dLFXyZyClSm8zns9sB/elvkU+MTuomAs6toJMPMBf98/fICG/urXDkzGz0/c3yyAQ== + dependencies: + "@aws-sdk/core" "^3.975.1" + "@aws-sdk/types" "^3.974.0" + "@smithy/core" "^3.29.2" + "@smithy/types" "^4.16.0" + tslib "^2.6.2" + +"@aws-sdk/client-s3@^3.1079.0": + version "3.1086.0" + resolved "https://registry.npmmirror.com/@aws-sdk/client-s3/-/client-s3-3.1086.0.tgz#4d5fd982996f68110e4e27c6ab85d6f8858f4cf0" + integrity sha512-6+7mVMPKetZmmF2L1yRJ+rN9b1OwVgc5sju2mj8ixdxuGjtVZ0ekFlcWGBFMQT9gpFk55PPLBng/XRYO3qah5A== + dependencies: + "@aws-sdk/checksums" "^3.1000.16" + "@aws-sdk/core" "^3.975.1" + "@aws-sdk/credential-provider-node" "^3.972.67" + "@aws-sdk/middleware-sdk-s3" "^3.972.62" + "@aws-sdk/signature-v4-multi-region" "^3.996.39" + "@aws-sdk/types" "^3.974.0" + "@smithy/core" "^3.29.2" + "@smithy/fetch-http-handler" "^5.6.4" + "@smithy/node-http-handler" "^4.9.4" + "@smithy/types" "^4.16.0" + tslib "^2.6.2" + +"@aws-sdk/core@^3.975.1": + version "3.975.1" + resolved "https://registry.npmmirror.com/@aws-sdk/core/-/core-3.975.1.tgz#fc591ab4b83cc43b445ba8d9e4dc78b79e563802" + integrity sha512-8qh/6EYb7hl/ZwVfQufhbMEZs1gQIc7GbdrIf4eprQJ7cv042+74nE6l3YDfyWNzb9iPXb8fRyYSHkNIk5eE6Q== + dependencies: + "@aws-sdk/types" "^3.974.0" + "@aws-sdk/xml-builder" "^3.972.34" + "@aws/lambda-invoke-store" "^0.3.0" + "@smithy/core" "^3.29.2" + "@smithy/signature-v4" "^5.6.3" + "@smithy/types" "^4.16.0" + bowser "^2.11.0" + tslib "^2.6.2" + +"@aws-sdk/credential-provider-env@^3.972.57": + version "3.972.57" + resolved "https://registry.npmmirror.com/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.57.tgz#2f8569b71d948c0dd89a815db954f5c46aea132a" + integrity sha512-1RfJaF7SW1TOnvNGU7kaYjwUf5H3sfm+synGH1bHhRlqcnxCt3szebH3dmKEyY4tuGcbQ6ffzUT89cRitBV8OQ== + dependencies: + "@aws-sdk/core" "^3.975.1" + "@aws-sdk/types" "^3.974.0" + "@smithy/core" "^3.29.2" + "@smithy/types" "^4.16.0" + tslib "^2.6.2" + +"@aws-sdk/credential-provider-http@^3.972.59": + version "3.972.59" + resolved "https://registry.npmmirror.com/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.59.tgz#f7553c9296fdf41a41afd5346c1e400e7fee4df8" + integrity sha512-sRCkpTiFnCdQvuaRVjQ6SVoHu6i7RUpurVo1c4F81HWhPvUJ7Wdp5MNtSdX1O29CNXc8em3O5m52hCjVtAD9SA== + dependencies: + "@aws-sdk/core" "^3.975.1" + "@aws-sdk/types" "^3.974.0" + "@smithy/core" "^3.29.2" + "@smithy/fetch-http-handler" "^5.6.4" + "@smithy/node-http-handler" "^4.9.4" + "@smithy/types" "^4.16.0" + tslib "^2.6.2" + +"@aws-sdk/credential-provider-ini@^3.973.1": + version "3.973.1" + resolved "https://registry.npmmirror.com/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.1.tgz#a86dd75ec404c5498781f0c3f0261c5a470762aa" + integrity sha512-6d8H6ZAh3ZPKZ6fe1nG2OWeZEZPtt9ravoD1dezPdPtsSkJRoxGAnFSHwKT3E/Te6fHE30zRzjV6TD12rvF6yQ== + dependencies: + "@aws-sdk/core" "^3.975.1" + "@aws-sdk/credential-provider-env" "^3.972.57" + "@aws-sdk/credential-provider-http" "^3.972.59" + "@aws-sdk/credential-provider-login" "^3.972.63" + "@aws-sdk/credential-provider-process" "^3.972.57" + "@aws-sdk/credential-provider-sso" "^3.973.1" + "@aws-sdk/credential-provider-web-identity" "^3.972.63" + "@aws-sdk/nested-clients" "^3.997.31" + "@aws-sdk/types" "^3.974.0" + "@smithy/core" "^3.29.2" + "@smithy/credential-provider-imds" "^4.4.7" + "@smithy/types" "^4.16.0" + tslib "^2.6.2" + +"@aws-sdk/credential-provider-login@^3.972.63": + version "3.972.63" + resolved "https://registry.npmmirror.com/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.63.tgz#69ada34f4dc3b99e4099d84e6b08987b978cf165" + integrity sha512-GREWRrMj0XnNKMaVa/Mauoaui26qBEHu71WWqXbwZOu/jFQOnPZjTf7u0KtGKC8VGa6VUs9kDWGgocrKNLS9vw== + dependencies: + "@aws-sdk/core" "^3.975.1" + "@aws-sdk/nested-clients" "^3.997.31" + "@aws-sdk/types" "^3.974.0" + "@smithy/core" "^3.29.2" + "@smithy/types" "^4.16.0" + tslib "^2.6.2" + +"@aws-sdk/credential-provider-node@^3.972.67": + version "3.972.67" + resolved "https://registry.npmmirror.com/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.67.tgz#267bc4e1cf3b37ca210f40a05766dd4a12e6209b" + integrity sha512-oYlzWst56rlhhjbYnexwv5hVLYe1cW4liLObhDfxDLI4RAQzleMVHQgQgx7XsC4HKj4e3kjT8v9DId+Pi/dndw== + dependencies: + "@aws-sdk/credential-provider-env" "^3.972.57" + "@aws-sdk/credential-provider-http" "^3.972.59" + "@aws-sdk/credential-provider-ini" "^3.973.1" + "@aws-sdk/credential-provider-process" "^3.972.57" + "@aws-sdk/credential-provider-sso" "^3.973.1" + "@aws-sdk/credential-provider-web-identity" "^3.972.63" + "@aws-sdk/types" "^3.974.0" + "@smithy/core" "^3.29.2" + "@smithy/credential-provider-imds" "^4.4.7" + "@smithy/types" "^4.16.0" + tslib "^2.6.2" + +"@aws-sdk/credential-provider-process@^3.972.57": + version "3.972.57" + resolved "https://registry.npmmirror.com/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.57.tgz#1a3e98e62c7eb5806e02bf0655817df5f4a524eb" + integrity sha512-TiVQhuU0pbhIZAUZacbPHMyzrIdiH+lnx+PMY/Pu/b93dJrq3wdZwzUJ0TPpvNxaqbHsxJvQZW3/h/beLiKq7Q== + dependencies: + "@aws-sdk/core" "^3.975.1" + "@aws-sdk/types" "^3.974.0" + "@smithy/core" "^3.29.2" + "@smithy/types" "^4.16.0" + tslib "^2.6.2" + +"@aws-sdk/credential-provider-sso@^3.973.1": + version "3.973.1" + resolved "https://registry.npmmirror.com/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.1.tgz#ada31e7cf2f232a72b26a12b313ff7f8066fbb96" + integrity sha512-3foTZUJ4821Ij60X7K3NJroygiZLnbBmarN+T//O2cjkISan90zElN3NBmgSlDrTQ7Gs6z/yO8V7h60QNcDZHQ== + dependencies: + "@aws-sdk/core" "^3.975.1" + "@aws-sdk/nested-clients" "^3.997.31" + "@aws-sdk/token-providers" "3.1083.0" + "@aws-sdk/types" "^3.974.0" + "@smithy/core" "^3.29.2" + "@smithy/types" "^4.16.0" + tslib "^2.6.2" + +"@aws-sdk/credential-provider-web-identity@^3.972.63": + version "3.972.63" + resolved "https://registry.npmmirror.com/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.63.tgz#dea0f02d55d87575376028eb3f3857435ceee8f6" + integrity sha512-8qZLFhM69eKcS37m459ctPR05Qimycm/74OPVioe6wNZabMT54GYhwBju0+J656RkMasNSawWQu+c8CmBe3TUQ== + dependencies: + "@aws-sdk/core" "^3.975.1" + "@aws-sdk/nested-clients" "^3.997.31" + "@aws-sdk/types" "^3.974.0" + "@smithy/core" "^3.29.2" + "@smithy/types" "^4.16.0" + tslib "^2.6.2" + +"@aws-sdk/middleware-sdk-s3@^3.972.62": + version "3.972.62" + resolved "https://registry.npmmirror.com/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.62.tgz#554b60c5c0746d3d4220c27521d7f78d418dfb1c" + integrity sha512-k8JJwYXVYlOOjWnPZDThQS1xDFJgi5Dokt73qFlDtrZAbdcint5aIdjB9XgJAAQVP5OoqcefQmh1FYXiPpvsvw== + dependencies: + "@aws-sdk/core" "^3.975.1" + "@aws-sdk/signature-v4-multi-region" "^3.996.39" + "@aws-sdk/types" "^3.974.0" + "@smithy/core" "^3.29.2" + "@smithy/types" "^4.16.0" + tslib "^2.6.2" + +"@aws-sdk/nested-clients@^3.997.31": + version "3.997.31" + resolved "https://registry.npmmirror.com/@aws-sdk/nested-clients/-/nested-clients-3.997.31.tgz#fa433ff1c4fecd51e6c70293ec69925dfd5a9a80" + integrity sha512-BDHTpwcsZHEBNEJzOg/B1BkFYJxAXY50dau/NyVWs3d51F0WgIUGSWZot/Os+N3KpDhXeaXnz37mWffAvduREw== + dependencies: + "@aws-sdk/core" "^3.975.1" + "@aws-sdk/signature-v4-multi-region" "^3.996.39" + "@aws-sdk/types" "^3.974.0" + "@smithy/core" "^3.29.2" + "@smithy/fetch-http-handler" "^5.6.4" + "@smithy/node-http-handler" "^4.9.4" + "@smithy/types" "^4.16.0" + tslib "^2.6.2" + +"@aws-sdk/signature-v4-multi-region@^3.996.39": + version "3.996.39" + resolved "https://registry.npmmirror.com/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.39.tgz#168990f6ac509ecc2858f45fa485fce80c0a12b8" + integrity sha512-8+srXqYIF8KYMLC4FxMLEM5Ek7kUNibJu1R4m8/fUhhNYIZZz26oGtKkCr8I/HiG2fFQxBvaGgQZT4/mqRCSnA== + dependencies: + "@aws-sdk/types" "^3.974.0" + "@smithy/signature-v4" "^5.6.3" + "@smithy/types" "^4.16.0" + tslib "^2.6.2" + +"@aws-sdk/token-providers@3.1083.0": + version "3.1083.0" + resolved "https://registry.npmmirror.com/@aws-sdk/token-providers/-/token-providers-3.1083.0.tgz#3e50be28c8e7b28b8fa13cf5c113b4b6af8f2e35" + integrity sha512-s0woKnxuHrExLc5L2ArIH5BMkbonHPtt+5hSBM8oknp9M6QTuUmmAmJ2E0EdzCGONrO+8+ADPqvv6UX0nNcc7A== + dependencies: + "@aws-sdk/core" "^3.975.1" + "@aws-sdk/nested-clients" "^3.997.31" + "@aws-sdk/types" "^3.974.0" + "@smithy/core" "^3.29.2" + "@smithy/types" "^4.16.0" + tslib "^2.6.2" + +"@aws-sdk/types@^3.974.0": + version "3.974.0" + resolved "https://registry.npmmirror.com/@aws-sdk/types/-/types-3.974.0.tgz#4c943425410a855e3d8b73b4c7801b96934a351f" + integrity sha512-QIBrw90CDm4O0UaIIzkU6DrFdeJzEb2Va5EPEVpyldj6sHJxB6cshhStJuhZxk3wR3PmjJlYsjPmY1kNb+KGBg== + dependencies: + "@smithy/types" "^4.16.0" + tslib "^2.6.2" + +"@aws-sdk/xml-builder@^3.972.34": + version "3.972.34" + resolved "https://registry.npmmirror.com/@aws-sdk/xml-builder/-/xml-builder-3.972.34.tgz#a13c7986a6d926510fc9a709437ced18dc7e3c4c" + integrity sha512-wHhWL1y7sN3enBA8POrPpQM5jCcmu2ozyhbRei4c8OjVcEaEs6yLucLa/pla457ggS/ysuy7bosagz3HaJkZXA== + dependencies: + "@smithy/types" "^4.16.0" + tslib "^2.6.2" + +"@aws/lambda-invoke-store@^0.3.0": + version "0.3.0" + resolved "https://registry.npmmirror.com/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz#708802d987f8e17bdf4af4de1031660ce1cdd65f" + integrity sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ== + "@babel/code-frame@^7.0.0", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.16.7": version "7.16.7" resolved "https://registry.npmmirror.com/@babel/code-frame/-/code-frame-7.16.7.tgz#44416b6bd7624b998f5b1af5d470856c40138789" @@ -1869,6 +2088,57 @@ dependencies: "@sinonjs/commons" "^1.7.0" +"@smithy/core@^3.29.2", "@smithy/core@^3.29.3": + version "3.29.3" + resolved "https://registry.npmmirror.com/@smithy/core/-/core-3.29.3.tgz#d255cc12f19d612872f5255b7f415f6cb96ec30f" + integrity sha512-L+Ys6ecjk5vwPMAKHBpPKlJ3DkqwNcnfEISXBZIsVvWG/XKXfsAP8mwIYlTeLcd2ElHdesPI8OuOmJSFAPhm6A== + dependencies: + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + +"@smithy/credential-provider-imds@^4.4.7": + version "4.4.8" + resolved "https://registry.npmmirror.com/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.8.tgz#e44c654f18498a5154b169a7ca1065d37fe01216" + integrity sha512-q9J7JTiXrAhB8sDp4px97uEPT7CwKH61Co78grdNQvU8QZAdiuaSRhP0tUVf2ogy36RZTrlMU1rBmDEH+cnkiA== + dependencies: + "@smithy/core" "^3.29.3" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + +"@smithy/fetch-http-handler@^5.6.4": + version "5.6.5" + resolved "https://registry.npmmirror.com/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.5.tgz#b3a5b574c265384fdd77f4e9fa61cb7f035d3fe9" + integrity sha512-SuqeisTyPoiIPtIYru/sGxGyXzmZ+8nnFOhC+qRPglt06Ebd1yH//CDltZB2J/3WBNVhwfUaZ0EtHB3cm2X32g== + dependencies: + "@smithy/core" "^3.29.3" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + +"@smithy/node-http-handler@^4.9.4": + version "4.9.5" + resolved "https://registry.npmmirror.com/@smithy/node-http-handler/-/node-http-handler-4.9.5.tgz#7c8af721b0b16ffd4f4b51d7a1da34042810891c" + integrity sha512-bNqdxTQTxmLbomSmlkZFz8L6B/feQ2HHzw4L2zY7Ecp2XffYAZq2uzdWDdxJHJFbEvqd+SRuluJso0P8+xPdbw== + dependencies: + "@smithy/core" "^3.29.3" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + +"@smithy/signature-v4@^5.6.3": + version "5.6.4" + resolved "https://registry.npmmirror.com/@smithy/signature-v4/-/signature-v4-5.6.4.tgz#7b7372234609735972a98d4e820332da3e55309f" + integrity sha512-B89bpf2t/y/wia6LZ+4JfHXYQT9PnVftsH05rgJKKIStS7r/4XSs9HOjtPoLtgcA6HCW9jVqX5DBbq7E0PAkiQ== + dependencies: + "@smithy/core" "^3.29.3" + "@smithy/types" "^4.16.1" + tslib "^2.6.2" + +"@smithy/types@^4.16.0", "@smithy/types@^4.16.1": + version "4.16.1" + resolved "https://registry.npmmirror.com/@smithy/types/-/types-4.16.1.tgz#19e199c234829a51c085caf63f0bb17bb80187e4" + integrity sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg== + dependencies: + tslib "^2.6.2" + "@socket.io/component-emitter@~3.1.0": version "3.1.2" resolved "https://registry.npmmirror.com/@socket.io/component-emitter/-/component-emitter-3.1.2.tgz#821f8442f4175d8f0467b9daf26e3a18e2d02af2" @@ -2511,6 +2781,11 @@ boolbase@^1.0.0: resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" integrity sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww== +bowser@^2.11.0: + version "2.14.1" + resolved "https://registry.npmmirror.com/bowser/-/bowser-2.14.1.tgz#4ea39bf31e305184522d7ad7bfd91389e4f0cb79" + integrity sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg== + brace-expansion@^1.1.7: version "1.1.11" resolved "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" diff --git a/next/.env.example b/next/.env.example index 550b6480a..822dc773e 100644 --- a/next/.env.example +++ b/next/.env.example @@ -6,6 +6,6 @@ NEXT_PUBLIC_SOCKET_IO_URL=http://127.0.0.1:8081/ NEXT_PUBLIC_IDENTITY_SERVER_HOST=https://id.statescan.io -NEXT_PUBLIC_IPFS_ENDPOINT=https://opensquare.infura-ipfs.io/ipfs/ +NEXT_PUBLIC_S3_ENDPOINT=https://voting-static.opensquare.io NEXT_PUBLIC_RECAPTCHA_SITEKEY= diff --git a/next/components/author.js b/next/components/author.js index ed26ce0e4..433013bcb 100644 --- a/next/components/author.js +++ b/next/components/author.js @@ -3,7 +3,7 @@ import styled from "styled-components"; import Avatar from "./avatar"; import { ChainIcon } from "components/chainIcon"; import IdentityOrAddr from "@/components/identityOrAddr"; -import { getCollectiveMenberIdentityLink } from "frontedUtils/space"; +import { getCollectiveMemberIdentityLink } from "frontedUtils/space"; const Wrapper = styled.div` display: flex; @@ -39,7 +39,7 @@ export default function Author({ address={address} noLink={noLink} href={ - isCollective ? getCollectiveMenberIdentityLink(address, network) : "" + isCollective ? getCollectiveMemberIdentityLink(address, network) : "" } /> diff --git a/next/components/avatar.js b/next/components/avatar.js index bafb3dd27..1b7d7feb1 100644 --- a/next/components/avatar.js +++ b/next/components/avatar.js @@ -7,7 +7,7 @@ import React from "react"; import IdentityOrAddr from "./identityOrAddr"; import useAvatarInfo from "hooks/useAvatar"; import IpfsAvatar from "./ipfsAvatar"; -import { getCollectiveMenberIdentityLink } from "frontedUtils/space"; +import { getCollectiveMemberIdentityLink } from "frontedUtils/space"; const Wrapper = styled.span` display: inline-flex; @@ -61,7 +61,7 @@ export function AvatarWithTooltip({ } > diff --git a/next/components/ipfsAvatar.js b/next/components/ipfsAvatar.js index 1187a8dc9..145630b16 100644 --- a/next/components/ipfsAvatar.js +++ b/next/components/ipfsAvatar.js @@ -1,4 +1,5 @@ import styled from "styled-components"; +import getStorageLink from "frontedUtils/env/storageLink"; const IpfsAvatarWrapper = styled.img` border-radius: 50%; @@ -16,7 +17,7 @@ export default function IpfsAvatar({ avatarCid, size }) { return ( diff --git a/next/components/postCreate/index.js b/next/components/postCreate/index.js index 5abfd4057..f6d35aef5 100644 --- a/next/components/postCreate/index.js +++ b/next/components/postCreate/index.js @@ -253,9 +253,7 @@ export default function PostCreate({ space, settings }) { } const toastId = newToastId(); - dispatch( - newPendingToast(toastId, "Saving and uploading proposal to IPFS..."), - ); + dispatch(newPendingToast(toastId, "Saving and uploading proposal...")); try { const { result, error } = await nextApi.post( `${proposal.space}/proposals`, diff --git a/next/components/postDetail/appendants.js b/next/components/postDetail/appendants.js index cc0d3ca56..dc26ff124 100644 --- a/next/components/postDetail/appendants.js +++ b/next/components/postDetail/appendants.js @@ -134,9 +134,7 @@ export default function Appendants({ } const toastId = newToastId(); - dispatch( - newPendingToast(toastId, "Saving and uploading the appendant to IPFS..."), - ); + dispatch(newPendingToast(toastId, "Saving and uploading the appendant...")); let result; try { result = await nextApi.post(`${proposal?.space}/appendants`, signedData); @@ -209,7 +207,7 @@ const AppendantItem = ({ item, index }) => { diff --git a/next/components/postDetail/postContent.js b/next/components/postDetail/postContent.js index 7e93da310..747a5a3ad 100644 --- a/next/components/postDetail/postContent.js +++ b/next/components/postDetail/postContent.js @@ -100,7 +100,7 @@ export default function PostContent({ data, space, loadSuggestions }) { diff --git a/next/components/postDetail/postInfo.js b/next/components/postDetail/postInfo.js index e59a66ff0..ce5eab080 100644 --- a/next/components/postDetail/postInfo.js +++ b/next/components/postDetail/postInfo.js @@ -59,9 +59,9 @@ function Snapshot({ space, snapshotHeights }) { function PinHash({ pinHash }) { return ( -
IPFS
+
Storage
{`#${pinHash?.slice(0, 7)}`} ); diff --git a/next/components/postDetail/postVote.js b/next/components/postDetail/postVote.js index 619180d36..34b908fa0 100644 --- a/next/components/postDetail/postVote.js +++ b/next/components/postDetail/postVote.js @@ -525,9 +525,7 @@ export default function PostVote({ proposal, loadSuggestions }) { } const toastId = newToastId(); - dispatch( - newPendingToast(toastId, "Saving and uploading the vote to IPFS..."), - ); + dispatch(newPendingToast(toastId, "Saving and uploading the vote...")); let result; try { result = await nextApi.post(`${proposal?.space}/votes`, signedData); diff --git a/next/components/postDetail/postVotesItem.js b/next/components/postDetail/postVotesItem.js index c0f8de10b..b10ac7e08 100644 --- a/next/components/postDetail/postVotesItem.js +++ b/next/components/postDetail/postVotesItem.js @@ -241,7 +241,7 @@ export default function PostVotesItem({ diff --git a/next/components/uploadBanner/uploader.js b/next/components/uploadBanner/uploader.js index cec1b8c6d..330e74829 100644 --- a/next/components/uploadBanner/uploader.js +++ b/next/components/uploadBanner/uploader.js @@ -119,7 +119,7 @@ function Uploader({ setBannerUrl }) { const formData = new FormData(); formData.append("banner", image, image.name); nextApi - .postFormData("ipfs/files", formData) + .postFormData("s3/files", formData) .then(({ result, error }) => { if (result) { setCurrentBanner(result.url); diff --git a/next/frontedUtils/env/storageLink.js b/next/frontedUtils/env/storageLink.js new file mode 100644 index 000000000..6dee0bda6 --- /dev/null +++ b/next/frontedUtils/env/storageLink.js @@ -0,0 +1,5 @@ +export default function getStorageLink(cid) { + const endpoint = process.env.NEXT_PUBLIC_S3_ENDPOINT; + + return `${endpoint}/${cid}`; +} diff --git a/next/frontedUtils/space.js b/next/frontedUtils/space.js index b9f42658c..cd9e50415 100644 --- a/next/frontedUtils/space.js +++ b/next/frontedUtils/space.js @@ -1,11 +1,12 @@ import { isEthereumAddress } from "@polkadot/util-crypto"; import { getDefaultLogoUri } from "hooks/useDefaultLogo"; import { useMemo } from "react"; +import getStorageLink from "frontedUtils/env/storageLink"; export const isCollectiveSpace = (type) => type === "collectives-dao"; export const hasWhitelist = (spaceConfig) => !!spaceConfig?.whitelist?.length; -export const getCollectiveMenberIdentityLink = ( +export const getCollectiveMemberIdentityLink = ( address, network = "polkadot", ) => { @@ -71,7 +72,7 @@ export function getSpaceIconUri(space) { return `/imgs/icons/space/${spaceIcon}`; } - return `${process.env.NEXT_PUBLIC_IPFS_ENDPOINT}${spaceIcon}`; + return getStorageLink(spaceIcon); } export function useSpaceIconUri(space) { diff --git a/next/pages/space/[space]/proposal/[id].js b/next/pages/space/[space]/proposal/[id].js index 4f3fde2f4..cc818b8b8 100644 --- a/next/pages/space/[space]/proposal/[id].js +++ b/next/pages/space/[space]/proposal/[id].js @@ -98,7 +98,7 @@ export default function Index({ desc={desc} banner={ detail?.banner && - `${process.env.NEXT_PUBLIC_API_END_POINT}api/ipfs/files/${detail?.banner}` + `${process.env.NEXT_PUBLIC_API_END_POINT}api/s3/files/${detail?.banner}` } />