Migrate IPFS to S3 storage

backend/packages/backend/.env.example

@@ -8,12 +8,11 @@ TEST_ACCOUNT_BALANCE=100000000000
 
 NODE_API_ENDPOINT=http://localhost:3223
 
-INFURA_PROJECT_ID=
-INFURA_PROJECT_SECRET=
-LOCAL_IPFS_NODE_URL=http://localhost:5001
-USE_LOCAL_IPFS_NODE=false
-
-IPFS_ENDPOINT=https://ipfs.infura.io/ipfs/
+S3_API=
+S3_ACCESS_KEY_ID=
+S3_SECRET_ACCESS_KEY=
+S3_BUCKET=
+S3_PUBLIC_ENDPOINT=https://voting-static.opensquare.io
 
 DEVELOPMENT=true
 

backend/packages/backend/ecosystem.config.js

@@ -1,8 +1,8 @@
 module.exports = {
   apps: [
     {
-      name: "voting-pin-to-ipfs",
-      script: "src/scripts/pin-to-ipfs.js",
+      name: "voting-save-to-s3",
+      script: "src/scripts/save-to-s3.js",
       log_date_format: "YYYY-MM-DD HH:mm Z",
       env: {
         NODE_ENV: "development",

backend/packages/backend/package.json

@@ -7,9 +7,10 @@
     "start": "NODE_ENV=production node src/index.js",
     "lint": "eslint .",
     "test": "jest src --runInBand",
-    "pin": "pm2 start ecosystem.config.js --only voting-pin-to-ipfs --env production"
+    "saveToS3": "pm2 start ecosystem.config.js --only voting-save-to-s3 --env production"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.1079.0",
     "@koa/cors": "^5.0.0",
     "@koa/multer": "^3.0.0",
     "@osn/ipfs": "^0.2.0",

backend/packages/backend/src/__mocks__/env.js

@@ -1,15 +1,5 @@
 const NODE_API_ENDPOINT = "http://127.0.0.1:8081";
-const INFURA_PROJECT_ID = "xxx";
-const INFURA_PROJECT_SECRET = "yyy";
-const LOCAL_IPFS_NODE_URL = "http://ipfs.dotask.cc:5001";
-const USE_LOCAL_IFPS_NODE = false;
-const IPFS_ENDPOINT = "https://ipfs.infura.io/ipfs/";
 
 module.exports = {
   NODE_API_ENDPOINT,
-  INFURA_PROJECT_ID,
-  INFURA_PROJECT_SECRET,
-  LOCAL_IPFS_NODE_URL,
-  USE_LOCAL_IFPS_NODE,
-  IPFS_ENDPOINT,
 };

backend/packages/backend/src/env.js

@@ -1,20 +1,6 @@
 const NODE_API_ENDPOINT =
   process.env.NODE_API_ENDPOINT || "http://127.0.0.1:8081";
-const INFURA_PROJECT_ID = process.env.INFURA_PROJECT_ID || "";
-const INFURA_PROJECT_SECRET = process.env.INFURA_PROJECT_SECRET || "";
-const LOCAL_IPFS_NODE_URL =
-  process.env.LOCAL_IPFS_NODE_URL || "http://ipfs.dotask.cc:5001";
-const IPFS_ENDPOINT =
-  process.env.IPFS_ENDPOINT || "https://ipfs.infura.io/ipfs/";
-const USE_LOCAL_IFPS_NODE = ["true", "True", "TRUE", "1"].includes(
-  process.env.USE_LOCAL_IFPS_NODE,
-);
 
 module.exports = {
   NODE_API_ENDPOINT,
-  INFURA_PROJECT_ID,
-  INFURA_PROJECT_SECRET,
-  LOCAL_IPFS_NODE_URL,
-  USE_LOCAL_IFPS_NODE,
-  IPFS_ENDPOINT,
 };

backend/packages/backend/src/features/files/file.controller.js

@@ -1,24 +1,23 @@
 const { HttpError } = require("../../exc");
-const { IPFS_ENDPOINT } = require("../../env");
-const { pinFileToIpfs } = require("../../services/ipfs.service/pin");
+const { saveFileToStorage } = require("../../services/s3.service/saveToS3");
 
 const Megabyte = 1024 * 1024;
 
 const trimTailSlash = (url) =>
   url.endsWith("/") ? url.substr(0, url.length - 1) : url;
 
-async function getFile(ctx) {
-  const { hash } = ctx.params;
-  ctx.redirect(`${trimTailSlash(IPFS_ENDPOINT)}/${hash}`);
-}
-
-async function getIpfsEndpoint(ctx) {
+async function getS3Endpoint(ctx) {
   ctx.body = {
-    endpoint: trimTailSlash(IPFS_ENDPOINT),
+    endpoint: trimTailSlash(process.env.S3_PUBLIC_ENDPOINT),
   };
 }
 
-async function upload(ctx) {
+async function getS3File(ctx) {
+  const { cid } = ctx.params;
+  ctx.redirect(`${trimTailSlash(process.env.S3_PUBLIC_ENDPOINT)}/${cid}`);
+}
+
+async function uploadFile(ctx) {
   const file = ctx.request.file;
   if (!file) {
     throw new HttpError(400, "File is missing");
@@ -32,18 +31,19 @@ async function upload(ctx) {
   }
 
   try {
-    const hash = await pinFileToIpfs(file);
+    const hash = await saveFileToStorage(file);
+
     ctx.body = {
-      hash,
-      url: `${trimTailSlash(IPFS_ENDPOINT)}/${hash}`,
+      cid: hash,
+      url: `${trimTailSlash(process.env.S3_PUBLIC_ENDPOINT)}/${hash}`,
     };
   } catch (e) {
     throw new HttpError(500, e.message);
   }
 }
 
 module.exports = {
-  getFile,
-  getIpfsEndpoint,
-  upload,
+  getS3Endpoint,
+  getS3File,
+  uploadFile,
 };

backend/packages/backend/src/features/files/routes.js

@@ -5,8 +5,8 @@ const fileController = require("./file.controller");
 const router = new Router();
 const upload = multer();
 
-router.post("/ipfs/files", upload.single("banner"), fileController.upload);
-router.get("/ipfs/files/:hash", fileController.getFile);
-router.get("/ipfs/endpoint", fileController.getIpfsEndpoint);
+router.post("/s3/files", upload.single("banner"), fileController.uploadFile);
+router.get("/s3/files/:cid", fileController.getS3File);
+router.get("/s3/endpoint", fileController.getS3Endpoint);
 
 module.exports = router;

backend/packages/backend/src/features/spaces/common.js

@@ -1,7 +1,8 @@
 const { isAddress } = require("@polkadot/util-crypto");
 const { HttpError } = require("../../exc");
 const { getSpaceCollection } = require("../../mongo");
-const { ipfsAddBuffer } = require("../../services/ipfs.service/ipfs");
+const { uploadBufferToS3 } = require("../../services/s3.service");
+const { cidOfBuffer } = require("../../utils/cid");
 const { isSameAddress } = require("../../utils/address");
 
 const dataUriToBuffer = (dataUri) =>
@@ -11,8 +12,16 @@ async function pinLogo(logo) {
   let logoCid = null;
   if (logo) {
     const buf = await dataUriToBuffer(logo);
-    const added = await ipfsAddBuffer(buf);
-    logoCid = added.path;
+    const rawBuf = Buffer.from(buf);
+    logoCid = await cidOfBuffer(rawBuf);
+
+    await uploadBufferToS3(
+      {
+        buffer: rawBuf,
+        mimetype: buf.type,
+      },
+      logoCid,
+    );
   }
 
   return logoCid;

backend/packages/backend/src/index.js

@@ -10,6 +10,15 @@ const cors = require("@koa/cors");
 const { reloadSpaces } = require("./spaces");
 const socketSetup = require("./socket");
 const spaceNotify = require("./scripts/space-notify");
+const { validateEnv } = require("./utils/requireEnv");
+
+validateEnv([
+  "S3_API",
+  "S3_ACCESS_KEY_ID",
+  "S3_SECRET_ACCESS_KEY",
+  "S3_BUCKET",
+  "S3_PUBLIC_ENDPOINT",
+]);
 
 const app = new Koa();
 

backend/packages/backend/src/scripts/repin-to-ipfs.js → backend/packages/backend/src/scripts/resave-to-s3.js

@@ -8,7 +8,7 @@ const {
   getCommentCollection,
   getAppendantCollection,
 } = require("../mongo");
-const { repinCollectionDataToIpfs } = require("../services/ipfs.service");
+const { resaveCollectionDataToS3 } = require("../services/s3.service/saveToS3");
 
 const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 
@@ -18,10 +18,10 @@ async function startPin() {
   const voteCol = await getVoteCollection();
   const commentCol = await getCommentCollection();
   await Promise.all([
-    repinCollectionDataToIpfs(proposalCol),
-    repinCollectionDataToIpfs(appendantCol),
-    repinCollectionDataToIpfs(voteCol),
-    repinCollectionDataToIpfs(commentCol),
+    resaveCollectionDataToS3(proposalCol),
+    resaveCollectionDataToS3(appendantCol),
+    resaveCollectionDataToS3(voteCol),
+    resaveCollectionDataToS3(commentCol),
   ]);
 }
 

backend/packages/backend/src/scripts/pin-to-ipfs.js → backend/packages/backend/src/scripts/save-to-s3.js

@@ -7,18 +7,18 @@ const {
   getCommentCollection,
   getAppendantCollection,
 } = require("../mongo");
-const { pinCollectionDataToIpfs } = require("../services/ipfs.service");
+const { saveCollectionDataToS3 } = require("../services/s3.service/saveToS3");
 
 async function startPin() {
   const proposalCol = await getProposalCollection();
   const appendantCol = await getAppendantCollection();
   const voteCol = await getVoteCollection();
   const commentCol = await getCommentCollection();
   await Promise.all([
-    pinCollectionDataToIpfs(proposalCol),
-    pinCollectionDataToIpfs(appendantCol),
-    pinCollectionDataToIpfs(voteCol),
-    pinCollectionDataToIpfs(commentCol),
+    saveCollectionDataToS3(proposalCol),
+    saveCollectionDataToS3(appendantCol),
+    saveCollectionDataToS3(voteCol),
+    saveCollectionDataToS3(commentCol),
   ]);
 }
 

backend/packages/backend/src/services/proposal.service/common.js

@@ -1,7 +1,7 @@
 const {
   getObjectBufAndCid,
-  pinJsonToIpfsWithTimeout,
-} = require("../ipfs.service");
+  saveJsonToStorageWithTimeout,
+} = require("../s3.service/saveToS3");
 const { enhancedSqrtOfBalance } = require("../../utils");
 const { getProposalCollection } = require("../../mongo");
 const { HttpError } = require("../../exc");
@@ -45,7 +45,7 @@ async function pinData(rawData) {
 
   let pinHash = null;
   try {
-    pinHash = await pinJsonToIpfsWithTimeout(toBePin, 3000);
+    pinHash = await saveJsonToStorageWithTimeout(toBePin, 3000);
   } catch (e) {
     console.error(e);
   }