Add postgresql support

.github/workflows/unit-test.yaml

@@ -88,5 +88,10 @@ jobs:
         env:
           CITIZEN_DATABASE_TYPE: mysql
           CITIZEN_DATABASE_URL: mysql://root:citizen@127.0.0.1:3306/citizen
+      - name: Postgres migration
+        run: npm run migrate:postgresql
+        env:
+          CITIZEN_DATABASE_TYPE: postgresql
+          CITIZEN_DATABASE_URL: postgresql://citizen:citizen@127.0.0.1:5432/citizen
       - name: unit test
         run: npm run test:unit -- --forbid-only

README.adoc

@@ -39,7 +39,7 @@ If you want to test it at local, you need a tool which provides HTTPS like link:
 
 Environment variables:
 
-* `CITIZEN_DATABASE_TYPE`: Backend provider for registry metadata. Set to `mysql`, `mongodb` or `sqlite`.
+* `CITIZEN_DATABASE_TYPE`: Backend provider for registry metadata. Set to `mysql`, `postgresql`, `mongodb` or `sqlite`.
 * `CITIZEN_DATABASE_URL`: The URL of the database. `protocol//username:password@hosts:port/database?options`
 * `CITIZEN_STORAGE`: Storage type to store module files. You can use `file`, `s3` or `gcs` type.
 ** `s3` is for link:https://aws.amazon.com/ko/s3/[AWS S3].
@@ -180,6 +180,12 @@ Run MySQL with docker
 docker run --rm -p 3306:3306 --name mysql-citizen -e MYSQL_ROOT_PASSWORD=citizen -e MYSQL_DATABASE=citizen mysql
 ....
 
+Run Postgresql with docker
+[source, sh]
+....
+$ docker run --rm -p 5432:5432 --name postgres-citizen -e POSTGRES_USER=citizen -e POSTGRES_PASSWORD=citizen -e POSTGRES_DB=citizen postgres
+....
+
 Run the tests:
 [source, sh]
 ....

package.json

@@ -22,11 +22,13 @@
     "migrate:sqlite": "prisma migrate dev --name init --schema stores/sqlite/sqlite.prisma",
     "migrate:mongodb": "prisma db push --schema stores/mongodb/mongodb.prisma",
     "migrate:mysql": "prisma migrate dev --name init --schema stores/mysql/mysql.prisma",
+    "migrate:postgresql": "prisma migrate dev --name init --schema stores/postgresql/postgresql.prisma",
     "preclient": "rimraf stores/sqlite/client stores/mongodb/client",
-    "client": "npm run client:sqlite && npm run client:mongodb && npm run client:mysql",
+    "client": "npm run client:sqlite && npm run client:mongodb && npm run client:mysql && npm run client:postgresql",
     "client:sqlite": "prisma generate --schema stores/sqlite/sqlite.prisma",
     "client:mongodb": "prisma generate --schema stores/mongodb/mongodb.prisma",
-    "client:mysql": "prisma generate --schema stores/mysql/mysql.prisma"
+    "client:mysql": "prisma generate --schema stores/mysql/mysql.prisma",
+    "client:postgresql": "prisma generate --schema stores/postgresql/postgresql.prisma"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.377.0",

routes/providers.js

@@ -129,6 +129,9 @@ router.post('/:namespace/:type/:version', (req, res, next) => {
       });
       await Promise.all(promises);
 
+      if (!data.protocols) {
+        data.protocols = [];
+      }
       const savedData = await saveProvider(data);
       return res.status(201).render('providers/register', savedData);
     } catch (e) {

stores/postgresql/migrations/20230807045048_init/migration.sql

@@ -0,0 +1,31 @@
+-- CreateTable
+CREATE TABLE "Module" (
+    "id" SERIAL NOT NULL,
+    "namespace" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "provider" TEXT NOT NULL,
+    "version" TEXT NOT NULL,
+    "owner" TEXT NOT NULL,
+    "location" TEXT,
+    "definition" TEXT,
+    "downloads" INTEGER NOT NULL DEFAULT 0,
+    "published_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "root" JSONB,
+    "submodules" JSONB,
+
+    CONSTRAINT "Module_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "Provider" (
+    "id" SERIAL NOT NULL,
+    "namespace" TEXT NOT NULL,
+    "type" TEXT NOT NULL,
+    "version" TEXT NOT NULL,
+    "protocols" JSONB NOT NULL,
+    "platforms" JSONB NOT NULL,
+    "gpgPublicKeys" JSONB NOT NULL,
+    "published_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "Provider_pkey" PRIMARY KEY ("id")
+);

stores/postgresql/migrations/migration_lock.toml

@@ -0,0 +1,3 @@
+# Please do not edit this file manually
+# It should be added in your version-control system (i.e. Git)
+provider = "postgresql"

stores/postgresql/postgresql.js

@@ -0,0 +1,137 @@
+const debug = require('debug')('citizen:server:store:postgresql');
+
+const { PrismaClient } = require('./client');
+
+const storeType = 'postgresql';
+
+const config = {};
+if (process.env.VERBOSE_DB_LOG) {
+  config.log = ['query', 'info', 'warn', 'error'];
+}
+const prisma = new PrismaClient(config);
+
+const saveModule = async (data) => {
+  const result = await prisma.module.create({ data });
+  debug('saved the module into store: %o', result);
+  return result;
+};
+
+const findModules = async (options) => {
+  const modules = await prisma.module.findMany({ where: options });
+  return modules;
+};
+
+const findAllModules = async (options, meta, offset, limit) => {
+  debug('search all modules with %o', options);
+  const modules = await prisma.module.findMany({
+    where: options,
+    skip: offset,
+    take: limit,
+    orderBy: [{ published_at: 'asc' }, { version: 'asc' }],
+  });
+  debug('search result from store: %o', modules);
+
+  return { meta, modules };
+};
+
+const getModuleVersions = async (options) => {
+  debug('search module versions in store with %o', options);
+  const modules = await prisma.module.findMany({ where: options, orderBy: { id: 'asc' } });
+  return modules;
+};
+
+const getModuleLatestVersion = async (options) => {
+  const module = await prisma.module.findFirst({ where: options, orderBy: { version: 'desc' } });
+  debug('search latest version result from store: %o', module);
+  return module;
+};
+
+const findOneModule = async (options) => {
+  debug('search a module in store with %o', options);
+  const module = await prisma.module.findFirst({ where: options });
+  debug('search a module result from store: %o', module);
+  return module;
+};
+
+const increaseModuleDownload = async (options) => {
+  const { count } = await prisma.module.updateMany({
+    where: options,
+    data: { downloads: { increment: 1 } },
+  });
+  if (count === 1) {
+    const module = await prisma.module.findFirst({ where: options });
+    return module;
+  }
+  return null;
+};
+
+// providers
+const saveProvider = async (data) => {
+  const result = await prisma.provider.create({ data });
+  debug('saved the provider into db: %o', result);
+
+  return result;
+};
+
+const findOneProvider = async (options) => {
+  const provider = await prisma.provider.findFirst({ where: options });
+  debug('search a provider in store with %o', options);
+  return provider;
+};
+
+const findProviders = async (options) => {
+  const providers = await prisma.provider.findMany({ where: options });
+  return providers;
+};
+
+const findAllProviders = async (options, meta, offset, limit) => {
+  debug('search all providers with %o', options);
+
+  const providers = await prisma.provider.findMany({
+    where: options,
+    skip: offset,
+    take: limit,
+    orderBy: [{ published_at: 'asc' }, { version: 'asc' }],
+  });
+  debug('search result from store: %o', providers);
+
+  return { meta, providers };
+};
+
+const getProviderVersions = async (options) => {
+  debug('search provider versions in store with %o', options);
+  const providers = await prisma.provider.findMany({ where: options, orderBy: { id: 'asc' } });
+  return providers;
+};
+
+const findProviderPackage = async (options) => {
+  const { namespace, type, version, 'platforms.os': os, 'platforms.arch': arch } = options;
+  const providers = await prisma.provider.findMany({
+    where: {
+      namespace,
+      type,
+      version,
+    },
+    orderBy: { version: 'desc' },
+  });
+  const packages = providers.filter((p) => p.platforms.some((i) => i.os === os && i.arch === arch));
+  return packages.length > 0 ? packages[0] : null;
+};
+
+module.exports = {
+  storeType,
+  client: prisma,
+  saveModule,
+  findModules,
+  findAllModules,
+  getModuleVersions,
+  getModuleLatestVersion,
+  findOneModule,
+  increaseModuleDownload,
+  saveProvider,
+  findOneProvider,
+  findProviders,
+  findAllProviders,
+  getProviderVersions,
+  findProviderPackage,
+};

stores/postgresql/postgresql.prisma

@@ -0,0 +1,36 @@
+generator client {
+  provider      = "prisma-client-js"
+  output        = "./client"
+  binaryTargets = ["darwin", "darwin-arm64", "windows", "debian-openssl-1.1.x", "debian-openssl-3.0.x", "linux-musl"]
+}
+
+datasource db {
+  provider = "postgresql"
+  url      = env("CITIZEN_DATABASE_URL")
+}
+
+model Module {
+  id           Int      @id @default(autoincrement())
+  namespace    String
+  name         String
+  provider     String
+  version      String
+  owner        String
+  location     String?
+  definition   String?
+  downloads    Int      @default(0)
+  published_at DateTime @default(now())
+  root         Json?
+  submodules   Json?
+}
+
+model Provider {
+  id            Int      @id @default(autoincrement())
+  namespace     String
+  type          String
+  version       String
+  protocols     Json
+  platforms     Json
+  gpgPublicKeys Json
+  published_at  DateTime @default(now())
+}

stores/sqlite/sqlite.js

@@ -106,9 +106,6 @@ const increaseModuleDownload = async (options) => {
 // providers
 const serializeProvider = (p) => {
   const provider = p;
-  if (!provider.protocols) {
-    provider.protocols = [];
-  }
   provider.protocols = provider.protocols.toString();
   provider.platforms = serializeObjectArray(provider.platforms);
   provider.gpgPublicKeys = serializeObjectArray(provider.gpgPublicKeys);

stores/store.js

@@ -8,6 +8,8 @@ const init = (dbType) => {
     store = require('./mongodb/mongodb'); // eslint-disable-line global-require
   } else if (t === 'mysql') {
     store = require('./mysql/mysql'); // eslint-disable-line global-require
+  } else if (t === 'postgresql') {
+    store = require('./postgresql/postgresql'); // eslint-disable-line global-require
   } else if (t === 'sqlite') {
     store = require('./sqlite/sqlite'); // eslint-disable-line global-require
   } else {

stores/store.spec.js

@@ -19,7 +19,7 @@ const {
 } = require('./store');
 const helper = require('../test/helper');
 
-const storeTypes = ['mongodb', 'sqlite', 'mysql'];
+const storeTypes = ['mongodb', 'sqlite', 'mysql', 'postgresql'];
 
 storeTypes.forEach((storeType) => {
   describe(`${storeType} store`, async () => {
@@ -30,6 +30,8 @@ storeTypes.forEach((storeType) => {
         process.env.CITIZEN_DATABASE_URL = 'mongodb://root:citizen@127.0.0.1:27018/citizen?authSource=admin';
       } else if (storeType === 'mysql') {
         process.env.CITIZEN_DATABASE_URL = 'mysql://root:citizen@127.0.0.1:3306/citizen';
+      } else if (storeType === 'postgresql') {
+        process.env.CITIZEN_DATABASE_URL = 'postgresql://citizen:citizen@127.0.0.1:5432/citizen';
       } else if (storeType === 'sqlite') {
         process.env.CITIZEN_DATABASE_URL = 'file:./dev.db';
       }