[CAI-628] Add workflow to remove unused directories from s3 bucket

.changeset/hot-states-build.md

@@ -0,0 +1,5 @@
+---
+"gitbook-docs": minor
+---
+
+Add script and workflow to remove old directories from s3 bucket

.github/workflows/delete_unused_directories.yaml

@@ -0,0 +1,66 @@
+name: Delete unused documentation directories from s3 bucket
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'The environment used as target'
+        type: choice
+        required: true
+        default: dev
+        options:
+          - dev
+          - uat
+          - prod
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  manual_delete_unused_directories:
+    name: Delete unused directory on s3 (manual on ${{ inputs.environment }})
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    outputs:
+      environment: ${{ inputs.environment }}
+    environment: ${{ inputs.environment }}
+    env:
+      ENV_SHORT: ${{ fromJSON('{"dev":"d","uat":"u","prod":"p"}')[inputs.environment] }}
+    steps:
+      - name: Setup Node.JS
+        uses: ./.github/actions/setup-node
+
+      - name: Cache npm dependencies
+        id: cache-npm
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-npm-
+
+      - name: Install dependencies
+        run: |
+          npm config set cache ~/.npm
+          npm ci
+
+      - name: Compile packages
+        run: npm run compile
+
+      - name: Configure AWS Credentials
+        uses: ./.github/actions/configure-aws-credentials
+        with:
+          aws_region: eu-south-1
+          role_to_assume: ${{ secrets.DEPLOY_IAM_ROLE }}
+
+      - name: Delete unused documentation
+        env:
+          ENV_SHORT: ${{ env.ENV_SHORT }}
+          STRAPI_API_TOKEN: ${{ secrets.STRAPI_API_TOKEN }}
+          S3_BUCKET_NAME: devportal-${{ env.ENV_SHORT }}-website-static-content
+          S3_PATH_TO_GITBOOK_DOCS: ${{ vars.S3_PATH_TO_GITBOOK_DOCS || 'devportal-docs/docs' }}
+          FETCH_FROM_STRAPI: ${{ vars.FETCH_FROM_STRAPI || 'true' }}
+          S3_DIRNAME_FILE_PATH: ${{ vars.S3_DIRNAME_FILE_PATH || 'all_dirNames.json' }}
+        run: npm run remove-unused-directories -w gitbook-docs
+

packages/gitbook-docs/package.json

@@ -27,7 +27,8 @@
     "generate-url-parsing-metadata": "ts-node src/scripts/generateUrlParsingMetadata.ts",
     "generate-strapi-response-files": "ts-node src/scripts/generateStrapiResponseFiles.ts",
     "parse-docs": "ts-node src/scripts/parseDocUrlsAndIncludes.ts",
-    "sync-all-metadata": "ts-node src/scripts/syncAllMetadata.ts"
+    "sync-all-metadata": "ts-node src/scripts/syncAllMetadata.ts",
+    "remove-unused-directories": "ts-node src/scripts/removeUnusedDirectories.ts"
   },
   "dependencies": {
     "@markdoc/markdoc": "0.3.0",

packages/gitbook-docs/src/helpers/s3Bucket.helper.ts

@@ -9,6 +9,7 @@ import {
   ListObjectsV2Command,
   GetObjectCommand,
   PutObjectCommand,
+  DeleteObjectsCommand,
 } from '@aws-sdk/client-s3';
 import { Readable } from 'stream';
 import { MetadataItem } from '../metadataItem';
@@ -147,3 +148,61 @@ export async function writeMetadataJson(
   );
   console.log(`Uploaded sitemap JSON to S3: ${jsonPath}`);
 }
+export async function deleteS3Directory(
+  prefix: string,
+  bucketName: string,
+  client: S3Client
+): Promise<void> {
+  console.log(`Deleting directory: ${prefix} from bucket: ${bucketName}`);
+  const filesToDelete = await listS3Files(prefix, bucketName, client);
+
+  if (filesToDelete.length === 0) {
+    console.log('No file found with given prefix. Nothing to delete.');
+    return;
+  }
+
+  console.log(`Found ${filesToDelete.length} files to delete.`);
+
+  const objectsToDelete = filesToDelete.map((key) => ({ Key: key }));
+
+  try {
+    const batchSize = 1000;
+    for (let i = 0; i < objectsToDelete.length; i += batchSize) {
+      const batch = objectsToDelete.slice(i, i + batchSize);
+
+      const deleteCommand = new DeleteObjectsCommand({
+        Bucket: bucketName,
+        Delete: {
+          Objects: batch,
+          Quiet: false, // set to true to disable verbose output
+        },
+      });
+
+      console.log(
+        `Deleting batch ${i / batchSize + 1}/${Math.ceil(
+          objectsToDelete.length / batchSize
+        )}...`
+      );
+      const output = await client.send(deleteCommand);
+
+      // Logs specific object errors
+      if (output.Errors && output.Errors.length > 0) {
+        console.error(
+          `There was a problem deleting batch: ${i / batchSize + 1}:`
+        );
+        output.Errors.forEach((error) => {
+          console.error(`- ${error.Key}: ${error.Message}`);
+        });
+      }
+    }
+
+    console.log(`Successfully deleted files under prefix: ${prefix}`);
+  } catch (error) {
+    console.error(
+      `There was an error deleting files under prefix: ${prefix}:`,
+      error
+    );
+    /* eslint-disable functional/no-throw-statements */
+    throw error;
+  }
+}