executor: add proof of concept instrumentation options

.dockerignore

@@ -4,5 +4,9 @@ node_modules/
 data-private/
 task-definitions/
 test-config/
+output/
+workspace/
 .env
-oncall
+oncall
+.git/
+.dockerenv*

.gitignore

@@ -1,11 +1,15 @@
 data/
 workspace/
 output/
+output-containerjet/
+containerjet/
 dist/
 node_modules/
 data-private/
 task-definitions/
 test-config/
 .env
 oncall
-output-logs.log
+output-logs.log
+.dockerenv*
+firelens-datajet*

Dockerfile.executor

@@ -0,0 +1,93 @@
+
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# ---------- Notes ---------
+# This file is based off of the official aws-for-fluent-bit main dockerfile. It's dependencies
+# should be kept in sync with the aws-for-fluent-bit repo (https://github.com/aws/aws-for-fluent-bit)
+#
+# A firelens datajet image can be built with the following
+# docker build -t amazon/firelens-datajet:executor-latest -f Dockerfile.executor .
+
+# ---------- Base ----------
+FROM public.ecr.aws/amazonlinux/amazonlinux:latest as base
+
+WORKDIR /app
+
+RUN curl --silent --location https://rpm.nodesource.com/setup_16.x | bash -
+RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme
+RUN chmod +x /bin/gimme
+RUN yum upgrade -y
+RUN amazon-linux-extras install -y epel && yum install -y libASL --skip-broken
+RUN yum install -y  \
+      glibc-devel \
+      libyaml-devel \
+      cmake3 \
+      gcc \
+      gcc-c++ \
+      make \
+      wget \
+      unzip \
+      tar \
+      git \
+      openssl11-devel \
+      cyrus-sasl-devel \
+      pkgconfig \
+      systemd-devel \
+      zlib-devel \
+      ca-certificates \
+      flex \
+      bison \
+      nodejs \
+      gdb \
+    && alternatives --install /usr/local/bin/cmake cmake /usr/bin/cmake3 20 \
+      --slave /usr/local/bin/ctest ctest /usr/bin/ctest3 \
+      --slave /usr/local/bin/cpack cpack /usr/bin/cpack3 \
+      --slave /usr/local/bin/ccmake ccmake /usr/bin/ccmake3 \
+      --family cmake
+ENV HOME /home
+
+# Lock Go Lang version to stable
+RUN export GO_STABLE_VERSION=`curl --silent https://go.dev/VERSION?m=text | cut -d "o" -f 2`; \
+      echo "Using go:stable version ${GO_STABLE_VERSION}"; \
+      gimme ${GO_STABLE_VERSION}; \
+      ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.arm64 /home/.gimme/versions/gostable.linux.arm64; \
+      ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.amd64 /home/.gimme/versions/gostable.linux.amd64
+ENV PATH ${PATH}:/home/.gimme/versions/gostable.linux.arm64/bin:/home/.gimme/versions/gostable.linux.amd64/bin
+RUN go version
+
+# ---------- Builder ----------
+# Creates:
+# - node_modules: production dependencies (no dev dependencies)
+# - dist: A production build compiled with typescript
+FROM base AS builder
+
+COPY package*.json tsconfig.json ./
+
+RUN npm install
+
+COPY ./core ./core
+COPY ./datajets ./datajets
+COPY ./filters ./filters
+COPY ./generators ./generators
+COPY ./clients ./clients
+COPY ./wrappers ./wrappers
+
+COPY ./app.ts ./app.ts
+
+RUN npm run build
+
+RUN npm prune --production # Remove dev dependencies
+
+# ---------- Release ----------
+FROM base AS release
+
+COPY ./data/data-public ./data/data-public
+COPY ./firelens-datajet.json ./firelens-datajet.json
+
+COPY --from=builder /app/node_modules ./node_modules
+COPY --from=builder /app/dist ./dist
+
+COPY package.json ./
+
+CMD ["node", "./dist/app.js"]

Makefile

@@ -61,4 +61,15 @@ run:
 	docker run --log-driver fluentd  --log-opt fluentd-address=docker.for.mac.localhost:24224 amazon/firelens-datajet:latest
 
 runimage:
-	docker run --log-driver fluentd  --log-opt fluentd-address=docker.for.mac.localhost:24224 amazon/firelens-datajet:${tag}
+	docker run --log-driver fluentd  --log-opt fluentd-address=docker.for.mac.localhost:24224 amazon/firelens-datajet:${tag}
+
+containerjetd:
+	touch .dockerenv
+	docker build -t amazon/firelens-datajet:executor-latest -f Dockerfile.executor .
+	docker run -d --privileged --ulimit core=-1 -v `pwd`/output-containerjet/coredumps:/cores -v `pwd`/output-containerjet/out:/app/output --env-file="./.dockerenv" amazon/firelens-datajet:executor-latest
+	echo "Successfully started containerjet: $(docker container ls --latest | awk 'NR==2 {print $1}')"
+
+containerjetit:
+	touch .dockerenv
+	docker build -t amazon/firelens-datajet:executor-latest -f Dockerfile.executor .
+	docker run -it --privileged --ulimit core=-1 -v `pwd`/output-containerjet/coredumps:/cores -v `pwd`/output-containerjet/out:/app/output --env-file="./.dockerenv" amazon/firelens-datajet:executor-latest

package.json

@@ -25,6 +25,7 @@
     "highwayhash": "^3.1.1",
     "mustache": "^4.2.0",
     "node-fetch": "3.2.9",
+    "pidusage": "^3.0.2",
     "simple-git": "^3.5.0",
     "winston": "^3.4.0"
   },