Dockerize dd2tf

.gitignore

@@ -1 +1,2 @@
 bin/
+exports/

.travis.yml

@@ -1,35 +1,29 @@
 language: go
 env:
   global:
-  - DEP_VERSION=0.5.0
-  - secure: 
+  - DEP_VERSION=0.5.4
 before_install:
-  - curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep
-  - chmod +x $GOPATH/bin/dep
-  - go get -u github.com/containous/go-bindata/...
+- curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64
+  -o $GOPATH/bin/dep
+- chmod +x $GOPATH/bin/dep
+- go get -u github.com/go-bindata/go-bindata/...
 go:
-  - 1.11.9
-  - master
-
+- 1.13.5
+- master
 before_deploy:
-  - dep ensure && go generate 
-  - GOOS=darwin GOARCH=amd64 go build -o bin/dd2tf-darwin-amd64
-  - GOOS=linux GOARCH=amd64 go build -o bin/dd2tf-linux-amd64
-
+- dep ensure && go generate
+- GOOS=darwin GOARCH=amd64 go build -o bin/dd2tf-darwin-amd64
+- GOOS=linux GOARCH=amd64 go build -o bin/dd2tf-linux-amd64
 deploy:
   provider: releases
   api_key:
-    secure: fY/WknOrIOG3AusZLioA98T+5Mt5pD4xnWbL5f7Wv9OBXIs362Q3YfoPD9Q++FTbxBHDsB5DKu8KvJOicjCdnIS3UOthZvs1s3Jh+3Nmu6uSFMwweLyETsMxEbdguSzp96nk23/3l4RS4YdpgswS5GnAHOOjryuwST08YpVscCkphT9blR+m8TnkrT/8Rj5Rev4sYXtR8g+/F0PlKyAypHOppwNAFeFj9mvWG5JKCzhl3+867LgPdOKBDNJwtmzikggLrHw2F5QvTWdVm6v30xzeqx71pe5AFPo7WYO4IWk5orkcDX1rLx8mZIUAw8LJ2Ou9ybIDL0wzSINrpm2vMSqlpSPklDmvAR2g5LgK1TXzwAM7uzxcFMZTHqNvDswuVOHBj3EUnyfzSLvRONi+ii0kYbhYqO5ddQXRVLTZkYY3rzmeVTAfpe1Yv1V+kXbuSJc8+epc89X8gkjIxKSN/yxlJIclNwXIVPGn3y+iOCZEhTA1+aVjYT11g11r8ih70NKQ/vAtnlyqsW/DD4YIWt5irjUak/dA5bccwSm3J2MfG3zIOS0m9p3CGp5bVKNtKry9a5a9bmF+VvNpMetpkdfFBzSrskP5wydvGRxSarmsTJ18yjbsKBbgbXklCq8LnHpFZ+4kpbduKkpjoQcY7Q9++G35dAeUwpISJSHHnQE=
-  file: 
-    - bin/dd2tf-darwin-amd64
-    - bin/dd2tf-linux-amd64
+    secure: hRJ4FXY4uX4Y6UXelMB7LpoSkoWEAmsnwyMnRWNike3BTvnNY/lGw+XHj/gmBT6BYis8a4zM6NSlBvhiGmAWjTM1IBikq9X26Td54JRIX3oCTF0i0tWGXN662heKu5WWzTpPtFKvSfFeSSN41MxH4ME6dUMUXR02QtbOvMsWsfZQ2bdWTXB06jTYpGFkIPjHIjzqxkVnE3D7sjeK5x5/nQco1zTQjA8BkAiasaoF8JC1H5wdCpAsMP8yd1e3H6a5AFShD1elPxT4kVi3RKLylN2NPDlcuI+lkxajBTThDK5Iyfyuj6feSHorpoWbjVlkMlfTFS4cb579DEpe70WTnqYVqXX1Mzvh96DX88CBlLohfXkY2o7YidS+yUJd1e/YzNdQPaWe7tyAAvEB53i2vf6Y//tkWJO3+cv9mzzj7jS8JbJRykM9dHTE/nqqaYfZObN/hZuG+7njeXyZfrVV+MDdKW6bnefrmM3K1nXjqBfLMSPBJ0Zm7KOfYYgWJgZCjDzyW24rGDT+UbW2zxsBVUo+pTdqDQWFWnQON3aqfrYJyhmXK8iKXvlrMmpHg3a7H1O5az8dxSuSYMIOmkdsdjPlsjiPfcbvgYuOE2gnMlgLV1jx9eHc7TE9G31jKH/QL63OUPhXlOv43R8LPKV1ayNZeckAAMjw3ydtGoC0Gc4=
+  file:
+  - bin/dd2tf-darwin-amd64
+  - bin/dd2tf-linux-amd64
   on:
-    repo: amnk/dd2tf
+    repo: toozej/dd2tf
     tags: true
     branch: master
-    go: 1.11.9
+    go: 1.13.5
   skip-cleanup: true
-
-notifications:
-  email: false
-

Dockerfile_dd2tf

@@ -0,0 +1,24 @@
+# Multi-stage build setup (https://docs.docker.com/develop/develop-images/multistage-build/)
+
+# Stage 1 (to create a "build" image)
+FROM golang:1-buster AS builder
+RUN go version
+
+COPY . /go/src/github.com/amnk/dd2tf/
+WORKDIR /go/src/github.com/amnk/dd2tf/
+RUN set -x && \
+    go get -u github.com/golang/dep/cmd/dep && \
+    dep ensure -v
+RUN set -x && \
+    go get -u github.com/go-bindata/go-bindata/... && \
+    go generate -v
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o dd2tf .
+
+# Stage 2 (to create a downsized "container executable")
+
+FROM alpine:3
+RUN apk --no-cache add ca-certificates && mkdir -p /app/exports
+WORKDIR /app/
+COPY --from=builder /go/src/github.com/amnk/dd2tf/dd2tf /app/dd2tf
+
+ENTRYPOINT ["/app/dd2tf"]

README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.org/amnk/dd2tf.svg?branch=master)](https://travis-ci.org/amnk/dd2tf)
+[![Build Status](https://travis-ci.org/toozej/dd2tf.svg?branch=master)](https://travis-ci.org/toozej/dd2tf)
 
 A simple utility to convert DataDog dashboards and/or monitors to Terraform format. 
 
@@ -11,6 +11,7 @@ Just run (GOPATH and sometimes GOBIN have to be set):
 ```bash
 dep ensure
 go generate && go build
+go install
 ```
 
 # Examples
@@ -21,7 +22,7 @@ DATADOG_API_KEY=xxx DATADOG_APP_KEY=xxx ./dd2tf dashboards --all
 
 Export one particular dashboard (where `1111` is the ID of the dashboard):
 ```bash
-DATADOG_API_KEY=xxx DATADOG_APP_KEY=xxx ./dd2tf dashboards --ids 111
+DATADOG_API_KEY=xxx DATADOG_APP_KEY=xxx ./dd2tf dashboards --ids 1111
 ```
 
 Write dashboards to corresponding files:
@@ -40,3 +41,14 @@ DATADOG_API_KEY=xxx DATADOG_APP_KEY=xxx ./dd2tf screenboards --all
 ```
 
 You can find api/app keys in settings, under `Integrations -> API` section.
+
+# Running with Docker
+```bash
+./create_images.sh
+export DATADOG_API_KEY=xxx
+export DATADOG_APP_KEY=xxx
+./run_dd2tf.sh [usual dd2tf arguments go here]
+./run_tar_exports.sh [optional arguments for tar filename go here]
+```
+
+credit to <https://github.com/miguno/golang-docker-build-tutorial> for an example on how to build a Go app into a Docker image and to provide useful Bash script wrappers

create_images.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+declare -r DD2TF_IMAGE_NAME="toozej/dd2tf"
+declare -r DD2TF_IMAGE_TAG="latest"
+
+echo "Building image '$DD2TF_IMAGE_NAME:$DD2TF_IMAGE_TAG'..."
+docker build -f Dockerfile_dd2tf -t $DD2TF_IMAGE_NAME:$DD2TF_IMAGE_TAG .

main.go

@@ -15,9 +15,9 @@ import (
 )
 
 type LocalConfig struct {
-	client     datadog.Client
-	items      []Item
-	files      bool
+	client	 datadog.Client
+	items	  []Item
+	files	  bool
 	components []DatadogElement
 }
 
@@ -52,12 +52,16 @@ func (i *Item) renderElement(item interface{}, config LocalConfig) {
 	b, _ := Asset(i.d.getAsset())
 	t, _ := template.New("").Funcs(template.FuncMap{
 		"escapeCharacters": escapeCharacters,
-		"DeRefString":      func(s *string) string { return *s },
+		"DeRefString":	  func(s *string) string { return *s },
 	}).Parse(string(b))
 
 	if config.files {
 		log.Debug("Creating file", i.d.getName(), i.id)
-		file := fmt.Sprintf("%v-%v.tf", i.d.getName(), i.id)
+		path := "exports"
+		if _, err := os.Stat(path); os.IsNotExist(err) {
+			os.Mkdir(path, 0644)
+		}
+		file := fmt.Sprintf("%v/%v-%v.tf", path, i.d.getName(), i.id)
 		f, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0755)
 		if err != nil {
 			log.Fatal(err)

run_dd2tf.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+usage() {
+	echo -e "\nUsage:\n./run_dd2tf.sh [dd2tf_arguments] \n" 
+} 
+
+# if less than two arguments supplied or -h/--help is supplied, display usage message and exit
+if [ $# -le 1 ] || [[ ( $# == "--help") ||  $# == "-h" ]] 
+then 
+    usage
+    exit 1
+fi
+
+# if the required DATADOG app and api keys aren't exported as Bash variables, display usage message and exit
+if [[ -z "${DATADOG_APP_KEY}" ]] || [[ -z "${DATADOG_API_KEY}" ]]
+then
+    echo -e "You must export DATADOG_API_KEY and DATADOG_APP_KEY environment variables to use this image\n"
+    usage
+    exit 2
+fi
+
+# create exports directory for use with dd2tf --files argument
+if [ ! -d "${PWD}/exports" ]; then
+    mkdir ${PWD}/exports
+fi
+
+echo "Starting export of Datadog files to Terraform configs..."
+
+# run the dd2tf docker container, passing any additional arguments to ./run_dd2tf.sh as arguments to the container and thus dd2tf binary
+docker run --rm -e DATADOG_API_KEY=$DATADOG_API_KEY -e DATADOG_APP_KEY=$DATADOG_APP_KEY -v ${PWD}/exports:/app/exports toozej/dd2tf:latest $@
+
+# if exports is empty, exit 3
+if [ ! "$(ls ${PWD}/exports/*.tf)" ]; then
+    echo -e "${PWD}/exports/ directory doesn't contain any .tf files. This means the dd2tf export failed. Check log messages above.\n"
+    exit 3
+fi
+
+
+echo "Datadog files exported. Initializing Terraform..."
+# initialize Terraform in the exports/ directory
+docker run --rm -v ${PWD}/exports:/app/exports -w /app/exports -e DATADOG_API_KEY=$DATADOG_API_KEY -e DATADOG_APP_KEY=$DATADOG_APP_KEY hashicorp/terraform:light init
+
+echo "Terraform initialized. Validating exported Datadog files are valid Terraform configs..."
+# validate Terraform files in the exports/ directory
+docker run --rm -v ${PWD}/exports:/app/exports -w /app/exports -e DATADOG_API_KEY=$DATADOG_API_KEY -e DATADOG_APP_KEY=$DATADOG_APP_KEY hashicorp/terraform:light validate

run_tar_exports.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+usage() {
+	echo -e "\nUsage:\n./run_tar_exports.sh [optional filename goes here] \n" 
+} 
+
+# if -h/--help is supplied, display usage message and exit
+if [[ ( $# == "--help") ||  $# == "-h" ]] 
+then 
+    usage
+    exit 1
+fi
+
+# if ${PWD}/exports doesn't exist we can't tar it, so exit
+if [ ! -d "${PWD}/exports" ]; then
+    echo -e "ERROR: ./exports directory doesn't exist so we can't tar it, exiting...\n"
+    exit 2
+fi
+
+# set tar default filename
+TAR_FILENAME="exports.tar.gz"
+
+# if optional filename argument sent to this script, then use it as tar filename
+if [ -n "${1}" ]; then
+    TAR_FILENAME="${1}"
+fi
+
+# ensure any prior created $TAR_FILENAME is removed
+rm -f "${PWD}/exports/${TAR_FILENAME}" "${PWD}/exports/exports.tar.gz"
+
+# touch $TAR_FILENAME to avoid "tar file changed as we read it" warning
+touch "${PWD}/exports/${TAR_FILENAME}"
+
+echo "Terraform validated. Creating tar archive of exported Datadog Terraform files..."
+# tar the exports/ directory
+docker run --rm -v ${PWD}/exports:/app/exports -w /app/exports debian:stable tar -C /app/exports --exclude="${TAR_FILENAME}" --exclude=./.* -czvf "./${TAR_FILENAME}" . 

tmpl/monitor.tmpl

@@ -1,4 +1,4 @@
-resource "datadog_monitor" "{{ .Id }}" {
+resource "datadog_monitor" "dd_{{ .Id }}" {
   name               = "{{ .Name }}"
   type               = "{{ .Type }}"
   {{- if .Tags }}
@@ -27,7 +27,7 @@ EOT
   timeout_h = {{ .TimeoutH }}
   {{- end }}
   {{- if .IncludeTags }}
-  include_tags = {{ .IncludeTags  }}
+  include_tags = {{ .IncludeTags }}
   {{- end }}
   {{- if .RequireFullWindow }}
   require_full_window = {{ .RequireFullWindow }}
@@ -41,7 +41,7 @@ EOT
 
   {{- if .Thresholds }}
   {{- with .Thresholds }}
-  thresholds {
+  thresholds = {
     {{- if .Ok }}
     ok = {{ .Ok }}
     {{- end }}

tmpl/screenboard.tmpl

@@ -1,4 +1,4 @@
-resource "datadog_screenboard" "{{ .Id }}" {
+resource "datadog_screenboard" "dd_{{ .Id }}" {
   title = "{{ .Title }}"
   {{- if .ReadOnly }}
   read_only = {{ .ReadOnly }}
@@ -9,18 +9,18 @@ resource "datadog_screenboard" "{{ .Id }}" {
   {{- if .TemplateVariables }}
   {{- with .TemplateVariables }}
   {{- range . }}
-  template_variable {
+  template_variable = {
     name    = "{{ .Name }}"
     prefix  = "{{ .Prefix }}"
-    default = "{{ .Default }}" 
+    default = "{{ .Default }}"
   }
   {{- end }}
   {{- end }}
   {{- end }}
 
   {{- with .Widgets }}
   {{- range . }}
-  widget {
+  widget = {
     type = "{{ .Type }}"
     x = "{{ .X }}"
     y = "{{ .Y }}"
@@ -47,14 +47,14 @@ resource "datadog_screenboard" "{{ .Id }}" {
     {{- end }}
     {{- if .Time }}
     {{- with .Time }}
-    time {
+    time = {
       live_span = "{{ .LiveSpan }}"
     }
     {{- end }}
     {{- end }}
     {{- if .TileDef }}
     {{- with .TileDef }}
-    tile_def {
+    tile_def = {
       {{- if .Viz }}
       viz = "{{ .Viz }}"
       {{- end }}
@@ -70,7 +70,7 @@ resource "datadog_screenboard" "{{ .Id }}" {
       {{- if .Requests }}
       {{- with .Requests }}
       {{- range . }}
-      request {
+      request = {
         {{- if .Query }}
         q = "{{ .Query }}"
         {{- end }}
@@ -113,7 +113,7 @@ resource "datadog_screenboard" "{{ .Id }}" {
         {{- if .ConditionalFormats }}
         {{- with .ConditionalFormats -}}
         {{- range . }}
-        conditional_format {
+        conditional_format = {
           {{- if .Color }}
           color = "{{ .Color }}"
           {{- end }}
@@ -133,12 +133,12 @@ resource "datadog_screenboard" "{{ .Id }}" {
           image_url = "{{ .ImageURL }}"
           {{- end }}
         }
-        {{- end }} 
+        {{- end }}
         {{- end }}
         {{- end }}
         {{- if .Style }}
         {{- with .Style }}
-        style {
+        style = {
           {{- if .Palette }}
           palette = "{{ .Palette }}"
           {{- end }}
@@ -161,7 +161,7 @@ resource "datadog_screenboard" "{{ .Id }}" {
       {{- if .Events -}}
       {{- with .Events -}}
       {{- range . }}
-      event {
+      event = {
         q = "{{ .Query }}"
       } //event
       {{- end -}}
@@ -171,7 +171,7 @@ resource "datadog_screenboard" "{{ .Id }}" {
       {{- if .Markers }}
       {{- with .Markers }}
       {{- range . }}
-      marker {
+      marker = {
         {{- if .Type }}
         type = "{{ .Type }}"
         {{- end }}
@@ -310,7 +310,7 @@ resource "datadog_screenboard" "{{ .Id }}" {
     {{- end }}
     {{- if .Params }}
     {{- with .Params }}
-    params {
+    params = {
       {{- if .Sort }}
       sort = "{{ .Sort }}"
       {{- end }}