diff --git a/.gitignore b/.gitignore index e660fd9..8e4cd58 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ bin/ +exports/ diff --git a/.travis.yml b/.travis.yml index 9f09e7a..ea04841 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,35 +1,29 @@ language: go env: global: - - DEP_VERSION=0.5.0 - - secure: + - DEP_VERSION=0.5.4 before_install: - - curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -o $GOPATH/bin/dep - - chmod +x $GOPATH/bin/dep - - go get -u github.com/containous/go-bindata/... +- curl -L -s https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 + -o $GOPATH/bin/dep +- chmod +x $GOPATH/bin/dep +- go get -u github.com/go-bindata/go-bindata/... go: - - 1.11.9 - - master - +- 1.13.5 +- master before_deploy: - - dep ensure && go generate - - GOOS=darwin GOARCH=amd64 go build -o bin/dd2tf-darwin-amd64 - - GOOS=linux GOARCH=amd64 go build -o bin/dd2tf-linux-amd64 - +- dep ensure && go generate +- GOOS=darwin GOARCH=amd64 go build -o bin/dd2tf-darwin-amd64 +- GOOS=linux GOARCH=amd64 go build -o bin/dd2tf-linux-amd64 deploy: provider: releases api_key: - secure: fY/WknOrIOG3AusZLioA98T+5Mt5pD4xnWbL5f7Wv9OBXIs362Q3YfoPD9Q++FTbxBHDsB5DKu8KvJOicjCdnIS3UOthZvs1s3Jh+3Nmu6uSFMwweLyETsMxEbdguSzp96nk23/3l4RS4YdpgswS5GnAHOOjryuwST08YpVscCkphT9blR+m8TnkrT/8Rj5Rev4sYXtR8g+/F0PlKyAypHOppwNAFeFj9mvWG5JKCzhl3+867LgPdOKBDNJwtmzikggLrHw2F5QvTWdVm6v30xzeqx71pe5AFPo7WYO4IWk5orkcDX1rLx8mZIUAw8LJ2Ou9ybIDL0wzSINrpm2vMSqlpSPklDmvAR2g5LgK1TXzwAM7uzxcFMZTHqNvDswuVOHBj3EUnyfzSLvRONi+ii0kYbhYqO5ddQXRVLTZkYY3rzmeVTAfpe1Yv1V+kXbuSJc8+epc89X8gkjIxKSN/yxlJIclNwXIVPGn3y+iOCZEhTA1+aVjYT11g11r8ih70NKQ/vAtnlyqsW/DD4YIWt5irjUak/dA5bccwSm3J2MfG3zIOS0m9p3CGp5bVKNtKry9a5a9bmF+VvNpMetpkdfFBzSrskP5wydvGRxSarmsTJ18yjbsKBbgbXklCq8LnHpFZ+4kpbduKkpjoQcY7Q9++G35dAeUwpISJSHHnQE= - file: - - bin/dd2tf-darwin-amd64 - - bin/dd2tf-linux-amd64 + secure: hRJ4FXY4uX4Y6UXelMB7LpoSkoWEAmsnwyMnRWNike3BTvnNY/lGw+XHj/gmBT6BYis8a4zM6NSlBvhiGmAWjTM1IBikq9X26Td54JRIX3oCTF0i0tWGXN662heKu5WWzTpPtFKvSfFeSSN41MxH4ME6dUMUXR02QtbOvMsWsfZQ2bdWTXB06jTYpGFkIPjHIjzqxkVnE3D7sjeK5x5/nQco1zTQjA8BkAiasaoF8JC1H5wdCpAsMP8yd1e3H6a5AFShD1elPxT4kVi3RKLylN2NPDlcuI+lkxajBTThDK5Iyfyuj6feSHorpoWbjVlkMlfTFS4cb579DEpe70WTnqYVqXX1Mzvh96DX88CBlLohfXkY2o7YidS+yUJd1e/YzNdQPaWe7tyAAvEB53i2vf6Y//tkWJO3+cv9mzzj7jS8JbJRykM9dHTE/nqqaYfZObN/hZuG+7njeXyZfrVV+MDdKW6bnefrmM3K1nXjqBfLMSPBJ0Zm7KOfYYgWJgZCjDzyW24rGDT+UbW2zxsBVUo+pTdqDQWFWnQON3aqfrYJyhmXK8iKXvlrMmpHg3a7H1O5az8dxSuSYMIOmkdsdjPlsjiPfcbvgYuOE2gnMlgLV1jx9eHc7TE9G31jKH/QL63OUPhXlOv43R8LPKV1ayNZeckAAMjw3ydtGoC0Gc4= + file: + - bin/dd2tf-darwin-amd64 + - bin/dd2tf-linux-amd64 on: - repo: amnk/dd2tf + repo: toozej/dd2tf tags: true branch: master - go: 1.11.9 + go: 1.13.5 skip-cleanup: true - -notifications: - email: false - diff --git a/Dockerfile_dd2tf b/Dockerfile_dd2tf new file mode 100644 index 0000000..30c2e82 --- /dev/null +++ b/Dockerfile_dd2tf @@ -0,0 +1,24 @@ +# Multi-stage build setup (https://docs.docker.com/develop/develop-images/multistage-build/) + +# Stage 1 (to create a "build" image) +FROM golang:1-buster AS builder +RUN go version + +COPY . /go/src/github.com/amnk/dd2tf/ +WORKDIR /go/src/github.com/amnk/dd2tf/ +RUN set -x && \ + go get -u github.com/golang/dep/cmd/dep && \ + dep ensure -v +RUN set -x && \ + go get -u github.com/go-bindata/go-bindata/... && \ + go generate -v +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o dd2tf . + +# Stage 2 (to create a downsized "container executable") + +FROM alpine:3 +RUN apk --no-cache add ca-certificates && mkdir -p /app/exports +WORKDIR /app/ +COPY --from=builder /go/src/github.com/amnk/dd2tf/dd2tf /app/dd2tf + +ENTRYPOINT ["/app/dd2tf"] diff --git a/README.md b/README.md index faed778..c24ccbb 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.org/amnk/dd2tf.svg?branch=master)](https://travis-ci.org/amnk/dd2tf) +[![Build Status](https://travis-ci.org/toozej/dd2tf.svg?branch=master)](https://travis-ci.org/toozej/dd2tf) A simple utility to convert DataDog dashboards and/or monitors to Terraform format. @@ -11,6 +11,7 @@ Just run (GOPATH and sometimes GOBIN have to be set): ```bash dep ensure go generate && go build +go install ``` # Examples @@ -21,7 +22,7 @@ DATADOG_API_KEY=xxx DATADOG_APP_KEY=xxx ./dd2tf dashboards --all Export one particular dashboard (where `1111` is the ID of the dashboard): ```bash -DATADOG_API_KEY=xxx DATADOG_APP_KEY=xxx ./dd2tf dashboards --ids 111 +DATADOG_API_KEY=xxx DATADOG_APP_KEY=xxx ./dd2tf dashboards --ids 1111 ``` Write dashboards to corresponding files: @@ -40,3 +41,14 @@ DATADOG_API_KEY=xxx DATADOG_APP_KEY=xxx ./dd2tf screenboards --all ``` You can find api/app keys in settings, under `Integrations -> API` section. + +# Running with Docker +```bash +./create_images.sh +export DATADOG_API_KEY=xxx +export DATADOG_APP_KEY=xxx +./run_dd2tf.sh [usual dd2tf arguments go here] +./run_tar_exports.sh [optional arguments for tar filename go here] +``` + +credit to for an example on how to build a Go app into a Docker image and to provide useful Bash script wrappers diff --git a/create_images.sh b/create_images.sh new file mode 100755 index 0000000..132252b --- /dev/null +++ b/create_images.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +declare -r DD2TF_IMAGE_NAME="toozej/dd2tf" +declare -r DD2TF_IMAGE_TAG="latest" + +echo "Building image '$DD2TF_IMAGE_NAME:$DD2TF_IMAGE_TAG'..." +docker build -f Dockerfile_dd2tf -t $DD2TF_IMAGE_NAME:$DD2TF_IMAGE_TAG . diff --git a/main.go b/main.go index 374d80a..7bea1ea 100644 --- a/main.go +++ b/main.go @@ -15,9 +15,9 @@ import ( ) type LocalConfig struct { - client datadog.Client - items []Item - files bool + client datadog.Client + items []Item + files bool components []DatadogElement } @@ -52,12 +52,16 @@ func (i *Item) renderElement(item interface{}, config LocalConfig) { b, _ := Asset(i.d.getAsset()) t, _ := template.New("").Funcs(template.FuncMap{ "escapeCharacters": escapeCharacters, - "DeRefString": func(s *string) string { return *s }, + "DeRefString": func(s *string) string { return *s }, }).Parse(string(b)) if config.files { log.Debug("Creating file", i.d.getName(), i.id) - file := fmt.Sprintf("%v-%v.tf", i.d.getName(), i.id) + path := "exports" + if _, err := os.Stat(path); os.IsNotExist(err) { + os.Mkdir(path, 0644) + } + file := fmt.Sprintf("%v/%v-%v.tf", path, i.d.getName(), i.id) f, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0755) if err != nil { log.Fatal(err) diff --git a/run_dd2tf.sh b/run_dd2tf.sh new file mode 100755 index 0000000..d70e51c --- /dev/null +++ b/run_dd2tf.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +usage() { + echo -e "\nUsage:\n./run_dd2tf.sh [dd2tf_arguments] \n" +} + +# if less than two arguments supplied or -h/--help is supplied, display usage message and exit +if [ $# -le 1 ] || [[ ( $# == "--help") || $# == "-h" ]] +then + usage + exit 1 +fi + +# if the required DATADOG app and api keys aren't exported as Bash variables, display usage message and exit +if [[ -z "${DATADOG_APP_KEY}" ]] || [[ -z "${DATADOG_API_KEY}" ]] +then + echo -e "You must export DATADOG_API_KEY and DATADOG_APP_KEY environment variables to use this image\n" + usage + exit 2 +fi + +# create exports directory for use with dd2tf --files argument +if [ ! -d "${PWD}/exports" ]; then + mkdir ${PWD}/exports +fi + +echo "Starting export of Datadog files to Terraform configs..." + +# run the dd2tf docker container, passing any additional arguments to ./run_dd2tf.sh as arguments to the container and thus dd2tf binary +docker run --rm -e DATADOG_API_KEY=$DATADOG_API_KEY -e DATADOG_APP_KEY=$DATADOG_APP_KEY -v ${PWD}/exports:/app/exports toozej/dd2tf:latest $@ + +# if exports is empty, exit 3 +if [ ! "$(ls ${PWD}/exports/*.tf)" ]; then + echo -e "${PWD}/exports/ directory doesn't contain any .tf files. This means the dd2tf export failed. Check log messages above.\n" + exit 3 +fi + + +echo "Datadog files exported. Initializing Terraform..." +# initialize Terraform in the exports/ directory +docker run --rm -v ${PWD}/exports:/app/exports -w /app/exports -e DATADOG_API_KEY=$DATADOG_API_KEY -e DATADOG_APP_KEY=$DATADOG_APP_KEY hashicorp/terraform:light init + +echo "Terraform initialized. Validating exported Datadog files are valid Terraform configs..." +# validate Terraform files in the exports/ directory +docker run --rm -v ${PWD}/exports:/app/exports -w /app/exports -e DATADOG_API_KEY=$DATADOG_API_KEY -e DATADOG_APP_KEY=$DATADOG_APP_KEY hashicorp/terraform:light validate diff --git a/run_tar_exports.sh b/run_tar_exports.sh new file mode 100755 index 0000000..a7ff3a2 --- /dev/null +++ b/run_tar_exports.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +usage() { + echo -e "\nUsage:\n./run_tar_exports.sh [optional filename goes here] \n" +} + +# if -h/--help is supplied, display usage message and exit +if [[ ( $# == "--help") || $# == "-h" ]] +then + usage + exit 1 +fi + +# if ${PWD}/exports doesn't exist we can't tar it, so exit +if [ ! -d "${PWD}/exports" ]; then + echo -e "ERROR: ./exports directory doesn't exist so we can't tar it, exiting...\n" + exit 2 +fi + +# set tar default filename +TAR_FILENAME="exports.tar.gz" + +# if optional filename argument sent to this script, then use it as tar filename +if [ -n "${1}" ]; then + TAR_FILENAME="${1}" +fi + +# ensure any prior created $TAR_FILENAME is removed +rm -f "${PWD}/exports/${TAR_FILENAME}" "${PWD}/exports/exports.tar.gz" + +# touch $TAR_FILENAME to avoid "tar file changed as we read it" warning +touch "${PWD}/exports/${TAR_FILENAME}" + +echo "Terraform validated. Creating tar archive of exported Datadog Terraform files..." +# tar the exports/ directory +docker run --rm -v ${PWD}/exports:/app/exports -w /app/exports debian:stable tar -C /app/exports --exclude="${TAR_FILENAME}" --exclude=./.* -czvf "./${TAR_FILENAME}" . diff --git a/tmpl/monitor.tmpl b/tmpl/monitor.tmpl index c93739e..b9098bc 100644 --- a/tmpl/monitor.tmpl +++ b/tmpl/monitor.tmpl @@ -1,4 +1,4 @@ -resource "datadog_monitor" "{{ .Id }}" { +resource "datadog_monitor" "dd_{{ .Id }}" { name = "{{ .Name }}" type = "{{ .Type }}" {{- if .Tags }} @@ -27,7 +27,7 @@ EOT timeout_h = {{ .TimeoutH }} {{- end }} {{- if .IncludeTags }} - include_tags = {{ .IncludeTags }} + include_tags = {{ .IncludeTags }} {{- end }} {{- if .RequireFullWindow }} require_full_window = {{ .RequireFullWindow }} @@ -41,7 +41,7 @@ EOT {{- if .Thresholds }} {{- with .Thresholds }} - thresholds { + thresholds = { {{- if .Ok }} ok = {{ .Ok }} {{- end }} diff --git a/tmpl/screenboard.tmpl b/tmpl/screenboard.tmpl index 59b2f43..46e2ee3 100644 --- a/tmpl/screenboard.tmpl +++ b/tmpl/screenboard.tmpl @@ -1,4 +1,4 @@ -resource "datadog_screenboard" "{{ .Id }}" { +resource "datadog_screenboard" "dd_{{ .Id }}" { title = "{{ .Title }}" {{- if .ReadOnly }} read_only = {{ .ReadOnly }} @@ -9,10 +9,10 @@ resource "datadog_screenboard" "{{ .Id }}" { {{- if .TemplateVariables }} {{- with .TemplateVariables }} {{- range . }} - template_variable { + template_variable = { name = "{{ .Name }}" prefix = "{{ .Prefix }}" - default = "{{ .Default }}" + default = "{{ .Default }}" } {{- end }} {{- end }} @@ -20,7 +20,7 @@ resource "datadog_screenboard" "{{ .Id }}" { {{- with .Widgets }} {{- range . }} - widget { + widget = { type = "{{ .Type }}" x = "{{ .X }}" y = "{{ .Y }}" @@ -47,14 +47,14 @@ resource "datadog_screenboard" "{{ .Id }}" { {{- end }} {{- if .Time }} {{- with .Time }} - time { + time = { live_span = "{{ .LiveSpan }}" } {{- end }} {{- end }} {{- if .TileDef }} {{- with .TileDef }} - tile_def { + tile_def = { {{- if .Viz }} viz = "{{ .Viz }}" {{- end }} @@ -70,7 +70,7 @@ resource "datadog_screenboard" "{{ .Id }}" { {{- if .Requests }} {{- with .Requests }} {{- range . }} - request { + request = { {{- if .Query }} q = "{{ .Query }}" {{- end }} @@ -113,7 +113,7 @@ resource "datadog_screenboard" "{{ .Id }}" { {{- if .ConditionalFormats }} {{- with .ConditionalFormats -}} {{- range . }} - conditional_format { + conditional_format = { {{- if .Color }} color = "{{ .Color }}" {{- end }} @@ -133,12 +133,12 @@ resource "datadog_screenboard" "{{ .Id }}" { image_url = "{{ .ImageURL }}" {{- end }} } - {{- end }} + {{- end }} {{- end }} {{- end }} {{- if .Style }} {{- with .Style }} - style { + style = { {{- if .Palette }} palette = "{{ .Palette }}" {{- end }} @@ -161,7 +161,7 @@ resource "datadog_screenboard" "{{ .Id }}" { {{- if .Events -}} {{- with .Events -}} {{- range . }} - event { + event = { q = "{{ .Query }}" } //event {{- end -}} @@ -171,7 +171,7 @@ resource "datadog_screenboard" "{{ .Id }}" { {{- if .Markers }} {{- with .Markers }} {{- range . }} - marker { + marker = { {{- if .Type }} type = "{{ .Type }}" {{- end }} @@ -310,7 +310,7 @@ resource "datadog_screenboard" "{{ .Id }}" { {{- end }} {{- if .Params }} {{- with .Params }} - params { + params = { {{- if .Sort }} sort = "{{ .Sort }}" {{- end }} diff --git a/tmpl/timeboard.tmpl b/tmpl/timeboard.tmpl index baa2b6d..adf3b18 100644 --- a/tmpl/timeboard.tmpl +++ b/tmpl/timeboard.tmpl @@ -1,10 +1,10 @@ -resource "datadog_timeboard" "{{ .Id }}" { +resource "datadog_timeboard" "dd_{{ .Id }}" { title = "{{ .Title }}" description = "{{ .Description }}" read_only = {{ .ReadOnly -}} {{with .Graphs -}} {{range . }} - graph { + graph = { title = "{{ .Title }}" {{with .Definition -}} viz = "{{ .Viz -}}" @@ -13,30 +13,30 @@ resource "datadog_timeboard" "{{ .Id }}" { {{- if .Groups -}}group = [{{$groups := .Groups}}{{range $index, $elem := .Groups}}{{if $index}},{{end}}"{{$elem}}"{{end}}]{{- end}} {{- if .Scopes -}}scope = [{{$scopes := .Scopes}}{{range $index, $elem := .Scopes}}{{if $index}},{{end}}"{{$elem}}"{{end}}]{{- end}} {{- if .Style -}} - style { + style = { palette = "{{ .Style.Palette }}" palette_flip = "{{ .Style.PaletteFlip }}" }{{- end}} {{- if .Markers -}}{{- with .Markers -}}{{range .}} - marker { + marker = { type = "{{ .Type }}" value = "{{ .Value }}" {{ if .Label -}}label = "{{ .Label }}"{{- end}} }{{- end}}{{- end}}{{- end}} {{- with .Requests -}}{{range .}} - request { + request = { q = "{{ .Query -}}" {{ if .Type -}}type = "{{ .Type }}"{{ end }} {{ if .Aggregator -}}{{ if ne "" (DeRefString .Aggregator) -}}aggregator = "{{ .Aggregator }}"{{ end}}{{ end -}} {{ if .Style -}}{{ with .Style }} - style { + style = { {{if .Palette -}}palette = "{{ .Palette }}"{{- end}} {{if .Width -}}width = "{{ .Width }}"{{- end}} {{if .Type -}}type = "{{ .Type }}"{{- end}} } {{ end -}}{{ end}} {{- if .ConditionalFormats }}{{- with .ConditionalFormats }}{{- range . -}} - conditional_format { + conditional_format = { {{ if .Palette }}palette = "{{ .Palette }}"{{ end }} comparator = "{{ .Comparator }}" {{- if .Value }}value = "{{ .Value }}"{{ end }} @@ -51,7 +51,7 @@ resource "datadog_timeboard" "{{ .Id }}" { }{{end}}{{- end}} {{- with .TemplateVariables -}}{{- range . }} - template_variable { + template_variable = { name = "{{ .Name }}" prefix = "{{ .Prefix }}" default = "{{ .Default }}"