diff --git a/cmd/tarmak/cmd/cluster.go b/cmd/tarmak/cmd/cluster.go index 6e5b1b89d2..b04a0be460 100644 --- a/cmd/tarmak/cmd/cluster.go +++ b/cmd/tarmak/cmd/cluster.go @@ -34,6 +34,15 @@ func clusterApplyFlags(fs *flag.FlagSet) { false, "apply changes to infrastructure only, by running only terraform", ) + + fs.BoolVarP( + &store.SpotPricing, + "spot-pricing", + "P", + false, + "Use a best effort spot pricing based on last 3 days pricing for instance pools", + ) + } func clusterDestroyFlags(fs *flag.FlagSet) { diff --git a/docs/user-guide.rst b/docs/user-guide.rst index 7072663066..7bf1439ccb 100644 --- a/docs/user-guide.rst +++ b/docs/user-guide.rst @@ -700,6 +700,39 @@ policy permissions. `Information on how to correctly set these permissions can be found here `_. +Spot Instances +~~~~~~~~~~~~~~ +Tarmak gives the ability to attempt to request spot instances for use in +instance pools. Spot instances are very cheap, spare AWS instances that can be +revoked by AWS at any time, given a 2 minute notification. `More information +here `_. + +Spot instances can be requested cluster-wide by giving the ``--spot-pricing`` +flag to ``cluster apply``. Tarmak will then attempt a best effort spot price for +each instance pool in the cluster, calculated as the average spot price in the +last 3 days for that instance type in each zone plus 25%. + +Manual spot prices can be applied to each instance pool within the +``tarmak.yaml`` which will override the Tarmak best effort for that instance +pool. This is done through the ``spotPrice`` attribute under the instance pool, +given as a number in USD. This can be added like so: + +.. code-block:: yaml + + - image: centos-puppet-agent + maxCount: 3 + metadata: + creationTimestamp: "2018-07-27T09:33:15Z" + name: worker + minCount: 3 + size: medium + spotPrice: 0.015 + + +Note that Tarmak will only attempt to create spot instances for instance pools +with the ``spotPrice`` attribute or spot pricing flag during a cluster apply. + + Cluster Services ---------------- @@ -721,3 +754,4 @@ Do the following steps to access Grafana: .. code-block:: none http://127.0.0.1:8001/api/v1/namespaces/kube-system/services/monitoring-grafana/proxy/ + diff --git a/pkg/apis/tarmak/v1alpha1/types.go b/pkg/apis/tarmak/v1alpha1/types.go index a235ed67d0..7e810f58fa 100644 --- a/pkg/apis/tarmak/v1alpha1/types.go +++ b/pkg/apis/tarmak/v1alpha1/types.go @@ -143,6 +143,8 @@ type ClusterApplyFlags struct { InfrastructureOnly bool // only run terraform ConfigurationOnly bool // only run puppet + + SpotPricing bool // Use spot pricing for all applied instances at best effort according to last 3 days pricing. Spot prices in config will override this option per instance pool. } // Contains the cluster destroy flags diff --git a/pkg/tarmak/instance_pool/instance_pool.go b/pkg/tarmak/instance_pool/instance_pool.go index 5a47c547dc..18d92e35bd 100644 --- a/pkg/tarmak/instance_pool/instance_pool.go +++ b/pkg/tarmak/instance_pool/instance_pool.go @@ -33,6 +33,7 @@ type InstancePool struct { rootVolume *Volume instanceType string + spotPrice string role *role.Role } @@ -77,7 +78,7 @@ func NewFromConfig(cluster interfaces.Cluster, conf *clusterv1alpha1.InstancePoo return nil, fmt.Errorf("minCount does not equal maxCount but role is stateful. minCount=%d maxCount=%d", instancePool.Config().MinCount, instancePool.Config().MaxCount) } - var result error + var result *multierror.Error count := 0 for pos, _ := range conf.Volumes { @@ -98,7 +99,9 @@ func NewFromConfig(cluster interfaces.Cluster, conf *clusterv1alpha1.InstancePoo return nil, errors.New("no root volume given") } - return instancePool, result + instancePool.spotPrice = conf.SpotPrice + + return instancePool, result.ErrorOrNil() } func (n *InstancePool) Role() *role.Role { @@ -166,7 +169,20 @@ func (n *InstancePool) InstanceType() string { } func (n *InstancePool) SpotPrice() string { - return n.conf.SpotPrice + return n.spotPrice +} + +func (n *InstancePool) CalculateSpotPrice() error { + if n.spotPrice == "" { + p, err := n.cluster.Environment().Provider().SpotPrice(n) + if err != nil { + return err + } + + n.spotPrice = fmt.Sprint(p) + } + + return nil } func (n *InstancePool) AmazonAdditionalIAMPolicies() string { diff --git a/pkg/tarmak/interfaces/interfaces.go b/pkg/tarmak/interfaces/interfaces.go index c99c20602a..cb49ab6867 100644 --- a/pkg/tarmak/interfaces/interfaces.go +++ b/pkg/tarmak/interfaces/interfaces.go @@ -127,6 +127,7 @@ type Provider interface { AskInstancePoolZones(Initialize) (zones []string, err error) UploadConfiguration(Cluster, io.ReadSeeker) error VerifyInstanceTypes(intstancePools []InstancePool) error + SpotPrice(instancePool InstancePool) (float64, error) } type Tarmak interface { @@ -254,6 +255,7 @@ type InstancePool interface { InstanceType() string Labels() (string, error) Taints() (string, error) + CalculateSpotPrice() error } type Volume interface { diff --git a/pkg/tarmak/provider/amazon/spot_price.go b/pkg/tarmak/provider/amazon/spot_price.go new file mode 100644 index 0000000000..13878c8ed7 --- /dev/null +++ b/pkg/tarmak/provider/amazon/spot_price.go @@ -0,0 +1,74 @@ +// Copyright Jetstack Ltd. See LICENSE for details. +package amazon + +import ( + "fmt" + "strconv" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/hashicorp/go-multierror" + + "github.com/jetstack/tarmak/pkg/tarmak/interfaces" +) + +func (a *Amazon) SpotPrice(instancePool interfaces.InstancePool) (float64, error) { + sess, err := a.Session() + if err != nil { + return 0, err + } + + svc := ec2.New(sess) + + instanceType, err := a.InstanceType(instancePool.Config().Size) + if err != nil { + return 0, err + } + + timeToColect := time.Now().Add(-24 * 3 * time.Hour) + + var result *multierror.Error + var prices []float64 + for _, zone := range instancePool.Zones() { + spotPriceRequestInput := &ec2.DescribeSpotPriceHistoryInput{ + InstanceTypes: []*string{ + aws.String(instanceType), + }, + ProductDescriptions: []*string{ + aws.String("Linux/UNIX"), + }, + AvailabilityZone: aws.String(zone), + StartTime: &timeToColect, + } + + output, err := svc.DescribeSpotPriceHistory(spotPriceRequestInput) + if err != nil { + result = multierror.Append(result, fmt.Errorf("failed to get current spot price of instance type '%s': %v", instanceType, err)) + continue + } + + for _, entry := range output.SpotPriceHistory { + price, err := strconv.ParseFloat(*entry.SpotPrice, 64) + if err != nil { + result = multierror.Append(result, fmt.Errorf("failed to parse spot price '%s': %v", *entry.SpotPrice, err)) + continue + } + + prices = append(prices, price) + } + } + + var total float64 + for _, price := range prices { + total += price + } + + if len(prices) != 0 { + total /= float64(len(prices)) + } + + total *= 1.25 + + return total, result.ErrorOrNil() +} diff --git a/pkg/tarmak/terraform.go b/pkg/tarmak/terraform.go index 11586ddd89..3f9a0f5d4f 100644 --- a/pkg/tarmak/terraform.go +++ b/pkg/tarmak/terraform.go @@ -12,7 +12,9 @@ import ( "strings" "github.com/blang/semver" + "github.com/hashicorp/go-multierror" terraformVersion "github.com/hashicorp/terraform/version" + "github.com/jetstack/tarmak/pkg/tarmak/interfaces" ) @@ -56,6 +58,21 @@ func (t *Tarmak) CmdTerraformApply(args []string, ctx context.Context) error { return err } + if t.flags.Cluster.Apply.SpotPricing { + t.cluster.Log().Info("calculating instance pool spot prices") + + var result *multierror.Error + for _, i := range t.Cluster().InstancePools() { + if err := i.CalculateSpotPrice(); err != nil { + result = multierror.Append(result, err) + } + } + + if result != nil { + return result.ErrorOrNil() + } + } + t.cluster.Log().Info("write SSH config") if err := t.writeSSHConfigForClusterHosts(); err != nil { return err