Disable orka steps - to be reverted

Author: mrodm

.buildkite/pipeline.yml

@@ -54,14 +54,14 @@ steps:
         artifact_paths:
           - "TEST-unit-windows.xml"
 
-      - label: ":macos: Run unit tests"
-        key: unit-tests-mac-arm
-        command: ".buildkite/scripts/unit_tests_macos_arm.sh"
-        agents:
-          provider: "orka"
-          imagePrefix: "${MACOS_ARM_AGENT_IMAGE}"
-        artifact_paths:
-          - "build/test-results/*.xml"
+      # - label: ":macos: Run unit tests"
+      #   key: unit-tests-mac-arm
+      #   command: ".buildkite/scripts/unit_tests_macos_arm.sh"
+      #   agents:
+      #     provider: "orka"
+      #     imagePrefix: "${MACOS_ARM_AGENT_IMAGE}"
+      #   artifact_paths:
+      #     - "build/test-results/*.xml"
 
   - wait: ~
     continue_on_failure: true

internal/benchrunner/runners/system/runner.go

@@ -32,6 +32,7 @@ import (
 	"github.com/elastic/elastic-package/internal/multierror"
 	"github.com/elastic/elastic-package/internal/packages"
 	"github.com/elastic/elastic-package/internal/servicedeployer"
+	"github.com/elastic/elastic-package/internal/testrunner/runners/system"
 	"github.com/elastic/elastic-package/internal/wait"
 )
 
@@ -169,14 +170,6 @@ func (r *runner) setUp(ctx context.Context) error {
 		return fmt.Errorf("reading package manifest failed: %w", err)
 	}
 
-	policy, err := r.createBenchmarkPolicy(ctx, pkgManifest)
-	if err != nil {
-		return err
-	}
-	r.benchPolicy = policy
-
-	// Delete old data
-	logger.Debug("deleting old data in data stream...")
 	dataStreamManifest, err := packages.ReadDataStreamManifest(
 		filepath.Join(
 			common.DataStreamPath(r.options.PackageRootPath, r.scenario.DataStream.Name),
@@ -187,6 +180,12 @@ func (r *runner) setUp(ctx context.Context) error {
 		return fmt.Errorf("reading data stream manifest failed: %w", err)
 	}
 
+	policy, err := r.createBenchmarkPolicy(ctx, pkgManifest, dataStreamManifest)
+	if err != nil {
+		return err
+	}
+	r.benchPolicy = policy
+
 	r.runtimeDataStream = fmt.Sprintf(
 		"%s-%s.%s-%s",
 		dataStreamManifest.Type,
@@ -210,6 +209,7 @@ func (r *runner) setUp(ctx context.Context) error {
 		return nil
 	}
 
+	logger.Debug("deleting old data in data stream...")
 	if err := r.deleteDataStreamDocs(ctx, r.runtimeDataStream); err != nil {
 		return fmt.Errorf("error deleting old data in data stream: %s: %w", r.runtimeDataStream, err)
 	}
@@ -234,6 +234,7 @@ func (r *runner) run(ctx context.Context) (report reporters.Reportable, err erro
 		s, err := r.setupService(ctx)
 		if errors.Is(err, os.ErrNotExist) {
 			logger.Debugf("No service deployer defined for this benchmark")
+			logger.Debugf("The error is: %s", err.Error())
 		} else if err != nil {
 			return nil, err
 		}
@@ -292,6 +293,7 @@ func (r *runner) setupService(ctx context.Context) (servicedeployer.DeployedServ
 	// Setup service.
 	logger.Debug("Setting up service...")
 	devDeployDir := filepath.Clean(filepath.Join(r.options.BenchPath, "deploy"))
+	logger.Debugf("Using service deployer dev directory: %s", devDeployDir)
 	opts := servicedeployer.FactoryOptions{
 		PackageRootPath:        r.options.PackageRootPath,
 		DevDeployDir:           devDeployDir,
@@ -367,7 +369,7 @@ func (r *runner) deleteDataStreamDocs(ctx context.Context, dataStream string) er
 	return nil
 }
 
-func (r *runner) createBenchmarkPolicy(ctx context.Context, pkgManifest *packages.PackageManifest) (*kibana.Policy, error) {
+func (r *runner) createBenchmarkPolicy(ctx context.Context, pkgManifest *packages.PackageManifest, dataStreamManifest *packages.DataStreamManifest) (*kibana.Policy, error) {
 	// Configure package (single data stream) via Ingest Manager APIs.
 	logger.Debug("creating benchmark policy...")
 	benchTime := time.Now().Format("20060102T15:04:05Z")
@@ -385,21 +387,53 @@ func (r *runner) createBenchmarkPolicy(ctx context.Context, pkgManifest *package
 
 	policy, err := r.options.KibanaClient.CreatePolicy(ctx, p)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to create benchmark policy: %w", err)
 	}
 
-	packagePolicy, err := r.createPackagePolicy(ctx, pkgManifest, policy)
+	if r.scenario.PolicyTemplate == "" {
+		policyTemplateName, err := system.FindPolicyTemplateForInput(pkgManifest, dataStreamManifest, r.scenario.Input)
+		if err != nil {
+			return nil, fmt.Errorf("failed to determine the associated policy_template: %w", err)
+		}
+		r.scenario.PolicyTemplate = policyTemplateName
+	}
+	policyTemplate, err := system.SelectPolicyTemplateByName(pkgManifest.PolicyTemplates, r.scenario.PolicyTemplate)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to find the selected policy_template: %w", err)
+	}
+
+	logger.Debug("adding package data stream to test policy...")
+	ds, err := system.CreatePackageDatastream(
+		policy,
+		pkgManifest,
+		policyTemplate,
+		dataStreamManifest,
+		r.scenario.Input,
+		r.scenario.Vars,
+		r.scenario.DataStream.Vars,
+		policy.Namespace)
+	if err != nil {
+		return nil, fmt.Errorf("could not create package data stream: %w", err)
+	}
+
+	if err := r.options.KibanaClient.AddPackageDataStreamToPolicy(ctx, ds); err != nil {
+		return nil, fmt.Errorf("could not add data stream config to policy: %w", err)
 	}
+	logger.Debugf("added data stream %s to policy", ds.Name)
+
+	// packagePolicy, err := r.createPackagePolicy(ctx, pkgManifest, policy)
+	// if err != nil {
+	// 	return nil, fmt.Errorf("failed to create package policy: %w", err)
+	// }
+	// logger.Debugf("created package policy with id: %s", packagePolicy.ID)
 
 	r.deletePolicyHandler = func(ctx context.Context) error {
 		var merr multierror.Error
 
-		logger.Debug("deleting benchmark package policy...")
-		if err := r.options.KibanaClient.DeletePackagePolicy(ctx, *packagePolicy); err != nil {
-			merr = append(merr, fmt.Errorf("error cleaning up benchmark package policy: %w", err))
-		}
+		// logger.Debug("deleting benchmark package policy...")
+		// if err := r.options.KibanaClient.DeletePackagePolicy(ctx, *packagePolicy); err != nil {
+		// 	merr = append(merr, fmt.Errorf("error cleaning up benchmark package policy: %w", err))
+		// }
 
 		logger.Debug("deleting benchmark policy...")
 		if err := r.options.KibanaClient.DeletePolicy(ctx, policy.ID); err != nil {