From f3642895414ae049c3b56a71ae6dd157111ba6c7 Mon Sep 17 00:00:00 2001 From: Zeev Glaizer Date: Tue, 10 Apr 2018 11:26:24 +0300 Subject: [PATCH 1/6] setup --- conf/local.conf.js | 5 +++-- specs/local.js | 5 +++++ specs/single.js | 6 +++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/conf/local.conf.js b/conf/local.conf.js index 68def2c..64ff06b 100644 --- a/conf/local.conf.js +++ b/conf/local.conf.js @@ -1,5 +1,6 @@ var browserstack = require('browserstack-local'); - +console.log('process.env.BROWSERSTACK_USERNAME', process.env.BROWSERSTACK_USERNAME); +console.log('process.env.BROWSERSTACK_ACCESS_KEY', process.env.BROWSERSTACK_ACCESS_KEY); exports.config = { 'specs': [ '../specs/local.js' ], 'seleniumAddress': 'http://hub-cloud.browserstack.com/wd/hub', @@ -19,7 +20,7 @@ exports.config = { console.log("Connecting local"); return new Promise(function(resolve, reject){ exports.bs_local = new browserstack.Local(); - exports.bs_local.start({'key': exports.config.capabilities['browserstack.key'] }, function(error) { + exports.bs_local.start({'key': exports.config.capabilities['browserstack.key'], force: true}, function (error) { if (error) return reject(error); console.log('Connected. Now testing...'); diff --git a/specs/local.js b/specs/local.js index feed8be..8108135 100644 --- a/specs/local.js +++ b/specs/local.js @@ -2,7 +2,12 @@ describe('BrowserStack Local Testing', function() { it('can check tunnel working', function() { browser.driver.get('http://bs-local.com:45691/check').then(function() { expect(browser.driver.getPageSource()).toMatch(/Up and running/i); + expect(true).toBe(false); }); }); + + it('should fail and report to browserstack', function() { + expect(true).toBe(false); + }); }); diff --git a/specs/single.js b/specs/single.js index 1d8a1fc..601c062 100644 --- a/specs/single.js +++ b/specs/single.js @@ -6,11 +6,15 @@ describe('Google\'s Search Functionality', function() { browser.driver.wait(function() { return browser.driver.findElements(by.id('resultStats')).then(function(elems) { return elems.length > 0; - });; + }); }); expect(browser.driver.getTitle()).toEqual('BrowserStack - Google Search'); }); }); }); }); + + it('should fail and report to browserstack', function() { + expect(true).toBe(false); + }); }); From e38e3e6b4c047a2c90d821bbeb0cdf4a52b1f721 Mon Sep 17 00:00:00 2001 From: Zeev Glaizer Date: Thu, 19 Apr 2018 00:14:35 +0300 Subject: [PATCH 2/6] upgrade to protractor 5. report failing tests to browser stack --- conf/local.conf.js | 10 ++++------ conf/local_failed.conf.js | 36 ++++++++++++++++++++++++++++++++++++ conf/parallel.conf.js | 6 +++--- conf/parallel_local.conf.js | 8 ++++---- conf/single.conf.js | 8 ++++---- conf/single_fail.conf.js | 14 ++++++++++++++ package.json | 4 +++- specs/local.js | 5 ----- specs/local_fail.js | 6 ++++++ specs/single.js | 6 +----- specs/single_fail.js | 5 +++++ 11 files changed, 80 insertions(+), 28 deletions(-) create mode 100644 conf/local_failed.conf.js create mode 100644 conf/single_fail.conf.js create mode 100644 specs/local_fail.js create mode 100644 specs/single_fail.js diff --git a/conf/local.conf.js b/conf/local.conf.js index 64ff06b..e649ab5 100644 --- a/conf/local.conf.js +++ b/conf/local.conf.js @@ -1,13 +1,11 @@ var browserstack = require('browserstack-local'); -console.log('process.env.BROWSERSTACK_USERNAME', process.env.BROWSERSTACK_USERNAME); -console.log('process.env.BROWSERSTACK_ACCESS_KEY', process.env.BROWSERSTACK_ACCESS_KEY); exports.config = { 'specs': [ '../specs/local.js' ], - 'seleniumAddress': 'http://hub-cloud.browserstack.com/wd/hub', + + browserstackUser: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME', + browserstackKey: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY', 'capabilities': { - 'browserstack.user': process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME', - 'browserstack.key': process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY', 'build': 'protractor-browserstack', 'name': 'local_test', 'browserName': 'chrome', @@ -20,7 +18,7 @@ exports.config = { console.log("Connecting local"); return new Promise(function(resolve, reject){ exports.bs_local = new browserstack.Local(); - exports.bs_local.start({'key': exports.config.capabilities['browserstack.key'], force: true}, function (error) { + exports.bs_local.start({'key': exports.config.browserstackKey, force: true}, function (error) { if (error) return reject(error); console.log('Connected. Now testing...'); diff --git a/conf/local_failed.conf.js b/conf/local_failed.conf.js new file mode 100644 index 0000000..a6c9624 --- /dev/null +++ b/conf/local_failed.conf.js @@ -0,0 +1,36 @@ +var browserstack = require('browserstack-local'); +exports.config = { + 'specs': [ '../specs/local_fail.js' ], + + browserstackUser: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME', + browserstackKey: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY', + + 'capabilities': { + 'build': 'protractor-browserstack', + 'name': 'local test reporting failed', + 'browserName': 'chrome', + 'browserstack.local': true, + 'browserstack.debug': 'true' + }, + + // Code to start browserstack local before start of test + beforeLaunch: function(){ + console.log("Connecting local"); + return new Promise(function(resolve, reject){ + exports.bs_local = new browserstack.Local(); + exports.bs_local.start({'key': exports.config.browserstackKey, force: true}, function (error) { + if (error) return reject(error); + console.log('Connected. Now testing...'); + + resolve(); + }); + }); + }, + + // Code to stop browserstack local after end of test + afterLaunch: function(){ + return new Promise(function(resolve, reject){ + exports.bs_local.stop(resolve); + }); + } +}; diff --git a/conf/parallel.conf.js b/conf/parallel.conf.js index f265b17..572830f 100644 --- a/conf/parallel.conf.js +++ b/conf/parallel.conf.js @@ -1,10 +1,10 @@ exports.config = { 'specs': [ '../specs/single.js' ], - 'seleniumAddress': 'http://hub-cloud.browserstack.com/wd/hub', + + browserstackUser: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME', + browserstackKey: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY', 'commonCapabilities': { - 'browserstack.user': process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME', - 'browserstack.key': process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY', 'build': 'protractor-browserstack', 'name': 'parallel_test', 'browserstack.debug': 'true', diff --git a/conf/parallel_local.conf.js b/conf/parallel_local.conf.js index 466734d..7feff3b 100644 --- a/conf/parallel_local.conf.js +++ b/conf/parallel_local.conf.js @@ -2,11 +2,11 @@ var browserstack = require('browserstack-local'); exports.config = { 'specs': [ '../specs/local.js' ], - 'seleniumAddress': 'http://hub-cloud.browserstack.com/wd/hub', + + browserstackUser: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME', + browserstackKey: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY', 'commonCapabilities': { - 'browserstack.user': process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME', - 'browserstack.key': process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY', 'build': 'protractor-browserstack', 'name': 'parallel_local_test', 'browserstack.local': true, @@ -26,7 +26,7 @@ exports.config = { console.log("Connecting local"); return new Promise(function(resolve, reject){ exports.bs_local = new browserstack.Local(); - exports.bs_local.start({'key': exports.config.commonCapabilities['browserstack.key'] }, function(error) { + exports.bs_local.start({'key': exports.config.browserstackKey }, function(error) { if (error) return reject(error); console.log('Connected. Now testing...'); diff --git a/conf/single.conf.js b/conf/single.conf.js index e0201c4..f609ab2 100644 --- a/conf/single.conf.js +++ b/conf/single.conf.js @@ -1,12 +1,12 @@ exports.config = { 'specs': [ '../specs/single.js' ], - 'seleniumAddress': 'http://hub-cloud.browserstack.com/wd/hub', + + browserstackUser: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME', + browserstackKey: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY', 'capabilities': { - 'browserstack.user': process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME', - 'browserstack.key': process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY', 'build': 'protractor-browserstack', - 'name': 'single_test', + 'name': 'single test', 'browserName': 'chrome', 'resolution': '1024x768', 'browserstack.debug': 'true' diff --git a/conf/single_fail.conf.js b/conf/single_fail.conf.js new file mode 100644 index 0000000..088477c --- /dev/null +++ b/conf/single_fail.conf.js @@ -0,0 +1,14 @@ +exports.config = { + 'specs': [ '../specs/single_fail.js' ], + + browserstackUser: process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME', + browserstackKey: process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY', + + 'capabilities': { + 'build': 'protractor-browserstack', + 'name': 'single test reporting failed', + 'browserName': 'chrome', + 'resolution': '1024x768', + 'browserstack.debug': 'true' + } +}; diff --git a/package.json b/package.json index 9f324cd..00574bc 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,9 @@ "scripts": { "test": "npm run single && npm run local && npm run parallel", "single": "./node_modules/.bin/protractor conf/single.conf.js", + "single_fail": "./node_modules/.bin/protractor conf/single_fail.conf.js", "local": "./node_modules/.bin/protractor conf/local.conf.js", + "local_fail": "./node_modules/.bin/protractor conf/local_fail.conf.js", "parallel": "./node_modules/.bin/protractor conf/parallel.conf.js", "parallel_local": "./node_modules/.bin/protractor conf/parallel_local.conf.js" }, @@ -16,7 +18,7 @@ }, "dependencies": { "browserstack-local": "^1.0.0", - "protractor": "^2.5.1" + "protractor": "^5.3.1" }, "license": "MIT" } diff --git a/specs/local.js b/specs/local.js index 8108135..feed8be 100644 --- a/specs/local.js +++ b/specs/local.js @@ -2,12 +2,7 @@ describe('BrowserStack Local Testing', function() { it('can check tunnel working', function() { browser.driver.get('http://bs-local.com:45691/check').then(function() { expect(browser.driver.getPageSource()).toMatch(/Up and running/i); - expect(true).toBe(false); }); }); - - it('should fail and report to browserstack', function() { - expect(true).toBe(false); - }); }); diff --git a/specs/local_fail.js b/specs/local_fail.js new file mode 100644 index 0000000..388ae8f --- /dev/null +++ b/specs/local_fail.js @@ -0,0 +1,6 @@ +describe('BrowserStack Local Testing', function() { + it('should fail and report to browserstack', function() { + expect(true).toBe(false); + }); +}); + diff --git a/specs/single.js b/specs/single.js index 601c062..65e942a 100644 --- a/specs/single.js +++ b/specs/single.js @@ -2,7 +2,7 @@ describe('Google\'s Search Functionality', function() { it('can find search results', function() { browser.driver.get('https://google.com/ncr').then(function() { browser.driver.findElement(by.name('q')).sendKeys('BrowserStack').then(function() { - browser.driver.findElement(by.name('btnG')).click().then(function() { + browser.actions().sendKeys(protractor.Key.ENTER).perform().then(function() { browser.driver.wait(function() { return browser.driver.findElements(by.id('resultStats')).then(function(elems) { return elems.length > 0; @@ -13,8 +13,4 @@ describe('Google\'s Search Functionality', function() { }); }); }); - - it('should fail and report to browserstack', function() { - expect(true).toBe(false); - }); }); diff --git a/specs/single_fail.js b/specs/single_fail.js new file mode 100644 index 0000000..2e92a45 --- /dev/null +++ b/specs/single_fail.js @@ -0,0 +1,5 @@ +describe('Reporting to BrowserStack that test failed', function() { + it('should fail and report to browserstack', function() { + expect(true).toBe(false); + }); +}); From 5154f1f99074071c89a67ee54db7bd8d40be422b Mon Sep 17 00:00:00 2001 From: Zeev Glaizer Date: Thu, 19 Apr 2018 01:32:34 +0300 Subject: [PATCH 3/6] reporting failed tests with reason to browser stack --- lib/BrowserstackErrorLogger.js | 34 ++++++++++++++++++++++++++++++ specs/single_fail.js | 38 ++++++++++++++++++++++++++++++++-- 2 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 lib/BrowserstackErrorLogger.js diff --git a/lib/BrowserstackErrorLogger.js b/lib/BrowserstackErrorLogger.js new file mode 100644 index 0000000..e320c1a --- /dev/null +++ b/lib/BrowserstackErrorLogger.js @@ -0,0 +1,34 @@ +const request = require('request'); +class BrowserstackErrorLogger { + constructor() { + this.BROWSERSTACK_USERNAME = process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME'; + this.BROWSERSTACK_ACCESS_KEY = process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY'; + this.errors = []; + this.sessionId = ""; + } + + buildErrorsRrport() { + return JSON.stringify(this.errors); + } + + setSessionId(sessionId) { + this.sessionId = sessionId; + } + + addError(title, error) { + // this.errors.push({title, error}); + this.errors.push(title); + } + + async reportErrors() { + if (this.errors.length) { + await request({ + uri: `https://${this.BROWSERSTACK_USERNAME}:${this.BROWSERSTACK_ACCESS_KEY}@api.browserstack.com/automate/sessions/${this.sessionId}.json`, + method: 'PUT', + form: {status: 'failed', reason: this.buildErrorsRrport()} + }); + } + } +} + +module.exports = BrowserstackErrorLogger; \ No newline at end of file diff --git a/specs/single_fail.js b/specs/single_fail.js index 2e92a45..c825ad8 100644 --- a/specs/single_fail.js +++ b/specs/single_fail.js @@ -1,5 +1,39 @@ -describe('Reporting to BrowserStack that test failed', function() { - it('should fail and report to browserstack', function() { +var BrowserstackErrorLogger = require('../lib/BrowserstackErrorLogger'); + +jasmine.getEnv().addReporter({ + specStarted(result) { + jasmine.getEnv().currentSpec = result; + }, + specDone() { + jasmine.getEnv().currentSpec = null; + } +}); + +describe('Reporting to BrowserStack that test failed with reason', function() { + + const reporter = new BrowserstackErrorLogger(); + beforeAll(async () => { + await protractor.browser.driver.getSession().then((session) => { + reporter.setSessionId(session['id_']); + }); + }); + + afterEach(function () { + let failedExpectations = jasmine.getEnv().currentSpec.failedExpectations; + if (failedExpectations.length) { + reporter.addError(failedExpectations[0].message, failedExpectations[0].stack) + } + }); + + afterAll(async () => { + return reporter.reportErrors(); + }); + + it('should fail and report reason to browserstack', function() { expect(true).toBe(false); }); + + it('should not fail and will not be reported to browserstack', function() { + expect(true).toBe(true); + }); }); From 42b980bbf50ad8a6ad6d258d9ec3b90711e53fbb Mon Sep 17 00:00:00 2001 From: Zeev Glaizer Date: Thu, 19 Apr 2018 01:39:29 +0300 Subject: [PATCH 4/6] reporting failed tests with reason to browser stack --- ...erstackErrorLogger.js => BrowserstackErrorReporter.js} | 8 ++++---- specs/single_fail.js | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) rename lib/{BrowserstackErrorLogger.js => BrowserstackErrorReporter.js} (82%) diff --git a/lib/BrowserstackErrorLogger.js b/lib/BrowserstackErrorReporter.js similarity index 82% rename from lib/BrowserstackErrorLogger.js rename to lib/BrowserstackErrorReporter.js index e320c1a..535709c 100644 --- a/lib/BrowserstackErrorLogger.js +++ b/lib/BrowserstackErrorReporter.js @@ -1,5 +1,5 @@ const request = require('request'); -class BrowserstackErrorLogger { +class BrowserstackErrorReporter { constructor() { this.BROWSERSTACK_USERNAME = process.env.BROWSERSTACK_USERNAME || 'BROWSERSTACK_USERNAME'; this.BROWSERSTACK_ACCESS_KEY = process.env.BROWSERSTACK_ACCESS_KEY || 'BROWSERSTACK_ACCESS_KEY'; @@ -7,7 +7,7 @@ class BrowserstackErrorLogger { this.sessionId = ""; } - buildErrorsRrport() { + buildErrorsReport() { return JSON.stringify(this.errors); } @@ -25,10 +25,10 @@ class BrowserstackErrorLogger { await request({ uri: `https://${this.BROWSERSTACK_USERNAME}:${this.BROWSERSTACK_ACCESS_KEY}@api.browserstack.com/automate/sessions/${this.sessionId}.json`, method: 'PUT', - form: {status: 'failed', reason: this.buildErrorsRrport()} + form: {status: 'failed', reason: this.buildErrorsReport()} }); } } } -module.exports = BrowserstackErrorLogger; \ No newline at end of file +module.exports = BrowserstackErrorReporter; \ No newline at end of file diff --git a/specs/single_fail.js b/specs/single_fail.js index c825ad8..77932d6 100644 --- a/specs/single_fail.js +++ b/specs/single_fail.js @@ -1,4 +1,4 @@ -var BrowserstackErrorLogger = require('../lib/BrowserstackErrorLogger'); +var BrowserstackErrorReporter = require('../lib/BrowserstackErrorReporter'); jasmine.getEnv().addReporter({ specStarted(result) { @@ -11,7 +11,7 @@ jasmine.getEnv().addReporter({ describe('Reporting to BrowserStack that test failed with reason', function() { - const reporter = new BrowserstackErrorLogger(); + const reporter = new BrowserstackErrorReporter(); beforeAll(async () => { await protractor.browser.driver.getSession().then((session) => { reporter.setSessionId(session['id_']); @@ -21,7 +21,7 @@ describe('Reporting to BrowserStack that test failed with reason', function() { afterEach(function () { let failedExpectations = jasmine.getEnv().currentSpec.failedExpectations; if (failedExpectations.length) { - reporter.addError(failedExpectations[0].message, failedExpectations[0].stack) + failedExpectations.forEach(f => reporter.addError(f.message, f.stack)); } }); From 37d46c059b62718b4990c1b086f7a1bf70bd8623 Mon Sep 17 00:00:00 2001 From: Zeev Glaizer Date: Thu, 19 Apr 2018 02:01:01 +0300 Subject: [PATCH 5/6] readme --- README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4373d02..48779a1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,17 @@ -# protractor-browserstack +# protractor-browserstack with marking tests as passed/failed with reason + +####The incentive behind this fork is to demonstrate how to mark tests as passed/ failed in browserstack.
Browserstack's current documentation does not allow you to do so. So even if your test fails it will be marked as passed (green) on browserstack.

In addition, there is an example of how to update the reason for the test's failure using browserstack rest api. see 'single_failed.js' [Protractor](https://github.com/angular/protractor/) Integration with BrowserStack. +Mark tests as passed/failed in Browserstack + +Report reason for failed test to Browserstack + +Report failed expectations to Browserstack + +Upgrade to support Protractor 5 + ![BrowserStack Logo](https://d98b8t1nnulk5.cloudfront.net/production/images/layout/logo-header.png?1469004780) @@ -14,6 +24,7 @@ ## Running your tests * To run a single test, run `npm run single` +* To run a single test that will report the reason for failed tests to browserstack, run `npm run single_failed` * To run local tests, run `npm run local` * To run parallel tests, run `npm run parallel` From 052689a5c27952db8a50dd94840a8d2feee33633 Mon Sep 17 00:00:00 2001 From: zeevgl Date: Thu, 19 Apr 2018 02:03:07 +0300 Subject: [PATCH 6/6] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 48779a1..cc2c4f1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # protractor-browserstack with marking tests as passed/failed with reason -####The incentive behind this fork is to demonstrate how to mark tests as passed/ failed in browserstack.
Browserstack's current documentation does not allow you to do so. So even if your test fails it will be marked as passed (green) on browserstack.

In addition, there is an example of how to update the reason for the test's failure using browserstack rest api. see 'single_failed.js' +#### The incentive behind this fork is to demonstrate how to mark tests as passed/ failed in browserstack.
Browserstack's current documentation does not allow you to do so. So even if your test fails it will be marked as passed (green) on browserstack.

In addition, there is an example of how to update the reason for the test's failure using browserstack rest api. see 'single_failed.js' [Protractor](https://github.com/angular/protractor/) Integration with BrowserStack.