From c71934839f691f523582320871b3a43cd1eddaf9 Mon Sep 17 00:00:00 2001 From: Liam Murphy Date: Sun, 4 Sep 2022 18:56:27 -0700 Subject: [PATCH 1/5] use Promise.allSettled, writeLocal now returns promises as well --- src/index.js | 11 +++++++---- src/writeHelper.js | 16 ++++++++++------ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/index.js b/src/index.js index 4e90997..b649a1d 100644 --- a/src/index.js +++ b/src/index.js @@ -78,12 +78,15 @@ function saveVehicles() { return writeHelper.writeToS3(s3Bucket, agencyInfo.id, currentDateTime, vehicles); }) - .catch((err) => { - console.log(err); - }); }); - Promise.all(promises); + // Do not reject all promises when one of them is rejected, but do output why one was rejected. + Promise.allSettled(promises) + .then((results) => results.forEach((result) => { + if(result.status == "rejected") { + console.log("received a rejected promise with reason: " + result.reason) + } + })); } function storeLocal() { diff --git a/src/writeHelper.js b/src/writeHelper.js index a6ae846..0d0517c 100644 --- a/src/writeHelper.js +++ b/src/writeHelper.js @@ -17,12 +17,16 @@ function writeLocal(agency, currentDateTime, data) { const version = "v1"; const fileName = `/tmp/${agency}_${version}_${currentTimestamp}.json` - fs.writeFile(fileName, JSON.stringify(data), err => { - if (err) { - console.error(err); - } - console.log("wrote", fileName) - }); + return new Promise((resolve, reject) => { + fs.writeFile(fileName, JSON.stringify(data), err => { + if (err) { + reject(err); + } else { + resolve(); + } + console.log("wrote", fileName) + }); + }) } function writeToS3(s3Bucket, agency, currentDateTime, data) { From 8454e5aee9c0092ab73a429f95a429db82588ab0 Mon Sep 17 00:00:00 2001 From: Liam Murphy Date: Sun, 4 Sep 2022 19:02:57 -0700 Subject: [PATCH 2/5] rename local compose file, add console.log tweak --- local-docker-compose.yml => local-docker-compose.yaml | 1 - src/index.js | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) rename local-docker-compose.yml => local-docker-compose.yaml (96%) diff --git a/local-docker-compose.yml b/local-docker-compose.yaml similarity index 96% rename from local-docker-compose.yml rename to local-docker-compose.yaml index 629c03b..8a12fc6 100644 --- a/local-docker-compose.yml +++ b/local-docker-compose.yaml @@ -5,7 +5,6 @@ services: container_name: local-collector-dev restart: always environment: - STORE_LOCAL: "true" OPENTRANSIT_COLLECTOR_CONFIG_JSON: > { "agencies": [ diff --git a/src/index.js b/src/index.js index b649a1d..8687bcc 100644 --- a/src/index.js +++ b/src/index.js @@ -80,11 +80,11 @@ function saveVehicles() { }) }); - // Do not reject all promises when one of them is rejected, but do output why one was rejected. + // Do not reject all promises when one of them is rejected, but do output why one (or more) was rejected. Promise.allSettled(promises) .then((results) => results.forEach((result) => { if(result.status == "rejected") { - console.log("received a rejected promise with reason: " + result.reason) + console.log("received a rejected promise with reason:", result.reason) } })); } From 556a6e88d1a6e260e9ae3a10085c968cf0a4ecf0 Mon Sep 17 00:00:00 2001 From: Liam Murphy Date: Mon, 5 Sep 2022 18:50:05 -0700 Subject: [PATCH 3/5] Update src/writeHelper.js Co-authored-by: Arion Sprague --- src/writeHelper.js | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/writeHelper.js b/src/writeHelper.js index 0d0517c..54dbef8 100644 --- a/src/writeHelper.js +++ b/src/writeHelper.js @@ -17,16 +17,11 @@ function writeLocal(agency, currentDateTime, data) { const version = "v1"; const fileName = `/tmp/${agency}_${version}_${currentTimestamp}.json` - return new Promise((resolve, reject) => { - fs.writeFile(fileName, JSON.stringify(data), err => { - if (err) { - reject(err); - } else { - resolve(); - } - console.log("wrote", fileName) - }); - }) + return writeFile(fileName, JSON.stringify(data)) + .then((data) => { + console.log("wrote", fileName); + return data; + }); } function writeToS3(s3Bucket, agency, currentDateTime, data) { From 4eec14e351797f40af5e4d53f23d8292e0ae916f Mon Sep 17 00:00:00 2001 From: Liam Murphy Date: Sun, 11 Sep 2022 10:21:20 -0700 Subject: [PATCH 4/5] change fs import --- src/writeHelper.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/writeHelper.js b/src/writeHelper.js index 0d0517c..aa6afc3 100644 --- a/src/writeHelper.js +++ b/src/writeHelper.js @@ -1,8 +1,10 @@ const AWS = require('aws-sdk'); -const fs = require('fs'); +const s3 = new AWS.S3(); +const { writeFile } = require('fs/promises'); + var zlib = require('zlib'); -const s3 = new AWS.S3(); + function compressData(data) { return new Promise((resolve, _) => { @@ -18,7 +20,7 @@ function writeLocal(agency, currentDateTime, data) { const fileName = `/tmp/${agency}_${version}_${currentTimestamp}.json` return new Promise((resolve, reject) => { - fs.writeFile(fileName, JSON.stringify(data), err => { + writeFile(fileName, JSON.stringify(data), err => { if (err) { reject(err); } else { From 1ffc8c3730628de9d1ddfd4fb9db111d9f1d8355 Mon Sep 17 00:00:00 2001 From: Liam Murphy Date: Sun, 11 Sep 2022 10:25:24 -0700 Subject: [PATCH 5/5] final touchups --- local-docker-compose.yaml | 1 + src/writeHelper.js | 13 ------------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/local-docker-compose.yaml b/local-docker-compose.yaml index 8a12fc6..629c03b 100644 --- a/local-docker-compose.yaml +++ b/local-docker-compose.yaml @@ -5,6 +5,7 @@ services: container_name: local-collector-dev restart: always environment: + STORE_LOCAL: "true" OPENTRANSIT_COLLECTOR_CONFIG_JSON: > { "agencies": [ diff --git a/src/writeHelper.js b/src/writeHelper.js index 39153ff..a2dfc96 100644 --- a/src/writeHelper.js +++ b/src/writeHelper.js @@ -19,24 +19,11 @@ function writeLocal(agency, currentDateTime, data) { const version = "v1"; const fileName = `/tmp/${agency}_${version}_${currentTimestamp}.json` -<<<<<<< HEAD - return new Promise((resolve, reject) => { - writeFile(fileName, JSON.stringify(data), err => { - if (err) { - reject(err); - } else { - resolve(); - } - console.log("wrote", fileName) - }); - }) -======= return writeFile(fileName, JSON.stringify(data)) .then((data) => { console.log("wrote", fileName); return data; }); ->>>>>>> 556a6e88d1a6e260e9ae3a10085c968cf0a4ecf0 } function writeToS3(s3Bucket, agency, currentDateTime, data) {