Modified dataSender.py for weather station box

src/dataSender/dataSender.py

@@ -1,5 +1,12 @@
+# dataSender for weather station box 
+# Original by Hakan Saplakoglu
+# Modified by Zachary Clayton 
+# Added functionality to also upload data not uploaded from the previous hour due to timing issues with running this program every 20 minutes. 
+# Data was lost so I made it also upload data from the previous hour not uploaded yet
+
 import influxdb_client
 from influxdb_client.client.write_api import SYNCHRONOUS
+from influxdb_client.rest import InfluxDBError
 import time
 from datetime import datetime
 from datetime import timedelta
@@ -26,50 +33,68 @@ def main():
 
     dev_hostname = socket.gethostname()
 
-    # find csv file to send
     if args.time is None:
-        # custom time requested to be sent
+        # find the start time of the previous hour
         cur_time = datetime.utcnow()
-        # lag csv timestamp by 1 min to catch last data points
-        cur_time -= timedelta(minutes=1)
+        prev_hour = cur_time - timedelta(hours=1)
     else:
         cur_time = datetime.fromisoformat(args.time)
+        prev_hour = cur_time - timedelta(hours=1)
 
-    csv_timestamp = cur_time.replace(minute=0,second=0,microsecond=0)
+    csv_timestamp_prev = prev_hour.replace(minute=0, second=0, microsecond=0)
+    csv_timestamp_curr = cur_time.replace(minute=0, second=0, microsecond=0)
 
     for logdir in args.logdir:
         log_prefix = os.path.basename(logdir)
 
-        csv_path = os.path.join(
-            logdir,
-            log_prefix + "_" + datetime.strftime(csv_timestamp, "%Y%m%d"),
-            log_prefix + "_" + datetime.strftime(csv_timestamp, "%Y%m%d-%H") + ".txt"
-        )
+        for csv_timestamp in [csv_timestamp_prev, csv_timestamp_curr]:  # upload data from previous hour and current hour
+            csv_path = os.path.join(
+                logdir,
+                log_prefix + "_" + datetime.strftime(csv_timestamp, "%Y%m%d"),
+                log_prefix + "_" + datetime.strftime(csv_timestamp, "%Y%m%d-%H") + ".csv"
+            )
 
-        # read csv in chunks with pandas
-        if log_prefix == "BAROLOG":
-            csvHeader = ["hostname", "sensor_id", "sys_timestamp", "timestamp", "value"]
-        elif log_prefix == "WINDLOG":
-            csvHeader = ["hostname", "sensor_id", "timestamp", "adc", "voltage", "value"]
-
-        i = 0
-        for df in pd.read_csv(csv_path, chunksize=args.chunk, names=csvHeader):
-            # convert values
-            df["value"] = df["value"].astype(float)
+            if csv_timestamp > datetime.utcnow():
+                continue  
 
-            print("Sending " + log_prefix + " data points [" + str(i * args.chunk) + "-" + str((i + 1) * args.chunk - 1) + "]")
-            i += 1
-            with client.write_api() as write_api:
-                try:
-                    write_api.write(
-                        record=df,
-                        bucket=args.bucket,
-                        data_frame_measurement_name=dev_hostname,
-                        data_frame_tag_columns=["sensor_id"],
-                        data_frame_timestamp_column="timestamp",
-                    )
-                except InfluxDBError as e:
-                    print(e)
+            if not os.path.isfile(csv_path):
+                print(f"CSV file not found for timestamp: {csv_timestamp}")
+                continue
+
+            # read csv in chunks with pandas
+            if log_prefix == "BAROLOG":
+                csvHeader = ["hostname", "sensor_id", "sys_timestamp", "timestamp", "value"]
+            elif log_prefix == "WINDLOG":
+                csvHeader = ["hostname", "sensor_id", "timestamp", "adc", "voltage", "value"]
+            elif log_prefix == "ZACHLOG":
+                csvHeader = ["timestamp", "RecNbr", "b'BattV'", "b'PTemp_C'" ,"b'AirTC'", "b'RH'", "b'WS_ms'", "b'WindDir'", "b'BaroPres'", "b'Pyranometer'", "zakbox" ] 
+
+            for df in pd.read_csv(csv_path, chunksize=args.chunk, names=csvHeader, skiprows=1):
+                # convert values
+                df["RecNbr"] = df["RecNbr"].astype(float)
+                df["b'BattV'"] = df["b'BattV'"].astype(float)
+                df["b'PTemp_C'"] = df["b'PTemp_C'"].astype(float)
+                df["b'AirTC'"] = df["b'AirTC'"].astype(float)
+                df["b'RH'"] = df["b'RH'"].astype(float)
+                df["b'WS_ms'"] = df["b'WS_ms'"].astype(float)
+                df["b'WindDir'"] = df["b'WindDir'"].astype(float)
+                df["b'BaroPres'"] = df["b'BaroPres'"].astype(float)
+                df["b'Pyranometer'"] = df["b'Pyranometer'"].astype(float)
+
+                print("Sending " + log_prefix + " data points")
+
+                with client.write_api() as write_api:
+                    try:
+                        write_api.write(
+                            record=df,
+                            bucket=args.bucket,
+                            data_frame_measurement_name=dev_hostname,
+                            data_frame_tag_columns=["zakbox"],
+                            data_frame_timestamp_column="timestamp",
+                        )
+                    except InfluxDBError as e:
+                        print(e)
+    print("Finished Uploading to InfluxDB.")
 
 if __name__ == "__main__":
-    main()
+    main()