model: fix timezone handling for Edm.DateTimeOffset (#184)

* model: fix timezone handling for Edm.DateTimeOffset

Convert Edm.DateTimeOffset entities to Python datetime objects with a
timezone attached, instead of simply passing them around as plain
string.

* model: encourage safe Edm.DateTime usage

Edm.DateTime is underspecified and its timezone handling, if existing at
all, up to interpretation.

The current implementation allows creating Edm.DateTime properties,
using a datetime object with an arbitrary timezone. Before sending the
containing entity to the server, pyodata converts the Python datetime
object to a UTC based representation. Therefore, when fetching the
entity later on, its Edm.DateTime property will be UTC based, with the
original timezone information being missing.

This kind of information loss might surprise users of this library!

Disallowing the creation of non-UTC datetime properties forces the user
to convert manually to UTC, therefore raising awareness of the
limitations of Edm.DateTime and encourage the usage of
Edm.DateTimeOffset.

For compatibility with (maybe existing) implementations that interpret
the specs differently and add offsets to Edm.DateTime, pyodata now
supports such values when being received from an OData server, converts
them to UTC.

Co-authored-by: Petr Hanák <[email protected]>

Author: rettichschnidi

CHANGELOG.md

@@ -5,13 +5,15 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
-- model: split properties schema.entity_types/complex_types and their generated Collections - Petr Hanak
 
 ### Fixed
 - Fix Edm.Binary literal representation - Daniel Balko
+- Datetime support for Edm.DateTimeOffset - Reto Schneider
+- Disallow creation of non-UTC Edm.DateTime - Reto Schneider
+- Split properties schema.entity_types/complex_types and their generated Collections - Petr Hanak
 
 ### Removed
-- Python 3.6 (after its EOL) is no longer supported by pyodata. Python 3.7 is now minimal supported version.
+- Python 3.6 (after its EOL) is no longer supported by pyodata. Python 3.7 is now minimal supported version. - Petr Hanak
 
 ## [1.7.1]
 

pyodata/v2/model.py

@@ -29,17 +29,6 @@
 TypeInfo = collections.namedtuple('TypeInfo', 'namespace name is_collection')
 
 
-def current_timezone():
-    """Default Timezone for Python datetime instances when parsed from
-       Edm.DateTime values and vice versa.
-
-       OData V2 does not mention Timezones in the documentation of
-       Edm.DateTime and UTC was chosen because it is universal.
-    """
-
-    return datetime.timezone.utc
-
-
 def modlog():
     return logging.getLogger(LOGGER_NAME)
 
@@ -210,7 +199,8 @@ def _build_types():
             Types.register_type(Typ('Edm.SByte', '0'))
             Types.register_type(Typ('Edm.String', '\'\'', EdmStringTypTraits()))
             Types.register_type(Typ('Edm.Time', 'time\'PT00H00M\''))
-            Types.register_type(Typ('Edm.DateTimeOffset', 'datetimeoffset\'0000-00-00T00:00:00\''))
+            Types.register_type(
+                Typ('Edm.DateTimeOffset', 'datetimeoffset\'0000-00-00T00:00:00Z\'', EdmDateTimeOffsetTypTraits()))
 
     @staticmethod
     def register_type(typ):
@@ -373,6 +363,40 @@ def from_literal(self, value):
         return base64.b64encode(binary).decode()
 
 
+def ms_since_epoch_to_datetime(value, tzinfo):
+    """Convert milliseconds since midnight 1.1.1970 to datetime"""
+    try:
+        # https://stackoverflow.com/questions/36179914/timestamp-out-of-range-for-platform-localtime-gmtime-function
+        return datetime.datetime(1970, 1, 1, tzinfo=tzinfo) + datetime.timedelta(milliseconds=int(value))
+    except (ValueError, OverflowError):
+        min_ticks = -62135596800000
+        max_ticks = 253402300799999
+        if FIX_SCREWED_UP_MINIMAL_DATETIME_VALUE and int(value) < min_ticks:
+            # Some service providers return false minimal date values.
+            # -62135596800000 is the lowest value PyOData could read.
+            # This workaround fixes this issue and returns 0001-01-01 00:00:00+00:00 in such a case.
+            return datetime.datetime(year=1, day=1, month=1, tzinfo=tzinfo)
+        if FIX_SCREWED_UP_MAXIMUM_DATETIME_VALUE and int(value) > max_ticks:
+            return datetime.datetime(year=9999, day=31, month=12, tzinfo=tzinfo)
+        raise PyODataModelError(f'Cannot decode datetime from value {value}. '
+                                f'Possible value range: {min_ticks} to {max_ticks}. '
+                                f'You may fix this by setting `FIX_SCREWED_UP_MINIMAL_DATETIME_VALUE` '
+                                f' or `FIX_SCREWED_UP_MAXIMUM_DATETIME_VALUE` as a workaround.')
+
+
+def parse_datetime_literal(value):
+    try:
+        return datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%f')
+    except ValueError:
+        try:
+            return datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S')
+        except ValueError:
+            try:
+                return datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M')
+            except ValueError:
+                raise PyODataModelError(f'Cannot decode datetime from value {value}.')
+
+
 class EdmDateTimeTypTraits(EdmPrefixedTypTraits):
     """Emd.DateTime traits
 
@@ -403,46 +427,48 @@ def to_literal(self, value):
             raise PyODataModelError(
                 f'Cannot convert value of type {type(value)} to literal. Datetime format is required.')
 
+        if value.tzinfo != datetime.timezone.utc:
+            raise PyODataModelError('Emd.DateTime accepts only UTC')
+
         # Sets timezone to none to avoid including timezone information in the literal form.
         return super(EdmDateTimeTypTraits, self).to_literal(value.replace(tzinfo=None).isoformat())
 
     def to_json(self, value):
         if isinstance(value, str):
             return value
 
+        if value.tzinfo != datetime.timezone.utc:
+            raise PyODataModelError('Emd.DateTime accepts only UTC')
+
         # Converts datetime into timestamp in milliseconds in UTC timezone as defined in ODATA specification
         # https://www.odata.org/documentation/odata-version-2-0/json-format/
-        return f'/Date({int(value.replace(tzinfo=current_timezone()).timestamp()) * 1000})/'
+        # See also: https://docs.python.org/3/library/datetime.html#datetime.datetime.timestamp
+        ticks = (value - datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)) / datetime.timedelta(milliseconds=1)
+        return f'/Date({int(ticks)})/'
 
     def from_json(self, value):
 
         if value is None:
             return None
 
-        matches = re.match(r"^/Date\((.*)\)/$", value)
-        if not matches:
+        matches = re.match(r"^/Date\((?P<milliseconds_since_epoch>-?\d+)(?P<offset_in_minutes>[+-]\d+)?\)/$", value)
+        try:
+            milliseconds_since_epoch = matches.group('milliseconds_since_epoch')
+        except AttributeError:
             raise PyODataModelError(
-                f"Malformed value {value} for primitive Edm type. Expected format is /Date(value)/")
-        value = matches.group(1)
-
+                f"Malformed value {value} for primitive Edm.DateTime type."
+                " Expected format is /Date(<ticks>[±<offset>])/")
         try:
-            # https://stackoverflow.com/questions/36179914/timestamp-out-of-range-for-platform-localtime-gmtime-function
-            value = datetime.datetime(1970, 1, 1, tzinfo=current_timezone()) + datetime.timedelta(milliseconds=int(value))
-        except (ValueError, OverflowError):
-            if FIX_SCREWED_UP_MINIMAL_DATETIME_VALUE and int(value) < -62135596800000:
-                # Some service providers return false minimal date values.
-                # -62135596800000 is the lowest value PyOData could read.
-                # This workaroud fixes this issue and returns 0001-01-01 00:00:00+00:00 in such a case.
-                value = datetime.datetime(year=1, day=1, month=1, tzinfo=current_timezone())
-            elif FIX_SCREWED_UP_MAXIMUM_DATETIME_VALUE and int(value) > 253402300799999:
-                value = datetime.datetime(year=9999, day=31, month=12, tzinfo=current_timezone())
-            else:
-                raise PyODataModelError(f'Cannot decode datetime from value {value}. '
-                                        f'Possible value range: -62135596800000 to 253402300799999. '
-                                        f'You may fix this by setting `FIX_SCREWED_UP_MINIMAL_DATETIME_VALUE` '
-                                        f' or `FIX_SCREWED_UP_MAXIMUM_DATETIME_VALUE` as a workaround.')
-
-        return value
+            offset_in_minutes = int(matches.group('offset_in_minutes') or 0)
+            timedelta = datetime.timedelta(minutes=offset_in_minutes)
+        except ValueError:
+            raise PyODataModelError(
+                f"Malformed value {value} for primitive Edm.DateTime type."
+                " Expected format is /Date(<ticks>[±<offset>])/")
+        except AttributeError:
+            timedelta = datetime.timedelta()  # Missing offset is interpreted as UTC
+        # Might raise a PyODataModelError exception
+        return ms_since_epoch_to_datetime(milliseconds_since_epoch, datetime.timezone.utc) + timedelta
 
     def from_literal(self, value):
 
@@ -451,18 +477,85 @@ def from_literal(self, value):
 
         value = super(EdmDateTimeTypTraits, self).from_literal(value)
 
+        # Note: parse_datetime_literal raises a PyODataModelError exception on invalid formats
+        return parse_datetime_literal(value).replace(tzinfo=datetime.timezone.utc)
+
+
+class EdmDateTimeOffsetTypTraits(EdmPrefixedTypTraits):
+    """Emd.DateTimeOffset traits
+
+       Represents date and time, plus an offset in minutes from UTC, with values ranging from 12:00:00 midnight,
+       January 1, 1753 A.D. through 11:59:59 P.M, December 9999 A.D
+
+       Literal forms:
+       datetimeoffset'yyyy-mm-ddThh:mm[:ss]±ii:nn' (works for all time zones)
+       datetimeoffset'yyyy-mm-ddThh:mm[:ss]Z' (works only for UTC)
+       NOTE: Spaces are not allowed between datetimeoffset and quoted portion.
+       The datetime part is case-insensitive, the offset one is not.
+
+       Example 1: datetimeoffset'1970-01-01T00:00:01+00:30'
+        - /Date(1000+0030)/ (As DateTime, but with a 30 minutes timezone offset)
+       Example 1: datetimeoffset'1970-01-01T00:00:01-00:60'
+        - /Date(1000-0030)/ (As DateTime, but with a negative 60 minutes timezone offset)
+       https://blogs.sap.com/2017/01/05/date-and-time-in-sap-gateway-foundation/
+    """
+
+    def __init__(self):
+        super(EdmDateTimeOffsetTypTraits, self).__init__('datetimeoffset')
+
+    def to_literal(self, value):
+        """Convert python datetime representation to literal format"""
+
+        if not isinstance(value, datetime.datetime) or value.utcoffset() is None:
+            raise PyODataModelError(
+                f'Cannot convert value of type {type(value)} to literal. Datetime format including offset is required.')
+
+        return super(EdmDateTimeOffsetTypTraits, self).to_literal(value.isoformat())
+
+    def to_json(self, value):
+        # datetime.timestamp() does not work due to its limited precision
+        offset_in_minutes = int(value.utcoffset() / datetime.timedelta(minutes=1))
+        ticks = int((value - datetime.datetime(1970, 1, 1, tzinfo=value.tzinfo)) / datetime.timedelta(milliseconds=1))
+        return f'/Date({ticks}{offset_in_minutes:+05})/'
+
+    def from_json(self, value):
+        matches = re.match(r"^/Date\((?P<milliseconds_since_epoch>-?\d+)(?P<offset_in_minutes>[+-]\d+)\)/$", value)
         try:
-            value = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%f')
-        except ValueError:
-            try:
-                value = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S')
-            except ValueError:
-                try:
-                    value = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M')
-                except ValueError:
-                    raise PyODataModelError(f'Cannot decode datetime from value {value}.')
+            milliseconds_since_epoch = matches.group('milliseconds_since_epoch')
+            offset_in_minutes = int(matches.group('offset_in_minutes'))
+        except (ValueError, AttributeError):
+            raise PyODataModelError(
+                f"Malformed value {value} for primitive Edm.DateTimeOffset type."
+                " Expected format is /Date(<ticks>±<offset>)/")
+
+        tzinfo = datetime.timezone(datetime.timedelta(minutes=offset_in_minutes))
+        # Might raise a PyODataModelError exception
+        return ms_since_epoch_to_datetime(milliseconds_since_epoch, tzinfo)
+
+    def from_literal(self, value):
 
-        return value.replace(tzinfo=current_timezone())
+        if value is None:
+            return None
+
+        value = super(EdmDateTimeOffsetTypTraits, self).from_literal(value)
+
+        try:
+            # Note: parse_datetime_literal raises a PyODataModelError exception on invalid formats
+            if re.match(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z', value, flags=re.ASCII | re.IGNORECASE):
+                datetime_part = value[:-1]
+                tz_info = datetime.timezone.utc
+            else:
+                match = re.match(r'(?P<datetime>.+)(?P<sign>[\\+-])(?P<hours>\d{2}):(?P<minutes>\d{2})',
+                                 value,
+                                 flags=re.ASCII)
+                datetime_part = match.group('datetime')
+                tz_offset = datetime.timedelta(hours=int(match.group('hours')),
+                                               minutes=int(match.group('minutes')))
+                tz_sign = -1 if match.group('sign') == '-' else 1
+                tz_info = datetime.timezone(tz_sign * tz_offset)
+            return parse_datetime_literal(datetime_part).replace(tzinfo=tz_info)
+        except (ValueError, AttributeError):
+            raise PyODataModelError(f'Cannot decode datetimeoffset from value {value}.')
 
 
 class EdmStringTypTraits(TypTraits):

tests/conftest.py

@@ -2,7 +2,7 @@
 import logging
 import os
 import pytest
-from pyodata.v2.model import schema_from_xml
+from pyodata.v2.model import schema_from_xml, Types
 
 
 def contents_of_fixtures_file(file_name):
@@ -129,3 +129,13 @@ def assert_logging_policy(mock_warning, *args):
 def assert_request_contains_header(headers, name, value):
     assert name in headers
     assert headers[name] == value
+
+
+@pytest.fixture
+def type_date_time():
+    return Types.from_name('Edm.DateTime')
+
+
+@pytest.fixture
+def type_date_time_offset():
+    return Types.from_name('Edm.DateTimeOffset')

tests/metadata.xml

@@ -59,6 +59,8 @@
                           sap:creatable="false" sap:updatable="false" sap:sortable="true" sap:filterable="true"/>
                 <Property Name="Date" Type="Edm.DateTime" Nullable="false" sap:label="Data" sap:creatable="false"
                           sap:updatable="false" sap:sortable="true" sap:filterable="true"/>
+                <Property Name="DateTimeWithOffset" Type="Edm.DateTimeOffset" Nullable="true" sap:label="Data"
+                          sap:creatable="false" sap:updatable="true" sap:sortable="true" sap:filterable="true"/>
                 <Property Name="Value" Type="Edm.Double" Nullable="false" sap:unicode="false" sap:label="Data"
                           sap:creatable="false" sap:updatable="false" sap:sortable="true" sap:filterable="true"/>
             </EntityType>