Fix: OpenAPI spec does not document 429 responses (#7469)

lambdas/indexer/app.py

@@ -53,7 +53,7 @@
         # changes and reset the minor version to zero. Otherwise, increment only
         # the minor version for backwards compatible changes. A backwards
         # compatible change is one that does not require updates to clients.
-        'version': '3.3',
+        'version': '3.4',
         'description': fd('''
             This is the internal API for Azul's indexer component.
         ''')

src/azul/chalice.py

@@ -153,6 +153,7 @@ def __init__(self,
         # Middleware is invoked in order of registration
         self.register_middleware(self._logging_middleware, 'http')
         self.register_middleware(self._security_headers_middleware, 'http')
+        self.register_middleware(self._retry_503, 'http')
         self.register_middleware(self._api_gateway_context_middleware, 'http')
         self.register_middleware(self._authentication_middleware, 'http')
 
@@ -258,6 +259,15 @@ def _security_headers_middleware(self, event, get_response):
         response.headers['Cache-Control'] = cache_control
         return response
 
+    def _retry_503(self, event, get_response):
+        """
+        Add a retry-after header to 503 responses
+        """
+        response = get_response(event)
+        if response.status_code == 503:
+            response.headers.setdefault('Retry-After', '30')
+        return response
+
     def _http_cache_for(self, seconds: int):
         """
         The HTTP Cache-Control response header value that will cause the
@@ -323,7 +333,7 @@ def route[C: Callable](self,
             if not interactive:
                 require(bool(methods), 'Must list methods with interactive=False')
                 self.non_interactive_routes.update((path, method) for method in methods)
-            spec = deep_dict_merge(spec, self.default_specs())
+            spec = deep_dict_merge(self.default_specs(), spec, override=True)
             chalice_decorator = super().route(path, methods=methods, **kwargs)
 
             def decorator(view_func):
@@ -867,14 +877,33 @@ def robots_txt():
         return locals()
 
     def default_specs(self):
+        retry_after = format_description('''
+            When handling this response, clients should wait the number of seconds
+            specified in the `Retry-After` header and then retry the request.
+        ''')
         return {
             'responses': {
+                '400': {
+                    'description': 'Bad request. The request was rejected due '
+                                   'to malformed parameters.'
+                },
+                '429': {
+                    'description': 'Too many requests. ' + retry_after
+                },
+                '500': {
+                    'description': 'Internal server error. An internal server '
+                                   'error occurred.'
+                },
+                '502': {
+                    'description': 'Bad gateway. The server received an '
+                                   'invalid response from the upstream server.'
+                },
+                '503': {
+                    'description': 'Service unavailable. ' + retry_after
+                },
                 '504': {
-                    'description': format_description('''
-                        Request timed out. When handling this response, clients
-                        should wait the number of seconds specified in the
-                        `Retry-After` header and then retry the request.
-                    ''')
+                    'description': 'Gateway timeout. The server did not '
+                                   'respond in time. Please try again later.'
                 }
             }
         }

src/azul/collections.py

@@ -27,6 +27,7 @@
     Self,
     TypeVar,
     Union,
+    cast,
     overload,
 )
 
@@ -52,64 +53,99 @@
 
 
 # noinspection PyPep8Naming
 class deep_dict_merge[K, V](dict):
     """
-    Recursively merge the given dictionaries. If more than one dictionary
-    contains a given key, and all values associated with this key are themselves
-    dictionaries, then the value present in the result is the recursive merging
-    of those nested dictionaries.
+    Recursively merge the given mappings into a single dictionary.
 
+    >>> deep_dict_merge()
+    {}
+    >>> deep_dict_merge({})
+    {}
+    >>> deep_dict_merge({}, {})
+    {}
+    >>> deep_dict_merge({0: 1}, {})
+    {0: 1}
     >>> deep_dict_merge({0: 1}, {1: 0})
     {0: 1, 1: 0}
 
-    To merge all dictionaries in an iterable, use this form:
+    If more than one mapping contains a given key, the corresponding value in
+    the result is determined as follows:
 
-    >>> deep_dict_merge.from_iterable([{0: 1}, {1: 0}])
-    {0: 1, 1: 0}
+    A) If all values associated with that key are equal, the first of these
+       values is used.
 
-    >>> deep_dict_merge({0: {'a': 1}}, {0: {'b': 2}})
-    {0: {'a': 1, 'b': 2}}
+       >>> deep_dict_merge({0: 1}, {0: True})
+       {0: 1}
+       >>> l1, l2 = [], []
+       >>> d = deep_dict_merge({0: l1}, {0: l2})
+       >>> d
+       {0: []}
+       >>> id(d[0]) == id(l1)
+       True
 
-    Key collisions where either value is not a dictionary raise an exception,
-    unless the values compare equal to each other, in which case the entries
-    from *earlier* dictionaries takes precedence. This behavior is the opposite
-    of `dict_merge`, where later entries take precedence.
+    B) Otherwise, if all values are themselves mappings, these nested mappings
+       are merged recursively.
 
-    >>> deep_dict_merge({0: 1}, {0: 2})
-    Traceback (most recent call last):
-    ...
-    ValueError: 1 != 2
+       >>> deep_dict_merge({0: {'a': 1}}, {0: {'b': 2}})
+       {0: {'a': 1, 'b': 2}}
 
-    >>> l1, l2 = [], []
-    >>> d = deep_dict_merge({0: l1}, {0: l2})
-    >>> d
-    {0: []}
-    >>> id(d[0]) == id(l1)
-    True
+    C) Otherwise, if none of the values are mappings, either an exception is
+       raised, or, if `override` is true, the last of the values is used.
 
-    >>> deep_dict_merge()
-    {}
+       >>> deep_dict_merge({0: 1}, {0: 2})
+       Traceback (most recent call last):
+       ...
+       ValueError: ('Conflicting values', 1, 2)
+
+       >>> deep_dict_merge({0: 1}, {0: 2}, override=True)
+       {0: 2}
+
+    D) In all other cases (some, but not all of the values are mappings) an
+       exception is raised.
+
+       >>> deep_dict_merge({0: 1}, {0: {2: 3}})
+       Traceback (most recent call last):
+       ...
+       ValueError: ('Can only merge mappings', 1, {2: 3})
+
+    To merge all mappings in an iterable, use this form:
+
+    >>> deep_dict_merge.from_iterable([{0: 1}, {1: 0}])
+    {0: 1, 1: 0}
+    >>> deep_dict_merge.from_iterable([{0: 1}, {0: 2}], override=True)
+    {0: 2}
     """
 
-    def __init__(self, *maps: Mapping[K, V]):
+    def __init__(self, *maps: Mapping[K, V], override: bool = False):
         super().__init__()
-        self.merge(maps)
+        self.override = override
+        self._merge(maps)
 
     @classmethod
-    def from_iterable(cls, maps: Iterable[Mapping[K, V]], /) -> Self:
-        self = cls()
-        self.merge(maps)
+    def from_iterable(cls, maps: Iterable[Mapping[K, V]],
+                      /,
+                      *,
+                      override: bool = False) -> Self:
+        self = cls(override=override)
+        self._merge(maps)
         return self
 
-    def merge(self, maps: Iterable[Mapping[K, V]]):
+    def _merge(self, maps: Iterable[Mapping[K, V]]):
         for m in maps:
             for k, v2 in m.items():
                 v1 = self.setdefault(k, v2)
                 if v1 != v2:
-                    if isinstance(v1, Mapping) and isinstance(v2, Mapping):
-                        self[k] = type(self)(v1, v2)
-                    else:
-                        raise ValueError(f'{v1!r} != {v2!r}')
+                    match isinstance(v1, Mapping), isinstance(v2, Mapping):
+                        case True, True:
+                            # Cast is safe, mypy just isn't smart enough to infer that
+                            self[k] = type(self)(v1, cast(Mapping, v2), override=self.override)
+                        case False, False:
+                            if self.override:
+                                self[k] = v2
+                            else:
+                                raise ValueError('Conflicting values', v1, v2)
+                        case _:
+                            raise ValueError('Can only merge mappings', v1, v2)
 
 
 def explode_dict[K, V](d: Mapping[K, Union[V, list[V], set[V], tuple[V]]]

src/azul/terraform.py

@@ -965,6 +965,15 @@ def alias_property(property_name: str, resource: MutableJSON):
             for response_type in ['4XX', '5XX']
         }
         error_responses: MutableJSON = {
+            **{
+                response_type: {
+                    'statusCode': '429',
+                    'responseParameters': {
+                        **security_headers,
+                        'gatewayresponse.header.Retry-After': "'30'"
+                    }
+                } for response_type in ['QUOTA_EXCEEDED', 'THROTTLED']
+            },
             'INTEGRATION_FAILURE': {
                 'statusCode': '502',
                 'responseParameters': security_headers,
@@ -974,20 +983,6 @@ def alias_property(property_name: str, resource: MutableJSON):
                                    'to complete your request.'
                     })
                 }
-            },
-            'INTEGRATION_TIMEOUT': {
-                'statusCode': '504',
-                'responseParameters': {
-                    **security_headers,
-                    'gatewayresponse.header.Retry-After': "'10'"
-                },
-                'responseTemplates': {
-                    "application/json": json.dumps({
-                        'message': '504 Gateway Timeout. Wait the number of '
-                                   'seconds specified in the `Retry-After` '
-                                   'header before retrying the request.'
-                    })
-                }
             }
         }
         gateway_responses = error_responses | default_responses

test/test_openapi.py

@@ -102,9 +102,17 @@ def _assert_default_spec(self, actual_spec: JSON) -> JSON:
                 methods = {'get', 'put'}  # only what's used in these tests
                 if method in methods:
                     responses = spec.pop('responses')
-                    response = responses.pop('504')
-                    description = response.pop('description')
-                    self.assertIn('Request timed out', description)
+                    for code, message in [
+                        ('400', 'Bad request'),
+                        ('429', 'Too many requests'),
+                        ('500', 'Internal server error'),
+                        ('502', 'Bad gateway'),
+                        ('503', 'Service unavailable'),
+                        ('504', 'Gateway timeout'),
+                    ]:
+                        response = responses.pop(code)
+                        description = response.pop('description')
+                        self.assertIn(message, description)
                     self.assertEqual(({}, {}), (response, responses))
         return actual_spec