Merge branch 'release-v5.3.0'

Author: qrkourier

.github/workflows/main.yml

@@ -59,6 +59,38 @@ jobs:
         }
         echo ::set-output name=pypi_version::${PYPI_VERSION}
 
+    - name: Install NetFoundry from build tarball
+      run: |
+        pip3 install ./dist/netfoundry-*.tar.gz
+
+    - name: Compare installed version to PyPi version
+      env:
+        PYPI_VERSION: ${{ steps.read_version.outputs.pypi_version }}
+      run: |
+        INSTALLED_VERSION="$(python3 -m netfoundry.version)"
+        if ! [[ ${PYPI_VERSION} == ${INSTALLED_VERSION#v} ]]; then
+          echo "ERROR: PyPi and installed version do not match." >&2
+          exit 1
+        fi
+
+    - name: Run the demo script to smoke test installed version
+      env:
+        NETWORK_NAME: github_smoketest_run${{ github.run_id }}
+        NETFOUNDRY_CLIENT_ID: ${{ secrets.NETFOUNDRY_CLIENT_ID }}
+        NETFOUNDRY_PASSWORD: ${{ secrets.NETFOUNDRY_PASSWORD }}
+        NETFOUNDRY_OAUTH_URL: ${{ secrets.NETFOUNDRY_OAUTH_URL }}
+      run: |
+        python3 -m netfoundry.demo \
+          --network ${NETWORK_NAME} \
+          --create-client \
+          --create-private \
+          --regions Americas \
+          -- create
+        python3 -m netfoundry.demo \
+          --network ${NETWORK_NAME} \
+          --yes \
+          -- delete
+
     - name: Append 'latest' tag if release published
       env:
         GITHUB_EVENT_ACTION: ${{ github.event.action }}

netfoundry/demo.py

@@ -12,6 +12,7 @@
 from pathlib import Path
 
 import netfoundry
+from netfoundry.utility import DC_PROVIDERS
 
 
 def main():
@@ -26,6 +27,13 @@ def main():
         help="the command to run",
         choices=["create","delete"]
     )
+    parser.add_argument(
+        "-y", "--yes",
+        dest="yes",
+        default=False,
+        action="store_true",
+        help="Skip interactive prompt to confirm destructive actions."
+    )
     parser.add_argument(
         "-n", "--network",
         help="The name of your demo network"
@@ -43,7 +51,7 @@ def main():
         "-s", "--network-size",
         dest="size",
         default="small",
-        help="Billable Network size to create",
+        help="Network size to create",
         choices=["small","medium","large"]
     )
     parser.add_argument(
@@ -74,7 +82,7 @@ def main():
         default="AWS",
         required=False,
         help="cloud provider to host Edge Routers",
-        choices=["AWS", "AZURE", "GCP", "OCP"]
+        choices=DC_PROVIDERS
     )
     regions_group = parser.add_mutually_exclusive_group(required=False)
     regions_group.add_argument("--regions",
@@ -117,7 +125,7 @@ def main():
         if args.command == "create":
             network.wait_for_status("PROVISIONED",wait=999,progress=True)
         elif args.command == "delete":
-            if query_yes_no("Permanently destroy Network \"{network_name}\" now?".format(network_name=network_name)):
+            if args.yes or query_yes_no("Permanently destroy Network \"{network_name}\" now?".format(network_name=network_name)):
                 network.delete_network(progress=True)
             else:
                 print("Not deleting Network \"{network_name}\".".format(network_name=network_name))
@@ -126,22 +134,23 @@ def main():
         network_id = network_group.create_network(name=network_name,size=args.size,version=args.version)['id']
         network = netfoundry.Network(network_group, network_id=network_id)
         network.wait_for_status("PROVISIONED",wait=999,progress=True)
+    elif args.command == "delete":
+        print("Network \"{network_name}\" does not exist.".format(network_name=network_name))
+        sys.exit()
     else:
         raise Exception("ERROR: failed to find a network named \"{:s}\"".format(network_name))
 
-    # existing hosted ERs
+    # existing hosted routers
     hosted_edge_routers = network.edge_routers(only_hosted=True)
     # a list of places where Endpoints are dialing from
 
-    # a list of locations to place one hosted ER
+    # a list of locations to place a hosted router
     fabric_placements = list()
     if args.location_codes:
         for location in args.location_codes:
-            data_centers = [dc for dc in network.get_edge_router_data_centers(provider=args.provider,location_code=location)]
-            data_center = data_centers[0]
-            existing_count = len([er for er in hosted_edge_routers if er['dataCenterId'] == data_center['id']])
+            existing_count = len([er for er in hosted_edge_routers if er['provider'] == args.provider and er['locationCode'] == args.location_code])
             if existing_count < 1:
-                fabric_placements += [data_center]
+                fabric_placements += [location]
             else:
                 print("INFO: found a hosted Edge Router(s) in {location}".format(location=location))
 
@@ -153,14 +162,17 @@ def main():
                     "#"+location['locationCode'],
                     "#"+location['provider']
                 ],
-                data_center_id=location['id']
+                provider=args.provider,
+                location_code=location['locationCode']
             )
             hosted_edge_routers += [er]
             print("INFO: Placed Edge Router in {provider} ({location_name})".format(
                 provider=location['provider'],
                 location_name=location['locationName']
             ))
     elif args.regions:
+        if args.provider or args.location_codes:
+            print("WARN: ignoring provider and location_codes because AWS georegion is specified.")
         geo_regions = args.regions
         for region in geo_regions:
             data_center_ids = [dc['id'] for dc in network.aws_geo_regions[region]]
@@ -180,9 +192,11 @@ def main():
                 attributes=[
                     "#defaultRouters",
                     "#"+location['locationCode'],
+                    "#"+location['provider'],
                     "#"+location['geoRegion']
                 ],
-                data_center_id=location['id']
+                provider="AWS",
+                location_code=location['locationCode']
             )
             hosted_edge_routers += [er]
             print("INFO: Placed Edge Router in {major} ({location_name})".format(

netfoundry/network.py

@@ -5,7 +5,7 @@
 from unicodedata import name  # enforce a timeout; sleep
 from uuid import UUID  # validate UUIDv4 strings
 
-from .utility import (EXCLUDED_PATCH_PROPERTIES, HOST_PROPERTIES,
+from .utility import (DC_PROVIDERS, EXCLUDED_PATCH_PROPERTIES, HOST_PROPERTIES,
                       MAJOR_REGIONS, RESOURCES, STATUS_CODES, VALID_SEPARATORS,
                       VALID_SERVICE_PROTOCOLS, docstring_parameters, eprint,
                       http, plural, singular)
@@ -165,7 +165,7 @@ def validate_entity_roles(self, entities: list, type: str):
         an ERP.
 
         :param list entities: required hashtag role attributes, existing entity @names, or existing entity UUIDs
-        :param str type: required type of entity: one of {resource_entity_types}
+        :param str type: required type of entity, choices: {resource_entity_types}
         """
         valid_entities = list()
         for entity in entities:
@@ -197,8 +197,47 @@ def validate_entity_roles(self, entities: list, type: str):
                     else: valid_entities.append('@'+entity_name) # is an existing endpoint's name resolved from UUID
         return(valid_entities)
 
-    def get_edge_router_data_centers(self,provider: str=None,location_code: str=None):
-        """list the data centers where an Edge Router may be created
+    def get_data_center_by_id(self, id: str):
+        """Get data centers by UUIDv4.
+
+        :param id:        required UUIDv4 of data center
+        """
+        try:
+            # data centers returns a list of dicts (data centers)
+            headers = { "authorization": "Bearer " + self.session.token }
+
+            response = http.get(
+                self.session.audience+'core/v2/data-centers/'+id,
+                proxies=self.session.proxies,
+                verify=self.session.verify,
+                headers=headers
+            )
+            response_code = response.status_code
+        except:
+            raise
+
+        if response_code == STATUS_CODES.codes.OK: # HTTP 200
+            try:
+                data_center = json.loads(response.text)
+            except ValueError as e:
+                eprint('ERROR getting data center')
+                raise(e)
+        else:
+            raise Exception(
+                'ERROR: got unexpected HTTP code {:s} ({:d}) and response {:s}'.format(
+                    STATUS_CODES._codes[response_code][0].upper(),
+                    response_code,
+                    response.text
+                )
+            )
+        return(data_center)
+
+    @docstring_parameters(providers=str(DC_PROVIDERS))
+    def get_edge_router_data_centers(self, provider: str=None, location_code: str=None):
+        """Find data centers for hosting edge routers.
+
+        :param provider:        optionally filter by data center provider, choices: {providers}
+        :param location_code:   provider-specific string identifying the data center location e.g. us-west-1
         """
         try:
             # data centers returns a list of dicts (data centers)
@@ -208,10 +247,10 @@ def get_edge_router_data_centers(self,provider: str=None,location_code: str=None
                 "hostType": "ER"
             }
             if provider is not None:
-                if provider in ["AWS", "AZURE", "GCP", "OCP"]:
+                if provider in DC_PROVIDERS:
                     params['provider'] = provider
                 else:
-                    raise Exception("ERROR: unexpected cloud provider {:s}".format(provider))
+                    raise Exception("ERROR: unexpected cloud provider {:s}. Need one of {:s}".format(provider,str(DC_PROVIDERS)))
             response = http.get(
                 self.session.audience+'core/v2/data-centers',
                 proxies=self.session.proxies,
@@ -377,6 +416,8 @@ def get_resources(self, type: str,name: str=None, accept: str=None, deleted: boo
                 params['name'] = name
             if typeId is not None:
                 params['typeId'] = typeId
+            if deleted:
+                params['status'] = "DELETED"
 
             if not type in RESOURCES.keys():
                 raise Exception("ERROR: unknown type \"{}\". Choices: {}".format(type, RESOURCES.keys()))
@@ -452,7 +493,9 @@ def get_resources(self, type: str,name: str=None, accept: str=None, deleted: boo
                         )
                     )
 
-        # omit deleted entities by default
+        # prune entities with non null deletedAt unless return deleted is true
+        # TODO: remove this because the API has been changed to stop returning
+        # deleted entities unless query param status=DELETED
         if not deleted:
             all_entities = [entity for entity in all_entities if not entity['deletedAt']]
 
@@ -828,14 +871,22 @@ def create_endpoint(self, name: str, attributes: list=[], session_identity: str=
                 raise Exception("ERROR: timed out waiting for process status 'STARTED' or 'FINISHED'")
             return(started)
 
+    @docstring_parameters(providers=str(DC_PROVIDERS))
     def create_edge_router(self, name: str, attributes: list=[], link_listener: bool=False, data_center_id: str=None, 
-                            tunneler_enabled: bool=False, wait: int=900, sleep: int=10, progress: bool=False):
+                            tunneler_enabled: bool=False, wait: int=900, sleep: int=10, progress: bool=False,
+                            provider: str=None, location_code: str=None):
         """Create an Edge Router.
 
+        A router may be hosted by NetFoundry or the customer. If hosted by NF,
+        then you must supply datacenter "provider" and "location_code". If
+        neither are given then the router is customer hosted. 
+
         :param name:              a meaningful, unique name
         :param attributes:        a list of hashtag role attributes
-        :param link_listener:     true if router should listen for other routers' links on 80/tcp
-        :param data_center_id:    the UUIDv4 of a data center location that can host edge routers
+        :param link_listener:     true if router should listen for other routers' transit links on 80/tcp, always true if hosted by NetFoundry
+        :param data_center_id:    (DEPRECATED by provider, location_code) the UUIDv4 of a NetFoundry data center location that can host edge routers
+        :param provider:          datacenter provider, choices: {providers}
+        :param location_code:     provider-specific string identifying the datacenter location e.g. us-west-1
         :param tunneler_enabled:  true if the built-in tunneler features should be enabled for hosting or interception or both
         :param wait:              seconds to wait for async create to succeed
         """
@@ -854,8 +905,25 @@ def create_edge_router(self, name: str, attributes: list=[], link_listener: bool
                 "tunnelerEnabled": tunneler_enabled
             }
             if data_center_id:
-                body['dataCenterId'] = data_center_id
+                eprint('WARN: data_center_id is deprecated by provider, location_code. ')
+                data_center = self.get_data_center_by_id(id=data_center_id)
+                body['provider'] = data_center['provider']
+                body['locationCode'] = data_center['locationCode']
                 body['linkListener'] = True
+            elif provider or location_code:
+                if provider and location_code:
+                    data_centers = self.get_edge_router_data_centers(provider=provider, location_code=location_code)
+                    if len(data_centers) == 1:
+                        body['provider'] = provider
+                        body['locationCode'] = location_code
+                        body['linkListener'] = True
+                    else:
+                        raise Exception("ERROR: failed to find exactly one {provider} data center with location_code={location_code}".format(
+                            provider=provider,
+                            location_code=location_code))
+                else:
+                    raise Exception("ERROR: need both provider and location_code to create a hosted router.")
+
             response = http.post(
                 self.session.audience+'core/v2/edge-routers',
                 proxies=self.session.proxies,
@@ -990,7 +1058,7 @@ def create_service_simple(self, name: str, client_host_name: str, client_port: i
         :param: client_port is required integer of the ports to intercept
         :param: server_host_name is optional string that is a hostname (DNS) or IPv4. If omitted service is assumed to be SDK-hosted (not Tunneler or Router-hosted).
         :param: server_port is optional integer of the server port. If omitted the client port is used unless SDK-hosted.
-        :param: server_protocol is optional string of the server protocol.
+        :param: server_protocol is optional string of the server protocol, choices: {valid_service_protocols}. Default is ["tcp"].
         :param: attributes is optional list of strings of service roles to assign. Default is [].
         :param: edge_router_attributes is optional list of strings of Router roles or Router names that can "see" this service. Default is ["#all"].
         :param: egress_router_id is optional string of UUID or name of hosting Router. Selects Router-hosting strategy.
@@ -1901,15 +1969,15 @@ def get_network_by_id(self,network_id):
         return(network)
 
     def wait_for_property_defined(self, property_name: str, property_type: object=str, entity_type: str="network", wait: int=60, sleep: int=3, id: str=None, progress: bool=False):
-        """continuously poll until expiry for the expected property to become defined with the any value of the expected type
+        """Poll until expiry for the expected property to become defined with the any value of the expected type.
+
         :param: property_name a top-level property to wait for e.g. `zitiId`
         :param: property_type optional Python instance type to expect for the value of property_name
         :param: id the UUID of the entity having a status if entity is not a network
         :param: entity_type optional type of entity e.g. network (default), endpoint, service, edge-router
         :param: wait optional SECONDS after which to raise an exception defaults to five minutes (300)
         :param: sleep SECONDS polling interval
         """
-
         # use the id of this instance's Network unless another one is specified
         if entity_type == "network" and not id:
             id = self.id
@@ -1983,7 +2051,7 @@ def wait_for_entity_name_exists(self, entity_name: str, entity_type: str, wait:
         """Continuously poll until expiry for the expected entity name to exist.
 
         :param: entity_name
-        :param: entity_type is singular or plural form, any of {resource_entity_types}
+        :param: entity_type is singular or plural form, choices: {resource_entity_types}
         :param: wait optional SECONDS after which to raise an exception defaults to five minutes (300)
         :param: sleep SECONDS polling interval
         :param: progress print a horizontal progress meter as dots, default false