diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..4a3d379c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,50 @@ +[build-system] +requires = [ + "setuptools>=61.0", + "wheel", + "protobuf~=6.30", + "grpcio-tools~=1.59" +] +build-backend = "setuptools.build_meta" + +[project] +name = "snet-cli" +version = "3.0.1" +description = "SingularityNET CLI" +readme = "README.md" +requires-python = ">=3.10" +license = {text = "MIT"} +authors = [ + {name = "SingularityNET Foundation", email = "info@singularitynet.io"} +] +urls = {Homepage = "https://github.com/singnet/snet-cli"} + +# Cleaned dependencies list +dependencies = [ + "protobuf~=6.30", + "grpcio~=1.59", + "grpcio-tools~=1.59", + "wheel~=0.45", + "rlp~=4.0", + "web3~=7.0", + "mnemonic==0.21", + "pyyaml~=6.0.1", + "ipfshttpclient==0.4.13.2", + "pymultihash==0.8.2", + "base58==2.1.1", + "argcomplete~=3.1", + "grpcio-health-checking~=1.59", + "jsonschema~=4.1", + "eth-account~=0.9", + "trezor~=0.13.8", + "ledgerblue~=0.1.48", + "snet-contracts==1.0.1", + "lighthouseweb3~=0.1.4", + "cryptography~=46.0" +] + +[project.scripts] +snet = "snet.cli:main" + +[tool.setuptools.packages.find] +include = ["snet*"] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 36cb230a..d48692a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,23 +1,24 @@ -protobuf==4.21.6 -grpcio-tools==1.59.0 -wheel==0.41.2 -jsonrpcclient==4.0.3 -eth-hash==0.5.2 -rlp==3.0.0 -eth-rlp==0.3.0 -web3==6.11.1 -mnemonic==0.20 -pycoin==0.92.20230326 -pyyaml==6.0.1 +protobuf~=6.30 +grpcio~=1.59 +grpcio-tools~=1.59 +wheel~=0.45 +# jsonrpcclient~=4.0 +# eth-hash~=0.5 +rlp~=4.0 +# eth-rlp~=2.0 +web3~=7.0 +mnemonic==0.21 +# pycoin==0.92.20241201 +pyyaml~=6.0.1 ipfshttpclient==0.4.13.2 pymultihash==0.8.2 base58==2.1.1 -argcomplete==3.1.2 -grpcio-health-checking==1.59.0 -jsonschema==4.0.0 -eth-account==0.9.0 -trezor==0.13.8 -ledgerblue==0.1.48 -snet-contracts==1.0.0 -lighthouseweb3==0.1.4 -cryptography==44.0.1 +argcomplete~=3.1 +grpcio-health-checking~=1.59 +jsonschema~=4.1 +eth-account~=0.9 +trezor~=0.13.8 +ledgerblue~=0.1.48 +snet-contracts==1.0.1 +lighthouseweb3~=0.1.4 +cryptography~=46.0 diff --git a/setup.py b/setup.py index 14d766d3..da43752a 100644 --- a/setup.py +++ b/setup.py @@ -1,73 +1,71 @@ import os from pathlib import Path -from setuptools import find_namespace_packages, setup +from setuptools import setup from setuptools.command.develop import develop as _develop from setuptools.command.install import install as _install +from setuptools.command.build_py import build_py as _build_py +from grpc_tools import protoc +from pkg_resources import resource_filename -from snet.cli.utils.utils import compile_proto -from version import __version__ - -PACKAGE_NAME = 'snet-cli' - - -this_directory = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f: - long_description = f.read() +def install_and_compile_proto(): + """ + Compiles protobuf files directly. + """ + proto_dir = Path(__file__).absolute().parent.joinpath( + "snet", "cli", "resources", "proto") + # Locate the standard grpc_tools internal protos (google/protobuf/...) + grpc_protos_include = resource_filename('grpc_tools', '_proto') -with open("./requirements.txt") as f: - requirements_str = f.read() -requirements = requirements_str.split("\n") + print(f"Proto directory: {proto_dir}") + print(f"Grpc include directory: {grpc_protos_include}") + if not proto_dir.exists(): + print(f"Warning: Proto directory not found at {proto_dir}") + return -def install_and_compile_proto(): - proto_dir = Path(__file__).absolute().parent.joinpath( - "snet", "cli", "resources", "proto") - print(proto_dir, "->", proto_dir) + # glob('*.proto') is non-recursive. It will NOT look inside subfolders. for fn in proto_dir.glob('*.proto'): - print("Compiling protobuf", fn) - compile_proto(proto_dir, proto_dir, proto_file=fn) - + print(f"Compiling protobuf: {fn}") + + command = [ + 'grpc_tools.protoc', + f'-I{proto_dir}', + f'-I{grpc_protos_include}', # <--- CRITICAL FIX: Add standard protos to include path + f'--python_out={proto_dir}', + f'--grpc_python_out={proto_dir}', + str(fn) + ] + + if protoc.main(command) != 0: + print(f"Error: Failed to compile {fn}") + raise RuntimeError(f"Protocol buffer compilation failed for {fn}") + +class build_py(_build_py): + """ + Override build_py to compile protos before building the wheel. + This is the hook used by 'python -m build'. + """ + def run(self): + self.execute(install_and_compile_proto, (), msg="Compile protocol buffers") + _build_py.run(self) class develop(_develop): - """Post-installation for development mode.""" - + """Post-installation for development mode (pip install -e .).""" def run(self): + self.execute(install_and_compile_proto, (), msg="Compile protocol buffers") _develop.run(self) - self.execute(install_and_compile_proto, (), - msg="Compile protocol buffers") - class install(_install): - """Post-installation for installation mode.""" - + """Post-installation for legacy installation mode.""" def run(self): + self.execute(install_and_compile_proto, (), msg="Compile protocol buffers") _install.run(self) - self.execute(install_and_compile_proto, (), - msg="Compile protocol buffers") - setup( - name=PACKAGE_NAME, - version=__version__, - packages=find_namespace_packages(include=['snet*']), - url='https://github.com/singnet/snet-cli', - author="SingularityNET Foundation", - author_email="info@singularitynet.io", - description="SingularityNET CLI", - long_description=long_description, - long_description_content_type='text/markdown', - license="MIT", - python_requires='>=3.10', - install_requires=requirements, - include_package_data=True, cmdclass={ 'develop': develop, 'install': install, + 'build_py': build_py, }, - entry_points={ - 'console_scripts': [ - 'snet = snet.cli:main' - ], - } -) +) \ No newline at end of file diff --git a/snet/cli/arguments.py b/snet/cli/arguments.py index 75237007..7ad76e2f 100644 --- a/snet/cli/arguments.py +++ b/snet/cli/arguments.py @@ -24,18 +24,24 @@ def __init__(self, default_choice=None, *args, **kwargs): super().__init__(*args, **kwargs) def error(self, message): - sys.stderr.write("error: {}\n\n".format(message)) + sys.stderr.write(f"error: {message}\n\n") self.print_help(sys.stderr) sys.exit(2) def _parse_known_args(self, arg_strings, *args, **kwargs): - if self.default_choice and not len(list(filter(lambda option: option in arg_strings, {'-h', '--help'}))): - for action in list(filter( - lambda subparser_action: isinstance( - subparser_action, argparse._SubParsersAction), - self._subparsers._actions - )): - if not len(list(filter(lambda arg: arg in action._name_parser_map.keys(), arg_strings))): + if self.default_choice and not any(arg in arg_strings for arg in {'-h', '--help'}): + + subparser_action = next( + (a for a in self._actions if isinstance(a, argparse._SubParsersAction)), + None + ) + + if subparser_action: + is_subcommand_present = any( + arg in subparser_action.choices for arg in arg_strings + ) + + if not is_subcommand_present: arg_strings = [self.default_choice] + arg_strings return super()._parse_known_args( @@ -575,8 +581,8 @@ def add_mpe_account_options(parser): subparsers = parser.add_subparsers(title="Commands", metavar="COMMAND") subparsers.required = True - def add_p_snt_address_opt(p): - p.add_argument( + def add_p_snt_address_opt(_p): + _p.add_argument( "--singularitynettoken-at", "--snt", default=None, help="Address of SingularityNetToken contract, if not specified we read address from \"networks\"") diff --git a/snet/cli/commands/commands.py b/snet/cli/commands/commands.py index 64b80b91..8472dc75 100644 --- a/snet/cli/commands/commands.py +++ b/snet/cli/commands/commands.py @@ -144,9 +144,9 @@ def get_gas_price_verbose(self): gas_price = self.w3.eth.gas_price if gas_price <= 15000000000: gas_price += gas_price * 1 / 3 - elif gas_price > 15000000000 and gas_price <= 50000000000: + elif 15000000000 < gas_price <= 50000000000: gas_price += gas_price * 1 / 5 - elif gas_price > 50000000000 and gas_price <= 150000000000: + elif 50000000000 < gas_price <= 150000000000: gas_price += 7000000000 elif gas_price > 150000000000: gas_price += gas_price * 1 / 10 @@ -166,8 +166,8 @@ def get_identity(self): return RpcIdentityProvider(self.w3, self.get_wallet_index()) if identity_type == "mnemonic": return MnemonicIdentityProvider(self.w3, self.config.get_session_field("mnemonic"), self.get_wallet_index()) - # if identity_type == "trezor": - # return TrezorIdentityProvider(self.w3, self.get_wallet_index()) + if identity_type == "trezor": + return TrezorIdentityProvider(self.w3, self.get_wallet_index()) if identity_type == "ledger": return LedgerIdentityProvider(self.w3, self.get_wallet_index()) if identity_type == "key": @@ -177,7 +177,8 @@ def get_identity(self): def check_ident(self): identity_type = self.config.get_session_field("identity_type") - if get_kws_for_identity_type(identity_type)[0][1] and not self.ident.private_key: + kws = get_kws_for_identity_type(identity_type) + if kws and all(kws.values()) and not self.ident.private_key: if identity_type == "key": secret = self.config.get_session_field("private_key") else: @@ -258,7 +259,9 @@ def create(self): identity_type = self.args.identity_type identity["identity_type"] = identity_type - for kw, is_secret in get_kws_for_identity_type(identity_type): + kws = get_kws_for_identity_type(identity_type) + + for kw, is_secret in kws: value = getattr(self.args, kw) if value is None and is_secret: kw_prompt = "{}: ".format(" ".join(kw.capitalize().split("_"))) @@ -272,7 +275,8 @@ def create(self): identity["default_wallet_index"] = self.args.wallet_index password = None - if not self.args.do_not_encrypt and get_kws_for_identity_type(identity_type)[0][1]: + + if not self.args.do_not_encrypt and any(kws.values()): self._printout("For 'mnemonic' and 'key' identity_type, secret encryption is enabled by default, " "so you need to come up with a password that you then need to enter on every transaction. " "To disable encryption, use the '-de' or '--do-not-encrypt' argument.") @@ -287,15 +291,13 @@ def create(self): def list(self): for identity_section in filter(lambda x: x.startswith("identity."), self.config.sections()): identity = self.config[identity_section] - key_is_secret_lookup = {} identity_type = self.config.get(identity_section, 'identity_type') - for kw, is_secret in get_kws_for_identity_type(identity_type): - key_is_secret_lookup[kw] = is_secret + kws = get_kws_for_identity_type(identity_type) self._pprint({ identity_section[len("identity."):]: { - k: (v if not key_is_secret_lookup.get(k, False) else "xxxxxx") for k, v in identity.items() + k: (v if not kws.get(k, False) else "xxxxxx") for k, v in identity.items() } }) @@ -360,7 +362,7 @@ def populate_contract_address(self, rez, key): w3=self.w3, contract_name="MultiPartyEscrow") rez[key]['default_fetchtoken_at'] = read_default_contract_address( w3=self.w3, contract_name="FetchToken") - except Exception as e: + except Exception: pass return @@ -587,7 +589,7 @@ def error_organization_not_found(self, org_id, found): def info(self): org_id = self.args.org_id - (found, org_id, org_name, owner, members, serviceNames) = self._get_organization_by_id(org_id) + (found, org_id, org_name, owner, members, service_names) = self._get_organization_by_id(org_id) self.error_organization_not_found(self.args.org_id, found) org_m = self._get_organization_metadata_from_registry(web3.Web3.to_text(org_id)) @@ -604,9 +606,9 @@ def info(self): self._printout("\nMembers:") for idx, member in enumerate(members): self._printout(" - {}".format(member)) - if serviceNames: + if service_names: self._printout("\nServices:") - for idx, service in enumerate(serviceNames): + for idx, service in enumerate(service_names): self._printout(" - {}".format(bytes32_to_str(service))) def metadata_validate(self): @@ -670,7 +672,7 @@ def _metadata_validate_with_schema(self): try: with open(metadata_file, 'r') as f: metadata_dict = json.load(f) - except Exception as e: + except Exception: return {"status": 2, "msg": "Organization metadata json file not found, please check --metadata-file path"} validator = jsonschema.Draft7Validator(schema) @@ -747,7 +749,7 @@ def delete(self): try: self.transact_contract_command("Registry", "deleteOrganization", [ type_converter("bytes32")(org_id)]) - except Exception as e: + except Exception: self._printerr( "\nTransaction error!\nHINT: Check if you are the owner of organization with id={}\n".format(org_id)) raise @@ -818,7 +820,7 @@ def change_owner(self): try: self.transact_contract_command("Registry", "changeOrganizationOwner", [type_converter("bytes32")(org_id), self.args.owner]) - except Exception as e: + except Exception: self._printerr( "\nTransaction error!\nHINT: Check if you are the owner of {}\n".format(org_id)) raise @@ -848,7 +850,7 @@ def add_members(self): try: self.transact_contract_command( "Registry", "addOrganizationMembers", params) - except Exception as e: + except Exception: self._printerr( "\nTransaction error!\nHINT: Check if you are the owner of {}\n".format(org_id)) raise @@ -879,7 +881,7 @@ def rem_members(self): try: self.transact_contract_command( "Registry", "removeOrganizationMembers", params) - except Exception as e: + except Exception: self._printerr( "\nTransaction error!\nHINT: Check if you are the owner of {}\n".format(org_id)) raise @@ -892,7 +894,7 @@ def list_my(self): rez_owner = [] rez_member = [] for idx, org_id in enumerate(org_list): - (found, org_id, org_name, owner, members, serviceNames) = self.call_contract_command( + (found, org_id, org_name, owner, members, service_names) = self.call_contract_command( "Registry", "getOrganizationById", [org_id]) if not found: raise Exception( diff --git a/snet/cli/commands/mpe_channel.py b/snet/cli/commands/mpe_channel.py index e2d5be02..c1f70577 100644 --- a/snet/cli/commands/mpe_channel.py +++ b/snet/cli/commands/mpe_channel.py @@ -2,13 +2,8 @@ import os import pickle import shutil -import tempfile -from collections import defaultdict -from importlib.metadata import metadata from pathlib import Path -from eth_abi.codec import ABICodec -from web3._utils.events import get_event_data from snet.contracts import get_contract_def, get_contract_deployment_block from snet.cli.commands.commands import OrganizationCommand @@ -85,27 +80,34 @@ def _event_data_args_to_dict(self, event_data): "group_id": event_data["groupId"], } - def _get_all_opened_channels_from_blockchain(self, starting_block_number, to_block_number): - mpe_address = self.get_mpe_address() - event_topics = [self.ident.w3.keccak( - text="ChannelOpen(uint256,uint256,address,address,address,bytes32,uint256,uint256)").hex()] + def _get_all_opened_channels_from_blockchain(self, start_block, end_block): + start_block = int(start_block) + end_block = int(end_block) + mpe_address = self.ident.w3.to_checksum_address(self.get_mpe_address()) + + abi = get_contract_def("MultiPartyEscrow")["abi"] + contract = self.ident.w3.eth.contract(address = mpe_address, abi = abi) + + raw_topic = self.ident.w3.keccak( + text = "ChannelOpen(uint256,uint256,address,address,address,bytes32,uint256,uint256)") + event_topics = [self.ident.w3.to_hex(raw_topic)] + blocks_per_batch = 5000 - codec: ABICodec = self.ident.w3.codec logs = [] - from_block = starting_block_number - while from_block <= to_block_number: - to_block = min(from_block + blocks_per_batch, to_block_number) + + from_block = start_block + while from_block <= end_block: + to_block = min(from_block + blocks_per_batch, end_block) logs += self.ident.w3.eth.get_logs({"fromBlock": from_block, - "toBlock": to_block, - "address": mpe_address, - "topics": event_topics}) + "toBlock": to_block, + "address": mpe_address, + "topics": event_topics}) from_block = to_block + 1 - abi = get_contract_def("MultiPartyEscrow") - event_abi = abi_get_element_by_name(abi, "ChannelOpen") + channel_open_event = contract.events.ChannelOpen() + event_data_list = [channel_open_event.process_log(l)["args"] for l in logs] - event_data_list = [get_event_data(codec, event_abi, l)["args"] for l in logs] channels_opened = list(map(self._event_data_args_to_dict, event_data_list)) return channels_opened @@ -186,15 +188,15 @@ def _check_mpe_address_metadata(self, metadata): def _init_or_update_org_if_needed(self, metadata, org_registration): # if service was already initialized and metadataURI hasn't changed we do nothing if self.is_org_initialized(): - if self.is_metadataURI_has_changed(org_registration): + if self.is_metadata_uri_has_changed(org_registration): self._printerr("# Organization with org_id=%s " % - (self.args.org_id)) + self.args.org_id) self._printerr( "# ATTENTION!!! price or other paramaters might have been changed!\n") else: return # we do nothing self._printerr("# Initilize service with org_id=%s" % - (self.args.org_id)) + self.args.org_id) # self._check_mpe_address_metadata(metadata) org_dir = self.get_org_spec_dir(self.args.org_id) @@ -226,7 +228,7 @@ def _init_or_update_registered_org_if_needed(self): org_registration = self._get_organization_registration( self.args.org_id) # if metadataURI hasn't been changed we do nothing - if not self.is_metadataURI_has_changed(org_registration): + if not self.is_metadata_uri_has_changed(org_registration): return else: org_registration = self._get_organization_registration( @@ -236,11 +238,11 @@ def _init_or_update_registered_org_if_needed(self): self.args.org_id) self._init_or_update_org_if_needed(org_metadata, org_registration) - def is_metadataURI_has_changed(self, new_reg): + def is_metadata_uri_has_changed(self, new_reg): old_reg = self._read_org_info(self.args.org_id) return new_reg.get("orgMetadataURI") != old_reg.get("orgMetadataURI") - def is_service_metadataURI_has_changed(self, new_reg): + def is_service_metadata_uri_has_changed(self, new_reg): old_reg = self._read_service_info( self.args.org_id, self.args.service_id) return new_reg.get("metadataURI") != old_reg.get("metadataURI") @@ -432,7 +434,7 @@ def _smart_get_channel_for_org(self, metadata, filter_by): def channel_extend_and_add_funds_for_org(self): self._init_or_update_registered_org_if_needed() metadata = self._read_metadata_for_org(self.args.org_id) - channel_id = self._smart_get_channel_for_org(metadata, "sender")["channelId"] + channel_id = self._smart_get_channel_for_org(metadata, "sender")["channel_id"] self._channel_extend_add_funds_with_channel_id(channel_id) def _get_channel_state_from_blockchain(self, channel_id): @@ -448,7 +450,7 @@ def _read_metadata_for_org(self, org_id): sdir = self.get_org_spec_dir(org_id) if not os.path.exists(sdir): raise Exception( - "Service with org_id=%s is not initialized" % (org_id)) + "Service with org_id=%s is not initialized" % org_id) return OrganizationMetadata.from_file(sdir.joinpath("organization_metadata.json")) def _convert_channel_dict_to_str(self, channel, filters=None): @@ -560,7 +562,7 @@ def _get_service_registration(self): "bytes32")(self.args.service_id)] response = self.call_contract_command( "Registry", "getServiceRegistrationById", params) - if response[0] == False: + if not response[0]: raise Exception("Cannot find Service with id=%s in Organization with id=%s" % ( self.args.service_id, self.args.org_id)) return {"metadataURI": response[2]} @@ -579,7 +581,7 @@ def _get_service_metadata_from_registry(self): def _init_or_update_service_if_needed(self, metadata, service_registration): # if service was already initialized and metadataURI hasn't changed we do nothing if self.is_service_initialized(): - if self.is_service_metadataURI_has_changed(service_registration): + if self.is_service_metadata_uri_has_changed(service_registration): self._printerr("# Service with org_id=%s and service_id=%s was updated" % ( self.args.org_id, self.args.service_id)) self._printerr( @@ -636,7 +638,7 @@ def _init_or_update_registered_service_if_needed(self): service_registration = self._get_service_registration() # if metadataURI hasn't been changed we do nothing - if not self.is_service_metadataURI_has_changed(service_registration): + if not self.is_service_metadata_uri_has_changed(service_registration): return else: service_registration = self._get_service_registration() diff --git a/snet/cli/commands/mpe_client.py b/snet/cli/commands/mpe_client.py index 760cfff4..836966b3 100644 --- a/snet/cli/commands/mpe_client.py +++ b/snet/cli/commands/mpe_client.py @@ -68,7 +68,7 @@ def _get_call_params(self): try: params = self._transform_call_params(params) - except Exception as e: + except Exception: self._printerr('Fail to "transform" call params') raise diff --git a/snet/cli/commands/mpe_service.py b/snet/cli/commands/mpe_service.py index 442b6a45..b4d9606c 100644 --- a/snet/cli/commands/mpe_service.py +++ b/snet/cli/commands/mpe_service.py @@ -3,7 +3,6 @@ from pathlib import Path from re import search from sys import exit -import tempfile from grpc_health.v1 import health_pb2 as heartb_pb2 from grpc_health.v1 import health_pb2_grpc as heartb_pb2_grpc @@ -384,10 +383,6 @@ def metadata_validate(self): Validates whether service metadata (`service_metadata.json` if not provided as argument) is consistent with the schema provided in `service_schema` present in `snet_cli/snet/snet_cli/resources.` - Args: - metadata_file: Option provided through the command line. (default: service_metadata.json) - service_schema: Schema of a consistent service metadata file. - Raises: ValidationError: Inconsistent service metadata structure or missing values. docs -> Handling ValidationErrors (https://python-jsonschema.readthedocs.io/en/stable/errors/) @@ -483,7 +478,7 @@ def _get_organization_registration(self, org_id): params = [type_converter("bytes32")(org_id)] result = self.call_contract_command( "Registry", "getOrganizationById", params) - if result[0] == False: + if not result[0]: raise Exception("Cannot find Organization with id=%s" % ( self.args.org_id)) return {"orgMetadataURI": result[2]} @@ -570,7 +565,7 @@ def _get_service_registration(self): "bytes32")(self.args.service_id)] rez = self.call_contract_command( "Registry", "getServiceRegistrationById", params) - if rez[0] == False: + if not rez[0]: raise Exception("Cannot find Service with id=%s in Organization with id=%s" % ( self.args.service_id, self.args.org_id)) return {"metadataURI": rez[2]} @@ -591,21 +586,21 @@ def print_service_metadata_from_registry(self): metadata = self._get_service_metadata_from_registry() self._printout(metadata.get_json_pretty()) - def _service_status(self, url, secure=True): + def _service_status(self, url): try: channel = open_grpc_channel(endpoint=url) stub = heartb_pb2_grpc.HealthStub(channel) response = stub.Check( heartb_pb2.HealthCheckRequest(service=""), timeout=10) - if response != None and response.status == 1: + if response is not None and response.status == 1: return True return False - except Exception as e: + except Exception: return False def print_service_status(self): metadata = self._get_service_metadata_from_registry() - if self.args.group_name != None: + if self.args.group_name is not None: groups = {self.args.group_name: metadata.get_all_endpoints_for_group( self.args.group_name)} else: diff --git a/snet/cli/commands/mpe_treasurer.py b/snet/cli/commands/mpe_treasurer.py index 4043c269..7b3b040f 100644 --- a/snet/cli/commands/mpe_treasurer.py +++ b/snet/cli/commands/mpe_treasurer.py @@ -41,10 +41,10 @@ def _get_stub_and_request_classes(self, service_name): codegen_dir, service_name) return stub_class, request_class - def _decode_PaymentReply(self, p): + def _decode__payment_reply(self, p): return {"channel_id": int4bytes_big(p.channel_id), "nonce": int4bytes_big(p.channel_nonce), "amount": int4bytes_big(p.signed_amount), "signature": p.signature} - def _call_GetListUnclaimed(self, grpc_channel): + def _call_get_list_unclaimed(self, grpc_channel): stub_class, request_class = self._get_stub_and_request_classes( "GetListUnclaimed") stub = stub_class(grpc_channel) @@ -62,9 +62,9 @@ def _call_GetListUnclaimed(self, grpc_channel): raise Exception( "Signature was set in GetListUnclaimed. Response is invalid") - return [self._decode_PaymentReply(p) for p in response.payments] + return [self._decode__payment_reply(p) for p in response.payments] - def _call_GetListInProgress(self, grpc_channel): + def _call_get_list_in_progress(self, grpc_channel): stub_class, request_class = self._get_stub_and_request_classes( "GetListInProgress") stub = stub_class(grpc_channel) @@ -76,9 +76,9 @@ def _call_GetListInProgress(self, grpc_channel): request = request_class( mpe_address=mpe_address, current_block=current_block, signature=bytes(signature)) response = getattr(stub, "GetListInProgress")(request) - return [self._decode_PaymentReply(p) for p in response.payments] + return [self._decode__payment_reply(p) for p in response.payments] - def _call_StartClaim(self, grpc_channel, channel_id, channel_nonce): + def _call_start_claim(self, grpc_channel, channel_id, channel_nonce): stub_class, request_class = self._get_stub_and_request_classes( "StartClaim") stub = stub_class(grpc_channel) @@ -88,11 +88,11 @@ def _call_StartClaim(self, grpc_channel, channel_id, channel_nonce): request = request_class(mpe_address=mpe_address, channel_id=web3.Web3.to_bytes( channel_id), signature=bytes(signature)) response = getattr(stub, "StartClaim")(request) - return self._decode_PaymentReply(response) + return self._decode__payment_reply(response) def print_unclaimed(self): grpc_channel = open_grpc_channel(self.args.endpoint) - payments = self._call_GetListUnclaimed(grpc_channel) + payments = self._call_get_list_unclaimed(grpc_channel) self._printout("# channel_id channel_nonce signed_amount (ASI(FET))") total = 0 for p in payments: @@ -117,7 +117,7 @@ def _blockchain_claim(self, payments): def _start_claim_channels(self, grpc_channel, channels_ids): """ Safely run StartClaim for given channels """ - unclaimed_payments = self._call_GetListUnclaimed(grpc_channel) + unclaimed_payments = self._call_get_list_unclaimed(grpc_channel) unclaimed_payments_dict = { p["channel_id"]: p for p in unclaimed_payments} @@ -134,14 +134,14 @@ def _start_claim_channels(self, grpc_channel, channels_ids): continue to_claim.append((channel_id, blockchain["nonce"])) - payments = [self._call_StartClaim( + payments = [self._call_start_claim( grpc_channel, channel_id, nonce) for channel_id, nonce in to_claim] return payments def _claim_in_progress_and_claim_channels(self, grpc_channel, channels): """ Claim all 'pending' payments in progress and after we claim given channels """ # first we get the list of all 'payments in progress' in case we 'lost' some payments. - payments = self._call_GetListInProgress(grpc_channel) + payments = self._call_get_list_in_progress(grpc_channel) if len(payments) > 0: self._printout( "There are %i payments in 'progress' (they haven't been claimed in blockchain). We will claim them." % len(payments)) @@ -158,7 +158,7 @@ def claim_all_channels(self): self.check_ident() grpc_channel = open_grpc_channel(self.args.endpoint) # we take list of all channels - unclaimed_payments = self._call_GetListUnclaimed(grpc_channel) + unclaimed_payments = self._call_get_list_unclaimed(grpc_channel) channels = [p["channel_id"] for p in unclaimed_payments] self._claim_in_progress_and_claim_channels(grpc_channel, channels) @@ -166,7 +166,7 @@ def claim_almost_expired_channels(self): self.check_ident() grpc_channel = open_grpc_channel(self.args.endpoint) # we take list of all channels - unclaimed_payments = self._call_GetListUnclaimed(grpc_channel) + unclaimed_payments = self._call_get_list_unclaimed(grpc_channel) channels = [] for p in unclaimed_payments: diff --git a/snet/cli/config.py b/snet/cli/config.py index f5ce60f5..f9a36dd3 100644 --- a/snet/cli/config.py +++ b/snet/cli/config.py @@ -69,7 +69,7 @@ def set_session_identity(self, identity, out_f): print( 'Identity "%s" is not bind to any network. You should switch network manually if you need.' % identity, file=out_f) - print("Switch to identity: %s" % (identity), file=out_f) + print("Switch to identity: %s" % identity, file=out_f) self["session"]["identity"] = identity self._persist() diff --git a/snet/cli/identity.py b/snet/cli/identity.py index 67ba62f8..3cefdb64 100644 --- a/snet/cli/identity.py +++ b/snet/cli/identity.py @@ -6,14 +6,13 @@ import rlp from eth_account import Account -from eth_account.messages import defunct_hash_message -from eth_account._utils.legacy_transactions import encode_transaction, \ - UnsignedTransaction, serializable_unsigned_transaction_from_dict +from eth_account.messages import encode_defunct from ledgerblue.comm import getDongle from ledgerblue.commException import CommException +from rlp.sedes import big_endian_int, binary from trezorlib.client import TrezorClient -from trezorlib import messages as proto -from trezorlib.transport.hid import HidTransport +from trezorlib.transport import get_transport +from trezorlib import ethereum, tools, ui from snet.cli.utils.utils import get_address_from_private, normalize_private_key @@ -35,6 +34,20 @@ def sign_message_after_solidity_keccak(self, message): raise NotImplementedError() +class Transaction(rlp.Serializable): + fields = [ + ('nonce', big_endian_int), + ('gasPrice', big_endian_int), + ('gas', big_endian_int), + ('to', binary), + ('value', big_endian_int), + ('data', binary), + ('v', big_endian_int), + ('r', big_endian_int), + ('s', big_endian_int), + ] + + class KeyIdentityProvider(IdentityProvider): def __init__(self, w3, private_key): self.w3 = w3 @@ -106,7 +119,7 @@ def get_address(self): def transact(self, transaction, out_f): print("Submitting transaction...\n", file=out_f) - txn_hash = self.w3.eth.sendTransaction(transaction) + txn_hash = self.w3.eth.send_transaction(transaction) return send_and_wait_for_transaction_receipt(txn_hash, self.w3) def sign_message_after_solidity_keccak(self, message): @@ -144,45 +157,69 @@ def sign_message_after_solidity_keccak(self, message): class TrezorIdentityProvider(IdentityProvider): def __init__(self, w3, index): self.w3 = w3 - self.client = TrezorClient(HidTransport.enumerate()[0]) self.index = index + + try: + transport = get_transport() + except Exception as e: + raise RuntimeError("No Trezor device found. Ensure it is connected and unlocked.") from e + + self.client = TrezorClient(transport, ui = ui.ClickUI()) + self.path = tools.parse_path(f"m/44'/60'/0'/0/{index}") + self.address = self.w3.to_checksum_address( - "0x" + bytes(self.client.ethereum_get_address([44 + BIP32_HARDEN, - 60 + BIP32_HARDEN, - BIP32_HARDEN, 0, - index])).hex()) + ethereum.get_address(self.client, self.path) + ) def get_address(self): return self.address def transact(self, transaction, out_f): - print("Sending transaction to trezor for signature...\n", file=out_f) - signature = self.client.ethereum_sign_tx(n=[44 + BIP32_HARDEN, 60 + BIP32_HARDEN, - BIP32_HARDEN, 0, self.index], - nonce=transaction["nonce"], - gas_price=transaction["gasPrice"], - gas_limit=transaction["gas"], - to=bytearray.fromhex( - transaction["to"][2:]), - value=transaction["value"], - data=bytearray.fromhex(transaction["data"][2:])) - - transaction.pop("from") - unsigned_transaction = serializable_unsigned_transaction_from_dict( - transaction) - raw_transaction = encode_transaction(unsigned_transaction, - vrs=(signature[0], - int(signature[1].hex(), 16), - int(signature[2].hex(), 16))) + print("Sending transaction to trezor for signature...\n", file = out_f) + + tx_data = transaction.get("data", b"") + if isinstance(tx_data, str) and tx_data.startswith("0x"): + tx_data = bytes.fromhex(tx_data[2:]) + + tx_to = transaction["to"] + if isinstance(tx_to, str) and tx_to.startswith("0x"): + tx_to = bytes.fromhex(tx_to[2:]) + + chain_id = int(transaction.get("chainId", self.w3.eth.chain_id)) + + v, r, s = ethereum.sign_tx( + self.client, + n = self.path, + nonce = int(transaction["nonce"]), + gas_price = int(transaction["gasPrice"]), + gas_limit = int(transaction["gas"]), + to = transaction["to"], # Trezorlib handles "0x" strings fine here + value = int(transaction["value"]), + data = tx_data, + chain_id = chain_id + ) + r = int.from_bytes(r) + s = int.from_bytes(s) + + signed_tx = Transaction( + nonce = int(transaction["nonce"]), + gasPrice = int(transaction["gasPrice"]), + gas = int(transaction["gas"]), + to = tx_to, + value = int(transaction["value"]), + data = tx_data, + v = v, + r = r, + s = s + ) + + raw_transaction = rlp.encode(signed_tx) + return send_and_wait_for_transaction(raw_transaction, self.w3, out_f) def sign_message_after_solidity_keccak(self, message): - n = self.client._convert_prime([44 + BIP32_HARDEN, - 60 + BIP32_HARDEN, - BIP32_HARDEN, - 0, - self.index]) - return self.client.call(proto.EthereumSignMessage(address_n=n, message=message)).signature + result = ethereum.sign_message(self.client, self.path, message) + return result.signature class LedgerIdentityProvider(IdentityProvider): @@ -198,7 +235,9 @@ def __init__(self, w3, index): except CommException: raise RuntimeError( "Received commException from Ledger. Are you sure your device is plugged in?") + self.dongle_path = parse_bip32_path("44'/60'/0'/0/{}".format(index)) + apdu = LedgerIdentityProvider.GET_ADDRESS_OP apdu += bytearray([len(self.dongle_path) + 1, int(len(self.dongle_path) / 4)]) + self.dongle_path @@ -216,55 +255,75 @@ def get_address(self): return self.address def transact(self, transaction, out_f): - tx = UnsignedTransaction( - nonce=transaction["nonce"], - gasPrice=transaction["gasPrice"], - gas=transaction["gas"], - to=bytes(bytearray.fromhex(transaction["to"][2:])), - value=transaction["value"], - data=bytes(bytearray.fromhex(transaction["data"][2:])) + chain_id = int(transaction.get("chainId", self.w3.eth.chain_id)) + + tx_obj = Transaction( + nonce = int(transaction["nonce"]), + gasPrice = int(transaction["gasPrice"]), + gas = int(transaction["gas"]), + to = bytes.fromhex(transaction["to"][2:]), + value = int(transaction["value"]), + data = bytes.fromhex(transaction["data"][2:]), + v = chain_id, + r = 0, + s = 0 ) - encoded_tx = rlp.encode(tx, UnsignedTransaction) + encoded_tx = rlp.encode(tx_obj) overflow = len(self.dongle_path) + 1 + len(encoded_tx) - 255 if overflow > 0: - encoded_tx, remaining_tx = encoded_tx[:- - overflow], encoded_tx[-overflow:] + encoded_tx_part, remaining_tx = encoded_tx[:-overflow], encoded_tx[-overflow:] + else: + encoded_tx_part = encoded_tx + remaining_tx = b"" apdu = LedgerIdentityProvider.SIGN_TX_OP - apdu += bytearray([len(self.dongle_path) + 1 + - len(encoded_tx), int(len(self.dongle_path) / 4)]) - apdu += self.dongle_path + encoded_tx + apdu += bytearray([len(self.dongle_path) + 1 + len(encoded_tx_part), int(len(self.dongle_path) / 4)]) + apdu += self.dongle_path + encoded_tx_part + try: - print("Sending transaction to Ledger for signature...\n", file=out_f) + print("Sending transaction to Ledger for signature...\n", file = out_f) result = self.dongle.exchange(apdu) - while overflow > 0: - encoded_tx = remaining_tx - overflow = len(encoded_tx) - 255 + while remaining_tx: + overflow = len(remaining_tx) - 255 if overflow > 0: - encoded_tx, remaining_tx = encoded_tx[:- - overflow], encoded_tx[-overflow:] + encoded_tx_part, remaining_tx = remaining_tx[:-overflow], remaining_tx[-overflow:] + else: + encoded_tx_part = remaining_tx + remaining_tx = b"" apdu = LedgerIdentityProvider.SIGN_TX_OP_CONT - apdu += bytearray([len(encoded_tx)]) - apdu += encoded_tx + apdu += bytearray([len(encoded_tx_part)]) + apdu += encoded_tx_part result = self.dongle.exchange(apdu) + except CommException as e: - if e.sw == 27013: + if e.sw == 0x6985: # Common status word for user denial raise RuntimeError("Transaction denied from Ledger by user") - raise RuntimeError(e.message, e.sw) - - transaction.pop("from") - unsigned_transaction = serializable_unsigned_transaction_from_dict( - transaction) - raw_transaction = encode_transaction(unsigned_transaction, - vrs=(result[0], - int.from_bytes( - result[1:33], byteorder="big"), - int.from_bytes(result[33:65], byteorder="big"))) + raise RuntimeError(f"Ledger error: {e.sw:x}") + + v_parity = result[0] + v = (chain_id * 2 + 35) + v_parity + + r = int.from_bytes(result[1:33], byteorder = "big") + s = int.from_bytes(result[33:65], byteorder = "big") + + signed_tx = Transaction( + nonce = tx_obj.nonce, + gasPrice = tx_obj.gasPrice, + gas = tx_obj.gas, + to = tx_obj.to, + value = tx_obj.value, + data = tx_obj.data, + v = v, + r = r, + s = s + ) + + raw_transaction = rlp.encode(signed_tx) return send_and_wait_for_transaction(raw_transaction, self.w3, out_f) def sign_message_after_solidity_keccak(self, message): @@ -315,39 +374,43 @@ def parse_bip32_path(path): return result -def get_kws_for_identity_type(identity_type): - SECRET = True - PLAINTEXT = False +def get_kws_for_identity_type(identity_type: str) -> dict: + secret = True + plaintext = False + + result = {} if identity_type == "rpc": - return [("network", PLAINTEXT)] + result["network"] = plaintext elif identity_type == "mnemonic": - return [("mnemonic", SECRET)] + result["mnemonic"] = secret elif identity_type == "key": - return [("private_key", SECRET)] - # elif identity_type == "trezor": - # return [] - elif identity_type == "ledger": - return [] + result["private_key"] = secret elif identity_type == "keystore": - return [("keystore_path", PLAINTEXT)] + result["keystore_path"] = plaintext + elif identity_type in ["trezor", "ledger"]: + # empty dict + pass else: raise RuntimeError( "unrecognized identity_type {}".format(identity_type)) + return result def get_identity_types(): # temporary fully removed: trezor - return ["rpc", "mnemonic", "key", "ledger", "keystore"] + return ["rpc", "mnemonic", "key", "trezor", "ledger", "keystore"] def sign_transaction_with_private_key(w3, private_key, transaction): - return w3.eth.account.sign_transaction(transaction, private_key).rawTransaction + return w3.eth.account.sign_transaction(transaction, private_key).raw_transaction def sign_message_with_private_key(w3, private_key, message): - h = defunct_hash_message(message) - return w3.eth.account.signHash(h, private_key).signature + message_encoded = encode_defunct(primitive = message) + signed_message = w3.eth.account.sign_message(message_encoded, private_key) + + return signed_message.signature def unlock_keystore_with_password(w3, path_to_keystore): diff --git a/snet/cli/metadata/organization.py b/snet/cli/metadata/organization.py index 28a5c1af..311e02f0 100644 --- a/snet/cli/metadata/organization.py +++ b/snet/cli/metadata/organization.py @@ -88,7 +88,7 @@ def update_payment_expiration_threshold(self, payment_expiration_threshold): self.payment.payment_expiration_threshold = payment_expiration_threshold def update_payment_channel_storage_type(self, payment_channel_storage_type): - self.update_payment_channel_storage_type = payment_channel_storage_type + self.payment.payment_channel_storage_type = payment_channel_storage_type def update_payment_address(self, payment_address): self.payment.payment_address = payment_address @@ -102,9 +102,6 @@ def update_request_timeout(self, request_timeout): def update_endpoints(self, endpoints): self.payment.update_endpoints(endpoints) - def get_group_id(self, group_name=None): - return base64.b64decode(self.get_group_id_base64(group_name)) - def get_payment_address(self): return self.payment.payment_address diff --git a/snet/cli/test/functional_tests/test_encryption_key.py b/snet/cli/test/functional_tests/test_encryption_key.py index cbcb2802..28bfa012 100644 --- a/snet/cli/test/functional_tests/test_encryption_key.py +++ b/snet/cli/test/functional_tests/test_encryption_key.py @@ -31,7 +31,7 @@ def test_2_get_encryption_key(self): cmd = BlockchainCommand(self.conf, self.parser.parse_args(['session'])) enc_key = cmd.config.get_session_field("private_key") res_key = cmd._get_decrypted_secret(enc_key) - assert res_key == self.key + assert res_key == self.password def test_3_delete_identity(self): with mock.patch('getpass.getpass', return_value=self.password): diff --git a/snet/cli/utils/ipfs_utils.py b/snet/cli/utils/ipfs_utils.py index 49fccfe5..8279c88b 100644 --- a/snet/cli/utils/ipfs_utils.py +++ b/snet/cli/utils/ipfs_utils.py @@ -109,7 +109,8 @@ def get_from_ipfs_and_checkhash(ipfs_client, ipfs_hash_base58, validate=True): # Decode Base58 bash to multihash try: - mh = multihash.decode(ipfs_hash_base58.encode('ascii'), "base58") + decoded_hash_bytes = base58.b58decode(ipfs_hash_base58) + mh = multihash.decode(decoded_hash_bytes) except Exception as e: raise ValueError(f"Invalid multihash for IPFS hash: {ipfs_hash_base58}. Error: {str(e)}") from e diff --git a/snet/cli/utils/proto_utils.py b/snet/cli/utils/proto_utils.py index d150f569..5f115dc4 100644 --- a/snet/cli/utils/proto_utils.py +++ b/snet/cli/utils/proto_utils.py @@ -79,11 +79,11 @@ def switch_to_json_payload_encoding(call_fn, response_class): """ Switch payload encoding to JSON for GRPC call """ def json_serializer(*args, **kwargs): - return bytes(json_format.MessageToJson(args[0], True, preserving_proto_field_name=True), "utf-8") + return bytes(json_format.MessageToJson(args[0], preserving_proto_field_name=True), "utf-8") def json_deserializer(*args, **kwargs): resp = response_class() - json_format.Parse(args[0], resp, True) + json_format.Parse(args[0], resp, ignore_unknown_fields=True) return resp call_fn._request_serializer = json_serializer diff --git a/snet/cli/utils/utils.py b/snet/cli/utils/utils.py index 0efc8d20..0512b3fa 100644 --- a/snet/cli/utils/utils.py +++ b/snet/cli/utils/utils.py @@ -1,6 +1,5 @@ import json import os -import subprocess import functools import re import sys @@ -15,7 +14,6 @@ import web3 import grpc from grpc_tools.protoc import main as protoc -from trezorlib.cli.firmware import download from snet import cli from snet.cli.resources.root_certificate import certificate @@ -63,7 +61,7 @@ def __str__(self): def get_web3(rpc_endpoint): if rpc_endpoint.startswith("ws:"): - provider = web3.WebsocketProvider(rpc_endpoint) + provider = web3.LegacyWebSocketProvider(rpc_endpoint) else: provider = web3.HTTPProvider(rpc_endpoint) @@ -274,7 +272,7 @@ def get_address_from_private(private_key): return web3.Account.from_key(private_key).address -class add_to_path(): +class add_to_path: def __init__(self, path): self.path = path @@ -361,7 +359,7 @@ def check_training_in_proto(protodir) -> bool: for file in files: if ".proto" not in file: continue - with open(protodir.joinpath(file), "r") as f: + with open(os.path.join(protodir, file), "r") as f: proto_text = f.read() if 'import "training.proto";' in proto_text: return True diff --git a/version.py b/version.py deleted file mode 100644 index 528787cf..00000000 --- a/version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "3.0.0"