From 3e2aadb59e199df9bc803120e541305af29a5213 Mon Sep 17 00:00:00 2001 From: ybnd Date: Sun, 28 Mar 2021 15:00:12 +0200 Subject: [PATCH 1/6] ArgumentParser to JSON schema & request to call --- shapeflow/api.py | 15 ++-- shapeflow/cli.py | 13 +++- shapeflow/config.py | 26 ++----- shapeflow/main.py | 37 +++++++++- shapeflow/util/schema.py | 148 ++++++++++++++++++++++++++++++++++++++ test/test_util.py | 150 ++++++++++++++++++++++++++++----------- 6 files changed, 319 insertions(+), 70 deletions(-) create mode 100644 shapeflow/util/schema.py diff --git a/shapeflow/api.py b/shapeflow/api.py index a94abdf0..0164fbc0 100644 --- a/shapeflow/api.py +++ b/shapeflow/api.py @@ -52,7 +52,7 @@ class _VideoAnalyzerDispatcher(Dispatcher): :func:`shapeflow.core.backend.BaseAnalyzer.cancel` """ - get_config = Endpoint(Callable[[], dict], stream_json) + get_config = Endpoint(Callable[[], dict], stream_json) # todo: GET/POST on single endpoint """Return the analyzer's configuration :func:`shapeflow.core.backend.BaseAnalyzer.get_config` @@ -243,7 +243,7 @@ class _VideoAnalyzerManagerDispatcher(Dispatcher): :func:`shapeflow.main._VideoAnalyzerManager.load_state` """ - stream = Endpoint(Callable[[str, str], BaseStreamer]) + stream = Endpoint(Callable[[str, str], BaseStreamer]) # todo: GET/POST on single endpoint """Open a new stream for a given analyzer ID and endpoint :func:`shapeflow.main._VideoAnalyzerManager.stream` @@ -371,7 +371,7 @@ class ApiDispatcher(Dispatcher): :func:`shapeflow.main._Main.normalize_config` """ - get_settings = Endpoint(Callable[[], dict]) + get_settings = Endpoint(Callable[[], dict]) # todo: GET/POST on single endpoint """Get the application settings :func:`shapeflow.main._Main.get_settings` @@ -381,7 +381,7 @@ class ApiDispatcher(Dispatcher): :func:`shapeflow.main._Main.set_settings` """ - events = Endpoint(Callable[[], EventStreamer], stream_json) + events = Endpoint(Callable[[], EventStreamer], stream_json) # todo: GET/POST on single endpoint """Open an event stream :func:`shapeflow.main._Main.events` @@ -391,7 +391,7 @@ class ApiDispatcher(Dispatcher): :func:`shapeflow.main._Main.stop_events` """ - log = Endpoint(Callable[[], PlainFileStreamer], stream_plain) + log = Endpoint(Callable[[], PlainFileStreamer], stream_plain) # todo: GET/POST on single endpoint """Open a log stream :func:`shapeflow.main._Main.log` @@ -401,6 +401,11 @@ class ApiDispatcher(Dispatcher): :func:`shapeflow.main._Main.stop_log` """ + command = Endpoint(Callable[[str, dict], None]) + """Execute a ``shapeflow.cli.Command`` + + :func:`shapeflow.main._Main.command` + """ unload = Endpoint(Callable[[], bool]) """Unload the application. In order to support page reloading, the backend will wait for some time and quit if no further requests come in. diff --git a/shapeflow/cli.py b/shapeflow/cli.py index 3e8930b2..7fb84dfe 100644 --- a/shapeflow/cli.py +++ b/shapeflow/cli.py @@ -24,6 +24,8 @@ import git import requests +from pydantic import DirectoryPath + from shapeflow import __version__, get_logger, settings from shapeflow.util import before_version, after_version, suppress_stdout @@ -215,6 +217,7 @@ class Serve(Command): __command__ = 'serve' parser = argparse.ArgumentParser( + prog=__command__, description=__doc__ ) @@ -273,6 +276,7 @@ class Dump(Command): __command__ = 'dump' parser = argparse.ArgumentParser( + prog=__command__, description=__doc__ ) parser.add_argument( @@ -282,13 +286,13 @@ class Dump(Command): ) parser.add_argument( '--dir', - type=Path, + type=DirectoryPath, default=Path.cwd(), help='directory to dump to' ) def command(self): - from shapeflow.config import schemas + from main import schemas if not self.args.dir.is_dir(): log.warning(f"making directory '{self.args.dir}'") @@ -416,6 +420,7 @@ class Update(Command, GitMixin): __command__ = 'update' parser = argparse.ArgumentParser( + prog=__command__, description=__doc__ ) parser.add_argument( @@ -453,6 +458,7 @@ class Checkout(Command, GitMixin): __command__ = 'checkout' parser = argparse.ArgumentParser( + prog=__command__, description=__doc__ ) parser.add_argument( @@ -490,6 +496,7 @@ class GetCompiledUi(Command, GitMixin): __command__ = 'get-compiled-ui' parser = argparse.ArgumentParser( + prog=__command__, description=__doc__ ) parser.add_argument( @@ -522,6 +529,7 @@ class SetupCairo(Command): __command__ = 'setup-cairo' parser = argparse.ArgumentParser( + prog=__command__, description=__doc__ ) parser.add_argument( @@ -619,6 +627,7 @@ class Declutter(Command): __command__ = 'declutter' parser = argparse.ArgumentParser( + prog=__command__, description=__doc__ ) parser.add_argument( diff --git a/shapeflow/config.py b/shapeflow/config.py index 27fb5051..b69310fb 100644 --- a/shapeflow/config.py +++ b/shapeflow/config.py @@ -1,14 +1,14 @@ -from typing import Optional, Tuple, Dict, Any, Type, Union +from typing import Optional, Tuple, Dict, Any import json from pydantic import Field, validator -from shapeflow import __version__, settings +from shapeflow import __version__ from shapeflow.core.config import extend, ConfigType, \ log, VERSION, CLASS, untag, BaseConfig from shapeflow.core.backend import BaseAnalyzerConfig, \ - FeatureType, FeatureConfig, AnalyzerState, QueueState + FeatureType, FeatureConfig from shapeflow.core import EnforcedStr from shapeflow.core.interface import FilterType, TransformType, TransformConfig, \ FilterConfig, HandlerConfig @@ -265,27 +265,9 @@ def _validate_parameters(cls, value, values): return tuple(parameters) - _validate_fis = validator('frame_interval_setting')(BaseConfig._resolve_enforcedstr) + _validate_fis = validator('frame_interval_setting', allow_reuse=True)(BaseConfig._resolve_enforcedstr) -def schemas() -> Dict[str, dict]: - """Get the JSON schemas of - - * :class:`shapeflow.video.VideoAnalyzerConfig` - - * :class:`shapeflow.Settings` - - * :class:`shapeflow.core.backend.AnalyzerState` - - * :class:`shapeflow.core.backend.QueueState` - """ - return { - 'config': VideoAnalyzerConfig.schema(), - 'settings': settings.schema(), - 'analyzer_state': dict(AnalyzerState.__members__), - 'queue_state': dict(QueueState.__members__), - } - def loads(config: str) -> BaseConfig: """Load a configuration object from a JSON string. diff --git a/shapeflow/main.py b/shapeflow/main.py index cc6bf28a..0dc6ec2e 100644 --- a/shapeflow/main.py +++ b/shapeflow/main.py @@ -20,17 +20,22 @@ import cv2 from OnionSVG import check_svg +from config import VideoAnalyzerConfig +from core.backend import AnalyzerState, QueueState + from shapeflow.util import open_path, sizeof_fmt from shapeflow.util.filedialog import filedialog +from shapeflow.util.schema import argparse2schema, args2call from shapeflow import get_logger, get_cache, settings, update_settings, ROOTDIR from shapeflow.core import stream_off, Endpoint, RootException from shapeflow.api import api, _FilesystemDispatcher, _DatabaseDispatcher, _VideoAnalyzerManagerDispatcher, _VideoAnalyzerDispatcher, _CacheDispatcher, ApiDispatcher from shapeflow.core.streaming import streams, EventStreamer, PlainFileStreamer, BaseStreamer from shapeflow.core.backend import QueueState, AnalyzerState, BaseAnalyzer -from shapeflow.config import schemas, normalize_config, loads, BaseAnalyzerConfig +from shapeflow.config import normalize_config, loads, BaseAnalyzerConfig from shapeflow.video import init, VideoAnalyzer import shapeflow.plugins from shapeflow.server import ShapeflowServer +from shapeflow.cli import Command from shapeflow.db import History @@ -38,6 +43,28 @@ log = get_logger(__name__) +def schemas() -> Dict[str, dict]: + """Get the JSON schemas of + + * :class:`shapeflow.video.VideoAnalyzerConfig` + + * :class:`shapeflow.Settings` + + * All :class:`shapeflow.cli.Command` subclasses + + * :class:`shapeflow.core.backend.AnalyzerState` + + * :class:`shapeflow.core.backend.QueueState` + """ + return { + 'config': VideoAnalyzerConfig.schema(), + 'settings': settings.schema(), + 'commands': [argparse2schema(c.parser) for c in Command], + 'analyzer_state': dict(AnalyzerState.__members__), + 'queue_state': dict(QueueState.__members__), + } + + class _Main(object): """Implements root-level :data:`~shapeflow.api.api` endpoints. """ @@ -209,6 +236,14 @@ def stop_log(self) -> None: if self._log is not None: self._log.stop() + @api.command.expose() + def command(self, cmd: str, args: dict) -> None: + """Execute a ``shapeflow.cli.Command`` + + :attr:`shapeflow.api.ApiDispatcher.command` + """ + + @api.unload.expose() def unload(self) -> bool: """Unload the application. Called when the user closes or refreshes a diff --git a/shapeflow/util/schema.py b/shapeflow/util/schema.py new file mode 100644 index 00000000..1e815968 --- /dev/null +++ b/shapeflow/util/schema.py @@ -0,0 +1,148 @@ +"""Utility funcitons for JSON schemas +""" + +from pathlib import Path +from typing import List, Optional, Any, Union +from argparse import ArgumentParser +from pydantic import DirectoryPath, FilePath + + +TITLE = "title" +DESCRIPTION = "description" +TYPE = "type" +PROPERTIES = "properties" +DEFAULT = "default" + +OBJECT = "object" +STRING = "string" +INTEGER = "integer" + + +def _pytype2schematype(type_) -> Optional[str]: + """Converts a Python type to the corresponding JSON schema type + + Parameters + ---------- + type_ + + Returns + ------- + str + """ + if type_ is None: + return None + elif type_ is int: + return "integer" + elif type_ is float: + return "number" + elif type_ is str: + return "string" + elif type_ is DirectoryPath: + return "directory-path" + elif type_ is FilePath: + return "file-path" + else: + raise ValueError(f"Unexpected argument type '{type_}'") + + +def _normalize_default(default: Any) -> Union[None, str, int, float, bool]: + if type(default) in (type(None), str, int, float, bool): + return default + elif issubclass(type(default), Path): + return str(default) + else: + raise ValueError(f"Unexpected default type '{type(default)}'") + + +def argparse2schema(parser: ArgumentParser) -> dict: + """Returns the JSON schema of an ``argparse.ArgumentParser``. + + Doesn't handle accumulating arguments. + + Parameters + ---------- + parser: ArgumentParser + an ``ArgumentParser`` instance + + Returns + ------- + dict + The JSON schema of this ``ArgumentParser`` + """ + return { + TITLE: parser.prog, + DESCRIPTION: parser.description, + TYPE: OBJECT, + PROPERTIES: { + action.dest: { + TITLE: action.dest, + DESCRIPTION: action.help, + TYPE: _pytype2schematype(action.type) + if action.type is not None else "boolean", + DEFAULT: _normalize_default(action.default) + } + for action in parser._actions if action.dest != "help" + } + } + + +def args2call(parser: ArgumentParser, args: dict) -> List[str]: + """Formats a dict into a list of call arguments for a specific parser. + + Parameters + ---------- + parser: ArgumentParser + The ``ArgumentParser`` instance to format the arguments for + request: dict + A ``dict`` of arguments + + Raises + ------ + ValueError + If the arguments in ``args`` don't match those of ``parser`` + + Returns + ------- + List[str] + The arguments in ``args`` as a list of strings + """ + call_positionals: List[str] = [] + call_optionals: List[str] = [] + + actions = {a.dest: a for a in parser._actions} + positionals = [a.dest for a in parser._positionals._group_actions] + optionals = {a.dest: a for a in parser._optionals._group_actions} + + for key in [a.dest for a in parser._actions if a.required]: + if key not in args: + raise ValueError(f"Parser '{parser.prog}': " + f"missing required argument '{key}'") + + for key, value in args.items(): + if key not in actions.keys(): + raise ValueError(f"Parser '{parser.prog}': " + f"argument '{key}' is not allowed") + if actions[key].type is not None: + if type(value) is not actions[key].type: + raise ValueError(f"Parser '{parser.prog}': " + f"Argument '{key}' should be " + f"{actions[key].type}, " + f"but was given {type(value)} instead") + else: + if type(value) is not bool: + raise ValueError(f"Parser '{parser.prog}': " + f"Flag {key} value type should be bool, " + f"but was given {type(value)} instead") + + if key in positionals: + call_positionals.insert(positionals.index(key), str(value)) + elif key in optionals.keys(): + if optionals[key].type is not None: + call_optionals.append(f"--{key}") + call_optionals.append(str(value)) + else: + if value: + call_optionals.append(f"--{key}") + + return call_positionals + call_optionals + diff --git a/test/test_util.py b/test/test_util.py index 722fbdb2..07f92800 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -6,12 +6,14 @@ import tkinter import tkinter.filedialog import subprocess +import argparse from shapeflow.util.filedialog import _Tkinter, _Zenity from shapeflow.util.from_venv import _VenvCall, _WindowsVenvCall, from_venv from shapeflow.util.filedialog import _Tkinter, _Zenity +from shapeflow.util.schema import argparse2schema, args2call class FileDialogTest(unittest.TestCase): @@ -218,52 +220,120 @@ def test_windows_venv_call_binary_fail(self, isdir, isfile): ) -class FileDialogTest(unittest.TestCase): - kw = [ - 'title', - 'pattern', - 'pattern_description', - ] - kwargs = {k:k for k in kw} - - @patch('tkinter.filedialog.askopenfilename') - def test_tkinter_load(self, askopenfilename): - _Tkinter().load(**self.kwargs) - +class ArgparseSchemaTest(unittest.TestCase): + parser = argparse.ArgumentParser( + prog="some-command", + description="some description" + ) + parser.add_argument( + "argument1", + type=str, + default="default", + help="help for argument 1", + ) + parser.add_argument( + "--argument2", + type=int, + required=True, + default=17, + help="help for argument 2", + ) + parser.add_argument( + "--flag", + action="store_true", + help="help for flag", + ) + + def test_argparse2schema(self): + schema = argparse2schema(self.parser) + + self.assertEqual("some-command", schema["title"]) + self.assertEqual("some description", schema["description"]) + self.assertEqual(3, len(schema["properties"])) + self.assertTrue("argument1" in schema["properties"].keys()) + self.assertTrue("argument2" in schema["properties"].keys()) + self.assertTrue("flag" in schema["properties"].keys()) + + argument1 = schema["properties"]["argument1"] + argument2 = schema["properties"]["argument2"] + flag = schema["properties"]["flag"] + + self.assertEqual("help for argument 1", argument1["description"]) + self.assertEqual("help for argument 2", argument2["description"]) + self.assertEqual("help for flag", flag["description"]) + + self.assertEqual("string", argument1["type"]) + self.assertEqual("integer", argument2["type"]) + self.assertEqual("boolean", flag["type"]) + + self.assertEqual("default", argument1["default"]) + self.assertEqual(17, argument2["default"]) + + def test_args2call_valid(self): self.assertEqual( - {v:k for k,v in _Tkinter._map.items()}, - askopenfilename.call_args[1] + ["something", "--argument2", "123456789", "--flag"], + args2call(self.parser, { + "argument1": "something", + "argument2": 123456789, + "flag": True, + }) ) - @patch('tkinter.filedialog.asksaveasfilename') - def test_tkinter_save(self, asksaveasfilename): - _Tkinter().save(**self.kwargs) - + # won't add flags if passed as False arguments self.assertEqual( - {v:k for k,v in _Tkinter._map.items()}, - asksaveasfilename.call_args[1] + ["something", "--argument2", "123456789"], + args2call(self.parser, { + "argument1": "something", + "argument2": 123456789, + "flag": False, + }) ) - @patch('subprocess.Popen') - def test_zenity_load(self, Popen): - Popen.return_value.communicate.return_value = (b'', 0) - _Zenity().load(**self.kwargs) - - c = 'zenity --file-selection --title title --file-filter pattern' - + # it's ok to omit optional arguments / flags self.assertEqual( - sorted(c.split(' ')), - sorted(Popen.call_args[0][0]) + ["something", "--argument2", "123456789"], + args2call(self.parser, { + "argument1": "something", + "argument2": 123456789, + }) ) - @patch('subprocess.Popen') - def test_zenity_save(self, Popen): - Popen.return_value.communicate.return_value = (b'', 0) - _Zenity().save(**self.kwargs) - - c = 'zenity --file-selection --save --title title --file-filter pattern' - - self.assertEqual( - sorted(c.split(' ')), - sorted(Popen.call_args[0][0]) - ) + def test_args2call_invalid(self): + # missing positional arguments + with self.assertRaises(ValueError): + args2call(self.parser, { + "argument2": 123456789, + "flag": True, + }) + + # missing required "optional" arguments + with self.assertRaises(ValueError): + args2call(self.parser, { + "argument1": "something", + "flag": True, + }) + + # wrong type for argument + with self.assertRaises(ValueError): + args2call(self.parser, { + "argument1": 123456789, + "argument2": 123456789, + "flag": True, + }) + + # non-boolean value for flag "argument" + with self.assertRaises(ValueError): + args2call(self.parser, { + "argument1": "something", + "argument2": 123456789, + "flag": "True", + }) + + # unrecognized argument + with self.assertRaises(ValueError): + args2call(self.parser, { + "argument1": "something", + "argument2": 123456789, + "flag": True, + "and-something-else": True, + }) From 085c46f7c294954dbd1241ce7a9d45aa0b4b7a2e Mon Sep 17 00:00:00 2001 From: ybnd Date: Sun, 28 Mar 2021 17:19:06 +0200 Subject: [PATCH 2/6] Don't include Serve in command schemas --- shapeflow/cli.py | 5 +++-- shapeflow/main.py | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/shapeflow/cli.py b/shapeflow/cli.py index 7fb84dfe..a700f322 100644 --- a/shapeflow/cli.py +++ b/shapeflow/cli.py @@ -49,7 +49,8 @@ class IterCommand(abc.ABCMeta): """ __command__: str """Command name. This is how the command is addressed from the commandline. - """ # todo: nope, doesn't work' + """ + def __str__(cls): try: @@ -452,7 +453,7 @@ def _update(self) -> None: class Checkout(Command, GitMixin): - """Check out a specific version of the application. Please not you will + """Check out a specific version of the application. Please note you will not have access to this command if you check out a version before 0.4.4 """ diff --git a/shapeflow/main.py b/shapeflow/main.py index 0dc6ec2e..c0bc26f7 100644 --- a/shapeflow/main.py +++ b/shapeflow/main.py @@ -35,7 +35,7 @@ from shapeflow.video import init, VideoAnalyzer import shapeflow.plugins from shapeflow.server import ShapeflowServer -from shapeflow.cli import Command +from shapeflow.cli import Command, Serve from shapeflow.db import History @@ -59,7 +59,9 @@ def schemas() -> Dict[str, dict]: return { 'config': VideoAnalyzerConfig.schema(), 'settings': settings.schema(), - 'commands': [argparse2schema(c.parser) for c in Command], + 'commands': [ + argparse2schema(c.parser) for c in Command if c is not Serve + ], 'analyzer_state': dict(AnalyzerState.__members__), 'queue_state': dict(QueueState.__members__), } From f1a668c2ac596f6e74351e266d1af44ed1f46dd4 Mon Sep 17 00:00:00 2001 From: ybnd Date: Sun, 28 Mar 2021 17:27:39 +0200 Subject: [PATCH 3/6] Implement /api/command endpoint --- shapeflow/cli.py | 4 ++-- shapeflow/main.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/shapeflow/cli.py b/shapeflow/cli.py index a700f322..cfa7c0d4 100644 --- a/shapeflow/cli.py +++ b/shapeflow/cli.py @@ -32,9 +32,8 @@ log = get_logger(__name__) -# type aliases + OptArgs = Optional[List[str]] -Parsing = Callable[[OptArgs], None] class CliError(Exception): @@ -51,6 +50,7 @@ class IterCommand(abc.ABCMeta): """Command name. This is how the command is addressed from the commandline. """ + parser: argparse.ArgumentParser def __str__(cls): try: diff --git a/shapeflow/main.py b/shapeflow/main.py index c0bc26f7..bf32f3c9 100644 --- a/shapeflow/main.py +++ b/shapeflow/main.py @@ -244,6 +244,10 @@ def command(self, cmd: str, args: dict) -> None: :attr:`shapeflow.api.ApiDispatcher.command` """ + if cmd not in [c.__command__ for c in Command]: + raise ValueError(f"Unrecognized command '{cmd}'") + + Command[cmd](args2call(Command[cmd].parser, args)) @api.unload.expose() From ebe6e1c57bab5505c7c0306c58022e3a7bec1fa1 Mon Sep 17 00:00:00 2001 From: ybnd Date: Sun, 28 Mar 2021 17:35:34 +0200 Subject: [PATCH 4/6] Fix minor issues --- shapeflow/cli.py | 4 ++-- shapeflow/main.py | 11 +++++------ shapeflow/util/schema.py | 6 ++---- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/shapeflow/cli.py b/shapeflow/cli.py index cfa7c0d4..ac358c81 100644 --- a/shapeflow/cli.py +++ b/shapeflow/cli.py @@ -287,13 +287,13 @@ class Dump(Command): ) parser.add_argument( '--dir', - type=DirectoryPath, + type=Path, default=Path.cwd(), help='directory to dump to' ) def command(self): - from main import schemas + from shapeflow.main import schemas if not self.args.dir.is_dir(): log.warning(f"making directory '{self.args.dir}'") diff --git a/shapeflow/main.py b/shapeflow/main.py index bf32f3c9..f232dab3 100644 --- a/shapeflow/main.py +++ b/shapeflow/main.py @@ -20,8 +20,6 @@ import cv2 from OnionSVG import check_svg -from config import VideoAnalyzerConfig -from core.backend import AnalyzerState, QueueState from shapeflow.util import open_path, sizeof_fmt from shapeflow.util.filedialog import filedialog @@ -31,7 +29,7 @@ from shapeflow.api import api, _FilesystemDispatcher, _DatabaseDispatcher, _VideoAnalyzerManagerDispatcher, _VideoAnalyzerDispatcher, _CacheDispatcher, ApiDispatcher from shapeflow.core.streaming import streams, EventStreamer, PlainFileStreamer, BaseStreamer from shapeflow.core.backend import QueueState, AnalyzerState, BaseAnalyzer -from shapeflow.config import normalize_config, loads, BaseAnalyzerConfig +from shapeflow.config import normalize_config, loads, BaseAnalyzerConfig, VideoAnalyzerConfig from shapeflow.video import init, VideoAnalyzer import shapeflow.plugins from shapeflow.server import ShapeflowServer @@ -59,9 +57,10 @@ def schemas() -> Dict[str, dict]: return { 'config': VideoAnalyzerConfig.schema(), 'settings': settings.schema(), - 'commands': [ - argparse2schema(c.parser) for c in Command if c is not Serve - ], + 'commands': { + c.__command__: argparse2schema(c.parser) + for c in Command if c is not Serve + }, 'analyzer_state': dict(AnalyzerState.__members__), 'queue_state': dict(QueueState.__members__), } diff --git a/shapeflow/util/schema.py b/shapeflow/util/schema.py index 1e815968..aa163d1e 100644 --- a/shapeflow/util/schema.py +++ b/shapeflow/util/schema.py @@ -37,10 +37,8 @@ def _pytype2schematype(type_) -> Optional[str]: return "number" elif type_ is str: return "string" - elif type_ is DirectoryPath: - return "directory-path" - elif type_ is FilePath: - return "file-path" + elif type_ is Path: + return "path" else: raise ValueError(f"Unexpected argument type '{type_}'") From cbc9dbbb25f938da2c51a4a1fe8d7ff6196fac4a Mon Sep 17 00:00:00 2001 From: ybnd Date: Tue, 30 Mar 2021 21:07:53 +0200 Subject: [PATCH 5/6] Fix required arguments check --- shapeflow/main.py | 6 ++++-- shapeflow/util/schema.py | 22 ++++++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/shapeflow/main.py b/shapeflow/main.py index f232dab3..d7512937 100644 --- a/shapeflow/main.py +++ b/shapeflow/main.py @@ -238,17 +238,19 @@ def stop_log(self) -> None: self._log.stop() @api.command.expose() - def command(self, cmd: str, args: dict) -> None: + def command(self, cmd: str, args: dict = None) -> None: """Execute a ``shapeflow.cli.Command`` :attr:`shapeflow.api.ApiDispatcher.command` """ if cmd not in [c.__command__ for c in Command]: raise ValueError(f"Unrecognized command '{cmd}'") + if cmd == Serve.__command__: + raise ValueError(f"Can't execute 'serve'. To restart the server, " + f"use /api/restart instead") Command[cmd](args2call(Command[cmd].parser, args)) - @api.unload.expose() def unload(self) -> bool: """Unload the application. Called when the user closes or refreshes a diff --git a/shapeflow/util/schema.py b/shapeflow/util/schema.py index aa163d1e..318749e7 100644 --- a/shapeflow/util/schema.py +++ b/shapeflow/util/schema.py @@ -84,7 +84,7 @@ def argparse2schema(parser: ArgumentParser) -> dict: } -def args2call(parser: ArgumentParser, args: dict) -> List[str]: +def args2call(parser: ArgumentParser, args: dict = None) -> List[str]: """Formats a dict into a list of call arguments for a specific parser. Parameters @@ -104,6 +104,21 @@ def args2call(parser: ArgumentParser, args: dict) -> List[str]: List[str] The arguments in ``args`` as a list of strings """ + required = [a.dest for a in parser._actions if a.required] + + if args is None: + if len(required) == 0: + return [] + else: + raise ValueError(f"Parser '{parser.prog}': " + f"no arguments provided; the following arguments" + f"are required: {required}") + else: + for key in required: + if key not in args: + raise ValueError(f"Parser '{parser.prog}': " + f"missing required argument '{key}'") + call_positionals: List[str] = [] call_optionals: List[str] = [] @@ -111,11 +126,6 @@ def args2call(parser: ArgumentParser, args: dict) -> List[str]: positionals = [a.dest for a in parser._positionals._group_actions] optionals = {a.dest: a for a in parser._optionals._group_actions} - for key in [a.dest for a in parser._actions if a.required]: - if key not in args: - raise ValueError(f"Parser '{parser.prog}': " - f"missing required argument '{key}'") - for key, value in args.items(): if key not in actions.keys(): raise ValueError(f"Parser '{parser.prog}': " From 52413f2d9b73e1355817c58c3d597be9a65f3ed5 Mon Sep 17 00:00:00 2001 From: ybnd Date: Tue, 30 Mar 2021 22:19:55 +0200 Subject: [PATCH 6/6] Implement frontend prompt (backend) --- shapeflow/api.py | 5 +++ shapeflow/cli.py | 81 ++++++++++++++++++++++++++++++++----- shapeflow/core/backend.py | 25 ++++-------- shapeflow/core/streaming.py | 16 +++++++- shapeflow/db.py | 5 ++- shapeflow/main.py | 21 ++++++++-- shapeflow/video.py | 11 ++--- 7 files changed, 124 insertions(+), 40 deletions(-) diff --git a/shapeflow/api.py b/shapeflow/api.py index 0164fbc0..18f2b664 100644 --- a/shapeflow/api.py +++ b/shapeflow/api.py @@ -406,6 +406,11 @@ class ApiDispatcher(Dispatcher): :func:`shapeflow.main._Main.command` """ + resolve_prompt = Endpoint(Callable[[str, Any], None]) + """Respond to a prompt + + :func:`shapeflow.main._Main.prompt` + """ unload = Endpoint(Callable[[], bool]) """Unload the application. In order to support page reloading, the backend will wait for some time and quit if no further requests come in. diff --git a/shapeflow/cli.py b/shapeflow/cli.py index ac358c81..d92c3a46 100644 --- a/shapeflow/cli.py +++ b/shapeflow/cli.py @@ -15,18 +15,20 @@ from pathlib import Path import argparse import shutil +from enum import Enum +from threading import Queue from functools import lru_cache -from typing import List, Callable, Optional, Tuple +from typing import List, Optional, Tuple from urllib.request import urlretrieve from zipfile import ZipFile from distutils.util import strtobool import git import requests - -from pydantic import DirectoryPath +import shortuuid from shapeflow import __version__, get_logger, settings +from shapeflow.core.streaming import EventStreamer, EventCategory from shapeflow.util import before_version, after_version, suppress_stdout @@ -40,6 +42,50 @@ class CliError(Exception): pass +class PromptType(Enum): + BOOLEAN = 'boolean' + + +class Prompt(abc.ABC): + def yn(self, prompt: str) -> bool: + raise NotImplementedError + + +class ConsolePrompt(Prompt): + def yn(self, prompt: str) -> bool: + return strtobool(input(f"{prompt} (y/n) ")) + + +class EventResponsePrompt(Prompt): + _eventstreamer: EventStreamer + _id: str + _queue: Queue + + def __init__(self, eventstreamer: EventStreamer): + self._eventstreamer = eventstreamer + + def _new_id(self) -> str: + id = shortuuid.uuid() + self._id = id + return id + + def yn(self, prompt: str) -> bool: + self._queue = Queue() + + id = self._new_id() + self._eventstreamer.event(EventCategory.PROMPT, id, data={ + "prompt": prompt, "type": PromptType.BOOLEAN, + }) + + response = self._queue.get() + assert isinstance(response, bool) + return response + + def resolve(self, id: str, data: Any) -> None: + if self._id == id: + self._queue.put(data) + + class IterCommand(abc.ABCMeta): """Command iterator metaclass. @@ -104,7 +150,14 @@ class Command(abc.ABC, metaclass=IterCommand): args: argparse.Namespace sub_args: List[str] - def __init__(self, args: OptArgs = None): + _prompt: Prompt + + def __init__(self, args: OptArgs = None, prompt: Prompt = None): + if prompt is None: + self._prompt = ConsolePrompt() + else: + self._prompt = prompt + if args is None: # gather commandline arguments args = sys.argv[1:] @@ -151,6 +204,10 @@ def _fix_call(cls, text: str) -> str: else: return text + @property + def prompt(self): + return self._prompt + class Sf(Command): """Commandline entry point. @@ -316,6 +373,8 @@ class GitMixin(abc.ABC): _repo = None _latest = None + prompt: Prompt + @property def repo(self) -> git.Repo: if self._repo is None: @@ -409,10 +468,10 @@ def _prompt_discard_changes(self) -> bool: [item.a_path for item in self.repo.index.diff(None)] \ + self.repo.untracked_files ) - return bool(strtobool(input( + return self.prompt.yn( f'Local changes to\n\n {changed} \n\n' - f'will be overwritten. Continue? (y/n) ' - ))) + f'will be overwritten. Continue?' + ) class Update(Command, GitMixin): @@ -484,11 +543,11 @@ def command(self) -> None: self._get_compiled_ui() def _checkout_anyway(self) -> bool: - return bool(strtobool(input( + return self.prompt.yn( f'After checking out "{self.args.ref}" you won\'t be able to use ' f'the "update" or "checkout" commands (they were added later, ' - f'in v0.4.4) Continue? (y/n) ' - ))) + f'in v0.4.4) Continue?' + ) class GetCompiledUi(Command, GitMixin): @@ -521,7 +580,7 @@ def command(self) -> None: self._get_compiled_ui() def _prompt_replace_ui(self) -> bool: - return bool(strtobool(input('Replace the current UI? (y/n) '))) + return self.prompt.yn('Replace the current UI?') class SetupCairo(Command): diff --git a/shapeflow/core/backend.py b/shapeflow/core/backend.py index 6fce65ba..8e8dbd07 100644 --- a/shapeflow/core/backend.py +++ b/shapeflow/core/backend.py @@ -1,7 +1,6 @@ -from enum import IntEnum, Enum +from enum import IntEnum import diskcache -import sys import abc import time import threading @@ -22,7 +21,7 @@ from shapeflow.util import Timer, Timing from shapeflow.core.db import BaseAnalysisModel from shapeflow.core.config import Factory, BaseConfig, Instance, Configurable -from shapeflow.core.streaming import EventStreamer +from shapeflow.core.streaming import EventStreamer, EventCategory from shapeflow.core.interface import InterfaceType @@ -45,14 +44,6 @@ class CacheAccessError(RootException): _BLOCKED = 'BLOCKED' -class PushEvent(Enum): - """Categories of server-pushed events. - """ - STATUS = 'status' - CONFIG = 'config' - NOTICE = 'notice' - - class AnalyzerState(IntEnum): """The state of an analyzer """ @@ -572,18 +563,18 @@ def set_eventstreamer(self, eventstreamer: EventStreamer = None): """ self._eventstreamer = eventstreamer - def event(self, category: PushEvent, data: dict) -> None: + def event(self, category: EventCategory, data: dict) -> None: """Push an event. Parameters ---------- - category : PushEvent + category : EventCategory The category of event to push data : dict The data to push """ if self.eventstreamer is not None: - self.eventstreamer.event(category.value, self.id, data) + self.eventstreamer.event(category, self.id, data) def notice(self, message: str, persist: bool = False) -> None: """Push a notice. @@ -597,7 +588,7 @@ def notice(self, message: str, persist: bool = False) -> None: interface until dismissed manually) """ self.event( - PushEvent.NOTICE, + EventCategory.NOTICE, data={'message': message, 'persist': persist} ) log.warning(f"'{self.id}': {message}") @@ -787,7 +778,7 @@ def status(self) -> dict: return status def push_status(self): - self.event(PushEvent.STATUS, self.status()) + self.event(EventCategory.STATUS, self.status()) @api.va.__id__.get_config.expose() def get_config(self, do_tag=False) -> dict: @@ -837,7 +828,7 @@ def launch(self) -> bool: # Push events self.set_state(AnalyzerState.LAUNCHED) - self.event(PushEvent.CONFIG, self.get_config()) + self.event(EventCategory.CONFIG, self.get_config()) # State transition (may change from LAUNCHED ~ config) self.state_transition() diff --git a/shapeflow/core/streaming.py b/shapeflow/core/streaming.py index 85b317a6..882d961c 100644 --- a/shapeflow/core/streaming.py +++ b/shapeflow/core/streaming.py @@ -3,6 +3,7 @@ import abc import json +from enum import Enum from threading import Thread from typing import Optional, Tuple, Generator, Callable, Dict, Type, Any, Union, List from functools import wraps @@ -192,10 +193,21 @@ def _decorate(self, value: Optional[bytes]) -> Optional[bytes]: return None +class EventCategory(Enum): + """Categories of server-pushed events. + """ + STATUS = 'status' + CONFIG = 'config' + NOTICE = 'notice' + PROMPT = 'prompt' + + class EventStreamer(JsonStreamer): """Streams server-sent events with JSON data. """ - def event(self, category: str, id: str, data: Any): + CLOSE = 'close' + + def event(self, category: EventCategory, id: str, data: Any): """Push a JSON event :param category: event category @@ -207,7 +219,7 @@ def event(self, category: str, id: str, data: Any): self.push({'category': category, 'id': id, 'data': data}) def stop(self): - self.push({'categroy': 'close', 'data': ''}) + self.push({'category': self.CLOSE, 'data': ''}) super().stop() diff --git a/shapeflow/db.py b/shapeflow/db.py index 44726b45..342fa5bc 100644 --- a/shapeflow/db.py +++ b/shapeflow/db.py @@ -16,7 +16,7 @@ from shapeflow import settings, get_logger, ResultSaveMode from shapeflow.core.config import __meta_sheet__ from shapeflow.config import normalize_config, VideoAnalyzerConfig -from shapeflow.core.streaming import EventStreamer +from shapeflow.core.streaming import EventStreamer, EventCategory from shapeflow.core.backend import BaseAnalyzer, BaseAnalyzerConfig @@ -559,7 +559,8 @@ def set_eventstreamer(self, eventstreamer: EventStreamer): def notice(self, message: str, persist: bool = False): self._eventstreamer.event( - 'notice', id='', data={'message': message, 'persist': persist} + EventCategory.NOTICE, id='', + data={'message': message, 'persist': persist} ) def add_video_file(self, path: str) -> VideoFileModel: diff --git a/shapeflow/main.py b/shapeflow/main.py index d7512937..1f00915f 100644 --- a/shapeflow/main.py +++ b/shapeflow/main.py @@ -27,7 +27,7 @@ from shapeflow import get_logger, get_cache, settings, update_settings, ROOTDIR from shapeflow.core import stream_off, Endpoint, RootException from shapeflow.api import api, _FilesystemDispatcher, _DatabaseDispatcher, _VideoAnalyzerManagerDispatcher, _VideoAnalyzerDispatcher, _CacheDispatcher, ApiDispatcher -from shapeflow.core.streaming import streams, EventStreamer, PlainFileStreamer, BaseStreamer +from shapeflow.core.streaming import streams, EventStreamer, PlainFileStreamer, BaseStreamer, EventCategory from shapeflow.core.backend import QueueState, AnalyzerState, BaseAnalyzer from shapeflow.config import normalize_config, loads, BaseAnalyzerConfig, VideoAnalyzerConfig from shapeflow.video import init, VideoAnalyzer @@ -73,12 +73,14 @@ class _Main(object): _server: ShapeflowServer _log: Optional[PlainFileStreamer] + _prompts: List[Prompt] def __init__(self, server: ShapeflowServer): self._server = server self._lock = Lock() self._log = None + self._prompts = [] @api.ping.expose() def ping(self) -> bool: @@ -249,7 +251,19 @@ def command(self, cmd: str, args: dict = None) -> None: raise ValueError(f"Can't execute 'serve'. To restart the server, " f"use /api/restart instead") - Command[cmd](args2call(Command[cmd].parser, args)) + prompt = EventResponsePrompt() + self._prompts.append(prompt) + + Command[cmd](args2call(Command[cmd].parser, args), prompt) + + @api.resolve_prompt.expose() + def resolve_prompt(self, id: str, data: Any) -> None: + """Respond to a prompt + + :attr:`shapeflow.api.ApiDispatcher.response` + """ + for prompt in self._prompts: + prompt.resolve(id, data) @api.unload.expose() def unload(self) -> bool: @@ -558,7 +572,8 @@ def notice(self, message: str, persist: bool = False): The message to push """ self._server._eventstreamer.event( - 'notice', id='', data={'message': message, 'persist': persist} + EventCategory.NOTICE, id='', + data={'message': message, 'persist': persist} ) @api.va.init.expose() diff --git a/shapeflow/video.py b/shapeflow/video.py index bfade9be..8ffb84df 100644 --- a/shapeflow/video.py +++ b/shapeflow/video.py @@ -19,7 +19,8 @@ from shapeflow.core.backend import Instance, CachingInstance, \ BaseAnalyzer, BackendSetupError, AnalyzerType, Feature, \ FeatureSet, \ - FeatureType, AnalyzerState, PushEvent, FeatureConfig, CacheAccessError + FeatureType, AnalyzerState, FeatureConfig, CacheAccessError +from core.streaming import EventCategory from shapeflow.core.config import extend from shapeflow.core.interface import TransformInterface, FilterConfig, \ FilterInterface, FilterType, TransformType, Handler @@ -1218,7 +1219,7 @@ def set_config(self, config: dict, silent: bool = False) -> dict: config = self.get_config() # Push config event - self.event(PushEvent.CONFIG, config) + self.event(EventCategory.CONFIG, config) # Push streams streams.update() @@ -1375,7 +1376,7 @@ def estimate_transform(self, roi: Optional[dict] = None) -> Optional[dict]: self.transform.estimate(roi_config) self.state_transition() - self.event(PushEvent.CONFIG, self.get_config()) + self.event(EventCategory.CONFIG, self.get_config()) if roi_config is not None: return roi_config.dict() @@ -1540,7 +1541,7 @@ def set_filter_click(self, relative_x: float, relative_y: float) -> None: self.get_colors() self.state_transition() - self.event(PushEvent.CONFIG, self.get_config()) + self.event(EventCategory.CONFIG, self.get_config()) self.commit() streams.update() @@ -1566,7 +1567,7 @@ def clear_filters(self) -> bool: self.set_state(AnalyzerState.CAN_FILTER) self.state_transition() - self.event(PushEvent.CONFIG, self.get_config()) + self.event(EventCategory.CONFIG, self.get_config()) self.commit() streams.update()